diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index e7f7c7423f..28c2c00af8 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -6,7 +6,7 @@ object Versions { const val clikt = "5.0.0" const val detekt = "1.23.7" const val ini4j = "0.5.4" - const val jacodb = "23199c1843" + const val jacodb = "37327563ab" const val juliet = "1.3.2" const val junit = "5.9.3" const val kotlin = "2.1.0" diff --git a/usvm-ts-calls/scripts/run-class-experiment.py b/usvm-ts-calls/scripts/run-class-experiment.py new file mode 100644 index 0000000000..645f4b4ffe --- /dev/null +++ b/usvm-ts-calls/scripts/run-class-experiment.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""Sequential paired class campaign with profile-independent source/native admission.""" +import argparse +import csv +import hashlib +import json +import subprocess +from pathlib import Path + +PROFILES = ('EMPTY_STOP', 'EMPTY_FRESH', 'FROZEN_STOP', 'FROZEN_FRESH') +BUDGETS = (30_000, 120_000) +COVERAGE_POLICY = 'native-original-instrumented-complete-v2' +SUSPENDED_COVERAGE_POLICY = 'reached-prefix-and-final-frontier-inputs-v1' + + +def validate_measurement_protocol(identity): + if (identity.get('schemaVersion') != 2 or identity.get('coveragePolicy') != COVERAGE_POLICY + or identity.get('suspendedCoveragePolicy') != SUSPENDED_COVERAGE_POLICY): + raise ValueError('A new schema-2 native coverage and reached-prefix acquisition is required') + + +def read(path): + return json.loads(path.read_text(encoding='utf-8')) + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def freeze(path, value): + if path.exists(): + if read(path) != value: + raise ValueError(f'Persisted campaign identity changed: {path}') + return + + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + '.tmp') + temporary.write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8') + temporary.replace(path) + + +def invoke(calls, arguments, log, timeout): + log.parent.mkdir(parents=True, exist_ok=True) + with log.open('w', encoding='utf-8') as output: + try: + result = subprocess.run([str(calls), *map(str, arguments)], stdout=output, + stderr=subprocess.STDOUT, timeout=timeout, check=False) + return {'exitCode': result.returncode, 'log': str(log)} + except subprocess.TimeoutExpired: + return {'exitCode': None, 'timeoutSeconds': timeout, 'log': str(log)} + + +def select_classes(ledger, eligible): + if ledger is None: + return sorted(eligible), [] + + decisions = ledger['decisions'] + chosen = [] + missing = [] + used = set() + for selected in sorted((item for item in decisions if item['origin'] == 'selected'), + key=lambda item: item['order']): + alternatives = [selected] + sorted( + (item for item in decisions if item['origin'] == 'reserve' + and item['projectId'] == selected['projectId'] and item['stratum'] == selected['stratum']), + key=lambda item: item['order']) + replacement = next((item for item in alternatives + if item.get('classId') in eligible and item['classId'] not in used), None) + if replacement is None: + missing.append({'identity': selected['identity'], 'projectId': selected['projectId'], + 'stratum': selected['stratum'], 'reason': 'No eligible preordered same-stratum reserve'}) + else: + used.add(replacement['classId']) + chosen.append(replacement['classId']) + + return chosen, missing + + +def admits_primary_comparison(preflight): + if preflight.get('admissionVersion') != 1: + raise ValueError('Legacy profile-dependent preflight cannot admit a new scientific comparison') + + return preflight['eligible'] and preflight['primaryInferenceEligible'] is True + + +def require_native_acquisition_health(output, stage, records): + if (output / 'acquisition-health-failure.json').exists(): + raise RuntimeError('Saved native acquisition failure invalidates the entire acquisition attempt') + + failures = [{'classId': record.get('classId', record.get('identity', {}).get('classId')), + 'nativeInfrastructureFailures': record['nativeInfrastructureFailures']} + for record in records if record.get('nativeInfrastructureFailures')] + if failures: + freeze(output / 'acquisition-health-failure.json', {'stage': stage, 'failures': failures}) + raise RuntimeError('Native instrumentation corruption invalidates the entire acquisition attempt') + + +def native_replay_failure(event, statement_ids): + original = event.get('original') + instrumented = event.get('instrumented') + if instrumented is not None and statement_ids is not None: + foreign = set(instrumented.get('coveredStatementKeys', [])) - statement_ids + if foreign: + return f'Instrumentation reported foreign statement IDs: {sorted(foreign)}' + if original is None or instrumented is None: + return None + if original['inputs'] != instrumented['inputs']: + return 'Instrumentation changed the frozen native input transport' + + original_observation = original['observation'] + instrumented_observation = instrumented['observation'] + if original_observation['completion'] != instrumented_observation['completion']: + return 'Instrumentation changed native completion' + original_representable = original_observation.get('representable', True) + instrumented_representable = instrumented_observation.get('representable', True) + if original_representable != instrumented_representable: + return 'Instrumentation changed native observability or observation schema' + if original_representable and original_observation != instrumented_observation: + return 'Instrumentation changed native returned values or receiver state' + + return None + + +def saved_replay_failures(output): + journals = [(path, path.parent.parent.name) + for path in sorted((output / 'prepared').glob('*/preflight/replays.jsonl'))] + journals.extend((path, path.relative_to(output / 'cells').parts[0]) + for path in sorted((output / 'cells').rglob('replays.jsonl'))) + universes = {} + failures = [] + for path, class_id in journals: + if class_id not in universes: + record = output / 'prepared' / class_id / 'class.json' + universes[class_id] = ({statement['id'] for statement in + read(record)['instrumentation']['universe']['statements']} + if record.exists() else None) + for line in path.read_text(encoding='utf-8').splitlines(): + if not line.strip(): + continue + event = json.loads(line) + failure = native_replay_failure(event, universes[class_id]) + if failure is not None: + failures.append({'classId': class_id, 'nativeInfrastructureFailures': [failure], + 'journal': str(path), 'eventIndex': event['index']}) + + return failures + + +def require_saved_acquisition_health(output): + require_native_acquisition_health(output, 'saved', []) + records = [read(path) for path in sorted((output / 'prepared').glob('*/preflight.json'))] + records.extend(read(path) for path in sorted((output / 'cells').rglob('result.json'))) + eligibility = output / 'eligibility.json' + if eligibility.exists(): + records.extend(read(eligibility)) + records.extend(saved_replay_failures(output)) + + require_native_acquisition_health(output, 'saved', records) + + +def scheduled_cells(plan, budgets=BUDGETS): + if plan['budgetsMillis'] != list(BUDGETS) or any(budget not in BUDGETS for budget in budgets): + raise ValueError('A phase must preserve the full frozen 30/120-second protocol') + + for budget in budgets: + for class_index, class_id in enumerate(plan['classIds']): + for seed in plan['seeds']: + rotation = (class_index + seed) % len(PROFILES) + profiles = PROFILES[rotation:] + PROFILES[:rotation] + for profile in profiles: + yield budget, class_id, seed, profile + + +def preserves_input_values(prepared, declared): + """Kotlin writes domain defaults; every explicitly frozen value must still agree.""" + if isinstance(declared, dict): + if not isinstance(prepared, dict): + return False + return all(preserves_input_values(prepared.get(key), value) for key, value in declared.items()) + if isinstance(declared, list): + if not isinstance(prepared, list) or len(prepared) != len(declared): + return False + return all(preserves_input_values(actual, expected) for actual, expected in zip(prepared, declared)) + return prepared == declared + + +def cell_directory(output, class_id, seed, budget, profile): + return output / 'cells' / class_id / str(seed) / str(budget) / profile + + +def validate_result(result, expected, prepared, universe): + if result.get('nativeInfrastructureFailures'): + raise ValueError('Native instrumentation corruption invalidates the acquisition attempt') + + identity = result['identity'] + validate_measurement_protocol(identity) + for key, value in expected.items(): + if identity.get(key) != value: + raise ValueError(f'Paired cell identity differs for {key}') + + if identity['preparedSha256'] != sha(prepared / identity['classId'] / 'class.json'): + raise ValueError('Prepared class identity differs') + if identity['preflightSha256'] != sha(prepared / identity['classId'] / 'preflight.json'): + raise ValueError('Frozen preflight identity differs') + if identity['universeSha256'] != universe['universeSha256']: + raise ValueError('Statement universe identity differs') + + ids = {statement['id'] for statement in universe['statements']} + previous = set() + previous_strict = set() + candidates = result['search']['search']['candidates'] + for candidate in candidates: + origin = candidate['origin'] + if origin not in ('TERMINATED', 'SUSPENDED') or candidate['emittedAtMillis'] < 0: + raise ValueError('Invalid candidate origin or emission time') + if origin == 'SUSPENDED' and (candidate.get('completion') is not None + or candidate.get('observation') is not None): + raise ValueError('Suspended input contains an invented terminal observation') + if origin == 'TERMINATED' and (candidate.get('completion') not in ('RETURNED', 'THREW') + or candidate.get('suspendedAtMachineStop', False)): + raise ValueError('Invalid terminal candidate provenance') + for point in result['checkpoints']: + covered = set(point['coveredStatementIds']) + uncovered = set(point['uncoveredStatementIds']) + if point['totalStatements'] != len(ids) or covered & uncovered or covered | uncovered != ids: + raise ValueError('Invalid exact statement numerator/denominator') + if not previous <= covered: + raise ValueError('Coverage history is not monotone') + if point['searchBudgetMillis'] > identity['budgetMillis']: + raise ValueError('Coverage checkpoint exceeds cell budget') + strict = set(point['strictCoveredStatementIds']) + strict_uncovered = set(point['strictUncoveredStatementIds']) + if strict & strict_uncovered or strict | strict_uncovered != ids or not strict <= covered: + raise ValueError('Invalid strict secondary statement coverage') + if not previous_strict <= strict: + raise ValueError('Strict coverage history is not monotone') + final = point['searchBudgetMillis'] == identity['budgetMillis'] + selected = [candidate for candidate in candidates + if (candidate['origin'] == 'SUSPENDED' and candidate.get('suspendedAtMachineStop', False) and final) + or (not candidate.get('suspendedAtMachineStop', False) + and candidate['emittedAtMillis'] <= point['searchBudgetMillis'])] + post_stop = sum(candidate.get('suspendedAtMachineStop', False) for candidate in selected) + suspended = sum(candidate['origin'] == 'SUSPENDED' for candidate in selected) + if (point['emittedCandidates'] != len(selected) or point['suspendedCandidates'] != suspended + or point['postStopSuspendedCandidates'] != post_stop + or point['includesSuspendedExtraction'] != final): + raise ValueError('Suspended extraction checkpoint attribution differs') + if point['acceptedCandidates'] + point['rejectedCandidates'] != len(selected): + raise ValueError('Native candidate accounting differs') + if (point['strictAcceptedCandidates'] + point['strictRejectedCandidates'] + + point['strictOracleNotApplicable'] != len(selected) + or point['strictOracleNotApplicable'] != suspended): + raise ValueError('Strict secondary candidate accounting differs') + previous = covered + previous_strict = strict + + if not result['checkpoints'] or result['checkpoints'][-1]['searchBudgetMillis'] != identity['budgetMillis']: + raise ValueError('Final requested-budget coverage checkpoint is absent') + + +def summarize(output, prepared, plan, scheduled_budgets=BUDGETS): + require_saved_acquisition_health(output) + manifest = read(prepared / 'manifest.json') + validate_measurement_protocol(manifest) + rows = [] + failures = [] + pending = [] + pairs = [] + for class_id in plan['classIds']: + record = read(prepared / class_id / 'class.json') + universe = record['instrumentation']['universe'] + for seed in manifest['seeds']: + for budget in BUDGETS: + pair = [] + for profile in PROFILES: + path = cell_directory(output, class_id, seed, budget, profile) / 'result.json' + if not path.exists(): + missing = {'classId': class_id, 'seed': seed, 'budgetMillis': budget, + 'profile': profile, 'reason': 'Missing completed cell'} + if budget in scheduled_budgets: + failures.append(missing) + else: + pending.append(missing) + continue + + result = read(path) + expected = {'experimentId': manifest['experimentId'], 'toolRevision': manifest['toolRevision'], + 'nativeFrontendRevision': manifest['nativeFrontendRevision'], + 'manifestSha256': sha(prepared / 'manifest.json'), + 'classId': class_id, 'seed': seed, 'budgetMillis': budget, 'profile': profile} + try: + validate_result(result, expected, prepared, universe) + except (KeyError, ValueError) as error: + failures.append({**expected, 'reason': str(error), 'path': str(path)}) + continue + + search = result['search']['search'] + final = result['checkpoints'][-1] + row = {'classId': class_id, 'profile': profile, 'seed': seed, 'budgetMillis': budget, + 'covered': len(final['coveredStatementIds']), 'total': final['totalStatements'], + 'fraction': len(final['coveredStatementIds']) / final['totalStatements'], + 'status': search['status'], 'acceptedCandidates': final['acceptedCandidates'], + 'machineStopReason': search.get('machineStopReason'), + 'rejectedCandidates': final['rejectedCandidates'], + 'strictCovered': len(final['strictCoveredStatementIds']), + 'strictAcceptedCandidates': final['strictAcceptedCandidates'], + 'strictRejectedCandidates': final['strictRejectedCandidates'], + 'strictOracleNotApplicable': final['strictOracleNotApplicable'], + 'suspendedCandidates': final['suspendedCandidates'], + 'postStopSuspendedCandidates': final['postStopSuspendedCandidates'], + 'suspendedExtractionElapsedMillis': search['suspendedExtractionElapsedMillis'], + 'frontierFlushElapsedMillis': search['frontierFlushElapsedMillis'], + 'searchElapsedMillis': search['searchElapsedMillis'], + 'setupElapsedMillis': search['preparationElapsedMillis'] + search['machineSetupElapsedMillis'], + 'replayElapsedMillis': result['replayElapsedMillis'], + 'solverCalls': result['search'].get('solverCalls'), + 'solverElapsedMillis': result['search'].get('solverElapsedMillis'), + 'runtimeLimited': search['runtimeLimited'], 'engineFailed': search['engineFailed'], + 'candidateCapReached': search['candidateCapReached'], + 'replayErrors': result['replayErrors'], 'oracleRejections': result['oracleRejections'], + 'nativeCoverageRejections': result['nativeCoverageRejections']} + rows.append(row) + pair.append(row) + + if len(pair) == len(PROFILES): + pairs.append({'classId': class_id, 'seed': seed, 'budgetMillis': budget, + 'coveredByProfile': {row['profile']: row['covered'] for row in pair}, + 'strictCoveredByProfile': {row['profile']: row['strictCovered'] for row in pair}, + 'total': pair[0]['total'], + 'measurementFailures': any(row['engineFailed'] or row['runtimeLimited'] + or row['candidateCapReached'] or row['replayErrors'] + or row['nativeCoverageRejections'] for row in pair), + 'symbolicOracleDisagreement': any(row['oracleRejections'] for row in pair)}) + + summary = {'schemaVersion': 2, 'coveragePolicy': COVERAGE_POLICY, + 'suspendedCoveragePolicy': SUSPENDED_COVERAGE_POLICY, 'experimentId': manifest['experimentId'], + 'classCount': len(plan['classIds']), 'completePairedGroups': len(pairs), + 'incompleteOrInvalidCells': failures, 'missingStrata': plan['missingStrata'], + 'scheduledBudgetsMillis': list(scheduled_budgets), 'pendingOtherPhaseCells': pending, + 'complete': bool(plan['classIds']) and not failures and not pending, + 'rows': rows, 'pairs': pairs} + (output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8') + if rows: + with (output / 'coverage.csv').open('w', encoding='utf-8', newline='') as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + lines = [f"Experiment: {manifest['experimentId']}", '', + 'Primary coverage requires complete agreeing native original/instrumented replay. Strict symbolic agreement is reported separately.', + 'In-search reached-prefix inputs use their actual emission time; post-stop frontier inputs appear only in the final budget checkpoint.', + f"Classes: {len(plan['classIds'])}; complete paired groups: {len(pairs)}; missing strata: {len(plan['missingStrata'])}.", + f"Complete acquisition: {summary['complete']}; pending cells in the other phase: {len(pending)}; invalid or missing scheduled cells: {len(failures)}.", + '', '| Class | Seed | Budget (s) | EMPTY_STOP | EMPTY_FRESH | FROZEN_STOP | FROZEN_FRESH |', + '|---|---:|---:|---:|---:|---:|---:|'] + for pair in pairs: + values = [f"{pair['coveredByProfile'][profile]}/{pair['total']}" for profile in PROFILES] + lines.append(f"| {pair['classId']} | {pair['seed']} | {pair['budgetMillis'] // 1000} | " + ' | '.join(values) + ' |') + lines.extend(['', '| Strict secondary class | Seed | Budget (s) | EMPTY_STOP | EMPTY_FRESH | FROZEN_STOP | FROZEN_FRESH |', + '|---|---:|---:|---:|---:|---:|---:|']) + for pair in pairs: + values = [f"{pair['strictCoveredByProfile'][profile]}/{pair['total']}" for profile in PROFILES] + lines.append(f"| {pair['classId']} | {pair['seed']} | {pair['budgetMillis'] // 1000} | " + ' | '.join(values) + ' |') + lines.extend(['', 'These are paired observations under the frozen domains. Unsupported candidates and missing strata remain in the eligibility ledger.', + 'Engine-suited datasets must be reported separately from the random/stratified corpus. No statistical superiority is inferred from one seed.']) + (output / 'RESULTS.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--calls', type=Path, required=True) + parser.add_argument('--manifest', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--selection-ledger', type=Path) + parser.add_argument('--preflight-ms', type=int, default=30_000) + parser.add_argument('--preflight-only', action='store_true') + parser.add_argument('--summarize-only', action='store_true') + parser.add_argument('--budget-ms', type=int, choices=BUDGETS, + help='Acquire or check one phase while retaining both budgets in the frozen plan') + args = parser.parse_args() + scheduled_budgets = (args.budget_ms,) if args.budget_ms is not None else BUDGETS + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + require_saved_acquisition_health(output) + prepared = output / 'prepared' + calls = args.calls.resolve() + ledger = read(args.selection_ledger) if args.selection_ledger else None + input_manifest = read(args.manifest) + validate_measurement_protocol(input_manifest) + if input_manifest['budgetsMillis'] != list(BUDGETS): + raise ValueError('Class budgets must be exactly 30 and 120 seconds') + freeze(output / 'campaign-input.json', {'manifest': input_manifest, 'selectionLedger': ledger, + 'preflightMillis': args.preflight_ms}) + if not (prepared / 'manifest.json').exists(): + outcome = invoke(calls, ['class-prepare', args.manifest.resolve(), prepared], + output / 'prepare.log', timeout=600) + freeze(output / 'prepare-process.json', outcome) + if outcome['exitCode'] != 0: + raise RuntimeError(f'Class preparation failed: {outcome}') + + manifest = read(prepared / 'manifest.json') + if not preserves_input_values(manifest, input_manifest): + raise ValueError('Prepared manifest differs from campaign input') + plan_path = output / 'run-plan.json' + if args.summarize_only: + summary = summarize(output, prepared, read(plan_path), scheduled_budgets=scheduled_budgets) + print(json.dumps(summary, indent=2)) + if summary['incompleteOrInvalidCells']: + raise RuntimeError('Incomplete or invalid scheduled campaign phase; see summary.json') + if not summary['classCount']: + raise RuntimeError('No primary admitted classes; no representative comparison was run') + return + + eligibility = [] + eligible = set() + for case in manifest['classes']: + class_id = case['classId'] + directory = prepared / class_id + failed = directory / 'preparation-failure.json' + if failed.exists(): + eligibility.append({'classId': class_id, 'eligible': False, 'preparationFailure': read(failed)}) + continue + preflight = directory / 'preflight.json' + if not preflight.exists(): + print(f'Preflight {class_id}', flush=True) + process = invoke(calls, ['class-preflight', prepared, class_id, args.preflight_ms], + output / 'logs' / f'{class_id}-preflight.log', timeout=600 + args.preflight_ms // 1000) + freeze(output / 'processes' / f'{class_id}-preflight.json', process) + if preflight.exists(): + outcome = read(preflight) + primary_admitted = admits_primary_comparison(outcome) + eligibility.append({'classId': class_id, 'eligible': outcome['eligible'], + 'primaryInferenceEligible': primary_admitted, + 'diagnosticReady': outcome['diagnosticReady'], + 'nativeSmoke': outcome.get('nativeSmoke'), + 'sourceAdmissionFailures': outcome['sourceAdmissionFailures'], + 'nativeInfrastructureFailures': outcome['nativeInfrastructureFailures'], + 'correctnessFrontierFailures': case.get('correctnessFrontierFailures', []), + 'missingRequiredScenarios': outcome['missingRequiredScenarios'], + 'reasons': outcome['reasons'], 'preflightSha256': sha(preflight)}) + if primary_admitted: + eligible.add(class_id) + else: + eligibility.append({'classId': class_id, 'eligible': False, 'reasons': ['Preflight failed before final record; see process log']}) + + chosen, missing = select_classes(ledger, eligible) + freeze(output / 'eligibility.json', eligibility) + require_native_acquisition_health(output, 'preflight', eligibility) + plan = {'schemaVersion': 2, 'coveragePolicy': COVERAGE_POLICY, + 'suspendedCoveragePolicy': SUSPENDED_COVERAGE_POLICY, 'admission': 'profile-independent-source-native-v1', + 'manifestSha256': sha(prepared / 'manifest.json'), 'classIds': chosen, + 'missingStrata': missing, 'profiles': list(PROFILES), 'budgetsMillis': list(BUDGETS), + 'seeds': manifest['seeds'], 'order': 'all 30s cells then all 120s; seeded rotation of profiles; sequential processes'} + freeze(plan_path, plan) + print(f'Frozen source/native admission: {len(chosen)} primary classes; {len(missing)} missing strata.', flush=True) + if args.preflight_only: + return + + for budget, class_id, seed, profile in scheduled_cells(plan, budgets=scheduled_budgets): + directory = cell_directory(output, class_id, seed, budget, profile) + if (directory / 'result.json').exists(): + require_native_acquisition_health(output, 'measurement', [read(directory / 'result.json')]) + continue + print(f'Measure {class_id} {profile} seed={seed} budget={budget}ms', flush=True) + process = invoke(calls, ['class-cell', prepared, class_id, profile, seed, budget, directory], + output / 'logs' / f'{class_id}-{seed}-{budget}-{profile}.log', timeout=600 + budget // 1000) + freeze(output / 'processes' / f'{class_id}-{seed}-{budget}-{profile}.json', process) + if (directory / 'result.json').exists(): + require_native_acquisition_health(output, 'measurement', [read(directory / 'result.json')]) + + summary = summarize(output, prepared, plan, scheduled_budgets=scheduled_budgets) + print(f"Finished: {len(summary['rows'])} cells, {summary['completePairedGroups']} complete paired groups.", flush=True) + if summary['incompleteOrInvalidCells']: + raise RuntimeError('Incomplete or invalid scheduled campaign phase; see summary.json') + if not chosen: + raise RuntimeError('No primary admitted classes; no representative comparison was run') + + +if __name__ == '__main__': + main() diff --git a/usvm-ts-calls/scripts/test_run_class_experiment.py b/usvm-ts-calls/scripts/test_run_class_experiment.py new file mode 100644 index 0000000000..994bb4929d --- /dev/null +++ b/usvm-ts-calls/scripts/test_run_class_experiment.py @@ -0,0 +1,447 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('campaign', Path(__file__).with_name('run-class-experiment.py')) +campaign = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(campaign) + + +class CampaignIntegrityTest(unittest.TestCase): + def make_phase_fixture(self, output): + prepared = output / 'prepared' + directory = prepared / 'class' + directory.mkdir(parents=True) + manifest = {'experimentId': 'phase-test', 'toolRevision': 'revision', + 'schemaVersion': 2, 'coveragePolicy': campaign.COVERAGE_POLICY, + 'suspendedCoveragePolicy': campaign.SUSPENDED_COVERAGE_POLICY, + 'nativeFrontendRevision': 'frontend', 'budgetsMillis': list(campaign.BUDGETS), + 'seeds': [17], 'classes': [{'classId': 'class'}]} + universe = {'universeSha256': 'universe', 'statements': [{'id': 'original'}]} + campaign.freeze(prepared / 'manifest.json', manifest) + campaign.freeze(directory / 'class.json', {'instrumentation': {'universe': universe}}) + campaign.freeze(directory / 'preflight.json', { + 'admissionVersion': 1, 'classId': 'class', 'eligible': True, + 'primaryInferenceEligible': True, 'diagnosticReady': False, + 'nativeSmoke': {'status': 'VERIFIED'}, 'sourceAdmissionFailures': [], + 'nativeInfrastructureFailures': [], 'missingRequiredScenarios': ['slow'], + 'reasons': ['Symbolic pilot reached the time budget'], + }) + plan = {'classIds': ['class'], 'seeds': [17], 'budgetsMillis': list(campaign.BUDGETS), + 'missingStrata': []} + + return prepared, manifest, plan + + def write_phase_result(self, output, prepared, manifest, budget, profile): + directory = campaign.cell_directory(output, 'class', 17, budget, profile) + identity = {'experimentId': manifest['experimentId'], 'toolRevision': manifest['toolRevision'], + 'schemaVersion': 2, 'coveragePolicy': campaign.COVERAGE_POLICY, + 'suspendedCoveragePolicy': campaign.SUSPENDED_COVERAGE_POLICY, + 'nativeFrontendRevision': manifest['nativeFrontendRevision'], + 'manifestSha256': campaign.sha(prepared / 'manifest.json'), + 'preparedSha256': campaign.sha(prepared / 'class' / 'class.json'), + 'preflightSha256': campaign.sha(prepared / 'class' / 'preflight.json'), + 'universeSha256': 'universe', 'classId': 'class', 'seed': 17, + 'budgetMillis': budget, 'profile': profile} + result = {'identity': identity, 'nativeInfrastructureFailures': [], + 'search': {'search': {'status': 'TIMEOUT', 'searchElapsedMillis': budget, + 'candidates': [], 'suspendedExtractionElapsedMillis': 0, 'frontierFlushElapsedMillis': 0, + 'preparationElapsedMillis': 0, 'machineSetupElapsedMillis': 0, + 'runtimeLimited': False, 'engineFailed': False, 'candidateCapReached': False}}, + 'checkpoints': [{'searchBudgetMillis': budget, 'totalStatements': 1, + 'coveredStatementIds': [], 'uncoveredStatementIds': ['original'], + 'strictCoveredStatementIds': [], 'strictUncoveredStatementIds': ['original'], + 'strictAcceptedCandidates': 0, 'strictRejectedCandidates': 0, + 'strictOracleNotApplicable': 0, 'emittedCandidates': 0, + 'suspendedCandidates': 0, 'postStopSuspendedCandidates': 0, + 'includesSuspendedExtraction': True, + 'acceptedCandidates': 0, 'rejectedCandidates': 0}], + 'replayElapsedMillis': 0, 'replayErrors': 0, 'oracleRejections': 0, 'nativeCoverageRejections': 0} + + campaign.freeze(directory / 'result.json', result) + + def test_phase_schedule_preserves_full_protocol_and_profile_rotation(self): + plan = {'classIds': ['first', 'second'], 'seeds': [17, 29], + 'budgetsMillis': list(campaign.BUDGETS)} + frozen = json.dumps(plan, sort_keys=True) + + full = list(campaign.scheduled_cells(plan)) + short = list(campaign.scheduled_cells(plan, budgets=(30_000,))) + long = list(campaign.scheduled_cells(plan, budgets=(120_000,))) + + self.assertEqual(full, short + long) + self.assertEqual(16, len(short)) + self.assertEqual(16, len(long)) + self.assertEqual(30_000, short[0][0]) + self.assertEqual(campaign.PROFILES[1], short[0][3]) + self.assertEqual(frozen, json.dumps(plan, sort_keys=True)) + with self.assertRaisesRegex(ValueError, 'full frozen'): + list(campaign.scheduled_cells(plan, budgets=(60_000,))) + + def test_short_phase_retains_timeouts_and_reports_pending_long_phase(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, plan = self.make_phase_fixture(output) + for profile in campaign.PROFILES: + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + + summary = campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertEqual(4, len(summary['rows'])) + self.assertTrue(all(row['status'] == 'TIMEOUT' for row in summary['rows'])) + self.assertEqual([], summary['incompleteOrInvalidCells']) + self.assertEqual(4, len(summary['pendingOtherPhaseCells'])) + self.assertEqual({120_000}, {cell['budgetMillis'] for cell in summary['pendingOtherPhaseCells']}) + self.assertFalse(summary['complete']) + + for profile in campaign.PROFILES: + self.write_phase_result(output, prepared, manifest, budget=120_000, profile=profile) + complete = campaign.summarize(output, prepared, plan) + + self.assertEqual(8, len(complete['rows'])) + self.assertEqual(2, complete['completePairedGroups']) + self.assertEqual([], complete['pendingOtherPhaseCells']) + self.assertTrue(complete['complete']) + + def test_scheduled_missing_cell_is_a_failure_not_a_pending_other_phase(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, plan = self.make_phase_fixture(output) + for profile in campaign.PROFILES[:-1]: + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + + summary = campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertEqual(1, len(summary['incompleteOrInvalidCells'])) + self.assertEqual(30_000, summary['incompleteOrInvalidCells'][0]['budgetMillis']) + self.assertEqual(4, len(summary['pendingOtherPhaseCells'])) + self.assertFalse(summary['complete']) + + def test_phase_resume_keeps_full_frozen_identity_and_launches_only_requested_budget(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + + def fake_invoke(calls, arguments, log, timeout): + self.assertEqual('class-cell', arguments[0]) + self.write_phase_result(output, prepared, manifest, budget=arguments[5], profile=arguments[3]) + return {'exitCode': 0} + + base_args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output)] + with patch('sys.argv', [*base_args, '--budget-ms', '30000']), patch.object( + campaign, 'invoke', side_effect=fake_invoke) as short_calls, patch('builtins.print'): + campaign.main() + frozen_input = (output / 'campaign-input.json').read_bytes() + frozen_plan = (output / 'run-plan.json').read_bytes() + + with patch('sys.argv', [*base_args, '--budget-ms', '120000']), patch.object( + campaign, 'invoke', side_effect=fake_invoke) as long_calls, patch('builtins.print'): + campaign.main() + + self.assertEqual(4, short_calls.call_count) + self.assertEqual({30_000}, {call.args[1][5] for call in short_calls.call_args_list}) + self.assertEqual(4, long_calls.call_count) + self.assertEqual({120_000}, {call.args[1][5] for call in long_calls.call_args_list}) + self.assertEqual(frozen_input, (output / 'campaign-input.json').read_bytes()) + self.assertEqual(frozen_plan, (output / 'run-plan.json').read_bytes()) + self.assertEqual(list(campaign.BUDGETS), campaign.read(output / 'run-plan.json')['budgetsMillis']) + self.assertTrue(campaign.read(output / 'summary.json')['complete']) + + def test_source_rejected_preflight_may_omit_null_native_smoke(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + preflight_path = prepared / 'class' / 'preflight.json' + preflight = campaign.read(preflight_path) + preflight.pop('nativeSmoke') + preflight.update(eligible=False, primaryInferenceEligible=False, + sourceAdmissionFailures=['TOP_LEVEL_INITIALIZATION_UNSUPPORTED: offset 0']) + preflight_path.write_text(json.dumps(preflight) + '\n') + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output), '--preflight-only'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke, patch('builtins.print'): + campaign.main() + + invoke.assert_not_called() + eligibility = campaign.read(output / 'eligibility.json') + self.assertIsNone(eligibility[0]['nativeSmoke']) + self.assertFalse(eligibility[0]['primaryInferenceEligible']) + self.assertEqual([], campaign.read(output / 'run-plan.json')['classIds']) + self.assertFalse((output / 'summary.json').exists()) + + def test_saved_corruption_in_later_unselected_class_blocks_any_resume_launch(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + _, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + later = output / 'cells' / 'unselected' / '29' / '120000' / 'FROZEN_FRESH' / 'result.json' + campaign.freeze(later, {'identity': {'classId': 'unselected'}, + 'nativeInfrastructureFailures': ['Native completion changed']}) + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output), '--budget-ms', '30000'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke: + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.main() + + invoke.assert_not_called() + self.assertFalse((output / 'run-plan.json').exists()) + self.assertEqual('unselected', campaign.read(output / 'acquisition-health-failure.json')['failures'][0]['classId']) + + def test_summarize_refuses_corruption_outside_plan_before_writing_results(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, _, plan = self.make_phase_fixture(output) + campaign.freeze(prepared / 'excluded' / 'preflight.json', { + 'classId': 'excluded', 'nativeInfrastructureFailures': ['Native receiver changed'], + }) + + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertFalse((output / 'summary.json').exists()) + self.assertFalse((output / 'RESULTS.md').exists()) + + def test_sticky_health_receipt_blocks_summarize_only_before_any_launch(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + campaign.freeze(output / 'acquisition-health-failure.json', {'stage': 'previous', 'failures': []}) + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(output / 'missing.json'), + '--output', str(output), '--summarize-only'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke: + with self.assertRaisesRegex(RuntimeError, 'Saved native acquisition failure'): + campaign.main() + + invoke.assert_not_called() + self.assertFalse((output / 'summary.json').exists()) + self.assertFalse((output / 'campaign-input.json').exists()) + + def test_interrupted_native_replay_mismatch_blocks_resume_before_result(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + manifest_path = output / 'input-manifest.json' + campaign.freeze(manifest_path, manifest) + journal = output / 'cells' / 'class' / '17' / '120000' / 'FROZEN_FRESH' / 'replays.jsonl' + journal.parent.mkdir(parents=True) + original = {'inputs': [], 'observation': {'completion': 'RETURNED', 'representable': True, + 'values': {'state': {'kind': 'boolean', 'value': True}}}} + instrumented = {'inputs': [], 'observation': {'completion': 'RETURNED', 'representable': True, + 'values': {'state': {'kind': 'boolean', 'value': False}}}, + 'coveredStatementKeys': ['original']} + event = {'index': 0, 'original': original, 'instrumented': instrumented, + 'oracle': {'accepted': False}} + journal.write_text(json.dumps(event) + '\n') + args = ['campaign', '--calls', str(output / 'calls'), '--manifest', str(manifest_path), + '--output', str(output), '--budget-ms', '30000'] + + with patch('sys.argv', args), patch.object(campaign, 'invoke') as invoke: + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.main() + + invoke.assert_not_called() + self.assertFalse((journal.parent / 'result.json').exists()) + self.assertEqual('class', campaign.read(output / 'acquisition-health-failure.json')['failures'][0]['classId']) + with self.assertRaisesRegex(RuntimeError, 'Saved native acquisition failure'): + campaign.summarize(output, prepared, {'classIds': []}) + + def test_interrupted_foreign_statement_is_corruption_even_without_original_replay(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, _, plan = self.make_phase_fixture(output) + journal = prepared / 'class' / 'preflight' / 'replays.jsonl' + journal.parent.mkdir() + event = {'index': 0, 'instrumented': {'coveredStatementKeys': ['foreign']}, + 'oracle': {'accepted': False}, 'error': 'original timed out'} + journal.write_text(json.dumps(event) + '\n') + + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.summarize(output, prepared, plan) + + self.assertFalse((output / 'summary.json').exists()) + receipt = campaign.read(output / 'acquisition-health-failure.json') + self.assertIn('foreign statement IDs', receipt['failures'][0]['nativeInfrastructureFailures'][0]) + + def test_lawful_incomplete_or_missing_native_replay_does_not_prove_corruption(self): + original = {'inputs': [], 'observation': {'completion': 'THREW', 'representable': False, + 'values': {}, 'diagnostic': 'original exception'}} + instrumented = {'inputs': [], 'observation': {'completion': 'THREW', 'representable': False, + 'values': {}, 'diagnostic': 'instrumented exception'}, 'coveredStatementKeys': ['original']} + + self.assertIsNone(campaign.native_replay_failure( + {'original': original, 'instrumented': instrumented}, statement_ids={'original'})) + self.assertIsNone(campaign.native_replay_failure( + {'instrumented': instrumented, 'error': 'original timed out'}, statement_ids={'original'})) + + def test_reserve_replacement_cannot_cross_project_or_stratum(self): + selected = {'origin': 'selected', 'order': 0, 'identity': 'selected', 'projectId': 'p', + 'stratum': 'complex', 'classId': 'original'} + reserves = [dict(selected, origin='reserve', order=0, classId='foreign', projectId='q'), + dict(selected, origin='reserve', order=1, classId='simple', stratum='moderate'), + dict(selected, origin='reserve', order=2, classId='second'), + dict(selected, origin='reserve', order=3, classId='third')] + + chosen, missing = campaign.select_classes({'decisions': [selected, *reserves]}, + {'foreign', 'simple', 'second', 'third'}) + + self.assertEqual(['second'], chosen) + self.assertEqual([], missing) + chosen, missing = campaign.select_classes({'decisions': [selected, *reserves]}, {'foreign', 'simple'}) + self.assertEqual([], chosen) + self.assertEqual('selected', missing[0]['identity']) + + def test_slow_or_incomplete_pilot_does_not_select_the_primary_population(self): + slow = {'admissionVersion': 1, 'eligible': True, 'primaryInferenceEligible': True, + 'diagnosticReady': False, 'missingRequiredScenarios': ['one', 'two'], + 'search': {'status': 'TIMEOUT'}} + broken = dict(slow, primaryInferenceEligible=False, + correctnessFrontierFailures=['Known lowering defect']) + selected = {'origin': 'selected', 'order': 0, 'identity': 'slow', 'projectId': 'p', + 'stratum': 'complex', 'classId': 'slow'} + + admitted = {class_id for class_id, preflight in [('slow', slow), ('broken', broken)] + if campaign.admits_primary_comparison(preflight)} + chosen, missing = campaign.select_classes({'decisions': [selected]}, admitted) + + self.assertEqual({'slow'}, admitted) + self.assertEqual(['slow'], chosen) + self.assertEqual([], missing) + + def test_native_corruption_stops_the_whole_attempt_instead_of_dropping_one_class(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + slow = {'classId': 'slow', 'missingRequiredScenarios': ['method'], + 'nativeInfrastructureFailures': [], 'replayErrors': 1} + broken = {'classId': 'broken', 'nativeInfrastructureFailures': ['Receiver state changed']} + + campaign.require_native_acquisition_health(output, 'preflight', [slow]) + self.assertFalse((output / 'acquisition-health-failure.json').exists()) + + with self.assertRaisesRegex(RuntimeError, 'entire acquisition attempt'): + campaign.require_native_acquisition_health(output, 'preflight', [slow, broken]) + + receipt = campaign.read(output / 'acquisition-health-failure.json') + self.assertEqual('preflight', receipt['stage']) + self.assertEqual('broken', receipt['failures'][0]['classId']) + self.assertFalse((output / 'run-plan.json').exists()) + + def test_legacy_pilot_success_cannot_silently_change_the_new_admission_protocol(self): + with self.assertRaisesRegex(ValueError, 'profile-dependent'): + campaign.admits_primary_comparison({'eligible': True, 'missingRequiredScenarios': []}) + + def test_same_reserve_cannot_fill_two_selected_slots(self): + selected = {'origin': 'selected', 'order': 0, 'identity': 'one', 'projectId': 'p', 'stratum': 'complex'} + other = dict(selected, order=1, identity='two') + reserve = dict(selected, origin='reserve', classId='reserve') + + chosen, missing = campaign.select_classes({'decisions': [selected, other, reserve]}, {'reserve'}) + + self.assertEqual(['reserve'], chosen) + self.assertEqual('two', missing[0]['identity']) + + def test_serializer_defaults_preserve_all_explicit_frozen_values(self): + declared = {'classes': [{'inputs': [{'domain': {'kind': 'string', 'maxLength': 8}}]}], 'seeds': [17]} + prepared = {'classes': [{'inputs': [{'domain': {'kind': 'string', 'minLength': 0, 'maxLength': 8}}]}], + 'seeds': [17], 'schemaVersion': 1} + + self.assertTrue(campaign.preserves_input_values(prepared, declared)) + prepared['classes'][0]['inputs'][0]['domain']['maxLength'] = 9 + self.assertFalse(campaign.preserves_input_values(prepared, declared)) + self.assertFalse(campaign.preserves_input_values({'seeds': [18]}, declared)) + + def test_saved_plan_rejects_changed_runtime_or_eligibility(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'plan.json' + campaign.freeze(path, {'classIds': ['first'], 'revision': 'one'}) + + with self.assertRaisesRegex(ValueError, 'identity changed'): + campaign.freeze(path, {'classIds': ['second'], 'revision': 'one'}) + + def test_coverage_cannot_admit_a_foreign_statement(self): + with tempfile.TemporaryDirectory() as temporary: + prepared = Path(temporary) + directory = prepared / 'class' + directory.mkdir() + (directory / 'class.json').write_text('{}') + (directory / 'preflight.json').write_text('{}') + identity = {'classId': 'class', 'budgetMillis': 30_000, 'universeSha256': 'universe', + 'schemaVersion': 2, 'coveragePolicy': campaign.COVERAGE_POLICY, + 'suspendedCoveragePolicy': campaign.SUSPENDED_COVERAGE_POLICY, + 'preparedSha256': campaign.sha(directory / 'class.json'), + 'preflightSha256': campaign.sha(directory / 'preflight.json')} + result = {'identity': identity, 'search': {'search': {'candidates': []}}, + 'checkpoints': [{'searchBudgetMillis': 30_000, + 'totalStatements': 1, 'coveredStatementIds': ['foreign'], 'uncoveredStatementIds': []}]} + universe = {'universeSha256': 'universe', 'statements': [{'id': 'original'}]} + + with self.assertRaisesRegex(ValueError, 'numerator/denominator'): + campaign.validate_result(result, {'classId': 'class'}, prepared, universe) + + def test_new_measurement_cannot_reinterpret_sealed_legacy_results(self): + with self.assertRaisesRegex(ValueError, 'schema-2'): + campaign.validate_measurement_protocol({'schemaVersion': 1}) + + def test_native_primary_and_strict_secondary_are_reported_separately(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, plan = self.make_phase_fixture(output) + for profile in campaign.PROFILES: + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + path = campaign.cell_directory(output, 'class', 17, 30_000, profile) / 'result.json' + result = campaign.read(path) + result['search']['search']['candidates'] = [ + {'origin': 'TERMINATED', 'completion': 'RETURNED', 'emittedAtMillis': 5}, + {'origin': 'SUSPENDED', 'suspendedAtMachineStop': True, 'emittedAtMillis': 30_050}, + {'origin': 'SUSPENDED', 'suspendedAtMachineStop': False, 'emittedAtMillis': 30_051}, + ] + final = result['checkpoints'][-1] + final.update(coveredStatementIds=['original'], uncoveredStatementIds=[], + acceptedCandidates=2, emittedCandidates=2, strictRejectedCandidates=1, + strictOracleNotApplicable=1, suspendedCandidates=1, postStopSuspendedCandidates=1) + result['oracleRejections'] = 1 + path.write_text(json.dumps(result) + '\n') + + summary = campaign.summarize(output, prepared, plan, scheduled_budgets=(30_000,)) + + self.assertEqual([], summary['incompleteOrInvalidCells']) + self.assertEqual({1}, {row['covered'] for row in summary['rows']}) + self.assertEqual({0}, {row['strictCovered'] for row in summary['rows']}) + self.assertFalse(summary['pairs'][0]['measurementFailures']) + self.assertTrue(summary['pairs'][0]['symbolicOracleDisagreement']) + self.assertEqual({1}, {row['postStopSuspendedCandidates'] for row in summary['rows']}) + + def test_post_stop_extraction_cannot_be_backdated_into_intermediate_checkpoint(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + prepared, manifest, _ = self.make_phase_fixture(output) + profile = campaign.PROFILES[0] + self.write_phase_result(output, prepared, manifest, budget=30_000, profile=profile) + path = campaign.cell_directory(output, 'class', 17, 30_000, profile) / 'result.json' + result = campaign.read(path) + result['search']['search']['candidates'] = [ + {'origin': 'SUSPENDED', 'suspendedAtMachineStop': True, 'emittedAtMillis': 10}, + ] + intermediate = dict(result['checkpoints'][0], searchBudgetMillis=1_000, + includesSuspendedExtraction=False, emittedCandidates=1, + acceptedCandidates=1, suspendedCandidates=1, postStopSuspendedCandidates=1, + strictOracleNotApplicable=1) + result['checkpoints'].insert(0, intermediate) + universe = campaign.read(prepared / 'class' / 'class.json')['instrumentation']['universe'] + + with self.assertRaisesRegex(ValueError, 'checkpoint attribution'): + campaign.validate_result(result, {}, prepared, universe) + + +if __name__ == '__main__': + unittest.main() diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt new file mode 100644 index 0000000000..35eb2ee287 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassCoverageEngine.kt @@ -0,0 +1,495 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import org.jacodb.ets.model.EtsClassSignature +import org.jacodb.ets.model.EtsMethod +import org.usvm.CoverageZone +import org.usvm.PathSelectorFairnessStrategy +import org.usvm.SolverType +import org.usvm.StateCollectionStrategy +import org.usvm.UMachineOptions +import org.usvm.machine.TsAnalysisStopReason +import org.usvm.machine.TsEntryPointTimeQuantumEvent +import org.usvm.machine.TsInterpreterObserver +import org.usvm.machine.TsMachine +import org.usvm.machine.TsOptions +import org.usvm.machine.TsRuntimeFeatureLimitationEvent +import org.usvm.machine.TsSolverQueryEvent +import org.usvm.machine.TsStoppedStatesObserver +import org.usvm.machine.call.TsUnknownCallDecision +import org.usvm.machine.call.TsUnknownCallEvent +import org.usvm.machine.call.TsUnknownCallModelSelection +import org.usvm.machine.interpreter.TsInitialStateSolverUnknownException +import org.usvm.machine.state.TsState +import org.usvm.statistics.CompositeUMachineObserver +import org.usvm.statistics.UMachineObserver +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.usvm.PlainObjectInputShapes +import kotlin.time.Duration +import kotlin.time.TimeSource + +internal data class CallsClassSearchRequest( + val harness: CallsClassPreparedHarness, + val profile: CallsExperimentProfile, + val frozenModelIds: Set, + val expectedNativeFrontendRevision: String, + val seed: Long, + val budget: Duration, + val timeQuantum: Duration, + val solverQueryLimit: Duration, + val solverType: SolverType, + val candidateCap: Int, + val requireEntryPointWitness: Boolean = false, + val onCandidate: (CallsCoverageCandidate) -> Unit = {}, +) { + init { + require(budget > Duration.ZERO && timeQuantum > Duration.ZERO && timeQuantum <= budget) + require(solverQueryLimit > Duration.ZERO && solverQueryLimit <= budget) + require(candidateCap > 0) + } +} + +@Serializable +internal data class CallsClassScenarioMetrics( + val scenarioId: String, + val executedSteps: Long, + val stepWallTimeMillis: Long, + val completedCandidates: Int, + val suspendedCandidates: Int = 0, +) + +@Serializable +internal data class CallsClassMethodMetrics( + val methodSignature: String, + val executedSteps: Long, + val stepWallTimeNanos: Long, +) + +/** Consecutive visits to an entry point; a single remaining entry can span multiple selector quanta. */ +@Serializable +internal data class CallsClassSchedulingVisit( + val scenarioId: String, + val startedAtMillis: Long, + val elapsedMillis: Long, + val requestedQuantumMillis: Long, +) + +@Serializable +internal data class CallsClassTimeQuantum( + val scenarioId: String, + val startedAtMillis: Long, + val requestedQuantumMillis: Long, + val elapsedNanos: Long, +) + +@Serializable +internal data class CallsClassSolverMetrics( + val phase: String, + val scenarioId: String?, + val methodSignature: String?, + val calls: Long, + val elapsedNanos: Long, + val failures: Long, + val statusCounts: Map, +) + +/** Ordered observer notifications for the entire search, without implying ancestry of any candidate. */ +@Serializable +internal data class CallsClassUnknownCallEvent( + val eventIndex: Int, + val callSite: CallsUnknownCallSite, + val callerMethodSignature: String, + val callee: CallsUnknownCallCallee, + val failureReason: String, + val decision: CallsUnknownCallDecisionKind, + val outcome: String, + val modelId: String? = null, + val residualPolicy: String? = null, +) { + init { + require(eventIndex >= 1) { "Unknown-call event index must be positive" } + require((decision == CallsUnknownCallDecisionKind.MODEL_APPLIED) == (modelId != null)) { + "Exactly a model-applied decision must carry a model ID" + } + require((decision == CallsUnknownCallDecisionKind.RESIDUAL_FALLBACK) == (residualPolicy != null)) { + "Exactly a residual-fallback decision must carry a residual policy" + } + } +} + +internal fun TsUnknownCallEvent.toCallsClassUnknownCallEvent(eventIndex: Int): CallsClassUnknownCallEvent { + val selectedDecision = decision + + return CallsClassUnknownCallEvent( + eventIndex = eventIndex, + callSite = callSite.toCallsUnknownCallSite(), + callerMethodSignature = callSite.location.method.signature.toString(), + callee = callee.toCallsUnknownCallCallee(), + failureReason = failureReason.name, + decision = when (selectedDecision) { + is TsUnknownCallDecision.ModelApplied -> CallsUnknownCallDecisionKind.MODEL_APPLIED + is TsUnknownCallDecision.ResidualFallback -> CallsUnknownCallDecisionKind.RESIDUAL_FALLBACK + }, + outcome = outcome.name, + modelId = (selectedDecision as? TsUnknownCallDecision.ModelApplied)?.modelId, + residualPolicy = (selectedDecision as? TsUnknownCallDecision.ResidualFallback)?.policy?.name, + ) +} + +@Serializable +internal data class CallsClassSearchResult( + val search: CallsCoverageSearchResult, + val scenarios: List, + val visits: List, + val unknownCallEvents: Int, + val methods: List = emptyList(), + val timeQuanta: List = emptyList(), + val solverQueries: List = emptyList(), + val solverCalls: Long? = null, + val solverElapsedMillis: Long? = null, + val unavailableMetrics: List = emptyList(), + val unknownCalls: List = emptyList(), +) + +/** A class owns one machine and one timeout; scenarios retain their path selectors between bounded visits. */ +internal class CurrentTsCallsClassCoverageEngine { + @Suppress("LongMethod") + fun search(request: CallsClassSearchRequest): CallsClassSearchResult { + val preparationStarted = TimeSource.Monotonic.markNow() + val sourceEngine = CurrentTsCallsSymbolicEngine() + val harness = request.harness + val preparations = harness.functions.map { function -> + sourceEngine.prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = harness.sourceRoot, + project = harness.project, + function = function, + expectedNativeFrontendRevision = request.expectedNativeFrontendRevision, + verifySourceCheckout = false, + ), + ) + } + val rejection = preparations.filterIsInstance().firstOrNull() + if (rejection != null) { + return rejectedSearch( + diagnostic = "${rejection.reasonCode}: ${rejection.diagnostic}", + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + status = if (rejection.status == CallsSymbolicStatus.TOOL_ERROR) { + CallsCoverageSearchStatus.TOOL_ERROR + } else { + CallsCoverageSearchStatus.UNSUPPORTED + }, + ) + } + val eligible = preparations.filterIsInstance() + val first = eligible.first() + require(eligible.all { it.scene === first.scene }) { "Class scenarios must share one source scene" } + val scope = resolveCallsSymbolicCoverageScope( + preparation = first, + target = CallsSymbolicCoverageClass(harness.surface.sourceFile, harness.surface.exportName), + ) + if (scope is CallsSymbolicCoverageScope.Rejected) { + return rejectedSearch( + diagnostic = scope.diagnostic, + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + status = scope.status, + ) + } + scope as CallsSymbolicCoverageScope.Selected + val objectShapes = PlainObjectInputShapes( + sourceScene = first.scene, + domains = harness.functions.flatMap { function -> function.inputs.map(PropertyInput::domain) }, + ) + val contexts = eligible.mapIndexed { index, preparation -> + val function = harness.functions[index] + CallsCoverageCandidateContext( + method = preparation.method, + inputDomains = function.inputs, + symbolicInputs = CallsSymbolicInputs( + inputs = function.inputs, + bindings = preparation.inputBindings, + lexicalEnvironment = preparation.lexicalEnvironment, + captures = preparation.captures, + sourceScene = first.scene, + objectShapes = objectShapes, + ), + entryPointId = harness.scenarios[index].id, + observationFields = harness.observations[index], + ) + } + val preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds + val byMethod = contexts.associateBy(CallsCoverageCandidateContext::method) + val candidates = mutableListOf() + val extractionFailures = mutableListOf() + val machineCallStarted = TimeSource.Monotonic.markNow() + val candidateObserver = CompletedCoverageCandidateObserver( + contexts = contexts, + coverageStatements = scope.statements, + candidateCap = request.candidateCap, + budgetMillis = request.budget.inWholeMilliseconds, + machineCallStarted = machineCallStarted, + onCandidate = { candidate -> + request.onCandidate(candidate) + candidates += candidate + }, + onExtractionFailure = extractionFailures::add, + requireEntryPointWitness = request.requireEntryPointWitness, + emitSuspendedCandidates = true, + ) + val schedulingObserver = ClassSchedulingObserver( + scenarioIds = contexts.associate { it.method to requireNotNull(it.entryPointId) }, + requestedQuantumMillis = request.timeQuantum.inWholeMilliseconds, + targetClass = requireNotNull(scope.classSignature), + ) + val limitations = linkedSetOf() + val unknownCalls = mutableListOf() + val stoppedObserver = TsStoppedStatesObserver(consumeState = candidateObserver::collectStoppedState) + val interpreterObserver = object : TsInterpreterObserver by stoppedObserver { + override fun onRuntimeFeatureLimitation(event: TsRuntimeFeatureLimitationEvent) { + limitations += "${event.reason.name}: ${event.detail}" + } + + override fun onUnknownCall(event: TsUnknownCallEvent) { + unknownCalls += event.toCallsClassUnknownCallEvent(eventIndex = unknownCalls.size + 1) + } + + override fun onSolverQuery(event: TsSolverQueryEvent) { + schedulingObserver.recordSolverQuery(event) + } + + override fun onEntryPointTimeQuantum(event: TsEntryPointTimeQuantumEvent) { + schedulingObserver.recordTimeQuantum(event) + } + } + val modelIds = if (request.profile.usesFrozenModels) request.frozenModelIds else emptySet() + val inputCapacity = contexts.flatMap { it.inputDomains }.maxOfOrNull { it.domain.callsInputCapacity() } ?: 0 + val machineResult = runCatching { + TsMachine( + scene = objectShapes.scene, + options = UMachineOptions( + pathSelectionStrategies = listOf(CALLS_PATH_SELECTION_STRATEGY), + pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.CONSTANT_TIME, + stateCollectionStrategy = StateCollectionStrategy.COVERED_NEW, + randomSeed = request.seed, + timeout = request.budget, + solverTimeout = request.solverQueryLimit, + solverType = request.solverType, + maxEntryPointTimeQuantum = request.timeQuantum, + coverageZone = CoverageZone.CLASS, + removeCoveredEntryPoints = false, + stopOnCoverage = CALLS_STOP_ON_COVERAGE, + stopOnTargetsReached = false, + throwExceptionOnStepFailure = true, + ), + tsOptions = TsOptions( + maxArraySize = maxOf(TsOptions().maxArraySize, inputCapacity), + unknownCallModelSelection = TsUnknownCallModelSelection.Only(modelIds), + unknownCallFallback = request.profile.fallback, + classCoverageTargets = setOf(requireNotNull(scope.classSignature)), + eraseTypeAssertions = true, + ), + machineObserver = CompositeUMachineObserver(candidateObserver, schedulingObserver), + observer = interpreterObserver, + machineObserverFactory = stoppedObserver::createMachineObserver, + ).use { machine -> + val result = machine.analyzeWithMetadata( + methods = contexts.map(CallsCoverageCandidateContext::method), + configureInitialState = { method, state -> + byMethod.getValue(method).symbolicInputs.initialize(state) + }, + ) + stoppedObserver.collectRemainingStates() + result + } + } + candidateObserver.finishTiming() + schedulingObserver.finishTiming() + val elapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds + val outcome = machineResult.getOrNull() + val status = when { + machineResult.exceptionOrNull() is TsInitialStateSolverUnknownException -> CallsCoverageSearchStatus.UNKNOWN + machineResult.isFailure -> CallsCoverageSearchStatus.TOOL_ERROR + outcome?.timedOut == true -> CallsCoverageSearchStatus.TIMEOUT + outcome?.stopReason == TsAnalysisStopReason.EXHAUSTED -> CallsCoverageSearchStatus.EXHAUSTED + else -> CallsCoverageSearchStatus.STOPPED + } + val search = CallsCoverageSearchResult( + status = status, + candidates = candidates, + selectedStates = candidateObserver.selectedStates, + executedSteps = candidateObserver.executedSteps, + stepsWithinBudget = candidateObserver.stepsWithinBudget, + extractionFailures = extractionFailures, + candidateCapReached = candidateObserver.capReached, + preparationElapsedMillis = preparationElapsedMillis, + machineSetupElapsedMillis = candidateObserver.machineSetupElapsedMillis, + searchElapsedMillis = candidateObserver.searchElapsedMillis, + machineTeardownElapsedMillis = ( + elapsedMillis - candidateObserver.machineSetupElapsedMillis - candidateObserver.searchElapsedMillis + ).coerceAtLeast(0), + unsupportedCall = outcome?.unsupportedCall == true, + engineFailed = outcome?.engineFailed == true, + runtimeLimited = outcome?.runtimeLimited == true || limitations.isNotEmpty(), + runtimeLimitationReasons = (limitations + outcome?.unsupportedPaths.orEmpty()).toList(), + diagnostic = machineResult.exceptionOrNull()?.let { it.message ?: it::class.java.name }, + suspendedStates = candidateObserver.suspendedStates, + suspendedExtractionElapsedMillis = candidateObserver.suspendedExtractionElapsedMillis, + frontierFlushElapsedMillis = candidateObserver.frontierFlushElapsedMillis, + machineStopReason = outcome?.stopReason?.name, + ) + + return CallsClassSearchResult( + search = search, + scenarios = schedulingObserver.metrics(candidates), + visits = schedulingObserver.visits, + unknownCallEvents = unknownCalls.size, + unknownCalls = unknownCalls.toList(), + methods = schedulingObserver.methodMetrics(), + timeQuanta = schedulingObserver.timeQuanta, + solverQueries = schedulingObserver.solverMetrics(), + solverCalls = schedulingObserver.solverMetrics().sumOf { it.calls }, + solverElapsedMillis = schedulingObserver.solverMetrics().sumOf { it.elapsedNanos } / 1_000_000, + ) + } +} + +private class ClassSchedulingObserver( + private val scenarioIds: Map, + private val requestedQuantumMillis: Long, + private val targetClass: EtsClassSignature, +) : UMachineObserver { + private var machineStarted = TimeSource.Monotonic.markNow() + private var stepStarted = TimeSource.Monotonic.markNow() + private var currentMethod: EtsMethod? = null + private var visitStartedAtMillis = 0L + private val steps = mutableMapOf() + private val timesNanos = mutableMapOf() + private var executingMethod: EtsMethod? = null + private val methodSteps = mutableMapOf() + private val methodTimesNanos = mutableMapOf() + val visits = mutableListOf() + val timeQuanta = mutableListOf() + private var solverPhase = "SETUP" + private val solverEvents = linkedMapOf, MutableList>() + + fun recordTimeQuantum(event: TsEntryPointTimeQuantumEvent) { + timeQuanta += CallsClassTimeQuantum( + scenarioId = scenarioIds.getValue(event.entrypoint), + startedAtMillis = (machineStarted.elapsedNow() - event.elapsed).inWholeMilliseconds.coerceAtLeast(0), + requestedQuantumMillis = event.requested.inWholeMilliseconds, + elapsedNanos = event.elapsed.inWholeNanoseconds, + ) + } + + fun recordSolverQuery(event: TsSolverQueryEvent) { + val scenarioId = currentMethod?.let(scenarioIds::getValue) + val methodSignature = executingMethod?.signature?.toString() + solverEvents.getOrPut(Triple(solverPhase, scenarioId, methodSignature)) { mutableListOf() } += event + } + + fun solverMetrics(): List = solverEvents.map { (key, events) -> + CallsClassSolverMetrics( + phase = key.first, + scenarioId = key.second, + methodSignature = key.third, + calls = events.size.toLong(), + elapsedNanos = events.sumOf { it.elapsedNanos }, + failures = events.count { it.failure != null }.toLong(), + statusCounts = events.groupingBy { it.status?.name ?: "ERROR" }.eachCount().mapValues { it.value.toLong() }, + ) + } + + override fun onMachineStarted() { + machineStarted = TimeSource.Monotonic.markNow() + solverPhase = "SEARCH" + } + + override fun onStatePeeked(state: TsState) { + val elapsed = machineStarted.elapsedNow().inWholeMilliseconds + if (currentMethod != state.entrypoint) { + finishVisit(elapsed) + currentMethod = state.entrypoint + visitStartedAtMillis = elapsed + } + val activeMethod = if (state.callStack.isNotEmpty()) state.callStack.lastMethod() else null + executingMethod = activeMethod?.takeIf { it.signature.enclosingClass == targetClass } + stepStarted = TimeSource.Monotonic.markNow() + } + + override fun onState(parent: TsState, forks: Sequence) { + val elapsedNanos = stepStarted.elapsedNow().inWholeNanoseconds + steps.merge(parent.entrypoint, 1L, Long::plus) + timesNanos.merge(parent.entrypoint, elapsedNanos, Long::plus) + executingMethod?.let { method -> + methodSteps.merge(method, 1L, Long::plus) + methodTimesNanos.merge(method, elapsedNanos, Long::plus) + } + } + + override fun onMachineStopped() = finishTiming() + + fun finishTiming() { + finishVisit(machineStarted.elapsedNow().inWholeMilliseconds) + currentMethod = null + executingMethod = null + solverPhase = "TEARDOWN" + } + + fun methodMetrics(): List = methodSteps.map { (method, executedSteps) -> + CallsClassMethodMetrics( + methodSignature = method.signature.toString(), + executedSteps = executedSteps, + stepWallTimeNanos = methodTimesNanos[method] ?: 0, + ) + } + + fun metrics(candidates: List): List = + scenarioIds.map { (method, id) -> + CallsClassScenarioMetrics( + scenarioId = id, + executedSteps = steps[method] ?: 0, + stepWallTimeMillis = (timesNanos[method] ?: 0) / 1_000_000, + completedCandidates = candidates.count { + it.entryPointId == id && it.origin == CallsCoverageCandidateOrigin.TERMINATED + }, + suspendedCandidates = candidates.count { + it.entryPointId == id && it.origin == CallsCoverageCandidateOrigin.SUSPENDED + }, + ) + } + + private fun finishVisit(elapsed: Long) { + val method = currentMethod ?: return + visits += CallsClassSchedulingVisit( + scenarioId = scenarioIds.getValue(method), + startedAtMillis = visitStartedAtMillis, + elapsedMillis = elapsed - visitStartedAtMillis, + requestedQuantumMillis = requestedQuantumMillis, + ) + } +} + +private fun rejectedSearch( + diagnostic: String, + preparationElapsedMillis: Long, + status: CallsCoverageSearchStatus, +): CallsClassSearchResult = CallsClassSearchResult( + search = CallsCoverageSearchResult( + status = status, + candidates = emptyList(), + selectedStates = 0, + executedSteps = 0, + stepsWithinBudget = 0, + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = preparationElapsedMillis, + machineSetupElapsedMillis = 0, + searchElapsedMillis = 0, + machineTeardownElapsedMillis = 0, + diagnostic = diagnostic, + ), + scenarios = emptyList(), + visits = emptyList(), + unknownCallEvents = 0, + unavailableMetrics = listOf("solverCalls", "solverElapsedMillis", "exactSelectorQuantumBoundaries"), +) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperiment.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperiment.kt new file mode 100644 index 0000000000..9da422ff6c --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperiment.kt @@ -0,0 +1,513 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import org.usvm.machine.call.TsBuiltInUnknownCallModels +import org.usvm.ts.pbt.model.PropertyDomain +import java.nio.file.Files +import java.nio.file.Path + +/** Frozen external correctness evidence, independent of the outcome of a treatment's symbolic pilot. */ +@Serializable +internal data class CallsClassCorrectnessFrontierFailure(val reason: String, val provenance: String) { + init { + require(reason.isNotBlank() && provenance.isNotBlank()) { + "Correctness limitations require evidence provenance" + } + } +} + +@Serializable +internal data class CallsClassCase( + val classId: String, + val projectId: String, + val revision: String, + val sourceRoot: String, + val sourceSha256: Map, + val surface: CallsClassSurface, + val correctnessFrontierFailures: List = emptyList(), +) { + init { + require(classId.matches(Regex("[A-Za-z0-9_-]+"))) { "Class ID must be a safe directory name" } + require(revision.matches(Regex("[0-9a-f]{40}"))) { "Source revision must be a Git commit" } + require(surface.sourceFile in sourceSha256) { "Source closure omits the class source" } + require(sourceSha256.values.all { it.matches(Regex("[0-9a-f]{64}")) }) { "Invalid source hashes" } + require( + sourceSha256.keys.all { path -> + val value = Path.of(path) + !value.isAbsolute && value.normalize() == value && !value.startsWith("..") + }, + ) { "Source closure contains an unsafe path" } + } + + fun originalRoot(manifestDirectory: Path): Path = manifestDirectory.resolve(sourceRoot).normalize().toRealPath() + + fun verifySources(root: Path, verifyCheckout: Boolean) { + val canonicalRoot = root.toRealPath() + if (verifyCheckout) verifyCallsGitCheckout(canonicalRoot, revision) + sourceSha256.forEach { (relative, expected) -> + val path = canonicalRoot.resolve(relative).normalize().toRealPath() + require(path.startsWith(canonicalRoot)) { "Source closure escapes its root: $relative" } + require(callsClassSourceSha256(Files.readAllBytes(path)) == expected) { + "Frozen class source changed: $relative" + } + } + } +} + +@Serializable +internal data class CallsClassExperimentManifest( + val schemaVersion: Int = 2, + val experimentId: String, + val toolRevision: String, + val nativeFrontendRevision: String, + val solver: String = "YICES", + val modelSet: CallsModelSetIdentity, + val seeds: List, + val budgetsMillis: List = CLASS_SEARCH_BUDGETS_MILLIS, + val maxSequenceLength: Int = 2, + val timeQuantumMillis: Long = 250, + val solverLimitMillis: Long = 1_000, + val candidateCap: Int = 10_000, + val classes: List, + val coveragePolicy: String = CLASS_NATIVE_COVERAGE_POLICY, + val suspendedCoveragePolicy: String = CLASS_SUSPENDED_COVERAGE_POLICY, +) { + init { + require(schemaVersion == 2) { "Class measurement requires the native coverage and suspended-input protocol" } + require(coveragePolicy == CLASS_NATIVE_COVERAGE_POLICY) + require(suspendedCoveragePolicy == CLASS_SUSPENDED_COVERAGE_POLICY) + require(experimentId.isNotBlank()) + require(toolRevision.matches(Regex("[0-9a-f]{40}"))) { "Tool revision must be clean" } + require(modelSet.toolRevision == toolRevision) { "Frozen model revision differs" } + require(solver == "YICES" || solver == "Z3") + require(seeds.isNotEmpty() && seeds.distinct().size == seeds.size) + require(budgetsMillis == CLASS_SEARCH_BUDGETS_MILLIS) { "Class treatments are exactly 30 and 120 seconds" } + require(maxSequenceLength in 1..2) + require(timeQuantumMillis in 1..budgetsMillis.first()) + require(solverLimitMillis in 1..budgetsMillis.first()) + require(candidateCap > 0) + require(classes.isNotEmpty() && classes.map { it.classId }.distinct().size == classes.size) + } + + fun verifyBuild(toolRevision: String, nativeFrontendRevision: String) { + require(this.toolRevision == toolRevision) { "Class manifest and running tool revision differ" } + require(this.nativeFrontendRevision == nativeFrontendRevision) { "Class manifest and frontend revision differ" } + val availableModels = TsBuiltInUnknownCallModels.catalog().modelIds.toSet() + require(availableModels.containsAll(modelSet.ids)) { "Frozen models are absent from this build" } + require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" } + } +} + +@Serializable +internal data class CallsClassPreparedRecord( + val classCase: CallsClassCase, + val originalSourceRoot: String, + val scenarios: List, + val functions: List, + val observations: List>, + val harnessFile: String, + val harnessSha256: String, + val instrumentation: CallsClassSourceInstrumentation, + val preparationElapsedMillis: Long = 0, +) { + fun harness(classDirectory: Path): CallsClassPreparedHarness { + val root = classDirectory.resolve("source").toRealPath() + classCase.verifySources(root, verifyCheckout = false) + require(callsClassSourceSha256(Files.readAllBytes(root.resolve(harnessFile))) == harnessSha256) { + "Generated class wrapper changed" + } + require(functions.map { it.functionId } == scenarios.map { it.id }) { "Scenario schedule differs" } + require(functions.all { it.sourceFile == harnessFile }) { "Scenarios do not share one wrapper module" } + require(observations.size == functions.size) + instrumentation.validateSurface(classCase.surface) + instrumentation.universe.verifyUnchanged(root) + + return CallsClassPreparedHarness( + surface = classCase.surface, + scenarios = scenarios, + sourceRoot = root, + project = CallsProjectCase( + projectId = classCase.projectId, + revision = classCase.revision, + sourceRoot = root.toString(), + development = true, + functions = functions, + ), + functions = functions, + observations = observations, + ) + } + + fun verifyOriginal() = classCase.verifySources(Path.of(originalSourceRoot), verifyCheckout = true) +} + +@Serializable +internal data class CallsClassPreparationFailure( + val classId: String, + val diagnostic: String, + val preparationElapsedMillis: Long = 0, +) + +@Serializable +internal enum class CallsClassNativeSmokeStatus { VERIFIED, INVALID, INCONCLUSIVE } + +@Serializable +internal data class CallsClassNativeSmokeResult( + val seed: Long, + val scenarioId: String, + val status: CallsClassNativeSmokeStatus, + val original: CallsClassReplayObservation? = null, + val instrumented: CallsClassReplayObservation? = null, + val diagnostic: String? = null, +) + +/** Check the native observation mechanism independently of symbolic policies and their selected candidates. */ +internal fun callsClassNativeConstructorSmoke( + harness: CallsClassPreparedHarness, + instrumentation: CallsClassSourceInstrumentation, + seed: Long, + timeoutMillis: Long, +): CallsClassNativeSmokeResult { + val index = harness.scenarios.indexOfFirst { it.memberIds.isEmpty() } + require(index >= 0) { "Class schedule omits the constructor observation scenario" } + val function = harness.functions[index] + val fields = harness.observations[index] + val scenarioId = harness.scenarios[index].id + val original = runCatching { + OriginalTypeScriptClassReplayer(harness.sourceRoot, function, fields).use { + it.sample(seed = seed, timeoutMillis = timeoutMillis) + } + } + val originalValue = original.getOrNull() + if (originalValue == null) { + val failure = requireNotNull(original.exceptionOrNull()) + return CallsClassNativeSmokeResult( + seed = seed, + scenarioId = scenarioId, + status = CallsClassNativeSmokeStatus.INCONCLUSIVE, + diagnostic = "Native original sampling did not complete: ${failure.message ?: failure::class.java.name}", + ) + } + + val instrumented = runCatching { + OriginalTypeScriptClassReplayer( + sourceRoot = harness.sourceRoot, + function = function, + observationFields = fields, + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = harness.surface.sourceFile, + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = originalValue.inputs, timeoutMillis = timeoutMillis) } + } + val instrumentedValue = instrumented.getOrNull() + if (instrumentedValue == null) { + val failure = requireNotNull(instrumented.exceptionOrNull()) + return CallsClassNativeSmokeResult( + seed = seed, + scenarioId = scenarioId, + status = CallsClassNativeSmokeStatus.INCONCLUSIVE, + original = originalValue, + diagnostic = "Native instrumented replay did not complete: ${failure.message ?: failure::class.java.name}", + ) + } + + val invalid = callsClassNativeInfrastructureFailure(originalValue, instrumentedValue) + val checkedIds = runCatching { instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys) } + val invalidDiagnostic = invalid ?: checkedIds.exceptionOrNull()?.let { it.message ?: it::class.java.name } + val representable = originalValue.observation.representable && instrumentedValue.observation.representable + val status = when { + invalidDiagnostic != null -> CallsClassNativeSmokeStatus.INVALID + representable -> CallsClassNativeSmokeStatus.VERIFIED + else -> CallsClassNativeSmokeStatus.INCONCLUSIVE + } + + val diagnostic = invalidDiagnostic ?: if (representable) { + null + } else { + "Native observation is incomplete at the sampled input" + } + return CallsClassNativeSmokeResult( + seed = seed, + scenarioId = scenarioId, + status = status, + original = originalValue, + instrumented = instrumentedValue, + diagnostic = diagnostic, + ) +} + +/** An incomplete lawful observation is not evidence of an instrumentation defect. */ +internal fun callsClassNativeInfrastructureFailure( + original: CallsClassReplayObservation, + instrumented: CallsClassReplayObservation, +): String? { + if (original.inputs != instrumented.inputs) return "Instrumentation changed the frozen native input transport" + if (original.observation.completion != instrumented.observation.completion) { + return "Instrumentation changed native completion" + } + if (original.observation.representable != instrumented.observation.representable) { + return "Instrumentation changed native observability or observation schema" + } + val completeObservations = original.observation.representable && instrumented.observation.representable + if (completeObservations && original.observation != instrumented.observation) { + return "Instrumentation changed native returned values or receiver state" + } + return null +} + +@Serializable +internal data class CallsClassPreflightResult( + val eligible: Boolean, + val classId: String, + val preparedSha256: String, + val toolRevision: String, + val search: CallsClassSearchResult, + val replays: List, + val missingRequiredScenarios: List, + val reasons: List, + val admissionVersion: Int = 0, + val sourceAdmissionFailures: List = emptyList(), + val nativeInfrastructureFailures: List = emptyList(), + val nativeSmoke: CallsClassNativeSmokeResult? = null, + val diagnosticReady: Boolean? = null, + val primaryInferenceEligible: Boolean? = null, + val correctnessFrontierFailures: List = emptyList(), +) { + init { + require(admissionVersion in 0..1) { "Unsupported class admission version" } + if (admissionVersion == 1) { + require(diagnosticReady != null && primaryInferenceEligible != null) { + "Version-one admission must distinguish pilot readiness and primary inference" + } + require(sourceAdmissionFailures.isNotEmpty() || nativeSmoke != null) { + "Source-ready admission requires a native probe" + } + } + } +} + +internal fun callsClassPreflightResult( + classCase: CallsClassCase, + preparedSha256: String, + toolRevision: String, + search: CallsClassSearchResult, + replays: List, + missingScenarios: List, + pilotDiagnostics: List, + sourceAdmissionFailures: List, + nativeSmoke: CallsClassNativeSmokeResult?, + nativeInfrastructureFailures: List, +): CallsClassPreflightResult { + require(sourceAdmissionFailures.isNotEmpty() || nativeSmoke != null) { + "Source-ready admission requires a native probe" + } + val invalidSmoke = nativeSmoke?.status == CallsClassNativeSmokeStatus.INVALID + val eligible = sourceAdmissionFailures.isEmpty() && !invalidSmoke + val primaryInferenceEligible = eligible && nativeSmoke?.status == CallsClassNativeSmokeStatus.VERIFIED && + classCase.correctnessFrontierFailures.isEmpty() + val admissionDiagnostics = sourceAdmissionFailures + nativeInfrastructureFailures + + listOfNotNull(nativeSmoke?.diagnostic) + + classCase.correctnessFrontierFailures.map { "${it.reason} (${it.provenance})" } + + return CallsClassPreflightResult( + eligible = eligible, + classId = classCase.classId, + preparedSha256 = preparedSha256, + toolRevision = toolRevision, + search = search, + replays = replays, + missingRequiredScenarios = missingScenarios, + reasons = admissionDiagnostics + pilotDiagnostics, + admissionVersion = 1, + sourceAdmissionFailures = sourceAdmissionFailures, + nativeInfrastructureFailures = nativeInfrastructureFailures, + nativeSmoke = nativeSmoke, + diagnosticReady = pilotDiagnostics.isEmpty() && nativeInfrastructureFailures.isEmpty(), + primaryInferenceEligible = primaryInferenceEligible, + correctnessFrontierFailures = classCase.correctnessFrontierFailures, + ) +} + +@Serializable +internal data class CallsClassReplayEvent( + val index: Int, + val scenarioId: String, + val original: CallsClassReplayObservation? = null, + val instrumented: CallsClassReplayObservation? = null, + val oracle: CallsClassOracleResult, + val error: String? = null, + val replayElapsedMillis: Long = 0, + val nativeCoverage: CallsClassOracleResult, + val origin: CallsCoverageCandidateOrigin = CallsCoverageCandidateOrigin.TERMINATED, + val strictOracleApplicable: Boolean = true, +) + +/** Primary coverage checks the native measurement, independently of symbolic execution predictions. */ +internal fun callsClassNativeCoverageAcceptance( + original: CallsClassReplayObservation?, + instrumented: CallsClassReplayObservation?, + universe: CallsClassSourceUniverse, + error: String? = null, +): CallsClassOracleResult { + if (error != null || original == null || instrumented == null) { + return CallsClassOracleResult(accepted = false, diagnostic = error ?: "Concrete replay missing") + } + + val foreignIds = runCatching { universe.checkedCoveredIds(instrumented.coveredStatementKeys) }.exceptionOrNull() + val failure = callsClassNativeInfrastructureFailure(original, instrumented) + ?: foreignIds?.let { it.message ?: it::class.java.name } + if (failure != null) return CallsClassOracleResult(accepted = false, diagnostic = failure) + + val complete = original.observation.representable && instrumented.observation.representable + if (!complete) { + return CallsClassOracleResult(accepted = false, diagnostic = "Native observation is incomplete") + } + + return CallsClassOracleResult(accepted = true) +} + +@Serializable +internal data class CallsClassCellIdentity( + val schemaVersion: Int = 2, + val experimentId: String, + val toolRevision: String, + val nativeFrontendRevision: String, + val manifestSha256: String, + val preparedSha256: String, + val preflightSha256: String, + val universeSha256: String, + val classId: String, + val profile: CallsExperimentProfile, + val seed: Long, + val budgetMillis: Long, + val coveragePolicy: String = CLASS_NATIVE_COVERAGE_POLICY, + val suspendedCoveragePolicy: String = CLASS_SUSPENDED_COVERAGE_POLICY, +) + +@Serializable +internal data class CallsClassCoverageCheckpoint( + val searchBudgetMillis: Long, + val emittedCandidates: Int, + val acceptedCandidates: Int, + val rejectedCandidates: Int, + val coveredStatementIds: Set, + val uncoveredStatementIds: Set, + val totalStatements: Int, + val strictAcceptedCandidates: Int, + val strictRejectedCandidates: Int, + val strictOracleNotApplicable: Int, + val strictCoveredStatementIds: Set, + val strictUncoveredStatementIds: Set, + val suspendedCandidates: Int, + val includesSuspendedExtraction: Boolean, + val postStopSuspendedCandidates: Int, +) + +@Serializable +internal data class CallsClassCellResult( + val identity: CallsClassCellIdentity, + val search: CallsClassSearchResult, + val checkpoints: List, + val replayErrors: Int, + val oracleRejections: Int, + val replayElapsedMillis: Long, + val cellElapsedMillis: Long, + val primaryInferenceEligible: Boolean? = null, + val correctnessFrontierFailures: List = emptyList(), + val nativeInfrastructureFailures: List = emptyList(), + val nativeCoverageRejections: Int = 0, + val strictOracleNotApplicable: Int = 0, +) + +internal fun callsClassCoverageCheckpoints( + search: CallsClassSearchResult, + events: List, + universe: CallsClassSourceUniverse, + budgetMillis: Long, +): List { + require(events.size == search.search.candidates.size) { "Incomplete class replay journal" } + val checkpoints = (CLASS_COVERAGE_CHECKPOINTS_MILLIS + budgetMillis) + .filter { it <= budgetMillis }.distinct().sorted() + + return checkpoints.map { checkpoint -> + val indices = search.search.candidates.indices.filter { index -> + val candidate = search.search.candidates[index] + when (candidate.origin) { + CallsCoverageCandidateOrigin.TERMINATED -> + candidate.emittedAtMillis <= checkpoint + CallsCoverageCandidateOrigin.SUSPENDED -> if (candidate.suspendedAtMachineStop) { + checkpoint == budgetMillis + } else { + candidate.emittedAtMillis <= checkpoint + } + } + } + val selectedEvents = indices.map { events[it] } + indices.forEach { index -> + val event = events[index] + val candidate = search.search.candidates[index] + val matchesCandidate = event.index == index && + event.scenarioId == candidate.entryPointId && event.origin == candidate.origin + require(matchesCandidate) { + "Class replay event differs from its candidate" + } + require(event.strictOracleApplicable == (candidate.origin == CallsCoverageCandidateOrigin.TERMINATED)) { + "Class replay symbolic oracle applicability differs from its candidate" + } + val expectedNative = callsClassNativeCoverageAcceptance( + original = event.original, + instrumented = event.instrumented, + universe = universe, + error = event.error, + ) + require(event.nativeCoverage == expectedNative) { "Class replay native coverage decision differs" } + } + val accepted = selectedEvents.filter { it.nativeCoverage.accepted && it.error == null } + val covered = universe.checkedCoveredIds( + accepted.flatMap { event -> requireNotNull(event.instrumented).coveredStatementKeys }, + ) + val strictAccepted = selectedEvents.filter { + it.strictOracleApplicable && it.oracle.accepted && it.error == null + } + val strictCovered = universe.checkedCoveredIds( + strictAccepted.flatMap { event -> requireNotNull(event.instrumented).coveredStatementKeys }, + ) + val strictNotApplicable = selectedEvents.count { !it.strictOracleApplicable } + CallsClassCoverageCheckpoint( + searchBudgetMillis = checkpoint, + emittedCandidates = indices.size, + acceptedCandidates = accepted.size, + rejectedCandidates = indices.size - accepted.size, + coveredStatementIds = covered, + uncoveredStatementIds = universe.statementIds - covered, + totalStatements = universe.statementIds.size, + strictAcceptedCandidates = strictAccepted.size, + strictRejectedCandidates = indices.size - strictAccepted.size - strictNotApplicable, + strictOracleNotApplicable = strictNotApplicable, + strictCoveredStatementIds = strictCovered, + strictUncoveredStatementIds = universe.statementIds - strictCovered, + suspendedCandidates = indices.count { + search.search.candidates[it].origin == CallsCoverageCandidateOrigin.SUSPENDED + }, + includesSuspendedExtraction = checkpoint == budgetMillis, + postStopSuspendedCandidates = indices.count { search.search.candidates[it].suspendedAtMachineStop }, + ) + } +} + +internal const val CLASS_NATIVE_COVERAGE_POLICY = "native-original-instrumented-complete-v2" +internal const val CLASS_SUSPENDED_COVERAGE_POLICY = "reached-prefix-and-final-frontier-inputs-v1" + +private const val SHORT_CLASS_BUDGET_MILLIS = 30_000L +private const val LONG_CLASS_BUDGET_MILLIS = 120_000L +private const val FIRST_CLASS_CHECKPOINT_MILLIS = 1_000L +private const val SECOND_CLASS_CHECKPOINT_MILLIS = 5_000L +private const val THIRD_CLASS_CHECKPOINT_MILLIS = 10_000L +private val CLASS_SEARCH_BUDGETS_MILLIS = listOf(SHORT_CLASS_BUDGET_MILLIS, LONG_CLASS_BUDGET_MILLIS) +private val CLASS_COVERAGE_CHECKPOINTS_MILLIS = listOf( + FIRST_CLASS_CHECKPOINT_MILLIS, + SECOND_CLASS_CHECKPOINT_MILLIS, + THIRD_CLASS_CHECKPOINT_MILLIS, + SHORT_CLASS_BUDGET_MILLIS, + LONG_CLASS_BUDGET_MILLIS, +) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt new file mode 100644 index 0000000000..203cb4eaac --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt @@ -0,0 +1,553 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.decodeFromString +import org.usvm.SolverType +import org.usvm.machine.call.TsBuiltInUnknownCallModels +import java.nio.file.Files +import java.nio.file.Path +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.TimeSource + +internal fun runClassCatalog(args: List) { + require(args.isEmpty()) { "class-catalog takes no arguments" } + val identity = CallsModelSetIdentity( + ids = TsBuiltInUnknownCallModels.catalog().modelIds.sorted().toSet(), + toolRevision = CallsBuildIdentity.toolRevision, + ) + System.out.appendLine(encodeCallsArtifact(identity)) +} + +internal fun runClassPrepare(args: List) { + require(args.size == 2) { "class-prepare requires " } + val manifestPath = Path.of(args[0]).toRealPath() + val manifest = decodeClassArtifact(manifestPath) + manifest.verifyBuild(CallsBuildIdentity.toolRevision, CallsBuildIdentity.nativeFrontendRevision) + val output = Path.of(args[1]).toAbsolutePath().normalize() + Files.createDirectories(output) + require(Files.list(output).use { it.findAny().isEmpty }) { "Preparation output must be empty" } + writeCallsArtifactAtomically(output.resolve("manifest.json"), encodeCallsArtifact(manifest)) + for (classCase in manifest.classes) { + val classDirectory = output.resolve(classCase.classId) + Files.createDirectories(classDirectory) + val preparationStarted = TimeSource.Monotonic.markNow() + val preparation = runCatching { + val originalRoot = classCase.originalRoot(requireNotNull(manifestPath.parent)) + classCase.verifySources(originalRoot, verifyCheckout = true) + val overlay = classDirectory.resolve("source") + Files.createDirectories(overlay) + classCase.sourceSha256.keys.forEach { relative -> + val copied = overlay.resolve(relative) + Files.createDirectories(copied.parent) + Files.copy(originalRoot.resolve(relative), copied) + } + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = originalRoot, + sourceFile = classCase.surface.sourceFile, + className = classCase.surface.exportName, + ) + instrumentation.validateSurface(classCase.surface) + val project = CallsProjectCase( + projectId = classCase.projectId, + revision = classCase.revision, + sourceRoot = overlay.toString(), + development = true, + functions = emptyList(), + ) + val harness = prepareCallsClassHarness( + sourceRoot = overlay, + project = project, + surface = classCase.surface, + maxSequenceLength = manifest.maxSequenceLength, + ) + val harnessFile = harness.functions.first().sourceFile + classCase.verifySources(originalRoot, verifyCheckout = true) + CallsClassPreparedRecord( + classCase = classCase, + originalSourceRoot = originalRoot.toString(), + scenarios = harness.scenarios, + functions = harness.functions, + observations = harness.observations, + harnessFile = harnessFile, + harnessSha256 = callsClassSourceSha256(Files.readAllBytes(overlay.resolve(harnessFile))), + instrumentation = instrumentation, + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + ) + } + val prepared = preparation.getOrNull() + if (prepared != null) { + writeCallsArtifactAtomically(classDirectory.resolve("class.json"), encodeCallsArtifact(prepared)) + val statementCount = prepared.instrumentation.universe.statements.size + System.out.appendLine("Prepared ${classCase.classId}: $statementCount statements") + } else { + val error = requireNotNull(preparation.exceptionOrNull()) + val failure = CallsClassPreparationFailure( + classId = classCase.classId, + diagnostic = error.message ?: error::class.java.name, + preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds, + ) + writeCallsArtifactAtomically( + classDirectory.resolve("preparation-failure.json"), + encodeCallsArtifact(failure), + ) + System.out.appendLine(encodeCallsArtifact(failure)) + } + } +} + +/** Source/native admission is independent of the diagnostic pilot and any treatment completion rate. */ +internal fun runClassPreflight( + args: List, + searchFunction: (CallsClassSearchRequest) -> CallsClassSearchResult = CurrentTsCallsClassCoverageEngine()::search, +) { + require(args.size == CLASS_PREFLIGHT_ARGUMENT_COUNT) { + "class-preflight requires " + } + val source = loadPreparedClass(Path.of(args[0]), classId = args[1]) + val budgetMillis = args[2].toLong() + require(budgetMillis >= source.manifest.solverLimitMillis && budgetMillis >= source.manifest.timeQuantumMillis) + val output = source.directory.resolve("preflight") + Files.createDirectories(output) + require(Files.list(output).use { it.findAny().isEmpty }) { "Class preflight already started" } + val sourceAdmissionFailures = callsClassSourceAdmissionFailures( + harness = source.harness, + expectedNativeFrontendRevision = source.manifest.nativeFrontendRevision, + ) + val nativeSmoke = if (sourceAdmissionFailures.isEmpty()) { + callsClassNativeConstructorSmoke( + harness = source.harness, + instrumentation = source.record.instrumentation, + seed = source.manifest.seeds.first(), + timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS, + ) + } else { + null + } + writeCallsArtifactAtomically(output.resolve("native-smoke.json"), encodeCallsArtifact(nativeSmoke)) + val candidatesPath = output.resolve("candidates.jsonl") + val search = searchFunction( + source.searchRequest( + profile = CallsExperimentProfile.FROZEN_FRESH, + seed = source.manifest.seeds.first(), + budgetMillis = budgetMillis, + requireEntryPointWitness = true, + onCandidate = { appendCallsJournalDurably(candidatesPath, encodeCallsArtifact(it)) }, + ), + ) + source.verify() + require(readCallsJournal(candidatesPath) == search.search.candidates) { + "Preflight candidate journal differs" + } + writeCallsArtifactAtomically(output.resolve("search.json"), encodeCallsArtifact(search)) + val events = replayClassCandidates(source, search, output) + val acceptedIds = events.filter { it.oracle.accepted && it.error == null }.mapTo(hashSetOf()) { it.scenarioId } + val missing = source.record.scenarios.map { it.id }.filter { it !in acceptedIds } + val pilotDiagnostics = callsClassPilotDiagnostics(search, events, missing) + val nativeInfrastructureFailures = callsClassPilotNativeFailures(events, source.record.instrumentation.universe) + val result = callsClassPreflightResult( + classCase = source.record.classCase, + preparedSha256 = source.preparedSha256, + toolRevision = source.manifest.toolRevision, + search = search, + replays = events, + missingScenarios = missing, + pilotDiagnostics = pilotDiagnostics, + sourceAdmissionFailures = sourceAdmissionFailures, + nativeSmoke = nativeSmoke, + nativeInfrastructureFailures = nativeInfrastructureFailures, + ) + source.verify() + writeCallsArtifactAtomically(source.directory.resolve("preflight.json"), encodeCallsArtifact(result)) + System.out.appendLine(encodeCallsArtifact(result)) +} + +private fun callsClassPilotDiagnostics( + search: CallsClassSearchResult, + events: List, + missing: List, +): List = buildList { + if (missing.isNotEmpty()) add("No exact returned-value/state witness for scenarios: $missing") + if (search.search.engineFailed) add("Symbolic engine failed") + if (search.search.unsupportedCall) add("Technical preflight stopped an unsupported call") + if (search.search.runtimeLimited) addAll(search.search.runtimeLimitationReasons) + if (search.search.extractionFailures.isNotEmpty()) addAll(search.search.extractionFailures) + if (search.search.candidateCapReached) add("Technical preflight reached candidate cap") + if (search.search.status == CallsCoverageSearchStatus.TIMEOUT) add("Symbolic pilot reached the time budget") + if (events.any { it.error != null }) add("Concrete preflight replay failed") + if (events.any { it.strictOracleApplicable && !it.oracle.accepted }) { + add("Technical preflight observation oracle rejected a candidate") + } + val terminalFailure = search.search.status in setOf( + CallsCoverageSearchStatus.UNSUPPORTED, + CallsCoverageSearchStatus.UNMAPPED, + CallsCoverageSearchStatus.AMBIGUOUS, + CallsCoverageSearchStatus.UNKNOWN, + CallsCoverageSearchStatus.TOOL_ERROR, + ) + if (terminalFailure) add(search.search.diagnostic ?: search.search.status.name) +} + +internal fun callsClassPilotNativeFailures( + events: List, + universe: CallsClassSourceUniverse, +): List = buildList { + for (event in events) { + val original = event.original + val instrumented = event.instrumented + if (original != null && instrumented != null) { + val failure = callsClassNativeInfrastructureFailure(original, instrumented) + if (failure != null) add("${event.scenarioId}: $failure") + } + if (instrumented != null) { + val checkedIds = runCatching { universe.checkedCoveredIds(instrumented.coveredStatementKeys) } + checkedIds.exceptionOrNull()?.let { add(it.message ?: it::class.java.name) } + } + } +} + +internal fun runClassCell( + args: List, + searchFunction: (CallsClassSearchRequest) -> CallsClassSearchResult = CurrentTsCallsClassCoverageEngine()::search, +) { + require(args.size == CLASS_CELL_ARGUMENT_COUNT) { + "class-cell requires " + } + val cellStarted = TimeSource.Monotonic.markNow() + val arguments = args.iterator() + val preparedDirectory = Path.of(arguments.next()) + val classId = arguments.next() + val source = loadPreparedClass(preparedDirectory, classId = classId) + val profile = CallsExperimentProfile.valueOf(arguments.next()) + val seed = arguments.next().toLong() + val budgetMillis = arguments.next().toLong() + require(seed in source.manifest.seeds && budgetMillis in source.manifest.budgetsMillis) { + "Cell is outside the frozen paired seed/budget schedule" + } + val preflightPath = source.directory.resolve("preflight.json") + val preflight = decodeClassArtifact(preflightPath) + source.validatePreflightAdmission(preflight) + val output = Path.of(arguments.next()).toAbsolutePath().normalize() + Files.createDirectories(output) + val identity = CallsClassCellIdentity( + experimentId = source.manifest.experimentId, + toolRevision = source.manifest.toolRevision, + nativeFrontendRevision = source.manifest.nativeFrontendRevision, + manifestSha256 = source.manifestSha256, + preparedSha256 = source.preparedSha256, + preflightSha256 = callsArtifactSha256(Files.readAllBytes(preflightPath)), + universeSha256 = source.record.instrumentation.universe.universeSha256, + classId = source.record.classCase.classId, + profile = profile, + seed = seed, + budgetMillis = budgetMillis, + ) + val identityPath = output.resolve("identity.json") + if (Files.exists(identityPath)) { + require(decodeClassArtifact(identityPath) == identity) { "Class cell identity changed" } + } else { + require(Files.list(output).use { it.findAny().isEmpty }) { "Cell output is not empty" } + writeCallsArtifactAtomically(identityPath, encodeCallsArtifact(identity)) + } + require(!Files.exists(output.resolve("result.json"))) { "Class cell is already complete" } + val candidatesPath = output.resolve("candidates.jsonl") + val searchPath = output.resolve("search.json") + val search = if (Files.exists(searchPath)) { + decodeClassArtifact(searchPath) + } else { + require(!Files.exists(candidatesPath)) { "Interrupted class search requires investigation before restart" } + val result = searchFunction( + source.searchRequest( + profile = profile, + seed = seed, + budgetMillis = budgetMillis, + requireEntryPointWitness = false, + onCandidate = { appendCallsJournalDurably(candidatesPath, encodeCallsArtifact(it)) }, + ), + ) + source.verify() + require(readCallsJournal(candidatesPath) == result.search.candidates) { + "Class candidate journal differs from search result" + } + writeCallsArtifactAtomically(searchPath, encodeCallsArtifact(result)) + result + } + require(readCallsJournal(candidatesPath) == search.search.candidates) { + "Persisted class search and candidate journal differ" + } + val events = replayClassCandidates(source, search, output) + source.verify() + val result = CallsClassCellResult( + identity = identity, + search = search, + checkpoints = callsClassCoverageCheckpoints( + search = search, + events = events, + universe = source.record.instrumentation.universe, + budgetMillis = budgetMillis, + ), + replayErrors = events.count { it.error != null }, + oracleRejections = events.count { it.strictOracleApplicable && !it.oracle.accepted }, + replayElapsedMillis = events.sumOf { it.replayElapsedMillis }, + cellElapsedMillis = cellStarted.elapsedNow().inWholeMilliseconds, + primaryInferenceEligible = preflight.primaryInferenceEligible, + correctnessFrontierFailures = preflight.correctnessFrontierFailures, + nativeInfrastructureFailures = callsClassPilotNativeFailures(events, source.record.instrumentation.universe), + nativeCoverageRejections = events.count { !it.nativeCoverage.accepted }, + strictOracleNotApplicable = events.count { !it.strictOracleApplicable }, + ) + writeCallsArtifactAtomically(output.resolve("result.json"), encodeCallsArtifact(result)) + System.out.appendLine(encodeCallsArtifact(result)) +} + +private fun PreparedClassSource.validatePreflightAdmission(preflight: CallsClassPreflightResult) { + require(preflight.admissionVersion == 1) { "Class requires a new profile-independent admission check" } + require(preflight.eligible) { "Class source/native admission failed: ${preflight.reasons}" } + require(preflight.correctnessFrontierFailures == record.classCase.correctnessFrontierFailures) { + "Frozen correctness-frontier evidence differs from the prepared class" + } + val expectedPrimaryEligibility = preflight.eligible && + preflight.nativeSmoke?.status == CallsClassNativeSmokeStatus.VERIFIED && + preflight.correctnessFrontierFailures.isEmpty() + require(preflight.primaryInferenceEligible == expectedPrimaryEligibility) { "Primary admission evidence differs" } + require(preflight.classId == record.classCase.classId) + val preflightIdentityMatches = preflight.preparedSha256 == preparedSha256 && + preflight.toolRevision == manifest.toolRevision + require(preflightIdentityMatches) { "Class preflight identity differs" } +} + +/** Parsing, domain binding and source coverage selection run without a symbolic treatment or machine. */ +internal fun callsClassSourceAdmissionFailures( + harness: CallsClassPreparedHarness, + expectedNativeFrontendRevision: String, +): List = runCatching { + val engine = CurrentTsCallsSymbolicEngine() + val preparations = harness.functions.map { function -> + engine.prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = harness.sourceRoot, + project = harness.project, + function = function, + expectedNativeFrontendRevision = expectedNativeFrontendRevision, + verifySourceCheckout = false, + ), + ) + } + val failures = preparations.filterIsInstance().map { + "${it.reasonCode}: ${it.diagnostic}" + } + if (failures.isNotEmpty()) { + failures + } else { + val first = preparations.first() as CallsFunctionPreparation.Eligible + val scope = resolveCallsSymbolicCoverageScope( + preparation = first, + target = CallsSymbolicCoverageClass(harness.surface.sourceFile, harness.surface.exportName), + ) + if (scope is CallsSymbolicCoverageScope.Rejected) listOf(scope.diagnostic) else emptyList() + } +}.getOrElse { listOf("Source preparation failed: ${it.message ?: it::class.java.name}") } + +private class PreparedClassSource( + val manifest: CallsClassExperimentManifest, + val manifestSha256: String, + val directory: Path, + val record: CallsClassPreparedRecord, + val preparedSha256: String, + val harness: CallsClassPreparedHarness, +) { + fun verify() { + record.verifyOriginal() + record.harness(directory) + } + + fun searchRequest( + profile: CallsExperimentProfile, + seed: Long, + budgetMillis: Long, + requireEntryPointWitness: Boolean, + onCandidate: (CallsCoverageCandidate) -> Unit, + ): CallsClassSearchRequest = CallsClassSearchRequest( + harness = harness, + profile = profile, + frozenModelIds = manifest.modelSet.ids, + expectedNativeFrontendRevision = manifest.nativeFrontendRevision, + seed = seed, + budget = budgetMillis.milliseconds, + timeQuantum = manifest.timeQuantumMillis.milliseconds, + solverQueryLimit = manifest.solverLimitMillis.milliseconds, + solverType = SolverType.valueOf(manifest.solver), + candidateCap = manifest.candidateCap, + requireEntryPointWitness = requireEntryPointWitness, + onCandidate = onCandidate, + ) +} + +private fun loadPreparedClass(preparedDirectory: Path, classId: String): PreparedClassSource { + val root = preparedDirectory.toRealPath() + val manifestPath = root.resolve("manifest.json") + val manifest = decodeClassArtifact(manifestPath) + manifest.verifyBuild(CallsBuildIdentity.toolRevision, CallsBuildIdentity.nativeFrontendRevision) + val classCase = manifest.classes.single { it.classId == classId } + val directory = root.resolve(classId).toRealPath() + require(directory.startsWith(root)) + val recordPath = directory.resolve("class.json") + val record = decodeClassArtifact(recordPath) + require(record.classCase == classCase) { "Prepared class differs from manifest" } + record.verifyOriginal() + + return PreparedClassSource( + manifest = manifest, + manifestSha256 = callsArtifactSha256(Files.readAllBytes(manifestPath)), + directory = directory, + record = record, + preparedSha256 = callsArtifactSha256(Files.readAllBytes(recordPath)), + harness = record.harness(directory), + ) +} + +private fun replayClassCandidates( + source: PreparedClassSource, + search: CallsClassSearchResult, + output: Path, +): List { + val path = output.resolve("replays.jsonl") + val events = readCallsJournal(path).toMutableList() + require(events.map { it.index } == events.indices.toList() && events.size <= search.search.candidates.size) { + "Class replay journal has missing, duplicate, or excess indices" + } + events.forEachIndexed { index, event -> + validateCallsClassReplayEvent( + candidate = search.search.candidates[index], + index = index, + event = event, + universe = source.record.instrumentation.universe, + ) + } + val scenarios = source.record.scenarios.withIndex().associate { it.value.id to it.index } + for (index in search.search.candidates.indices.drop(events.size)) { + source.verify() + val candidate = search.search.candidates[index] + val scenarioId = requireNotNull(candidate.entryPointId) + val scenarioIndex = scenarios.getValue(scenarioId) + val function = source.record.functions[scenarioIndex] + val observations = source.record.observations[scenarioIndex] + val replayStarted = TimeSource.Monotonic.markNow() + val original = runCatching { + OriginalTypeScriptClassReplayer( + sourceRoot = source.harness.sourceRoot, + function = function, + observationFields = observations, + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) } + } + val instrumentation = source.record.instrumentation + val instrumented = runCatching { + OriginalTypeScriptClassReplayer( + sourceRoot = source.harness.sourceRoot, + function = function, + observationFields = observations, + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = source.record.classCase.surface.sourceFile, + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) } + } + source.verify() + val originalValue = original.getOrNull() + val instrumentedValue = instrumented.getOrNull() + val validation = runCatching { + require(originalValue != null && instrumentedValue != null) { "Concrete replay failed" } + require(originalValue.inputs == candidate.inputs && instrumentedValue.inputs == candidate.inputs) { + "Class replay input snapshot differs from candidate" + } + instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys) + classCandidateStrictOracle(candidate, originalValue, instrumentedValue) + } + val failures = listOfNotNull( + original.exceptionOrNull()?.let { "original: ${it.message ?: it::class.java.name}" }, + instrumented.exceptionOrNull()?.let { "instrumented: ${it.message ?: it::class.java.name}" }, + validation.exceptionOrNull()?.let { "validation: ${it.message ?: it::class.java.name}" }, + ) + val replayError = failures.takeIf { it.isNotEmpty() }?.joinToString(separator = "; ") + val event = CallsClassReplayEvent( + index = index, + scenarioId = scenarioId, + original = originalValue, + instrumented = instrumentedValue, + oracle = validation.getOrNull() ?: CallsClassOracleResult( + accepted = false, + diagnostic = "Concrete replay failed", + ), + error = replayError, + replayElapsedMillis = replayStarted.elapsedNow().inWholeMilliseconds, + nativeCoverage = callsClassNativeCoverageAcceptance( + original = originalValue, + instrumented = instrumentedValue, + universe = instrumentation.universe, + error = replayError, + ), + origin = candidate.origin, + strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED, + ) + appendCallsJournalDurably(path, encodeCallsArtifact(event)) + events += event + } + + return events +} + +internal fun validateCallsClassReplayEvent( + candidate: CallsCoverageCandidate, + index: Int, + event: CallsClassReplayEvent, + universe: CallsClassSourceUniverse, +) { + require(event.index == index && event.scenarioId == candidate.entryPointId) { + "Persisted replay index or scenario differs from candidate" + } + require(event.origin == candidate.origin) { "Persisted replay candidate origin differs" } + require(event.strictOracleApplicable == (candidate.origin == CallsCoverageCandidateOrigin.TERMINATED)) { + "Persisted symbolic oracle applicability differs from candidate origin" + } + if (event.error == null) { + val original = requireNotNull(event.original) + val instrumented = requireNotNull(event.instrumented) + require(original.inputs == candidate.inputs && instrumented.inputs == candidate.inputs) { + "Persisted replay input snapshot differs" + } + universe.checkedCoveredIds(instrumented.coveredStatementKeys) + val expectedOracle = classCandidateStrictOracle(candidate, original, instrumented) + require(event.oracle == expectedOracle) { "Persisted replay oracle decision differs" } + } else { + require(!event.oracle.accepted) { "Failed persisted replay is marked accepted" } + } + + val expectedNative = callsClassNativeCoverageAcceptance( + original = event.original, + instrumented = event.instrumented, + universe = universe, + error = event.error, + ) + require(event.nativeCoverage == expectedNative) { "Persisted native coverage decision differs" } +} + +private fun classCandidateStrictOracle( + candidate: CallsCoverageCandidate, + original: CallsClassReplayObservation, + instrumented: CallsClassReplayObservation, +): CallsClassOracleResult = when (candidate.origin) { + CallsCoverageCandidateOrigin.TERMINATED -> compareCallsClassObservations( + symbolic = candidate.observation, + original = original.observation, + instrumented = instrumented.observation, + ) + CallsCoverageCandidateOrigin.SUSPENDED -> CallsClassOracleResult( + accepted = false, + diagnostic = "Not applicable: suspended input has no symbolic completion or observation", + ) +} + +private inline fun decodeClassArtifact(path: Path): T = + CallsExperimentJson.json.decodeFromString(Files.readString(path)) + +private const val CLASS_REPLAY_TIMEOUT_MILLIS = 20_000L + +private const val CLASS_PREFLIGHT_ARGUMENT_COUNT = 3 +private const val CLASS_CELL_ARGUMENT_COUNT = 6 diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassHarness.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassHarness.kt new file mode 100644 index 0000000000..7cde0af0c1 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassHarness.kt @@ -0,0 +1,415 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.ConstantDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.OptionalDomain +import org.usvm.ts.pbt.model.PropertyDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.StringDomain +import org.usvm.ts.pbt.model.TupleDomain +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import java.nio.file.Files +import java.nio.file.Path + +@Serializable +internal enum class CallsClassMemberKind { METHOD, GETTER, SETTER, FUNCTION_FIELD } + +@Serializable +internal data class CallsClassMember( + val id: String, + val name: String, + val kind: CallsClassMemberKind, + val inputs: List, + val resultDomain: PropertyDomain, + val isStatic: Boolean = false, + val resultClassName: String? = null, +) + +/** A same-module exported class whose complete state consists only of these own data fields. */ +@Serializable +internal data class CallsClassDataClass( + val id: String, + val exportName: String, + val fields: Map, +) + +/** A complete source-inventoried callable surface and receiver-state shape, frozen before comparing policies. */ +@Serializable +internal data class CallsClassSurface( + val sourceFile: String, + val exportName: String, + val constructorInputs: List, + val members: List, + val stateFields: Map, + val dataClasses: List = emptyList(), +) { + init { + CallsSymbolicCoverageClass(sourceFile = sourceFile, className = exportName) + require(exportName.matches(CLASS_IDENTIFIER)) { "Class export must be an identifier" } + require(members.map(CallsClassMember::id).distinct().size == members.size) { "Member IDs must be unique" } + require(members.all { it.id.isNotBlank() && it.name.matches(CLASS_IDENTIFIER) }) { + "Every callable member needs a stable ID and an identifier name" + } + require(dataClasses.map(CallsClassDataClass::id).distinct().size == dataClasses.size) { + "Data-class IDs must be unique" + } + require(dataClasses.map(CallsClassDataClass::exportName).distinct().size == dataClasses.size) { + "Data-class exports must be unique" + } + for (dataClass in dataClasses) { + require(dataClass.id.matches(CLASS_IDENTIFIER) && dataClass.id !in setOf("plain", "target")) { + "Data-class IDs must be non-reserved identifiers" + } + require(dataClass.exportName.matches(CLASS_IDENTIFIER) && dataClass.exportName != exportName) { + "Data classes must name distinct same-module exports" + } + require(dataClass.fields.keys.all { it.matches(CLASS_IDENTIFIER) }) { + "Data-class fields must be identifiers" + } + require(dataClass.fields.values.all { it.isSupportedCallsSymbolicDomain() }) { + "Unsupported data-class observation domain" + } + } + val domains = stateFields.values + constructorInputs.map(PropertyInput::domain) + + members.flatMap { member -> member.inputs.map(PropertyInput::domain) + member.resultDomain } + + dataClasses.flatMap { it.fields.values } + if (dataClasses.isNotEmpty()) { + require(domains.none { it.hasClassIdentityField() } && CLASS_IDENTITY_FIELD !in stateFields) { + "Observation fields collide with the reserved class-identity field" + } + require(dataClasses.none { CLASS_IDENTITY_FIELD in it.fields }) { + "Data-class fields collide with the reserved class-identity field" + } + } + require(stateFields.keys.all { it.matches(CLASS_IDENTIFIER) }) { "State fields must be identifiers" } + for (member in members) { + require(member.resultClassName == null || member.resultClassName == exportName) { + "Only the selected concrete class can be an explicitly declared class result" + } + when (member.kind) { + CallsClassMemberKind.GETTER -> require(member.inputs.isEmpty()) { "Getters have no inputs" } + CallsClassMemberKind.SETTER -> require(member.inputs.size == 1) { "Setters have one input" } + else -> Unit + } + } + require(constructorInputs.map(PropertyInput::name).distinct().size == constructorInputs.size) + require(callsSymbolicInputPreflight(constructorInputs) == null) { "Unsupported constructor input domain" } + require(members.all { callsSymbolicInputPreflight(it.inputs) == null }) { "Unsupported callable input domain" } + require( + stateFields.values.all { it.isSupportedCallsSymbolicDomain() } + ) { "Unsupported receiver observation domain" } + require( + members.all { it.resultDomain.isSupportedCallsSymbolicDomain() } + ) { "Unsupported returned-value observation domain" } + } +} + +@Serializable +internal data class CallsClassScenario(val id: String, val memberIds: List) + +internal data class CallsClassPreparedHarness( + val surface: CallsClassSurface, + val scenarios: List, + val sourceRoot: Path, + val project: CallsProjectCase, + val functions: List, + val observations: List>, +) + +/** Each scenario constructs a fresh receiver; every invocation inside that scenario uses the same receiver. */ +internal fun prepareCallsClassHarness( + sourceRoot: Path, + project: CallsProjectCase, + surface: CallsClassSurface, + maxSequenceLength: Int = 2, +): CallsClassPreparedHarness { + require(maxSequenceLength in 1..2) { "Class scenarios support a frozen sequence bound of one or two" } + val root = sourceRoot.toRealPath() + require(Files.isRegularFile(root.resolve(surface.sourceFile))) { "Class source is missing" } + val sequences = mutableListOf(emptyList()) + sequences += surface.members.map { member -> listOf(member.id) } + if (maxSequenceLength == 2) { + sequences += surface.members.flatMap { first -> surface.members.map { second -> listOf(first.id, second.id) } } + } + val scenarios = sequences.mapIndexed { index, ids -> CallsClassScenario(id = "scenario-$index", memberIds = ids) } + val fileName = ".usvm-calls-class-harness.ts" + require(!Files.exists(root.resolve(fileName))) { "A generated class harness already exists in the overlay" } + val source = StringBuilder() + val moduleName = CallsExperimentJson.json.encodeToUtf8SafeString("./${surface.sourceFile}") + val imports = listOf("${surface.exportName} as CallsTargetClass") + + surface.dataClasses.mapIndexed { index, dataClass -> "${dataClass.exportName} as CallsDataClass$index" } + source.appendLine("import { ${imports.joinToString()} } from $moduleName;") + if (surface.dataClasses.isNotEmpty()) { + source.appendLine("// __usvm_class_registry ${CallsExperimentJson.json.encodeToString(surface.dataClasses)}") + } + val functions = mutableListOf() + val observations = mutableListOf>() + for ((index, scenario) in scenarios.withIndex()) { + val generated = ClassScenarioSource(surface, scenario, index).generate() + source.appendLine(generated.source) + functions += CallsFunctionCase( + functionId = scenario.id, + sourceFile = fileName, + entryPoint = TypeScriptEntryPoint(module = fileName, exportName = generated.exportName), + inputs = generated.inputs, + targets = emptyList(), + symbolicCoverageClass = CallsSymbolicCoverageClass(surface.sourceFile, surface.exportName), + ) + observations += generated.observationFields + } + Files.writeString(root.resolve(fileName), source) + + return CallsClassPreparedHarness( + surface = surface, + scenarios = scenarios, + sourceRoot = root, + project = project.copy(functions = functions), + functions = functions, + observations = observations, + ) +} + +@Serializable +internal data class CallsClassRawShapeCheck( + val expression: String, + val domain: PropertyDomain, + val skipCallableProperties: Boolean, + val allowTargetClass: Boolean = false, + val callablePropertyNames: Set = emptySet(), + val dataClasses: List = emptyList(), +) + +private data class GeneratedClassScenario( + val source: String, + val exportName: String, + val inputs: List, + val observationFields: Map, +) + +private class ClassScenarioSource( + private val surface: CallsClassSurface, + private val scenario: CallsClassScenario, + private val index: Int, +) { + private val inputs = mutableListOf() + private val fields = linkedMapOf() + private val declarations = StringBuilder() + private var snapshotIndex = 0 + + fun generate(): GeneratedClassScenario { + val constructorArgs = addInputs(surface.constructorInputs) + declarations.appendLine(" const receiver = new CallsTargetClass($constructorArgs);") + observeState(step = 0) + val membersById = surface.members.associateBy(CallsClassMember::id) + scenario.memberIds.forEachIndexed { callIndex, id -> + val member = membersById.getValue(id) + val args = addInputs(member.inputs) + val owner = if (member.isStatic) "CallsTargetClass" else "receiver" + val access = "$owner.${member.name}" + val expression = when (member.kind) { + CallsClassMemberKind.GETTER -> { access } + CallsClassMemberKind.SETTER -> { + declarations.appendLine(" $access = $args;") + "undefined" + } + CallsClassMemberKind.METHOD, CallsClassMemberKind.FUNCTION_FIELD -> { "$access($args)" } + } + declarations.appendLine(" const value_result_$callIndex = $expression;") + observe( + name = "result_$callIndex", + expression = "value_result_$callIndex", + domain = member.resultDomain, + allowTargetClass = member.resultClassName == surface.exportName, + ) + observeState(step = callIndex + 1) + } + val observationType = fields.entries.joinToString(separator = "; ") { (name, domain) -> + "$name: ${domain.classTypeScriptType()}" + } + val parameters = inputs.joinToString { input -> "${input.name}: ${input.domain.classTypeScriptType()}" } + val exportName = "callsClassScenario$index" + val source = """ + interface CallsObservation$index { $observationType } + export function $exportName($parameters): CallsObservation$index { + $declarations + return { ${fields.keys.joinToString()} }; + } + """.trimIndent() + + return GeneratedClassScenario(source, exportName, inputs.toList(), fields.toMap()) + } + + private fun addInputs(declared: List): String = declared.joinToString { input -> + val name = "input_${inputs.size}" + inputs += input.copy(name = name) + name + } + + private fun observeState(step: Int) { + rawShapeCheck( + expression = "receiver", + domain = ObjectDomain(surface.stateFields), + skipCallableProperties = true, + allowTargetClass = true, + ) + if (surface.dataClasses.isNotEmpty()) { + observe( + name = "state_${step}_$CLASS_IDENTITY_FIELD", + expression = classIdentity("receiver"), + domain = classIdentityDomain(), + ) + } + surface.stateFields.forEach { (name, domain) -> + val type = domain.classTypeScriptType() + observe( + name = "state_${step}_$name", + expression = "(receiver as unknown as { $name: $type }).$name", + domain = domain, + ) + } + } + + private fun observe( + name: String, + expression: String, + domain: PropertyDomain, + allowTargetClass: Boolean = false, + ) { + rawShapeCheck( + expression = expression, + domain = domain, + skipCallableProperties = true, + allowTargetClass = allowTargetClass, + ) + val snapshot = snapshot(expression, domain) + declarations.appendLine(" const $name = $snapshot;") + fields[name] = observationDomain(domain) + } + + private fun rawShapeCheck( + expression: String, + domain: PropertyDomain, + skipCallableProperties: Boolean, + allowTargetClass: Boolean = false, + ) { + val callablePropertyNames = surface.members.filter { member -> + member.kind == CallsClassMemberKind.FUNCTION_FIELD && !member.isStatic + }.mapTo(linkedSetOf()) { it.name } + val check = CallsClassRawShapeCheck( + expression = expression, + domain = domain, + skipCallableProperties = skipCallableProperties, + allowTargetClass = allowTargetClass, + callablePropertyNames = callablePropertyNames, + dataClasses = surface.dataClasses, + ) + val encoded = CallsExperimentJson.json.encodeToString(check) + declarations.appendLine(" // __usvm_class_observe $encoded") + } + + private fun classIdentityDomain(): StringDomain = StringDomain( + maxLength = maxOf("target".length, "plain".length, surface.dataClasses.maxOf { it.id.length }), + ) + + private fun classIdentity(expression: String): String { + val registered = surface.dataClasses.mapIndexed { index, dataClass -> + val id = CallsExperimentJson.json.encodeToUtf8SafeString(dataClass.id) + "($expression) instanceof CallsDataClass$index ? $id : " + }.joinToString(separator = "") + return "(($expression) instanceof CallsTargetClass ? \"target\" : ${registered}\"plain\")" + } + + private fun observationDomain(domain: PropertyDomain): PropertyDomain { + if (surface.dataClasses.isEmpty()) return domain + + return when (domain) { + is ObjectDomain -> ObjectDomain( + fields = domain.fields.mapValues { (_, value) -> observationDomain(value) } + + (CLASS_IDENTITY_FIELD to classIdentityDomain()), + ) + is OptionalDomain -> domain.copy(value = observationDomain(domain.value)) + is ArrayDomain -> domain.copy(element = observationDomain(domain.element)) + else -> domain + } + } + + private fun snapshot(expression: String, domain: PropertyDomain): String = when (domain) { + is ObjectDomain -> { + val entries = domain.fields.map { (name, nested) -> + require(name.matches(CLASS_IDENTIFIER)) { "Observed object fields must be identifiers" } + "$name: ${snapshot("($expression).$name", nested)}" + }.toMutableList() + if (surface.dataClasses.isNotEmpty()) { + entries += "$CLASS_IDENTITY_FIELD: ${classIdentity(expression)}" + } + entries.joinToString(prefix = "{ ", postfix = " }") + } + is OptionalDomain -> { + val nil = if (domain.nil == JsConcreteValue.Null) "null" else "undefined" + val temporary = "snapshot_${snapshotIndex++}" + val original = "snapshot_${snapshotIndex++}" + declarations.appendLine(" const $original = $expression;") + declarations.appendLine(" let $temporary: ${observationDomain(domain).classTypeScriptType()};") + declarations.appendLine(" if ($original === $nil) {") + declarations.appendLine(" $temporary = $nil;") + declarations.appendLine(" } else {") + val nested = snapshot(original, domain.value) + declarations.appendLine(" $temporary = $nested;") + declarations.appendLine(" }") + temporary + } + is ArrayDomain -> { + val temporary = "snapshot_${snapshotIndex++}" + val variable = "snapshot_index_${snapshotIndex++}" + declarations.appendLine(" const $temporary: ${observationDomain(domain).classTypeScriptType()} = [];") + declarations.appendLine(" for (let $variable = 0; $variable < ($expression).length; $variable++) {") + val element = snapshot("($expression)[$variable]", domain.element) + declarations.appendLine(" $temporary[$variable] = $element;") + declarations.appendLine(" }") + temporary + } + is TupleDomain -> { error("Tuple class observations are unsupported") } + else -> { expression } + } +} + +internal fun PropertyDomain.classTypeScriptType(): String = when (this) { + BooleanDomain -> "boolean" + is IntegerDomain, is NumberDomain -> "number" + is StringDomain -> "string" + is ConstantDomain -> when (value) { + JsConcreteValue.Null -> "null" + JsConcreteValue.Undefined -> "undefined" + is JsConcreteValue.Boolean -> "boolean" + is JsConcreteValue.Number -> "number" + is JsConcreteValue.String -> "string" + else -> error("Structured constants are unsupported class domains") + } + is OptionalDomain -> "(${value.classTypeScriptType()} | ${ConstantDomain(nil).classTypeScriptType()})" + is ArrayDomain -> "(${element.classTypeScriptType()})[]" + is TupleDomain -> elements.joinToString(prefix = "[", postfix = "]") { it.classTypeScriptType() } + is ObjectDomain -> fields.entries.joinToString(prefix = "{ ", postfix = " }") { (name, domain) -> + val key = CallsExperimentJson.json.encodeToUtf8SafeString(name) + "$key: ${domain.classTypeScriptType()}" + } +} + +private fun PropertyDomain.hasClassIdentityField(): Boolean = when (this) { + is ObjectDomain -> CLASS_IDENTITY_FIELD in fields || fields.values.any { it.hasClassIdentityField() } + is OptionalDomain -> value.hasClassIdentityField() + is ArrayDomain -> element.hasClassIdentityField() + is TupleDomain -> elements.any { it.hasClassIdentityField() } + else -> false +} + +internal const val CLASS_IDENTITY_FIELD = "__usvmCallsClassIdentity" + +private val CLASS_IDENTIFIER = Regex("[A-Za-z_$][A-Za-z0-9_$]*") diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassOracle.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassOracle.kt new file mode 100644 index 0000000000..d2aa0e5d23 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassOracle.kt @@ -0,0 +1,683 @@ +package org.usvm.ts.calls + +import io.ksmt.utils.asExpr +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import org.jacodb.ets.model.EtsArrayType +import org.jacodb.ets.model.EtsMethod +import org.jacodb.ets.model.EtsType +import org.jacodb.ets.model.EtsUnionType +import org.usvm.UExpr +import org.usvm.USort +import org.usvm.isTrue +import org.usvm.machine.expr.TsUnresolvedSort +import org.usvm.machine.expr.extractDouble +import org.usvm.machine.expr.extractInt +import org.usvm.machine.expr.toConcreteBoolValue +import org.usvm.machine.state.TsMethodResult +import org.usvm.machine.state.TsState +import org.usvm.model.UModelBase +import org.usvm.ts.pbt.backend.PropertyRunConfiguration +import org.usvm.ts.pbt.backend.PropertyRunStatus +import org.usvm.ts.pbt.fastcheck.FastCheckBackend +import org.usvm.ts.pbt.fastcheck.FastCheckRuntime +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.ConstantDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.OptionalDomain +import org.usvm.ts.pbt.model.PropertyDefinition +import org.usvm.ts.pbt.model.PropertyDomain +import org.usvm.ts.pbt.model.PropertyId +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.StringDomain +import org.usvm.ts.pbt.model.TupleDomain +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import org.usvm.ts.pbt.model.contains +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import org.usvm.util.EtsHierarchy +import org.usvm.util.componentType +import org.usvm.util.declaredInstanceProperty +import org.usvm.util.mkArrayIndexLValue +import org.usvm.util.mkArrayLengthLValue +import org.usvm.util.mkFieldLValue +import org.usvm.util.resolveStringValue +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import kotlin.time.TimeSource + +/** Unrepresentable observations remain evidence, never a successful equivalence check. */ +@Serializable +internal data class CallsClassObservation( + val completion: CallsCoverageCompletion, + val values: Map = emptyMap(), + val representable: Boolean = true, + val diagnostic: String? = null, +) + +@Serializable +internal data class CallsClassReplayObservation( + val inputs: List, + val observation: CallsClassObservation, + val coveredStatementKeys: Set = emptySet(), + val replayElapsedMillis: Long = 0, +) + +@Serializable +internal data class CallsClassOracleResult(val accepted: Boolean, val diagnostic: String? = null) + +internal fun compareCallsClassObservations( + symbolic: CallsClassObservation?, + original: CallsClassObservation, + instrumented: CallsClassObservation, +): CallsClassOracleResult { + if (symbolic == null) return CallsClassOracleResult(accepted = false, diagnostic = "Symbolic observation missing") + val observations = listOf(symbolic, original, instrumented) + val unrepresentable = observations.firstOrNull { !it.representable } + if (unrepresentable != null) { + return CallsClassOracleResult( + accepted = false, + diagnostic = unrepresentable.diagnostic ?: "Unrepresentable observation" + ) + } + if (original != instrumented) { + return CallsClassOracleResult( + accepted = false, + diagnostic = "Instrumentation changed returned values or receiver state" + ) + } + if (symbolic != original) { + return CallsClassOracleResult( + accepted = false, + diagnostic = "Symbolic and original returned values or receiver state disagree" + ) + } + + return CallsClassOracleResult(accepted = true) +} + +/** Uses the same first model as CallsSymbolicInputs.resolve; no solver query changes candidate selection. */ +internal fun extractCallsClassObservation( + state: TsState, + method: EtsMethod, + fields: Map, +): CallsClassObservation { + val methodResult = state.methodResult + if (methodResult is TsMethodResult.TsException) { + return CallsClassObservation( + completion = CallsCoverageCompletion.THREW, + representable = false, + diagnostic = "Thrown class scenarios have no complete return/state snapshot; " + + "exception type=${methodResult.type}, value=${methodResult.value}", + ) + } + + val completion = CallsCoverageCompletion.RETURNED + return runCatching { + val result = methodResult as? TsMethodResult.Success.RegularCall + ?: error("Class scenario did not produce a regular returned snapshot") + require(result.method == method) { "Class scenario did not complete its wrapper" } + val value = ClassObservationValueResolver(state, state.models.first()).resolve( + domain = ObjectDomain(fields), + type = method.returnType, + expression = result.value, + ) as JsConcreteValue.Object + CallsClassObservation(completion = completion, values = value.fields) + }.getOrElse { error -> + CallsClassObservation( + completion = completion, + representable = false, + diagnostic = error.message ?: error::class.java.name, + ) + } +} + +private class ClassObservationValueResolver( + private val state: TsState, + private val model: UModelBase, +) { + private val hierarchy = EtsHierarchy(state.ctx.scene) + + fun resolve(domain: PropertyDomain, type: EtsType, expression: UExpr): JsConcreteValue = with( + state.ctx + ) { + val value = unbox(expression) + when (domain) { + BooleanDomain -> JsConcreteValue.Boolean(model.eval(value.asExpr(boolSort)).toConcreteBoolValue()) + is NumberDomain, is IntegerDomain -> JsConcreteValue.number( + model.eval(value.asExpr(fp64Sort)).extractDouble() + ) + is StringDomain -> JsConcreteValue.String( + state.resolveStringValue( + string = value.asExpr(addressSort), + model = model, + maxLength = domain.maxLength + ), + ) + is ConstantDomain -> resolveConstant(domain, value) + is OptionalDomain -> resolveOptional(domain, type, value) + is ObjectDomain -> resolveObject(domain, type, value) + is ArrayDomain -> resolveArray(domain, type, value) + is TupleDomain -> error("Tuple class observations are unsupported") + } + } + + private fun unbox(value: UExpr): UExpr = with(state.ctx) { + if (value.sort != addressSort) return@with value + + val reference = model.eval(value.asExpr(addressSort)) + if (!reference.isFakeObject()) return@with value + + val fakeType = reference.getFakeType(state.memory) + when { + model.eval(fakeType.boolTypeExpr).isTrue -> reference.extractBool(state.memory) + model.eval(fakeType.fpTypeExpr).isTrue -> reference.extractFp(state.memory) + model.eval(fakeType.refTypeExpr).isTrue -> reference.extractRef(state.memory) + else -> error("Observed fake value has no selected runtime type") + } + } + + private fun resolveOptional( + domain: OptionalDomain, + type: EtsType, + value: UExpr, + ): JsConcreteValue = with(state.ctx) { + val nil = if (domain.nil == JsConcreteValue.Null) mkTsNullValue() else mkUndefinedValue() + val isNil = value.sort == addressSort && model.eval(mkHeapRefEq(value.asExpr(addressSort), nil)).isTrue + if (isNil) return@with domain.nil + + val nestedType = (type as? EtsUnionType)?.types?.firstOrNull { nested -> + nested != org.jacodb.ets.model.EtsNullType && nested != org.jacodb.ets.model.EtsUndefinedType + } ?: type + resolve(domain.value, nestedType, value) + } + + private fun resolveObject( + domain: ObjectDomain, + type: EtsType, + value: UExpr, + ): JsConcreteValue = with(state.ctx) { + val reference = value.asExpr(addressSort) + val values = domain.fields.mapValues { (name, nested) -> + val fieldType = declaredInstanceProperty(type = type, name = name, hierarchy = hierarchy)?.type + ?: error("Observed field $name has no declared type in $type") + val sort = typeToSort(fieldType) + val storageSort = if (sort is TsUnresolvedSort) addressSort else sort + val fieldValue = state.memory.read(mkFieldLValue(storageSort, reference, name)) + resolve(nested, fieldType, fieldValue) + } + JsConcreteValue.Object(values) + } + + private fun resolveConstant(domain: ConstantDomain, value: UExpr): JsConcreteValue = with(state.ctx) { + val resolved = when (val expected = domain.value) { + JsConcreteValue.Undefined, JsConcreteValue.Null -> { + val nil = if (expected == JsConcreteValue.Null) mkTsNullValue() else mkUndefinedValue() + require(value.sort == addressSort && model.eval(mkHeapRefEq(value.asExpr(addressSort), nil)).isTrue) { + "Observed constant differs from its declared value" + } + expected + } + is JsConcreteValue.Boolean -> { + JsConcreteValue.Boolean(model.eval(value.asExpr(boolSort)).toConcreteBoolValue()) + } + is JsConcreteValue.Number -> { + JsConcreteValue.number(model.eval(value.asExpr(fp64Sort)).extractDouble()) + } + is JsConcreteValue.String -> { + JsConcreteValue.String( + state.resolveStringValue( + string = value.asExpr(addressSort), + model = model, + maxLength = expected.value.length, + ), + ) + } + else -> { error("Structured constant class observations are unsupported") } + } + require(resolved == domain.value) { "Observed constant differs from its declared value" } + resolved + } + + private fun resolveArray(domain: ArrayDomain, type: EtsType, value: UExpr): JsConcreteValue = with( + state.ctx + ) { + val arrayType = type as? EtsArrayType ?: error("Observed array has no declared array type: $type") + val reference = value.asExpr(addressSort) + val length = model.eval(state.memory.read(mkArrayLengthLValue(reference, arrayType))).extractInt() + require(length in 0..domain.maxLength) { "Observed array exceeds the frozen observation capacity" } + val elementSort = typeToSort(arrayType.componentType) + val storageSort = if (elementSort is TsUnresolvedSort) addressSort else elementSort + val elements = (0 until length).map { index -> + val lValue = mkArrayIndexLValue(storageSort, reference, mkBv(index), arrayType) + resolve(domain.element, arrayType.componentType, state.memory.read(lValue)) + } + JsConcreteValue.Array(elements) + } +} + +/** Replays the exact same scenario in a fresh process against original or counter-instrumented source. */ +internal class OriginalTypeScriptClassReplayer( + sourceRoot: Path, + private val function: CallsFunctionCase, + private val observationFields: Map, + sourceOverride: String? = null, + overrideSourceFile: String? = null, + private val counterGlobalName: String? = null, +) : AutoCloseable { + private val root = sourceRoot.toRealPath() + private val workspace = Files.createTempDirectory("usvm-calls-class-replay-") + private val overlay = workspace.resolve("overlay") + private val observationPath = workspace.resolve("observation.json") + private val wrapperName = ".usvm-class-replay-${UUID.randomUUID()}.mts" + private val backend: FastCheckBackend + + init { + val setup = runCatching { + require( + (sourceOverride == null) == (overrideSourceFile == null) + ) { "Source override needs a relative target file" } + createClassReplayOverlay(root, overlay, overrideSourceFile, sourceOverride) + val replayEntry = overlay.resolve(function.sourceFile) + Files.deleteIfExists(replayEntry) + val harnessSource = Files.readString(root.resolve(function.sourceFile)) + Files.writeString(replayEntry, nativeCheckedClassHarness(harnessSource)) + Files.writeString( + overlay.resolve(wrapperName), + classReplayWrapper( + sourceFile = function.sourceFile, + exportName = function.entryPoint.exportName, + observationPath = observationPath, + dropSentinel = function.inputs.isEmpty(), + counterGlobalName = counterGlobalName, + fields = observationFields.keys, + ) + ) + FastCheckBackend(sourceRoots = listOf(overlay, root)) + } + backend = setup.getOrElse { error -> + close() + throw error + } + } + + fun replay(inputs: List, timeoutMillis: Long): CallsClassReplayObservation { + require(inputs.size == function.inputs.size) + val emptyInputs = function.inputs.isEmpty() + val observation = execute( + seed = 0, + timeoutMillis = timeoutMillis, + examples = listOf(if (emptyInputs) listOf(JsConcreteValue.Boolean(true)) else inputs), + ) + require(observation.inputs == inputs) { "Class replay input transport changed" } + return observation + } + + /** Native-only sampling uses the declared domains; it does not consult any symbolic profile or candidate. */ + fun sample(seed: Long, timeoutMillis: Long): CallsClassReplayObservation = execute( + seed = seed, + timeoutMillis = timeoutMillis, + examples = emptyList(), + ) + + private fun execute( + seed: Long, + timeoutMillis: Long, + examples: List>, + ): CallsClassReplayObservation { + require(timeoutMillis > 0) + require(seed in Int.MIN_VALUE.toLong()..Int.MAX_VALUE.toLong()) { + "Native PBT seed must fit a signed 32-bit integer" + } + val nativeSeed = seed.toInt() + Files.deleteIfExists(observationPath) + val started = TimeSource.Monotonic.markNow() + val emptyInputs = function.inputs.isEmpty() + val propertyInputs = if (emptyInputs) { + listOf(PropertyInput(name = "sentinel", domain = BooleanDomain)) + } else { + function.inputs + } + val result = backend.run( + property = PropertyDefinition( + id = PropertyId("calls.class.exact-replay"), + inputs = propertyInputs, + predicate = TypeScriptEntryPoint(module = wrapperName, exportName = "replayClassScenario"), + ), + configuration = PropertyRunConfiguration( + seed = nativeSeed, + numRuns = 1, + timeoutMillis = timeoutMillis, + examples = examples, + ), + ) + require(result.status == PropertyRunStatus.SUCCESS && result.numRuns == 1) { + "Class replay did not complete exactly once: $result" + } + val observation = CallsExperimentJson.json.decodeFromString( + Files.readString(observationPath) + ) + val inputsInsideDomains = observation.inputs.size == function.inputs.size && + observation.inputs.zip(function.inputs).all { (value, input) -> value in input.domain } + require(inputsInsideDomains) { "Native sample is outside the frozen constructor input domains" } + val expectedFields = observationFields.keys + val hasExactFields = !observation.observation.representable || + observation.observation.values.keys == expectedFields + require(hasExactFields) { "Class replay omitted or added observation fields" } + return observation.copy(replayElapsedMillis = started.elapsedNow().inWholeMilliseconds) + } + + override fun close() { + Files.walk(workspace).use { paths -> paths.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) } + } +} + +private fun createClassReplayOverlay(root: Path, overlay: Path, relative: String?, source: String?) { + Files.createDirectories(overlay) + val target = relative?.let(Path::of) + val isSafeTarget = target == null || !target.isAbsolute && target.normalize() == target && !target.startsWith("..") + require(isSafeTarget) { "Replay source override must stay inside source root" } + fun mirror(directory: Path, destination: Path, remaining: Path?) { + Files.newDirectoryStream(directory).use { entries -> + for (entry in entries) { + val targetSuffix = remaining?.takeIf { entry.fileName == it.getName(0) } + val output = destination.resolve(entry.fileName) + if (targetSuffix == null) { + Files.createSymbolicLink(output, entry.toAbsolutePath()) + } else if (targetSuffix.nameCount == 1) { + Files.writeString(output, requireNotNull(source)) + } else { + Files.createDirectories(output) + mirror(entry, output, targetSuffix.subpath(1, targetSuffix.nameCount)) + } + } + } + } + mirror(root, overlay, target) +} + +private fun classReplayWrapper( + sourceFile: String, + exportName: String, + observationPath: Path, + dropSentinel: Boolean, + counterGlobalName: String?, + fields: Set, +): String { + val json = CallsExperimentJson.json + val protocolModule = json.encodeToUtf8SafeString(FastCheckRuntime.protocolJsonModule().toUri().toString()) + val sourceModule = json.encodeToUtf8SafeString("./$sourceFile") + val exportKey = json.encodeToUtf8SafeString(exportName) + val outputPath = json.encodeToUtf8SafeString(observationPath.toString()) + val expectedFields = json.encodeToUtf8SafeString(fields.sorted()) + val actualArgs = if (dropSentinel) "args.slice(1)" else "args" + val counterSetup = counterGlobalName?.let { name -> + "(globalThis as any)[${json.encodeToUtf8SafeString( + name + )}] = new Set();" + }.orEmpty() + val counterRead = counterGlobalName?.let { name -> + "(globalThis as any)[${json.encodeToUtf8SafeString(name)}]" + } ?: "[]" + return """ + import { writeFileSync } from 'node:fs'; + import { stringifyProtocolJson } from $protocolModule; + import * as scenarioModule from $sourceModule; + + function encode(value: unknown, seen = new Set()): unknown { + if (value === undefined) return { kind: 'undefined' }; + if (value === null) return { kind: 'null' }; + if (typeof value === 'boolean') return { kind: 'boolean', value }; + if (typeof value === 'string') return { kind: 'string', value }; + if (typeof value === 'number') { + if (Number.isNaN(value)) return { kind: 'number', value: 'nan' }; + if (value === Infinity) return { kind: 'number', value: 'positive-infinity' }; + if (value === -Infinity) return { kind: 'number', value: 'negative-infinity' }; + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value, false); + const bits = view.getBigUint64(0, false).toString(16).padStart(16, '0'); + return { kind: 'number', value: 'finite', bits }; + } + if (typeof value !== 'object') throw new Error('Unrepresentable observed value'); + if (seen.has(value)) throw new Error('Cyclic observed value'); + seen.add(value); + let encoded: unknown; + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + if (!Object.hasOwn(value, i)) throw new Error('Sparse observed array'); + } + encoded = { kind: 'array', elements: value.map(element => encode(element, seen)) }; + } else { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) throw new Error('Non-plain observed object'); + const fields: Record = {}; + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== 'string' || ['__proto__', 'constructor', 'prototype'].includes(key)) { + throw new Error('Unsupported observed property'); + } + const descriptor = Object.getOwnPropertyDescriptor(value, key)!; + if (!('value' in descriptor)) throw new Error('Accessor observed property'); + fields[key] = encode(descriptor.value, seen); + } + encoded = { kind: 'object', fields }; + } + seen.delete(value); + return encoded; + } + +${classReplayThrownValueDiagnosticSource().prependIndent(indent = " ")} + + export function replayClassScenario(...args: unknown[]): boolean { + const actualArgs = $actualArgs; + const inputs = actualArgs.map(value => encode(value)); + $counterSetup + let observation: any; + let returned: any; + let completion = 'RETURNED'; + try { + returned = (scenarioModule as any)[$exportKey](...actualArgs); + } catch (error) { + completion = 'THREW'; + const isShapeFailure = error instanceof Error && error.message.startsWith('USVM class observation shape:'); + const diagnostic = isShapeFailure ? error.message : + 'Thrown class scenarios have no complete return/state snapshot; ' + describeThrownValue(error); + observation = { completion, representable: false, diagnostic }; + } + if (completion === 'RETURNED') { + try { + const encoded = encode(returned) as any; + const exactFields = encoded.kind === 'object' && + JSON.stringify(Object.keys(encoded.fields).sort()) === JSON.stringify($expectedFields); + if (!exactFields) { + throw new Error('Class scenario observation fields differ from frozen surface'); + } + observation = { completion, values: encoded.fields }; + } catch (error) { + const diagnostic = error instanceof Error ? error.message : String(error); + observation = { completion, representable: false, diagnostic }; + } + } + const coveredStatementKeys = Array.from($counterRead); + writeFileSync($outputPath, stringifyProtocolJson({ inputs, observation, coveredStatementKeys }), 'utf8'); + return true; + } + """.trimIndent() + "\n" +} + +private fun classReplayThrownValueDiagnosticSource(): String = """ + function describeThrownValue(error: unknown): string { + if (error instanceof Error) { + return 'exception name=' + stringifyProtocolJson(error.name) + + ', message=' + stringifyProtocolJson(error.message); + } + const valueType = typeof error; + if (valueType === 'bigint' || valueType === 'symbol') { + return 'thrown value type=' + valueType + ', value=' + stringifyProtocolJson(String(error)); + } + if ((valueType === 'object' && error !== null) || valueType === 'function') { + return 'thrown value type=' + valueType; + } + return 'thrown value=' + stringifyProtocolJson(encode(error)); + } +""".trimIndent() + + +/** Check raw values before projection, so snapshots cannot silently hide an unlisted concrete property. */ +private fun nativeCheckedClassHarness(source: String): String { + val marker = "// __usvm_class_observe " + val registryMarker = "// __usvm_class_registry " + val checked = source.lines().joinToString(separator = "\n") { line -> + val offset = line.indexOf(marker) + val registryOffset = line.indexOf(registryMarker) + if (registryOffset >= 0) { + val raw = line.substring(registryOffset + registryMarker.length) + val registry = CallsExperimentJson.json.decodeFromString>(raw) + val entries = registry.mapIndexed { index, dataClass -> + val fields = CallsExperimentJson.json.encodeToString(ObjectDomain(dataClass.fields)) + val name = CallsExperimentJson.json.encodeToUtf8SafeString(dataClass.exportName) + "{ constructor: CallsDataClass$index, prototype: CallsDataClass$index.prototype, " + + "name: $name, length: CallsDataClass$index.length, domain: $fields }" + } + "const __usvmClassDataRegistry = [${entries.joinToString()}];" + } else if (offset < 0) { + line + } else { + val raw = line.substring(offset + marker.length) + val check = CallsExperimentJson.json.decodeFromString(raw) + val domain = CallsExperimentJson.json.encodeToString(check.domain) + val allowCallableFields = if (check.skipCallableProperties) { + "(${check.expression}) === receiver" + } else { + "false" + } + val allowedClass = if (check.allowTargetClass) "CallsTargetClass" else "undefined" + val callableNames = CallsExperimentJson.json.encodeToUtf8SafeString(check.callablePropertyNames.sorted()) + val registry = if (check.dataClasses.isEmpty()) "[]" else "__usvmClassDataRegistry" + " __usvmAssertObservedShape(${check.expression}, $domain, " + + "$allowCallableFields, $allowedClass, $callableNames, $registry);" + } + } + return "$NATIVE_CLASS_SHAPE_VALIDATOR\n$checked" +} + +private val NATIVE_CLASS_SHAPE_VALIDATOR = """ + function __usvmCanonicalDomain(domain: any): any { + if (domain === null || typeof domain !== 'object') return domain; + if (Array.isArray(domain)) return domain.map(__usvmCanonicalDomain); + return Object.fromEntries(Object.keys(domain).sort().map(key => [key, __usvmCanonicalDomain(domain[key])])); + } + function __usvmAssertObservedShape( + value: any, domain: any, skipCallableProperties = false, allowedClass?: any, + callablePropertyNames: string[] = [], dataClasses: any[] = [], + ): void { + const fail = (reason: string): never => { throw new Error('USVM class observation shape: ' + reason); }; + const exactOwnKeys = (value: any, names: string[]): boolean => { + const keys = Reflect.ownKeys(value); + return keys.every(key => typeof key === 'string') && + JSON.stringify(keys.sort()) === JSON.stringify([...names].sort()); + }; + for (const entry of dataClasses) { + const ctor = entry.constructor; + const prototype = entry.prototype; + const validConstructor = Object.getPrototypeOf(ctor) === Function.prototype && + ctor.prototype === prototype && ctor.name === entry.name && ctor.length === entry.length && + exactOwnKeys(ctor, ['length', 'name', 'prototype']); + const descriptor = Object.getOwnPropertyDescriptor(prototype, 'constructor'); + const validPrototype = Object.getPrototypeOf(prototype) === Object.prototype && + exactOwnKeys(prototype, ['constructor']) && descriptor && 'value' in descriptor && + descriptor.value === ctor && descriptor.writable && descriptor.configurable && !descriptor.enumerable; + const validConstructorProperties = ['length', 'name', 'prototype'].every(name => { + const property = Object.getOwnPropertyDescriptor(ctor, name)!; + return property && 'value' in property && !property.enumerable && !property.writable && + property.configurable === (name !== 'prototype'); + }); + if (!validConstructor || !validPrototype || !validConstructorProperties) { + fail('Registered data-class constructor or prototype changed'); + } + } + if (domain.kind === 'optional') { + const nil = domain.nil?.kind === 'null' ? null : undefined; + if (value !== nil) { + __usvmAssertObservedShape( + value, domain.value, skipCallableProperties, allowedClass, callablePropertyNames, dataClasses, + ); + } + } else if (domain.kind === 'object') { + if (value === null || typeof value !== 'object' || Array.isArray(value)) fail('Expected object'); + const prototype = Object.getPrototypeOf(value); + const knownClass = allowedClass !== undefined && value instanceof allowedClass && + prototype === allowedClass.prototype; + const dataClass = dataClasses.find(entry => + prototype === entry.prototype && value instanceof entry.constructor); + if (prototype !== Object.prototype && prototype !== null && !knownClass && !dataClass) { + fail('Unlisted object prototype'); + } + if (dataClass && JSON.stringify(__usvmCanonicalDomain(domain)) !== + JSON.stringify(__usvmCanonicalDomain(dataClass.domain))) { + fail('Registered data-class observation domain changed'); + } + if (dataClass && !exactOwnKeys(value, Object.keys(dataClass.domain.fields))) { + fail('Registered data-class own fields changed'); + } + if (dataClass) { + for (const [key, nested] of Object.entries(dataClass.domain.fields)) { + const descriptor = Object.getOwnPropertyDescriptor(value, key)!; + const ordinaryField = 'value' in descriptor && descriptor.enumerable && + descriptor.writable && descriptor.configurable; + if (!ordinaryField) fail('Registered data-class field is not an ordinary data property'); + __usvmAssertObservedShape(value[key], nested, false, undefined, [], dataClasses); + } + } + const allowedCallables = new Set(skipCallableProperties ? callablePropertyNames : []); + for (const name of allowedCallables) { + const descriptor = Object.getOwnPropertyDescriptor(value, name); + if (!descriptor || !('value' in descriptor) || typeof descriptor.value !== 'function') { + fail('Declared callable property changed kind or disappeared'); + } + } + const keys = Reflect.ownKeys(value).filter(key => { + const descriptor = Object.getOwnPropertyDescriptor(value, key)!; + if (!('value' in descriptor)) fail('Accessor in observed data'); + return !(typeof key === 'string' && allowedCallables.has(key) && typeof descriptor.value === 'function'); + }); + const exactKeys = !keys.some(key => typeof key !== 'string') && + JSON.stringify(keys.sort()) === JSON.stringify(Object.keys(domain.fields).sort()); + if (!exactKeys) { + fail('Unlisted or missing object property'); + } + for (const [key, nested] of Object.entries(domain.fields)) { + __usvmAssertObservedShape(value[key], nested, false, undefined, [], dataClasses); + } + } else if (domain.kind === 'array') { + const validArray = Array.isArray(value) && value.length <= (domain.maxLength ?? 10); + if (!validArray) fail('Array exceeds observation capacity'); + for (let i = 0; i < value.length; i++) { + if (!Object.hasOwn(value, i)) fail('Sparse array'); + __usvmAssertObservedShape(value[i], domain.element, false, undefined, [], dataClasses); + } + const allowed = new Set(['length', ...value.map((_: any, index: number) => String(index))]); + const unlistedProperty = Reflect.ownKeys(value).some(key => typeof key !== 'string' || !allowed.has(key)); + if (unlistedProperty) fail('Unlisted array property'); + } else if (domain.kind === 'string') { + const validString = typeof value === 'string' && value.length <= (domain.maxLength ?? 10); + if (!validString) fail('String exceeds observation capacity'); + } else if (domain.kind === 'number' || domain.kind === 'integer') { + if (typeof value !== 'number') fail('Expected number'); + } else if (domain.kind === 'boolean') { + if (typeof value !== 'boolean') fail('Expected boolean'); + } else if (domain.kind === 'constant') { + const expected = domain.value; + if (expected.kind === 'undefined' && value !== undefined) fail('Expected undefined'); + if (expected.kind === 'null' && value !== null) fail('Expected null'); + if (expected.kind === 'boolean' || expected.kind === 'string') { + if (value !== expected.value) fail('Constant changed'); + } + if (expected.kind === 'number' && typeof value !== 'number') fail('Expected constant number'); + } else fail('Unsupported observation domain'); + } +""".trimIndent() diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceInstrumentation.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceInstrumentation.kt new file mode 100644 index 0000000000..62d9dc2a51 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceInstrumentation.kt @@ -0,0 +1,129 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import org.usvm.ts.pbt.fastcheck.TypeScriptClassStatementDescriptor +import org.usvm.ts.pbt.fastcheck.TypeScriptClassSurfaceInventory +import org.usvm.ts.pbt.fastcheck.TypeScriptSourceInspector +import java.nio.file.Files +import java.nio.file.Path + +/** Instrument only a source overlay; the original source and frozen AST offsets remain untouched. */ +@Serializable +internal data class CallsClassSourceInstrumentation( + val universe: CallsClassSourceUniverse, + val instrumentedSource: String, + val instrumentedSha256: String, + val hitGlobalName: String, + val inventory: TypeScriptClassSurfaceInventory, +) { + init { + require(callsClassSourceSha256(instrumentedSource.toByteArray(Charsets.UTF_8)) == instrumentedSha256) { + "Class instrumentation source hash mismatch" + } + } + + fun validateSurface(surface: CallsClassSurface) { + require(surface.sourceFile == universe.sourceFile && surface.exportName == universe.className) { + "Class harness surface selects a different source class" + } + require(surface.constructorInputs.map { input -> input.name } == inventory.constructorParameterNames) { + "Class harness must represent every constructor parameter in source order" + } + val expectedMembers = inventory.publicMembers.map { member -> + Triple(member.name, member.kind, member.isStatic) + }.toSet() + val actualMembers = surface.members.map { member -> + Triple(member.name, member.kind.name, member.isStatic) + }.toSet() + require(actualMembers == expectedMembers && actualMembers.size == surface.members.size) { + "Class harness must represent the complete original public callable surface" + } + for (member in surface.members) { + val original = inventory.publicMembers.single { original -> + original.name == member.name && original.kind == member.kind.name && + original.isStatic == member.isStatic + } + require(member.inputs.map { input -> input.name } == original.parameterNames) { + "Class harness must represent every parameter of ${member.name} in source order" + } + } + val hasUnobservablePrivateState = inventory.instanceDataFields.any { field -> field.isEcmaPrivate } + require(!hasUnobservablePrivateState) { "ECMAScript private receiver state cannot be observed by this harness" } + val expectedFields = inventory.instanceDataFields.map { field -> field.name }.toSet() + require(surface.stateFields.keys == expectedFields) { + "Class harness must observe every original receiver data field: $expectedFields" + } + validateDataClasses(surface) + } + + private fun validateDataClasses(surface: CallsClassSurface) { + val duplicateExports = surface.dataClasses.map { dataClass -> dataClass.exportName }.distinct().size != + surface.dataClasses.size + require(!duplicateExports) { "Companion data-class exports must be unique" } + for (dataClass in surface.dataClasses) { + require(dataClass.exportName != universe.className) { "The target class has its own observation identity" } + val original = inventory.dataClasses.singleOrNull { original -> + original.exportName == dataClass.exportName + } + requireNotNull(original) { + "Unverified companion data class ${dataClass.exportName} in ${universe.sourceFile}" + } + require(dataClass.fields.keys == original.fields.toSet()) { + "Companion data class ${dataClass.exportName} must preserve every original own data field" + } + } + } + + companion object { + fun create(sourceRoot: Path, sourceFile: String, className: String): CallsClassSourceInstrumentation { + val root = sourceRoot.toRealPath() + val source = root.resolve(sourceFile).normalize().toRealPath() + require(source.startsWith(root)) { "Class source is outside the frozen source root" } + val sourceBytes = Files.readAllBytes(source) + val sourceSha256 = callsClassSourceSha256(sourceBytes) + val inspected = TypeScriptSourceInspector.instrumentClassCoverage(source = source, className = className) + val instrumentedSource = requireNotNull(inspected.source) { + inspected.reason ?: "Unsupported class statement instrumentation" + } + require(inspected.sourceSha256 == sourceSha256) { "Class source changed during AST inspection" } + + val universeIdentity = CallsExperimentJson.json.encodeToString( + CallsClassUniverseHashInput( + sourceFile = sourceFile, + className = className, + sourceSha256 = sourceSha256, + instrumenterVersion = inspected.instrumenterVersion, + statements = inspected.statements, + inventory = requireNotNull(inspected.inventory), + ), + ) + val universe = CallsClassSourceUniverse( + sourceFile = sourceFile, + className = className, + sourceSha256 = sourceSha256, + universeSha256 = callsClassSourceSha256(universeIdentity.toByteArray(Charsets.UTF_8)), + instrumenterVersion = inspected.instrumenterVersion, + statements = inspected.statements, + ) + + return CallsClassSourceInstrumentation( + universe = universe, + instrumentedSource = instrumentedSource, + instrumentedSha256 = requireNotNull(inspected.instrumentedSha256), + hitGlobalName = inspected.counterGlobalName, + inventory = requireNotNull(inspected.inventory), + ) + } + } +} + +@Serializable +private data class CallsClassUniverseHashInput( + val sourceFile: String, + val className: String, + val sourceSha256: String, + val instrumenterVersion: String, + val statements: List, + val inventory: TypeScriptClassSurfaceInventory, +) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceUniverse.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceUniverse.kt new file mode 100644 index 0000000000..f920110c4a --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassSourceUniverse.kt @@ -0,0 +1,57 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import org.usvm.ts.pbt.fastcheck.TypeScriptClassStatementDescriptor +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** Frozen original-source executable statements, independent of policy, seed and replay hits. */ +@Serializable +internal data class CallsClassSourceUniverse( + val sourceFile: String, + val className: String, + val sourceSha256: String, + val universeSha256: String, + val instrumenterVersion: String, + val statements: List, +) { + init { + require(sourceFile.isNotBlank() && className.isNotBlank()) + require(statements.isNotEmpty()) { "Class coverage universe has no executable statements" } + require(statements.map { statement -> statement.id }.toSet().size == statements.size) { + "Class statement IDs must be unique" + } + val hasExactOriginalRanges = statements.all { statement -> + statement.startOffset >= 0 && statement.endOffset > statement.startOffset && + statement.id.startsWith("$sourceSha256:") + } + require(hasExactOriginalRanges) { "Class statements must identify exact original-source AST ranges" } + } + + val statementIds: Set + get() = statements.mapTo(linkedSetOf()) { statement -> statement.id } + + fun verifyUnchanged(sourceRoot: Path) { + val root = sourceRoot.toRealPath() + val source = root.resolve(sourceFile).normalize().toRealPath() + require(source.startsWith(root)) { "Class source is outside the frozen source root" } + require(callsClassSourceSha256(Files.readAllBytes(source)) == sourceSha256) { + "Class source changed after freezing its statement universe" + } + } + + /** Counter IDs are native hits of these exact AST statements, never inferred from c8 line records. */ + fun checkedCoveredIds(nativeHits: Collection): Set { + val covered = nativeHits.toSet() + val unknownIds = covered - statementIds + require(unknownIds.isEmpty()) { "Native class coverage reported unknown statement IDs: $unknownIds" } + + return covered + } +} + +internal fun callsClassSourceSha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256") + .digest(bytes).joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and UNSIGNED_BYTE_MASK) } + +private const val UNSIGNED_BYTE_MASK = 0xff diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt index ff96c00951..80e427551e 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt @@ -124,8 +124,8 @@ internal fun runCoverageUniverse( val universe = probeFunction(source) source.verifyUnchanged() val record = CallsCoverageUniverseRecord(identity = identity, universe = universe) - atomicWrite(output.resolve("universe.json"), encode(record)) - System.out.appendLine(encode(record)) + writeCallsArtifactAtomically(output.resolve("universe.json"), encodeCallsArtifact(record)) + System.out.appendLine(encodeCallsArtifact(record)) } internal fun runCoverageCell( @@ -147,12 +147,12 @@ internal fun runCoverageCell( Files.createDirectories(output) val identityPath = output.resolve("identity.json") - val expectedIdentity = encode(identity) + val expectedIdentity = encodeCallsArtifact(identity) if (Files.exists(identityPath)) { require(Files.readString(identityPath).trim() == expectedIdentity) { "Cell identity changed" } } else { require(Files.list(output).use { entries -> entries.findAny().isEmpty }) { "Output directory is not empty" } - atomicWrite(identityPath, expectedIdentity) + writeCallsArtifactAtomically(identityPath, expectedIdentity) } require(!Files.exists(output.resolve("result.json"))) { "Cell already completed" } @@ -182,8 +182,8 @@ internal fun runCoverageCell( budgetMillis = identity.budgetMillis, ), ) - atomicWrite(output.resolve("result.json"), encode(result)) - System.out.appendLine(encode(result)) + writeCallsArtifactAtomically(output.resolve("result.json"), encodeCallsArtifact(result)) + System.out.appendLine(encodeCallsArtifact(result)) } private data class CoverageCellRequest( @@ -254,7 +254,7 @@ private fun parseCoverageCellRequest( budgetMillis = budgetMillis, solverLimitMillis = solverLimitMillis, candidateCap = candidateCap, - coverageUniverseSha256 = coverageUniverseSha256(universeBytes), + coverageUniverseSha256 = callsArtifactSha256(universeBytes), ) return CoverageCellRequest(source = source, identity = identity, output = output, universe = universe) @@ -286,13 +286,13 @@ private fun loadOrRunCoverageSearch( solverQueryLimit = identity.solverLimitMillis.milliseconds, solverType = SolverType.valueOf(identity.solver), candidateCap = identity.candidateCap, - onCandidate = { candidate -> appendDurably(candidatePath, encode(candidate)) }, + onCandidate = { candidate -> appendCallsJournalDurably(candidatePath, encodeCallsArtifact(candidate)) }, ) val result = searchFunction(request) source.verifyUnchanged() - val persistedCandidates = readLines(candidatePath) + val persistedCandidates = readCallsJournal(candidatePath) require(result.candidates == persistedCandidates) { "Persisted candidate journal differs from search result" } - atomicWrite(searchPath, encode(result)) + writeCallsArtifactAtomically(searchPath, encodeCallsArtifact(result)) result } } @@ -303,7 +303,7 @@ private fun replayCoverageCandidates( output: Path, ): List { val replayPath = output.resolve("replays.jsonl") - val events = readLines(replayPath).toMutableList() + val events = readCallsJournal(replayPath).toMutableList() require(events.map(CallsCoverageReplayEvent::index) == events.indices.toList()) { "Replay journal has missing or duplicate indices" } @@ -325,7 +325,7 @@ private fun replayCoverageCandidates( replay = replay.getOrNull(), error = replay.exceptionOrNull()?.let { error -> error.message ?: error::class.java.name }, ) - appendDurably(replayPath, encode(event)) + appendCallsJournalDurably(replayPath, encodeCallsArtifact(event)) events += event } } @@ -379,7 +379,7 @@ private fun validateCoverageManifestBuild( } } -private inline fun readLines(path: Path): List = if (Files.exists(path)) { +internal inline fun readCallsJournal(path: Path): List = if (Files.exists(path)) { Files.readAllLines(path).filter(String::isNotBlank).map { line -> CallsExperimentJson.json.decodeFromString(line) } @@ -387,9 +387,10 @@ private inline fun readLines(path: Path): List = if (Files.exists emptyList() } -private inline fun encode(value: T): String = CallsExperimentJson.json.encodeToUtf8SafeString(value) +internal inline fun encodeCallsArtifact(value: T): String = + CallsExperimentJson.json.encodeToUtf8SafeString(value) -private fun appendDurably(path: Path, line: String) { +internal fun appendCallsJournalDurably(path: Path, line: String) { val bytes = ByteBuffer.wrap((line + "\n").toByteArray(Charsets.UTF_8)) FileChannel.open( path, @@ -402,7 +403,7 @@ private fun appendDurably(path: Path, line: String) { } } -private fun atomicWrite(path: Path, contents: String) { +internal fun writeCallsArtifactAtomically(path: Path, contents: String) { val temporary = path.resolveSibling("${path.fileName}.tmp") Files.writeString(temporary, contents + "\n") FileChannel.open(temporary, StandardOpenOption.READ).use { it.force(true) } @@ -420,7 +421,7 @@ private val COVERAGE_CHECKPOINT_MILLIS = listOf( THIRD_CHECKPOINT_MILLIS, ) -private fun coverageUniverseSha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes) +internal fun callsArtifactSha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes) .joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and UNSIGNED_BYTE_MASK) } private const val COVERAGE_UNIVERSE_ARGUMENT_COUNT = 4 diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt index 969e3b2ade..0b583009c2 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt @@ -29,8 +29,27 @@ internal data class CallsCoverageCandidate( val emittedAtStep: Long, val inputs: List, val newSymbolicStatements: Int, - val completion: CallsCoverageCompletion, -) + val completion: CallsCoverageCompletion?, + val entryPointId: String? = null, + val observation: CallsClassObservation? = null, + val origin: CallsCoverageCandidateOrigin = CallsCoverageCandidateOrigin.TERMINATED, + val suspendedAtMachineStop: Boolean = false, + val suspensionReason: String? = null, +) { + init { + when (origin) { + CallsCoverageCandidateOrigin.TERMINATED -> require( + completion != null && !suspendedAtMachineStop && suspensionReason == null, + ) + CallsCoverageCandidateOrigin.SUSPENDED -> require(completion == null && observation == null) { + "Suspended candidates carry inputs only, without a symbolic completion or observation" + } + } + } +} + +@Serializable +internal enum class CallsCoverageCandidateOrigin { TERMINATED, SUSPENDED } @Serializable internal enum class CallsCoverageCompletion { @@ -68,6 +87,10 @@ internal data class CallsCoverageSearchResult( val runtimeLimited: Boolean = false, val runtimeLimitationReasons: List = emptyList(), val diagnostic: String? = null, + val suspendedStates: Int = 0, + val suspendedExtractionElapsedMillis: Long = 0, + val frontierFlushElapsedMillis: Long = 0, + val machineStopReason: String? = null, ) internal data class CallsCoverageSearchRequest( @@ -95,6 +118,7 @@ internal data class CallsCoverageSearchRequest( internal class CurrentTsCallsCoverageEngine( private val sourceEngine: CurrentTsCallsSymbolicEngine = CurrentTsCallsSymbolicEngine(), ) { + @Suppress("LongMethod") fun search(request: CallsCoverageSearchRequest): CallsCoverageSearchResult { val preparationStarted = TimeSource.Monotonic.markNow() val preparation = sourceEngine.prepareFunction(request.toFunctionPreflightRequest()) @@ -122,10 +146,14 @@ internal class CurrentTsCallsCoverageEngine( val extractionFailures = mutableListOf() val machineCallStarted = TimeSource.Monotonic.markNow() val observer = CompletedCoverageCandidateObserver( - method = preparation.method, + contexts = listOf( + CallsCoverageCandidateContext( + method = preparation.method, + inputDomains = request.function.inputs, + symbolicInputs = symbolicInputs, + ), + ), coverageStatements = scope.statements, - inputDomains = request.function.inputs, - symbolicInputs = symbolicInputs, candidateCap = request.candidateCap, budgetMillis = request.budget.inWholeMilliseconds, machineCallStarted = machineCallStarted, @@ -167,6 +195,7 @@ internal class CurrentTsCallsCoverageEngine( ) } } + observer.finishTiming() val machineCallElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds val machineTeardownElapsedMillis = ( machineCallElapsedMillis - observer.machineSetupElapsedMillis - observer.searchElapsedMillis @@ -256,20 +285,35 @@ private fun emptyCoverageResult( ) } -private class CompletedCoverageCandidateObserver( - private val method: EtsMethod, +internal data class CallsCoverageCandidateContext( + val method: EtsMethod, + val inputDomains: List, + val symbolicInputs: CallsSymbolicInputs, + val entryPointId: String? = null, + val observationFields: Map? = null, +) + +/** + * Shared snapshot cap and input deduplication per origin and entry point. + * Suspended snapshots require neither results nor novelty. + */ +internal class CompletedCoverageCandidateObserver( + contexts: List, private val coverageStatements: Set, - private val inputDomains: List, - private val symbolicInputs: CallsSymbolicInputs, private val candidateCap: Int, private val budgetMillis: Long, private val machineCallStarted: TimeSource.Monotonic.ValueTimeMark, private val onCandidate: (CallsCoverageCandidate) -> Unit, private val onExtractionFailure: (String) -> Unit, + private val requireEntryPointWitness: Boolean = false, + private val emitSuspendedCandidates: Boolean = false, ) : UMachineObserver { + private val contextsByMethod = contexts.associateBy(CallsCoverageCandidateContext::method) private val coveredStatements = hashSetOf() private val inputKeys = hashSetOf() + private val witnessedMethods = hashSetOf() private var machineStarted: TimeSource.Monotonic.ValueTimeMark? = null + private var machineStopped = false var selectedStates: Int = 0 private set @@ -289,6 +333,18 @@ private class CompletedCoverageCandidateObserver( var searchElapsedMillis: Long = 0 private set + var suspendedStates: Int = 0 + private set + + private var suspendedExtractionElapsedNanos: Long = 0 + private var frontierFlushElapsedNanos: Long = 0 + + val suspendedExtractionElapsedMillis: Long + get() = suspendedExtractionElapsedNanos / NANOS_PER_MILLISECOND + + val frontierFlushElapsedMillis: Long + get() = frontierFlushElapsedNanos / NANOS_PER_MILLISECOND + override fun onMachineStarted() { machineSetupElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds machineStarted = TimeSource.Monotonic.markNow() @@ -296,6 +352,16 @@ private class CompletedCoverageCandidateObserver( override fun onMachineStopped() { searchElapsedMillis = elapsedSinceMachineStart() + machineStopped = true + } + + fun finishTiming() { + if (machineStarted == null) { + machineSetupElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds + } else if (!machineStopped) { + searchElapsedMillis = elapsedSinceMachineStart() + machineStopped = true + } } override fun onState(parent: TsState, forks: Sequence) { @@ -308,6 +374,8 @@ private class CompletedCoverageCandidateObserver( override fun onStateTerminated(state: TsState, stateReachable: Boolean) { if (!stateReachable) return + val context = contextsByMethod[state.entrypoint] ?: return + val method = context.method val completion = when (val result = state.methodResult) { is TsMethodResult.Success.RegularCall -> { if (result.method != method) return @@ -324,20 +392,73 @@ private class CompletedCoverageCandidateObserver( statement in coverageStatements } val newStatements = pathStatements.count { statement -> statement !in coveredStatements } - if (newStatements == 0) return + val needsWitness = requireEntryPointWitness && method !in witnessedMethods + if (newStatements == 0 && !needsWitness) return + + emitCandidate( + state = state, + context = context, + pathStatements = pathStatements, + newStatements = newStatements, + completion = completion, + origin = CallsCoverageCandidateOrigin.TERMINATED, + ) + } + + fun collectStoppedState(state: TsState) { + if (!emitSuspendedCandidates) return + val started = TimeSource.Monotonic.markNow() + suspendedStates++ + try { + val context = contextsByMethod[state.entrypoint] ?: return + emitCandidate( + state = state, + context = context, + pathStatements = emptySet(), + newStatements = 0, + completion = null, + origin = CallsCoverageCandidateOrigin.SUSPENDED, + ) + } finally { + val elapsedNanos = started.elapsedNow().inWholeNanoseconds + suspendedExtractionElapsedNanos += elapsedNanos + if (machineStopped) frontierFlushElapsedNanos += elapsedNanos + } + } + + private fun emitCandidate( + state: TsState, + context: CallsCoverageCandidateContext, + pathStatements: Set, + newStatements: Int, + completion: CallsCoverageCompletion?, + origin: CallsCoverageCandidateOrigin, + ) { if (selectedStates >= candidateCap) { capReached = true return } selectedStates++ val candidate = runCatching { - val inputs = symbolicInputs.resolve(state).firstOrNull { candidate -> + val inputDomains = context.inputDomains + val symbolicInputs = context.symbolicInputs + val extractedInputs = symbolicInputs.resolve(state) + val canResolveAnyModel = context.observationFields == null || + origin == CallsCoverageCandidateOrigin.SUSPENDED + val eligibleInputs = if (canResolveAnyModel) { + extractedInputs + } else { + // Output/state extraction uses the first model too; do not pair it with another model's inputs. + extractedInputs.take(1) + } + val inputs = eligibleInputs.firstOrNull { candidate -> candidate.size == inputDomains.size && candidate.zip(inputDomains).all { (value, input) -> value in input.domain } } requireNotNull(inputs) { "No extracted input is inside every frozen domain" } - val key = CallsExperimentJson.json.encodeToUtf8SafeString(inputs) + val key = origin.name + ":" + context.entryPointId.orEmpty() + + CallsExperimentJson.json.encodeToUtf8SafeString(inputs) if (inputKeys.add(key)) { CallsCoverageCandidate( emittedAtMillis = elapsedSinceMachineStart(), @@ -345,6 +466,17 @@ private class CompletedCoverageCandidateObserver( inputs = inputs, newSymbolicStatements = newStatements, completion = completion, + entryPointId = context.entryPointId, + observation = context.observationFields?.takeIf { + origin == CallsCoverageCandidateOrigin.TERMINATED + }?.let { fields -> + extractCallsClassObservation(state = state, method = context.method, fields = fields) + }, + origin = origin, + suspendedAtMachineStop = origin == CallsCoverageCandidateOrigin.SUSPENDED && machineStopped, + suspensionReason = state.unsupportedReason.takeIf { + origin == CallsCoverageCandidateOrigin.SUSPENDED + }, ) } else { null @@ -356,7 +488,10 @@ private class CompletedCoverageCandidateObserver( if (candidate != null) { onCandidate(candidate) - coveredStatements += pathStatements + if (origin == CallsCoverageCandidateOrigin.TERMINATED) { + coveredStatements += pathStatements + witnessedMethods += context.method + } } } @@ -371,3 +506,5 @@ private fun CallsSymbolicStatus.toCoverageStatus(): CallsCoverageSearchStatus = CallsSymbolicStatus.AMBIGUOUS -> CallsCoverageSearchStatus.AMBIGUOUS else -> CallsCoverageSearchStatus.TOOL_ERROR } + +private const val NANOS_PER_MILLISECOND = 1_000_000L diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt index 9588cf70ef..587aa2237f 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt @@ -9,6 +9,10 @@ fun main(args: Array) { require(args.isNotEmpty()) { usage() } when (args.first()) { + "class-catalog" -> runClassCatalog(args.drop(1)) + "class-prepare" -> runClassPrepare(args.drop(1)) + "class-preflight" -> runClassPreflight(args.drop(1)) + "class-cell" -> runClassCell(args.drop(1)) "run" -> runExperiment(args.drop(1)) "coverage-universe" -> runCoverageUniverse(args.drop(1)) "coverage-cell" -> runCoverageCell(args.drop(1)) @@ -89,6 +93,10 @@ private fun summarize(args: List) { private fun usage(): String = """ Usage: + calls class-catalog + calls class-prepare + calls class-preflight + calls class-cell calls run calls coverage-universe calls coverage-cell diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt index a4f314ff79..fd3c927b18 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSymbolicInputs.kt @@ -52,8 +52,11 @@ internal class CallsSymbolicInputs( private val lexicalEnvironment: EtsLexicalEnvironmentBinding?, private val captures: List, sourceScene: EtsScene, + private val objectShapes: PlainObjectInputShapes = PlainObjectInputShapes( + sourceScene = sourceScene, + domains = inputs.map(PropertyInput::domain), + ), ) { - private val objectShapes = PlainObjectInputShapes(sourceScene, inputs.map(PropertyInput::domain)) val scene: EtsScene = objectShapes.scene private val boundInputs: List> diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt index 9af0ae2733..c2015aaf42 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsUnknownCallTelemetry.kt @@ -138,7 +138,7 @@ internal fun CallsExperimentCellIdentity.unknownCallRecord( ) } -private fun EtsStmt.toCallsUnknownCallSite(): CallsUnknownCallSite { +internal fun EtsStmt.toCallsUnknownCallSite(): CallsUnknownCallSite { val stmtLocation = location val origin = stmtLocation.origin val containingMethod = stmtLocation.method.signature @@ -165,7 +165,7 @@ internal fun callsUnknownCallEventSink( } } -private fun EtsMethodSignature.toCallsUnknownCallCallee(): CallsUnknownCallCallee { +internal fun EtsMethodSignature.toCallsUnknownCallCallee(): CallsUnknownCallCallee { val parameterTypes = parameters.map { parameter -> parameter.type.toString() } val namespace = enclosingClass.namespace?.qualifiedName() val className = listOfNotNull(namespace, enclosingClass.name).joinToString(separator = "::") diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt index 3e4ce47438..1e1375a164 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt @@ -48,6 +48,7 @@ internal data class CallsFunctionPreflightRequest( val project: CallsProjectCase, val function: CallsFunctionCase, val expectedNativeFrontendRevision: String, + val verifySourceCheckout: Boolean = true, ) internal sealed interface CallsFunctionPreparation { @@ -109,6 +110,7 @@ internal class CurrentTsCallsSymbolicEngine( private val bundledNativeFrontendRevision: String = CallsBuildIdentity.nativeFrontendRevision, ) : CallsSymbolicEngine { private val verifiedProjects = mutableMapOf() + private val preparedSources = mutableMapOf, CallsSourceProject>() private var preparedTarget: Pair? = null private var verifiedNativeFrontendIdentity: String? = null @@ -311,17 +313,25 @@ internal class CurrentTsCallsSymbolicEngine( ) } - verifyGitCheckoutOnce( - checkout = request.sourceRoot, - expectedRevision = request.project.revision, - cache = verifiedProjects, - ) + if (request.verifySourceCheckout) { + verifyGitCheckoutOnce( + checkout = request.sourceRoot, + expectedRevision = request.project.revision, + cache = verifiedProjects, + ) + } verifyNativeFrontendOnce(expectedRevision = request.expectedNativeFrontendRevision) val source = request.sourceRoot.resolve(request.function.sourceFile).normalize() require(source.startsWith(request.sourceRoot)) { "Function source escapes its frozen source root" } - val project = loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source) + val project = if (request.verifySourceCheckout) { + loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source) + } else { + preparedSources.getOrPut(request.sourceRoot to source) { + loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source) + } + } val loaded = when (project) { is CallsSourceProject.Loaded -> project is CallsSourceProject.Unsupported -> return CallsFunctionPreparation.Rejected( diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassAdmissionTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassAdmissionTest.kt new file mode 100644 index 0000000000..e71f012d1e --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassAdmissionTest.kt @@ -0,0 +1,403 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.io.TempDir +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.StringDomain +import org.usvm.ts.pbt.model.contains +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +class CallsClassAdmissionTest { + @TempDir + lateinit var directory: Path + + @Test + fun `class admission does not require a symbolic pilot to complete any hashed method scenario`() { + val harness = fixture() + val instrumentation = instrumentation() + val sourceFailures = callsClassSourceAdmissionFailures(harness, CallsBuildIdentity.nativeFrontendRevision) + + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation, seed = 17, timeoutMillis = 20_000) + val repeated = callsClassNativeConstructorSmoke(harness, instrumentation, seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures, smoke) + + assertEquals(emptyList(), sourceFailures) + assertEquals(CallsClassNativeSmokeStatus.VERIFIED, smoke.status) + val original = assertNotNull(smoke.original) + assertEquals(original.inputs, assertNotNull(repeated.original).inputs) + assertEquals(original.observation, repeated.original.observation) + assertTrue(original.inputs.single() in harness.surface.constructorInputs.single().domain) + assertTrue(result.eligible) + assertEquals(true, result.primaryInferenceEligible) + assertEquals(false, result.diagnosticReady) + assertEquals(CallsCoverageSearchStatus.TIMEOUT, result.search.search.status) + assertEquals(harness.scenarios.map { it.id }, result.missingRequiredScenarios) + assertEquals(harness.scenarios.size, result.missingRequiredScenarios.size) + } + + @Test + fun `native sampler rejects seed overflow instead of truncating the frozen seed`() { + val harness = fixture() + + OriginalTypeScriptClassReplayer( + directory, + harness.functions.first(), + harness.observations.first(), + ).use { replayer -> + assertFailsWith { + replayer.sample(seed = Int.MAX_VALUE.toLong() + 1, timeoutMillis = 20_000) + } + assertFailsWith { + replayer.sample(seed = Int.MIN_VALUE.toLong() - 1, timeoutMillis = 20_000) + } + } + } + + @Test + fun `pilot native corruption remains an acquisition failure without selecting the class population`() { + val harness = fixture() + val originalInstrumentation = instrumentation() + val modified = originalInstrumentation.instrumentedSource.replace( + "return key.length === this.size;", + "return false;", + ) + val changed = originalInstrumentation.copy( + instrumentedSource = modified, + instrumentedSha256 = callsClassSourceSha256(modified.toByteArray(Charsets.UTF_8)), + ) + val smoke = callsClassNativeConstructorSmoke(harness, changed, seed = 17, timeoutMillis = 20_000) + val inputs = listOf(JsConcreteValue.String("aa"), JsConcreteValue.String("aa")) + + val original = OriginalTypeScriptClassReplayer(directory, harness.functions[1], harness.observations[1]).use { + it.replay(inputs = inputs, timeoutMillis = 20_000) + } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + sourceOverride = changed.instrumentedSource, + overrideSourceFile = "SlowMap.ts", + counterGlobalName = changed.hitGlobalName, + ).use { it.replay(inputs = inputs, timeoutMillis = 20_000) } + val replay = CallsClassReplayEvent( + index = 0, + scenarioId = harness.scenarios[1].id, + original = original, + instrumented = instrumented, + oracle = compareCallsClassObservations( + original.observation, + original.observation, + instrumented.observation, + ), + nativeCoverage = callsClassNativeCoverageAcceptance(original, instrumented, changed.universe), + ) + val failures = callsClassPilotNativeFailures(listOf(replay), changed.universe) + val baseline = admission(harness, sourceFailures = emptyList(), smoke = smoke) + val result = callsClassPreflightResult( + classCase = classCase(harness), + preparedSha256 = baseline.preparedSha256, + toolRevision = baseline.toolRevision, + search = baseline.search, + replays = listOf(replay), + missingScenarios = baseline.missingRequiredScenarios, + pilotDiagnostics = emptyList(), + sourceAdmissionFailures = emptyList(), + nativeSmoke = smoke, + nativeInfrastructureFailures = failures, + ) + + assertEquals(CallsClassNativeSmokeStatus.VERIFIED, smoke.status) + assertTrue(failures.isNotEmpty(), replay.toString()) + assertFalse(replay.oracle.accepted) + assertTrue(result.eligible) + assertEquals(true, result.primaryInferenceEligible) + assertEquals(false, result.diagnosticReady) + assertEquals(failures, result.nativeInfrastructureFailures) + } + + @Test + fun `changed native constructor state rejects the observation mechanism independently of pilot coverage`() { + val harness = fixture() + val originalInstrumentation = instrumentation() + val modified = originalInstrumentation.instrumentedSource.replace( + "this.size = initial.length;", + "this.size = initial.length + 1;", + ) + val changed = originalInstrumentation.copy( + instrumentedSource = modified, + instrumentedSha256 = callsClassSourceSha256(modified.toByteArray(Charsets.UTF_8)), + ) + + val smoke = callsClassNativeConstructorSmoke(harness, changed, seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke) + + assertEquals(CallsClassNativeSmokeStatus.INVALID, smoke.status) + assertFalse(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + val changedState = smoke.diagnostic?.contains( + "Instrumentation changed native returned values or receiver state", + ) == true + assertTrue(changedState) + } + + @Test + fun `instrumentation changing native observability is invalid even with unchanged frozen inputs`() { + val harness = fixture() + val originalInstrumentation = instrumentation() + val modified = originalInstrumentation.instrumentedSource.replace( + "this.size = initial.length;", + "this.size = 'outside-the-number-schema' as any;", + ) + val changed = originalInstrumentation.copy( + instrumentedSource = modified, + instrumentedSha256 = callsClassSourceSha256(modified.toByteArray(Charsets.UTF_8)), + ) + + val smoke = callsClassNativeConstructorSmoke(harness, changed, seed = 17, timeoutMillis = 20_000) + val original = assertNotNull(smoke.original) + val instrumented = assertNotNull(smoke.instrumented) + val sameCompletionIncomplete = instrumented.copy( + observation = instrumented.observation.copy(completion = original.observation.completion), + ) + + assertEquals(original.inputs, instrumented.inputs) + assertTrue(original.observation.representable) + assertFalse(instrumented.observation.representable) + assertEquals(CallsClassNativeSmokeStatus.INVALID, smoke.status) + assertEquals( + "Instrumentation changed native observability or observation schema", + callsClassNativeInfrastructureFailure(original, sameCompletionIncomplete), + ) + } + + @Test + fun `lawful incomplete native constructor observation remains inconclusive without silently excluding raw cells`() { + val harness = fixture(source = source.replace("this.size = initial.length;", "throw 7;")) + + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke) + + assertEquals(CallsClassNativeSmokeStatus.INCONCLUSIVE, smoke.status) + assertTrue(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + assertNotNull(smoke.diagnostic) + } + + @Test + fun `source closure preparation failure is a hard admission gate independent of pilot outcomes`() { + val harness = fixture(source = "const cache: number[][] = [];\n" + source) + val failures = callsClassSourceAdmissionFailures(harness, CallsBuildIdentity.nativeFrontendRevision) + + val result = admission(harness, sourceFailures = failures, smoke = null) + + assertTrue(failures.any { it.contains("TOP_LEVEL_INITIALIZATION_UNSUPPORTED") }, failures.toString()) + assertFalse(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + assertEquals(failures, result.sourceAdmissionFailures) + } + + @Test + fun `symbolic fresh-result disagreement remains a pilot outcome with strict candidate rejection`() { + val harness = fixture() + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val original = assertNotNull(smoke.original) + val instrumented = assertNotNull(smoke.instrumented) + val symbolic = original.observation.copy(values = mapOf("state_0_size" to JsConcreteValue.number(-1.0))) + val oracle = compareCallsClassObservations(symbolic, original.observation, instrumented.observation) + val baseline = admission(harness, sourceFailures = emptyList(), smoke = smoke) + val replay = CallsClassReplayEvent( + index = 0, + scenarioId = smoke.scenarioId, + original = original, + instrumented = instrumented, + oracle = oracle, + nativeCoverage = callsClassNativeCoverageAcceptance(original, instrumented, instrumentation().universe), + ) + val candidate = CallsCoverageCandidate( + emittedAtMillis = 1, + emittedAtStep = 1, + inputs = original.inputs, + newSymbolicStatements = 1, + completion = CallsCoverageCompletion.RETURNED, + entryPointId = smoke.scenarioId, + observation = symbolic, + ) + val pilot = baseline.search.copy( + search = baseline.search.search.copy( + candidates = listOf(candidate), + selectedStates = 1, + ), + ) + + val result = callsClassPreflightResult( + classCase = classCase(harness), + preparedSha256 = baseline.preparedSha256, + toolRevision = baseline.toolRevision, + search = pilot, + replays = listOf(replay), + missingScenarios = baseline.missingRequiredScenarios, + pilotDiagnostics = listOf("Symbolic and original returned values or receiver state disagree"), + sourceAdmissionFailures = emptyList(), + nativeSmoke = smoke, + nativeInfrastructureFailures = listOfNotNull(callsClassNativeInfrastructureFailure(original, instrumented)), + ) + + assertFalse(oracle.accepted) + assertTrue(replay.nativeCoverage.accepted) + assertTrue(result.eligible) + assertEquals(true, result.primaryInferenceEligible) + assertEquals(false, result.diagnosticReady) + assertEquals(listOf(replay), result.replays) + } + + @Test + fun `documented frozen correctness frontier excludes primary inference while preserving raw diagnostics`() { + val harness = fixture() + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val failure = CallsClassCorrectnessFrontierFailure( + reason = "Known constructor lowering defect", + provenance = "Pinned frontend regression and independently replayed counterexample", + ) + val classCase = classCase(harness).copy(correctnessFrontierFailures = listOf(failure)) + + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke, selectedCase = classCase) + + assertTrue(result.eligible) + assertEquals(false, result.primaryInferenceEligible) + assertEquals(listOf(failure), result.correctnessFrontierFailures) + } + + @Test + fun `new admission serializes explicitly and legacy strict preflight remains version zero`() { + val harness = fixture() + val smoke = callsClassNativeConstructorSmoke(harness, instrumentation(), seed = 17, timeoutMillis = 20_000) + val result = admission(harness, sourceFailures = emptyList(), smoke = smoke) + + val encoded = CallsExperimentJson.json.encodeToString(result) + val decoded = CallsExperimentJson.json.decodeFromString(encoded) + val addedFields = setOf( + "admissionVersion", + "sourceAdmissionFailures", + "nativeInfrastructureFailures", + "nativeSmoke", + "diagnosticReady", + "primaryInferenceEligible", + "correctnessFrontierFailures", + ) + val legacy = JsonObject( + CallsExperimentJson.json.parseToJsonElement(encoded).jsonObject.filterKeys { + it !in addedFields + }, + ) + val decodedLegacy = CallsExperimentJson.json.decodeFromString(legacy.toString()) + + assertEquals(result, decoded) + assertEquals(1, decoded.admissionVersion) + assertEquals(0, decodedLegacy.admissionVersion) + assertEquals(null, decodedLegacy.primaryInferenceEligible) + assertEquals(null, decodedLegacy.nativeSmoke) + } + + private fun fixture(source: String = this.source): CallsClassPreparedHarness { + Files.writeString(directory.resolve("SlowMap.ts"), source) + val surface = CallsClassSurface( + sourceFile = "SlowMap.ts", + exportName = "SlowMap", + constructorInputs = listOf( + PropertyInput(name = "initial", domain = StringDomain(minLength = 2, maxLength = 6)), + ), + members = listOf( + CallsClassMember( + id = "has", + name = "has", + kind = CallsClassMemberKind.METHOD, + inputs = listOf(PropertyInput(name = "key", domain = StringDomain(minLength = 2, maxLength = 6))), + resultDomain = BooleanDomain, + ), + ), + stateFields = mapOf("size" to NumberDomain()), + ) + val project = CallsProjectCase( + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = true, + functions = emptyList(), + ) + + return prepareCallsClassHarness(directory, project, surface, maxSequenceLength = 2) + } + + private fun instrumentation() = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "SlowMap.ts", + className = "SlowMap", + ) + + private fun classCase(harness: CallsClassPreparedHarness) = CallsClassCase( + classId = "slow-map", + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = directory.toString(), + sourceSha256 = mapOf( + "SlowMap.ts" to callsClassSourceSha256(Files.readAllBytes(directory.resolve("SlowMap.ts"))), + ), + surface = harness.surface, + ) + + private fun admission( + harness: CallsClassPreparedHarness, + sourceFailures: List, + smoke: CallsClassNativeSmokeResult?, + selectedCase: CallsClassCase = classCase(harness), + ) = callsClassPreflightResult( + classCase = selectedCase, + preparedSha256 = "a".repeat(64), + toolRevision = "b".repeat(40), + search = CallsClassSearchResult( + search = CallsCoverageSearchResult( + status = CallsCoverageSearchStatus.TIMEOUT, + candidates = emptyList(), + selectedStates = 0, + executedSteps = 10, + stepsWithinBudget = 10, + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = 0, + machineSetupElapsedMillis = 0, + searchElapsedMillis = 120_000, + machineTeardownElapsedMillis = 0, + ), + scenarios = emptyList(), + visits = emptyList(), + unknownCallEvents = 0, + ), + replays = emptyList(), + missingScenarios = harness.scenarios.map { it.id }, + pilotDiagnostics = listOf("Symbolic pilot reached the time budget", "No scenario witnesses"), + sourceAdmissionFailures = sourceFailures, + nativeSmoke = smoke, + nativeInfrastructureFailures = emptyList(), + ) + + private val source = """ + export class SlowMap { + size: number; + constructor(initial: string) { this.size = initial.length; } + has(key: string): boolean { return key.length === this.size; } + } + """.trimIndent() +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassCoverageSourceTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassCoverageSourceTest.kt new file mode 100644 index 0000000000..5b234a7af2 --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassCoverageSourceTest.kt @@ -0,0 +1,171 @@ +package org.usvm.ts.calls + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.PropertyInput +import java.nio.file.Files +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +internal class CallsClassCoverageSourceTest { + @Test + fun `class universe freezes exact statements and checks native IDs`() { + val sourceRoot = Files.createTempDirectory("calls-class-universe-") + try { + Files.writeString( + sourceRoot.resolve("Target.ts"), + "export class Target { run(x: boolean) { if (x) return 1; return 2; } }", + ) + + val instrumented = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + val universe = instrumented.universe + val surface = CallsClassSurface( + sourceFile = "Target.ts", + exportName = "Target", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + id = "run", + name = "run", + kind = CallsClassMemberKind.METHOD, + inputs = listOf(PropertyInput(name = "x", domain = BooleanDomain)), + resultDomain = IntegerDomain(min = 1, max = 2), + ), + ), + stateFields = emptyMap(), + ) + instrumented.validateSurface(surface = surface) + assertFailsWith { + instrumented.validateSurface(surface = surface.copy(members = emptyList())) + } + assertFailsWith { + instrumented.validateSurface(surface = surface.copy(stateFields = mapOf("foreign" to BooleanDomain))) + } + universe.verifyUnchanged(sourceRoot = sourceRoot) + + assertEquals(expected = 3, actual = universe.statements.size) + assertEquals(expected = 3, actual = universe.statementIds.size) + assertTrue(universe.statements.all { statement -> statement.startLine == 1 }) + assertNotEquals(illegal = universe.sourceSha256, actual = instrumented.instrumentedSha256) + val firstId = universe.statements.first().id + assertEquals(expected = setOf(firstId), actual = universe.checkedCoveredIds(listOf(firstId, firstId))) + assertFailsWith { universe.checkedCoveredIds(listOf("foreign-id")) } + + val repeated = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + assertEquals(expected = instrumented, actual = repeated) + } finally { + Files.walk(sourceRoot).use { files -> + files.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + } + + @Test + fun `companion registry preserves exact same-source data class fields`() { + val sourceRoot = Files.createTempDirectory("calls-class-data-registry-") + try { + Files.writeString( + sourceRoot.resolve("Target.ts"), + """ + export class Target { run() { return 1; } } + export class MapEntry { + key: K + value: V + + constructor(key: K, value: V) { + this.key = key + this.value = value + } + } + export class CallableCompanion { value = 1; run() { return 1; } } + """.trimIndent(), + ) + val companion = CallsClassDataClass( + id = "entry", + exportName = "MapEntry", + fields = mapOf("key" to IntegerDomain(min = 0, max = 1), "value" to BooleanDomain), + ) + val surface = CallsClassSurface( + sourceFile = "Target.ts", + exportName = "Target", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + id = "run", + name = "run", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = IntegerDomain(min = 1, max = 1), + ), + ), + stateFields = emptyMap(), + dataClasses = listOf(companion), + ) + val instrumented = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + + instrumented.validateSurface(surface = surface) + val missingField = companion.copy(fields = companion.fields - "value") + val extraField = companion.copy(fields = companion.fields + ("foreign" to BooleanDomain)) + val unknownClass = companion.copy(exportName = "Unknown") + val callableClass = companion.copy( + exportName = "CallableCompanion", + fields = mapOf("value" to BooleanDomain), + ) + for (invalid in listOf(missingField, extraField, unknownClass, callableClass)) { + assertFailsWith { + instrumented.validateSurface(surface = surface.copy(dataClasses = listOf(invalid))) + } + } + } finally { + Files.walk(sourceRoot).use { files -> + files.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + } + + @Test + fun `class source drift and unsupported reachability remain explicit failures`() { + val sourceRoot = Files.createTempDirectory("calls-class-source-drift-") + try { + val source = sourceRoot.resolve("Target.ts") + Files.writeString(source, "export class Target { run() { return 1; } }") + val universe = CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ).universe + + Files.writeString(source, "export class Target { run() { return 2; } }") + assertFailsWith { universe.verifyUnchanged(sourceRoot = sourceRoot) } + + Files.writeString(source, "export class Target { run(key: string) { return this[key]; } }") + val failure = assertFailsWith { + CallsClassSourceInstrumentation.create( + sourceRoot = sourceRoot, + sourceFile = "Target.ts", + className = "Target", + ) + } + assertTrue(failure.message.orEmpty().contains("Computed receiver")) + } finally { + Files.walk(sourceRoot).use { files -> + files.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + } +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassDataRegistryOracleTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassDataRegistryOracleTest.kt new file mode 100644 index 0000000000..f575a1071f --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassDataRegistryOracleTest.kt @@ -0,0 +1,388 @@ +package org.usvm.ts.calls + +import org.junit.jupiter.api.io.TempDir +import org.usvm.SolverType +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.PropertyDomain +import org.usvm.ts.pbt.model.PropertyInput +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class CallsClassDataRegistryOracleTest { + @TempDir + lateinit var directory: Path + + @Test + fun `registered nested data class preserves nominal identity against a plain object`() { + val harness = dataClassFixture() + val function = harness.functions[1] + val observations = harness.observations[1] + val original = OriginalTypeScriptClassReplayer(directory, function, observations).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + val plainSource = dataClassSource.replace("new Entry(1)", "{ value: 1 }") + val plain = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = function, + observationFields = observations, + sourceOverride = plainSource, + overrideSourceFile = "Entries.ts", + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + val entry = JsConcreteValue.Object( + fields = mapOf("value" to number(1), CLASS_IDENTITY_FIELD to JsConcreteValue.String("entry")), + ) + val plainEntry = entry.copy(fields = entry.fields + (CLASS_IDENTITY_FIELD to JsConcreteValue.String("plain"))) + assertTrue(original.observation.representable, original.toString()) + assertTrue(plain.observation.representable, plain.toString()) + assertEquals(JsConcreteValue.Array(listOf(entry)), original.observation.values["result_0"]) + assertEquals(JsConcreteValue.Array(listOf(plainEntry)), plain.observation.values["result_0"]) + assertEquals(JsConcreteValue.String("target"), original.observation.values["state_1_$CLASS_IDENTITY_FIELD"]) + assertFalse( + compareCallsClassObservations(original.observation, original.observation, plain.observation).accepted + ) + + val self = OriginalTypeScriptClassReplayer(directory, harness.functions[2], harness.observations[2]).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + val selfResult = self.observation.values["result_0"] as JsConcreteValue.Object + assertTrue(self.observation.representable, self.toString()) + assertEquals(JsConcreteValue.String("target"), selfResult.fields[CLASS_IDENTITY_FIELD]) + assertEquals(JsConcreteValue.Array(listOf(entry)), selfResult.fields["items"]) + } + + @Test + fun `registered nested data class rejects unlisted prototypes and changed own fields`() { + val harness = dataClassFixture() + val replacements = listOf( + "new OtherEntry(1)" to "Unlisted object prototype", + "new ChildEntry(1)" to "Unlisted object prototype", + "Object.assign(new Entry(1), { extra: 2 })" to "Registered data-class own fields changed", + "Object.defineProperty(new Entry(1), 'value', { get: () => 1 })" to + "Registered data-class field is not an ordinary data property", + "new Date(1)" to "Unlisted object prototype", + ) + for ((replacement, expectedDiagnostic) in replacements) { + val changed = dataClassSource.replace("new Entry(1)", replacement) + """ + + class OtherEntry { value: number; constructor(value: number) { this.value = value; } } + class ChildEntry extends Entry {} + """.trimIndent() + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + sourceOverride = changed, + overrideSourceFile = "Entries.ts", + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertFalse(replay.observation.representable, replay.toString()) + assertTrue(replay.observation.diagnostic?.contains(expectedDiagnostic) == true, replay.toString()) + } + } + + @Test + fun `registered data class rejects constructor and prototype mutation`() { + val harness = dataClassFixture() + val mutations = listOf( + "(Entry as any).hidden = 2;", + "(Entry.prototype as any).hidden = 2;", + "Object.defineProperty(Entry.prototype, 'value', { get: () => 1 });", + "Object.setPrototypeOf(Entry.prototype, {});", + "Object.defineProperty(Entry, 'name', { get: () => 'Entry' });", + "Object.defineProperty(Entry, 'name', { value: 'Changed' });", + "Object.defineProperty(Entry, 'length', { value: 99 });", + ) + for (mutation in mutations) { + val changed = dataClassSource.replace("read(): Entry[] {", "read(): Entry[] { $mutation") + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + sourceOverride = changed, + overrideSourceFile = "Entries.ts", + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertFalse(replay.observation.representable, replay.toString()) + val changedContract = replay.observation.diagnostic?.contains( + "Registered data-class constructor or prototype changed" + ) == true + assertTrue(changedContract, replay.toString()) + } + } + + @Test + fun `registered class fields require the exact frozen observation domain`() { + val harness = dataClassFixture(observationFieldDomain = IntegerDomain(min = 0, max = 2)) + + val replay = OriginalTypeScriptClassReplayer(directory, harness.functions[1], harness.observations[1]).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + + assertFalse(replay.observation.representable, replay.toString()) + assertTrue( + replay.observation.diagnostic?.contains("Registered data-class observation domain changed") == true, + replay.toString(), + ) + } + + @Test + fun `registry identity field collisions reject before execution`() { + val entryFields = mapOf("value" to NumberDomain()) + val registry = listOf(CallsClassDataClass(id = "entry", exportName = "Entry", fields = entryFields)) + val collision = ObjectDomain(fields = mapOf(CLASS_IDENTITY_FIELD to NumberDomain())) + + assertFailsWith { + surface().copy(stateFields = mapOf("nested" to ArrayDomain(element = collision)), dataClasses = registry) + } + assertFailsWith { + surface().copy(dataClasses = registry + registry) + } + assertFailsWith { + surface().copy(dataClasses = listOf(registry.single().copy(fields = collision.fields))) + } + assertFailsWith { + surface().copy(dataClasses = listOf(registry.single().copy(id = "plain"))) + } + } + + @Test + fun `symbolic nested registered instances and target identity agree with native observations`() { + val harness = dataClassFixture() + val result = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = CallsExperimentProfile.EMPTY_STOP, + frozenModelIds = emptySet(), + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 3.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 10, + requireEntryPointWitness = true, + ), + ) + + assertEquals(harness.scenarios.map { it.id }.toSet(), result.search.candidates.map { it.entryPointId }.toSet()) + for (candidate in result.search.candidates) { + val index = harness.scenarios.indexOfFirst { it.id == candidate.entryPointId } + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index], + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + sourceOverride = dataClassSource.replace( + "return this.items;", + "; (globalThis as any).entryHits?.add('read'); return this.items;", + ), + overrideSourceFile = "Entries.ts", + counterGlobalName = "entryHits", + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + + val checked = compareCallsClassObservations( + candidate.observation, + original.observation, + instrumented.observation + ) + assertTrue(checked.accepted, "$candidate / $original / $instrumented / $checked") + } + } + + @Test + fun `boxed conditional array elements retain registered or plain identity on both branches`() { + val conditionalSource = dataClassSource + .replace("items: Entry[];", "items: any[];") + .replace( + "constructor() { this.items = [new Entry(1)]; }", + "constructor(flag: boolean) { this.items = [flag ? new Entry(1) : { value: 1 }]; }", + ) + val harness = dataClassFixture( + fixtureSource = conditionalSource, + constructorInputs = listOf(PropertyInput(name = "flag", domain = BooleanDomain)), + ) + + val result = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = CallsExperimentProfile.EMPTY_STOP, + frozenModelIds = emptySet(), + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 3.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 20, + requireEntryPointWitness = true, + ), + ) + + val constructorScenarioId = harness.scenarios.first().id + val constructorBranches = result.search.candidates.filter { it.entryPointId == constructorScenarioId } + .map { it.inputs.single() }.toSet() + assertEquals(setOf(JsConcreteValue.Boolean(true), JsConcreteValue.Boolean(false)), constructorBranches) + assertEquals(harness.scenarios.map { it.id }.toSet(), result.search.candidates.map { it.entryPointId }.toSet()) + for (candidate in result.search.candidates) { + val index = harness.scenarios.indexOfFirst { it.id == candidate.entryPointId } + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index], + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + sourceOverride = conditionalSource.replace( + "return this.items;", + "; (globalThis as any).entryHits?.add('read'); return this.items;", + ), + overrideSourceFile = "Entries.ts", + counterGlobalName = "entryHits", + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + + val checked = compareCallsClassObservations( + candidate.observation, + original.observation, + instrumented.observation + ) + assertTrue(checked.accepted, "$candidate / $original / $instrumented / $checked") + val createsRegisteredEntry = (candidate.inputs.single() as JsConcreteValue.Boolean).value + val expectedIdentity = if (createsRegisteredEntry) "entry" else "plain" + val symbolic = requireNotNull(candidate.observation) + val items = symbolic.values.getValue("state_0_items") as JsConcreteValue.Array + val observedEntry = items.elements.single() as JsConcreteValue.Object + assertEquals(JsConcreteValue.String(expectedIdentity), observedEntry.fields[CLASS_IDENTITY_FIELD]) + } + } + + @Test + fun `frozen registry domains compare fields without relying on insertion order`() { + Files.writeString( + directory.resolve("Entries.ts"), + """ + export class PairEntry { + key: number; + value: number; + constructor(key: number, value: number) { this.key = key; this.value = value; } + } + export class Entries { + item: PairEntry = new PairEntry(1, 2); + } + """.trimIndent() + ) + val registryFields = mapOf("key" to NumberDomain(), "value" to NumberDomain()) + val observationFields = registryFields.entries.reversed().associate { it.toPair() } + val surface = CallsClassSurface( + sourceFile = "Entries.ts", + exportName = "Entries", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = mapOf("item" to ObjectDomain(fields = observationFields)), + dataClasses = listOf(CallsClassDataClass(id = "pair", exportName = "PairEntry", fields = registryFields)), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + val replay = OriginalTypeScriptClassReplayer( + directory, + harness.functions.single(), + harness.observations.single() + ) + .use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertTrue(replay.observation.representable, replay.toString()) + val item = replay.observation.values.getValue("state_0_item") as JsConcreteValue.Object + assertEquals(number(1), item.fields["key"]) + assertEquals(number(2), item.fields["value"]) + assertEquals(JsConcreteValue.String("pair"), item.fields[CLASS_IDENTITY_FIELD]) + } + + private fun dataClassFixture( + observationFieldDomain: PropertyDomain = NumberDomain(), + fixtureSource: String = dataClassSource, + constructorInputs: List = emptyList(), + ): CallsClassPreparedHarness { + Files.writeString(directory.resolve("Entries.ts"), fixtureSource) + val entryFields = mapOf("value" to NumberDomain()) + val array = ArrayDomain( + element = ObjectDomain(fields = mapOf("value" to observationFieldDomain)), + maxLength = 4, + ) + val stateFields = mapOf("items" to array) + val surface = CallsClassSurface( + sourceFile = "Entries.ts", + exportName = "Entries", + constructorInputs = constructorInputs, + members = listOf( + CallsClassMember( + id = "read", + name = "read", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = array, + ), + CallsClassMember( + id = "self", + name = "self", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = ObjectDomain(fields = stateFields), + resultClassName = "Entries", + ), + ), + stateFields = stateFields, + dataClasses = listOf(CallsClassDataClass(id = "entry", exportName = "Entry", fields = entryFields)), + ) + + return prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + } + + private val dataClassSource = """ + export class Entry { + value: number; + constructor(value: number) { this.value = value; } + } + export class Entries { + items: Entry[]; + constructor() { this.items = [new Entry(1)]; } + read(): Entry[] { return this.items; } + self(): Entries { return this; } + } + """.trimIndent() + + private fun surface() = CallsClassSurface( + sourceFile = "Entries.ts", + exportName = "Entries", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = emptyMap(), + ) + + private fun project() = CallsProjectCase( + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = false, + functions = emptyList(), + ) + + private fun number(value: Int) = JsConcreteValue.number(value.toDouble()) +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassExperimentTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassExperimentTest.kt new file mode 100644 index 0000000000..aabe0bbc0a --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassExperimentTest.kt @@ -0,0 +1,289 @@ +package org.usvm.ts.calls + +import org.usvm.ts.pbt.fastcheck.TypeScriptClassStatementDescriptor +import org.usvm.ts.pbt.model.JsConcreteValue +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class CallsClassExperimentTest { + @Test + fun `class treatments reject sixty seconds and duplicate paired seeds`() { + val manifest = manifest() + + assertEquals(listOf(30_000L, 120_000L), manifest.budgetsMillis) + assertFailsWith { manifest.copy(budgetsMillis = listOf(30_000L, 60_000L, 120_000L)) } + assertFailsWith { manifest.copy(seeds = listOf(17, 17)) } + } + + @Test + fun `native primary coverage survives strict symbolic rejection and excludes late terminal candidates`() { + val hash = "a".repeat(64) + val ids = (0..2).map { "$hash:$it" } + val universe = CallsClassSourceUniverse( + sourceFile = "Counter.ts", + className = "Counter", + sourceSha256 = hash, + universeSha256 = "b".repeat(64), + instrumenterVersion = "test", + statements = ids.mapIndexed { index, id -> statement(id, index) }, + ) + val candidates = listOf( + candidate(emittedAtMillis = 20, scenarioId = "first"), + candidate(emittedAtMillis = 30, scenarioId = "second"), + candidate(emittedAtMillis = 40, scenarioId = "second"), + candidate(emittedAtMillis = 30_001, scenarioId = "late"), + ) + val events = listOf( + replay(index = 0, scenarioId = "first", covered = setOf(ids[0]), accepted = true), + replay(index = 1, scenarioId = "second", covered = setOf(ids[0], ids[1]), accepted = true), + replay(index = 2, scenarioId = "second", covered = setOf(ids[2]), accepted = false), + replay(index = 3, scenarioId = "late", covered = setOf(ids[2]), accepted = true), + ) + + val checkpoint = callsClassCoverageCheckpoints( + search = search(candidates), + events = events, + universe = universe, + budgetMillis = 30_000, + ).last() + + assertEquals(ids.toSet(), checkpoint.coveredStatementIds) + assertEquals(emptySet(), checkpoint.uncoveredStatementIds) + assertEquals(3, checkpoint.totalStatements) + assertEquals(3, checkpoint.acceptedCandidates) + assertEquals(0, checkpoint.rejectedCandidates) + assertEquals(setOf(ids[0], ids[1]), checkpoint.strictCoveredStatementIds) + assertEquals(2, checkpoint.strictAcceptedCandidates) + assertEquals(1, checkpoint.strictRejectedCandidates) + assertTrue(checkpoint.strictCoveredStatementIds.size < checkpoint.totalStatements) + } + + @Test + fun `reached prefix uses real search time and post stop extraction appears only in final checkpoint`() { + val universe = universe() + val id = universe.statementIds.single() + val prefix = candidate(emittedAtMillis = 20, scenarioId = "prefix").copy( + completion = null, + origin = CallsCoverageCandidateOrigin.SUSPENDED, + suspensionReason = "Unsupported native call", + ) + val frontier = prefix.copy(entryPointId = "frontier", emittedAtMillis = 30_050, suspendedAtMachineStop = true) + val lateTerminal = candidate(emittedAtMillis = 30_051, scenarioId = "late") + val latePrefix = prefix.copy(entryPointId = "late-prefix", emittedAtMillis = 30_052) + val candidates = listOf(prefix, frontier, lateTerminal, latePrefix) + val events = candidates.mapIndexed { index, candidate -> + replay(index, requireNotNull(candidate.entryPointId), setOf(id), accepted = false).copy( + origin = candidate.origin, + strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED, + ) + } + + val checkpoints = callsClassCoverageCheckpoints(search(candidates), events, universe, budgetMillis = 30_000) + + assertEquals(30_050, candidates[1].emittedAtMillis) + assertEquals(1, checkpoints.first().emittedCandidates) + assertEquals(0, checkpoints.first().postStopSuspendedCandidates) + assertFalse(checkpoints.first().includesSuspendedExtraction) + assertEquals(2, checkpoints.last().emittedCandidates) + assertEquals(2, checkpoints.last().acceptedCandidates) + assertEquals(2, checkpoints.last().strictOracleNotApplicable) + assertEquals(1, checkpoints.last().postStopSuspendedCandidates) + assertTrue(checkpoints.last().includesSuspendedExtraction) + } + + @Test + fun `primary coverage rejects transport mismatch foreign hits and incomplete native observations`() { + val universe = universe() + val observation = CallsClassReplayObservation( + inputs = emptyList(), + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + coveredStatementKeys = universe.statementIds, + ) + val incomplete = observation.copy(observation = observation.observation.copy(representable = false)) + val mismatch = observation.copy( + observation = observation.observation.copy(completion = CallsCoverageCompletion.THREW), + ) + val foreign = observation.copy(coveredStatementKeys = setOf("foreign")) + val changedInputs = observation.copy(inputs = listOf(JsConcreteValue.Boolean(value = true))) + + assertTrue(callsClassNativeCoverageAcceptance(observation, observation, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, mismatch, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, foreign, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, changedInputs, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(incomplete, incomplete, universe).accepted) + assertFalse(callsClassNativeCoverageAcceptance(observation, observation, universe, error = "timeout").accepted) + assertFalse(callsClassNativeCoverageAcceptance(null, observation, universe).accepted) + } + + @Test + fun `suspended candidates reject invented terminal observations and old measurement schema`() { + val terminal = candidate(emittedAtMillis = 1, scenarioId = "constructor") + + assertFailsWith { terminal.copy(origin = CallsCoverageCandidateOrigin.SUSPENDED) } + assertFailsWith { + terminal.copy( + completion = null, + origin = CallsCoverageCandidateOrigin.SUSPENDED, + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + ) + } + assertFailsWith { manifest().copy(schemaVersion = 1) } + } + + @Test + fun `resume recomputes native and strict decisions and refuses changed candidate origin`() { + val universe = universe() + val observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED) + val candidate = candidate(emittedAtMillis = 1, scenarioId = "constructor").copy(observation = observation) + val event = replay(index = 0, scenarioId = "constructor", covered = universe.statementIds, accepted = true) + + validateCallsClassReplayEvent(candidate, index = 0, event = event, universe = universe) + + assertFailsWith { + validateCallsClassReplayEvent( + candidate, + index = 0, + event = event.copy(nativeCoverage = CallsClassOracleResult(accepted = false)), + universe = universe, + ) + } + assertFailsWith { + validateCallsClassReplayEvent( + candidate, + index = 0, + event = event.copy(oracle = CallsClassOracleResult(accepted = false)), + universe = universe, + ) + } + assertFailsWith { + validateCallsClassReplayEvent( + candidate, + index = 0, + event = event.copy(origin = CallsCoverageCandidateOrigin.SUSPENDED), + universe = universe, + ) + } + } + + @Test + fun `class cell refuses a truncated replay journal`() { + val hash = "a".repeat(64) + val universe = CallsClassSourceUniverse( + sourceFile = "Counter.ts", + className = "Counter", + sourceSha256 = hash, + universeSha256 = hash, + instrumenterVersion = "test", + statements = listOf(statement("$hash:0", index = 0)), + ) + + assertFailsWith { + callsClassCoverageCheckpoints( + search = search(listOf(candidate(emittedAtMillis = 1, scenarioId = "constructor"))), + events = emptyList(), + universe = universe, + budgetMillis = 30_000, + ) + } + } + + private fun manifest(): CallsClassExperimentManifest = CallsClassExperimentManifest( + experimentId = "class-pilot", + toolRevision = "a".repeat(40), + nativeFrontendRevision = "bundled:test", + modelSet = CallsModelSetIdentity(ids = emptySet(), toolRevision = "a".repeat(40)), + seeds = listOf(17), + classes = listOf( + CallsClassCase( + classId = "counter", + projectId = "fixture", + revision = "c".repeat(40), + sourceRoot = "fixture", + sourceSha256 = mapOf("Counter.ts" to "a".repeat(64)), + surface = CallsClassSurface( + sourceFile = "Counter.ts", + exportName = "Counter", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = emptyMap(), + ), + ), + ), + ) + + private fun candidate(emittedAtMillis: Long, scenarioId: String): CallsCoverageCandidate = CallsCoverageCandidate( + emittedAtMillis = emittedAtMillis, + emittedAtStep = 1, + inputs = emptyList(), + newSymbolicStatements = 1, + completion = CallsCoverageCompletion.RETURNED, + entryPointId = scenarioId, + ) + + private fun replay( + index: Int, + scenarioId: String, + covered: Set, + accepted: Boolean, + ): CallsClassReplayEvent = CallsClassReplayEvent( + index = index, + scenarioId = scenarioId, + original = CallsClassReplayObservation( + inputs = emptyList(), + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + ), + instrumented = CallsClassReplayObservation( + inputs = emptyList(), + observation = CallsClassObservation(completion = CallsCoverageCompletion.RETURNED), + coveredStatementKeys = covered, + ), + oracle = CallsClassOracleResult(accepted = accepted), + nativeCoverage = CallsClassOracleResult(accepted = true), + ) + + private fun universe(): CallsClassSourceUniverse { + val hash = "a".repeat(64) + return CallsClassSourceUniverse( + sourceFile = "Counter.ts", + className = "Counter", + sourceSha256 = hash, + universeSha256 = hash, + instrumenterVersion = "test", + statements = listOf(statement("$hash:0", index = 0)), + ) + } + + private fun search(candidates: List): CallsClassSearchResult = CallsClassSearchResult( + search = CallsCoverageSearchResult( + status = CallsCoverageSearchStatus.EXHAUSTED, + candidates = candidates, + selectedStates = candidates.size, + executedSteps = candidates.size.toLong(), + stepsWithinBudget = candidates.size.toLong(), + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = 0, + machineSetupElapsedMillis = 0, + searchElapsedMillis = 0, + machineTeardownElapsedMillis = 0, + ), + scenarios = emptyList(), + visits = emptyList(), + unknownCallEvents = 0, + ) + + private fun statement(id: String, index: Int) = TypeScriptClassStatementDescriptor( + id = id, + memberName = "increment", + kind = "ExpressionStatement", + startOffset = index, + endOffset = index + 1, + startLine = 1, + startColumn = index, + endLine = 1, + endColumn = index + 1, + ) +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt new file mode 100644 index 0000000000..11abffc97c --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt @@ -0,0 +1,473 @@ +package org.usvm.ts.calls + +import org.junit.jupiter.api.io.TempDir +import org.usvm.SolverType +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.ConstantDomain +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.ObjectDomain +import org.usvm.ts.pbt.model.OptionalDomain +import org.usvm.ts.pbt.model.PropertyInput +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class CallsClassHarnessOracleTest { + @TempDir + lateinit var directory: Path + + @Test + fun `scenario schedule freezes constructor every member and every ordered pair`() { + val harness = fixture() + + assertEquals(expected = 21, actual = harness.scenarios.size) + assertEquals(emptyList(), harness.scenarios.first().memberIds) + assertEquals( + listOf("add", "read", "write", "arrow"), + harness.scenarios.drop(1).take(4).map { it.memberIds.single() } + ) + assertTrue(harness.scenarios.any { it.memberIds == listOf("write", "read") }) + assertTrue(harness.scenarios.any { it.memberIds == listOf("add", "add") }) + assertTrue(harness.functions.all { it.symbolicCoverageClass?.className == "Counter" }) + assertTrue(harness.functions[1].inputs[1].domain is ObjectDomain) + } + + @Test + fun `same receiver observes setter mutation before getter and resets between replays`() { + val harness = fixture() + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("write", "read") } + OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + ).use { replayer -> + val first = replayer.replay(inputs = listOf(number(2), number(9)), timeoutMillis = 20_000) + val second = replayer.replay(inputs = listOf(number(3), number(4)), timeoutMillis = 20_000) + + assertTrue(first.observation.representable, first.observation.diagnostic) + assertEquals(number(2), first.observation.values["state_0_count"]) + assertEquals(number(9), first.observation.values["state_1_count"]) + assertEquals(number(9), first.observation.values["result_1"]) + assertEquals(JsConcreteValue.Undefined, first.observation.values["result_0"]) + assertEquals(number(3), second.observation.values["state_0_count"]) + assertEquals(number(4), second.observation.values["result_1"]) + } + } + + @Test + fun `object parameter transport stays frozen when callable mutates it`() { + val harness = fixture() + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("add", "read") } + val options = JsConcreteValue.Object(mapOf("amount" to number(3))) + val inputs = listOf(number(2), options) + + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + ).use { it.replay(inputs = inputs, timeoutMillis = 20_000) } + + assertEquals(inputs, replay.inputs) + assertTrue(replay.observation.representable, replay.observation.diagnostic) + assertEquals(number(5), replay.observation.values["result_0"]) + assertEquals(number(5), replay.observation.values["result_1"]) + } + + @Test + fun `exact counters come only from equivalent instrumented scenario execution`() { + val harness = fixture() + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("arrow") } + val inputs = listOf(number(2), number(5)) + val source = Files.readString(directory.resolve("Counter.ts")) + val instrumented = source.replace( + "return this.count * x;", + "(globalThis as any).classHits?.add('arrow-body'); return this.count * x;" + ) + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(inputs = inputs, timeoutMillis = 20_000) + } + val counted = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[index], + observationFields = harness.observations[index], + sourceOverride = instrumented, + overrideSourceFile = "Counter.ts", + counterGlobalName = "classHits", + ).use { it.replay(inputs = inputs, timeoutMillis = 20_000) } + + assertEquals(setOf("arrow-body"), counted.coveredStatementKeys) + assertTrue( + compareCallsClassObservations(original.observation, original.observation, counted.observation).accepted + ) + } + + @Test + fun `returned values and receiver state are independently checked`() { + val original = CallsClassObservation( + completion = CallsCoverageCompletion.RETURNED, + values = mapOf("result_0" to number(1), "state_1_count" to number(2)), + ) + val wrongResult = original.copy(values = original.values + ("result_0" to number(0))) + val wrongState = original.copy(values = original.values + ("state_1_count" to number(0))) + + assertFalse(compareCallsClassObservations(wrongResult, original, original).accepted) + assertFalse(compareCallsClassObservations(wrongState, original, original).accepted) + assertFalse(compareCallsClassObservations(original, original, wrongState).accepted) + assertTrue(compareCallsClassObservations(original, original, original).accepted) + } + + @Test + fun `matching unavailable outcomes are never equivalent`() { + val unavailable = CallsClassObservation( + completion = CallsCoverageCompletion.THREW, + representable = false, + diagnostic = "Incomplete thrown receiver state", + ) + + assertFalse(compareCallsClassObservations(unavailable, unavailable, unavailable).accepted) + assertFalse(compareCallsClassObservations(null, unavailable, unavailable).accepted) + } + + @Test + fun `invalid getter and sequence bound reject before execution`() { + assertFailsWith { + surface().copy( + members = listOf( + CallsClassMember( + id = "bad", + name = "bad", + kind = CallsClassMemberKind.GETTER, + inputs = listOf(PropertyInput("x", IntegerDomain())), + resultDomain = NumberDomain(), + ) + ) + ) + } + Files.writeString(directory.resolve("Counter.ts"), source) + + assertFailsWith { + prepareCallsClassHarness(directory, project(), surface(), maxSequenceLength = 3) + } + } + + @Test + fun `array result is snapshotted before subsequent mutation`() { + Files.writeString( + directory.resolve("Bag.ts"), + """ + export class Bag { + items: number[] = [1]; + take(): number[] { return this.items; } + replace(x: number): number { this.items[0] = x; return x; } + } + """.trimIndent() + ) + val arrayDomain = ArrayDomain(element = NumberDomain(), maxLength = 4) + val surface = CallsClassSurface( + sourceFile = "Bag.ts", + exportName = "Bag", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember("take", "take", CallsClassMemberKind.METHOD, emptyList(), arrayDomain), + CallsClassMember( + "replace", + "replace", + CallsClassMemberKind.METHOD, + listOf(PropertyInput("x", NumberDomain())), + NumberDomain() + ), + ), + stateFields = mapOf("items" to arrayDomain), + ) + val harness = prepareCallsClassHarness(directory, project(), surface) + val index = harness.scenarios.indexOfFirst { it.memberIds == listOf("take", "replace") } + + val replay = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(inputs = listOf(number(9)), timeoutMillis = 20_000) + } + + assertTrue(replay.observation.representable, replay.observation.diagnostic) + assertEquals(JsConcreteValue.Array(listOf(number(1))), replay.observation.values["result_0"]) + assertEquals(JsConcreteValue.Array(listOf(number(1))), replay.observation.values["state_1_items"]) + assertEquals(JsConcreteValue.Array(listOf(number(9))), replay.observation.values["state_2_items"]) + } + + @Test + fun `symbolic constructed state and return snapshots agree with original replay`() { + Files.writeString( + directory.resolve("Scalar.ts"), + """ + export class Scalar { + count: number; + constructor(start: number) { this.count = start; } + add(amount: number): number { this.count += amount; return this.count; } + read(): number { return this.count; } + } + """.trimIndent() + ) + val surface = CallsClassSurface( + sourceFile = "Scalar.ts", + exportName = "Scalar", + constructorInputs = listOf(PropertyInput("start", IntegerDomain(min = 0, max = 4))), + members = listOf( + CallsClassMember( + "add", + "add", + CallsClassMemberKind.METHOD, + listOf(PropertyInput("amount", IntegerDomain(min = 0, max = 4))), + NumberDomain() + ), + CallsClassMember("read", "read", CallsClassMemberKind.METHOD, emptyList(), NumberDomain()), + ), + stateFields = mapOf("count" to NumberDomain()), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 2) + + val result = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = CallsExperimentProfile.EMPTY_STOP, + frozenModelIds = emptySet(), + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 2.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 20, + requireEntryPointWitness = true, + ) + ) + + assertTrue(result.search.candidates.isNotEmpty(), result.toString()) + assertTrue((result.solverCalls ?: 0) > 0, result.toString()) + assertTrue(result.solverQueries.isNotEmpty(), result.toString()) + assertTrue(result.timeQuanta.isNotEmpty(), result.toString()) + assertTrue( + result.timeQuanta.all { it.requestedQuantumMillis <= 100 && it.elapsedNanos >= 0 }, + result.toString() + ) + assertTrue(result.search.executedSteps > 0, result.toString()) + assertTrue(result.search.searchElapsedMillis <= 3_000, result.toString()) + assertEquals( + harness.scenarios.map { it.id }.toSet(), + result.search.candidates.map { it.entryPointId }.toSet(), + result.toString() + ) + for (candidate in result.search.candidates) { + val index = harness.scenarios.indexOfFirst { it.id == candidate.entryPointId } + val original = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(candidate.inputs, timeoutMillis = 20_000) + } + val checked = + compareCallsClassObservations(candidate.observation, original.observation, original.observation) + assertTrue(checked.accepted, "$candidate / $original / $checked") + } + } + + @Test + fun `optional array and nested object snapshots guard nullish values before copying`() { + Files.writeString( + directory.resolve("OptionalBag.ts"), + """ + export class OptionalBag { + items: number[] | undefined = undefined; + object: { items: number[] } | null = null; + read(): number[] | undefined { return this.items; } + } + """.trimIndent() + ) + val array = ArrayDomain(element = NumberDomain(), maxLength = 4) + val optionalArray = OptionalDomain(value = array, nil = JsConcreteValue.Undefined) + val surface = CallsClassSurface( + sourceFile = "OptionalBag.ts", + exportName = "OptionalBag", + constructorInputs = emptyList(), + members = listOf(CallsClassMember("read", "read", CallsClassMemberKind.METHOD, emptyList(), optionalArray)), + stateFields = mapOf( + "items" to optionalArray, + "object" to OptionalDomain(value = ObjectDomain(mapOf("items" to array)), nil = JsConcreteValue.Null), + ), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + val replay = OriginalTypeScriptClassReplayer(directory, harness.functions[1], harness.observations[1]).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + + assertTrue(replay.observation.representable, replay.observation.diagnostic) + assertEquals(JsConcreteValue.Undefined, replay.observation.values["result_0"]) + assertEquals(JsConcreteValue.Null, replay.observation.values["state_1_object"]) + } + + @Test + fun `raw output and receiver shape checks reject fields hidden by projection`() { + Files.writeString( + directory.resolve("Hidden.ts"), + """ + export class Hidden { + count: number = 1; + result(): { kept: number; omitted: number } { return { kept: 1, omitted: 2 }; } + mutate(): number { (this as any).extra = 3; return this.count; } + mutateCallback(): number { (this as any).extraCallback = () => 3; return this.count; } + } + """.trimIndent() + ) + val surface = CallsClassSurface( + sourceFile = "Hidden.ts", + exportName = "Hidden", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + "result", + "result", + CallsClassMemberKind.METHOD, + emptyList(), + ObjectDomain(mapOf("kept" to NumberDomain())) + ), + CallsClassMember("mutate", "mutate", CallsClassMemberKind.METHOD, emptyList(), NumberDomain()), + CallsClassMember( + id = "mutateCallback", + name = "mutateCallback", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = NumberDomain(), + ), + ), + stateFields = mapOf("count" to NumberDomain()), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + for (index in 1..3) { + val replay = OriginalTypeScriptClassReplayer( + directory, + harness.functions[index], + harness.observations[index] + ).use { + it.replay(inputs = emptyList(), timeoutMillis = 20_000) + } + + assertFalse(replay.observation.representable) + assertTrue( + replay.observation.diagnostic?.contains("Unlisted or missing object property") == true, + replay.toString() + ) + } + } + + @Test + fun `native empty object shape rejects builtin internal slot values`() { + Files.writeString( + directory.resolve("Builtin.ts"), + """ + export class Builtin { + result(): object { return new Date(1234); } + } + """.trimIndent(), + ) + val surface = CallsClassSurface( + sourceFile = "Builtin.ts", + exportName = "Builtin", + constructorInputs = emptyList(), + members = listOf( + CallsClassMember( + id = "result", + name = "result", + kind = CallsClassMemberKind.METHOD, + inputs = emptyList(), + resultDomain = ObjectDomain(fields = emptyMap()), + ), + ), + stateFields = emptyMap(), + ) + val harness = prepareCallsClassHarness(directory, project(), surface, maxSequenceLength = 1) + + val replay = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions[1], + observationFields = harness.observations[1], + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertFalse(replay.observation.representable) + assertTrue(replay.observation.diagnostic?.contains("Unlisted object prototype") == true) + } + + private fun fixture(): CallsClassPreparedHarness { + Files.writeString(directory.resolve("Counter.ts"), source) + return prepareCallsClassHarness(directory, project(), surface()) + } + + private fun surface(): CallsClassSurface = CallsClassSurface( + sourceFile = "Counter.ts", + exportName = "Counter", + constructorInputs = listOf(PropertyInput("start", IntegerDomain(min = 0, max = 10))), + members = listOf( + CallsClassMember( + "add", + "add", + CallsClassMemberKind.METHOD, + listOf(PropertyInput("options", ObjectDomain(mapOf("amount" to IntegerDomain(min = 0, max = 10))))), + NumberDomain() + ), + CallsClassMember("read", "value", CallsClassMemberKind.GETTER, emptyList(), NumberDomain()), + CallsClassMember( + "write", + "value", + CallsClassMemberKind.SETTER, + listOf(PropertyInput("value", IntegerDomain(min = 0, max = 10))), + ConstantDomain(JsConcreteValue.Undefined) + ), + CallsClassMember( + "arrow", + "multiply", + CallsClassMemberKind.FUNCTION_FIELD, + listOf(PropertyInput("x", IntegerDomain(min = 0, max = 10))), + NumberDomain() + ), + ), + stateFields = mapOf("count" to NumberDomain()), + ) + + private fun project() = CallsProjectCase( + "fixture", + "0".repeat(40), + ".", + development = false, + functions = emptyList() + ) + + private fun number(value: Int) = JsConcreteValue.number(value.toDouble()) + + private val source = """ + export class Counter { + count: number; + constructor(start: number) { this.count = start; } + add(options: { amount: number }): number { + this.count += options.amount; options.amount = 99; return this.count; + } + get value(): number { return this.count; } + set value(value: number) { this.count = value; } + multiply = (x: number): number => { return this.count * x; }; + } + """.trimIndent() +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassNativeDiagnosticTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassNativeDiagnosticTest.kt new file mode 100644 index 0000000000..fdaffa9a0d --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassNativeDiagnosticTest.kt @@ -0,0 +1,120 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.encodeToString +import org.junit.jupiter.api.io.TempDir +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.NumberDomain +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class CallsClassNativeDiagnosticTest { + @TempDir + lateinit var directory: Path + + @Test + fun `native numeric throws retain the actual primitive value`() { + val harness = fixture(constructorBody = "throw 7;") + + val replay = replay(harness) + + assertEquals(CallsCoverageCompletion.THREW, replay.observation.completion) + assertFalse(replay.observation.representable) + val expectedValue = CallsExperimentJson.json.encodeToString(JsConcreteValue.number(7.0)) + assertEquals("$thrownPrefix; thrown value=$expectedValue", replay.observation.diagnostic) + } + + @Test + fun `native string throws retain the actual string value`() { + val harness = fixture(constructorBody = "throw 'actual thrown string';") + + val replay = replay(harness) + + assertEquals(CallsCoverageCompletion.THREW, replay.observation.completion) + assertFalse(replay.observation.representable) + val expectedValue = CallsExperimentJson.json.encodeToString( + JsConcreteValue.String("actual thrown string"), + ) + assertEquals("$thrownPrefix; thrown value=$expectedValue", replay.observation.diagnostic) + } + + @Test + fun `native errors retain the actual name and message`() { + val harness = fixture(constructorBody = "throw new TypeError('actual native message');") + + val replay = replay(harness) + + assertEquals(CallsCoverageCompletion.THREW, replay.observation.completion) + assertFalse(replay.observation.representable) + assertEquals( + "$thrownPrefix; exception name=\"TypeError\", message=\"actual native message\"", + replay.observation.diagnostic, + ) + } + + @Test + fun `repeated loop coverage preserves native snapshots and contains only unique statement keys`() { + val harness = fixture(constructorBody = "for (let i = 0; i < 20000; i++) { this.count++; }") + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "NativeOutcome.ts", + className = "NativeOutcome", + ) + + val original = replay(harness) + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = "NativeOutcome.ts", + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + assertTrue(original.observation.representable, original.observation.diagnostic) + assertEquals(JsConcreteValue.number(20_000.0), original.observation.values["state_0_count"]) + assertEquals(original.observation, instrumented.observation) + val coveredIds = instrumentation.universe.checkedCoveredIds(instrumented.coveredStatementKeys) + assertEquals(instrumentation.universe.statements.size, coveredIds.size) + } + + private fun fixture(constructorBody: String): CallsClassPreparedHarness { + Files.writeString( + directory.resolve("NativeOutcome.ts"), + """ + export class NativeOutcome { + count: number = 0; + constructor() { $constructorBody } + } + """.trimIndent(), + ) + val surface = CallsClassSurface( + sourceFile = "NativeOutcome.ts", + exportName = "NativeOutcome", + constructorInputs = emptyList(), + members = emptyList(), + stateFields = mapOf("count" to NumberDomain()), + ) + val project = CallsProjectCase( + projectId = "native-diagnostic-fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = true, + functions = emptyList(), + ) + + return prepareCallsClassHarness(directory, project, surface, maxSequenceLength = 2) + } + + private fun replay(harness: CallsClassPreparedHarness): CallsClassReplayObservation = + OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + ).use { it.replay(inputs = emptyList(), timeoutMillis = 20_000) } + + private val thrownPrefix = "Thrown class scenarios have no complete return/state snapshot" +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt new file mode 100644 index 0000000000..29c27908d5 --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassUnknownCallJournalTest.kt @@ -0,0 +1,323 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.io.TempDir +import org.usvm.CoverageZone +import org.usvm.SolverType +import org.usvm.UMachineOptions +import org.usvm.machine.TsMachine +import org.usvm.machine.TsOptions +import org.usvm.machine.state.TsMethodResult +import org.usvm.machine.state.TsState +import org.usvm.statistics.UMachineObserver +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.contains +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource + +class CallsClassUnknownCallJournalTest { + @TempDir + lateinit var directory: Path + + @Test + fun `actual repeated source calls retain ordered model and residual decisions in serialized search`() { + val harness = prepareFixture( + source = """ + export class Telemetry { + value: number = 0; + constructor(input: number) { + for (let index = 0; index < 2; index++) { + this.value = Math.floor(input); + } + } + } + """.trimIndent(), + inputs = listOf(PropertyInput(name = "input", domain = IntegerDomain(min = 0, max = 3))), + stateFields = mapOf("value" to NumberDomain()), + ) + + val modeled = search(harness, profile = CallsExperimentProfile.FROZEN_FRESH, models = setOf("ts.math.floor")) + val residual = search(harness, profile = CallsExperimentProfile.EMPTY_FRESH, models = emptySet()) + + assertTrue(modeled.search.candidates.isNotEmpty(), modeled.toString()) + assertTrue(residual.search.candidates.isNotEmpty(), residual.toString()) + for (result in listOf(modeled, residual)) { + assertTrue(result.unknownCalls.size >= 2, result.toString()) + assertEquals(result.unknownCallEvents, result.unknownCalls.size) + assertEquals((1..result.unknownCalls.size).toList(), result.unknownCalls.map { it.eventIndex }) + val sites = result.unknownCalls.groupingBy { it.callSite }.eachCount() + assertTrue(sites.values.any { it >= 2 }, result.toString()) + assertTrue(result.unknownCalls.all { it.callSite.sourcePath.endsWith("Telemetry.ts") }, result.toString()) + assertTrue(result.unknownCalls.all { it.callerMethodSignature.contains("Telemetry::constructor") }) + assertTrue(result.unknownCalls.all { it.callee.name == "floor" }) + assertTrue(result.unknownCalls.all { it.failureReason == "PARTIAL_APPROXIMATION" }) + assertNotNull(result.unknownCalls.first().callSite.startOffset) + + val encoded = CallsExperimentJson.json.encodeToString(result) + assertEquals(result, CallsExperimentJson.json.decodeFromString(encoded)) + val legacy = JsonObject( + CallsExperimentJson.json.parseToJsonElement(encoded).jsonObject.filterKeys { it != "unknownCalls" }, + ) + val decodedLegacy = CallsExperimentJson.json.decodeFromString(legacy.toString()) + assertEquals(emptyList(), decodedLegacy.unknownCalls) + assertEquals(result.unknownCallEvents, decodedLegacy.unknownCallEvents) + } + for (event in modeled.unknownCalls) { + assertEquals(CallsUnknownCallDecisionKind.MODEL_APPLIED, event.decision) + assertEquals("MODEL_APPLIED", event.outcome) + assertEquals("ts.math.floor", event.modelId) + assertNull(event.residualPolicy) + } + for (event in residual.unknownCalls) { + assertEquals(CallsUnknownCallDecisionKind.RESIDUAL_FALLBACK, event.decision) + assertEquals("FRESH_SYMBOLIC_RETURN", event.outcome) + assertNull(event.modelId) + assertEquals("FRESH_SYMBOLIC_RETURN", event.residualPolicy) + } + } + + @Test + fun `stop path harvests reached prefix inputs without pretending that constructor returned`() { + val domain = IntegerDomain(min = 0, max = 3) + val harness = prepareFixture( + source = """ + export class Telemetry { + value: number = 0; + constructor(input: number) { + this.value = Math.floor(input); + } + } + """.trimIndent(), + inputs = listOf(PropertyInput(name = "input", domain = domain)), + stateFields = mapOf("value" to NumberDomain()), + ) + + val result = search(harness, profile = CallsExperimentProfile.EMPTY_STOP, models = emptySet()) + + val candidate = result.search.candidates.single() + assertEquals(CallsCoverageCandidateOrigin.SUSPENDED, candidate.origin) + assertNull(candidate.completion) + assertNull(candidate.observation) + assertFalse(candidate.suspendedAtMachineStop) + assertNotNull(candidate.suspensionReason) + assertTrue(candidate.inputs.single() in domain) + assertTrue(result.search.extractionFailures.isEmpty(), result.toString()) + assertEquals(expected = "PATH_STOPPED", actual = result.unknownCalls.single().outcome) + + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "Telemetry.ts", + className = "Telemetry", + ) + val original = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + val instrumented = OriginalTypeScriptClassReplayer( + sourceRoot = directory, + function = harness.functions.single(), + observationFields = harness.observations.single(), + sourceOverride = instrumentation.instrumentedSource, + overrideSourceFile = "Telemetry.ts", + counterGlobalName = instrumentation.hitGlobalName, + ).use { it.replay(inputs = candidate.inputs, timeoutMillis = 20_000) } + + assertTrue(callsClassNativeCoverageAcceptance(original, instrumented, instrumentation.universe).accepted) + assertTrue(instrumented.coveredStatementKeys.isNotEmpty()) + val strictOracle = compareCallsClassObservations( + symbolic = candidate.observation, + original = original.observation, + instrumented = instrumented.observation, + ) + assertFalse(strictOracle.accepted) + } + + @Test + fun `prefix snapshot does not hide completed novelty or strict observation for the same inputs`() { + val harness = prepareFixture( + source = """ + export class Telemetry { + value: number = 0; + constructor() { this.value = 1; } + } + """.trimIndent(), + inputs = emptyList(), + stateFields = mapOf("value" to NumberDomain()), + ) + + val candidates = terminalAfterPrefix(harness) + + assertEquals( + listOf(CallsCoverageCandidateOrigin.SUSPENDED, CallsCoverageCandidateOrigin.TERMINATED), + candidates.map { it.origin }, + ) + assertEquals(candidates[0].inputs, candidates[1].inputs) + assertEquals(expected = 0, actual = candidates[0].newSymbolicStatements) + assertNull(candidates[0].completion) + assertNull(candidates[0].observation) + assertTrue(candidates[1].newSymbolicStatements > 0) + assertEquals(CallsCoverageCompletion.RETURNED, candidates[1].completion) + assertTrue(assertNotNull(candidates[1].observation).representable) + } + + @Test + fun `symbolic thrown diagnostic retains actual exception type and expression`() { + val harness = prepareFixture( + source = """ + export class Telemetry { + constructor() { throw 7; } + } + """.trimIndent(), + inputs = emptyList(), + stateFields = emptyMap(), + ) + + val result = search(harness, profile = CallsExperimentProfile.EMPTY_STOP, models = emptySet()) + + val observation = assertNotNull(result.search.candidates.single().observation) + assertEquals(CallsCoverageCompletion.THREW, observation.completion) + assertFalse(observation.representable) + assertEquals(emptyMap(), observation.values) + val diagnostic = assertNotNull(observation.diagnostic) + assertTrue(diagnostic.contains("exception type=number"), diagnostic) + assertTrue(diagnostic.contains("value=(fp "), diagnostic) + assertFalse(compareCallsClassObservations(observation, observation, observation).accepted) + } + + private fun prepareFixture( + source: String, + inputs: List, + stateFields: Map, + ): CallsClassPreparedHarness { + Files.writeString(directory.resolve("Telemetry.ts"), source) + val surface = CallsClassSurface( + sourceFile = "Telemetry.ts", + exportName = "Telemetry", + constructorInputs = inputs, + members = emptyList(), + stateFields = stateFields, + ) + val project = CallsProjectCase( + projectId = "fixture", + revision = "0".repeat(40), + sourceRoot = ".", + development = false, + functions = emptyList(), + ) + + return prepareCallsClassHarness(directory, project, surface, maxSequenceLength = 1) + } + + private fun terminalAfterPrefix(harness: CallsClassPreparedHarness): List { + val function = harness.functions.single() + val preparation = CurrentTsCallsSymbolicEngine().prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = directory, + project = harness.project, + function = function, + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + verifySourceCheckout = false, + ), + ) as CallsFunctionPreparation.Eligible + val scope = resolveCallsSymbolicCoverageScope( + preparation, + target = CallsSymbolicCoverageClass(sourceFile = "Telemetry.ts", className = "Telemetry"), + ) as CallsSymbolicCoverageScope.Selected + val inputs = CallsSymbolicInputs( + inputs = function.inputs, + bindings = preparation.inputBindings, + lexicalEnvironment = preparation.lexicalEnvironment, + captures = preparation.captures, + sourceScene = preparation.scene, + ) + val candidates = mutableListOf() + val observer = CompletedCoverageCandidateObserver( + contexts = listOf( + CallsCoverageCandidateContext( + method = preparation.method, + inputDomains = function.inputs, + symbolicInputs = inputs, + entryPointId = harness.scenarios.single().id, + observationFields = harness.observations.single(), + ), + ), + coverageStatements = scope.statements, + candidateCap = 2, + budgetMillis = 2_000, + machineCallStarted = TimeSource.Monotonic.markNow(), + onCandidate = { candidates += it }, + onExtractionFailure = { error(it) }, + emitSuspendedCandidates = true, + ) + val prefixThenTerminal = object : UMachineObserver by observer { + override fun onStateTerminated(state: TsState, stateReachable: Boolean) { + if (stateReachable) { + val prefix = state.clone() + prefix.methodResult = TsMethodResult.NoCall + prefix.unsupportedReason = "Synthetic reached-prefix observer contract" + observer.collectStoppedState(prefix) + } + observer.onStateTerminated(state = state, stateReachable = stateReachable) + } + } + + TsMachine( + scene = inputs.scene, + options = UMachineOptions( + timeout = 2.seconds, + solverTimeout = 1.seconds, + coverageZone = CoverageZone.CLASS, + stopOnCoverage = 0, + removeCoveredEntryPoints = false, + throwExceptionOnStepFailure = true, + ), + tsOptions = TsOptions( + classCoverageTargets = setOf(requireNotNull(scope.classSignature)), + eraseTypeAssertions = true, + ), + machineObserver = prefixThenTerminal, + ).use { machine -> + machine.analyzeWithMetadata( + methods = listOf(preparation.method), + configureInitialState = { _, state -> inputs.initialize(state) }, + ) + } + + return candidates + } + + private fun search( + harness: CallsClassPreparedHarness, + profile: CallsExperimentProfile, + models: Set, + ): CallsClassSearchResult = CurrentTsCallsClassCoverageEngine().search( + CallsClassSearchRequest( + harness = harness, + profile = profile, + frozenModelIds = models, + expectedNativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + seed = 1, + budget = 2.seconds, + timeQuantum = 100.milliseconds, + solverQueryLimit = 1.seconds, + solverType = SolverType.YICES, + candidateCap = 10, + requireEntryPointWitness = true, + ), + ) +} diff --git a/usvm-ts-fast-check/fast-check-adapter/package.json b/usvm-ts-fast-check/fast-check-adapter/package.json index 6764b2e13b..3fcbf35f61 100644 --- a/usvm-ts-fast-check/fast-check-adapter/package.json +++ b/usvm-ts-fast-check/fast-check-adapter/package.json @@ -9,7 +9,7 @@ "build": "tsc --project tsconfig.json", "pretest": "npm run build", "test": "npm run test:compiled", - "test:compiled": "node --test dist/test/coverage-branches.test.js dist/test/entry-point.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/observe-object.test.js dist/test/process-group-shutdown.test.js dist/test/process-supervisor.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js dist/test/source-replay.test.js dist/test/source-captures.test.js dist/test/local-source-closure.test.js" + "test:compiled": "node --test dist/test/coverage-branches.test.js dist/test/entry-point.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/observe-object.test.js dist/test/process-group-shutdown.test.js dist/test/process-supervisor.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js dist/test/source-replay.test.js dist/test/source-captures.test.js dist/test/local-source-closure.test.js dist/test/class-coverage-source.test.js" }, "dependencies": { "@jridgewell/trace-mapping": "0.3.31", diff --git a/usvm-ts-fast-check/fast-check-adapter/src/class-coverage-source.ts b/usvm-ts-fast-check/fast-check-adapter/src/class-coverage-source.ts new file mode 100644 index 0000000000..7ea82753fb --- /dev/null +++ b/usvm-ts-fast-check/fast-check-adapter/src/class-coverage-source.ts @@ -0,0 +1,671 @@ +import { createHash } from 'node:crypto' +import ts from 'typescript' + +export interface ClassStatementDescriptor { + id: string + memberName: string + kind: string + startOffset: number + endOffset: number + startLine: number + startColumn: number + endLine: number + endColumn: number +} + +export interface ClassCallableDescriptor { + name: string + kind: 'METHOD' | 'GETTER' | 'SETTER' | 'FUNCTION_FIELD' + isStatic: boolean + parameterNames: string[] + parameterTypeTexts: (string | null)[] + resultTypeText: string | null +} + +export interface ClassStateFieldDescriptor { name: string; isEcmaPrivate: boolean; typeText: string | null } +export interface ClassDataClassDescriptor { exportName: string; fields: string[] } +export interface ClassSurfaceInventory { + constructorParameterNames: string[] + publicMembers: ClassCallableDescriptor[] + instanceDataFields: ClassStateFieldDescriptor[] + dataClasses: ClassDataClassDescriptor[] +} + +export interface ClassCoverageInstrumentation { + source: string | null + reason: string | null + sourceSha256: string + instrumentedSha256: string | null + counterGlobalName: string + instrumenterVersion: string + statements: ClassStatementDescriptor[] + inventory: ClassSurfaceInventory | null +} + +interface MemberBody { name: string; node: ts.ClassElement; body: ts.Node; publicSurface: boolean; isStatic: boolean } +interface Insertion { offset: number; text: string; order: number } +const VERSION = 'class-statements-v2' + +/** Exact original AST IDs. Native counters avoid c8's ambiguous same-line statement records. */ +export function instrumentClassCoverage(file: ts.SourceFile, className: string): ClassCoverageInstrumentation { + const sourceSha256 = hash(file.text) + const counterGlobalName = `__usvmCallsClassStatementHits_${hash(`${sourceSha256}:${className}`).slice(0, 24)}` + const base = { sourceSha256, counterGlobalName, instrumenterVersion: VERSION } + const unsupported = (reason: string): ClassCoverageInstrumentation => + ({ ...base, source: null, reason, instrumentedSha256: null, statements: [], inventory: null }) + const matches = file.statements.filter(ts.isClassDeclaration).filter((node) => node.name?.text === className) + if (matches.length !== 1) return unsupported(`Expected one top-level named class ${className}`) + const selected = matches[0]! + if (selected.heritageClauses?.some((clause) => clause.token === ts.SyntaxKind.ExtendsKeyword)) { + return unsupported('Inherited class lifecycle is outside the checked class statement subset') + } + const decorated = ts.canHaveDecorators(selected) && (ts.getDecorators(selected)?.length ?? 0) > 0 + if (decorated) return unsupported('Decorated classes are outside the checked class statement subset') + + const members: MemberBody[] = [] + const initializers: MemberBody[] = [] + for (const member of selected.members) { + const modifiers = ts.canHaveModifiers(member) ? ts.getModifiers(member) ?? [] : [] + const isStatic = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + const memberDecorated = ts.canHaveDecorators(member) && (ts.getDecorators(member)?.length ?? 0) > 0 + if (memberDecorated) return unsupported('Decorated members are outside the checked class statement subset') + if (ts.isClassStaticBlockDeclaration(member)) return unsupported('Static initialization is not constructor coverage') + if (ts.isIndexSignatureDeclaration(member) || ts.isSemicolonClassElement(member)) continue + const name = ts.isConstructorDeclaration(member) ? 'constructor' : memberName(member) + if (name === null) return unsupported('Computed or unnamed class members are outside the checked subset') + const publicSurface = !modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.PrivateKeyword + || modifier.kind === ts.SyntaxKind.ProtectedKeyword) && !name.startsWith('#') + if (ts.isMethodDeclaration(member) || ts.isGetAccessorDeclaration(member) + || ts.isSetAccessorDeclaration(member) || ts.isConstructorDeclaration(member)) { + if (member.body === undefined) continue + const asynchronous = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) + || (ts.isMethodDeclaration(member) && member.asteriskToken !== undefined) + if (asynchronous) return unsupported('Async or generator members are outside the synchronous class subset') + members.push({ name, node: member, body: member.body, publicSurface, isStatic }) + } else if (ts.isPropertyDeclaration(member) && member.initializer !== undefined) { + const initializer = member.initializer + if (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer)) { + const asynchronous = initializer.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) + || (ts.isFunctionExpression(initializer) && initializer.asteriskToken !== undefined) + if (asynchronous) return unsupported('Async or generator callable properties are unsupported') + members.push({ name, node: member, body: initializer.body, publicSurface, isStatic }) + } else { + if (isStatic) return unsupported('Static property initialization is outside constructor coverage') + initializers.push({ name, node: member, body: initializer, publicSurface: true, isStatic: false }) + } + } + } + + const memberKey = (name: string, isStatic: boolean): string => `${isStatic ? 'static' : 'instance'}:${name}` + const callableMemberKeys = new Set(selected.members.flatMap((member) => { + const name = memberName(member) + const functionProperty = ts.isPropertyDeclaration(member) + && ((member.initializer !== undefined && (ts.isArrowFunction(member.initializer) || ts.isFunctionExpression(member.initializer))) + || (member.type !== undefined && ts.isFunctionTypeNode(member.type))) + if (name === null || (!ts.isMethodDeclaration(member) && !functionProperty)) return [] + const isStatic = ts.canHaveModifiers(member) && (ts.getModifiers(member)?.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) ?? false) + return [memberKey(name, isStatic)] + })) + const reachable = new Set(members.filter((member) => member.publicSurface).map((member) => memberKey(member.name, member.isStatic))) + const hasPrivateCallableBodies = members.some((member) => !member.publicSurface) + const nativeGlobals = new Set(['String', 'Number', 'Boolean', 'parseFloat', 'parseInt', 'isNaN', 'isFinite', + 'encodeURI', 'encodeURIComponent', 'decodeURI', 'decodeURIComponent', 'escape', 'unescape', 'atob', 'btoa']) + const shadowedNativeGlobals = declaredValueBindings(file) + let invalidReason: string | null = null + const localFunctionCache = new Map() + const nestedFunctions = (body: ts.Node): LocalFunctionReachability => { + const cached = localFunctionCache.get(body) + if (cached !== undefined) return cached + const result = inspectLocalFunctions(body) + localFunctionCache.set(body, result) + return result + } + const receiverReferences = (body: ts.Node, isStatic: boolean): Set => { + const names = new Set() + const locals = nestedFunctions(body) + if (locals.reason !== null) invalidReason = locals.reason + const visit = (node: ts.Node): void => { + if (ts.isClassExpression(node) || ts.isClassDeclaration(node)) { + invalidReason = 'Nested classes cannot be attributed to the selected class' + return + } + if (ts.isFunctionDeclaration(node)) { + if (locals.reachable.has(node) && node.body !== undefined) visit(node.body) + return + } + if (ts.isFunctionExpression(node) || ts.isArrowFunction(node)) { + invalidReason = 'Nested closures require a checked callable reachability graph' + return + } + if (ts.isElementAccessExpression(node) && node.expression.kind === ts.SyntaxKind.ThisKeyword) { + invalidReason = 'Computed receiver access cannot establish a frozen private-member reachability graph' + } + if (node.kind === ts.SyntaxKind.ThisKeyword) { + const directMemberReceiver = (ts.isPropertyAccessExpression(node.parent) + || ts.isElementAccessExpression(node.parent)) && node.parent.expression === node + const directFluentReturn = ts.isReturnStatement(node.parent) && node.parent.expression === node + if (!directMemberReceiver && !directFluentReturn) invalidReason = 'Aliased or escaping receivers require a checked private-member reachability graph' + } + const written = assignmentTarget(node) + if (written !== null && ts.isPropertyAccessExpression(written)) { + const receiver = written.expression + const receiverKey = receiver.kind === ts.SyntaxKind.ThisKeyword ? memberKey(written.name.text, isStatic) + : ts.isIdentifier(receiver) && receiver.text === className ? memberKey(written.name.text, true) : null + if (receiverKey !== null && callableMemberKeys.has(receiverKey)) { + invalidReason = 'Mutable callable member identity or captures cannot be represented by a complete state snapshot' + } + if (hasPrototypeAccess(written) || written.name.text === 'constructor') { + invalidReason = 'Prototype and constructor mutations are outside the checked class lifecycle subset' + } + } + if (ts.isDeleteExpression(node)) invalidReason = 'Deleted properties are outside the complete fixed-state snapshot subset' + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression) + && ts.isIdentifier(node.expression.expression) && ['Object', 'Reflect'].includes(node.expression.expression.text) + && ['setPrototypeOf', 'defineProperty', 'defineProperties', 'set', 'deleteProperty'].includes(node.expression.name.text)) { + invalidReason = 'Dynamic property or prototype mutation is outside the checked class lifecycle subset' + } + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression) + && ts.isIdentifier(node.expression.expression) && node.expression.expression.text === 'Object' + && node.expression.name.text === 'assign' && node.arguments[0] !== undefined + && (hasPrototypeAccess(node.arguments[0]) + || (ts.isIdentifier(node.arguments[0]) && node.arguments[0].text === className))) { + invalidReason = 'Dynamic property or prototype mutation is outside the checked class lifecycle subset' + } + if (hasPrivateCallableBodies && ts.isCallExpression(node) && ts.isIdentifier(node.expression)) { + const name = node.expression.text + const checkedLocal = [...locals.reachable].some((declaration) => declaration.name?.text === name) + const nativeGlobal = nativeGlobals.has(name) && !shadowedNativeGlobals.has(name) + if (!checkedLocal && !nativeGlobal) { + invalidReason = 'Unchecked free helper calls cannot establish the selected private callable statement universe' + } + } + if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === 'eval') { + invalidReason = 'Dynamic evaluation cannot establish an original AST statement universe' + } + if (ts.isPropertyAccessExpression(node) && node.expression.kind === ts.SyntaxKind.ThisKeyword) { + names.add(memberKey(node.name.text, isStatic)) + } + if ((ts.isParameter(node) || ts.isVariableDeclaration(node) || ts.isBindingElement(node)) + && ts.isIdentifier(node.name) && node.name.text === className) { + invalidReason = 'A shadowed class binding would make static private-member reachability ambiguous' + } + if (ts.isIdentifier(node) && node.text === className) { + const parent = node.parent + const typeReference = ts.isTypeReferenceNode(parent) && parent.typeName === node + const constructorCall = ts.isNewExpression(parent) && parent.expression === node + const directClassMember = ts.isPropertyAccessExpression(parent) && parent.expression === node + const memberNameOnly = (ts.isPropertyAccessExpression(parent) && parent.name === node) + || (ts.isMethodDeclaration(parent) && parent.name === node) + || (ts.isPropertyAssignment(parent) && parent.name === node) + if (!typeReference && !constructorCall && !directClassMember && !memberNameOnly) { + invalidReason = 'Aliased, escaping or computed class receivers require a checked static private-member reachability graph' + } + } + if (ts.isPropertyAccessExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === className) { + names.add(memberKey(node.name.text, true)) + } + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.name.text === 'globalThis') { + invalidReason = 'A local globalThis binding would shadow native statement counter storage' + } + ts.forEachChild(node, visit) + } + visit(body) + return names + } + for (const initializer of initializers) receiverReferences(initializer.body, initializer.isStatic).forEach((name) => reachable.add(name)) + let changed = true + while (changed) { + changed = false + for (const member of members) { + if (!reachable.has(memberKey(member.name, member.isStatic))) continue + for (const name of receiverReferences(member.body, member.isStatic)) { + if (!reachable.has(name)) { reachable.add(name); changed = true } + } + } + } + if (invalidReason !== null) return unsupported(invalidReason) + if (hasGlobalThisBinding(file)) return unsupported('A globalThis binding would shadow native statement counter storage') + + const inventoried = inspectClassSurface(selected, file) + if (typeof inventoried === 'string') return unsupported(inventoried) + + const statements: ClassStatementDescriptor[] = [] + const insertions: Insertion[] = [] + const marker = (id: string): string => `(globalThis as unknown as Record>)[${JSON.stringify(counterGlobalName)}]?.add(${JSON.stringify(id)});` + const describe = (node: ts.Node, name: string, kind: string): ClassStatementDescriptor => { + const startOffset = node.getStart(file) + const endOffset = node.getEnd() + const start = file.getLineAndCharacterOfPosition(startOffset) + const end = file.getLineAndCharacterOfPosition(endOffset) + const descriptor = { id: `${sourceSha256}:${startOffset}:${endOffset}:${kind}`, memberName: name, kind, + startOffset, endOffset, startLine: start.line + 1, startColumn: start.character, + endLine: end.line + 1, endColumn: end.character } + statements.push(descriptor) + return descriptor + } + const instrumentExpression = (node: ts.Node, name: string, kind: string): void => { + const statement = describe(node, name, kind) + const hit = marker(statement.id).slice(0, -1) + insertions.push({ offset: node.getStart(file), text: `(${hit}, (`, order: 0 }) + insertions.push({ offset: node.getEnd(), text: '))', order: 1 }) + } + const instrumentBody = (body: ts.Node, name: string): void => { + const locals = nestedFunctions(body) + const visit = (node: ts.Node): void => { + if (ts.isFunctionDeclaration(node)) { + if (locals.reachable.has(node) && node.body !== undefined) visit(node.body) + return + } + if (ts.isStatement(node) && executableStatement(node)) { + const statement = describe(node, name, ts.SyntaxKind[node.kind]) + let anchor = node + while (ts.isLabeledStatement(anchor.parent)) anchor = anchor.parent + const parent = anchor.parent + const inList = ts.isBlock(parent) || ts.isCaseClause(parent) || ts.isDefaultClause(parent) + const prefix = `${marker(statement.id)} ` + if (inList) { + // A parenthesized counter must not continue a preceding semicolonless expression. + insertions.push({ offset: anchor.getStart(file), text: `; ${prefix}`, order: 0 }) + } else { + insertions.push({ offset: anchor.getStart(file), text: `{ ${prefix}`, order: 0 }) + insertions.push({ offset: anchor.getEnd(), text: ' }', order: 1 }) + } + } + ts.forEachChild(node, visit) + } + visit(body) + } + for (const initializer of initializers) instrumentExpression(initializer.body, initializer.name, 'PropertyInitializer') + for (const member of members) { + if (!reachable.has(memberKey(member.name, member.isStatic))) continue + if (ts.isBlock(member.body)) instrumentBody(member.body, member.name) + else instrumentExpression(member.body, member.name, 'CallableExpressionBody') + } + statements.sort((left, right) => left.startOffset - right.startOffset || left.endOffset - right.endOffset) + if (statements.length === 0) return unsupported('The selected class has no executable statements') + // For coincident close/open positions, close the earlier statement before opening the next one. + insertions.sort((left, right) => right.offset - left.offset || left.order - right.order) + let source = file.text + for (const insertion of insertions) source = source.slice(0, insertion.offset) + insertion.text + source.slice(insertion.offset) + return { ...base, source, reason: null, instrumentedSha256: hash(source), statements, inventory: inventoried } +} + +function memberName(member: ts.ClassElement): string | null { + const name = member.name + if (name !== undefined && (ts.isIdentifier(name) || ts.isPrivateIdentifier(name) || ts.isStringLiteral(name))) return name.text + return null +} + +function executableStatement(node: ts.Statement): boolean { + if (ts.isBlock(node) || ts.isEmptyStatement(node) || ts.isLabeledStatement(node) + || ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node) || ts.isInterfaceDeclaration(node) + || ts.isTypeAliasDeclaration(node)) return false + if (ts.isVariableStatement(node) && node.declarationList.declarations.every((declaration) => declaration.initializer === undefined)) return false + const directive = ts.isExpressionStatement(node) && ts.isStringLiteral(node.expression) + return !directive +} + +function hasGlobalThisBinding(file: ts.SourceFile): boolean { + let found = false + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node) && node.text === 'globalThis') { + const parent = node.parent + const declaration = (ts.isParameter(parent) || ts.isVariableDeclaration(parent) || ts.isBindingElement(parent) + || ts.isImportSpecifier(parent) || ts.isImportClause(parent) || ts.isNamespaceImport(parent) + || ts.isFunctionDeclaration(parent) || ts.isClassDeclaration(parent)) && parent.name === node + if (declaration) found = true + } + ts.forEachChild(node, visit) + } + visit(file) + return found +} + +function hash(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex') } + + +interface LocalFunctionReachability { reachable: Set; reason: string | null } + +/** Checked local subset: uniquely named declarations in the member body, used only by direct calls. */ +function inspectLocalFunctions(body: ts.Node): LocalFunctionReachability { + const reachable = new Set() + let reason: string | null = null + const declarations = ts.isBlock(body) ? body.statements.filter(ts.isFunctionDeclaration) : [] + const byName = new Map() + for (const declaration of declarations) { + if (declaration.name === undefined || declaration.body === undefined) { + reason = 'Local callable declarations must have an exact name and body' + continue + } + const asynchronous = declaration.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) + || declaration.asteriskToken !== undefined + if (asynchronous || byName.has(declaration.name.text)) { + reason = 'Async, generator or overloaded local callables are outside the checked reachability subset' + continue + } + byName.set(declaration.name.text, declaration) + } + const scan = (node: ts.Node, localOwner: ts.FunctionDeclaration | null): void => { + if (ts.isFunctionDeclaration(node)) { + if (!declarations.includes(node)) reason = 'Nested local function scopes require a checked callable reachability graph' + return + } + if (ts.isFunctionExpression(node) || ts.isArrowFunction(node)) { + reason = 'Nested closures require a checked callable reachability graph' + return + } + if (localOwner !== null && node.kind === ts.SyntaxKind.ThisKeyword) { + reason = 'Direct-called local functions cannot use a class receiver implicitly' + } + if (ts.isIdentifier(node) && byName.has(node.text)) { + const parent = node.parent + const directCall = ts.isCallExpression(parent) && parent.expression === node + const declarationName = ts.isFunctionDeclaration(parent) && parent.name === node + if (directCall) reachable.add(byName.get(node.text)!) + else if (!declarationName) reason = 'Aliased, escaped or shadowed local callables require a checked reachability graph' + } + ts.forEachChild(node, (child) => scan(child, localOwner)) + } + scan(body, null) + const scanned = new Set() + while (scanned.size < reachable.size) { + for (const declaration of reachable) { + if (scanned.has(declaration)) continue + scanned.add(declaration) + if (declaration.body !== undefined) scan(declaration.body, declaration) + } + } + return { reachable, reason } +} + + +function inspectClassSurface(selected: ts.ClassDeclaration, file: ts.SourceFile): ClassSurfaceInventory | string { + const constructorParameterNames: string[] = [] + const publicMembers: ClassCallableDescriptor[] = [] + const stateFields = new Map() + const callableOrAccessorNames = new Set() + let reason: string | null = null + const parameterNames = (parameters: ts.NodeArray): string[] => parameters.map((parameter) => { + if (!ts.isIdentifier(parameter.name) || parameter.dotDotDotToken !== undefined) { + reason = 'Destructured and rest class parameters are outside the frozen harness subset' + return '' + } + return parameter.name.text + }) + for (const member of selected.members) { + const modifiers = ts.canHaveModifiers(member) ? ts.getModifiers(member) ?? [] : [] + const isStatic = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + if (ts.isConstructorDeclaration(member)) { + if (member.body === undefined) continue + constructorParameterNames.push(...parameterNames(member.parameters)) + for (const parameter of member.parameters) { + const parameterProperty = parameter.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.PublicKeyword + || modifier.kind === ts.SyntaxKind.PrivateKeyword || modifier.kind === ts.SyntaxKind.ProtectedKeyword + || modifier.kind === ts.SyntaxKind.ReadonlyKeyword) + if (parameterProperty && ts.isIdentifier(parameter.name)) { + stateFields.set(parameter.name.text, { name: parameter.name.text, isEcmaPrivate: false, + typeText: parameter.type?.getText(file) ?? null }) + } + } + continue + } + const name = memberName(member) + if (name === null) continue + const isPrivate = name.startsWith('#') || modifiers.some((modifier) => + modifier.kind === ts.SyntaxKind.PrivateKeyword || modifier.kind === ts.SyntaxKind.ProtectedKeyword) + const callable = ts.isPropertyDeclaration(member) + ? (member.initializer !== undefined && (ts.isArrowFunction(member.initializer) || ts.isFunctionExpression(member.initializer)) + ? member.initializer : member.type !== undefined && ts.isFunctionTypeNode(member.type) ? member.type : null) + : ts.isMethodDeclaration(member) || ts.isGetAccessorDeclaration(member) || ts.isSetAccessorDeclaration(member) ? member : null + if (callable !== null) { + if (!isStatic) callableOrAccessorNames.add(name) + const bodylessMethod = ts.isMethodDeclaration(callable) && callable.body === undefined + if (isPrivate || bodylessMethod) continue + const kind = ts.isGetAccessorDeclaration(member) ? 'GETTER' : ts.isSetAccessorDeclaration(member) ? 'SETTER' + : ts.isPropertyDeclaration(member) ? 'FUNCTION_FIELD' : 'METHOD' + publicMembers.push({ name, kind, isStatic, parameterNames: parameterNames(callable.parameters), + parameterTypeTexts: callable.parameters.map((parameter) => parameter.type?.getText(file) ?? null), + resultTypeText: callable.type?.getText(file) ?? null }) + } else if (ts.isPropertyDeclaration(member) && !isStatic) { + stateFields.set(name, { name, isEcmaPrivate: name.startsWith('#'), typeText: member.type?.getText(file) ?? null }) + } + } + const visitWrites = (node: ts.Node): void => { + if (node !== selected && (ts.isClassDeclaration(node) || ts.isClassExpression(node))) return + let written: ts.Node | undefined + if (ts.isBinaryExpression(node) && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment + && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment) written = node.left + if (ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) { + if (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken) written = node.operand + } + if (written !== undefined && ts.isPropertyAccessExpression(written) && written.expression.kind === ts.SyntaxKind.ThisKeyword) { + const name = written.name.text + if (!callableOrAccessorNames.has(name) && !stateFields.has(name)) { + stateFields.set(name, { name, isEcmaPrivate: name.startsWith('#'), typeText: null }) + } + } + ts.forEachChild(node, visitWrites) + } + for (const member of selected.members) { + const isStatic = ts.canHaveModifiers(member) && ts.getModifiers(member)?.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + if (!isStatic) visitWrites(member) + } + if (reason !== null) return reason + return { constructorParameterNames, publicMembers, + instanceDataFields: [...stateFields.values()].sort((left, right) => left.name.localeCompare(right.name)), + dataClasses: inspectDataClasses(file) } +} + + +function assignmentTarget(node: ts.Node): ts.Node | null { + if (ts.isBinaryExpression(node) && node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment + && node.operatorToken.kind <= ts.SyntaxKind.LastAssignment) return node.left + if ((ts.isPrefixUnaryExpression(node) || ts.isPostfixUnaryExpression(node)) + && (node.operator === ts.SyntaxKind.PlusPlusToken || node.operator === ts.SyntaxKind.MinusMinusToken)) return node.operand + return null +} + +function hasPrototypeAccess(node: ts.Node): boolean { + let found = false + const visit = (part: ts.Node): void => { + if (ts.isPropertyAccessExpression(part) && ['prototype', '__proto__'].includes(part.name.text)) found = true + if (ts.isElementAccessExpression(part) && ts.isStringLiteral(part.argumentExpression) + && ['prototype', '__proto__'].includes(part.argumentExpression.text)) found = true + ts.forEachChild(part, visit) + } + visit(node) + return found +} + + +function declaredValueBindings(file: ts.SourceFile): Set { + const names = new Set() + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node)) { + const parent = node.parent + const valueBinding = (ts.isParameter(parent) || ts.isVariableDeclaration(parent) || ts.isBindingElement(parent) + || ts.isImportSpecifier(parent) || ts.isImportClause(parent) || ts.isNamespaceImport(parent) + || ts.isFunctionDeclaration(parent) || ts.isClassDeclaration(parent)) && parent.name === node + if (valueBinding) names.add(node.text) + } + ts.forEachChild(node, visit) + } + visit(file) + return names +} + + +/** Source identity is the same captured module; registries cannot admit arbitrary instance prototypes. */ +function inspectDataClasses(file: ts.SourceFile): ClassDataClassDescriptor[] { + const candidates = file.statements.filter(ts.isClassDeclaration) + const typeAliases = new Map(file.statements.filter(ts.isTypeAliasDeclaration).map((node) => [node.name.text, node.type])) + const valueNames = declaredValueBindings(file) + const safeName = (name: string): boolean => /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(name) + && !['__proto__', 'prototype', 'constructor', '__usvmCallsClassIdentity'].includes(name) + const callableType = (type: ts.TypeNode | undefined, parameters: Map, seen = new Set()): boolean => { + if (type === undefined) return false + if (ts.isFunctionTypeNode(type) || ts.isConstructorTypeNode(type) || ts.isTypeQueryNode(type)) return true + if (ts.isTypeReferenceNode(type) && ts.isIdentifier(type.typeName)) { + const name = type.typeName.text + if (name === 'Function' || seen.has(name)) return true + const referenced = typeAliases.get(name) ?? parameters.get(name) + if (referenced !== undefined) { + const nestedSeen = new Set(seen) + nestedSeen.add(name) + if (callableType(referenced, parameters, nestedSeen)) return true + } + } + let found = false + ts.forEachChild(type, (node) => { + if (ts.isTypeNode(node) && callableType(node, parameters, seen)) found = true + if (ts.isCallSignatureDeclaration(node) || ts.isConstructSignatureDeclaration(node)) found = true + }) + return found + } + const pureValue = (node: ts.Expression, inputs: Set): boolean => { + if (ts.isParenthesizedExpression(node) || ts.isAsExpression(node) || ts.isTypeAssertionExpression(node) + || ts.isSatisfiesExpression(node) || ts.isNonNullExpression(node)) return pureValue(node.expression, inputs) + if (ts.isStringLiteral(node) || ts.isNumericLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node) + || node.kind === ts.SyntaxKind.TrueKeyword || node.kind === ts.SyntaxKind.FalseKeyword + || node.kind === ts.SyntaxKind.NullKeyword) return true + if (ts.isIdentifier(node)) return inputs.has(node.text) + || (['undefined', 'NaN', 'Infinity'].includes(node.text) && !valueNames.has(node.text)) + if (ts.isPrefixUnaryExpression(node) + && [ts.SyntaxKind.PlusToken, ts.SyntaxKind.MinusToken, ts.SyntaxKind.ExclamationToken, ts.SyntaxKind.TildeToken].includes(node.operator)) { + // Coercing an unknown constructor input can invoke valueOf or Symbol.toPrimitive. + return pureValue(node.operand, new Set()) + } + if (ts.isArrayLiteralExpression(node)) return node.elements.every((element) => + !ts.isSpreadElement(element) && !ts.isOmittedExpression(element) && pureValue(element, inputs)) + if (ts.isObjectLiteralExpression(node)) return node.properties.every((property) => { + if (!ts.isPropertyAssignment(property)) return false + const name = property.name + const literalName = ts.isIdentifier(name) || ts.isStringLiteral(name) + return literalName && safeName(name.text) && pureValue(property.initializer, inputs) + }) + return false + } + const result: ClassDataClassDescriptor[] = [] + for (const candidate of candidates) { + const modifiers = ts.getModifiers(candidate) ?? [] + const exported = modifiers.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword) + const disallowedClassModifier = modifiers.some((modifier) => [ts.SyntaxKind.DefaultKeyword, + ts.SyntaxKind.AbstractKeyword, ts.SyntaxKind.DeclareKeyword].includes(modifier.kind)) + const exportName = candidate.name?.text + if (!exported || disallowedClassModifier || exportName === undefined || !safeName(exportName)) continue + const decorated = (ts.getDecorators(candidate)?.length ?? 0) > 0 + if (decorated || (candidate.heritageClauses?.length ?? 0) > 0 || !uniqueDirectClassExport(file, candidate, exportName)) continue + const parameters = new Map((candidate.typeParameters ?? []).map((parameter) => [parameter.name.text, parameter.constraint])) + const fields = new Set() + let eligible = true + let constructor: ts.ConstructorDeclaration | undefined + for (const member of candidate.members) { + if (ts.isSemicolonClassElement(member)) continue + const memberModifiers = ts.canHaveModifiers(member) ? ts.getModifiers(member) ?? [] : [] + const memberDecorated = ts.canHaveDecorators(member) && (ts.getDecorators(member)?.length ?? 0) > 0 + const isStatic = memberModifiers.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) + if (memberDecorated || isStatic) { eligible = false; break } + if (ts.isConstructorDeclaration(member)) { + if (member.body === undefined) continue + if (constructor !== undefined) { eligible = false; break } + constructor = member + continue + } + if (!ts.isPropertyDeclaration(member) || member.name === undefined || ts.isPrivateIdentifier(member.name) + || (!ts.isIdentifier(member.name) && !ts.isStringLiteral(member.name))) { eligible = false; break } + const name = member.name.text + const valueInitializer = member.initializer + const safeInitializer = valueInitializer === undefined || pureValue(valueInitializer, new Set()) + if (!safeName(name) || fields.has(name) || callableType(member.type, parameters) || !safeInitializer) { + eligible = false + break + } + fields.add(name) + } + if (!eligible) continue + if (constructor !== undefined) { + const inputs = new Set() + for (const parameter of constructor.parameters) { + if (!ts.isIdentifier(parameter.name) || parameter.dotDotDotToken !== undefined + || inputs.has(parameter.name.text) || callableType(parameter.type, parameters)) { eligible = false; break } + inputs.add(parameter.name.text) + if (parameter.initializer !== undefined && !pureValue(parameter.initializer, inputs)) { eligible = false; break } + const parameterProperty = parameter.modifiers?.some((modifier) => [ts.SyntaxKind.PublicKeyword, + ts.SyntaxKind.PrivateKeyword, ts.SyntaxKind.ProtectedKeyword, ts.SyntaxKind.ReadonlyKeyword].includes(modifier.kind)) + if (parameterProperty) { + const name = parameter.name.text + if (!safeName(name) || fields.has(name)) { eligible = false; break } + fields.add(name) + } + } + if (!eligible) continue + for (const statement of constructor.body!.statements) { + if (ts.isEmptyStatement(statement)) continue + if (!ts.isExpressionStatement(statement) || !ts.isBinaryExpression(statement.expression) + || statement.expression.operatorToken.kind !== ts.SyntaxKind.EqualsToken) { eligible = false; break } + const assignment = statement.expression + const target = assignment.left + const ownField = ts.isPropertyAccessExpression(target) && target.expression.kind === ts.SyntaxKind.ThisKeyword + && fields.has(target.name.text) + if (!ownField || !pureValue(assignment.right, inputs)) { eligible = false; break } + } + } + if (!eligible) continue + // An exported class binding must retain the source-declared constructor identity. + const modified = classBindingModified(file, exportName) + if (modified) continue + result.push({ exportName, fields: [...fields].sort() }) + } + return result.sort((left, right) => left.exportName.localeCompare(right.exportName)) +} + +function uniqueDirectClassExport(file: ts.SourceFile, candidate: ts.ClassDeclaration, name: string): boolean { + const bindsName = (binding: ts.BindingName): boolean => ts.isIdentifier(binding) ? binding.text === name + : binding.elements.some((element) => !ts.isOmittedExpression(element) && bindsName(element.name)) + let declarations = 0 + for (const statement of file.statements) { + if ((ts.isClassDeclaration(statement) || ts.isFunctionDeclaration(statement) || ts.isEnumDeclaration(statement) + || ts.isModuleDeclaration(statement)) && statement.name?.getText(file) === name) declarations++ + if (ts.isVariableStatement(statement) && statement.declarationList.declarations.some((declaration) => + bindsName(declaration.name))) declarations++ + if (ts.isImportEqualsDeclaration(statement) && !statement.isTypeOnly && statement.name.text === name) declarations++ + if (ts.isImportDeclaration(statement) && statement.importClause !== undefined && !statement.importClause.isTypeOnly) { + const clause = statement.importClause + const bindings = clause.namedBindings + const defaultBinding = clause.name?.text === name + const namespaceBinding = bindings !== undefined && ts.isNamespaceImport(bindings) && bindings.name.text === name + const namedBinding = bindings !== undefined && ts.isNamedImports(bindings) + && bindings.elements.some((element) => !element.isTypeOnly && element.name.text === name) + if (defaultBinding || namespaceBinding || namedBinding) declarations++ + } + const conflictingExport = ts.isExportDeclaration(statement) && !statement.isTypeOnly + && statement.exportClause !== undefined && ts.isNamedExports(statement.exportClause) + && statement.exportClause.elements.some((element) => !element.isTypeOnly && element.name.text === name) + if (conflictingExport) return false + } + return declarations === 1 && candidate.name?.text === name +} + +function classBindingModified(file: ts.SourceFile, name: string): boolean { + const writesName = (target: ts.Node): boolean => { + if (ts.isIdentifier(target)) return target.text === name + if (ts.isParenthesizedExpression(target)) return writesName(target.expression) + if (ts.isArrayLiteralExpression(target)) return target.elements.some(writesName) + if (ts.isObjectLiteralExpression(target)) return target.properties.some((property) => { + if (ts.isShorthandPropertyAssignment(property)) return property.name.text === name + if (ts.isPropertyAssignment(property)) return writesName(property.initializer) + return ts.isSpreadAssignment(property) && writesName(property.expression) + }) + if (ts.isSpreadElement(target)) return writesName(target.expression) + if (ts.isBinaryExpression(target) && target.operatorToken.kind === ts.SyntaxKind.EqualsToken) return writesName(target.left) + return false + } + let modified = false + const visit = (node: ts.Node): void => { + const target = assignmentTarget(node) + if (target !== null && writesName(target)) modified = true + ts.forEachChild(node, visit) + } + visit(file) + return modified +} diff --git a/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts b/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts index 60d867772f..27513b6712 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts @@ -253,7 +253,9 @@ function safeModuleStatement(statement: ts.Statement, source: ts.SourceFile): bo } if (ts.isExportAssignment(statement)) return !statement.isExportEquals && safeInitializer(statement.expression, source) if (ts.isClassDeclaration(statement)) { - if (statement.heritageClauses?.length || ts.getDecorators(statement)?.length) return false + const extendsRuntimeBase = statement.heritageClauses?.some(clause => clause.token === ts.SyntaxKind.ExtendsKeyword) ?? false + const decorated = (ts.getDecorators(statement)?.length ?? 0) > 0 + if (extendsRuntimeBase || decorated) return false return statement.members.every(member => !ts.isClassStaticBlockDeclaration(member) && !(member.name && ts.isComputedPropertyName(member.name)) && !(ts.canHaveDecorators(member) && ts.getDecorators(member)?.length) diff --git a/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts b/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts index 5869fe5313..54e6e76795 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/source-inspector-cli.ts @@ -3,6 +3,7 @@ import ts from 'typescript' import { instrumentSourceReplay } from './source-replay.js' import { inspectSourceCaptures } from './source-captures.js' import { inspectLocalSourceClosure } from './local-source-closure.js' +import { instrumentClassCoverage } from './class-coverage-source.js' const [operation, sourcePath, ...arguments_] = process.argv.slice(2) if (sourcePath === undefined) throw new Error('Expected a TypeScript source path') @@ -20,7 +21,11 @@ if (operation === 'inspect-local-closure') { }).diagnostics?.filter((diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error) ?? [] if (diagnostics.length > 0) throw new Error(ts.flattenDiagnosticMessageText(diagnostics[0]?.messageText ?? '', '\n')) - if (operation === 'inspect-captures') { + if (operation === 'instrument-class-coverage') { + const className = arguments_[0] + if (className === undefined) throw new Error('Expected a target class name') + process.stdout.write(JSON.stringify(instrumentClassCoverage(sourceFile, className))) + } else if (operation === 'inspect-captures') { const [exportName, namesText] = arguments_ if (exportName === undefined || namesText === undefined) throw new Error('Expected export and capture names') const names: unknown = JSON.parse(namesText) diff --git a/usvm-ts-fast-check/fast-check-adapter/test/class-coverage-source.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/class-coverage-source.test.ts new file mode 100644 index 0000000000..d2a12330dc --- /dev/null +++ b/usvm-ts-fast-check/fast-check-adapter/test/class-coverage-source.test.ts @@ -0,0 +1,396 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { runInNewContext } from 'node:vm' +import ts from 'typescript' +import { instrumentClassCoverage } from '../src/class-coverage-source.js' + +function instrument(source: string, className = 'Target') { + const file = ts.createSourceFile('target.ts', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS) + return instrumentClassCoverage(file, className) +} + +function load(source: string, counter: string | null) { + const exports: Record = {} + const sandbox: Record = { exports, module: { exports } } + if (counter !== null) sandbox[counter] = new Set() + const js = ts.transpileModule(source, { compilerOptions: { + target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS, + } }).outputText + runInNewContext(js, sandbox) + return { sandbox, Target: exports['Target'] as new (...args: unknown[]) => Record unknown> } +} + +test('freezes distinct original AST statements on one line and only native hits', () => { + const source = 'export class Target { run(x: boolean) { if (x) return 1; return 2; } }' + const result = instrument(source) + assert.equal(result.reason, null) + assert.equal(result.statements.length, 3) + assert.equal(new Set(result.statements.map((statement) => statement.id)).size, 3) + assert.deepEqual(result.statements.map((statement) => statement.kind), ['IfStatement', 'ReturnStatement', 'ReturnStatement']) + + const runtime = load(result.source!, result.counterGlobalName) + const receiver = new runtime.Target() + assert.equal(receiver['run']!(true), 1) + const covered = Array.from(runtime.sandbox[result.counterGlobalName] as Set) + assert.deepEqual(covered, result.statements.slice(0, 2).map((statement) => statement.id)) + assert.equal(covered.includes(result.statements[2]!.id), false) + assert.deepEqual(instrument(source), result) +}) + +test('counts initialization and reachable private bodies while excluding foreign and dead code', () => { + const source = ` + interface TypeOnly { value: number } + export class Foreign { run() { return 9; } } + export class Target { + value = 0; + constructor(x: number) { this.value = this.helper(x); } + run() { return this.value; } + private helper(x: number) { if (x > 0) return x; return -x; } + private dead() { return 99; } + } + export function wrapper() { return 100; } + ` + const result = instrument(source) + assert.equal(result.reason, null) + assert.deepEqual(new Set(result.statements.map((statement) => statement.memberName)), + new Set(['value', 'constructor', 'run', 'helper'])) + assert.equal(result.statements[0]!.kind, 'PropertyInitializer') + + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + assert.equal(new original.Target(-2)['run']!(), new instrumented.Target(-2)['run']!()) + const covered = Array.from(instrumented.sandbox[result.counterGlobalName] as Set) + assert.equal(covered.length, 5) + assert.equal(covered.every((id) => result.statements.some((statement) => statement.id === id)), true) +}) + +test('preserves labelled loops, unbraced nested controls and else binding', () => { + const source = `export class Target { run(x: number) { + let value = 0; + outer: for (let i = 0; i < 4; i++) { + if (i === x) continue outer; + if (i < x) if (x > 1) value += i; else value -= i; + } + return value; + } }` + const result = instrument(source) + assert.equal(result.reason, null) + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + + for (const value of [-1, 0, 1, 2, 3, 8]) { + assert.equal(new instrumented.Target()['run']!(value), new original.Target()['run']!(value)) + } + assert.equal(result.statements.some((statement) => statement.kind === 'ContinueStatement'), true) + assert.equal(result.statements.some((statement) => statement.kind === 'LabeledStatement'), false) +}) + +test('supports private accessors and expression callable properties', () => { + const result = instrument(`export class Target { + private value = 2; + private get doubled() { return this.value * 2; } + run = () => this.doubled; + }`) + assert.equal(result.reason, null) + assert.deepEqual(result.statements.map((statement) => statement.kind), + ['PropertyInitializer', 'ReturnStatement', 'CallableExpressionBody']) + const runtime = load(result.source!, result.counterGlobalName) + assert.equal(new runtime.Target()['run']!(), 4) + assert.equal((Array.from(runtime.sandbox[result.counterGlobalName] as Set)).length, 3) +}) + +test('retains explicit unsupported dynamic reachability and closure outcomes', () => { + assert.match(instrument('export class Target { run(name: string) { return this[name]; } }').reason!, /Computed receiver/) + assert.match(instrument('export class Target { run() { return [1].map(x => x + 1); } }').reason!, /Nested closures/) + assert.match(instrument('export class Target { run() { const alias = this; return alias.hidden(); } private hidden() { return 1; } }').reason!, /Aliased or escaping/) + assert.match(instrument('export class Target { run() { return eval("this.hidden()"); } private hidden() { return 1; } }').reason!, /Dynamic evaluation/) + assert.match(instrument('export class Target extends Base { run() { return 1; } }').reason!, /Inherited/) + assert.match(instrument('export class Target { static value = 1; static run() { return 1; } }').reason!, /Static/) + assert.match(instrument('export class Target { run(globalThis: unknown) { return 1; } }').reason!, /globalThis binding/) +}) + + +test('includes static private helpers reachable through the class or static receiver', () => { + const source = `export class Target { + static run(x: number) { return this.helper(x) + Target.helper(x); } + private static helper(x: number) { if (x > 0) return x; return -x; } + private helper() { return 99; } + private static dead() { return 100; } + }` + const result = instrument(source) + assert.equal(result.reason, null) + assert.equal(result.statements.length, 4) + assert.equal(result.statements.some((statement) => source.slice(statement.startOffset, statement.endOffset) === 'return 99;'), false) + + const runtime = load(result.source!, result.counterGlobalName) + const target = runtime.Target as unknown as Record unknown> + assert.equal(target['run']!(-2), 4) + const covered = Array.from(runtime.sandbox[result.counterGlobalName] as Set) + assert.equal(new Set(covered).size, 3) +}) + + +test('includes exact bodies of direct-called local functions without adding unused local bodies', () => { + const source = `export class Target { + static run(value: number) { + function pad(input: number) { + let result = String(input); + while (result.length < 2) result = "0" + result; + return result; + } + function dead() { return 99; } + let declarationOnly: string; + return pad(value); + } + }` + const result = instrument(source) + assert.equal(result.reason, null) + assert.equal(result.statements.length, 5) + assert.equal(result.statements.some((statement) => source.slice(statement.startOffset, statement.endOffset) === 'return 99;'), false) + + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + const originalClass = original.Target as unknown as Record unknown> + const instrumentedClass = instrumented.Target as unknown as Record unknown> + assert.equal(instrumentedClass['run']!(1), originalClass['run']!(1)) + assert.equal(new Set(Array.from(instrumented.sandbox[result.counterGlobalName] as Set)).size, 5) + assert.match(instrument('export class Target { run() { function local() { return 1; } return [1].map(local); } }').reason!, /escaped/) +}) + + +test('freezes the complete constructor callable and receiver data inventory', () => { + const source = `export class Target { + private flag = false; + constructor(public value: number) { this.extra = 1; } + run(x: number) { this.value += x; return this; } + get current() { return this.value; } + set current(value: number) { this.value = value; } + callback = (input: number) => input + 1; + static utility(value: number) { return value + 1; } + }` + const result = instrument(source) + assert.equal(result.reason, null) + assert.deepEqual(result.inventory!.constructorParameterNames, ['value']) + assert.deepEqual(result.inventory!.instanceDataFields.map((field) => field.name), ['extra', 'flag', 'value']) + assert.deepEqual(result.inventory!.publicMembers.map((member) => [member.name, member.kind, member.isStatic, member.parameterNames]), + [['run', 'METHOD', false, ['x']], ['current', 'GETTER', false, []], ['current', 'SETTER', false, ['value']], + ['callback', 'FUNCTION_FIELD', false, ['input']], ['utility', 'METHOD', true, ['value']]]) +}) + + +test('rejects mutable callable identity and dynamic lifecycle state without rejecting stable initializers', () => { + assert.equal(instrument('export class Target { run = () => 1; other() { return this.run(); } }').reason, null) + assert.match(instrument('export class Target { run = () => 1; other() { this.run = this.other; return 1; } }').reason!, /Mutable callable/) + assert.match(instrument('export class Target { run() { this.run = this.other; return 1; } other() { return 2; } }').reason!, /Mutable callable/) + assert.match(instrument('export class Target { static run() { Target.run = Target.other; return 1; } static other() { return 2; } }').reason!, /Mutable callable/) + assert.match(instrument('export class Target { value = 1; run() { delete this.value; return 1; } }').reason!, /Deleted properties/) + assert.match(instrument('export class Target { run() { Target.prototype.run = Target.prototype.other; return 1; } other() { return 2; } }').reason!, /Prototype/) + assert.match(instrument('export class Target { run() { Object.defineProperty(Target.prototype, "extra", {value: 1}); return 1; } }').reason!, /Dynamic property/) + assert.match(instrument('export class Target { static run() { const Alias = Target; return Alias.hidden(); } private static hidden() { return 1; } }').reason!, /Aliased, escaping or computed class/) + assert.match(instrument('export class Target { static run(key: string) { return Target[key](); } private static hidden() { return 1; } }').reason!, /Aliased, escaping or computed class/) + assert.match(instrument('export class Target { static run() { Object.assign(Target.prototype, {extra: 1}); return 1; } }').reason!, /Dynamic property/) + assert.match(instrument('export class Target { static run() { Reflect.set(Target.prototype, "extra", 1); return 1; } }').reason!, /Dynamic property/) +}) + + +test('rejects unchecked free helpers that could enter private class bodies', () => { + const unsafe = instrument('function bridge(){return Target.hidden()} export class Target{static run(){return bridge()} private static hidden(){return 1}}') + assert.equal(unsafe.source, null) + assert.match(unsafe.reason!, /Unchecked free helper/) + assert.equal(instrument('export class Target { static run(value: number) { return this.helper(String(value)); } private static helper(value: string) { return value.length; } }').reason, null) + assert.equal(instrument('export class Target { static run(value: number) { function local(input: number) { return String(input); } return this.helper(local(value)); } private static helper(value: string) { return value.length; } }').reason, null) + assert.match(instrument('function String(value: unknown) { return Target.hidden(); } export class Target { static run() { return String(1); } private static hidden() { return 2; } }').reason!, /Unchecked free helper/) +}) + +test('preserves automatic semicolon insertion between assignments and statement counters', () => { + const source = `export class Target { + private size!: number + private buckets!: number[][] + private readonly loadFactor = 0.75 + + constructor() { + this.clear() + } + + clear() { + this.size = 0 + this.initializeBuckets(3) + } + + private initializeBuckets(amount: number) { + this.buckets = [] + for (let i = 0; i < amount; i++) { + this.buckets.push([]) + } + } + + run() { + this.size = 0 + if (this.size === 0) + this.initializeBuckets(3) + else + this.buckets = [] + return this.buckets.length + } + }` + const result = instrument(source) + assert.equal(result.reason, null) + + const original = load(source, null) + const instrumented = load(result.source!, result.counterGlobalName) + const originalReceiver = new original.Target() + const instrumentedReceiver = new instrumented.Target() + + assert.equal(instrumentedReceiver['run']!(), originalReceiver['run']!()) + assert.equal(instrumentedReceiver['run']!(), 3) + const nativeHits = Array.from(instrumented.sandbox[result.counterGlobalName] as Set) + const coveredMemberNames = result.statements.filter((statement) => nativeHits.includes(statement.id)) + .map((statement) => statement.memberName) + assert.equal(coveredMemberNames.includes('loadFactor'), true) + assert.equal(coveredMemberNames.includes('constructor'), true) + assert.equal(coveredMemberNames.includes('initializeBuckets'), true) +}) + + +test('inventories same-source exported generic data classes with complete own fields', () => { + const source = ` + export class Target { run() { return 1; } } + export class MapEntry { + key: K + value: V + + constructor(key: K, value: V) { + this.key = key + this.value = value + } + } + export class Coordinates { + readonly label = 'point'; + readonly negative = -1; + constructor(public x: number, private y: number = 0) {} + } + ` + const result = instrument(source) + + assert.equal(result.reason, null) + assert.deepEqual(result.inventory!.dataClasses, [ + { exportName: 'Coordinates', fields: ['label', 'negative', 'x', 'y'] }, + { exportName: 'MapEntry', fields: ['key', 'value'] }, + ]) + assert.deepEqual(result.statements.map((statement) => statement.memberName), ['run']) +}) + + +test('excludes non-data companion classes without rejecting unrelated target coverage', () => { + const companions = [ + 'export class Companion { value = 1; method() {} }', + 'export class Companion { get value() { return 1; } }', + 'export class Companion extends Base { value = 1; }', + 'export class Companion implements Shape { value = 1; }', + 'export class Companion { static value = 1; }', + 'export class Companion { static {} }', + 'export class Companion { ["value"] = 1; }', + 'export class Companion { #value = 1; }', + 'export class Companion { constructor: number = 1; }', + 'export class Companion { __usvmCallsClassIdentity = "fake"; }', + 'export class Companion { callback: () => number; }', + 'type Callback = () => number; export class Companion { callback: Callback; }', + 'export class Companion number> { callback: T; }', + 'export class Companion { callback = () => 1; }', + 'export class Companion { value = effect(); }', + 'export class Companion { value: number; constructor(value: number) { this.value = effect(value); } }', + 'export class Companion { amount: number; constructor(value: unknown) { this.amount = +value; } }', + 'export class Companion { amount: number; constructor(value: unknown) { this.amount = -value; } }', + 'export class Companion { amount: number; constructor(value: unknown) { this.amount = ~value; } }', + 'export class Companion { value: number; constructor(value: number) { this.extra = value; } }', + 'export class Companion { constructor(callback: () => number) {} }', + 'export abstract class Companion { value = 1; }', + 'export default class Companion { value = 1; }', + 'export declare class Companion { value: number; }', + '@decorate export class Companion { value = 1; }', + ] + + for (const companion of companions) { + const result = instrument(`export class Target { run() { return 1; } } ${companion}`) + + assert.equal(result.reason, null, companion) + assert.deepEqual(result.inventory!.dataClasses, [], companion) + } +}) + + +test('excludes companion exports with shadowed or re-exported constructor identity', () => { + const conflicts = [ + 'export { Foreign as Companion } from "./foreign";', + 'const Companion = replacement;', + 'const { original: Companion } = replacement;', + 'const [Companion] = replacement;', + 'import Companion from "./foreign";', + 'import { Foreign as Companion } from "./foreign";', + 'import * as Companion from "./foreign";', + 'import Companion = require("./foreign");', + ] + + for (const conflict of conflicts) { + const result = instrument(`export class Target { run() { return 1; } } + export class Companion { value = 1; } ${conflict}`) + + assert.equal(result.reason, null, conflict) + assert.deepEqual(result.inventory!.dataClasses, [], conflict) + } +}) + + +test('excludes reassigned companion constructors including destructuring writes', () => { + const mutations = [ + 'Companion = replacement;', + 'Companion++;', + '[Companion] = replacement;', + '({ Companion } = replacement);', + '({ original: Companion } = replacement);', + '[Companion = replacement] = replacements;', + '[...Companion] = replacements;', + ] + + for (const mutation of mutations) { + const result = instrument(`export class Target { run() { return 1; } } + export class Companion { value = 1; } ${mutation}`) + + assert.equal(result.reason, null, mutation) + assert.deepEqual(result.inventory!.dataClasses, [], mutation) + } +}) + + +test('stores only distinct statement hits across long loops and repeated calls', () => { + const source = `export class Target { + run(amount: number) { + let total = 0; + try { + for (let index = 0; index < amount; index++) { + total += index; + } + } catch { + return -1; + } + return total; + } + }` + const result = instrument(source) + assert.equal(result.reason, null) + const original = load(source, null) + const runtime = load(result.source!, result.counterGlobalName) + const receiver = new runtime.Target() + + assert.equal(receiver['run']!(10_000), new original.Target()['run']!(10_000)) + const firstHits = Array.from(runtime.sandbox[result.counterGlobalName] as Iterable) + assert.equal(firstHits.length, new Set(firstHits).size) + assert.equal(firstHits.length <= result.statements.length, true) + assert.equal(firstHits.every((id) => result.statements.some((statement) => statement.id === id)), true) + const caughtReturn = result.statements.find((statement) => source.slice(statement.startOffset, statement.endOffset) === 'return -1;')! + assert.equal(firstHits.includes(caughtReturn.id), false) + + for (let iteration = 0; iteration < 20; iteration++) receiver['run']!(10_000) + const repeatedHits = Array.from(runtime.sandbox[result.counterGlobalName] as Iterable) + assert.deepEqual(repeatedHits, firstHits) +}) diff --git a/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts index 524e77d848..eaf07bad7e 100644 --- a/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts +++ b/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts @@ -158,7 +158,6 @@ test('binding inspection does not read ignored type imports outside the runtime }) }) - test('single-source captures retain their dedicated validation route', () => { workspace({ 'entry.ts': 'const source = { value: 7 }; const alias = source; ' @@ -167,3 +166,35 @@ test('single-source captures retain their dedicated validation route', () => { assert.deepEqual(inspectLocalSourceClosure(root, path.join(root, 'entry.ts')), { files: ['entry.ts'] }) }) }) + +test('imported class implements an erased interface without a runtime initialization effect', () => { + workspace({ + 'wrapper.ts': "import { HashMap } from './subject'; export function scenario() { return new HashMap().size; }", + 'subject.ts': "import { MapContract } from './type-contract'; " + + 'export class HashMap implements MapContract { size = 0; set(_key: K, _value: V) { this.size++; } }', + 'type-contract.ts': 'export interface MapContract { size: number; set(key: K, value: V): void }', + }, root => { + assert.deepEqual(inspectLocalSourceClosure(root, path.join(root, 'wrapper.ts')), { + files: ['subject.ts', 'wrapper.ts'], + }) + }) +}) + +for (const [name, declaration] of [ + ['effectful extends expression', 'function base() { throw new Error("must not execute"); } export class Subject extends base() {}'], + ['plain runtime extends', 'class Base {} export class Subject extends Base {}'], + ['decorated implements class', 'interface Contract {} function decorate(value: unknown) { return value; } @decorate export class Subject implements Contract {}'], + ['static initializer with implements', 'interface Contract {} export class Subject implements Contract { static value = Math.random(); }'], + ['static block with implements', 'interface Contract {} export class Subject implements Contract { static { throw new Error("must not execute"); } }'], +] as const) { + test(`imported class retains module initialization rejection for ${name}`, () => { + workspace({ + 'wrapper.ts': "import { Subject } from './subject'; export function scenario() { return new Subject(); }", + 'subject.ts': declaration, + }, root => { + const result = inspectLocalSourceClosure(root, path.join(root, 'wrapper.ts')) + assert.equal(result.reasonCode, 'TOP_LEVEL_INITIALIZATION_UNSUPPORTED') + assert.match(result.diagnostic!, /subject\.ts/) + }) + }) +} diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt index 992fa65184..aa815efc14 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/TypeScriptSourceInspector.kt @@ -10,6 +10,16 @@ import java.util.concurrent.TimeUnit /** Uses the native TypeScript AST when lowered source ranges need validation or replay instrumentation. */ object TypeScriptSourceInspector { + fun instrumentClassCoverage(source: Path, className: String): TypeScriptClassCoverageInstrumentation { + val output = inspect( + source = source, + operation = "instrument-class-coverage", + arguments = listOf(className), + ) + + return PropertyManifestJson.json.decodeFromString(output) + } + fun localSourceClosure(sourceRoot: Path, source: Path): TypeScriptLocalSourceClosure { val output = inspect( source = sourceRoot, @@ -148,3 +158,59 @@ data class TypeScriptLocalSourceClosure( val reasonCode: String? = null, val diagnostic: String? = null, ) + +@Serializable +data class TypeScriptClassStatementDescriptor( + val id: String, + val memberName: String, + val kind: String, + val startOffset: Int, + val endOffset: Int, + val startLine: Int, + val startColumn: Int, + val endLine: Int, + val endColumn: Int, +) + +@Serializable +data class TypeScriptClassCoverageInstrumentation( + val source: String?, + val reason: String?, + val sourceSha256: String, + val instrumentedSha256: String?, + val counterGlobalName: String, + val instrumenterVersion: String, + val statements: List, + val inventory: TypeScriptClassSurfaceInventory?, +) + +@Serializable +data class TypeScriptClassCallableDescriptor( + val name: String, + val kind: String, + val isStatic: Boolean, + val parameterNames: List, + val parameterTypeTexts: List, + val resultTypeText: String?, +) + +@Serializable +data class TypeScriptClassStateFieldDescriptor( + val name: String, + val isEcmaPrivate: Boolean, + val typeText: String?, +) + +@Serializable +data class TypeScriptClassSurfaceInventory( + val constructorParameterNames: List, + val publicMembers: List, + val instanceDataFields: List, + val dataClasses: List = emptyList(), +) + +@Serializable +data class TypeScriptClassDataClassDescriptor( + val exportName: String, + val fields: List, +)