From f8cd9fd0ed43a09a5c2fa8005bb4cbf78e11acc3 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Thu, 15 Oct 2026 23:39:40 +0300 Subject: [PATCH 01/15] [Core] Accept externally supplied coverage goals Reuse closest-to-uncovered and weighted selectors with revisioned goals supplied by callers. Refresh pending priorities after coverage changes while retaining queued states and existing reporting semantics. --- usvm-core/README.md | 59 ++++ .../org/usvm/ps/CoverageGuidedPathSelector.kt | 30 ++ .../kotlin/org/usvm/ps/PathSelectorFactory.kt | 76 +++-- .../org/usvm/ps/WeightedPathSelector.kt | 27 +- .../org/usvm/statistics/CoverageStatistics.kt | 10 +- .../org/usvm/statistics/CoverageTargets.kt | 43 +++ .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 278 ++++++++++++++++++ .../statistics/ExternalCoverageTargetsTest.kt | 47 +++ 8 files changed, 540 insertions(+), 30 deletions(-) create mode 100644 usvm-core/README.md create mode 100644 usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt create mode 100644 usvm-core/src/main/kotlin/org/usvm/statistics/CoverageTargets.kt create mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt create mode 100644 usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt diff --git a/usvm-core/README.md b/usvm-core/README.md new file mode 100644 index 0000000000..853a55b309 --- /dev/null +++ b/usvm-core/README.md @@ -0,0 +1,59 @@ +# External coverage goals + +`CoverageTargets` supplies the currently uncovered goals and a revision that changes whenever that set changes. +The normal `CoverageStatistics` implements this contract for statements. `ExternalCoverageTargets` lets a caller +register goals and report confirmed coverage between analysis steps. + +The standard factory accepts this source independently of the statistics used for reporting and stopping: + +```kotlin +val coverage = ExternalCoverageTargets(targets = knownStatements) +coverage.markCovered(targets = statementsCoveredByConcreteTests) + +val selector = createPathSelector( + initialState = initialState, + options = options.copy(pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED)), + applicationGraph = graph, + cfgStatisticsFactory = { cfgStatistics }, + coverageTargetsFactory = { coverage }, +) + +// The analysis can continue with the same queue after the caller confirms more coverage. +coverage.markCovered(targets = newlyConfirmedStatements) +``` + +Both `CLOSEST_TO_UNCOVERED` variants use cached CFG distances in the active call stack. Existing entry-point fairness, +exception propagation, loop bounds and selector composition remain controlled by `UMachineOptions`. +External goals do not change reported coverage percentages or coverage-based stop conditions. + +For targets that are not statements, use the same weighted queue with a caller-supplied distance: + +```kotlin +data class Edge(val from: Statement, val to: Statement) + +val coverage = ExternalCoverageTargets(targets = knownEdges) +coverage.markCovered(targets = edgesCoveredByConcreteTests) + +val selector = createCoverageGuidedPathSelector( + coverageTargets = coverage, + distance = ::distanceFromStateToEdge, +) +``` + +The distance function owns the meaning of each target and returns `UInt.MAX_VALUE` when it is unreachable. +An edge must retain both endpoints: reaching its successor through a different predecessor does not cover that edge. +The core does not turn an edge into a statement or infer coverage from the distance function. +Language-specific code can reuse the existing CFG and interprocedural distance calculators in this function. + +Updates are applied on the analysis thread, between machine steps. The next `peek()` refreshes all queued weights +after a revision change; repeated peeks with unchanged evidence reuse the weights. No listener keeps the selector +attached to the external source after the analysis ends. Draining stopped work does not calculate new distances. + +`markCovered` ignores unknown goals, and repeated coverage reports do not change the revision. Use `addTargets` +to register newly identified goals; re-registering a covered goal does not make it uncovered again. Missing records +are not automatically registered as uncovered goals. + +An empty goal set retains every queued state. Covered prefixes remain traversable, and coverage alone does not +establish a property on other inputs. Property search must keep its own stop conditions and exploration policy. +Runtime admission, source identity and concrete replay confirmation belong to the caller; this core API accepts +its coverage decisions without interpreting them. diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt new file mode 100644 index 0000000000..2b266d7bb7 --- /dev/null +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -0,0 +1,30 @@ +package org.usvm.ps + +import org.usvm.UPathSelector +import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.statistics.CoverageTargets + +/** + * Selects the closest uncovered goal without pruning states. Goals may be statements, edges or other caller types. + * [distance] owns their semantics; [UInt.MAX_VALUE] means unreachable. Updates become visible on the next peek. + */ +fun createCoverageGuidedPathSelector( + coverageTargets: CoverageTargets, + distance: (State, Target) -> UInt, +): UPathSelector { + var revision = coverageTargets.revision + var targets = coverageTargets.uncoveredTargets.toList() + + return WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, + weighter = { state -> + if (revision != coverageTargets.revision) { + targets = coverageTargets.uncoveredTargets.toList() + revision = coverageTargets.revision + } + + targets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE + }, + weightRevision = { coverageTargets.revision }, + ) +} diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt index 9a3d0b4c94..e7214bb419 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt @@ -13,6 +13,7 @@ import org.usvm.merging.CloseStatesSearcherImpl import org.usvm.merging.MergingPathSelector import org.usvm.statistics.ApplicationGraph import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.CoverageTargets import org.usvm.statistics.TimeStatistics import org.usvm.statistics.distances.CallGraphStatistics import org.usvm.statistics.distances.CallStackDistanceCalculator @@ -36,6 +37,7 @@ private fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { @@ -66,18 +68,24 @@ private fun createPathSelector( PathSelectionStrategy.FORK_DEPTH -> createForkDepthPathSelector() PathSelectionStrategy.FORK_DEPTH_RANDOM -> createForkDepthPathSelector(random) - PathSelectionStrategy.CLOSEST_TO_UNCOVERED -> createClosestToUncoveredPathSelector( - requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, - requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, - applicationGraph - ) + PathSelectionStrategy.CLOSEST_TO_UNCOVERED, + PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM, + -> { + val coverageTargets = requireNotNull(coverageTargetsFactory()) { + "Coverage targets are required for closest to uncovered path selector" + } + val cfgStatistics = requireNotNull(cfgStatisticsFactory()) { + "CFG statistics is required for closest to uncovered path selector" + } + val selectorRandom = if (strategy == PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM) random else null - PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM -> createClosestToUncoveredPathSelector( - requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, - requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, - applicationGraph, - random - ) + createClosestToUncoveredPathSelector( + coverageTargets = coverageTargets, + cfgStatistics = cfgStatistics, + applicationGraph = applicationGraph, + random = selectorRandom, + ) + } PathSelectionStrategy.TARGETED -> createTargetedPathSelector( requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for targeted path selector" }, @@ -145,6 +153,7 @@ private fun createPathSelector( return selector } +/** [coverageTargetsFactory] overrides closest-to-uncovered goals, leaving reported coverage unchanged. */ fun createPathSelector( initialState: State, options: UMachineOptions, @@ -153,6 +162,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> = createPathSelector( listOf(initialState), @@ -161,9 +171,11 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageTargetsFactory, ) +/** External goals are shared by the entry points; coverage statistics still own reporting and stop conditions. */ fun createPathSelector( initialStates: Map, options: UMachineOptions, @@ -173,6 +185,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { val hasSingleEntryPoint = initialStates.size == 1 val hasUnlimitedScheduling = options.timeout == Duration.INFINITE && @@ -185,7 +198,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageTargetsFactory, ) } @@ -202,7 +216,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageTargetsFactory, ) val coverageStatistics = coverageStatisticsFactory() @@ -312,29 +327,35 @@ private fun > createDepthPathSelector(rando ) } -private fun > createClosestToUncoveredPathSelector( - coverageStatistics: CoverageStatistics, +/** Uses cached CFG distances within the active call stack and refreshes priorities after goal changes. */ +fun > createClosestToUncoveredPathSelector( + coverageTargets: CoverageTargets, cfgStatistics: CfgStatistics, applicationGraph: ApplicationGraph, random: Random? = null, ): UPathSelector { - val distanceCalculator = CallStackDistanceCalculator( - targets = coverageStatistics.getUncoveredStatements(), + fun createDistanceCalculator() = CallStackDistanceCalculator( + targets = coverageTargets.uncoveredTargets, cfgStatistics = cfgStatistics, - applicationGraph + applicationGraph = applicationGraph, ) + var distanceCalculator = createDistanceCalculator() + var revision = coverageTargets.revision - coverageStatistics.addOnCoveredObserver { _, method, statement -> - distanceCalculator.removeTarget( - method, - statement - ) + fun calculateDistance(state: State): UInt { + if (revision != coverageTargets.revision) { + distanceCalculator = createDistanceCalculator() + revision = coverageTargets.revision + } + + return distanceCalculator.calculateDistance(state.currentStatement, state.callStack) } if (random == null) { return WeightedPathSelector( priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = { distanceCalculator.calculateDistance(it.currentStatement, it.callStack) } + weighter = ::calculateDistance, + weightRevision = { coverageTargets.revision }, ) } @@ -342,10 +363,11 @@ private fun ( priorityCollectionFactory: () -> UPriorityCollection, - private val weighter: StateWeighter + private val weighter: StateWeighter, + private val weightRevision: (() -> Long)? = null, ) : UPathSelector { private val priorityCollection = priorityCollectionFactory() + private var currentWeightRevision = weightRevision?.invoke() override fun isEmpty(): Boolean = priorityCollection.count == 0 - override fun peek(): State = priorityCollection.peek() + override fun peek(): State { + val revision = weightRevision?.invoke() + if (revision != currentWeightRevision) { + val states = drainPriorityCollection().toList() + add(states) + currentWeightRevision = revision + } + + return priorityCollection.peek() + } override fun update(state: State) = priorityCollection.update(state, weighter.weight(state)) @@ -30,4 +42,15 @@ open class WeightedPathSelector( } override fun remove(state: State) = priorityCollection.remove(state) + + override fun drain(): Sequence = drainPriorityCollection() + + private fun drainPriorityCollection(): Sequence = sequence { + // Draining stopped work must not recalculate weights or consult external goals. + while (priorityCollection.count > 0) { + val state = priorityCollection.peek() + priorityCollection.remove(state) + yield(state) + } + } } diff --git a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt index df263e6ac1..a4b767d25b 100644 --- a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt +++ b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt @@ -16,7 +16,13 @@ import java.util.concurrent.ConcurrentHashMap class CoverageStatistics>( methods: Set, private val applicationGraph: ApplicationGraph, -) : UMachineObserver { +) : UMachineObserver, CoverageTargets { + + override val uncoveredTargets: Collection + get() = getUncoveredStatements() + + override var revision: Long = 0 + private set private val onStatementCoveredObservers: MutableSet<(State, Method, Statement) -> Unit> = ConcurrentHashMap.newKeySet() @@ -56,6 +62,7 @@ class CoverageStatistics { + val uncoveredTargets: Collection + val revision: Long +} + +/** + * Caller-owned coverage goals, updated on the analysis thread between machine steps. + * Reporting coverage does not remove queued states or establish a property on other inputs. + */ +class ExternalCoverageTargets(targets: Collection) : CoverageTargets { + private val knownTargets = targets.toMutableSet() + private val pendingTargets = targets.toMutableSet() + + override val uncoveredTargets: Set + get() = pendingTargets.toSet() + + override var revision: Long = 0 + private set + + /** Registers newly identified goals; previously covered goals stay covered. */ + fun addTargets(targets: Collection): Boolean { + val additions = targets.filter { knownTargets.add(it) } + if (additions.isEmpty()) return false + + pendingTargets.addAll(additions) + revision++ + return true + } + + /** Accepts caller-confirmed coverage of known goals. Unknown goals are ignored. */ + fun markCovered(targets: Collection): Boolean { + val changed = pendingTargets.removeAll(targets.toSet()) + if (changed) revision++ + + return changed + } +} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt new file mode 100644 index 0000000000..515fc8561e --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt @@ -0,0 +1,278 @@ +package org.usvm.ps + +import io.mockk.every +import org.junit.jupiter.api.Test +import org.usvm.PathNode +import org.usvm.PathSelectionStrategy +import org.usvm.TestInstruction +import org.usvm.TestState +import org.usvm.TestTarget +import org.usvm.UMachineOptions +import org.usvm.UPathSelector +import org.usvm.appGraph +import org.usvm.mockState +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.ExternalCoverageTargets +import org.usvm.statistics.TimeStatistics +import org.usvm.statistics.distances.CfgStatisticsImpl +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class CoverageGuidedPathSelectorTest { + @Test + fun `external coverage reprioritizes every pending state without state updates`() { + val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + val selector = selector(targets) + selector.add(states = listOf("left", "right")) + + targets.markCovered(targets = listOf("left")) + assertEquals(expected = "right", actual = selector.peek()) + + targets.addTargets(targets = listOf("new-left")) + targets.markCovered(targets = listOf("right")) + assertEquals(expected = "left", actual = selector.peek()) + assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) + } + + @Test + fun `stable evidence does not recalculate queued weights`() { + val targets = ExternalCoverageTargets(targets = listOf("goal")) + var evaluations = 0 + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { _, _ -> + evaluations++ + 0u + }, + ) + selector.add(states = listOf("first", "second")) + val afterAdd = evaluations + + selector.peek() + selector.peek() + assertFalse(targets.markCovered(targets = listOf("unknown"))) + selector.peek() + + assertEquals(expected = afterAdd, actual = evaluations) + } + + @Test + fun `saturated coverage retains all queued states and accepts later work`() { + val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + val selector = selector(targets) + selector.add(states = listOf("left", "right")) + targets.markCovered(targets = listOf("left", "right")) + + selector.peek() + val drained = selector.drain().toList() + selector.add(states = listOf("later")) + + assertEquals(expected = setOf("left", "right"), actual = drained.toSet()) + assertEquals(expected = 2, actual = drained.size) + assertEquals(expected = "later", actual = selector.peek()) + } + + @Test + fun `edges sharing a successor remain distinct coverage goals`() { + val firstEdge = Edge(from = 1, to = 3) + val secondEdge = Edge(from = 2, to = 3) + val targets = ExternalCoverageTargets(targets = listOf(firstEdge, secondEdge)) + val firstPath = setOf(firstEdge) + val secondPath = setOf(secondEdge) + val selector = createCoverageGuidedPathSelector, Edge>( + coverageTargets = targets, + distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, + ) + selector.add(states = listOf(firstPath, secondPath)) + + targets.markCovered(targets = listOf(firstEdge)) + + assertSame(expected = secondPath, actual = selector.peek()) + assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) + } + + @Test + fun `empty initial evidence does not invoke the distance function or remove states`() { + val targets = ExternalCoverageTargets(targets = emptyList()) + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { _, _ -> error("There is no known goal") }, + ) + selector.add(states = listOf("first", "second")) + + selector.peek() + + assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) + assertTrue(selector.isEmpty()) + } + + @Test + fun `draining stopped work does not refresh weights after new evidence`() { + val targets = ExternalCoverageTargets(targets = listOf("left")) + var stopped = false + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { _, _ -> + check(!stopped) + 0u + }, + ) + selector.add(states = listOf("first", "second")) + stopped = true + targets.addTargets(targets = listOf("right")) + + val drained = selector.drain().toList() + + assertEquals(expected = setOf("first", "second"), actual = drained.toSet()) + assertTrue(selector.isEmpty()) + } + + @Test + fun `state updates additions and removals work across coverage revisions`() { + val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + val distances = mutableMapOf("first" to 0u, "second" to 2u, "third" to 1u) + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { state, _ -> distances.getValue(state) }, + ) + selector.add(states = listOf("first", "second")) + assertEquals(expected = "first", actual = selector.peek()) + + distances["first"] = 3u + selector.update(state = "first") + selector.add(states = listOf("third")) + targets.markCovered(targets = listOf("left")) + + assertEquals(expected = "third", actual = selector.peek()) + selector.remove(state = "third") + assertEquals(expected = "second", actual = selector.peek()) + assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) + } + + @Test + fun `normal factory accepts external goals without replacing machine statistics`() { + val statistics = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 2), instruction(offset = 4))) + targets.markCovered(targets = listOf(instruction(offset = 2))) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createPathSelector( + initialState = left, + options = options(), + applicationGraph = graph, + coverageStatisticsFactory = { statistics }, + cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, + coverageTargetsFactory = { targets }, + ) + selector.add(states = listOf(right)) + assertSame(expected = right, actual = selector.peek()) + + targets.markCovered(targets = listOf(instruction(offset = 4))) + targets.addTargets(targets = listOf(instruction(offset = 1))) + + assertSame(expected = left, actual = selector.peek()) + assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) + assertEquals(expected = 0, actual = statistics.getTotalCoveredStatements()) + } + + @Test + fun `map factory forwards external goals to the standard selector`() { + val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createPathSelector( + initialStates = mapOf("method" to left), + options = options(), + applicationGraph = graph, + timeStatistics = TimeStatistics(), + cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, + coverageTargetsFactory = { targets }, + ) + selector.add(states = listOf(right)) + + assertSame(expected = right, actual = selector.peek()) + } + + @Test + fun `default statistics refresh weights for states that were not stepped`() { + val statistics = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val completed = mockState(id = 3u, startMethod = "method", startInstruction = 2) + every { completed.pathNode } returns PathNode.root() + + instruction(offset = 0) + instruction(offset = 1) + instruction(offset = 2) + val selector = createPathSelector( + initialState = left, + options = options(), + applicationGraph = graph, + coverageStatisticsFactory = { statistics }, + cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, + ) + selector.add(states = listOf(right)) + selector.peek() + + statistics.onStateTerminated(state = completed, stateReachable = true) + + assertSame(expected = right, actual = selector.peek()) + assertEquals(expected = 3, actual = statistics.getTotalCoveredStatements()) + } + + @Test + fun `randomized closest selector also refreshes all pending weights`() { + val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createClosestToUncoveredPathSelector( + coverageTargets = targets, + cfgStatistics = CfgStatisticsImpl(graph), + applicationGraph = graph, + random = Random(seed = 42), + ) + selector.add(states = listOf(left, right)) + assertSame(expected = right, actual = selector.peek()) + + targets.markCovered(targets = listOf(instruction(offset = 4))) + targets.addTargets(targets = listOf(instruction(offset = 1))) + + assertSame(expected = left, actual = selector.peek()) + } + + private fun selector(targets: ExternalCoverageTargets): UPathSelector = + createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { state, target -> + val destination = if (target == "new-left") "left" else target + if (state == destination) 0u else 10u + }, + ) + + private fun instruction(offset: Int) = TestInstruction(method = "method", offset = offset) + + private fun options() = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED), + exceptionsPropagation = false, + ) + + private val graph = appGraph { + method(name = "method", instructionsCount = 5) { + entryPoint(offset = 0) + edge(from = 0, to = 1) + edge(from = 0, to = 3) + edge(from = 1, to = 2) + edge(from = 3, to = 4) + exitPoint(offset = 2) + exitPoint(offset = 4) + } + } + + private data class Edge(val from: Int, val to: Int) +} diff --git a/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt b/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt new file mode 100644 index 0000000000..dbf0c03795 --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt @@ -0,0 +1,47 @@ +package org.usvm.statistics + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ExternalCoverageTargetsTest { + @Test + fun `coverage only changes registered goals and repeated reports are idempotent`() { + val targets = ExternalCoverageTargets(targets = listOf("first", "second")) + + assertFalse(targets.markCovered(targets = listOf("unknown"))) + assertEquals(expected = 0L, actual = targets.revision) + assertTrue(targets.markCovered(targets = listOf("first"))) + assertFalse(targets.markCovered(targets = listOf("first"))) + + assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) + assertEquals(expected = 1L, actual = targets.revision) + } + + @Test + fun `new goals can be registered without resurrecting covered goals`() { + val targets = ExternalCoverageTargets(targets = listOf("first")) + targets.markCovered(targets = listOf("first")) + + assertTrue(targets.addTargets(targets = listOf("first", "second", "second"))) + assertFalse(targets.addTargets(targets = listOf("first", "second"))) + + assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) + assertEquals(expected = 2L, actual = targets.revision) + } + + @Test + fun `goal snapshots and constructor inputs do not share mutable storage`() { + val initial = mutableListOf("first", "second") + val targets = ExternalCoverageTargets(targets = initial) + val snapshot = targets.uncoveredTargets + + initial.clear() + targets.markCovered(targets = listOf("first", "second")) + + assertEquals(expected = setOf("first", "second"), actual = snapshot) + assertTrue(targets.uncoveredTargets.isEmpty()) + assertEquals(expected = 1L, actual = targets.revision) + } +} From e69a9bb0d52da0980fbde4a4cd5cbbff15a30a73 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 00:25:38 +0300 Subject: [PATCH 02/15] [Core] Isolate external coverage updates in one selector Restore existing statistics and selector APIs. Keep goal updates and lazy priority refresh inside CoverageGuidedPathSelector, and remove the unsolicited README. --- usvm-core/README.md | 59 ---- .../org/usvm/ps/CoverageGuidedPathSelector.kt | 73 +++-- .../kotlin/org/usvm/ps/PathSelectorFactory.kt | 76 ++--- .../org/usvm/ps/WeightedPathSelector.kt | 27 +- .../org/usvm/statistics/CoverageStatistics.kt | 10 +- .../org/usvm/statistics/CoverageTargets.kt | 43 --- .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 265 +++++++++--------- .../statistics/ExternalCoverageTargetsTest.kt | 47 ---- 8 files changed, 209 insertions(+), 391 deletions(-) delete mode 100644 usvm-core/README.md delete mode 100644 usvm-core/src/main/kotlin/org/usvm/statistics/CoverageTargets.kt delete mode 100644 usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt diff --git a/usvm-core/README.md b/usvm-core/README.md deleted file mode 100644 index 853a55b309..0000000000 --- a/usvm-core/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# External coverage goals - -`CoverageTargets` supplies the currently uncovered goals and a revision that changes whenever that set changes. -The normal `CoverageStatistics` implements this contract for statements. `ExternalCoverageTargets` lets a caller -register goals and report confirmed coverage between analysis steps. - -The standard factory accepts this source independently of the statistics used for reporting and stopping: - -```kotlin -val coverage = ExternalCoverageTargets(targets = knownStatements) -coverage.markCovered(targets = statementsCoveredByConcreteTests) - -val selector = createPathSelector( - initialState = initialState, - options = options.copy(pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED)), - applicationGraph = graph, - cfgStatisticsFactory = { cfgStatistics }, - coverageTargetsFactory = { coverage }, -) - -// The analysis can continue with the same queue after the caller confirms more coverage. -coverage.markCovered(targets = newlyConfirmedStatements) -``` - -Both `CLOSEST_TO_UNCOVERED` variants use cached CFG distances in the active call stack. Existing entry-point fairness, -exception propagation, loop bounds and selector composition remain controlled by `UMachineOptions`. -External goals do not change reported coverage percentages or coverage-based stop conditions. - -For targets that are not statements, use the same weighted queue with a caller-supplied distance: - -```kotlin -data class Edge(val from: Statement, val to: Statement) - -val coverage = ExternalCoverageTargets(targets = knownEdges) -coverage.markCovered(targets = edgesCoveredByConcreteTests) - -val selector = createCoverageGuidedPathSelector( - coverageTargets = coverage, - distance = ::distanceFromStateToEdge, -) -``` - -The distance function owns the meaning of each target and returns `UInt.MAX_VALUE` when it is unreachable. -An edge must retain both endpoints: reaching its successor through a different predecessor does not cover that edge. -The core does not turn an edge into a statement or infer coverage from the distance function. -Language-specific code can reuse the existing CFG and interprocedural distance calculators in this function. - -Updates are applied on the analysis thread, between machine steps. The next `peek()` refreshes all queued weights -after a revision change; repeated peeks with unchanged evidence reuse the weights. No listener keeps the selector -attached to the external source after the analysis ends. Draining stopped work does not calculate new distances. - -`markCovered` ignores unknown goals, and repeated coverage reports do not change the revision. Use `addTargets` -to register newly identified goals; re-registering a covered goal does not make it uncovered again. Missing records -are not automatically registered as uncovered goals. - -An empty goal set retains every queued state. Covered prefixes remain traversable, and coverage alone does not -establish a property on other inputs. Property search must keep its own stop conditions and exploration policy. -Runtime admission, source identity and concrete replay confirmation belong to the caller; this core API accepts -its coverage decisions without interpreting them. diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt index 2b266d7bb7..df0f0efe87 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -2,29 +2,62 @@ package org.usvm.ps import org.usvm.UPathSelector import org.usvm.algorithms.DeterministicPriorityCollection -import org.usvm.statistics.CoverageTargets /** - * Selects the closest uncovered goal without pruning states. Goals may be statements, edges or other caller types. - * [distance] owns their semantics; [UInt.MAX_VALUE] means unreachable. Updates become visible on the next peek. + * Caller-confirmed coverage changes priorities, never removes states. Updates belong on the analysis thread. + * [distance] defines statement, edge or other goal semantics; [UInt.MAX_VALUE] means unreachable. */ -fun createCoverageGuidedPathSelector( - coverageTargets: CoverageTargets, +class CoverageGuidedPathSelector( + targets: Collection, distance: (State, Target) -> UInt, -): UPathSelector { - var revision = coverageTargets.revision - var targets = coverageTargets.uncoveredTargets.toList() - - return WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = { state -> - if (revision != coverageTargets.revision) { - targets = coverageTargets.uncoveredTargets.toList() - revision = coverageTargets.revision - } - - targets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE - }, - weightRevision = { coverageTargets.revision }, +) : UPathSelector { + private val knownTargets = targets.toMutableSet() + private val pendingTargets = targets.toMutableSet() + private var prioritiesChanged = false + private val selector = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, + weighter = { state -> pendingTargets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE }, ) + + val uncoveredTargets: Set + get() = pendingTargets.toSet() + + /** Registers newly identified goals; previously covered goals stay covered. */ + fun addTargets(targets: Collection): Boolean { + val additions = targets.filter { knownTargets.add(it) } + if (additions.isEmpty()) return false + + pendingTargets.addAll(additions) + prioritiesChanged = true + return true + } + + /** Unknown goals are ignored. Changed priorities become visible on the next [peek]. */ + fun markCovered(targets: Collection): Boolean { + val changed = pendingTargets.removeAll(targets.toSet()) + if (changed) prioritiesChanged = true + + return changed + } + + override fun isEmpty(): Boolean = selector.isEmpty() + + override fun peek(): State { + if (prioritiesChanged) { + val states = selector.drain().toList() + selector.add(states) + prioritiesChanged = false + } + + return selector.peek() + } + + override fun add(states: Collection) = selector.add(states) + + override fun update(state: State) = selector.update(state) + + override fun remove(state: State) = selector.remove(state) + + // Bypass pending priority updates when collecting stopped work. + override fun drain(): Sequence = selector.drain() } diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt index e7214bb419..9a3d0b4c94 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt @@ -13,7 +13,6 @@ import org.usvm.merging.CloseStatesSearcherImpl import org.usvm.merging.MergingPathSelector import org.usvm.statistics.ApplicationGraph import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.CoverageTargets import org.usvm.statistics.TimeStatistics import org.usvm.statistics.distances.CallGraphStatistics import org.usvm.statistics.distances.CallStackDistanceCalculator @@ -37,7 +36,6 @@ private fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, - coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { @@ -68,24 +66,18 @@ private fun createPathSelector( PathSelectionStrategy.FORK_DEPTH -> createForkDepthPathSelector() PathSelectionStrategy.FORK_DEPTH_RANDOM -> createForkDepthPathSelector(random) - PathSelectionStrategy.CLOSEST_TO_UNCOVERED, - PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM, - -> { - val coverageTargets = requireNotNull(coverageTargetsFactory()) { - "Coverage targets are required for closest to uncovered path selector" - } - val cfgStatistics = requireNotNull(cfgStatisticsFactory()) { - "CFG statistics is required for closest to uncovered path selector" - } - val selectorRandom = if (strategy == PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM) random else null + PathSelectionStrategy.CLOSEST_TO_UNCOVERED -> createClosestToUncoveredPathSelector( + requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, + requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, + applicationGraph + ) - createClosestToUncoveredPathSelector( - coverageTargets = coverageTargets, - cfgStatistics = cfgStatistics, - applicationGraph = applicationGraph, - random = selectorRandom, - ) - } + PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM -> createClosestToUncoveredPathSelector( + requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, + requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, + applicationGraph, + random + ) PathSelectionStrategy.TARGETED -> createTargetedPathSelector( requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for targeted path selector" }, @@ -153,7 +145,6 @@ private fun createPathSelector( return selector } -/** [coverageTargetsFactory] overrides closest-to-uncovered goals, leaving reported coverage unchanged. */ fun createPathSelector( initialState: State, options: UMachineOptions, @@ -162,7 +153,6 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, - coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> = createPathSelector( listOf(initialState), @@ -171,11 +161,9 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory, - coverageTargetsFactory, + loopStatisticFactory ) -/** External goals are shared by the entry points; coverage statistics still own reporting and stop conditions. */ fun createPathSelector( initialStates: Map, options: UMachineOptions, @@ -185,7 +173,6 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, - coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { val hasSingleEntryPoint = initialStates.size == 1 val hasUnlimitedScheduling = options.timeout == Duration.INFINITE && @@ -198,8 +185,7 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory, - coverageTargetsFactory, + loopStatisticFactory ) } @@ -216,8 +202,7 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory, - coverageTargetsFactory, + loopStatisticFactory ) val coverageStatistics = coverageStatisticsFactory() @@ -327,35 +312,29 @@ private fun > createDepthPathSelector(rando ) } -/** Uses cached CFG distances within the active call stack and refreshes priorities after goal changes. */ -fun > createClosestToUncoveredPathSelector( - coverageTargets: CoverageTargets, +private fun > createClosestToUncoveredPathSelector( + coverageStatistics: CoverageStatistics, cfgStatistics: CfgStatistics, applicationGraph: ApplicationGraph, random: Random? = null, ): UPathSelector { - fun createDistanceCalculator() = CallStackDistanceCalculator( - targets = coverageTargets.uncoveredTargets, + val distanceCalculator = CallStackDistanceCalculator( + targets = coverageStatistics.getUncoveredStatements(), cfgStatistics = cfgStatistics, - applicationGraph = applicationGraph, + applicationGraph ) - var distanceCalculator = createDistanceCalculator() - var revision = coverageTargets.revision - - fun calculateDistance(state: State): UInt { - if (revision != coverageTargets.revision) { - distanceCalculator = createDistanceCalculator() - revision = coverageTargets.revision - } - return distanceCalculator.calculateDistance(state.currentStatement, state.callStack) + coverageStatistics.addOnCoveredObserver { _, method, statement -> + distanceCalculator.removeTarget( + method, + statement + ) } if (random == null) { return WeightedPathSelector( priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = ::calculateDistance, - weightRevision = { coverageTargets.revision }, + weighter = { distanceCalculator.calculateDistance(it.currentStatement, it.callStack) } ) } @@ -363,11 +342,10 @@ fun > creat priorityCollectionFactory = { RandomizedPriorityCollection(compareById()) { random.nextDouble() } }, weighter = { 1.0 / max( - calculateDistance(it).toDouble(), + distanceCalculator.calculateDistance(it.currentStatement, it.callStack).toDouble(), 1.0 ) - }, - weightRevision = { coverageTargets.revision }, + } ) } diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt index 0b29f65294..7844295c55 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt @@ -9,29 +9,17 @@ import org.usvm.algorithms.UPriorityCollection * * @param priorityCollectionFactory function to create a priority collection for states. * @param weighter [StateWeighter] used to get states' weight which is used as priority in collection. - * @param weightRevision invalidates all queued weights when shared guidance changes between steps. */ open class WeightedPathSelector( priorityCollectionFactory: () -> UPriorityCollection, - private val weighter: StateWeighter, - private val weightRevision: (() -> Long)? = null, + private val weighter: StateWeighter ) : UPathSelector { private val priorityCollection = priorityCollectionFactory() - private var currentWeightRevision = weightRevision?.invoke() override fun isEmpty(): Boolean = priorityCollection.count == 0 - override fun peek(): State { - val revision = weightRevision?.invoke() - if (revision != currentWeightRevision) { - val states = drainPriorityCollection().toList() - add(states) - currentWeightRevision = revision - } - - return priorityCollection.peek() - } + override fun peek(): State = priorityCollection.peek() override fun update(state: State) = priorityCollection.update(state, weighter.weight(state)) @@ -42,15 +30,4 @@ open class WeightedPathSelector( } override fun remove(state: State) = priorityCollection.remove(state) - - override fun drain(): Sequence = drainPriorityCollection() - - private fun drainPriorityCollection(): Sequence = sequence { - // Draining stopped work must not recalculate weights or consult external goals. - while (priorityCollection.count > 0) { - val state = priorityCollection.peek() - priorityCollection.remove(state) - yield(state) - } - } } diff --git a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt index a4b767d25b..df263e6ac1 100644 --- a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt +++ b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt @@ -16,13 +16,7 @@ import java.util.concurrent.ConcurrentHashMap class CoverageStatistics>( methods: Set, private val applicationGraph: ApplicationGraph, -) : UMachineObserver, CoverageTargets { - - override val uncoveredTargets: Collection - get() = getUncoveredStatements() - - override var revision: Long = 0 - private set +) : UMachineObserver { private val onStatementCoveredObservers: MutableSet<(State, Method, Statement) -> Unit> = ConcurrentHashMap.newKeySet() @@ -62,7 +56,6 @@ class CoverageStatistics { - val uncoveredTargets: Collection - val revision: Long -} - -/** - * Caller-owned coverage goals, updated on the analysis thread between machine steps. - * Reporting coverage does not remove queued states or establish a property on other inputs. - */ -class ExternalCoverageTargets(targets: Collection) : CoverageTargets { - private val knownTargets = targets.toMutableSet() - private val pendingTargets = targets.toMutableSet() - - override val uncoveredTargets: Set - get() = pendingTargets.toSet() - - override var revision: Long = 0 - private set - - /** Registers newly identified goals; previously covered goals stay covered. */ - fun addTargets(targets: Collection): Boolean { - val additions = targets.filter { knownTargets.add(it) } - if (additions.isEmpty()) return false - - pendingTargets.addAll(additions) - revision++ - return true - } - - /** Accepts caller-confirmed coverage of known goals. Unknown goals are ignored. */ - fun markCovered(targets: Collection): Boolean { - val changed = pendingTargets.removeAll(targets.toSet()) - if (changed) revision++ - - return changed - } -} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt index 515fc8561e..84c2a8c88e 100644 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt @@ -1,21 +1,13 @@ package org.usvm.ps -import io.mockk.every import org.junit.jupiter.api.Test -import org.usvm.PathNode -import org.usvm.PathSelectionStrategy import org.usvm.TestInstruction import org.usvm.TestState -import org.usvm.TestTarget -import org.usvm.UMachineOptions -import org.usvm.UPathSelector import org.usvm.appGraph import org.usvm.mockState -import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.ExternalCoverageTargets -import org.usvm.statistics.TimeStatistics +import org.usvm.statistics.distances.CallStackDistanceCalculator import org.usvm.statistics.distances.CfgStatisticsImpl -import kotlin.random.Random +import org.usvm.statistics.distances.MultiTargetDistanceCalculator import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertSame @@ -23,26 +15,24 @@ import kotlin.test.assertTrue class CoverageGuidedPathSelectorTest { @Test - fun `external coverage reprioritizes every pending state without state updates`() { - val targets = ExternalCoverageTargets(targets = listOf("left", "right")) - val selector = selector(targets) + fun `reported coverage reprioritizes every pending state without state updates`() { + val selector = selector(targets = listOf("left", "right")) selector.add(states = listOf("left", "right")) - targets.markCovered(targets = listOf("left")) + selector.markCovered(targets = listOf("left")) assertEquals(expected = "right", actual = selector.peek()) - targets.addTargets(targets = listOf("new-left")) - targets.markCovered(targets = listOf("right")) + selector.addTargets(targets = listOf("new-left")) + selector.markCovered(targets = listOf("right")) assertEquals(expected = "left", actual = selector.peek()) assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) } @Test - fun `stable evidence does not recalculate queued weights`() { - val targets = ExternalCoverageTargets(targets = listOf("goal")) + fun `unchanged coverage does not recalculate queued weights`() { var evaluations = 0 - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector( + targets = listOf("goal"), distance = { _, _ -> evaluations++ 0u @@ -53,18 +43,43 @@ class CoverageGuidedPathSelectorTest { selector.peek() selector.peek() - assertFalse(targets.markCovered(targets = listOf("unknown"))) + assertFalse(selector.markCovered(targets = listOf("unknown"))) + assertFalse(selector.addTargets(targets = listOf("goal"))) selector.peek() assertEquals(expected = afterAdd, actual = evaluations) } + @Test + fun `multiple reports are applied together before the next selection`() { + var refreshing = false + var evaluations = 0 + val selector = CoverageGuidedPathSelector( + targets = listOf("first", "second", "third"), + distance = { state, target -> + if (refreshing) { + check(target == "third") + evaluations++ + } + + if (state == target) 0u else 10u + }, + ) + selector.add(states = listOf("first", "third")) + refreshing = true + + selector.markCovered(targets = listOf("first")) + selector.markCovered(targets = listOf("second")) + + assertEquals(expected = "third", actual = selector.peek()) + assertEquals(expected = 2, actual = evaluations) + } + @Test fun `saturated coverage retains all queued states and accepts later work`() { - val targets = ExternalCoverageTargets(targets = listOf("left", "right")) - val selector = selector(targets) + val selector = selector(targets = listOf("left", "right")) selector.add(states = listOf("left", "right")) - targets.markCovered(targets = listOf("left", "right")) + selector.markCovered(targets = listOf("left", "right")) selector.peek() val drained = selector.drain().toList() @@ -79,26 +94,24 @@ class CoverageGuidedPathSelectorTest { fun `edges sharing a successor remain distinct coverage goals`() { val firstEdge = Edge(from = 1, to = 3) val secondEdge = Edge(from = 2, to = 3) - val targets = ExternalCoverageTargets(targets = listOf(firstEdge, secondEdge)) val firstPath = setOf(firstEdge) val secondPath = setOf(secondEdge) - val selector = createCoverageGuidedPathSelector, Edge>( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector, Edge>( + targets = listOf(firstEdge, secondEdge), distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, ) selector.add(states = listOf(firstPath, secondPath)) - targets.markCovered(targets = listOf(firstEdge)) + selector.markCovered(targets = listOf(firstEdge)) assertSame(expected = secondPath, actual = selector.peek()) assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) } @Test - fun `empty initial evidence does not invoke the distance function or remove states`() { - val targets = ExternalCoverageTargets(targets = emptyList()) - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + fun `empty goals do not invoke the distance function or remove states`() { + val selector = CoverageGuidedPathSelector( + targets = emptyList(), distance = { _, _ -> error("There is no known goal") }, ) selector.add(states = listOf("first", "second")) @@ -110,11 +123,10 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `draining stopped work does not refresh weights after new evidence`() { - val targets = ExternalCoverageTargets(targets = listOf("left")) + fun `draining stopped work does not refresh weights after coverage changes`() { var stopped = false - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector( + targets = listOf("left"), distance = { _, _ -> check(!stopped) 0u @@ -122,7 +134,7 @@ class CoverageGuidedPathSelectorTest { ) selector.add(states = listOf("first", "second")) stopped = true - targets.addTargets(targets = listOf("right")) + selector.addTargets(targets = listOf("right")) val drained = selector.drain().toList() @@ -131,11 +143,10 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `state updates additions and removals work across coverage revisions`() { - val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + fun `state updates additions and removals work after coverage changes`() { val distances = mutableMapOf("first" to 0u, "second" to 2u, "third" to 1u) - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), distance = { state, _ -> distances.getValue(state) }, ) selector.add(states = listOf("first", "second")) @@ -144,7 +155,7 @@ class CoverageGuidedPathSelectorTest { distances["first"] = 3u selector.update(state = "first") selector.add(states = listOf("third")) - targets.markCovered(targets = listOf("left")) + selector.markCovered(targets = listOf("left")) assertEquals(expected = "third", actual = selector.peek()) selector.remove(state = "third") @@ -153,126 +164,102 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `normal factory accepts external goals without replacing machine statistics`() { - val statistics = CoverageStatistics( - methods = setOf("method"), - applicationGraph = graph, - ) - val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 2), instruction(offset = 4))) - targets.markCovered(targets = listOf(instruction(offset = 2))) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val selector = createPathSelector( - initialState = left, - options = options(), - applicationGraph = graph, - coverageStatisticsFactory = { statistics }, - cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, - coverageTargetsFactory = { targets }, - ) - selector.add(states = listOf(right)) - assertSame(expected = right, actual = selector.peek()) + fun `coverage reports are idempotent and unknown goals are ignored`() { + val selector = selector(targets = listOf("first", "second")) - targets.markCovered(targets = listOf(instruction(offset = 4))) - targets.addTargets(targets = listOf(instruction(offset = 1))) + assertFalse(selector.markCovered(targets = listOf("unknown"))) + assertTrue(selector.markCovered(targets = listOf("first"))) + assertFalse(selector.markCovered(targets = listOf("first"))) - assertSame(expected = left, actual = selector.peek()) - assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) - assertEquals(expected = 0, actual = statistics.getTotalCoveredStatements()) + assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) } @Test - fun `map factory forwards external goals to the standard selector`() { - val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val selector = createPathSelector( - initialStates = mapOf("method" to left), - options = options(), - applicationGraph = graph, - timeStatistics = TimeStatistics(), - cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, - coverageTargetsFactory = { targets }, - ) - selector.add(states = listOf(right)) + fun `new goals can be registered without resurrecting covered goals`() { + val selector = selector(targets = listOf("first")) + selector.markCovered(targets = listOf("first")) - assertSame(expected = right, actual = selector.peek()) + assertTrue(selector.addTargets(targets = listOf("first", "second", "second"))) + assertFalse(selector.addTargets(targets = listOf("first", "second"))) + + assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) } @Test - fun `default statistics refresh weights for states that were not stepped`() { - val statistics = CoverageStatistics( - methods = setOf("method"), - applicationGraph = graph, - ) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val completed = mockState(id = 3u, startMethod = "method", startInstruction = 2) - every { completed.pathNode } returns PathNode.root() + - instruction(offset = 0) + instruction(offset = 1) + instruction(offset = 2) - val selector = createPathSelector( - initialState = left, - options = options(), - applicationGraph = graph, - coverageStatisticsFactory = { statistics }, - cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, - ) - selector.add(states = listOf(right)) - selector.peek() + fun `goal snapshots and constructor inputs do not share mutable storage`() { + val initial = mutableListOf("first", "second") + val selector = selector(targets = initial) + val snapshot = selector.uncoveredTargets - statistics.onStateTerminated(state = completed, stateReachable = true) + initial.clear() + selector.markCovered(targets = listOf("first", "second")) - assertSame(expected = right, actual = selector.peek()) - assertEquals(expected = 3, actual = statistics.getTotalCoveredStatements()) + assertEquals(expected = setOf("first", "second"), actual = snapshot) + assertTrue(selector.uncoveredTargets.isEmpty()) } @Test - fun `randomized closest selector also refreshes all pending weights`() { - val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) + fun `caller can reuse existing CFG distance calculators`() { + val graph = appGraph { + method(name = "method", instructionsCount = 5) { + entryPoint(offset = 0) + edge(from = 0, to = 1) + edge(from = 0, to = 3) + edge(from = 1, to = 2) + edge(from = 3, to = 4) + exitPoint(offset = 2) + exitPoint(offset = 4) + } + } + val cfgStatistics = CfgStatisticsImpl(graph) + val distances = MultiTargetDistanceCalculator { target -> + CallStackDistanceCalculator( + targets = listOf(target), + cfgStatistics = cfgStatistics, + applicationGraph = graph, + ) + } + val firstTarget = TestInstruction(method = "method", offset = 2) + val secondTarget = TestInstruction(method = "method", offset = 4) + val selector = CoverageGuidedPathSelector( + targets = listOf(firstTarget, secondTarget), + distance = { state, target -> + distances.calculateDistance(state.currentStatement, state.callStack, target) + }, + ) val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val selector = createClosestToUncoveredPathSelector( - coverageTargets = targets, - cfgStatistics = CfgStatisticsImpl(graph), - applicationGraph = graph, - random = Random(seed = 42), - ) selector.add(states = listOf(left, right)) - assertSame(expected = right, actual = selector.peek()) - targets.markCovered(targets = listOf(instruction(offset = 4))) - targets.addTargets(targets = listOf(instruction(offset = 1))) + selector.markCovered(targets = listOf(firstTarget)) - assertSame(expected = left, actual = selector.peek()) + assertSame(expected = right, actual = selector.peek()) + assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) } - private fun selector(targets: ExternalCoverageTargets): UPathSelector = - createCoverageGuidedPathSelector( - coverageTargets = targets, - distance = { state, target -> - val destination = if (target == "new-left") "left" else target - if (state == destination) 0u else 10u - }, - ) - - private fun instruction(offset: Int) = TestInstruction(method = "method", offset = offset) + @Test + fun `coverage guidance composes with a standard selector on shared states`() { + val coverage = selector(targets = listOf("left", "right")) + val combined = InterleavedPathSelector(selectors = listOf(coverage, BfsPathSelector())) + combined.add(states = listOf("left", "right")) + assertEquals(expected = "left", actual = combined.peek()) + + coverage.markCovered(targets = listOf("left")) + combined.add(states = listOf("fork")) + + assertEquals(expected = "right", actual = combined.peek()) + combined.remove(state = "right") + assertEquals(expected = setOf("left", "fork"), actual = combined.drain().toSet()) + assertTrue(combined.isEmpty()) + } - private fun options() = UMachineOptions( - pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED), - exceptionsPropagation = false, + private fun selector(targets: Collection) = CoverageGuidedPathSelector( + targets = targets, + distance = { state, target -> + val destination = if (target == "new-left") "left" else target + if (state == destination) 0u else 10u + }, ) - private val graph = appGraph { - method(name = "method", instructionsCount = 5) { - entryPoint(offset = 0) - edge(from = 0, to = 1) - edge(from = 0, to = 3) - edge(from = 1, to = 2) - edge(from = 3, to = 4) - exitPoint(offset = 2) - exitPoint(offset = 4) - } - } - private data class Edge(val from: Int, val to: Int) } diff --git a/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt b/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt deleted file mode 100644 index dbf0c03795..0000000000 --- a/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt +++ /dev/null @@ -1,47 +0,0 @@ -package org.usvm.statistics - -import org.junit.jupiter.api.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class ExternalCoverageTargetsTest { - @Test - fun `coverage only changes registered goals and repeated reports are idempotent`() { - val targets = ExternalCoverageTargets(targets = listOf("first", "second")) - - assertFalse(targets.markCovered(targets = listOf("unknown"))) - assertEquals(expected = 0L, actual = targets.revision) - assertTrue(targets.markCovered(targets = listOf("first"))) - assertFalse(targets.markCovered(targets = listOf("first"))) - - assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) - assertEquals(expected = 1L, actual = targets.revision) - } - - @Test - fun `new goals can be registered without resurrecting covered goals`() { - val targets = ExternalCoverageTargets(targets = listOf("first")) - targets.markCovered(targets = listOf("first")) - - assertTrue(targets.addTargets(targets = listOf("first", "second", "second"))) - assertFalse(targets.addTargets(targets = listOf("first", "second"))) - - assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) - assertEquals(expected = 2L, actual = targets.revision) - } - - @Test - fun `goal snapshots and constructor inputs do not share mutable storage`() { - val initial = mutableListOf("first", "second") - val targets = ExternalCoverageTargets(targets = initial) - val snapshot = targets.uncoveredTargets - - initial.clear() - targets.markCovered(targets = listOf("first", "second")) - - assertEquals(expected = setOf("first", "second"), actual = snapshot) - assertTrue(targets.uncoveredTargets.isEmpty()) - assertEquals(expected = 1L, actual = targets.revision) - } -} From 25fc41323e6537b7578a69f6aed6b75dea497123 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 00:35:23 +0300 Subject: [PATCH 03/15] Fix cached priority ordering after insertion --- .../DeterministicPriorityCollection.kt | 2 +- .../DeterministicPriorityCollectionTest.kt | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt diff --git a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt index af9eb77830..844a16487c 100644 --- a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt +++ b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt @@ -44,7 +44,7 @@ class DeterministicPriorityCollection(private val comparator: Compa assert({ !priorityQueue.any { (e, _) -> e == element } }) { "Element already exists in priority queue" } val currentTop = topElement - if (currentTop != null && comparator.compare(currentTop.second, priority) < 0) { + if (currentTop != null && comparator.compare(currentTop.second, priority) > 0) { topElement = element to priority priorityQueue.add(currentTop) return diff --git a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt new file mode 100644 index 0000000000..c5d182a099 --- /dev/null +++ b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt @@ -0,0 +1,62 @@ +package org.usvm.algorithms + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals + +internal class DeterministicPriorityCollectionTest { + @Test + fun `adding a worse priority keeps the cached minimum`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + collection.add(element = "first", priority = 1) + assertEquals("first", collection.peek()) + + collection.add(element = "second", priority = 2) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } + + @Test + fun `adding a better priority replaces the cached minimum`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + collection.add(element = "second", priority = 2) + assertEquals("second", collection.peek()) + + collection.add(element = "first", priority = 1) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } + + @Test + fun `adding a worse priority keeps the cached maximum with reverse order`() { + val collection = DeterministicPriorityCollection(reverseOrder()) + collection.add(element = "first", priority = 2) + assertEquals("first", collection.peek()) + + collection.add(element = "second", priority = 1) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } + + @Test + fun `adding a better priority replaces the cached maximum with reverse order`() { + val collection = DeterministicPriorityCollection(reverseOrder()) + collection.add(element = "second", priority = 1) + assertEquals("second", collection.peek()) + + collection.add(element = "first", priority = 2) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } +} From db1becc39acba5a81cfa706f98d31a42d020b6da Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 16:15:19 +0300 Subject: [PATCH 04/15] Batch coverage priority refreshes and heapify updated weights --- .../org/usvm/ps/CoverageGuidedPathSelector.kt | 80 +++++- .../CoverageGuidedPathSelectorRefreshTest.kt | 234 ++++++++++++++++++ .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 27 +- .../DeterministicPriorityCollection.kt | 50 +++- ...erministicPriorityCollectionRefreshTest.kt | 108 ++++++++ 5 files changed, 470 insertions(+), 29 deletions(-) create mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt create mode 100644 usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt index df0f0efe87..6dc86cdfcf 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -6,17 +6,33 @@ import org.usvm.algorithms.DeterministicPriorityCollection /** * Caller-confirmed coverage changes priorities, never removes states. Updates belong on the analysis thread. * [distance] defines statement, edge or other goal semantics; [UInt.MAX_VALUE] means unreachable. + * + * Weights use a snapshot of uncovered goals. Changes are applied before selection after at least + * [priorityRefreshInterval] completed selections since the previous priority refresh. A selection completes when + * its state is updated or removed; repeated [peek] calls do not advance the interval. + * The default interval is a tuning starting point, not a measured optimum. */ class CoverageGuidedPathSelector( targets: Collection, distance: (State, Target) -> UInt, + private val priorityRefreshInterval: Int = DEFAULT_PRIORITY_REFRESH_INTERVAL, ) : UPathSelector { + init { + require(priorityRefreshInterval > 0) { "Priority refresh interval must be positive" } + } + private val knownTargets = targets.toMutableSet() private val pendingTargets = targets.toMutableSet() + private var priorityTargets: Set = targets.toSet() + private val distanceToTarget = distance private var prioritiesChanged = false + private var completedSelections = 0 + private var selectionPending = false + private var selectedState: State? = null + private val priorities = DeterministicPriorityCollection(Comparator.naturalOrder()) private val selector = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = { state -> pendingTargets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE }, + priorityCollectionFactory = { priorities }, + weighter = { state -> weight(state, priorityTargets) }, ) val uncoveredTargets: Set @@ -32,10 +48,16 @@ class CoverageGuidedPathSelector( return true } - /** Unknown goals are ignored. Changed priorities become visible on the next [peek]. */ + /** Unknown goals are ignored. Changed priorities are applied at the next eligible selection. */ fun markCovered(targets: Collection): Boolean { val changed = pendingTargets.removeAll(targets.toSet()) - if (changed) prioritiesChanged = true + if (changed) { + prioritiesChanged = pendingTargets.isNotEmpty() + if (!prioritiesChanged) { + // With no goals, ordering is irrelevant and new work needs no distance calculations. + priorityTargets = emptySet() + } + } return changed } @@ -43,21 +65,55 @@ class CoverageGuidedPathSelector( override fun isEmpty(): Boolean = selector.isEmpty() override fun peek(): State { - if (prioritiesChanged) { - val states = selector.drain().toList() - selector.add(states) + val refreshDue = prioritiesChanged && completedSelections >= priorityRefreshInterval + if (refreshDue) { + val refreshedTargets = pendingTargets.toSet() + priorities.updatePriorities { state -> weight(state, refreshedTargets) } + priorityTargets = refreshedTargets prioritiesChanged = false + completedSelections = 0 } - return selector.peek() + val state = selector.peek() + selectedState = state + selectionPending = true + return state } override fun add(states: Collection) = selector.add(states) - override fun update(state: State) = selector.update(state) + override fun update(state: State) { + selector.update(state) + completeSelection(state) + } - override fun remove(state: State) = selector.remove(state) + override fun remove(state: State) { + selector.remove(state) + completeSelection(state) + } + + // Bypass pending priority updates and selection accounting when collecting stopped work. + override fun drain(): Sequence { + selectionPending = false + selectedState = null + return selector.drain() + } - // Bypass pending priority updates when collecting stopped work. - override fun drain(): Sequence = selector.drain() + private fun weight(state: State, targets: Set): UInt = + targets.minOfOrNull { target -> distanceToTarget(state, target) } ?: UInt.MAX_VALUE + + private fun completeSelection(state: State) { + val completesSelection = selectionPending && selectedState == state + if (!completesSelection) return + + if (completedSelections < priorityRefreshInterval) { + completedSelections++ + } + selectionPending = false + selectedState = null + } + + private companion object { + const val DEFAULT_PRIORITY_REFRESH_INTERVAL = 128 + } } diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt new file mode 100644 index 0000000000..62b2200096 --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt @@ -0,0 +1,234 @@ +package org.usvm.ps + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class CoverageGuidedPathSelectorRefreshTest { + @Test + fun `default interval keeps weights stale for 128 completed selections`() { + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + ) + selector.add(states = listOf("left", "right")) + selector.markCovered(targets = listOf("left")) + + repeat(127) { + assertEquals(expected = "left", actual = selector.peek()) + evaluatedTargets.clear() + selector.update(state = "left") + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + } + evaluatedTargets.clear() + repeat(256) { + assertEquals(expected = "left", actual = selector.peek()) + } + assertTrue(evaluatedTargets.isEmpty()) + + selector.update(state = "left") + evaluatedTargets.clear() + + assertEquals(expected = "right", actual = selector.peek()) + assertEquals(expected = listOf("right", "right"), actual = evaluatedTargets) + evaluatedTargets.clear() + selector.peek() + assertTrue(evaluatedTargets.isEmpty()) + } + + @Test + fun `new and updated states use the old snapshot until the interval expires`() { + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("old"), + distance = { state, target -> + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("old")) + selector.peek() + + selector.addTargets(targets = listOf("new")) + selector.markCovered(targets = listOf("old")) + evaluatedTargets.clear() + selector.add(states = listOf("new")) + selector.update(state = "old") + + assertEquals(expected = listOf("old", "old"), actual = evaluatedTargets) + assertEquals(expected = setOf("new"), actual = selector.uncoveredTargets) + assertEquals(expected = "old", actual = selector.peek()) + + selector.update(state = "old") + evaluatedTargets.clear() + + assertEquals(expected = "new", actual = selector.peek()) + assertEquals(expected = listOf("new", "new"), actual = evaluatedTargets) + } + + @Test + fun `only one completion of the selected state advances the interval`() { + var evaluations = 0 + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + evaluations++ + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("left", "right", "other")) + selector.peek() + selector.markCovered(targets = listOf("left")) + + selector.update(state = "other") + selector.remove(state = "other") + selector.update(state = "left") + selector.update(state = "left") + val beforePeek = evaluations + + assertEquals(expected = "left", actual = selector.peek()) + assertEquals(expected = beforePeek, actual = evaluations) + + selector.remove(state = "left") + val beforeRefresh = evaluations + + assertEquals(expected = "right", actual = selector.peek()) + assertEquals(expected = beforeRefresh + 1, actual = evaluations) + } + + @Test + fun `each refresh starts a new interval including newly registered goals`() { + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + val destination = if (target == "left-again") "left" else target + if (state == destination) 0u else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("left", "right")) + selector.markCovered(targets = listOf("left")) + repeat(2) { selector.update(state = selector.peek()) } + assertEquals(expected = "right", actual = selector.peek()) + + selector.addTargets(targets = listOf("left-again")) + selector.markCovered(targets = listOf("right")) + selector.update(state = "right") + + assertEquals(expected = "right", actual = selector.peek()) + + selector.update(state = "right") + + assertEquals(expected = "left", actual = selector.peek()) + assertEquals(expected = setOf("left-again"), actual = selector.uncoveredTargets) + } + + @Test + fun `unchanged goals do not trigger refresh even after the interval`() { + var evaluations = 0 + val selector = CoverageGuidedPathSelector( + targets = listOf("goal"), + distance = { _, _ -> + evaluations++ + 0u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("first", "second")) + + repeat(8) { selector.update(state = selector.peek()) } + val beforePeek = evaluations + selector.markCovered(targets = listOf("unknown")) + selector.addTargets(targets = listOf("goal")) + selector.peek() + + assertEquals(expected = 2 + 8, actual = evaluations) + assertEquals(expected = beforePeek, actual = evaluations) + } + + @Test + fun `saturation avoids distance work and later goals resume guidance`() { + var distancesAllowed = true + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("old"), + distance = { state, target -> + check(distancesAllowed) + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("old", "new")) + distancesAllowed = false + + selector.markCovered(targets = listOf("old")) + repeat(4) { selector.update(state = selector.peek()) } + selector.add(states = listOf("later")) + + assertTrue(selector.uncoveredTargets.isEmpty()) + + selector.addTargets(targets = listOf("new")) + distancesAllowed = true + evaluatedTargets.clear() + + assertEquals(expected = "new", actual = selector.peek()) + assertEquals(expected = listOf("new", "new", "new"), actual = evaluatedTargets) + assertEquals(expected = setOf("old", "new", "later"), actual = selector.drain().toSet()) + } + + @Test + fun `failed refresh preserves the snapshot and can be retried`() { + var failRefresh = false + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + check(!failRefresh) + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 1, + ) + selector.add(states = listOf("left", "right")) + selector.peek() + selector.markCovered(targets = listOf("left")) + selector.update(state = "left") + failRefresh = true + + assertFailsWith { selector.peek() } + + failRefresh = false + evaluatedTargets.clear() + selector.add(states = listOf("other")) + + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + assertEquals(expected = "right", actual = selector.peek()) + assertEquals(expected = setOf("left", "right", "other"), actual = selector.drain().toSet()) + } + + @Test + fun `refresh interval must be positive`() { + for (interval in listOf(0, -1)) { + assertFailsWith { + CoverageGuidedPathSelector( + targets = emptyList(), + distance = { _, _ -> 0u }, + priorityRefreshInterval = interval, + ) + } + } + } +} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt index 84c2a8c88e..825c93380e 100644 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt @@ -15,15 +15,21 @@ import kotlin.test.assertTrue class CoverageGuidedPathSelectorTest { @Test - fun `reported coverage reprioritizes every pending state without state updates`() { + fun `reported coverage reprioritizes every pending state after a completed selection`() { val selector = selector(targets = listOf("left", "right")) selector.add(states = listOf("left", "right")) + assertEquals(expected = "left", actual = selector.peek()) + selector.markCovered(targets = listOf("left")) + selector.update(state = "left") + assertEquals(expected = "right", actual = selector.peek()) selector.addTargets(targets = listOf("new-left")) selector.markCovered(targets = listOf("right")) + selector.update(state = "right") + assertEquals(expected = "left", actual = selector.peek()) assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) } @@ -51,7 +57,7 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `multiple reports are applied together before the next selection`() { + fun `multiple reports are applied together at the refresh boundary`() { var refreshing = false var evaluations = 0 val selector = CoverageGuidedPathSelector( @@ -64,12 +70,15 @@ class CoverageGuidedPathSelectorTest { if (state == target) 0u else 10u }, + priorityRefreshInterval = 1, ) selector.add(states = listOf("first", "third")) - refreshing = true + val selected = selector.peek() selector.markCovered(targets = listOf("first")) selector.markCovered(targets = listOf("second")) + selector.update(state = selected) + refreshing = true assertEquals(expected = "third", actual = selector.peek()) assertEquals(expected = 2, actual = evaluations) @@ -99,10 +108,13 @@ class CoverageGuidedPathSelectorTest { val selector = CoverageGuidedPathSelector, Edge>( targets = listOf(firstEdge, secondEdge), distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, + priorityRefreshInterval = 1, ) selector.add(states = listOf(firstPath, secondPath)) + val selected = selector.peek() selector.markCovered(targets = listOf(firstEdge)) + selector.update(state = selected) assertSame(expected = secondPath, actual = selector.peek()) assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) @@ -226,12 +238,15 @@ class CoverageGuidedPathSelectorTest { distance = { state, target -> distances.calculateDistance(state.currentStatement, state.callStack, target) }, + priorityRefreshInterval = 1, ) val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) selector.add(states = listOf(left, right)) + val selected = selector.peek() selector.markCovered(targets = listOf(firstTarget)) + selector.update(state = selected) assertSame(expected = right, actual = selector.peek()) assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) @@ -240,14 +255,15 @@ class CoverageGuidedPathSelectorTest { @Test fun `coverage guidance composes with a standard selector on shared states`() { val coverage = selector(targets = listOf("left", "right")) - val combined = InterleavedPathSelector(selectors = listOf(coverage, BfsPathSelector())) + val combined = InterleavedPathSelector(selectors = listOf(BfsPathSelector(), coverage)) combined.add(states = listOf("left", "right")) assertEquals(expected = "left", actual = combined.peek()) coverage.markCovered(targets = listOf("left")) - combined.add(states = listOf("fork")) + combined.update(state = "left") assertEquals(expected = "right", actual = combined.peek()) + combined.add(states = listOf("fork")) combined.remove(state = "right") assertEquals(expected = setOf("left", "fork"), actual = combined.drain().toSet()) assertTrue(combined.isEmpty()) @@ -259,6 +275,7 @@ class CoverageGuidedPathSelectorTest { val destination = if (target == "new-left") "left" else target if (state == destination) 0u else 10u }, + priorityRefreshInterval = 1, ) private data class Edge(val from: Int, val to: Int) diff --git a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt index 844a16487c..1c5248ffef 100644 --- a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt +++ b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt @@ -11,16 +11,22 @@ import kotlin.NoSuchElementException // TODO: what to do if elements have same priority? class DeterministicPriorityCollection(private val comparator: Comparator) : UPriorityCollection { - private var topElement: Pair? = null - private val priorityQueue = PriorityQueue> { (_, p1), (_, p2) -> comparator.compare(p1, p2) } + private inner class Entry(val element: T, val priority: Priority) : Comparable { + override fun compareTo(other: Entry): Int = comparator.compare(priority, other.priority) + } + + private var topElement: Entry? = null + private var priorityQueue = PriorityQueue() override val count: Int get() = priorityQueue.size + if (topElement == null) 0 else 1 override fun peek(): T { - if (topElement == null) { - topElement = priorityQueue.remove() - } - return topElement!!.first + val currentTop = topElement + if (currentTop != null) return currentTop.element + + val next = priorityQueue.remove() + topElement = next + return next.element } override fun update(element: T, priority: Priority) { @@ -30,25 +36,45 @@ class DeterministicPriorityCollection(private val comparator: Compa override fun remove(element: T) { // Don't traverse the whole queue if we remove the top element - if (topElement != null && topElement?.first == element) { + if (topElement != null && topElement?.element == element) { topElement = null return } - if (!priorityQueue.removeIf { (e, _) -> e == element }) { + if (!priorityQueue.removeIf { it.element == element }) { throw NoSuchElementException("Element not found in priority queue") } } override fun add(element: T, priority: Priority) { - assert({ !priorityQueue.any { (e, _) -> e == element } }) { "Element already exists in priority queue" } + assert({ !priorityQueue.any { it.element == element } }) { "Element already exists in priority queue" } val currentTop = topElement - if (currentTop != null && comparator.compare(currentTop.second, priority) > 0) { - topElement = element to priority + if (currentTop != null && comparator.compare(currentTop.priority, priority) > 0) { + topElement = Entry(element, priority) priorityQueue.add(currentTop) return } - priorityQueue.add(element to priority) + priorityQueue.add(Entry(element, priority)) + } + + /** + * Recalculates each priority once and rebuilds the heap in linear time, without removing and reinserting elements. + * If [priority] throws, the collection keeps its previous priorities and cached selection. + */ + fun updatePriorities(priority: (T) -> Priority) { + val entries = ArrayList(count) + val currentTop = topElement + if (currentTop != null) { + entries.add(Entry(currentTop.element, priority(currentTop.element))) + } + for (entry in priorityQueue) { + entries.add(Entry(entry.element, priority(entry.element))) + } + + // The collection constructor heapifies Comparable entries; addAll would perform individual insertions. + val refreshedQueue = PriorityQueue(entries) + priorityQueue = refreshedQueue + topElement = null } } diff --git a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt new file mode 100644 index 0000000000..2537c630db --- /dev/null +++ b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt @@ -0,0 +1,108 @@ +package org.usvm.algorithms + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +internal class DeterministicPriorityCollectionRefreshTest { + @Test + fun `refresh reorders every element including the cached selection`() { + for (comparator in listOf(naturalOrder(), reverseOrder())) { + val collection = DeterministicPriorityCollection(comparator) + collection.add(element = "first", priority = 1) + collection.add(element = "second", priority = 2) + collection.add(element = "third", priority = 3) + collection.peek() + val evaluations = mutableListOf() + val priorities = mapOf("first" to 3, "second" to 1, "third" to 2) + + collection.updatePriorities { element -> + evaluations.add(element) + priorities.getValue(element) + } + + assertEquals(expected = 3, actual = collection.count) + assertEquals(expected = 3, actual = evaluations.size) + assertEquals(expected = priorities.keys, actual = evaluations.toSet()) + val expected = priorities.keys.sortedWith { left, right -> + comparator.compare(priorities.getValue(left), priorities.getValue(right)) + } + assertEquals(expected = expected, actual = drain(collection)) + } + } + + @Test + fun `heap rebuilding uses a linear number of comparisons`() { + var comparisons = 0 + val comparator = Comparator { left, right -> + comparisons++ + left.compareTo(right) + } + val collection = DeterministicPriorityCollection(comparator) + val size = 1024 + repeat(size) { element -> collection.add(element = element, priority = element) } + collection.peek() + comparisons = 0 + + collection.updatePriorities { element -> size - element } + + assertTrue(comparisons <= 2 * size, "Heapify used $comparisons comparisons for $size elements") + assertEquals(expected = (0 until size).reversed().toList(), actual = drain(collection)) + } + + @Test + fun `equal refreshed priorities retain every element`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + repeat(8) { element -> collection.add(element = element, priority = element) } + collection.peek() + + collection.updatePriorities { 0 } + + val elements = drain(collection) + assertEquals(expected = 8, actual = elements.size) + assertEquals(expected = (0 until 8).toSet(), actual = elements.toSet()) + } + + @Test + fun `failed refresh leaves old ordering and all elements intact`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + collection.add(element = "first", priority = 1) + collection.add(element = "second", priority = 2) + collection.peek() + + assertFailsWith { + collection.updatePriorities { element -> + check(element != "second") + 100 + } + } + + assertEquals(expected = "first", actual = collection.peek()) + assertEquals(expected = listOf("first", "second"), actual = drain(collection)) + } + + @Test + fun `empty and singleton collections can be refreshed repeatedly`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + + collection.updatePriorities { error("Empty collection") } + collection.add(element = "only", priority = 1) + collection.peek() + repeat(2) { collection.updatePriorities { 2 } } + + assertEquals(expected = listOf("only"), actual = drain(collection)) + collection.updatePriorities { error("Empty collection") } + assertEquals(expected = 0, actual = collection.count) + } + + private fun drain(collection: DeterministicPriorityCollection): List { + val elements = mutableListOf() + while (collection.count > 0) { + val element = collection.peek() + collection.remove(element) + elements.add(element) + } + return elements + } +} From deca5d3bc0c505aa5efd8a597d234476e94f533e Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 17:09:50 +0300 Subject: [PATCH 05/15] Use unordered sets for coverage goals and snapshots --- .../kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt | 12 ++++++------ .../usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt index 6dc86cdfcf..416043bf11 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -21,9 +21,9 @@ class CoverageGuidedPathSelector( require(priorityRefreshInterval > 0) { "Priority refresh interval must be positive" } } - private val knownTargets = targets.toMutableSet() - private val pendingTargets = targets.toMutableSet() - private var priorityTargets: Set = targets.toSet() + private val knownTargets = targets.toHashSet() + private val pendingTargets = targets.toHashSet() + private var priorityTargets: Set = targets.toHashSet() private val distanceToTarget = distance private var prioritiesChanged = false private var completedSelections = 0 @@ -36,7 +36,7 @@ class CoverageGuidedPathSelector( ) val uncoveredTargets: Set - get() = pendingTargets.toSet() + get() = pendingTargets.toHashSet() /** Registers newly identified goals; previously covered goals stay covered. */ fun addTargets(targets: Collection): Boolean { @@ -50,7 +50,7 @@ class CoverageGuidedPathSelector( /** Unknown goals are ignored. Changed priorities are applied at the next eligible selection. */ fun markCovered(targets: Collection): Boolean { - val changed = pendingTargets.removeAll(targets.toSet()) + val changed = pendingTargets.removeAll(targets.toHashSet()) if (changed) { prioritiesChanged = pendingTargets.isNotEmpty() if (!prioritiesChanged) { @@ -67,7 +67,7 @@ class CoverageGuidedPathSelector( override fun peek(): State { val refreshDue = prioritiesChanged && completedSelections >= priorityRefreshInterval if (refreshDue) { - val refreshedTargets = pendingTargets.toSet() + val refreshedTargets = pendingTargets.toHashSet() priorities.updatePriorities { state -> weight(state, refreshedTargets) } priorityTargets = refreshedTargets prioritiesChanged = false diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt index 62b2200096..9421b89c4c 100644 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt @@ -24,7 +24,7 @@ class CoverageGuidedPathSelectorRefreshTest { assertEquals(expected = "left", actual = selector.peek()) evaluatedTargets.clear() selector.update(state = "left") - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) } evaluatedTargets.clear() repeat(256) { @@ -214,7 +214,7 @@ class CoverageGuidedPathSelectorRefreshTest { evaluatedTargets.clear() selector.add(states = listOf("other")) - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) assertEquals(expected = "right", actual = selector.peek()) assertEquals(expected = setOf("left", "right", "other"), actual = selector.drain().toSet()) } From 3ecbad1ed98d00abebe91417afe898ec57e600d6 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 17:14:50 +0300 Subject: [PATCH 06/15] Measure coverage selector costs on real JVM programs --- .../coverage-selector-performance/README.md | 91 +++++ .../coverage-selector-performance/logback.xml | 3 + .../manifest.json | 36 ++ .../performance.gradle | 19 + .../results.jsonl | 16 + .../coverage-selector-performance/run.py | 83 +++++ .../summarize.py | 39 ++ .../coverage-selector-performance/summary.csv | 13 + .../CoverageSelectorPerformanceTest.kt | 336 ++++++++++++++++++ 9 files changed, 636 insertions(+) create mode 100644 docs/experiments/coverage-selector-performance/README.md create mode 100644 docs/experiments/coverage-selector-performance/logback.xml create mode 100644 docs/experiments/coverage-selector-performance/manifest.json create mode 100644 docs/experiments/coverage-selector-performance/performance.gradle create mode 100644 docs/experiments/coverage-selector-performance/results.jsonl create mode 100644 docs/experiments/coverage-selector-performance/run.py create mode 100644 docs/experiments/coverage-selector-performance/summarize.py create mode 100644 docs/experiments/coverage-selector-performance/summary.csv create mode 100644 usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md new file mode 100644 index 0000000000..bb43ada212 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/README.md @@ -0,0 +1,91 @@ +# Coverage selector performance on real JVM bytecode + +This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. + +## Recorded results (2026-10-16) + +The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. + +The retained dataset contains 16 runs and 1,328,129 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. + +| Program | Mode | Steps | Stop | Analysis seconds | Selector seconds | Selector share | Refresh ms | +| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | +| Codec | existing | 100,000 | steps | 17.145 | 0.134 | 0.78% | 0 | +| Codec | eager | 100,000 | steps | 15.300 | 4.284 | 27.96% | 13.86 | +| Codec | 128 | 100,000 | steps | 21.468 | 6.319 | 29.48% | 90.32 | +| Codec | 512 | 100,000 | steps | 19.110 | 5.842 | 30.61% | 146.87 | +| Guava gcd | existing | 61,538 | timeout | 120.167 | 0.235 | 0.20% | 0 | +| Guava gcd | eager | 57,141 | timeout | 120.024 | 10.822 | 9.02% | 5.36 | +| Guava gcd | 128 | 63,808 | timeout | 120.189 | 14.249 | 11.86% | 5.73 | +| Guava gcd | 512 | 51,566 | timeout | 120.269 | 8.584 | 7.14% | 3.48 | +| LoanExam | existing | 73,519 | exhausted | 2.227 | 0.069 | 3.08% | 0 | +| LoanExam | eager | 73,519 | exhausted | 2.003 | 0.248 | 12.38% | 29.85 | +| LoanExam | 128 | 73,519 | exhausted | 2.333 | 0.266 | 11.39% | 27.08 | +| LoanExam | 512 | 73,519 | exhausted | 2.319 | 0.244 | 10.51% | 16.97 | + +For interval 128: + +| Program | Initial goals | Covered goals | Peak queued states | Per-target distance calls | Analysis-thread allocated GB (decimal) | +| --- | ---: | ---: | ---: | ---: | ---: | +| Codec | 273 | 74 | 3,364 | 23,826,680 | 5.952 | +| Guava gcd | 673 | 17 | 3,022 | 43,862,759 | 10.762 | +| LoanExam | 126 | 123 | 4,378 | 886,279 | 0.717 | + +All modes reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. + +Codec whole-analysis times varied substantially between repetitions: interval 128 took 15.007–27.930 seconds, and `existing` took 14.066–20.224 seconds. Its per-target callback count was identical in both repetitions, and its selector share was 29.33–29.62%. Do not treat the median wall-time ratio as a stable speedup estimate. + +Coverage-guided runs produced only 1–26 peeks with distance refresh work per analysis. They do not establish the worst-case cost of continuously discovering new goals or changing coverage while the queue is large. Initial goals are static; dynamic `addTargets` performance is not exercised. + +A useful next change is a caller-supplied batched minimum-distance function over the snapshot, allowing domain-specific caches such as the aggregate `CallStackDistanceCalculator`. A generic per-target callback may depend on arbitrary mutable state, so the selector cannot safely introduce a statement/call-stack-only cache for every caller. The proposed batched API itself has not been implemented or benchmarked here. + +The production selector files match the recorded source hashes throughout the retained runs. The manifest records the final reproducible harness; setup-helper extraction and report formatting were refined between exploratory runs and the repeated Codec runs, without changes to the measured execution loop, selector algorithms or settings. + +Raw data: [results.jsonl](results.jsonl). Full median summary: [summary.csv](summary.csv). Source/settings checksums: [manifest.json](manifest.json). + +Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 16 retained experiment processes completed successfully; Guava analyses ended at the explicit time limit. The experiment is excluded from ordinary JVM test runs. + +## Programs and setup + +- Apache Commons Codec 1.16.0: `org.apache.commons.codec.binary.BinaryCodec.toAsciiBytes(byte[])`. +- Guava 32.0.1-jre: `com.google.common.math.IntMath.gcd(int, int)`. +- Repository sample: `org.usvm.samples.psbenchmarks.LoanExam.getCreditPercent(CreditRequest)`. + +Coverage goals are the statements of all declared, non-abstract, non-native, non-constructor methods in the entry point's class, including static initialization. Coverage is reported by the standard `CoverageStatistics` observer for terminated reachable symbolic states. Class-wide goals intentionally include methods not visited by a particular entry point. + +Each analysis allows 100,000 interpreter steps or 120 seconds, whichever comes first. The timeout is checked between steps, so an in-flight solver operation may overshoot. Reaching the time limit is recorded as `stop=timeout`, not treated as exhaustive analysis. There is no early stop for coverage saturation or coverage stagnation, and no loop-iteration cap. + +Settings: Yices, 250 ms solver/type-operation timeout, implicit exceptions enabled, 1,500-element array bound, soft constraints disabled, 4 GiB Java heap, assertions disabled. Each run uses a fresh test JVM. JacoDB/classpath setup and Gradle startup are outside the analysis timer; initial queue insertion and the complete execution loop are inside it. These are cold-start analysis measurements, not JMH steady-state measurements. + +The recorded host was an Apple M3 Pro (arm64), 18 GiB RAM, Zulu JDK 11.0.32.1+1. Runs were sequential; other host activity was not controlled. + +## Selectors + +- `existing`: the aggregate `CallStackDistanceCalculator` / `WeightedPathSelector` strategy used by `CLOSEST_TO_UNCOVERED`. Coverage reports remove goals from that calculator. Queued state weights are refreshed on normal updates, without a global rebuild. +- `eager`: the original #526 algorithm, refreshing via `drain().toList()` and re-adding states after changed coverage. It uses the same cached-ordering fix and queue implementation as the periodic variants. +- `128` and `512`: the new `CoverageGuidedPathSelector`, using `HashSet` snapshots and the respective refresh interval. + +The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. + +`selectorMs` measures add/update/remove/peek and coverage-report handling, including distance evaluation. `refreshMs` identifies distance work inside `peek` and includes heap rebuilding plus selecting its root. `distanceCalls` counts per-target callbacks in eager/periodic modes, but aggregate calculator calls in `existing`; those are different units of work. `allocatedBytes` is all allocation on the analysis thread, not retained heap, native memory, or allocation attributed specifically to the selector. + +## Reproduction + +Use JDK 11 and cached project dependencies. The runner locates the two library jars in the Gradle cache, or accepts explicit `--codec-jar` and `--guava-jar` paths. It runs Gradle offline and retains per-run logs, raw JSONL measurements and process outcomes. + +```sh +JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ + --cases codec,gcd,loan --modes existing,128,eager,512 \ + --steps 100000 --timeout-ms 120000 --repetitions 1 \ + --output build/coverage-selector-performance + +python3 docs/experiments/coverage-selector-performance/summarize.py \ + build/coverage-selector-performance/results.jsonl +``` + +The experiment is tagged `manual` and excluded from ordinary test runs. The runner's dedicated Gradle task selects only this experiment and disables assertions, avoiding the debug duplicate-element scan in queue insertion. + +Library SHA-256: + +- Commons Codec: `56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d`. +- Guava: `bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5`. diff --git a/docs/experiments/coverage-selector-performance/logback.xml b/docs/experiments/coverage-selector-performance/logback.xml new file mode 100644 index 0000000000..463a09ebff --- /dev/null +++ b/docs/experiments/coverage-selector-performance/logback.xml @@ -0,0 +1,3 @@ + + + diff --git a/docs/experiments/coverage-selector-performance/manifest.json b/docs/experiments/coverage-selector-performance/manifest.json new file mode 100644 index 0000000000..8d44a5cd67 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/manifest.json @@ -0,0 +1,36 @@ +{ + "recordedDate": "2026-10-16", + "productionRevision": "deca5d3bc", + "cpu": "Apple M3 Pro", + "architecture": "arm64", + "memoryBytes": 19327352832, + "java": "Zulu 11.0.32.1+1", + "testHeapBytes": 4294967296, + "settings": { + "stepLimit": 100000, + "timeoutMillis": 120000, + "solver": "YICES", + "solverTimeoutMillis": 250, + "typeTimeoutMillis": 250, + "arrayMaxSize": 1500, + "useSoftConstraints": false, + "assertions": false, + "dynamicGoals": false + }, + "counts": { + "runs": 16, + "interpreterSteps": 1328129 + }, + "librarySha256": { + "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", + "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" + }, + "sourceSha256": { + "usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt": "5905b62b890e94b7390058e46c07fbd9dbb7285521ccc3c65abfceb458ce6b1e", + "usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt": "e55fd73657139ec726574f5ddfca43eccfe8d615520b5c30111146a65019bc32", + "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "602ecb571d50078a628fcc0b8d05ee7259e1a67949040b752db52dcae4b890cd", + "docs/experiments/coverage-selector-performance/run.py": "fa4dfa0bc6a328daf8bcdb736c08a719581bdf8bd46f29e388f03e77a74da0f5", + "docs/experiments/coverage-selector-performance/performance.gradle": "119f882bfba0d7222daaab9a844e7f7498b5a4b82600d9fd108ffe7ecab2bfbf" + }, + "harnessProvenance": "Final reproducible harness; setup-helper extraction and report-formatting refinements occurred during exploratory runs. Production selector sources, measured execution loop and settings were unchanged." +} diff --git a/docs/experiments/coverage-selector-performance/performance.gradle b/docs/experiments/coverage-selector-performance/performance.gradle new file mode 100644 index 0000000000..adc379a57e --- /dev/null +++ b/docs/experiments/coverage-selector-performance/performance.gradle @@ -0,0 +1,19 @@ +gradle.projectsEvaluated { + def jvm = gradle.rootProject.findProject(':usvm-jvm') + if (jvm == null) return + jvm.tasks.register('coverageSelectorPerformance', Test) { + testClassesDirs = jvm.sourceSets.test.output.classesDirs + classpath = jvm.sourceSets.test.runtimeClasspath + dependsOn(jvm.tasks.named('testClasses')) + useJUnitPlatform { includeTags 'manual' } + filter { includeTestsMatching 'org.usvm.experiments.CoverageSelectorPerformanceTest' } + enableAssertions = false + maxHeapSize = '4g' + def logging = jvm.rootProject.file('docs/experiments/coverage-selector-performance/logback.xml') + jvmArgs "-Dlogback.configurationFile=${logging.absolutePath}" + ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition'].each { key -> + systemProperty("coverage.perf.${key}", jvm.findProperty("coveragePerf${key.capitalize()}")) + } + outputs.upToDateWhen { false } + } +} diff --git a/docs/experiments/coverage-selector-performance/results.jsonl b/docs/experiments/coverage-selector-performance/results.jsonl new file mode 100644 index 0000000000..ff93c66817 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/results.jsonl @@ -0,0 +1,16 @@ +{"program":"gcd","mode":"existing","steps":61538,"stop":"timeout","elapsedMs":120167.436333,"selectorMs":235.437487,"refreshMs":0.0,"refreshes":0,"distanceCalls":64765,"peakQueue":3227,"pending":3227,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":856651016,"gcCount":5,"gcMs":449,"repetition":1} +{"program":"gcd","mode":"128","steps":63808,"stop":"timeout","elapsedMs":120189.039625,"selectorMs":14249.244039,"refreshMs":5.728084,"refreshes":2,"distanceCalls":43862759,"peakQueue":3022,"pending":3022,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":10761566008,"gcCount":10,"gcMs":304,"repetition":1} +{"program":"gcd","mode":"eager","repetition":1,"steps":57141,"stop":"timeout","elapsedMs":120024.317916,"selectorMs":10822.395495,"refreshMs":5.358375,"refreshes":3,"distanceCalls":39557171,"peakQueue":3124,"pending":3124,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":9568604488,"gcCount":12,"gcMs":435} +{"program":"gcd","mode":"512","repetition":1,"steps":51566,"stop":"timeout","elapsedMs":120269.191542,"selectorMs":8584.144519,"refreshMs":3.480708,"refreshes":1,"distanceCalls":35593159,"peakQueue":2655,"pending":2655,"goals":673,"covered":17,"coverageReports":17,"terminated":5,"allocatedBytes":8698358712,"gcCount":9,"gcMs":175} +{"program":"loan","mode":"existing","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2227.168709,"selectorMs":68.547639,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":513509984,"gcCount":2,"gcMs":26} +{"program":"loan","mode":"128","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2332.535792,"selectorMs":265.641903,"refreshMs":27.08429,"refreshes":18,"distanceCalls":886279,"peakQueue":4378,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":716986848,"gcCount":2,"gcMs":24} +{"program":"loan","mode":"eager","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2003.228,"selectorMs":248.054217,"refreshMs":29.854126,"refreshes":26,"distanceCalls":944047,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":734125672,"gcCount":2,"gcMs":36} +{"program":"loan","mode":"512","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2319.290542,"selectorMs":243.802455,"refreshMs":16.965501,"refreshes":12,"distanceCalls":760175,"peakQueue":4360,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":701963040,"gcCount":2,"gcMs":19} +{"program":"codec","mode":"existing","repetition":1,"steps":100000,"stop":"steps","elapsedMs":14065.793084,"selectorMs":103.386872,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2071408672,"gcCount":5,"gcMs":327} +{"program":"codec","mode":"128","repetition":1,"steps":100000,"stop":"steps","elapsedMs":27929.542917,"selectorMs":8192.190751,"refreshMs":93.001749,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952381656,"gcCount":10,"gcMs":367} +{"program":"codec","mode":"eager","repetition":1,"steps":100000,"stop":"steps","elapsedMs":17828.648083,"selectorMs":5027.923518,"refreshMs":13.558208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5478178352,"gcCount":12,"gcMs":287} +{"program":"codec","mode":"512","repetition":1,"steps":100000,"stop":"steps","elapsedMs":18655.336042,"selectorMs":6038.100888,"refreshMs":193.471708,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":6332651112,"gcCount":9,"gcMs":260} +{"program":"codec","mode":"512","repetition":2,"steps":100000,"stop":"steps","elapsedMs":19563.939083,"selectorMs":5645.342032,"refreshMs":100.26425,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5946781568,"gcCount":8,"gcMs":210} +{"program":"codec","mode":"eager","repetition":2,"steps":100000,"stop":"steps","elapsedMs":12771.742584,"selectorMs":3540.902094,"refreshMs":14.152208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5479713280,"gcCount":8,"gcMs":186} +{"program":"codec","mode":"128","repetition":2,"steps":100000,"stop":"steps","elapsedMs":15007.014042,"selectorMs":4445.708042,"refreshMs":87.639833,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952287184,"gcCount":7,"gcMs":173} +{"program":"codec","mode":"existing","repetition":2,"steps":100000,"stop":"steps","elapsedMs":20224.028875,"selectorMs":165.022394,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2078604128,"gcCount":6,"gcMs":202} diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py new file mode 100644 index 0000000000..fea8bdeb59 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/run.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Run isolated JVM analyses; keep successes, failures and timeouts in the output directory.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import time + + +def cached_jar(group, artifact, version): + cache = Path.home() / '.gradle/caches/modules-2/files-2.1' + matches = list((cache / group / artifact / version).glob(f'*/{artifact}-{version}.jar')) + if len(matches) != 1: + raise RuntimeError(f'Pass the jar explicitly: {group}:{artifact}:{version}') + return matches[0] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cases', default='codec,gcd,loan') + parser.add_argument('--modes', default='existing,128,eager,512') + parser.add_argument('--steps', type=int, default=100000) + parser.add_argument('--timeout-ms', type=int, default=120000) + parser.add_argument('--repetitions', type=int, default=2) + parser.add_argument('--output', type=Path) + parser.add_argument('--codec-jar', type=Path) + parser.add_argument('--guava-jar', type=Path) + args = parser.parse_args() + + experiment = Path(__file__).resolve().parent + root = experiment.parents[2] + output = (args.output or root / 'build/coverage-selector-performance').resolve() + output.mkdir(parents=True, exist_ok=True) + codec = args.codec_jar or cached_jar('commons-codec', 'commons-codec', '1.16.0') + guava = args.guava_jar or cached_jar('com.google.guava', 'guava', '32.0.1-jre') + jars = os.pathsep.join(str(path.resolve()) for path in [codec, guava]) + cases = args.cases.split(',') + modes = args.modes.split(',') + results_file = output / 'results.jsonl' + outcomes_file = output / 'outcomes.jsonl' + + for repetition in range(1, args.repetitions + 1): + # Alternate mode order to reduce a fixed ordering effect. Runs remain sequential. + run_modes = modes if repetition % 2 else list(reversed(modes)) + for case in cases: + for mode in run_modes: + label = f'{case}-{mode}-r{repetition}' + print(f'START {label}', flush=True) + log_file = output / f'{label}.log' + command = [ + str(root / 'gradlew'), ':usvm-jvm:coverageSelectorPerformance', + '-I', str(experiment / 'performance.gradle'), + f'-PcoveragePerfCase={case}', f'-PcoveragePerfMode={mode}', + f'-PcoveragePerfSteps={args.steps}', + f'-PcoveragePerfTimeoutMillis={args.timeout_ms}', + f'-PcoveragePerfRepetition={repetition}', + f'-PcoveragePerfResults={results_file}', f'-PcoveragePerfJars={jars}', + '--offline', '--console=plain', + ] + started = time.monotonic() + outcome = dict(program=case, mode=mode, repetition=repetition, log=log_file.name) + try: + with log_file.open('w') as log: + process = subprocess.run( + command, cwd=root, env=os.environ.copy(), stdout=log, + stderr=subprocess.STDOUT, timeout=args.timeout_ms / 1000 + 180, + ) + outcome['exitCode'] = process.returncode + outcome['status'] = 'completed' if process.returncode == 0 else 'failed' + except subprocess.TimeoutExpired: + outcome['status'] = 'process-timeout' + outcome['wallSeconds'] = round(time.monotonic() - started, 3) + with outcomes_file.open('a') as outcomes: + outcomes.write(json.dumps(outcome) + '\n') + print(json.dumps(outcome), flush=True) + if outcome['status'] != 'completed': + print(log_file.read_text()[-4000:], flush=True) + + +if __name__ == '__main__': + main() diff --git a/docs/experiments/coverage-selector-performance/summarize.py b/docs/experiments/coverage-selector-performance/summarize.py new file mode 100644 index 0000000000..6b0a63e766 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/summarize.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Write per-program/per-mode medians from benchmark JSONL to CSV.""" + +import argparse +import csv +import json +from collections import defaultdict +from pathlib import Path +from statistics import median +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('results', type=Path) + args = parser.parse_args() + groups = defaultdict(list) + for line in args.results.read_text().splitlines(): + row = json.loads(line) + groups[row['program'], row['mode']].append(row) + + fields = ['program', 'mode', 'runs', 'stops', 'steps', 'elapsedMs', 'selectorMs', + 'selectorPercent', 'refreshMs', 'refreshes', 'distanceCalls', 'peakQueue', + 'covered', 'goals', 'allocatedBytes', 'gcMs'] + writer = csv.DictWriter(sys.stdout, fieldnames=fields, lineterminator='\n') + writer.writeheader() + for (program, mode), rows in sorted(groups.items()): + summary = dict(program=program, mode=mode, runs=len(rows)) + summary['stops'] = ','.join(sorted({row['stop'] for row in rows})) + for field in fields[4:]: + if field == 'selectorPercent': + summary[field] = median(100 * row['selectorMs'] / row['elapsedMs'] for row in rows) + else: + summary[field] = median(row[field] for row in rows) + writer.writerow(summary) + + +if __name__ == '__main__': + main() diff --git a/docs/experiments/coverage-selector-performance/summary.csv b/docs/experiments/coverage-selector-performance/summary.csv new file mode 100644 index 0000000000..0baf0df1c0 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/summary.csv @@ -0,0 +1,13 @@ +program,mode,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs +codec,128,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 +codec,512,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 +codec,eager,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 +codec,existing,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 +gcd,128,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 +gcd,512,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 +gcd,eager,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 +gcd,existing,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 +loan,128,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 +loan,512,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 +loan,eager,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 +loan,existing,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt new file mode 100644 index 0000000000..829671d918 --- /dev/null +++ b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt @@ -0,0 +1,336 @@ +package org.usvm.experiments + +import kotlinx.coroutines.runBlocking +import org.jacodb.api.jvm.JcClasspath +import org.jacodb.api.jvm.JcMethod +import org.jacodb.api.jvm.cfg.JcInst +import org.jacodb.api.jvm.ext.findClass +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test +import org.usvm.SolverType +import org.usvm.UMachine +import org.usvm.UMachineOptions +import org.usvm.UPathSelector +import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.machine.JcApplicationGraph +import org.usvm.machine.JcComponents +import org.usvm.machine.JcContext +import org.usvm.machine.JcMachineOptions +import org.usvm.machine.JcTypeSystem +import org.usvm.machine.interpreter.JcInterpreter +import org.usvm.machine.state.JcState +import org.usvm.ps.BfsPathSelector +import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.WeightedPathSelector +import org.usvm.samples.JacoDBContainer +import org.usvm.statistics.CompositeUMachineObserver +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.UMachineObserver +import org.usvm.statistics.distances.CallStackDistanceCalculator +import org.usvm.statistics.distances.CfgStatisticsImpl +import org.usvm.statistics.distances.MultiTargetDistanceCalculator +import org.usvm.stopstrategies.StopStrategy +import org.usvm.util.originalInst +import java.io.File +import java.lang.management.ManagementFactory +import kotlin.time.Duration.Companion.milliseconds + +/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ +@Tag("manual") +class CoverageSelectorPerformanceTest { + @Test + fun runProgram() { + val program = System.getProperty("coverage.perf.case", "codec") + val mode = System.getProperty("coverage.perf.mode", "128") + val steps = System.getProperty("coverage.perf.steps", "100000").toLong() + val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() + val resultFile = File(System.getProperty("coverage.perf.results")) + val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) + val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) + val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) + try { + val (className, methodName) = when (program) { + "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" + "gcd" -> "com.google.common.math.IntMath" to "gcd" + "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" + else -> error("Unknown program: $program") + } + val cls = container.cp.findClass(className) + val method = cls.declaredMethods.single { it.name == methodName } + val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } + PerformanceMachine(container.cp).use { machine -> + val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) + resultFile.parentFile.mkdirs() + resultFile.appendText(result + "\n") + } + } finally { + runBlocking { + container.cp.close() + container.db.close() + } + } + } +} + +private class PerformanceMachine(cp: JcClasspath) : UMachine() { + override val options = UMachineOptions( + solverType = SolverType.YICES, + solverTimeout = 250.milliseconds, + typeOperationsTimeout = 250.milliseconds, + throwExceptionOnStepFailure = true, + useSoftConstraints = false, + ) + private val graph = JcApplicationGraph(cp) + private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) + private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) + + fun measure( + program: String, + mode: String, + method: JcMethod, + methodsToCover: List, + stepLimit: Long, + timeoutMillis: Long, + ): String { + val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) + val goals = coverage.getUncoveredStatements() + val metrics = Metrics() + val (base, markCovered) = createGuidance(mode, goals, metrics) + val selector = MeasuredSelector(base, metrics) + coverage.addOnCoveredObserver { _, _, statement -> + val started = System.nanoTime() + markCovered(statement) + metrics.selectorNanos += System.nanoTime() - started + metrics.coverageReports++ + } + val observer = metrics.observer() + val initial = interpreter.getInitialState(method) + val gcCountBefore = gcCount() + val gcMillisBefore = gcMillis() + val allocatedBefore = allocatedBytes() + val started = System.nanoTime() + var stop = "exhausted" + selector.add(states = listOf(initial)) + run( + interpreter = interpreter, + pathSelector = selector, + observer = CompositeUMachineObserver(coverage, observer), + isStateTerminated = { state -> state.callStack.isEmpty() }, + stopStrategy = StopStrategy { + val elapsedMillis = (System.nanoTime() - started) / 1_000_000 + when { + metrics.steps >= stepLimit -> { + stop = "steps" + true + } + elapsedMillis >= timeoutMillis -> { + stop = "timeout" + true + } + else -> { + false + } + } + }, + ) + val elapsedNanos = System.nanoTime() - started + val allocated = allocatedBytes() - allocatedBefore + val totals = AnalysisTotals( + elapsedNanos = elapsedNanos, + allocatedBytes = allocated, + gcCount = gcCount() - gcCountBefore, + gcMillis = gcMillis() - gcMillisBefore, + ) + return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) + } + + private fun createGuidance( + mode: String, + goals: Collection, + metrics: Metrics, + ): Pair, (JcInst) -> Unit> { + val cfg = CfgStatisticsImpl(graph) + val calculators = MultiTargetDistanceCalculator { target -> + CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) + } + val distance: (JcState, JcInst) -> UInt = { state, target -> + metrics.distanceCalls++ + calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) + } + val markCovered: (JcInst) -> Unit + val base: UPathSelector + if (mode == "eager") { + val eager = EagerCoverageSelector(goals, distance) + base = eager + markCovered = { eager.markCovered(it) } + } else if (mode == "existing") { + val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) + base = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state: JcState -> + metrics.distanceCalls++ + aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) + }, + ) + markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } + } else if (mode == "bfs") { + base = BfsPathSelector() + markCovered = {} + } else { + val periodic = CoverageGuidedPathSelector( + targets = goals, + distance = distance, + priorityRefreshInterval = mode.toInt(), + ) + base = periodic + markCovered = { periodic.markCovered(targets = listOf(it)) } + } + return base to markCovered + } + + override fun close() = components.close() +} + +private class Metrics { + var steps = 0L + var terminated = 0L + var selectorNanos = 0L + var refreshNanos = 0L + var refreshes = 0L + var distanceCalls = 0L + var coverageReports = 0L + var queueSize = 0 + var peakQueue = 0 +} + +private class MeasuredSelector( + private val delegate: UPathSelector, + private val metrics: Metrics, +) : UPathSelector { + override fun isEmpty(): Boolean = delegate.isEmpty() + + override fun peek(): JcState { + val beforeCalls = metrics.distanceCalls + val started = System.nanoTime() + val result = delegate.peek() + val elapsed = System.nanoTime() - started + metrics.selectorNanos += elapsed + if (metrics.distanceCalls > beforeCalls) { + metrics.refreshes++ + metrics.refreshNanos += elapsed + } + return result + } + + override fun add(states: Collection) { + val started = System.nanoTime() + delegate.add(states) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize += states.size + metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) + } + + override fun update(state: JcState) { + val started = System.nanoTime() + delegate.update(state) + metrics.selectorNanos += System.nanoTime() - started + } + + override fun remove(state: JcState) { + val started = System.nanoTime() + delegate.remove(state) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize-- + } +} + +/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ +private class EagerCoverageSelector( + targets: Collection, + distance: (JcState, JcInst) -> UInt, +) : UPathSelector { + private val pending = targets.toMutableSet() + private var dirty = false + private val delegate = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, + ) + + fun markCovered(target: JcInst) { + if (pending.remove(target)) dirty = true + } + + override fun isEmpty(): Boolean = delegate.isEmpty() + override fun peek(): JcState { + if (dirty) { + val states = delegate.drain().toList() + delegate.add(states) + dirty = false + } + return delegate.peek() + } + override fun add(states: Collection) = delegate.add(states) + override fun update(state: JcState) = delegate.update(state) + override fun remove(state: JcState) = delegate.remove(state) +} + +private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionCount.coerceAtLeast(minimumValue = 0) +} + +private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionTime.coerceAtLeast(minimumValue = 0) +} +private fun allocatedBytes(): Long { + val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean + return bean.getThreadAllocatedBytes(Thread.currentThread().id) +} + +private data class AnalysisTotals( + val elapsedNanos: Long, + val allocatedBytes: Long, + val gcCount: Long, + val gcMillis: Long, +) + +private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { + override fun onStatePeeked(state: JcState) { + steps++ + } + + override fun onStateTerminated(state: JcState, stateReachable: Boolean) { + terminated++ + } +} + +private fun Metrics.report( + program: String, + mode: String, + goals: Int, + covered: Int, + stop: String, + totals: AnalysisTotals, +): String { + val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() + return """ + {"program":"$program", + "mode":"$mode", + "repetition":$repetition, + "steps":$steps, + "stop":"$stop", + "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, + "selectorMs":${selectorNanos / 1_000_000.0}, + "refreshMs":${refreshNanos / 1_000_000.0}, + "refreshes":$refreshes, + "distanceCalls":$distanceCalls, + "peakQueue":$peakQueue, + "pending":$queueSize, + "goals":$goals, + "covered":$covered, + "coverageReports":$coverageReports, + "terminated":$terminated, + "allocatedBytes":${totals.allocatedBytes}, + "gcCount":${totals.gcCount}, + "gcMs":${totals.gcMillis}} + """.trimIndent().replace("\n", "") +} From 44869512bca6b1ce89511b9f6799f4475efdf972 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 17:25:35 +0300 Subject: [PATCH 07/15] Compare coverage guidance with random selection baselines --- .../coverage-selector-performance/README.md | 45 +++++++++++++++++-- .../manifest.json | 18 +++++--- .../performance.gradle | 2 +- .../results.jsonl | 12 +++++ .../coverage-selector-performance/run.py | 7 +-- .../summarize.py | 16 ++++--- .../coverage-selector-performance/summary.csv | 38 ++++++++++------ .../CoverageSelectorPerformanceTest.kt | 34 +++++++++++++- 8 files changed, 136 insertions(+), 36 deletions(-) diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md index bb43ada212..fa9cbe8ac9 100644 --- a/docs/experiments/coverage-selector-performance/README.md +++ b/docs/experiments/coverage-selector-performance/README.md @@ -6,7 +6,7 @@ This experiment drives the production `JcInterpreter` through `UMachine.run`, wi The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. -The retained dataset contains 16 runs and 1,328,129 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. +The distance-guidance comparison below contains 16 runs and 1,328,129 interpreter steps. The random comparison adds 12 runs; the complete retained dataset contains 28 runs and 2,422,205 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. | Program | Mode | Steps | Stop | Analysis seconds | Selector seconds | Selector share | Refresh ms | | --- | --- | ---: | --- | ---: | ---: | ---: | ---: | @@ -31,7 +31,7 @@ For interval 128: | Guava gcd | 673 | 17 | 3,022 | 43,862,759 | 10.762 | | LoanExam | 126 | 123 | 4,378 | 886,279 | 0.717 | -All modes reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. +All distance-guidance modes in the preceding table reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. Codec whole-analysis times varied substantially between repetitions: interval 128 took 15.007–27.930 seconds, and `existing` took 14.066–20.224 seconds. Its per-target callback count was identical in both repetitions, and its selector share was 29.33–29.62%. Do not treat the median wall-time ratio as a stable speedup estimate. @@ -43,7 +43,36 @@ The production selector files match the recorded source hashes throughout the re Raw data: [results.jsonl](results.jsonl). Full median summary: [summary.csv](summary.csv). Source/settings checksums: [manifest.json](manifest.json). -Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 16 retained experiment processes completed successfully; Guava analyses ended at the explicit time limit. The experiment is excluded from ordinary JVM test runs. +Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 28 retained experiment processes completed successfully. The four distance-guidance Guava analyses ended at the explicit time limit; the random Guava runs reached 100,000 steps. The experiment is excluded from ordinary JVM test runs. + + +## Random baselines + +Two additional strategies were measured once for each selector RNG seed, 0 and 1, with the same step/time limits and production interpreter: + +- `random-path`: the actual `RandomTreePathSelector.fromRoot` used by `RANDOM_PATH`; it samples branches of the execution tree and traverses that tree on selection. +- `uniform`: the existing `RandomizedPriorityCollection` with constant weight 1.0 in a `WeightedPathSelector`, giving uniform random selection among queued states. It performs no goal-distance calculations and does not traverse the execution tree. + +| Program | Mode | Seed | Steps | Analysis seconds | Selector seconds | Covered goals | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| codec | random-path | 0 | 100,000 | 13.359 | 6.585 | 74 | +| codec | random-path | 1 | 100,000 | 7.283 | 3.991 | 74 | +| codec | uniform | 0 | 100,000 | 3.937 | 0.339 | 74 | +| codec | uniform | 1 | 100,000 | 3.430 | 0.315 | 74 | +| gcd | random-path | 0 | 100,000 | 19.171 | 3.411 | 28 | +| gcd | random-path | 1 | 100,000 | 18.281 | 3.186 | 28 | +| gcd | uniform | 0 | 100,000 | 10.153 | 0.572 | 28 | +| gcd | uniform | 1 | 100,000 | 7.328 | 0.351 | 28 | +| loan | random-path | 0 | 73,519 | 3.868 | 0.883 | 123 | +| loan | random-path | 1 | 73,519 | 5.003 | 1.155 | 123 | +| loan | uniform | 0 | 73,519 | 3.046 | 0.254 | 123 | +| loan | uniform | 1 | 73,519 | 2.745 | 0.224 | 123 | + +Uniform random was a strong inexpensive baseline on both library methods in these runs. Codec reached the same covered-goal count (74) in 3.430–3.937 seconds, versus a 21.468-second median for interval 128. Guava uniform random reached 100,000 steps and 28 covered goals in 7.328–10.153 seconds; interval 128 reached 63,808 steps and 17 covered goals at the 120-second timeout. LoanExam is a counterexample: uniform random took 2.745–3.046 seconds, versus 2.333 seconds for interval 128, with the same 73,519 steps and 123 covered goals. + +The standard `RANDOM_PATH` strategy was itself relatively expensive on Codec: 3.991–6.585 seconds spent in its selector, versus 0.315–0.339 seconds for uniform random. A random strategy's data structure and selection distribution matter. + +Different exploration order changes work outside the selector as well. Guava's wall-time difference cannot be explained by selector overhead alone; its interpreter/solver cost has not been separately profiled. Two seeds and three programs support using cheap uniform random as a baseline, not a universal claim that it dominates all coverage guidance. Goal counts do not establish equality of covered-goal sets or concrete replay coverage. ## Programs and setup @@ -65,7 +94,7 @@ The recorded host was an Apple M3 Pro (arm64), 18 GiB RAM, Zulu JDK 11.0.32.1+1. - `eager`: the original #526 algorithm, refreshing via `drain().toList()` and re-adding states after changed coverage. It uses the same cached-ordering fix and queue implementation as the periodic variants. - `128` and `512`: the new `CoverageGuidedPathSelector`, using `HashSet` snapshots and the respective refresh interval. -The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. +For the random strategies, the selector RNG seed is recorded separately for every row and CSV group. The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. `selectorMs` measures add/update/remove/peek and coverage-report handling, including distance evaluation. `refreshMs` identifies distance work inside `peek` and includes heap rebuilding plus selecting its root. `distanceCalls` counts per-target callbacks in eager/periodic modes, but aggregate calculator calls in `existing`; those are different units of work. `allocatedBytes` is all allocation on the analysis thread, not retained heap, native memory, or allocation attributed specifically to the selector. @@ -79,6 +108,14 @@ JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/ --steps 100000 --timeout-ms 120000 --repetitions 1 \ --output build/coverage-selector-performance +JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ + --cases codec,gcd,loan --modes random-path,uniform --seed 0 --repetitions 1 \ + --output build/coverage-selector-performance + +JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ + --cases codec,gcd,loan --modes random-path,uniform --seed 1 --repetitions 1 \ + --output build/coverage-selector-performance + python3 docs/experiments/coverage-selector-performance/summarize.py \ build/coverage-selector-performance/results.jsonl ``` diff --git a/docs/experiments/coverage-selector-performance/manifest.json b/docs/experiments/coverage-selector-performance/manifest.json index 8d44a5cd67..1e41f1c164 100644 --- a/docs/experiments/coverage-selector-performance/manifest.json +++ b/docs/experiments/coverage-selector-performance/manifest.json @@ -15,11 +15,16 @@ "arrayMaxSize": 1500, "useSoftConstraints": false, "assertions": false, - "dynamicGoals": false + "dynamicGoals": false, + "randomSeeds": [ + 0, + 1 + ] }, "counts": { - "runs": 16, - "interpreterSteps": 1328129 + "runs": 28, + "interpreterSteps": 2422205, + "peakQueue": 4921 }, "librarySha256": { "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", @@ -28,9 +33,10 @@ "sourceSha256": { "usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt": "5905b62b890e94b7390058e46c07fbd9dbb7285521ccc3c65abfceb458ce6b1e", "usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt": "e55fd73657139ec726574f5ddfca43eccfe8d615520b5c30111146a65019bc32", - "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "602ecb571d50078a628fcc0b8d05ee7259e1a67949040b752db52dcae4b890cd", - "docs/experiments/coverage-selector-performance/run.py": "fa4dfa0bc6a328daf8bcdb736c08a719581bdf8bd46f29e388f03e77a74da0f5", - "docs/experiments/coverage-selector-performance/performance.gradle": "119f882bfba0d7222daaab9a844e7f7498b5a4b82600d9fd108ffe7ecab2bfbf" + "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "42c2c518e157ff8edca81044c7561c6832a9868eef0ef0514c223c9ee05209a2", + "docs/experiments/coverage-selector-performance/run.py": "9c6affa7309e0a6bbacfcf6fd5fdb005ee63817aa3d1a8103a3caf635c667a50", + "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0", + "docs/experiments/coverage-selector-performance/summarize.py": "67b695a12348503a3700d26b3fc938337ff9215a392353060ece05713dbf2fa2" }, "harnessProvenance": "Final reproducible harness; setup-helper extraction and report-formatting refinements occurred during exploratory runs. Production selector sources, measured execution loop and settings were unchanged." } diff --git a/docs/experiments/coverage-selector-performance/performance.gradle b/docs/experiments/coverage-selector-performance/performance.gradle index adc379a57e..8a301e7f07 100644 --- a/docs/experiments/coverage-selector-performance/performance.gradle +++ b/docs/experiments/coverage-selector-performance/performance.gradle @@ -11,7 +11,7 @@ gradle.projectsEvaluated { maxHeapSize = '4g' def logging = jvm.rootProject.file('docs/experiments/coverage-selector-performance/logback.xml') jvmArgs "-Dlogback.configurationFile=${logging.absolutePath}" - ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition'].each { key -> + ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition', 'seed'].each { key -> systemProperty("coverage.perf.${key}", jvm.findProperty("coveragePerf${key.capitalize()}")) } outputs.upToDateWhen { false } diff --git a/docs/experiments/coverage-selector-performance/results.jsonl b/docs/experiments/coverage-selector-performance/results.jsonl index ff93c66817..1fe7bae02f 100644 --- a/docs/experiments/coverage-selector-performance/results.jsonl +++ b/docs/experiments/coverage-selector-performance/results.jsonl @@ -14,3 +14,15 @@ {"program":"codec","mode":"eager","repetition":2,"steps":100000,"stop":"steps","elapsedMs":12771.742584,"selectorMs":3540.902094,"refreshMs":14.152208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5479713280,"gcCount":8,"gcMs":186} {"program":"codec","mode":"128","repetition":2,"steps":100000,"stop":"steps","elapsedMs":15007.014042,"selectorMs":4445.708042,"refreshMs":87.639833,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952287184,"gcCount":7,"gcMs":173} {"program":"codec","mode":"existing","repetition":2,"steps":100000,"stop":"steps","elapsedMs":20224.028875,"selectorMs":165.022394,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2078604128,"gcCount":6,"gcMs":202} +{"program":"codec","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":13359.287291,"selectorMs":6585.102705,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4615,"pending":4615,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513024960,"gcCount":7,"gcMs":539} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3936.713375,"selectorMs":338.87798,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1080632408,"gcCount":3,"gcMs":62} +{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":19170.84275,"selectorMs":3410.622702,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4921,"pending":4920,"goals":673,"covered":28,"coverageReports":28,"terminated":895,"allocatedBytes":2143921128,"gcCount":3,"gcMs":41} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10152.718417,"selectorMs":572.30041,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1025193416,"gcCount":5,"gcMs":409} +{"program":"loan","mode":"random-path","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3868.197,"selectorMs":882.931681,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1527,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":881496912,"gcCount":2,"gcMs":50} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3045.632291,"selectorMs":253.601349,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743016424,"gcCount":3,"gcMs":98} +{"program":"codec","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7282.688708,"selectorMs":3990.851612,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4562,"pending":4562,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513769624,"gcCount":3,"gcMs":111} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":3429.624834,"selectorMs":314.7182,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4780,"pending":4780,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1087211024,"gcCount":4,"gcMs":122} +{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":18280.874209,"selectorMs":3186.388872,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4898,"pending":4897,"goals":673,"covered":28,"coverageReports":28,"terminated":911,"allocatedBytes":2151045424,"gcCount":3,"gcMs":32} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7327.864792,"selectorMs":351.211791,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4444,"pending":4440,"goals":673,"covered":28,"coverageReports":28,"terminated":1521,"allocatedBytes":1029454208,"gcCount":4,"gcMs":259} +{"program":"loan","mode":"random-path","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":5003.377792,"selectorMs":1155.413364,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1513,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":878046240,"gcCount":2,"gcMs":39} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":2744.655583,"selectorMs":224.198052,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":2902,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":747822944,"gcCount":2,"gcMs":26} diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py index fea8bdeb59..590a47f71e 100644 --- a/docs/experiments/coverage-selector-performance/run.py +++ b/docs/experiments/coverage-selector-performance/run.py @@ -24,6 +24,7 @@ def main(): parser.add_argument('--steps', type=int, default=100000) parser.add_argument('--timeout-ms', type=int, default=120000) parser.add_argument('--repetitions', type=int, default=2) + parser.add_argument('--seed', type=int, default=0) parser.add_argument('--output', type=Path) parser.add_argument('--codec-jar', type=Path) parser.add_argument('--guava-jar', type=Path) @@ -46,7 +47,7 @@ def main(): run_modes = modes if repetition % 2 else list(reversed(modes)) for case in cases: for mode in run_modes: - label = f'{case}-{mode}-r{repetition}' + label = f'{case}-{mode}-s{args.seed}-r{repetition}' print(f'START {label}', flush=True) log_file = output / f'{label}.log' command = [ @@ -55,12 +56,12 @@ def main(): f'-PcoveragePerfCase={case}', f'-PcoveragePerfMode={mode}', f'-PcoveragePerfSteps={args.steps}', f'-PcoveragePerfTimeoutMillis={args.timeout_ms}', - f'-PcoveragePerfRepetition={repetition}', + f'-PcoveragePerfRepetition={repetition}', f'-PcoveragePerfSeed={args.seed}', f'-PcoveragePerfResults={results_file}', f'-PcoveragePerfJars={jars}', '--offline', '--console=plain', ] started = time.monotonic() - outcome = dict(program=case, mode=mode, repetition=repetition, log=log_file.name) + outcome = dict(program=case, mode=mode, repetition=repetition, randomSeed=args.seed, log=log_file.name) try: with log_file.open('w') as log: process = subprocess.run( diff --git a/docs/experiments/coverage-selector-performance/summarize.py b/docs/experiments/coverage-selector-performance/summarize.py index 6b0a63e766..5ba0af99d9 100644 --- a/docs/experiments/coverage-selector-performance/summarize.py +++ b/docs/experiments/coverage-selector-performance/summarize.py @@ -17,17 +17,19 @@ def main(): groups = defaultdict(list) for line in args.results.read_text().splitlines(): row = json.loads(line) - groups[row['program'], row['mode']].append(row) + seed = row.get('randomSeed') if row['mode'] in ['random-path', 'uniform'] else None + groups[row['program'], row['mode'], seed].append(row) - fields = ['program', 'mode', 'runs', 'stops', 'steps', 'elapsedMs', 'selectorMs', - 'selectorPercent', 'refreshMs', 'refreshes', 'distanceCalls', 'peakQueue', - 'covered', 'goals', 'allocatedBytes', 'gcMs'] + metrics = ['steps', 'elapsedMs', 'selectorMs', 'selectorPercent', 'refreshMs', + 'refreshes', 'distanceCalls', 'peakQueue', 'covered', 'goals', + 'allocatedBytes', 'gcMs'] + fields = ['program', 'mode', 'randomSeed', 'runs', 'stops', *metrics] writer = csv.DictWriter(sys.stdout, fieldnames=fields, lineterminator='\n') writer.writeheader() - for (program, mode), rows in sorted(groups.items()): - summary = dict(program=program, mode=mode, runs=len(rows)) + for (program, mode, seed), rows in sorted(groups.items()): + summary = dict(program=program, mode=mode, randomSeed=seed, runs=len(rows)) summary['stops'] = ','.join(sorted({row['stop'] for row in rows})) - for field in fields[4:]: + for field in metrics: if field == 'selectorPercent': summary[field] = median(100 * row['selectorMs'] / row['elapsedMs'] for row in rows) else: diff --git a/docs/experiments/coverage-selector-performance/summary.csv b/docs/experiments/coverage-selector-performance/summary.csv index 0baf0df1c0..fc5876db60 100644 --- a/docs/experiments/coverage-selector-performance/summary.csv +++ b/docs/experiments/coverage-selector-performance/summary.csv @@ -1,13 +1,25 @@ -program,mode,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs -codec,128,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 -codec,512,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 -codec,eager,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 -codec,existing,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 -gcd,128,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 -gcd,512,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 -gcd,eager,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 -gcd,existing,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 -loan,128,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 -loan,512,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 -loan,eager,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 -loan,existing,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 +program,mode,randomSeed,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs +codec,128,,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 +codec,512,,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 +codec,eager,,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 +codec,existing,,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 +codec,random-path,0,1,steps,100000,13359.287291,6585.102705,49.29232047757748,0.0,0,0,4615,74,273,2513024960,539 +codec,random-path,1,1,steps,100000,7282.688708,3990.851612,54.79915141252801,0.0,0,0,4562,74,273,2513769624,111 +codec,uniform,0,1,steps,100000,3936.713375,338.87798,8.608144604888842,0.0,0,0,4790,74,273,1080632408,62 +codec,uniform,1,1,steps,100000,3429.624834,314.7182,9.176461427500849,0.0,0,0,4780,74,273,1087211024,122 +gcd,128,,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 +gcd,512,,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 +gcd,eager,,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 +gcd,existing,,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 +gcd,random-path,0,1,steps,100000,19170.84275,3410.622702,17.79067694872204,0.0,0,0,4921,28,673,2143921128,41 +gcd,random-path,1,1,steps,100000,18280.874209,3186.388872,17.430177767052758,0.0,0,0,4898,28,673,2151045424,32 +gcd,uniform,0,1,steps,100000,10152.718417,572.30041,5.636917980919514,0.0,0,0,4338,28,673,1025193416,409 +gcd,uniform,1,1,steps,100000,7327.864792,351.211791,4.7928257544192965,0.0,0,0,4444,28,673,1029454208,259 +loan,128,,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 +loan,512,,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 +loan,eager,,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 +loan,existing,,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 +loan,random-path,0,1,exhausted,73519,3868.197,882.931681,22.82540628101413,0.0,0,0,1527,123,126,881496912,50 +loan,random-path,1,1,exhausted,73519,5003.377792,1155.413364,23.09266683494125,0.0,0,0,1513,123,126,878046240,39 +loan,uniform,0,1,exhausted,73519,3045.632291,253.601349,8.326722492055428,0.0,0,0,3049,123,126,743016424,98 +loan,uniform,1,1,exhausted,73519,2744.655583,224.198052,8.168531359222275,0.0,0,0,2902,123,126,747822944,26 diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt index 829671d918..df0da45000 100644 --- a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt +++ b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt @@ -12,6 +12,7 @@ import org.usvm.UMachine import org.usvm.UMachineOptions import org.usvm.UPathSelector import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.algorithms.RandomizedPriorityCollection import org.usvm.machine.JcApplicationGraph import org.usvm.machine.JcComponents import org.usvm.machine.JcContext @@ -21,6 +22,7 @@ import org.usvm.machine.interpreter.JcInterpreter import org.usvm.machine.state.JcState import org.usvm.ps.BfsPathSelector import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.RandomTreePathSelector import org.usvm.ps.WeightedPathSelector import org.usvm.samples.JacoDBContainer import org.usvm.statistics.CompositeUMachineObserver @@ -33,6 +35,7 @@ import org.usvm.stopstrategies.StopStrategy import org.usvm.util.originalInst import java.io.File import java.lang.management.ManagementFactory +import kotlin.random.Random import kotlin.time.Duration.Companion.milliseconds /** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ @@ -74,6 +77,7 @@ class CoverageSelectorPerformanceTest { private class PerformanceMachine(cp: JcClasspath) : UMachine() { override val options = UMachineOptions( + randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), solverType = SolverType.YICES, solverTimeout = 250.milliseconds, typeOperationsTimeout = 250.milliseconds, @@ -95,7 +99,8 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) val goals = coverage.getUncoveredStatements() val metrics = Metrics() - val (base, markCovered) = createGuidance(mode, goals, metrics) + val initial = interpreter.getInitialState(method) + val (base, markCovered) = createGuidance(mode, goals, metrics, initial) val selector = MeasuredSelector(base, metrics) coverage.addOnCoveredObserver { _, _, statement -> val started = System.nanoTime() @@ -104,7 +109,6 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { metrics.coverageReports++ } val observer = metrics.observer() - val initial = interpreter.getInitialState(method) val gcCountBefore = gcCount() val gcMillisBefore = gcMillis() val allocatedBefore = allocatedBytes() @@ -148,6 +152,7 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { mode: String, goals: Collection, metrics: Metrics, + initial: JcState, ): Pair, (JcInst) -> Unit> { val cfg = CfgStatisticsImpl(graph) val calculators = MultiTargetDistanceCalculator { target -> @@ -173,6 +178,9 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { }, ) markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } + } else if (mode == "random-path" || mode == "uniform") { + base = createRandomSelector(mode, initial) + markCovered = {} } else if (mode == "bfs") { base = BfsPathSelector() markCovered = {} @@ -188,6 +196,26 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { return base to markCovered } + private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { + val random = Random(seed = options.randomSeed) + if (mode == "uniform") { + return WeightedPathSelector( + priorityCollectionFactory = { + RandomizedPriorityCollection( + comparator = compareBy { it.id }, + unitIntervalRandom = { random.nextDouble() }, + ) + }, + weighter = { 1.0 }, + ) + } + + return RandomTreePathSelector.fromRoot( + root = initial.pathNode, + randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, + ) + } + override fun close() = components.close() } @@ -312,10 +340,12 @@ private fun Metrics.report( totals: AnalysisTotals, ): String { val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() + val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() return """ {"program":"$program", "mode":"$mode", "repetition":$repetition, + "randomSeed":$randomSeed, "steps":$steps, "stop":"$stop", "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, From 465c303de7f1968a85511fd12d6b7d651ec29b97 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 19:22:06 +0300 Subject: [PATCH 08/15] Reuse lazy NURS guidance for external TypeScript coverage --- .../CoverageSelectorPerformanceTest.kt | 366 ++++++++++++++++++ .../coverage-selector-performance/README.md | 2 + .../coverage-selector-performance/run.py | 9 +- .../org/usvm/ps/CoverageGuidedPathSelector.kt | 119 ------ .../kotlin/org/usvm/ps/PathSelectorFactory.kt | 45 ++- ...losestToUncoveredRandomPathSelectorTest.kt | 149 +++++++ .../CoverageGuidedPathSelectorRefreshTest.kt | 234 ----------- .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 282 -------------- .../CoverageSelectorPerformanceTest.kt | 87 ++--- .../org/usvm/ts/calls/CallsClassStartSmoke.kt | 1 + .../usvm/ts/pbt/usvm/UsvmPropertySearcher.kt | 6 +- .../main/kotlin/org/usvm/machine/TsMachine.kt | 47 ++- .../machine/TsMachineExternalCoverageTest.kt | 103 +++++ .../DeterministicPriorityCollection.kt | 44 +-- ...erministicPriorityCollectionRefreshTest.kt | 108 ------ 15 files changed, 736 insertions(+), 866 deletions(-) create mode 100644 docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt delete mode 100644 usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt create mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt delete mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt delete mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt create mode 100644 usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt delete mode 100644 usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt diff --git a/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt b/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt new file mode 100644 index 0000000000..df0da45000 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt @@ -0,0 +1,366 @@ +package org.usvm.experiments + +import kotlinx.coroutines.runBlocking +import org.jacodb.api.jvm.JcClasspath +import org.jacodb.api.jvm.JcMethod +import org.jacodb.api.jvm.cfg.JcInst +import org.jacodb.api.jvm.ext.findClass +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test +import org.usvm.SolverType +import org.usvm.UMachine +import org.usvm.UMachineOptions +import org.usvm.UPathSelector +import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.algorithms.RandomizedPriorityCollection +import org.usvm.machine.JcApplicationGraph +import org.usvm.machine.JcComponents +import org.usvm.machine.JcContext +import org.usvm.machine.JcMachineOptions +import org.usvm.machine.JcTypeSystem +import org.usvm.machine.interpreter.JcInterpreter +import org.usvm.machine.state.JcState +import org.usvm.ps.BfsPathSelector +import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.RandomTreePathSelector +import org.usvm.ps.WeightedPathSelector +import org.usvm.samples.JacoDBContainer +import org.usvm.statistics.CompositeUMachineObserver +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.UMachineObserver +import org.usvm.statistics.distances.CallStackDistanceCalculator +import org.usvm.statistics.distances.CfgStatisticsImpl +import org.usvm.statistics.distances.MultiTargetDistanceCalculator +import org.usvm.stopstrategies.StopStrategy +import org.usvm.util.originalInst +import java.io.File +import java.lang.management.ManagementFactory +import kotlin.random.Random +import kotlin.time.Duration.Companion.milliseconds + +/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ +@Tag("manual") +class CoverageSelectorPerformanceTest { + @Test + fun runProgram() { + val program = System.getProperty("coverage.perf.case", "codec") + val mode = System.getProperty("coverage.perf.mode", "128") + val steps = System.getProperty("coverage.perf.steps", "100000").toLong() + val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() + val resultFile = File(System.getProperty("coverage.perf.results")) + val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) + val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) + val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) + try { + val (className, methodName) = when (program) { + "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" + "gcd" -> "com.google.common.math.IntMath" to "gcd" + "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" + else -> error("Unknown program: $program") + } + val cls = container.cp.findClass(className) + val method = cls.declaredMethods.single { it.name == methodName } + val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } + PerformanceMachine(container.cp).use { machine -> + val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) + resultFile.parentFile.mkdirs() + resultFile.appendText(result + "\n") + } + } finally { + runBlocking { + container.cp.close() + container.db.close() + } + } + } +} + +private class PerformanceMachine(cp: JcClasspath) : UMachine() { + override val options = UMachineOptions( + randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), + solverType = SolverType.YICES, + solverTimeout = 250.milliseconds, + typeOperationsTimeout = 250.milliseconds, + throwExceptionOnStepFailure = true, + useSoftConstraints = false, + ) + private val graph = JcApplicationGraph(cp) + private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) + private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) + + fun measure( + program: String, + mode: String, + method: JcMethod, + methodsToCover: List, + stepLimit: Long, + timeoutMillis: Long, + ): String { + val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) + val goals = coverage.getUncoveredStatements() + val metrics = Metrics() + val initial = interpreter.getInitialState(method) + val (base, markCovered) = createGuidance(mode, goals, metrics, initial) + val selector = MeasuredSelector(base, metrics) + coverage.addOnCoveredObserver { _, _, statement -> + val started = System.nanoTime() + markCovered(statement) + metrics.selectorNanos += System.nanoTime() - started + metrics.coverageReports++ + } + val observer = metrics.observer() + val gcCountBefore = gcCount() + val gcMillisBefore = gcMillis() + val allocatedBefore = allocatedBytes() + val started = System.nanoTime() + var stop = "exhausted" + selector.add(states = listOf(initial)) + run( + interpreter = interpreter, + pathSelector = selector, + observer = CompositeUMachineObserver(coverage, observer), + isStateTerminated = { state -> state.callStack.isEmpty() }, + stopStrategy = StopStrategy { + val elapsedMillis = (System.nanoTime() - started) / 1_000_000 + when { + metrics.steps >= stepLimit -> { + stop = "steps" + true + } + elapsedMillis >= timeoutMillis -> { + stop = "timeout" + true + } + else -> { + false + } + } + }, + ) + val elapsedNanos = System.nanoTime() - started + val allocated = allocatedBytes() - allocatedBefore + val totals = AnalysisTotals( + elapsedNanos = elapsedNanos, + allocatedBytes = allocated, + gcCount = gcCount() - gcCountBefore, + gcMillis = gcMillis() - gcMillisBefore, + ) + return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) + } + + private fun createGuidance( + mode: String, + goals: Collection, + metrics: Metrics, + initial: JcState, + ): Pair, (JcInst) -> Unit> { + val cfg = CfgStatisticsImpl(graph) + val calculators = MultiTargetDistanceCalculator { target -> + CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) + } + val distance: (JcState, JcInst) -> UInt = { state, target -> + metrics.distanceCalls++ + calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) + } + val markCovered: (JcInst) -> Unit + val base: UPathSelector + if (mode == "eager") { + val eager = EagerCoverageSelector(goals, distance) + base = eager + markCovered = { eager.markCovered(it) } + } else if (mode == "existing") { + val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) + base = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state: JcState -> + metrics.distanceCalls++ + aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) + }, + ) + markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } + } else if (mode == "random-path" || mode == "uniform") { + base = createRandomSelector(mode, initial) + markCovered = {} + } else if (mode == "bfs") { + base = BfsPathSelector() + markCovered = {} + } else { + val periodic = CoverageGuidedPathSelector( + targets = goals, + distance = distance, + priorityRefreshInterval = mode.toInt(), + ) + base = periodic + markCovered = { periodic.markCovered(targets = listOf(it)) } + } + return base to markCovered + } + + private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { + val random = Random(seed = options.randomSeed) + if (mode == "uniform") { + return WeightedPathSelector( + priorityCollectionFactory = { + RandomizedPriorityCollection( + comparator = compareBy { it.id }, + unitIntervalRandom = { random.nextDouble() }, + ) + }, + weighter = { 1.0 }, + ) + } + + return RandomTreePathSelector.fromRoot( + root = initial.pathNode, + randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, + ) + } + + override fun close() = components.close() +} + +private class Metrics { + var steps = 0L + var terminated = 0L + var selectorNanos = 0L + var refreshNanos = 0L + var refreshes = 0L + var distanceCalls = 0L + var coverageReports = 0L + var queueSize = 0 + var peakQueue = 0 +} + +private class MeasuredSelector( + private val delegate: UPathSelector, + private val metrics: Metrics, +) : UPathSelector { + override fun isEmpty(): Boolean = delegate.isEmpty() + + override fun peek(): JcState { + val beforeCalls = metrics.distanceCalls + val started = System.nanoTime() + val result = delegate.peek() + val elapsed = System.nanoTime() - started + metrics.selectorNanos += elapsed + if (metrics.distanceCalls > beforeCalls) { + metrics.refreshes++ + metrics.refreshNanos += elapsed + } + return result + } + + override fun add(states: Collection) { + val started = System.nanoTime() + delegate.add(states) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize += states.size + metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) + } + + override fun update(state: JcState) { + val started = System.nanoTime() + delegate.update(state) + metrics.selectorNanos += System.nanoTime() - started + } + + override fun remove(state: JcState) { + val started = System.nanoTime() + delegate.remove(state) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize-- + } +} + +/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ +private class EagerCoverageSelector( + targets: Collection, + distance: (JcState, JcInst) -> UInt, +) : UPathSelector { + private val pending = targets.toMutableSet() + private var dirty = false + private val delegate = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, + ) + + fun markCovered(target: JcInst) { + if (pending.remove(target)) dirty = true + } + + override fun isEmpty(): Boolean = delegate.isEmpty() + override fun peek(): JcState { + if (dirty) { + val states = delegate.drain().toList() + delegate.add(states) + dirty = false + } + return delegate.peek() + } + override fun add(states: Collection) = delegate.add(states) + override fun update(state: JcState) = delegate.update(state) + override fun remove(state: JcState) = delegate.remove(state) +} + +private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionCount.coerceAtLeast(minimumValue = 0) +} + +private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionTime.coerceAtLeast(minimumValue = 0) +} +private fun allocatedBytes(): Long { + val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean + return bean.getThreadAllocatedBytes(Thread.currentThread().id) +} + +private data class AnalysisTotals( + val elapsedNanos: Long, + val allocatedBytes: Long, + val gcCount: Long, + val gcMillis: Long, +) + +private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { + override fun onStatePeeked(state: JcState) { + steps++ + } + + override fun onStateTerminated(state: JcState, stateReachable: Boolean) { + terminated++ + } +} + +private fun Metrics.report( + program: String, + mode: String, + goals: Int, + covered: Int, + stop: String, + totals: AnalysisTotals, +): String { + val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() + val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() + return """ + {"program":"$program", + "mode":"$mode", + "repetition":$repetition, + "randomSeed":$randomSeed, + "steps":$steps, + "stop":"$stop", + "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, + "selectorMs":${selectorNanos / 1_000_000.0}, + "refreshMs":${refreshNanos / 1_000_000.0}, + "refreshes":$refreshes, + "distanceCalls":$distanceCalls, + "peakQueue":$peakQueue, + "pending":$queueSize, + "goals":$goals, + "covered":$covered, + "coverageReports":$coverageReports, + "terminated":$terminated, + "allocatedBytes":${totals.allocatedBytes}, + "gcCount":${totals.gcCount}, + "gcMs":${totals.gcMillis}} + """.trimIndent().replace("\n", "") +} diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md index fa9cbe8ac9..5df8bce855 100644 --- a/docs/experiments/coverage-selector-performance/README.md +++ b/docs/experiments/coverage-selector-performance/README.md @@ -1,5 +1,7 @@ # Coverage selector performance on real JVM bytecode +These are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). + This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. ## Recorded results (2026-10-16) diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py index 590a47f71e..c1e9ef53d9 100644 --- a/docs/experiments/coverage-selector-performance/run.py +++ b/docs/experiments/coverage-selector-performance/run.py @@ -20,7 +20,7 @@ def cached_jar(group, artifact, version): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--cases', default='codec,gcd,loan') - parser.add_argument('--modes', default='existing,128,eager,512') + parser.add_argument('--modes', default='nurs,uniform') parser.add_argument('--steps', type=int, default=100000) parser.add_argument('--timeout-ms', type=int, default=120000) parser.add_argument('--repetitions', type=int, default=2) @@ -32,6 +32,13 @@ def main(): experiment = Path(__file__).resolve().parent root = experiment.parents[2] + revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() + historical_revision = '44869512bca6b1ce89511b9f6799f4475efdf972' + historical_modes = {'128', '512', 'eager'} + uses_historical_modes = bool(set(args.modes.split(',')) & historical_modes) + if uses_historical_modes and revision != historical_revision: + parser.error(f'Historical experiment: run this script from a checkout of {historical_revision}') + output = (args.output or root / 'build/coverage-selector-performance').resolve() output.mkdir(parents=True, exist_ok=True) codec = args.codec_jar or cached_jar('commons-codec', 'commons-codec', '1.16.0') diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt deleted file mode 100644 index 416043bf11..0000000000 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ /dev/null @@ -1,119 +0,0 @@ -package org.usvm.ps - -import org.usvm.UPathSelector -import org.usvm.algorithms.DeterministicPriorityCollection - -/** - * Caller-confirmed coverage changes priorities, never removes states. Updates belong on the analysis thread. - * [distance] defines statement, edge or other goal semantics; [UInt.MAX_VALUE] means unreachable. - * - * Weights use a snapshot of uncovered goals. Changes are applied before selection after at least - * [priorityRefreshInterval] completed selections since the previous priority refresh. A selection completes when - * its state is updated or removed; repeated [peek] calls do not advance the interval. - * The default interval is a tuning starting point, not a measured optimum. - */ -class CoverageGuidedPathSelector( - targets: Collection, - distance: (State, Target) -> UInt, - private val priorityRefreshInterval: Int = DEFAULT_PRIORITY_REFRESH_INTERVAL, -) : UPathSelector { - init { - require(priorityRefreshInterval > 0) { "Priority refresh interval must be positive" } - } - - private val knownTargets = targets.toHashSet() - private val pendingTargets = targets.toHashSet() - private var priorityTargets: Set = targets.toHashSet() - private val distanceToTarget = distance - private var prioritiesChanged = false - private var completedSelections = 0 - private var selectionPending = false - private var selectedState: State? = null - private val priorities = DeterministicPriorityCollection(Comparator.naturalOrder()) - private val selector = WeightedPathSelector( - priorityCollectionFactory = { priorities }, - weighter = { state -> weight(state, priorityTargets) }, - ) - - val uncoveredTargets: Set - get() = pendingTargets.toHashSet() - - /** Registers newly identified goals; previously covered goals stay covered. */ - fun addTargets(targets: Collection): Boolean { - val additions = targets.filter { knownTargets.add(it) } - if (additions.isEmpty()) return false - - pendingTargets.addAll(additions) - prioritiesChanged = true - return true - } - - /** Unknown goals are ignored. Changed priorities are applied at the next eligible selection. */ - fun markCovered(targets: Collection): Boolean { - val changed = pendingTargets.removeAll(targets.toHashSet()) - if (changed) { - prioritiesChanged = pendingTargets.isNotEmpty() - if (!prioritiesChanged) { - // With no goals, ordering is irrelevant and new work needs no distance calculations. - priorityTargets = emptySet() - } - } - - return changed - } - - override fun isEmpty(): Boolean = selector.isEmpty() - - override fun peek(): State { - val refreshDue = prioritiesChanged && completedSelections >= priorityRefreshInterval - if (refreshDue) { - val refreshedTargets = pendingTargets.toHashSet() - priorities.updatePriorities { state -> weight(state, refreshedTargets) } - priorityTargets = refreshedTargets - prioritiesChanged = false - completedSelections = 0 - } - - val state = selector.peek() - selectedState = state - selectionPending = true - return state - } - - override fun add(states: Collection) = selector.add(states) - - override fun update(state: State) { - selector.update(state) - completeSelection(state) - } - - override fun remove(state: State) { - selector.remove(state) - completeSelection(state) - } - - // Bypass pending priority updates and selection accounting when collecting stopped work. - override fun drain(): Sequence { - selectionPending = false - selectedState = null - return selector.drain() - } - - private fun weight(state: State, targets: Set): UInt = - targets.minOfOrNull { target -> distanceToTarget(state, target) } ?: UInt.MAX_VALUE - - private fun completeSelection(state: State) { - val completesSelection = selectionPending && selectedState == state - if (!completesSelection) return - - if (completedSelections < priorityRefreshInterval) { - completedSelections++ - } - selectionPending = false - selectedState = null - } - - private companion object { - const val DEFAULT_PRIORITY_REFRESH_INTERVAL = 128 - } -} diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt index 9a3d0b4c94..60173bd83b 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt @@ -36,6 +36,7 @@ private fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageDistanceCalculatorFactory: () -> CallStackDistanceCalculator? = { null }, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { @@ -69,14 +70,16 @@ private fun createPathSelector( PathSelectionStrategy.CLOSEST_TO_UNCOVERED -> createClosestToUncoveredPathSelector( requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, - applicationGraph + applicationGraph, + distanceCalculator = coverageDistanceCalculatorFactory(), ) PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM -> createClosestToUncoveredPathSelector( requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, applicationGraph, - random + random, + distanceCalculator = coverageDistanceCalculatorFactory(), ) PathSelectionStrategy.TARGETED -> createTargetedPathSelector( @@ -153,6 +156,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageDistanceCalculatorFactory: () -> CallStackDistanceCalculator? = { null }, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> = createPathSelector( listOf(initialState), @@ -161,7 +165,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageDistanceCalculatorFactory, ) fun createPathSelector( @@ -173,6 +178,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageDistanceCalculatorFactory: () -> CallStackDistanceCalculator? = { null }, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { val hasSingleEntryPoint = initialStates.size == 1 val hasUnlimitedScheduling = options.timeout == Duration.INFINITE && @@ -185,7 +191,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageDistanceCalculatorFactory, ) } @@ -202,7 +209,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageDistanceCalculatorFactory, ) val coverageStatistics = coverageStatisticsFactory() @@ -317,15 +325,16 @@ private fun , applicationGraph: ApplicationGraph, random: Random? = null, + distanceCalculator: CallStackDistanceCalculator? = null, ): UPathSelector { - val distanceCalculator = CallStackDistanceCalculator( + val calculator = distanceCalculator ?: CallStackDistanceCalculator( targets = coverageStatistics.getUncoveredStatements(), cfgStatistics = cfgStatistics, applicationGraph ) coverageStatistics.addOnCoveredObserver { _, method, statement -> - distanceCalculator.removeTarget( + calculator.removeTarget( method, statement ) @@ -334,21 +343,25 @@ private fun > createClosestToUncoveredRandomPathSelector( + distance: StateWeighter, + random: Random, +): UPathSelector = WeightedPathSelector( + priorityCollectionFactory = { RandomizedPriorityCollection(compareById()) { random.nextDouble() } }, + weighter = { 1.0 / max(distance.weight(it).toDouble(), 1.0) }, +) + fun > createForkDepthPathSelector( random: Random? = null, ): UPathSelector { diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt new file mode 100644 index 0000000000..a7365dde31 --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt @@ -0,0 +1,149 @@ +package org.usvm.ps + +import io.mockk.every +import org.junit.jupiter.api.Test +import org.usvm.PathNode +import org.usvm.PathSelectionStrategy +import org.usvm.TestInstruction +import org.usvm.TestState +import org.usvm.UMachineOptions +import org.usvm.appGraph +import org.usvm.mockState +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.distances.CallStackDistanceCalculator +import org.usvm.statistics.distances.CfgStatistics +import org.usvm.statistics.distances.CfgStatisticsImpl +import kotlin.test.assertEquals +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class ClosestToUncoveredRandomPathSelectorTest { + private val graph = appGraph { + method(name = "method", instructionsCount = 7) { + entryPoint(offset = 0) + edge(from = 0, to = 1) + edge(from = 1, to = 2) + edge(from = 0, to = 3) + edge(from = 3, to = 4) + edge(from = 4, to = 5) + edge(from = 5, to = 6) + exitPoint(offset = 2) + exitPoint(offset = 6) + } + } + + @Test + fun `coverage and peek never recalculate queued weights while update touches only its state`() { + val cfg = CfgStatisticsImpl(graph) + val evaluated = mutableListOf() + val countedCfg = object : CfgStatistics by cfg { + override fun getShortestDistanceToExit(method: String, stmtFrom: TestInstruction): UInt { + evaluated.add(stmtFrom) + return cfg.getShortestDistanceToExit(method, stmtFrom) + } + } + val leftGoal = TestInstruction(method = "method", offset = 1) + val rightGoal = TestInstruction(method = "method", offset = 6) + val calculator = CallStackDistanceCalculator( + targets = listOf(leftGoal, rightGoal), + cfgStatistics = countedCfg, + applicationGraph = graph, + ) + val nativeCoverage = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createPathSelector( + initialState = left, + options = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + randomSeed = 0, + ), + applicationGraph = graph, + coverageStatisticsFactory = { nativeCoverage }, + cfgStatisticsFactory = { countedCfg }, + coverageDistanceCalculatorFactory = { calculator }, + ) + selector.add(states = listOf(right)) + assertEquals(expected = listOf(left.currentStatement, right.currentStatement), actual = evaluated) + assertSame(expected = left, actual = selector.peek()) + evaluated.clear() + + assertTrue(calculator.removeTarget(method = "method", statement = leftGoal)) + repeat(256) { selector.peek() } + + assertTrue(evaluated.isEmpty()) + assertEquals(expected = 0, actual = nativeCoverage.getTotalCoveredStatements()) + + selector.update(state = left) + + assertEquals(expected = listOf(left.currentStatement), actual = evaluated) + assertSame(expected = right, actual = selector.peek()) + evaluated.clear() + + val fork = mockState(id = 3u, startMethod = "method", startInstruction = 2) + selector.add(states = listOf(fork)) + + assertEquals(expected = listOf(fork.currentStatement), actual = evaluated) + assertSame(expected = right, actual = selector.peek()) + assertEquals(expected = setOf(left, right, fork), actual = selector.drain().toHashSet()) + } + + @Test + fun `native coverage still removes targets from the supplied calculator`() { + val cfg = CfgStatisticsImpl(graph) + val goal = TestInstruction(method = "method", offset = 1) + val calculator = CallStackDistanceCalculator( + targets = listOf(goal), + cfgStatistics = cfg, + applicationGraph = graph, + ) + val state = mockState(id = 1u, startMethod = "method", startInstruction = 1) + every { state.pathNode } returns PathNode.root() + goal + val coverage = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val selector = createPathSelector( + initialState = state, + options = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + ), + applicationGraph = graph, + coverageStatisticsFactory = { coverage }, + cfgStatisticsFactory = { cfg }, + coverageDistanceCalculatorFactory = { calculator }, + ) + + coverage.onStateTerminated(state = state, stateReachable = true) + selector.update(state = state) + + assertEquals(expected = 1, actual = coverage.getTotalCoveredStatements()) + assertEquals( + expected = UInt.MAX_VALUE, + actual = calculator.calculateDistance(state.currentStatement, state.callStack), + ) + assertSame(expected = state, actual = selector.peek()) + assertEquals(expected = listOf(state), actual = selector.drain().toList()) + } + + @Test + fun `covered prefixes can still lead to uncovered successors`() { + val goal = TestInstruction(method = "method", offset = 6) + val prefix = mockState(id = 1u, startMethod = "method", startInstruction = 3) + val calculator = CallStackDistanceCalculator( + targets = listOf(prefix.currentStatement, goal), + cfgStatistics = CfgStatisticsImpl(graph), + applicationGraph = graph, + ) + + calculator.removeTarget(method = "method", statement = prefix.currentStatement) + + assertEquals( + expected = 3u, + actual = calculator.calculateDistance(prefix.currentStatement, prefix.callStack), + ) + } +} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt deleted file mode 100644 index 9421b89c4c..0000000000 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt +++ /dev/null @@ -1,234 +0,0 @@ -package org.usvm.ps - -import org.junit.jupiter.api.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertTrue - -class CoverageGuidedPathSelectorRefreshTest { - @Test - fun `default interval keeps weights stale for 128 completed selections`() { - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - ) - selector.add(states = listOf("left", "right")) - selector.markCovered(targets = listOf("left")) - - repeat(127) { - assertEquals(expected = "left", actual = selector.peek()) - evaluatedTargets.clear() - selector.update(state = "left") - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) - } - evaluatedTargets.clear() - repeat(256) { - assertEquals(expected = "left", actual = selector.peek()) - } - assertTrue(evaluatedTargets.isEmpty()) - - selector.update(state = "left") - evaluatedTargets.clear() - - assertEquals(expected = "right", actual = selector.peek()) - assertEquals(expected = listOf("right", "right"), actual = evaluatedTargets) - evaluatedTargets.clear() - selector.peek() - assertTrue(evaluatedTargets.isEmpty()) - } - - @Test - fun `new and updated states use the old snapshot until the interval expires`() { - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("old"), - distance = { state, target -> - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("old")) - selector.peek() - - selector.addTargets(targets = listOf("new")) - selector.markCovered(targets = listOf("old")) - evaluatedTargets.clear() - selector.add(states = listOf("new")) - selector.update(state = "old") - - assertEquals(expected = listOf("old", "old"), actual = evaluatedTargets) - assertEquals(expected = setOf("new"), actual = selector.uncoveredTargets) - assertEquals(expected = "old", actual = selector.peek()) - - selector.update(state = "old") - evaluatedTargets.clear() - - assertEquals(expected = "new", actual = selector.peek()) - assertEquals(expected = listOf("new", "new"), actual = evaluatedTargets) - } - - @Test - fun `only one completion of the selected state advances the interval`() { - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - evaluations++ - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("left", "right", "other")) - selector.peek() - selector.markCovered(targets = listOf("left")) - - selector.update(state = "other") - selector.remove(state = "other") - selector.update(state = "left") - selector.update(state = "left") - val beforePeek = evaluations - - assertEquals(expected = "left", actual = selector.peek()) - assertEquals(expected = beforePeek, actual = evaluations) - - selector.remove(state = "left") - val beforeRefresh = evaluations - - assertEquals(expected = "right", actual = selector.peek()) - assertEquals(expected = beforeRefresh + 1, actual = evaluations) - } - - @Test - fun `each refresh starts a new interval including newly registered goals`() { - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - val destination = if (target == "left-again") "left" else target - if (state == destination) 0u else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("left", "right")) - selector.markCovered(targets = listOf("left")) - repeat(2) { selector.update(state = selector.peek()) } - assertEquals(expected = "right", actual = selector.peek()) - - selector.addTargets(targets = listOf("left-again")) - selector.markCovered(targets = listOf("right")) - selector.update(state = "right") - - assertEquals(expected = "right", actual = selector.peek()) - - selector.update(state = "right") - - assertEquals(expected = "left", actual = selector.peek()) - assertEquals(expected = setOf("left-again"), actual = selector.uncoveredTargets) - } - - @Test - fun `unchanged goals do not trigger refresh even after the interval`() { - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("goal"), - distance = { _, _ -> - evaluations++ - 0u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("first", "second")) - - repeat(8) { selector.update(state = selector.peek()) } - val beforePeek = evaluations - selector.markCovered(targets = listOf("unknown")) - selector.addTargets(targets = listOf("goal")) - selector.peek() - - assertEquals(expected = 2 + 8, actual = evaluations) - assertEquals(expected = beforePeek, actual = evaluations) - } - - @Test - fun `saturation avoids distance work and later goals resume guidance`() { - var distancesAllowed = true - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("old"), - distance = { state, target -> - check(distancesAllowed) - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("old", "new")) - distancesAllowed = false - - selector.markCovered(targets = listOf("old")) - repeat(4) { selector.update(state = selector.peek()) } - selector.add(states = listOf("later")) - - assertTrue(selector.uncoveredTargets.isEmpty()) - - selector.addTargets(targets = listOf("new")) - distancesAllowed = true - evaluatedTargets.clear() - - assertEquals(expected = "new", actual = selector.peek()) - assertEquals(expected = listOf("new", "new", "new"), actual = evaluatedTargets) - assertEquals(expected = setOf("old", "new", "later"), actual = selector.drain().toSet()) - } - - @Test - fun `failed refresh preserves the snapshot and can be retried`() { - var failRefresh = false - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - check(!failRefresh) - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 1, - ) - selector.add(states = listOf("left", "right")) - selector.peek() - selector.markCovered(targets = listOf("left")) - selector.update(state = "left") - failRefresh = true - - assertFailsWith { selector.peek() } - - failRefresh = false - evaluatedTargets.clear() - selector.add(states = listOf("other")) - - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) - assertEquals(expected = "right", actual = selector.peek()) - assertEquals(expected = setOf("left", "right", "other"), actual = selector.drain().toSet()) - } - - @Test - fun `refresh interval must be positive`() { - for (interval in listOf(0, -1)) { - assertFailsWith { - CoverageGuidedPathSelector( - targets = emptyList(), - distance = { _, _ -> 0u }, - priorityRefreshInterval = interval, - ) - } - } - } -} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt deleted file mode 100644 index 825c93380e..0000000000 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt +++ /dev/null @@ -1,282 +0,0 @@ -package org.usvm.ps - -import org.junit.jupiter.api.Test -import org.usvm.TestInstruction -import org.usvm.TestState -import org.usvm.appGraph -import org.usvm.mockState -import org.usvm.statistics.distances.CallStackDistanceCalculator -import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.statistics.distances.MultiTargetDistanceCalculator -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertSame -import kotlin.test.assertTrue - -class CoverageGuidedPathSelectorTest { - @Test - fun `reported coverage reprioritizes every pending state after a completed selection`() { - val selector = selector(targets = listOf("left", "right")) - selector.add(states = listOf("left", "right")) - - assertEquals(expected = "left", actual = selector.peek()) - - selector.markCovered(targets = listOf("left")) - selector.update(state = "left") - - assertEquals(expected = "right", actual = selector.peek()) - - selector.addTargets(targets = listOf("new-left")) - selector.markCovered(targets = listOf("right")) - selector.update(state = "right") - - assertEquals(expected = "left", actual = selector.peek()) - assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) - } - - @Test - fun `unchanged coverage does not recalculate queued weights`() { - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("goal"), - distance = { _, _ -> - evaluations++ - 0u - }, - ) - selector.add(states = listOf("first", "second")) - val afterAdd = evaluations - - selector.peek() - selector.peek() - assertFalse(selector.markCovered(targets = listOf("unknown"))) - assertFalse(selector.addTargets(targets = listOf("goal"))) - selector.peek() - - assertEquals(expected = afterAdd, actual = evaluations) - } - - @Test - fun `multiple reports are applied together at the refresh boundary`() { - var refreshing = false - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("first", "second", "third"), - distance = { state, target -> - if (refreshing) { - check(target == "third") - evaluations++ - } - - if (state == target) 0u else 10u - }, - priorityRefreshInterval = 1, - ) - selector.add(states = listOf("first", "third")) - val selected = selector.peek() - - selector.markCovered(targets = listOf("first")) - selector.markCovered(targets = listOf("second")) - selector.update(state = selected) - refreshing = true - - assertEquals(expected = "third", actual = selector.peek()) - assertEquals(expected = 2, actual = evaluations) - } - - @Test - fun `saturated coverage retains all queued states and accepts later work`() { - val selector = selector(targets = listOf("left", "right")) - selector.add(states = listOf("left", "right")) - selector.markCovered(targets = listOf("left", "right")) - - selector.peek() - val drained = selector.drain().toList() - selector.add(states = listOf("later")) - - assertEquals(expected = setOf("left", "right"), actual = drained.toSet()) - assertEquals(expected = 2, actual = drained.size) - assertEquals(expected = "later", actual = selector.peek()) - } - - @Test - fun `edges sharing a successor remain distinct coverage goals`() { - val firstEdge = Edge(from = 1, to = 3) - val secondEdge = Edge(from = 2, to = 3) - val firstPath = setOf(firstEdge) - val secondPath = setOf(secondEdge) - val selector = CoverageGuidedPathSelector, Edge>( - targets = listOf(firstEdge, secondEdge), - distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, - priorityRefreshInterval = 1, - ) - selector.add(states = listOf(firstPath, secondPath)) - val selected = selector.peek() - - selector.markCovered(targets = listOf(firstEdge)) - selector.update(state = selected) - - assertSame(expected = secondPath, actual = selector.peek()) - assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) - } - - @Test - fun `empty goals do not invoke the distance function or remove states`() { - val selector = CoverageGuidedPathSelector( - targets = emptyList(), - distance = { _, _ -> error("There is no known goal") }, - ) - selector.add(states = listOf("first", "second")) - - selector.peek() - - assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) - assertTrue(selector.isEmpty()) - } - - @Test - fun `draining stopped work does not refresh weights after coverage changes`() { - var stopped = false - val selector = CoverageGuidedPathSelector( - targets = listOf("left"), - distance = { _, _ -> - check(!stopped) - 0u - }, - ) - selector.add(states = listOf("first", "second")) - stopped = true - selector.addTargets(targets = listOf("right")) - - val drained = selector.drain().toList() - - assertEquals(expected = setOf("first", "second"), actual = drained.toSet()) - assertTrue(selector.isEmpty()) - } - - @Test - fun `state updates additions and removals work after coverage changes`() { - val distances = mutableMapOf("first" to 0u, "second" to 2u, "third" to 1u) - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, _ -> distances.getValue(state) }, - ) - selector.add(states = listOf("first", "second")) - assertEquals(expected = "first", actual = selector.peek()) - - distances["first"] = 3u - selector.update(state = "first") - selector.add(states = listOf("third")) - selector.markCovered(targets = listOf("left")) - - assertEquals(expected = "third", actual = selector.peek()) - selector.remove(state = "third") - assertEquals(expected = "second", actual = selector.peek()) - assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) - } - - @Test - fun `coverage reports are idempotent and unknown goals are ignored`() { - val selector = selector(targets = listOf("first", "second")) - - assertFalse(selector.markCovered(targets = listOf("unknown"))) - assertTrue(selector.markCovered(targets = listOf("first"))) - assertFalse(selector.markCovered(targets = listOf("first"))) - - assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) - } - - @Test - fun `new goals can be registered without resurrecting covered goals`() { - val selector = selector(targets = listOf("first")) - selector.markCovered(targets = listOf("first")) - - assertTrue(selector.addTargets(targets = listOf("first", "second", "second"))) - assertFalse(selector.addTargets(targets = listOf("first", "second"))) - - assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) - } - - @Test - fun `goal snapshots and constructor inputs do not share mutable storage`() { - val initial = mutableListOf("first", "second") - val selector = selector(targets = initial) - val snapshot = selector.uncoveredTargets - - initial.clear() - selector.markCovered(targets = listOf("first", "second")) - - assertEquals(expected = setOf("first", "second"), actual = snapshot) - assertTrue(selector.uncoveredTargets.isEmpty()) - } - - @Test - fun `caller can reuse existing CFG distance calculators`() { - val graph = appGraph { - method(name = "method", instructionsCount = 5) { - entryPoint(offset = 0) - edge(from = 0, to = 1) - edge(from = 0, to = 3) - edge(from = 1, to = 2) - edge(from = 3, to = 4) - exitPoint(offset = 2) - exitPoint(offset = 4) - } - } - val cfgStatistics = CfgStatisticsImpl(graph) - val distances = MultiTargetDistanceCalculator { target -> - CallStackDistanceCalculator( - targets = listOf(target), - cfgStatistics = cfgStatistics, - applicationGraph = graph, - ) - } - val firstTarget = TestInstruction(method = "method", offset = 2) - val secondTarget = TestInstruction(method = "method", offset = 4) - val selector = CoverageGuidedPathSelector( - targets = listOf(firstTarget, secondTarget), - distance = { state, target -> - distances.calculateDistance(state.currentStatement, state.callStack, target) - }, - priorityRefreshInterval = 1, - ) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - selector.add(states = listOf(left, right)) - val selected = selector.peek() - - selector.markCovered(targets = listOf(firstTarget)) - selector.update(state = selected) - - assertSame(expected = right, actual = selector.peek()) - assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) - } - - @Test - fun `coverage guidance composes with a standard selector on shared states`() { - val coverage = selector(targets = listOf("left", "right")) - val combined = InterleavedPathSelector(selectors = listOf(BfsPathSelector(), coverage)) - combined.add(states = listOf("left", "right")) - assertEquals(expected = "left", actual = combined.peek()) - - coverage.markCovered(targets = listOf("left")) - combined.update(state = "left") - - assertEquals(expected = "right", actual = combined.peek()) - combined.add(states = listOf("fork")) - combined.remove(state = "right") - assertEquals(expected = setOf("left", "fork"), actual = combined.drain().toSet()) - assertTrue(combined.isEmpty()) - } - - private fun selector(targets: Collection) = CoverageGuidedPathSelector( - targets = targets, - distance = { state, target -> - val destination = if (target == "new-left") "left" else target - if (state == destination) 0u else 10u - }, - priorityRefreshInterval = 1, - ) - - private data class Edge(val from: Int, val to: Int) -} diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt index df0da45000..4f41716094 100644 --- a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt +++ b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt @@ -21,7 +21,7 @@ import org.usvm.machine.JcTypeSystem import org.usvm.machine.interpreter.JcInterpreter import org.usvm.machine.state.JcState import org.usvm.ps.BfsPathSelector -import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.createClosestToUncoveredRandomPathSelector import org.usvm.ps.RandomTreePathSelector import org.usvm.ps.WeightedPathSelector import org.usvm.samples.JacoDBContainer @@ -30,7 +30,6 @@ import org.usvm.statistics.CoverageStatistics import org.usvm.statistics.UMachineObserver import org.usvm.statistics.distances.CallStackDistanceCalculator import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.statistics.distances.MultiTargetDistanceCalculator import org.usvm.stopstrategies.StopStrategy import org.usvm.util.originalInst import java.io.File @@ -44,7 +43,7 @@ class CoverageSelectorPerformanceTest { @Test fun runProgram() { val program = System.getProperty("coverage.perf.case", "codec") - val mode = System.getProperty("coverage.perf.mode", "128") + val mode = System.getProperty("coverage.perf.mode", "nurs") val steps = System.getProperty("coverage.perf.steps", "100000").toLong() val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() val resultFile = File(System.getProperty("coverage.perf.results")) @@ -154,45 +153,33 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { metrics: Metrics, initial: JcState, ): Pair, (JcInst) -> Unit> { - val cfg = CfgStatisticsImpl(graph) - val calculators = MultiTargetDistanceCalculator { target -> - CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) + if (mode == "random-path" || mode == "uniform") { + return createRandomSelector(mode, initial) to {} } - val distance: (JcState, JcInst) -> UInt = { state, target -> + if (mode == "bfs") return BfsPathSelector() to {} + require(mode == "nurs" || mode == "existing") { "Historical modes require checkout 44869512b" } + + val aggregate = CallStackDistanceCalculator( + targets = goals, + cfgStatistics = CfgStatisticsImpl(graph), + applicationGraph = graph, + ) + val distance: (JcState) -> UInt = { state -> metrics.distanceCalls++ - calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) + aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) } - val markCovered: (JcInst) -> Unit - val base: UPathSelector - if (mode == "eager") { - val eager = EagerCoverageSelector(goals, distance) - base = eager - markCovered = { eager.markCovered(it) } - } else if (mode == "existing") { - val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) - base = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state: JcState -> - metrics.distanceCalls++ - aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) - }, + val base = if (mode == "nurs") { + createClosestToUncoveredRandomPathSelector( + distance = distance, + random = Random(seed = options.randomSeed), ) - markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } - } else if (mode == "random-path" || mode == "uniform") { - base = createRandomSelector(mode, initial) - markCovered = {} - } else if (mode == "bfs") { - base = BfsPathSelector() - markCovered = {} } else { - val periodic = CoverageGuidedPathSelector( - targets = goals, - distance = distance, - priorityRefreshInterval = mode.toInt(), + WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = distance, ) - base = periodic - markCovered = { periodic.markCovered(targets = listOf(it)) } } + val markCovered: (JcInst) -> Unit = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } return base to markCovered } @@ -272,36 +259,6 @@ private class MeasuredSelector( } } -/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ -private class EagerCoverageSelector( - targets: Collection, - distance: (JcState, JcInst) -> UInt, -) : UPathSelector { - private val pending = targets.toMutableSet() - private var dirty = false - private val delegate = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, - ) - - fun markCovered(target: JcInst) { - if (pending.remove(target)) dirty = true - } - - override fun isEmpty(): Boolean = delegate.isEmpty() - override fun peek(): JcState { - if (dirty) { - val states = delegate.drain().toList() - delegate.add(states) - dirty = false - } - return delegate.peek() - } - override fun add(states: Collection) = delegate.add(states) - override fun update(state: JcState) = delegate.update(state) - override fun remove(state: JcState) = delegate.remove(state) -} - private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { it.collectionCount.coerceAtLeast(minimumValue = 0) } diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt index 548c40027b..60bece8260 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt @@ -571,6 +571,7 @@ private fun runRawClassAnalysis(args: List, exploratory: Boolean) { TsMachine( scene = scene, options = UMachineOptions( + pathSelectionStrategies = listOf(CALLS_PATH_SELECTION_STRATEGY), stateCollectionStrategy = StateCollectionStrategy.ALL, randomSeed = manifest.seed, collectedStatesLimit = manifest.collectedStatesLimit, diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt index 10bf82289f..8e2cadda37 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt @@ -3,6 +3,7 @@ package org.usvm.ts.pbt.usvm import io.ksmt.utils.asExpr import org.jacodb.ets.model.EtsScene import org.jacodb.ets.model.EtsType +import org.usvm.PathSelectionStrategy import org.usvm.SolverType import org.usvm.StateCollectionStrategy import org.usvm.UMachineOptions @@ -31,7 +32,10 @@ import org.usvm.ts.pbt.model.PropertyId /** Searches mapped TypeScript predicates for concrete property violations with USVM. */ class UsvmPropertySearcher( private val scene: EtsScene, - machineOptions: UMachineOptions = UMachineOptions(solverType = SolverType.YICES), + machineOptions: UMachineOptions = UMachineOptions( + solverType = SolverType.YICES, + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + ), private val tsOptions: TsOptions = TsOptions(), projectionOptions: UsvmProjectionOptions = UsvmProjectionOptions(), ) { diff --git a/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt b/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt index 80e389983f..39695c703a 100644 --- a/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt +++ b/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt @@ -5,6 +5,7 @@ import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsScene import org.jacodb.ets.model.EtsStmt import org.usvm.CoverageZone +import org.usvm.PathSelectionStrategy import org.usvm.SolverType import org.usvm.StateCollectionStrategy import org.usvm.UMachine @@ -37,6 +38,7 @@ import org.usvm.statistics.collectors.CoveredNewStatesCollector import org.usvm.statistics.collectors.StatesCollector import org.usvm.statistics.collectors.TargetsReachedStatesCollector import org.usvm.statistics.constraints.SoftConstraintsObserver +import org.usvm.statistics.distances.CallStackDistanceCalculator import org.usvm.statistics.distances.CfgStatisticsImpl import org.usvm.statistics.distances.PlainCallGraphStatistics import org.usvm.stopstrategies.StopStrategy @@ -78,9 +80,13 @@ data class TsMachineAnalysisResult( val runtimeLimited: Boolean get() = unsupportedPaths.isNotEmpty() } +/** Symbolically executes TypeScript entry points with shared coverage guidance. */ class TsMachine( scene: EtsScene, - override val options: UMachineOptions = UMachineOptions(solverType = SolverType.YICES), + override val options: UMachineOptions = UMachineOptions( + solverType = SolverType.YICES, + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + ), private val tsOptions: TsOptions, private val machineObserver: UMachineObserver? = null, private val observer: TsInterpreterObserver? = null, @@ -143,6 +149,24 @@ class TsMachine( throwExceptionOnStepFailure = options.throwExceptionOnStepFailure, ) private val cfgStatistics = CfgStatisticsImpl(graph) + private val externallyCoveredStatements = hashSetOf() + private var coverageDistanceCalculator: CallStackDistanceCalculator? = null + + /** + * Reports caller-confirmed coverage on the analysis thread, including before analysis starts. + * This removes distance goals without changing native coverage counters or refreshing queued weights. + * Reports are retained across analyses on this machine. + */ + fun markCovered(statements: Collection): Boolean { + var changed = false + for (statement in statements) { + if (externallyCoveredStatements.add(statement)) { + coverageDistanceCalculator?.removeTarget(graph.methodOf(statement), statement) + changed = true + } + } + return changed + } /** Returns supported states only. Call [analyzeWithOutcome] to inspect excluded unsupported paths. */ fun analyze( @@ -186,6 +210,7 @@ class TsMachine( targets: List = emptyList(), configureInitialState: (EtsMethod, TsState) -> Unit = { _, _ -> }, ): TsMachineAnalysisResult { + coverageDistanceCalculator = null interpreter.resetStepFailure() failureTrackingUnknownCallObserver.reset() @@ -207,11 +232,7 @@ class TsMachine( applicationGraph = graph, ) - val callGraphStatistics: PlainCallGraphStatistics = - when (options.targetSearchDepth) { - 0u -> PlainCallGraphStatistics() - else -> TODO("Unsupported yet") - } + val callGraphStatistics = createCallGraphStatistics() val timeStatistics = createTimeStatistics() @@ -223,6 +244,7 @@ class TsMachine( coverageStatisticsFactory = { coverageStatistics }, cfgStatisticsFactory = { cfgStatistics }, callGraphStatisticsFactory = { callGraphStatistics }, + coverageDistanceCalculatorFactory = { getCoverageDistanceCalculator(coverageStatistics) }, ) val statesCollector = createStatesCollector(coverageStatistics) @@ -319,6 +341,19 @@ class TsMachine( ) } + private fun createCallGraphStatistics(): PlainCallGraphStatistics = when (options.targetSearchDepth) { + 0u -> PlainCallGraphStatistics() + else -> TODO("Unsupported yet") + } + + private fun getCoverageDistanceCalculator( + coverageStatistics: CoverageStatistics, + ): CallStackDistanceCalculator = coverageDistanceCalculator ?: CallStackDistanceCalculator( + targets = coverageStatistics.getUncoveredStatements().filterNot { it in externallyCoveredStatements }, + cfgStatistics = cfgStatistics, + applicationGraph = graph, + ).also { coverageDistanceCalculator = it } + private fun createStatesCollector( coverageStatistics: CoverageStatistics, ): StatesCollector = when (options.stateCollectionStrategy) { diff --git a/usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt b/usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt new file mode 100644 index 0000000000..d2ce585a46 --- /dev/null +++ b/usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt @@ -0,0 +1,103 @@ +package org.usvm.machine + +import org.jacodb.ets.model.EtsScene +import org.jacodb.ets.utils.EtsIrProvider +import org.jacodb.ets.utils.loadEtsFileAutoConvert +import org.junit.jupiter.api.Test +import org.usvm.CoverageZone +import org.usvm.PathSelectionStrategy +import org.usvm.StateCollectionStrategy +import org.usvm.machine.state.TsState +import org.usvm.statistics.UMachineObserver +import org.usvm.util.TsMethodTestRunner +import org.usvm.util.getResourcePath +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration + +/** External coverage affects scheduling only, independently of native collection and stopping. */ +class TsMachineExternalCoverageTest : TsMethodTestRunner() { + override val scene = EtsScene( + projectFiles = listOf( + loadEtsFileAutoConvert( + getResourcePath("/samples/coverage/ClassCoverage.ts"), + provider = EtsIrProvider.TS_FRONTEND, + ), + ), + ) + + @Test + fun `TypeScript defaults to NURS and respects an explicit strategy`() { + TsMachine(scene = scene, tsOptions = TsOptions()).use { machine -> + assertEquals( + expected = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + actual = machine.options.pathSelectionStrategies, + ) + } + val explicitOptions = options.copy(pathSelectionStrategies = listOf(PathSelectionStrategy.BFS)) + + TsMachine(scene = scene, options = explicitOptions, tsOptions = TsOptions()).use { machine -> + assertEquals(expected = listOf(PathSelectionStrategy.BFS), actual = machine.options.pathSelectionStrategies) + } + } + + @Test + fun `external saturation before analysis neither stops execution nor suppresses native witnesses`() { + val method = getMethod(methodName = "entry", className = "HelperCoverage") + val statements = scene.projectClasses.flatMap { owner -> owner.methods.flatMap { it.cfg.stmts } } + val analysisOptions = options.copy( + coverageZone = CoverageZone.CLASS, + stateCollectionStrategy = StateCollectionStrategy.COVERED_NEW, + stopOnCoverage = 100, + timeout = Duration.INFINITE, + ) + + TsMachine(scene = scene, options = analysisOptions, tsOptions = TsOptions()).use { machine -> + assertTrue(machine.markCovered(statements)) + assertFalse(machine.markCovered(statements)) + + repeat(2) { + val result = machine.analyzeWithMetadata(methods = listOf(method)) + + assertEquals(expected = TsAnalysisStopReason.EXHAUSTED, actual = result.stopReason) + assertEquals(expected = 2, actual = result.states.size) + assertFalse(result.engineFailed) + } + } + } + + @Test + fun `external coverage can arrive on the analysis thread while paths are queued`() { + val method = getMethod(methodName = "entry", className = "HelperCoverage") + val statements = scene.projectClasses.flatMap { owner -> owner.methods.flatMap { it.cfg.stmts } } + lateinit var machine: TsMachine + var reported = false + val observer = object : UMachineObserver { + override fun onStatePeeked(state: TsState) { + if (!reported) { + reported = machine.markCovered(statements) + } + } + } + val analysisOptions = options.copy( + coverageZone = CoverageZone.CLASS, + stateCollectionStrategy = StateCollectionStrategy.COVERED_NEW, + stopOnCoverage = 0, + timeout = Duration.INFINITE, + ) + machine = TsMachine( + scene = scene, + options = analysisOptions, + tsOptions = TsOptions(), + machineObserver = observer, + ) + + val result = machine.use { it.analyzeWithMetadata(methods = listOf(method)) } + + assertTrue(reported) + assertEquals(expected = TsAnalysisStopReason.EXHAUSTED, actual = result.stopReason) + assertEquals(expected = 2, actual = result.states.size) + assertFalse(result.engineFailed) + } +} diff --git a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt index 1c5248ffef..a2b866e182 100644 --- a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt +++ b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt @@ -11,22 +11,18 @@ import kotlin.NoSuchElementException // TODO: what to do if elements have same priority? class DeterministicPriorityCollection(private val comparator: Comparator) : UPriorityCollection { - private inner class Entry(val element: T, val priority: Priority) : Comparable { - override fun compareTo(other: Entry): Int = comparator.compare(priority, other.priority) - } - - private var topElement: Entry? = null - private var priorityQueue = PriorityQueue() + private var topElement: Pair? = null + private val priorityQueue = PriorityQueue> { (_, p1), (_, p2) -> comparator.compare(p1, p2) } override val count: Int get() = priorityQueue.size + if (topElement == null) 0 else 1 override fun peek(): T { val currentTop = topElement - if (currentTop != null) return currentTop.element + if (currentTop != null) return currentTop.first val next = priorityQueue.remove() topElement = next - return next.element + return next.first } override fun update(element: T, priority: Priority) { @@ -36,45 +32,25 @@ class DeterministicPriorityCollection(private val comparator: Compa override fun remove(element: T) { // Don't traverse the whole queue if we remove the top element - if (topElement != null && topElement?.element == element) { + if (topElement != null && topElement?.first == element) { topElement = null return } - if (!priorityQueue.removeIf { it.element == element }) { + if (!priorityQueue.removeIf { (e, _) -> e == element }) { throw NoSuchElementException("Element not found in priority queue") } } override fun add(element: T, priority: Priority) { - assert({ !priorityQueue.any { it.element == element } }) { "Element already exists in priority queue" } + assert({ !priorityQueue.any { (e, _) -> e == element } }) { "Element already exists in priority queue" } val currentTop = topElement - if (currentTop != null && comparator.compare(currentTop.priority, priority) > 0) { - topElement = Entry(element, priority) + if (currentTop != null && comparator.compare(currentTop.second, priority) > 0) { + topElement = element to priority priorityQueue.add(currentTop) return } - priorityQueue.add(Entry(element, priority)) - } - - /** - * Recalculates each priority once and rebuilds the heap in linear time, without removing and reinserting elements. - * If [priority] throws, the collection keeps its previous priorities and cached selection. - */ - fun updatePriorities(priority: (T) -> Priority) { - val entries = ArrayList(count) - val currentTop = topElement - if (currentTop != null) { - entries.add(Entry(currentTop.element, priority(currentTop.element))) - } - for (entry in priorityQueue) { - entries.add(Entry(entry.element, priority(entry.element))) - } - - // The collection constructor heapifies Comparable entries; addAll would perform individual insertions. - val refreshedQueue = PriorityQueue(entries) - priorityQueue = refreshedQueue - topElement = null + priorityQueue.add(element to priority) } } diff --git a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt deleted file mode 100644 index 2537c630db..0000000000 --- a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt +++ /dev/null @@ -1,108 +0,0 @@ -package org.usvm.algorithms - -import org.junit.jupiter.api.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertTrue - -internal class DeterministicPriorityCollectionRefreshTest { - @Test - fun `refresh reorders every element including the cached selection`() { - for (comparator in listOf(naturalOrder(), reverseOrder())) { - val collection = DeterministicPriorityCollection(comparator) - collection.add(element = "first", priority = 1) - collection.add(element = "second", priority = 2) - collection.add(element = "third", priority = 3) - collection.peek() - val evaluations = mutableListOf() - val priorities = mapOf("first" to 3, "second" to 1, "third" to 2) - - collection.updatePriorities { element -> - evaluations.add(element) - priorities.getValue(element) - } - - assertEquals(expected = 3, actual = collection.count) - assertEquals(expected = 3, actual = evaluations.size) - assertEquals(expected = priorities.keys, actual = evaluations.toSet()) - val expected = priorities.keys.sortedWith { left, right -> - comparator.compare(priorities.getValue(left), priorities.getValue(right)) - } - assertEquals(expected = expected, actual = drain(collection)) - } - } - - @Test - fun `heap rebuilding uses a linear number of comparisons`() { - var comparisons = 0 - val comparator = Comparator { left, right -> - comparisons++ - left.compareTo(right) - } - val collection = DeterministicPriorityCollection(comparator) - val size = 1024 - repeat(size) { element -> collection.add(element = element, priority = element) } - collection.peek() - comparisons = 0 - - collection.updatePriorities { element -> size - element } - - assertTrue(comparisons <= 2 * size, "Heapify used $comparisons comparisons for $size elements") - assertEquals(expected = (0 until size).reversed().toList(), actual = drain(collection)) - } - - @Test - fun `equal refreshed priorities retain every element`() { - val collection = DeterministicPriorityCollection(naturalOrder()) - repeat(8) { element -> collection.add(element = element, priority = element) } - collection.peek() - - collection.updatePriorities { 0 } - - val elements = drain(collection) - assertEquals(expected = 8, actual = elements.size) - assertEquals(expected = (0 until 8).toSet(), actual = elements.toSet()) - } - - @Test - fun `failed refresh leaves old ordering and all elements intact`() { - val collection = DeterministicPriorityCollection(naturalOrder()) - collection.add(element = "first", priority = 1) - collection.add(element = "second", priority = 2) - collection.peek() - - assertFailsWith { - collection.updatePriorities { element -> - check(element != "second") - 100 - } - } - - assertEquals(expected = "first", actual = collection.peek()) - assertEquals(expected = listOf("first", "second"), actual = drain(collection)) - } - - @Test - fun `empty and singleton collections can be refreshed repeatedly`() { - val collection = DeterministicPriorityCollection(naturalOrder()) - - collection.updatePriorities { error("Empty collection") } - collection.add(element = "only", priority = 1) - collection.peek() - repeat(2) { collection.updatePriorities { 2 } } - - assertEquals(expected = listOf("only"), actual = drain(collection)) - collection.updatePriorities { error("Empty collection") } - assertEquals(expected = 0, actual = collection.count) - } - - private fun drain(collection: DeterministicPriorityCollection): List { - val elements = mutableListOf() - while (collection.count > 0) { - val element = collection.peek() - collection.remove(element) - elements.add(element) - } - return elements - } -} From d5f0f07d5036405d5ff44057c374a32f94e07224 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 19:29:53 +0300 Subject: [PATCH 09/15] Use NURS in the TS DSL and retain lazy guidance measurements --- .../coverage-selector-performance/README.md | 23 +++++++++- .../lazy-nurs/exploratory-outcomes.jsonl | 6 +++ .../lazy-nurs/exploratory-results.jsonl | 6 +++ .../lazy-nurs/manifest.json | 46 +++++++++++++++++++ .../lazy-nurs/sequential-outcomes.jsonl | 6 +++ .../lazy-nurs/sequential-results.jsonl | 6 +++ .../org/usvm/ts/pbt/dsl/TsPropertyTest.kt | 2 + 7 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md index 5df8bce855..e944031cf8 100644 --- a/docs/experiments/coverage-selector-performance/README.md +++ b/docs/experiments/coverage-selector-performance/README.md @@ -1,10 +1,29 @@ # Coverage selector performance on real JVM bytecode -These are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). +The original 28 runs below are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. -## Recorded results (2026-10-16) +## Lazy NURS follow-up (2026-10-16) + +The current harness uses the production `createClosestToUncoveredRandomPathSelector` with one aggregate `CallStackDistanceCalculator`. Coverage reports remove goals; normal add/update operations recompute only the affected state's weight. No queue-wide refresh or per-target calculator is used. This checks core NURS on real JVM bytecode, not TS-machine performance or concrete replay coverage. + +Two batches contain 12 successful runs and 1,094,076 interpreter steps, with seed 0 and the same limits as above. The first batch partly overlapped a dependent PBT build; its raw measurements are retained separately. The table uses only the subsequent sequential batch, which ran after that build finished. Other host activity was not controlled. + +| Program | Mode | Steps | Analysis seconds | Selector seconds | Covered goals | Global refreshes | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| codec | nurs | 100,000 | 11.829 | 1.277 | 74 | 0 | +| codec | uniform | 100,000 | 10.851 | 1.029 | 74 | 0 | +| gcd | nurs | 100,000 | 9.247 | 0.508 | 28 | 0 | +| gcd | uniform | 100,000 | 9.641 | 0.608 | 28 | 0 | +| loan | nurs | 73,519 | 3.344 | 0.328 | 123 | 0 | +| loan | uniform | 73,519 | 7.063 | 0.682 | 123 | 0 | + +Both strategies reached the step limit for Codec and Guava and exhausted LoanExam. NURS made 104,817 / 104,581 / 73,519 aggregate distance calls respectively, only through state add/update. All runs recorded zero distance work in peek and zero global refreshes. Goal counts match the uniform baseline in these runs; equal counts do not prove equal covered sets. Timing varied considerably between batches, so these short runs do not establish a stable speedup or a TS performance conclusion. + +Raw measurements, process outcomes, source hashes and batch conditions are in [`lazy-nurs/`](lazy-nurs/). The old results and their manifest are unchanged. To reproduce the current comparison, use the current checkout with `--modes nurs,uniform --seed 0 --repetitions 1`; to reproduce the original periodic modes, use the historical commit noted above. + +## Original recorded results (2026-10-16) The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl new file mode 100644 index 0000000000..b8bbbb4988 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl @@ -0,0 +1,6 @@ +{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 34.428} +{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.777} +{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 23.742} +{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 37.788} +{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.323} +{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 67.84} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl new file mode 100644 index 0000000000..ca88bc184c --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl @@ -0,0 +1,6 @@ +{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":8079.354458,"selectorMs":672.783741,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1092642416,"gcCount":7,"gcMs":502} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3722.539417,"selectorMs":318.351377,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1086488832,"gcCount":3,"gcMs":98} +{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":17174.910917,"selectorMs":1584.688466,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1053153056,"gcCount":6,"gcMs":735} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":21399.094625,"selectorMs":1286.346016,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1029830680,"gcCount":4,"gcMs":265} +{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":4847.856875,"selectorMs":534.124959,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":755228528,"gcCount":2,"gcMs":115} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":17242.797666,"selectorMs":1517.318704,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":737997944,"gcCount":5,"gcMs":343} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json b/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json new file mode 100644 index 0000000000..54358caa3d --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json @@ -0,0 +1,46 @@ +{ + "recordedDate": "2026-10-16", + "productionRevision": "465c303de7f1968a85511fd12d6b7d651ec29b97", + "cpu": "Apple M3 Pro", + "architecture": "arm64", + "memoryBytes": 19327352832, + "java": "Zulu 11.0.32.1+1", + "testHeapBytes": 4294967296, + "settings": { + "stepLimit": 100000, + "timeoutMillis": 120000, + "solver": "YICES", + "solverTimeoutMillis": 250, + "typeTimeoutMillis": 250, + "arrayMaxSize": 1500, + "useSoftConstraints": false, + "assertions": false, + "dynamicGoals": false, + "randomSeeds": [ + 0 + ] + }, + "counts": { + "runs": 12, + "interpreterSteps": 1094076, + "peakQueue": 4817 + }, + "librarySha256": { + "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", + "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" + }, + "sourceSha256": { + "usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt": "6fe66b001e94c4fca877626f109c79eb02b9a3e65c4e75af2f6c33f77fffb379", + "usvm-core/src/main/kotlin/org/usvm/statistics/distances/CallStackDistanceCalculator.kt": "7121a8c6152361afe49ef7b92d10d36ce1a9a76a226c03d0fd206acc5751b770", + "usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt": "674c113ed05d6ec26f7dbe2e608415cf8893f8b765bac58c9e26ac660fab3b6c", + "usvm-util/src/main/kotlin/org/usvm/algorithms/RandomizedPriorityCollection.kt": "f4b859ccef3f263d70853c6148c6809a205734d75197a7eccf41287fe569c9b8", + "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "be70752c25cae757597770a35731ebc9e675d427efaa9d5eb9a92448f67625ac", + "docs/experiments/coverage-selector-performance/run.py": "316e4eed00fd3f0c7cb4808df9401c714d760597931b0ccefbe089792aa708ab", + "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0" + }, + "harnessProvenance": "Current production NURS factory and aggregate CallStackDistanceCalculator; external coverage-report callback removes targets on the analysis thread. No concrete replay or public TsMachine run.", + "batches": { + "exploratory": "Some runs overlapped compilation and tests of the dependent PBT checkout. Retained for transparency; do not interpret timing differences as selector effects.", + "sequential": "Fresh test JVM per run, sequential runs after the dependent build finished. No concurrent build or test was launched by this chat; other host activity was not controlled." + } +} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl new file mode 100644 index 0000000000..2560960892 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl @@ -0,0 +1,6 @@ +{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 22.268} +{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 19.444} +{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 15.125} +{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 16.442} +{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 10.253} +{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 13.169} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl new file mode 100644 index 0000000000..8fdf8de67f --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl @@ -0,0 +1,6 @@ +{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":11829.20275,"selectorMs":1276.802646,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1098245584,"gcCount":5,"gcMs":356} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10851.263541,"selectorMs":1029.121608,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1076156496,"gcCount":6,"gcMs":263} +{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9246.969,"selectorMs":508.044296,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1047281200,"gcCount":2,"gcMs":112} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9640.541209,"selectorMs":608.421144,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1026080192,"gcCount":6,"gcMs":162} +{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3343.874542,"selectorMs":327.584665,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":749417280,"gcCount":2,"gcMs":88} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":7062.51475,"selectorMs":681.600274,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743486632,"gcCount":3,"gcMs":577} diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt index 9b32e22127..b0974e53b2 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt @@ -3,6 +3,7 @@ package org.usvm.ts.pbt.dsl import kotlinx.serialization.json.Json import org.jacodb.ets.utils.EtsIrProvider import org.jacodb.ets.utils.loadEtsProjectAutoConvert +import org.usvm.PathSelectionStrategy import org.usvm.SolverType import org.usvm.UMachineOptions import org.usvm.ts.pbt.backend.ProjectionCapability @@ -149,6 +150,7 @@ class DslPropertyRunner(sourceRoots: List) { val manifest = compiled.definition.toManifest() val mapping = PropertyEtsMapper(scene = scene, sourceRoots = sourceRoots).map(manifest) val options = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), solverType = SolverType.YICES, timeout = configuration.timeoutMillis.milliseconds, ) From 9b311ae3c81e4826c3f9f38dca0f5ef5efd474fe Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 19:35:51 +0300 Subject: [PATCH 10/15] Remove coverage selector benchmark artifacts --- .../CoverageSelectorPerformanceTest.kt | 366 ------------------ .../coverage-selector-performance/README.md | 149 ------- .../lazy-nurs/exploratory-outcomes.jsonl | 6 - .../lazy-nurs/exploratory-results.jsonl | 6 - .../lazy-nurs/manifest.json | 46 --- .../lazy-nurs/sequential-outcomes.jsonl | 6 - .../lazy-nurs/sequential-results.jsonl | 6 - .../coverage-selector-performance/logback.xml | 3 - .../manifest.json | 42 -- .../performance.gradle | 19 - .../results.jsonl | 28 -- .../coverage-selector-performance/run.py | 91 ----- .../summarize.py | 41 -- .../coverage-selector-performance/summary.csv | 25 -- .../CoverageSelectorPerformanceTest.kt | 323 ---------------- 15 files changed, 1157 deletions(-) delete mode 100644 docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt delete mode 100644 docs/experiments/coverage-selector-performance/README.md delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/logback.xml delete mode 100644 docs/experiments/coverage-selector-performance/manifest.json delete mode 100644 docs/experiments/coverage-selector-performance/performance.gradle delete mode 100644 docs/experiments/coverage-selector-performance/results.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/run.py delete mode 100644 docs/experiments/coverage-selector-performance/summarize.py delete mode 100644 docs/experiments/coverage-selector-performance/summary.csv delete mode 100644 usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt diff --git a/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt b/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt deleted file mode 100644 index df0da45000..0000000000 --- a/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt +++ /dev/null @@ -1,366 +0,0 @@ -package org.usvm.experiments - -import kotlinx.coroutines.runBlocking -import org.jacodb.api.jvm.JcClasspath -import org.jacodb.api.jvm.JcMethod -import org.jacodb.api.jvm.cfg.JcInst -import org.jacodb.api.jvm.ext.findClass -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.usvm.SolverType -import org.usvm.UMachine -import org.usvm.UMachineOptions -import org.usvm.UPathSelector -import org.usvm.algorithms.DeterministicPriorityCollection -import org.usvm.algorithms.RandomizedPriorityCollection -import org.usvm.machine.JcApplicationGraph -import org.usvm.machine.JcComponents -import org.usvm.machine.JcContext -import org.usvm.machine.JcMachineOptions -import org.usvm.machine.JcTypeSystem -import org.usvm.machine.interpreter.JcInterpreter -import org.usvm.machine.state.JcState -import org.usvm.ps.BfsPathSelector -import org.usvm.ps.CoverageGuidedPathSelector -import org.usvm.ps.RandomTreePathSelector -import org.usvm.ps.WeightedPathSelector -import org.usvm.samples.JacoDBContainer -import org.usvm.statistics.CompositeUMachineObserver -import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.UMachineObserver -import org.usvm.statistics.distances.CallStackDistanceCalculator -import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.statistics.distances.MultiTargetDistanceCalculator -import org.usvm.stopstrategies.StopStrategy -import org.usvm.util.originalInst -import java.io.File -import java.lang.management.ManagementFactory -import kotlin.random.Random -import kotlin.time.Duration.Companion.milliseconds - -/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ -@Tag("manual") -class CoverageSelectorPerformanceTest { - @Test - fun runProgram() { - val program = System.getProperty("coverage.perf.case", "codec") - val mode = System.getProperty("coverage.perf.mode", "128") - val steps = System.getProperty("coverage.perf.steps", "100000").toLong() - val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() - val resultFile = File(System.getProperty("coverage.perf.results")) - val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) - val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) - val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) - try { - val (className, methodName) = when (program) { - "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" - "gcd" -> "com.google.common.math.IntMath" to "gcd" - "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" - else -> error("Unknown program: $program") - } - val cls = container.cp.findClass(className) - val method = cls.declaredMethods.single { it.name == methodName } - val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } - PerformanceMachine(container.cp).use { machine -> - val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) - resultFile.parentFile.mkdirs() - resultFile.appendText(result + "\n") - } - } finally { - runBlocking { - container.cp.close() - container.db.close() - } - } - } -} - -private class PerformanceMachine(cp: JcClasspath) : UMachine() { - override val options = UMachineOptions( - randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), - solverType = SolverType.YICES, - solverTimeout = 250.milliseconds, - typeOperationsTimeout = 250.milliseconds, - throwExceptionOnStepFailure = true, - useSoftConstraints = false, - ) - private val graph = JcApplicationGraph(cp) - private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) - private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) - - fun measure( - program: String, - mode: String, - method: JcMethod, - methodsToCover: List, - stepLimit: Long, - timeoutMillis: Long, - ): String { - val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) - val goals = coverage.getUncoveredStatements() - val metrics = Metrics() - val initial = interpreter.getInitialState(method) - val (base, markCovered) = createGuidance(mode, goals, metrics, initial) - val selector = MeasuredSelector(base, metrics) - coverage.addOnCoveredObserver { _, _, statement -> - val started = System.nanoTime() - markCovered(statement) - metrics.selectorNanos += System.nanoTime() - started - metrics.coverageReports++ - } - val observer = metrics.observer() - val gcCountBefore = gcCount() - val gcMillisBefore = gcMillis() - val allocatedBefore = allocatedBytes() - val started = System.nanoTime() - var stop = "exhausted" - selector.add(states = listOf(initial)) - run( - interpreter = interpreter, - pathSelector = selector, - observer = CompositeUMachineObserver(coverage, observer), - isStateTerminated = { state -> state.callStack.isEmpty() }, - stopStrategy = StopStrategy { - val elapsedMillis = (System.nanoTime() - started) / 1_000_000 - when { - metrics.steps >= stepLimit -> { - stop = "steps" - true - } - elapsedMillis >= timeoutMillis -> { - stop = "timeout" - true - } - else -> { - false - } - } - }, - ) - val elapsedNanos = System.nanoTime() - started - val allocated = allocatedBytes() - allocatedBefore - val totals = AnalysisTotals( - elapsedNanos = elapsedNanos, - allocatedBytes = allocated, - gcCount = gcCount() - gcCountBefore, - gcMillis = gcMillis() - gcMillisBefore, - ) - return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) - } - - private fun createGuidance( - mode: String, - goals: Collection, - metrics: Metrics, - initial: JcState, - ): Pair, (JcInst) -> Unit> { - val cfg = CfgStatisticsImpl(graph) - val calculators = MultiTargetDistanceCalculator { target -> - CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) - } - val distance: (JcState, JcInst) -> UInt = { state, target -> - metrics.distanceCalls++ - calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) - } - val markCovered: (JcInst) -> Unit - val base: UPathSelector - if (mode == "eager") { - val eager = EagerCoverageSelector(goals, distance) - base = eager - markCovered = { eager.markCovered(it) } - } else if (mode == "existing") { - val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) - base = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state: JcState -> - metrics.distanceCalls++ - aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) - }, - ) - markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } - } else if (mode == "random-path" || mode == "uniform") { - base = createRandomSelector(mode, initial) - markCovered = {} - } else if (mode == "bfs") { - base = BfsPathSelector() - markCovered = {} - } else { - val periodic = CoverageGuidedPathSelector( - targets = goals, - distance = distance, - priorityRefreshInterval = mode.toInt(), - ) - base = periodic - markCovered = { periodic.markCovered(targets = listOf(it)) } - } - return base to markCovered - } - - private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { - val random = Random(seed = options.randomSeed) - if (mode == "uniform") { - return WeightedPathSelector( - priorityCollectionFactory = { - RandomizedPriorityCollection( - comparator = compareBy { it.id }, - unitIntervalRandom = { random.nextDouble() }, - ) - }, - weighter = { 1.0 }, - ) - } - - return RandomTreePathSelector.fromRoot( - root = initial.pathNode, - randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, - ) - } - - override fun close() = components.close() -} - -private class Metrics { - var steps = 0L - var terminated = 0L - var selectorNanos = 0L - var refreshNanos = 0L - var refreshes = 0L - var distanceCalls = 0L - var coverageReports = 0L - var queueSize = 0 - var peakQueue = 0 -} - -private class MeasuredSelector( - private val delegate: UPathSelector, - private val metrics: Metrics, -) : UPathSelector { - override fun isEmpty(): Boolean = delegate.isEmpty() - - override fun peek(): JcState { - val beforeCalls = metrics.distanceCalls - val started = System.nanoTime() - val result = delegate.peek() - val elapsed = System.nanoTime() - started - metrics.selectorNanos += elapsed - if (metrics.distanceCalls > beforeCalls) { - metrics.refreshes++ - metrics.refreshNanos += elapsed - } - return result - } - - override fun add(states: Collection) { - val started = System.nanoTime() - delegate.add(states) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize += states.size - metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) - } - - override fun update(state: JcState) { - val started = System.nanoTime() - delegate.update(state) - metrics.selectorNanos += System.nanoTime() - started - } - - override fun remove(state: JcState) { - val started = System.nanoTime() - delegate.remove(state) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize-- - } -} - -/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ -private class EagerCoverageSelector( - targets: Collection, - distance: (JcState, JcInst) -> UInt, -) : UPathSelector { - private val pending = targets.toMutableSet() - private var dirty = false - private val delegate = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, - ) - - fun markCovered(target: JcInst) { - if (pending.remove(target)) dirty = true - } - - override fun isEmpty(): Boolean = delegate.isEmpty() - override fun peek(): JcState { - if (dirty) { - val states = delegate.drain().toList() - delegate.add(states) - dirty = false - } - return delegate.peek() - } - override fun add(states: Collection) = delegate.add(states) - override fun update(state: JcState) = delegate.update(state) - override fun remove(state: JcState) = delegate.remove(state) -} - -private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionCount.coerceAtLeast(minimumValue = 0) -} - -private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionTime.coerceAtLeast(minimumValue = 0) -} -private fun allocatedBytes(): Long { - val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean - return bean.getThreadAllocatedBytes(Thread.currentThread().id) -} - -private data class AnalysisTotals( - val elapsedNanos: Long, - val allocatedBytes: Long, - val gcCount: Long, - val gcMillis: Long, -) - -private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { - override fun onStatePeeked(state: JcState) { - steps++ - } - - override fun onStateTerminated(state: JcState, stateReachable: Boolean) { - terminated++ - } -} - -private fun Metrics.report( - program: String, - mode: String, - goals: Int, - covered: Int, - stop: String, - totals: AnalysisTotals, -): String { - val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() - val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() - return """ - {"program":"$program", - "mode":"$mode", - "repetition":$repetition, - "randomSeed":$randomSeed, - "steps":$steps, - "stop":"$stop", - "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, - "selectorMs":${selectorNanos / 1_000_000.0}, - "refreshMs":${refreshNanos / 1_000_000.0}, - "refreshes":$refreshes, - "distanceCalls":$distanceCalls, - "peakQueue":$peakQueue, - "pending":$queueSize, - "goals":$goals, - "covered":$covered, - "coverageReports":$coverageReports, - "terminated":$terminated, - "allocatedBytes":${totals.allocatedBytes}, - "gcCount":${totals.gcCount}, - "gcMs":${totals.gcMillis}} - """.trimIndent().replace("\n", "") -} diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md deleted file mode 100644 index e944031cf8..0000000000 --- a/docs/experiments/coverage-selector-performance/README.md +++ /dev/null @@ -1,149 +0,0 @@ -# Coverage selector performance on real JVM bytecode - -The original 28 runs below are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). - -This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. - -## Lazy NURS follow-up (2026-10-16) - -The current harness uses the production `createClosestToUncoveredRandomPathSelector` with one aggregate `CallStackDistanceCalculator`. Coverage reports remove goals; normal add/update operations recompute only the affected state's weight. No queue-wide refresh or per-target calculator is used. This checks core NURS on real JVM bytecode, not TS-machine performance or concrete replay coverage. - -Two batches contain 12 successful runs and 1,094,076 interpreter steps, with seed 0 and the same limits as above. The first batch partly overlapped a dependent PBT build; its raw measurements are retained separately. The table uses only the subsequent sequential batch, which ran after that build finished. Other host activity was not controlled. - -| Program | Mode | Steps | Analysis seconds | Selector seconds | Covered goals | Global refreshes | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| codec | nurs | 100,000 | 11.829 | 1.277 | 74 | 0 | -| codec | uniform | 100,000 | 10.851 | 1.029 | 74 | 0 | -| gcd | nurs | 100,000 | 9.247 | 0.508 | 28 | 0 | -| gcd | uniform | 100,000 | 9.641 | 0.608 | 28 | 0 | -| loan | nurs | 73,519 | 3.344 | 0.328 | 123 | 0 | -| loan | uniform | 73,519 | 7.063 | 0.682 | 123 | 0 | - -Both strategies reached the step limit for Codec and Guava and exhausted LoanExam. NURS made 104,817 / 104,581 / 73,519 aggregate distance calls respectively, only through state add/update. All runs recorded zero distance work in peek and zero global refreshes. Goal counts match the uniform baseline in these runs; equal counts do not prove equal covered sets. Timing varied considerably between batches, so these short runs do not establish a stable speedup or a TS performance conclusion. - -Raw measurements, process outcomes, source hashes and batch conditions are in [`lazy-nurs/`](lazy-nurs/). The old results and their manifest are unchanged. To reproduce the current comparison, use the current checkout with `--modes nurs,uniform --seed 0 --repetitions 1`; to reproduce the original periodic modes, use the historical commit noted above. - -## Original recorded results (2026-10-16) - -The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. - -The distance-guidance comparison below contains 16 runs and 1,328,129 interpreter steps. The random comparison adds 12 runs; the complete retained dataset contains 28 runs and 2,422,205 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. - -| Program | Mode | Steps | Stop | Analysis seconds | Selector seconds | Selector share | Refresh ms | -| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | -| Codec | existing | 100,000 | steps | 17.145 | 0.134 | 0.78% | 0 | -| Codec | eager | 100,000 | steps | 15.300 | 4.284 | 27.96% | 13.86 | -| Codec | 128 | 100,000 | steps | 21.468 | 6.319 | 29.48% | 90.32 | -| Codec | 512 | 100,000 | steps | 19.110 | 5.842 | 30.61% | 146.87 | -| Guava gcd | existing | 61,538 | timeout | 120.167 | 0.235 | 0.20% | 0 | -| Guava gcd | eager | 57,141 | timeout | 120.024 | 10.822 | 9.02% | 5.36 | -| Guava gcd | 128 | 63,808 | timeout | 120.189 | 14.249 | 11.86% | 5.73 | -| Guava gcd | 512 | 51,566 | timeout | 120.269 | 8.584 | 7.14% | 3.48 | -| LoanExam | existing | 73,519 | exhausted | 2.227 | 0.069 | 3.08% | 0 | -| LoanExam | eager | 73,519 | exhausted | 2.003 | 0.248 | 12.38% | 29.85 | -| LoanExam | 128 | 73,519 | exhausted | 2.333 | 0.266 | 11.39% | 27.08 | -| LoanExam | 512 | 73,519 | exhausted | 2.319 | 0.244 | 10.51% | 16.97 | - -For interval 128: - -| Program | Initial goals | Covered goals | Peak queued states | Per-target distance calls | Analysis-thread allocated GB (decimal) | -| --- | ---: | ---: | ---: | ---: | ---: | -| Codec | 273 | 74 | 3,364 | 23,826,680 | 5.952 | -| Guava gcd | 673 | 17 | 3,022 | 43,862,759 | 10.762 | -| LoanExam | 126 | 123 | 4,378 | 886,279 | 0.717 | - -All distance-guidance modes in the preceding table reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. - -Codec whole-analysis times varied substantially between repetitions: interval 128 took 15.007–27.930 seconds, and `existing` took 14.066–20.224 seconds. Its per-target callback count was identical in both repetitions, and its selector share was 29.33–29.62%. Do not treat the median wall-time ratio as a stable speedup estimate. - -Coverage-guided runs produced only 1–26 peeks with distance refresh work per analysis. They do not establish the worst-case cost of continuously discovering new goals or changing coverage while the queue is large. Initial goals are static; dynamic `addTargets` performance is not exercised. - -A useful next change is a caller-supplied batched minimum-distance function over the snapshot, allowing domain-specific caches such as the aggregate `CallStackDistanceCalculator`. A generic per-target callback may depend on arbitrary mutable state, so the selector cannot safely introduce a statement/call-stack-only cache for every caller. The proposed batched API itself has not been implemented or benchmarked here. - -The production selector files match the recorded source hashes throughout the retained runs. The manifest records the final reproducible harness; setup-helper extraction and report formatting were refined between exploratory runs and the repeated Codec runs, without changes to the measured execution loop, selector algorithms or settings. - -Raw data: [results.jsonl](results.jsonl). Full median summary: [summary.csv](summary.csv). Source/settings checksums: [manifest.json](manifest.json). - -Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 28 retained experiment processes completed successfully. The four distance-guidance Guava analyses ended at the explicit time limit; the random Guava runs reached 100,000 steps. The experiment is excluded from ordinary JVM test runs. - - -## Random baselines - -Two additional strategies were measured once for each selector RNG seed, 0 and 1, with the same step/time limits and production interpreter: - -- `random-path`: the actual `RandomTreePathSelector.fromRoot` used by `RANDOM_PATH`; it samples branches of the execution tree and traverses that tree on selection. -- `uniform`: the existing `RandomizedPriorityCollection` with constant weight 1.0 in a `WeightedPathSelector`, giving uniform random selection among queued states. It performs no goal-distance calculations and does not traverse the execution tree. - -| Program | Mode | Seed | Steps | Analysis seconds | Selector seconds | Covered goals | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| codec | random-path | 0 | 100,000 | 13.359 | 6.585 | 74 | -| codec | random-path | 1 | 100,000 | 7.283 | 3.991 | 74 | -| codec | uniform | 0 | 100,000 | 3.937 | 0.339 | 74 | -| codec | uniform | 1 | 100,000 | 3.430 | 0.315 | 74 | -| gcd | random-path | 0 | 100,000 | 19.171 | 3.411 | 28 | -| gcd | random-path | 1 | 100,000 | 18.281 | 3.186 | 28 | -| gcd | uniform | 0 | 100,000 | 10.153 | 0.572 | 28 | -| gcd | uniform | 1 | 100,000 | 7.328 | 0.351 | 28 | -| loan | random-path | 0 | 73,519 | 3.868 | 0.883 | 123 | -| loan | random-path | 1 | 73,519 | 5.003 | 1.155 | 123 | -| loan | uniform | 0 | 73,519 | 3.046 | 0.254 | 123 | -| loan | uniform | 1 | 73,519 | 2.745 | 0.224 | 123 | - -Uniform random was a strong inexpensive baseline on both library methods in these runs. Codec reached the same covered-goal count (74) in 3.430–3.937 seconds, versus a 21.468-second median for interval 128. Guava uniform random reached 100,000 steps and 28 covered goals in 7.328–10.153 seconds; interval 128 reached 63,808 steps and 17 covered goals at the 120-second timeout. LoanExam is a counterexample: uniform random took 2.745–3.046 seconds, versus 2.333 seconds for interval 128, with the same 73,519 steps and 123 covered goals. - -The standard `RANDOM_PATH` strategy was itself relatively expensive on Codec: 3.991–6.585 seconds spent in its selector, versus 0.315–0.339 seconds for uniform random. A random strategy's data structure and selection distribution matter. - -Different exploration order changes work outside the selector as well. Guava's wall-time difference cannot be explained by selector overhead alone; its interpreter/solver cost has not been separately profiled. Two seeds and three programs support using cheap uniform random as a baseline, not a universal claim that it dominates all coverage guidance. Goal counts do not establish equality of covered-goal sets or concrete replay coverage. - -## Programs and setup - -- Apache Commons Codec 1.16.0: `org.apache.commons.codec.binary.BinaryCodec.toAsciiBytes(byte[])`. -- Guava 32.0.1-jre: `com.google.common.math.IntMath.gcd(int, int)`. -- Repository sample: `org.usvm.samples.psbenchmarks.LoanExam.getCreditPercent(CreditRequest)`. - -Coverage goals are the statements of all declared, non-abstract, non-native, non-constructor methods in the entry point's class, including static initialization. Coverage is reported by the standard `CoverageStatistics` observer for terminated reachable symbolic states. Class-wide goals intentionally include methods not visited by a particular entry point. - -Each analysis allows 100,000 interpreter steps or 120 seconds, whichever comes first. The timeout is checked between steps, so an in-flight solver operation may overshoot. Reaching the time limit is recorded as `stop=timeout`, not treated as exhaustive analysis. There is no early stop for coverage saturation or coverage stagnation, and no loop-iteration cap. - -Settings: Yices, 250 ms solver/type-operation timeout, implicit exceptions enabled, 1,500-element array bound, soft constraints disabled, 4 GiB Java heap, assertions disabled. Each run uses a fresh test JVM. JacoDB/classpath setup and Gradle startup are outside the analysis timer; initial queue insertion and the complete execution loop are inside it. These are cold-start analysis measurements, not JMH steady-state measurements. - -The recorded host was an Apple M3 Pro (arm64), 18 GiB RAM, Zulu JDK 11.0.32.1+1. Runs were sequential; other host activity was not controlled. - -## Selectors - -- `existing`: the aggregate `CallStackDistanceCalculator` / `WeightedPathSelector` strategy used by `CLOSEST_TO_UNCOVERED`. Coverage reports remove goals from that calculator. Queued state weights are refreshed on normal updates, without a global rebuild. -- `eager`: the original #526 algorithm, refreshing via `drain().toList()` and re-adding states after changed coverage. It uses the same cached-ordering fix and queue implementation as the periodic variants. -- `128` and `512`: the new `CoverageGuidedPathSelector`, using `HashSet` snapshots and the respective refresh interval. - -For the random strategies, the selector RNG seed is recorded separately for every row and CSV group. The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. - -`selectorMs` measures add/update/remove/peek and coverage-report handling, including distance evaluation. `refreshMs` identifies distance work inside `peek` and includes heap rebuilding plus selecting its root. `distanceCalls` counts per-target callbacks in eager/periodic modes, but aggregate calculator calls in `existing`; those are different units of work. `allocatedBytes` is all allocation on the analysis thread, not retained heap, native memory, or allocation attributed specifically to the selector. - -## Reproduction - -Use JDK 11 and cached project dependencies. The runner locates the two library jars in the Gradle cache, or accepts explicit `--codec-jar` and `--guava-jar` paths. It runs Gradle offline and retains per-run logs, raw JSONL measurements and process outcomes. - -```sh -JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ - --cases codec,gcd,loan --modes existing,128,eager,512 \ - --steps 100000 --timeout-ms 120000 --repetitions 1 \ - --output build/coverage-selector-performance - -JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ - --cases codec,gcd,loan --modes random-path,uniform --seed 0 --repetitions 1 \ - --output build/coverage-selector-performance - -JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ - --cases codec,gcd,loan --modes random-path,uniform --seed 1 --repetitions 1 \ - --output build/coverage-selector-performance - -python3 docs/experiments/coverage-selector-performance/summarize.py \ - build/coverage-selector-performance/results.jsonl -``` - -The experiment is tagged `manual` and excluded from ordinary test runs. The runner's dedicated Gradle task selects only this experiment and disables assertions, avoiding the debug duplicate-element scan in queue insertion. - -Library SHA-256: - -- Commons Codec: `56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d`. -- Guava: `bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5`. diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl deleted file mode 100644 index b8bbbb4988..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 34.428} -{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.777} -{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 23.742} -{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 37.788} -{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.323} -{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 67.84} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl deleted file mode 100644 index ca88bc184c..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":8079.354458,"selectorMs":672.783741,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1092642416,"gcCount":7,"gcMs":502} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3722.539417,"selectorMs":318.351377,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1086488832,"gcCount":3,"gcMs":98} -{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":17174.910917,"selectorMs":1584.688466,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1053153056,"gcCount":6,"gcMs":735} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":21399.094625,"selectorMs":1286.346016,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1029830680,"gcCount":4,"gcMs":265} -{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":4847.856875,"selectorMs":534.124959,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":755228528,"gcCount":2,"gcMs":115} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":17242.797666,"selectorMs":1517.318704,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":737997944,"gcCount":5,"gcMs":343} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json b/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json deleted file mode 100644 index 54358caa3d..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "recordedDate": "2026-10-16", - "productionRevision": "465c303de7f1968a85511fd12d6b7d651ec29b97", - "cpu": "Apple M3 Pro", - "architecture": "arm64", - "memoryBytes": 19327352832, - "java": "Zulu 11.0.32.1+1", - "testHeapBytes": 4294967296, - "settings": { - "stepLimit": 100000, - "timeoutMillis": 120000, - "solver": "YICES", - "solverTimeoutMillis": 250, - "typeTimeoutMillis": 250, - "arrayMaxSize": 1500, - "useSoftConstraints": false, - "assertions": false, - "dynamicGoals": false, - "randomSeeds": [ - 0 - ] - }, - "counts": { - "runs": 12, - "interpreterSteps": 1094076, - "peakQueue": 4817 - }, - "librarySha256": { - "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", - "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" - }, - "sourceSha256": { - "usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt": "6fe66b001e94c4fca877626f109c79eb02b9a3e65c4e75af2f6c33f77fffb379", - "usvm-core/src/main/kotlin/org/usvm/statistics/distances/CallStackDistanceCalculator.kt": "7121a8c6152361afe49ef7b92d10d36ce1a9a76a226c03d0fd206acc5751b770", - "usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt": "674c113ed05d6ec26f7dbe2e608415cf8893f8b765bac58c9e26ac660fab3b6c", - "usvm-util/src/main/kotlin/org/usvm/algorithms/RandomizedPriorityCollection.kt": "f4b859ccef3f263d70853c6148c6809a205734d75197a7eccf41287fe569c9b8", - "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "be70752c25cae757597770a35731ebc9e675d427efaa9d5eb9a92448f67625ac", - "docs/experiments/coverage-selector-performance/run.py": "316e4eed00fd3f0c7cb4808df9401c714d760597931b0ccefbe089792aa708ab", - "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0" - }, - "harnessProvenance": "Current production NURS factory and aggregate CallStackDistanceCalculator; external coverage-report callback removes targets on the analysis thread. No concrete replay or public TsMachine run.", - "batches": { - "exploratory": "Some runs overlapped compilation and tests of the dependent PBT checkout. Retained for transparency; do not interpret timing differences as selector effects.", - "sequential": "Fresh test JVM per run, sequential runs after the dependent build finished. No concurrent build or test was launched by this chat; other host activity was not controlled." - } -} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl deleted file mode 100644 index 2560960892..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 22.268} -{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 19.444} -{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 15.125} -{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 16.442} -{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 10.253} -{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 13.169} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl deleted file mode 100644 index 8fdf8de67f..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":11829.20275,"selectorMs":1276.802646,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1098245584,"gcCount":5,"gcMs":356} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10851.263541,"selectorMs":1029.121608,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1076156496,"gcCount":6,"gcMs":263} -{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9246.969,"selectorMs":508.044296,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1047281200,"gcCount":2,"gcMs":112} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9640.541209,"selectorMs":608.421144,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1026080192,"gcCount":6,"gcMs":162} -{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3343.874542,"selectorMs":327.584665,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":749417280,"gcCount":2,"gcMs":88} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":7062.51475,"selectorMs":681.600274,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743486632,"gcCount":3,"gcMs":577} diff --git a/docs/experiments/coverage-selector-performance/logback.xml b/docs/experiments/coverage-selector-performance/logback.xml deleted file mode 100644 index 463a09ebff..0000000000 --- a/docs/experiments/coverage-selector-performance/logback.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/docs/experiments/coverage-selector-performance/manifest.json b/docs/experiments/coverage-selector-performance/manifest.json deleted file mode 100644 index 1e41f1c164..0000000000 --- a/docs/experiments/coverage-selector-performance/manifest.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "recordedDate": "2026-10-16", - "productionRevision": "deca5d3bc", - "cpu": "Apple M3 Pro", - "architecture": "arm64", - "memoryBytes": 19327352832, - "java": "Zulu 11.0.32.1+1", - "testHeapBytes": 4294967296, - "settings": { - "stepLimit": 100000, - "timeoutMillis": 120000, - "solver": "YICES", - "solverTimeoutMillis": 250, - "typeTimeoutMillis": 250, - "arrayMaxSize": 1500, - "useSoftConstraints": false, - "assertions": false, - "dynamicGoals": false, - "randomSeeds": [ - 0, - 1 - ] - }, - "counts": { - "runs": 28, - "interpreterSteps": 2422205, - "peakQueue": 4921 - }, - "librarySha256": { - "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", - "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" - }, - "sourceSha256": { - "usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt": "5905b62b890e94b7390058e46c07fbd9dbb7285521ccc3c65abfceb458ce6b1e", - "usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt": "e55fd73657139ec726574f5ddfca43eccfe8d615520b5c30111146a65019bc32", - "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "42c2c518e157ff8edca81044c7561c6832a9868eef0ef0514c223c9ee05209a2", - "docs/experiments/coverage-selector-performance/run.py": "9c6affa7309e0a6bbacfcf6fd5fdb005ee63817aa3d1a8103a3caf635c667a50", - "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0", - "docs/experiments/coverage-selector-performance/summarize.py": "67b695a12348503a3700d26b3fc938337ff9215a392353060ece05713dbf2fa2" - }, - "harnessProvenance": "Final reproducible harness; setup-helper extraction and report-formatting refinements occurred during exploratory runs. Production selector sources, measured execution loop and settings were unchanged." -} diff --git a/docs/experiments/coverage-selector-performance/performance.gradle b/docs/experiments/coverage-selector-performance/performance.gradle deleted file mode 100644 index 8a301e7f07..0000000000 --- a/docs/experiments/coverage-selector-performance/performance.gradle +++ /dev/null @@ -1,19 +0,0 @@ -gradle.projectsEvaluated { - def jvm = gradle.rootProject.findProject(':usvm-jvm') - if (jvm == null) return - jvm.tasks.register('coverageSelectorPerformance', Test) { - testClassesDirs = jvm.sourceSets.test.output.classesDirs - classpath = jvm.sourceSets.test.runtimeClasspath - dependsOn(jvm.tasks.named('testClasses')) - useJUnitPlatform { includeTags 'manual' } - filter { includeTestsMatching 'org.usvm.experiments.CoverageSelectorPerformanceTest' } - enableAssertions = false - maxHeapSize = '4g' - def logging = jvm.rootProject.file('docs/experiments/coverage-selector-performance/logback.xml') - jvmArgs "-Dlogback.configurationFile=${logging.absolutePath}" - ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition', 'seed'].each { key -> - systemProperty("coverage.perf.${key}", jvm.findProperty("coveragePerf${key.capitalize()}")) - } - outputs.upToDateWhen { false } - } -} diff --git a/docs/experiments/coverage-selector-performance/results.jsonl b/docs/experiments/coverage-selector-performance/results.jsonl deleted file mode 100644 index 1fe7bae02f..0000000000 --- a/docs/experiments/coverage-selector-performance/results.jsonl +++ /dev/null @@ -1,28 +0,0 @@ -{"program":"gcd","mode":"existing","steps":61538,"stop":"timeout","elapsedMs":120167.436333,"selectorMs":235.437487,"refreshMs":0.0,"refreshes":0,"distanceCalls":64765,"peakQueue":3227,"pending":3227,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":856651016,"gcCount":5,"gcMs":449,"repetition":1} -{"program":"gcd","mode":"128","steps":63808,"stop":"timeout","elapsedMs":120189.039625,"selectorMs":14249.244039,"refreshMs":5.728084,"refreshes":2,"distanceCalls":43862759,"peakQueue":3022,"pending":3022,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":10761566008,"gcCount":10,"gcMs":304,"repetition":1} -{"program":"gcd","mode":"eager","repetition":1,"steps":57141,"stop":"timeout","elapsedMs":120024.317916,"selectorMs":10822.395495,"refreshMs":5.358375,"refreshes":3,"distanceCalls":39557171,"peakQueue":3124,"pending":3124,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":9568604488,"gcCount":12,"gcMs":435} -{"program":"gcd","mode":"512","repetition":1,"steps":51566,"stop":"timeout","elapsedMs":120269.191542,"selectorMs":8584.144519,"refreshMs":3.480708,"refreshes":1,"distanceCalls":35593159,"peakQueue":2655,"pending":2655,"goals":673,"covered":17,"coverageReports":17,"terminated":5,"allocatedBytes":8698358712,"gcCount":9,"gcMs":175} -{"program":"loan","mode":"existing","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2227.168709,"selectorMs":68.547639,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":513509984,"gcCount":2,"gcMs":26} -{"program":"loan","mode":"128","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2332.535792,"selectorMs":265.641903,"refreshMs":27.08429,"refreshes":18,"distanceCalls":886279,"peakQueue":4378,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":716986848,"gcCount":2,"gcMs":24} -{"program":"loan","mode":"eager","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2003.228,"selectorMs":248.054217,"refreshMs":29.854126,"refreshes":26,"distanceCalls":944047,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":734125672,"gcCount":2,"gcMs":36} -{"program":"loan","mode":"512","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2319.290542,"selectorMs":243.802455,"refreshMs":16.965501,"refreshes":12,"distanceCalls":760175,"peakQueue":4360,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":701963040,"gcCount":2,"gcMs":19} -{"program":"codec","mode":"existing","repetition":1,"steps":100000,"stop":"steps","elapsedMs":14065.793084,"selectorMs":103.386872,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2071408672,"gcCount":5,"gcMs":327} -{"program":"codec","mode":"128","repetition":1,"steps":100000,"stop":"steps","elapsedMs":27929.542917,"selectorMs":8192.190751,"refreshMs":93.001749,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952381656,"gcCount":10,"gcMs":367} -{"program":"codec","mode":"eager","repetition":1,"steps":100000,"stop":"steps","elapsedMs":17828.648083,"selectorMs":5027.923518,"refreshMs":13.558208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5478178352,"gcCount":12,"gcMs":287} -{"program":"codec","mode":"512","repetition":1,"steps":100000,"stop":"steps","elapsedMs":18655.336042,"selectorMs":6038.100888,"refreshMs":193.471708,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":6332651112,"gcCount":9,"gcMs":260} -{"program":"codec","mode":"512","repetition":2,"steps":100000,"stop":"steps","elapsedMs":19563.939083,"selectorMs":5645.342032,"refreshMs":100.26425,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5946781568,"gcCount":8,"gcMs":210} -{"program":"codec","mode":"eager","repetition":2,"steps":100000,"stop":"steps","elapsedMs":12771.742584,"selectorMs":3540.902094,"refreshMs":14.152208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5479713280,"gcCount":8,"gcMs":186} -{"program":"codec","mode":"128","repetition":2,"steps":100000,"stop":"steps","elapsedMs":15007.014042,"selectorMs":4445.708042,"refreshMs":87.639833,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952287184,"gcCount":7,"gcMs":173} -{"program":"codec","mode":"existing","repetition":2,"steps":100000,"stop":"steps","elapsedMs":20224.028875,"selectorMs":165.022394,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2078604128,"gcCount":6,"gcMs":202} -{"program":"codec","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":13359.287291,"selectorMs":6585.102705,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4615,"pending":4615,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513024960,"gcCount":7,"gcMs":539} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3936.713375,"selectorMs":338.87798,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1080632408,"gcCount":3,"gcMs":62} -{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":19170.84275,"selectorMs":3410.622702,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4921,"pending":4920,"goals":673,"covered":28,"coverageReports":28,"terminated":895,"allocatedBytes":2143921128,"gcCount":3,"gcMs":41} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10152.718417,"selectorMs":572.30041,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1025193416,"gcCount":5,"gcMs":409} -{"program":"loan","mode":"random-path","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3868.197,"selectorMs":882.931681,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1527,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":881496912,"gcCount":2,"gcMs":50} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3045.632291,"selectorMs":253.601349,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743016424,"gcCount":3,"gcMs":98} -{"program":"codec","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7282.688708,"selectorMs":3990.851612,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4562,"pending":4562,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513769624,"gcCount":3,"gcMs":111} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":3429.624834,"selectorMs":314.7182,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4780,"pending":4780,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1087211024,"gcCount":4,"gcMs":122} -{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":18280.874209,"selectorMs":3186.388872,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4898,"pending":4897,"goals":673,"covered":28,"coverageReports":28,"terminated":911,"allocatedBytes":2151045424,"gcCount":3,"gcMs":32} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7327.864792,"selectorMs":351.211791,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4444,"pending":4440,"goals":673,"covered":28,"coverageReports":28,"terminated":1521,"allocatedBytes":1029454208,"gcCount":4,"gcMs":259} -{"program":"loan","mode":"random-path","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":5003.377792,"selectorMs":1155.413364,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1513,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":878046240,"gcCount":2,"gcMs":39} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":2744.655583,"selectorMs":224.198052,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":2902,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":747822944,"gcCount":2,"gcMs":26} diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py deleted file mode 100644 index c1e9ef53d9..0000000000 --- a/docs/experiments/coverage-selector-performance/run.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Run isolated JVM analyses; keep successes, failures and timeouts in the output directory.""" - -import argparse -import json -import os -from pathlib import Path -import subprocess -import time - - -def cached_jar(group, artifact, version): - cache = Path.home() / '.gradle/caches/modules-2/files-2.1' - matches = list((cache / group / artifact / version).glob(f'*/{artifact}-{version}.jar')) - if len(matches) != 1: - raise RuntimeError(f'Pass the jar explicitly: {group}:{artifact}:{version}') - return matches[0] - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--cases', default='codec,gcd,loan') - parser.add_argument('--modes', default='nurs,uniform') - parser.add_argument('--steps', type=int, default=100000) - parser.add_argument('--timeout-ms', type=int, default=120000) - parser.add_argument('--repetitions', type=int, default=2) - parser.add_argument('--seed', type=int, default=0) - parser.add_argument('--output', type=Path) - parser.add_argument('--codec-jar', type=Path) - parser.add_argument('--guava-jar', type=Path) - args = parser.parse_args() - - experiment = Path(__file__).resolve().parent - root = experiment.parents[2] - revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() - historical_revision = '44869512bca6b1ce89511b9f6799f4475efdf972' - historical_modes = {'128', '512', 'eager'} - uses_historical_modes = bool(set(args.modes.split(',')) & historical_modes) - if uses_historical_modes and revision != historical_revision: - parser.error(f'Historical experiment: run this script from a checkout of {historical_revision}') - - output = (args.output or root / 'build/coverage-selector-performance').resolve() - output.mkdir(parents=True, exist_ok=True) - codec = args.codec_jar or cached_jar('commons-codec', 'commons-codec', '1.16.0') - guava = args.guava_jar or cached_jar('com.google.guava', 'guava', '32.0.1-jre') - jars = os.pathsep.join(str(path.resolve()) for path in [codec, guava]) - cases = args.cases.split(',') - modes = args.modes.split(',') - results_file = output / 'results.jsonl' - outcomes_file = output / 'outcomes.jsonl' - - for repetition in range(1, args.repetitions + 1): - # Alternate mode order to reduce a fixed ordering effect. Runs remain sequential. - run_modes = modes if repetition % 2 else list(reversed(modes)) - for case in cases: - for mode in run_modes: - label = f'{case}-{mode}-s{args.seed}-r{repetition}' - print(f'START {label}', flush=True) - log_file = output / f'{label}.log' - command = [ - str(root / 'gradlew'), ':usvm-jvm:coverageSelectorPerformance', - '-I', str(experiment / 'performance.gradle'), - f'-PcoveragePerfCase={case}', f'-PcoveragePerfMode={mode}', - f'-PcoveragePerfSteps={args.steps}', - f'-PcoveragePerfTimeoutMillis={args.timeout_ms}', - f'-PcoveragePerfRepetition={repetition}', f'-PcoveragePerfSeed={args.seed}', - f'-PcoveragePerfResults={results_file}', f'-PcoveragePerfJars={jars}', - '--offline', '--console=plain', - ] - started = time.monotonic() - outcome = dict(program=case, mode=mode, repetition=repetition, randomSeed=args.seed, log=log_file.name) - try: - with log_file.open('w') as log: - process = subprocess.run( - command, cwd=root, env=os.environ.copy(), stdout=log, - stderr=subprocess.STDOUT, timeout=args.timeout_ms / 1000 + 180, - ) - outcome['exitCode'] = process.returncode - outcome['status'] = 'completed' if process.returncode == 0 else 'failed' - except subprocess.TimeoutExpired: - outcome['status'] = 'process-timeout' - outcome['wallSeconds'] = round(time.monotonic() - started, 3) - with outcomes_file.open('a') as outcomes: - outcomes.write(json.dumps(outcome) + '\n') - print(json.dumps(outcome), flush=True) - if outcome['status'] != 'completed': - print(log_file.read_text()[-4000:], flush=True) - - -if __name__ == '__main__': - main() diff --git a/docs/experiments/coverage-selector-performance/summarize.py b/docs/experiments/coverage-selector-performance/summarize.py deleted file mode 100644 index 5ba0af99d9..0000000000 --- a/docs/experiments/coverage-selector-performance/summarize.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -"""Write per-program/per-mode medians from benchmark JSONL to CSV.""" - -import argparse -import csv -import json -from collections import defaultdict -from pathlib import Path -from statistics import median -import sys - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('results', type=Path) - args = parser.parse_args() - groups = defaultdict(list) - for line in args.results.read_text().splitlines(): - row = json.loads(line) - seed = row.get('randomSeed') if row['mode'] in ['random-path', 'uniform'] else None - groups[row['program'], row['mode'], seed].append(row) - - metrics = ['steps', 'elapsedMs', 'selectorMs', 'selectorPercent', 'refreshMs', - 'refreshes', 'distanceCalls', 'peakQueue', 'covered', 'goals', - 'allocatedBytes', 'gcMs'] - fields = ['program', 'mode', 'randomSeed', 'runs', 'stops', *metrics] - writer = csv.DictWriter(sys.stdout, fieldnames=fields, lineterminator='\n') - writer.writeheader() - for (program, mode, seed), rows in sorted(groups.items()): - summary = dict(program=program, mode=mode, randomSeed=seed, runs=len(rows)) - summary['stops'] = ','.join(sorted({row['stop'] for row in rows})) - for field in metrics: - if field == 'selectorPercent': - summary[field] = median(100 * row['selectorMs'] / row['elapsedMs'] for row in rows) - else: - summary[field] = median(row[field] for row in rows) - writer.writerow(summary) - - -if __name__ == '__main__': - main() diff --git a/docs/experiments/coverage-selector-performance/summary.csv b/docs/experiments/coverage-selector-performance/summary.csv deleted file mode 100644 index fc5876db60..0000000000 --- a/docs/experiments/coverage-selector-performance/summary.csv +++ /dev/null @@ -1,25 +0,0 @@ -program,mode,randomSeed,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs -codec,128,,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 -codec,512,,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 -codec,eager,,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 -codec,existing,,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 -codec,random-path,0,1,steps,100000,13359.287291,6585.102705,49.29232047757748,0.0,0,0,4615,74,273,2513024960,539 -codec,random-path,1,1,steps,100000,7282.688708,3990.851612,54.79915141252801,0.0,0,0,4562,74,273,2513769624,111 -codec,uniform,0,1,steps,100000,3936.713375,338.87798,8.608144604888842,0.0,0,0,4790,74,273,1080632408,62 -codec,uniform,1,1,steps,100000,3429.624834,314.7182,9.176461427500849,0.0,0,0,4780,74,273,1087211024,122 -gcd,128,,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 -gcd,512,,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 -gcd,eager,,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 -gcd,existing,,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 -gcd,random-path,0,1,steps,100000,19170.84275,3410.622702,17.79067694872204,0.0,0,0,4921,28,673,2143921128,41 -gcd,random-path,1,1,steps,100000,18280.874209,3186.388872,17.430177767052758,0.0,0,0,4898,28,673,2151045424,32 -gcd,uniform,0,1,steps,100000,10152.718417,572.30041,5.636917980919514,0.0,0,0,4338,28,673,1025193416,409 -gcd,uniform,1,1,steps,100000,7327.864792,351.211791,4.7928257544192965,0.0,0,0,4444,28,673,1029454208,259 -loan,128,,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 -loan,512,,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 -loan,eager,,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 -loan,existing,,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 -loan,random-path,0,1,exhausted,73519,3868.197,882.931681,22.82540628101413,0.0,0,0,1527,123,126,881496912,50 -loan,random-path,1,1,exhausted,73519,5003.377792,1155.413364,23.09266683494125,0.0,0,0,1513,123,126,878046240,39 -loan,uniform,0,1,exhausted,73519,3045.632291,253.601349,8.326722492055428,0.0,0,0,3049,123,126,743016424,98 -loan,uniform,1,1,exhausted,73519,2744.655583,224.198052,8.168531359222275,0.0,0,0,2902,123,126,747822944,26 diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt deleted file mode 100644 index 4f41716094..0000000000 --- a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt +++ /dev/null @@ -1,323 +0,0 @@ -package org.usvm.experiments - -import kotlinx.coroutines.runBlocking -import org.jacodb.api.jvm.JcClasspath -import org.jacodb.api.jvm.JcMethod -import org.jacodb.api.jvm.cfg.JcInst -import org.jacodb.api.jvm.ext.findClass -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.usvm.SolverType -import org.usvm.UMachine -import org.usvm.UMachineOptions -import org.usvm.UPathSelector -import org.usvm.algorithms.DeterministicPriorityCollection -import org.usvm.algorithms.RandomizedPriorityCollection -import org.usvm.machine.JcApplicationGraph -import org.usvm.machine.JcComponents -import org.usvm.machine.JcContext -import org.usvm.machine.JcMachineOptions -import org.usvm.machine.JcTypeSystem -import org.usvm.machine.interpreter.JcInterpreter -import org.usvm.machine.state.JcState -import org.usvm.ps.BfsPathSelector -import org.usvm.ps.createClosestToUncoveredRandomPathSelector -import org.usvm.ps.RandomTreePathSelector -import org.usvm.ps.WeightedPathSelector -import org.usvm.samples.JacoDBContainer -import org.usvm.statistics.CompositeUMachineObserver -import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.UMachineObserver -import org.usvm.statistics.distances.CallStackDistanceCalculator -import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.stopstrategies.StopStrategy -import org.usvm.util.originalInst -import java.io.File -import java.lang.management.ManagementFactory -import kotlin.random.Random -import kotlin.time.Duration.Companion.milliseconds - -/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ -@Tag("manual") -class CoverageSelectorPerformanceTest { - @Test - fun runProgram() { - val program = System.getProperty("coverage.perf.case", "codec") - val mode = System.getProperty("coverage.perf.mode", "nurs") - val steps = System.getProperty("coverage.perf.steps", "100000").toLong() - val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() - val resultFile = File(System.getProperty("coverage.perf.results")) - val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) - val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) - val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) - try { - val (className, methodName) = when (program) { - "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" - "gcd" -> "com.google.common.math.IntMath" to "gcd" - "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" - else -> error("Unknown program: $program") - } - val cls = container.cp.findClass(className) - val method = cls.declaredMethods.single { it.name == methodName } - val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } - PerformanceMachine(container.cp).use { machine -> - val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) - resultFile.parentFile.mkdirs() - resultFile.appendText(result + "\n") - } - } finally { - runBlocking { - container.cp.close() - container.db.close() - } - } - } -} - -private class PerformanceMachine(cp: JcClasspath) : UMachine() { - override val options = UMachineOptions( - randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), - solverType = SolverType.YICES, - solverTimeout = 250.milliseconds, - typeOperationsTimeout = 250.milliseconds, - throwExceptionOnStepFailure = true, - useSoftConstraints = false, - ) - private val graph = JcApplicationGraph(cp) - private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) - private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) - - fun measure( - program: String, - mode: String, - method: JcMethod, - methodsToCover: List, - stepLimit: Long, - timeoutMillis: Long, - ): String { - val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) - val goals = coverage.getUncoveredStatements() - val metrics = Metrics() - val initial = interpreter.getInitialState(method) - val (base, markCovered) = createGuidance(mode, goals, metrics, initial) - val selector = MeasuredSelector(base, metrics) - coverage.addOnCoveredObserver { _, _, statement -> - val started = System.nanoTime() - markCovered(statement) - metrics.selectorNanos += System.nanoTime() - started - metrics.coverageReports++ - } - val observer = metrics.observer() - val gcCountBefore = gcCount() - val gcMillisBefore = gcMillis() - val allocatedBefore = allocatedBytes() - val started = System.nanoTime() - var stop = "exhausted" - selector.add(states = listOf(initial)) - run( - interpreter = interpreter, - pathSelector = selector, - observer = CompositeUMachineObserver(coverage, observer), - isStateTerminated = { state -> state.callStack.isEmpty() }, - stopStrategy = StopStrategy { - val elapsedMillis = (System.nanoTime() - started) / 1_000_000 - when { - metrics.steps >= stepLimit -> { - stop = "steps" - true - } - elapsedMillis >= timeoutMillis -> { - stop = "timeout" - true - } - else -> { - false - } - } - }, - ) - val elapsedNanos = System.nanoTime() - started - val allocated = allocatedBytes() - allocatedBefore - val totals = AnalysisTotals( - elapsedNanos = elapsedNanos, - allocatedBytes = allocated, - gcCount = gcCount() - gcCountBefore, - gcMillis = gcMillis() - gcMillisBefore, - ) - return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) - } - - private fun createGuidance( - mode: String, - goals: Collection, - metrics: Metrics, - initial: JcState, - ): Pair, (JcInst) -> Unit> { - if (mode == "random-path" || mode == "uniform") { - return createRandomSelector(mode, initial) to {} - } - if (mode == "bfs") return BfsPathSelector() to {} - require(mode == "nurs" || mode == "existing") { "Historical modes require checkout 44869512b" } - - val aggregate = CallStackDistanceCalculator( - targets = goals, - cfgStatistics = CfgStatisticsImpl(graph), - applicationGraph = graph, - ) - val distance: (JcState) -> UInt = { state -> - metrics.distanceCalls++ - aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) - } - val base = if (mode == "nurs") { - createClosestToUncoveredRandomPathSelector( - distance = distance, - random = Random(seed = options.randomSeed), - ) - } else { - WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = distance, - ) - } - val markCovered: (JcInst) -> Unit = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } - return base to markCovered - } - - private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { - val random = Random(seed = options.randomSeed) - if (mode == "uniform") { - return WeightedPathSelector( - priorityCollectionFactory = { - RandomizedPriorityCollection( - comparator = compareBy { it.id }, - unitIntervalRandom = { random.nextDouble() }, - ) - }, - weighter = { 1.0 }, - ) - } - - return RandomTreePathSelector.fromRoot( - root = initial.pathNode, - randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, - ) - } - - override fun close() = components.close() -} - -private class Metrics { - var steps = 0L - var terminated = 0L - var selectorNanos = 0L - var refreshNanos = 0L - var refreshes = 0L - var distanceCalls = 0L - var coverageReports = 0L - var queueSize = 0 - var peakQueue = 0 -} - -private class MeasuredSelector( - private val delegate: UPathSelector, - private val metrics: Metrics, -) : UPathSelector { - override fun isEmpty(): Boolean = delegate.isEmpty() - - override fun peek(): JcState { - val beforeCalls = metrics.distanceCalls - val started = System.nanoTime() - val result = delegate.peek() - val elapsed = System.nanoTime() - started - metrics.selectorNanos += elapsed - if (metrics.distanceCalls > beforeCalls) { - metrics.refreshes++ - metrics.refreshNanos += elapsed - } - return result - } - - override fun add(states: Collection) { - val started = System.nanoTime() - delegate.add(states) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize += states.size - metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) - } - - override fun update(state: JcState) { - val started = System.nanoTime() - delegate.update(state) - metrics.selectorNanos += System.nanoTime() - started - } - - override fun remove(state: JcState) { - val started = System.nanoTime() - delegate.remove(state) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize-- - } -} - -private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionCount.coerceAtLeast(minimumValue = 0) -} - -private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionTime.coerceAtLeast(minimumValue = 0) -} -private fun allocatedBytes(): Long { - val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean - return bean.getThreadAllocatedBytes(Thread.currentThread().id) -} - -private data class AnalysisTotals( - val elapsedNanos: Long, - val allocatedBytes: Long, - val gcCount: Long, - val gcMillis: Long, -) - -private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { - override fun onStatePeeked(state: JcState) { - steps++ - } - - override fun onStateTerminated(state: JcState, stateReachable: Boolean) { - terminated++ - } -} - -private fun Metrics.report( - program: String, - mode: String, - goals: Int, - covered: Int, - stop: String, - totals: AnalysisTotals, -): String { - val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() - val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() - return """ - {"program":"$program", - "mode":"$mode", - "repetition":$repetition, - "randomSeed":$randomSeed, - "steps":$steps, - "stop":"$stop", - "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, - "selectorMs":${selectorNanos / 1_000_000.0}, - "refreshMs":${refreshNanos / 1_000_000.0}, - "refreshes":$refreshes, - "distanceCalls":$distanceCalls, - "peakQueue":$peakQueue, - "pending":$queueSize, - "goals":$goals, - "covered":$covered, - "coverageReports":$coverageReports, - "terminated":$terminated, - "allocatedBytes":${totals.allocatedBytes}, - "gcCount":${totals.gcCount}, - "gcMs":${totals.gcMillis}} - """.trimIndent().replace("\n", "") -} From 5d90bcfb3c3f2699c882c499f68de7bf65aa579e Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Thu, 15 Oct 2026 18:18:43 +0300 Subject: [PATCH 11/15] [TS PBT] Guide original property search with admitted edge coverage --- usvm-ts-fast-check/README.md | 49 ++- .../fast-check-adapter/package-lock.json | 3 +- .../fast-check-adapter/package.json | 1 + .../src/coverage-branches.ts | 17 +- .../fast-check-adapter/src/diagnostics.ts | 2 +- .../src/execute-property.ts | 37 +- .../fast-check-adapter/src/execution-cli.ts | 14 +- .../src/execution-coverage-worker.ts | 43 +- .../src/execution-coverage.ts | 48 ++- .../fast-check-adapter/src/v8-coverage.d.ts | 7 + .../test/coverage-branches.test.ts | 45 +++ .../test/execution-cli.test.ts | 123 ++++++ .../org/usvm/ts/pbt/dsl/TsPropertyTest.kt | 145 ++++++- .../usvm/ts/pbt/fastcheck/FastCheckBackend.kt | 21 +- .../pbt/fastcheck/FastCheckCoverageSession.kt | 8 +- .../fastcheck/FastCheckExecutionProtocol.kt | 2 + .../usvm/ts/pbt/dsl/PbtEdgeGuidanceTest.kt | 140 +++++++ .../fastcheck/UsvmEdgeFeedbackContractTest.kt | 212 ++++++++++ .../resources/properties/dsl/edge-guidance.ts | 35 ++ .../properties/dsl/guidance-property.ts | 14 + .../backend/PropertyBasedTestingBackend.kt | 5 + .../usvm/ts/pbt/backend/PropertyCoverage.kt | 14 + .../pbt/coverage/PropertyCoverageIdentity.kt | 37 ++ .../usvm/ts/pbt/usvm/UsvmCoverageFeedback.kt | 75 ++++ .../org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt | 367 ++++++++++++++++++ .../ts/pbt/usvm/UsvmPropertySearchModel.kt | 1 + .../usvm/ts/pbt/usvm/UsvmPropertySearcher.kt | 41 +- .../ts/pbt/usvm/UsvmPreconditionSearchTest.kt | 3 +- .../main/kotlin/org/usvm/machine/TsMachine.kt | 14 +- 29 files changed, 1470 insertions(+), 53 deletions(-) create mode 100644 usvm-ts-fast-check/fast-check-adapter/src/v8-coverage.d.ts create mode 100644 usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/PbtEdgeGuidanceTest.kt create mode 100644 usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt create mode 100644 usvm-ts-fast-check/src/test/resources/properties/dsl/edge-guidance.ts create mode 100644 usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmCoverageFeedback.kt create mode 100644 usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt diff --git a/usvm-ts-fast-check/README.md b/usvm-ts-fast-check/README.md index 97bf0d95c6..ff5df9bdb9 100644 --- a/usvm-ts-fast-check/README.md +++ b/usvm-ts-fast-check/README.md @@ -12,14 +12,57 @@ remain in [`usvm-ts-pbt`](../usvm-ts-pbt/README.md). One c8 execution produces the existing source-mapped Istanbul statement report and raw V8 ranges. The bounded TypeScript branch converter reads those same raw ranges, the executed source snapshot, and the source map. It emits ordered `if` arms only when the original TypeScript points map back exactly, each arm has a distinct V8 execution -point, and the arm counts add up to the count at the condition. A supported `if` without `else` additionally needs -a single terminating `return` or `throw` in its true arm and a following statement that is reached only on false. +point, and the arm counts add up to the count at the condition. An `if` without `else` uses either a terminating +true arm and a distinct false successor, or a following join whose execution count equals the condition count. +In the latter case the false count is the condition count minus the true count. Missing points, early exits and +throwing conditions that violate this conservation rule remain unknown. Nested `if` statements are supported under those conditions. Ambiguous counts or ranges and unsupported constructs produce diagnostics; exact statement mappings remain available. Loops, `switch`, conditional expressions, logical short-circuit ranges, and function or script ranges are never reinterpreted as `if` arms. The converter does not instrument the TypeScript program or execute it a second time. -The resulting edges are coverage artifacts; target-selection integration belongs to #399/#355. +The DSL's `BOTH` mode uses exact admitted coverage to guide the original property search as described below. + +## PBT edge feedback in `BOTH` + +`DslPropertyRunner.run(program, engine = PropertyEngine.BOTH, configuration = ...)` first runs FastCheck and then +passes its confirmed source edges to the same USVM analysis that searches for a violation of the author checks. +Coverage is enabled automatically in this mode. The predicate, domains and precondition keep their original roles; +coverage affects scheduling only and adds no constraints. + +This mode collects V8 counters only during admitted predicate invocations, excluding the precondition, rejected +inputs, import initialization and shrinking trials. A synchronous worker handshake keeps synchronous precondition +and predicate calls uninterrupted. Each report carries a digest of the manifest and local TypeScript source files. +Absent, stale, unsupported and no-admission evidence produce distinct classifications. Only exact mapped `if` arms +provide known covered or uncovered edges; statement hits and missing branch records provide neither. + +The deterministic selector alternates a nearest known-uncovered edge priority with a fair original-objective turn. +Covered prefixes stay in the graph. Full edge coverage stops the coverage objective while property exploration +continues; statement saturation does not stop a property search. Distances follow calls and active return sites and +are scheduling overapproximations, not claims of feasibility. +Configurations using state merging or selector loop bounds retain their existing selector with an explicit +unsupported-feedback diagnostic; this first policy does not silently bypass those controls. + +When a completed property path crosses a new target, its input is replayed natively as a single explicit example: +no fallback generation and no shrinking. A rejected input, changed oracle result, incompatible source/runtime or +missing claimed edge leaves coverage unchanged. Confirmed additions immediately change subsequent selections in +the running machine. Exact branches first observed during replay also become eligible goals. Each edge has at most +two replay attempts with distinct inputs; failed targets remain listed +as uncovered after that limit. At least half of scheduling turns remain available to the original objective. + +Inspect `run.symbolic?.edgeGuidance` for initial coverage/goals, actual selected states and traversed goals, +evidence revisions, replay outcomes/additions, remaining goals and diagnostics. Selection history is bounded to +4096 entries with an explicit omitted count. `run.elapsedMillis` and `run.budgetMillis` report the shared wall-time +budget; replay entries report native replay costs. FastCheck receives at most one third of the total budget. +After frontend preparation, one third of the remaining time is reserved for final witness replay and shrinking; +feedback replays use at most one third of the symbolic allocation. Mapping, startup, search and replay use the +same monotonic deadline. Stop checks occur between interpreter steps and backend calls; an already-running solver +query, frontend conversion already in progress, or process shutdown can exceed the deadline and is included in +elapsed time. A frontend conversion is never started after the deadline; `symbolicSkippedDueToDeadline` reports +that no symbolic analysis ran, and `requireSuccess()` treats it as inconclusive. + +The admitted collector covers the main Node isolate only. This slice does not infer relations, change PBT +generation, retain a session between separate runner calls or implement the repeated two-way campaign of #355. Run the backend checks with: diff --git a/usvm-ts-fast-check/fast-check-adapter/package-lock.json b/usvm-ts-fast-check/fast-check-adapter/package-lock.json index 3db9388792..3b7a99bb51 100644 --- a/usvm-ts-fast-check/fast-check-adapter/package-lock.json +++ b/usvm-ts-fast-check/fast-check-adapter/package-lock.json @@ -8,6 +8,7 @@ "name": "@usvm/fast-check-adapter", "version": "0.1.0", "dependencies": { + "@bcoe/v8-coverage": "1.0.2", "@jridgewell/trace-mapping": "0.3.31", "c8": "10.1.3", "fast-check": "4.9.0", @@ -23,7 +24,7 @@ }, "node_modules/@bcoe/v8-coverage": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "resolved": "https://nexus.vkteam.ru/repository/npm-remote/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", "license": "MIT", "engines": { diff --git a/usvm-ts-fast-check/fast-check-adapter/package.json b/usvm-ts-fast-check/fast-check-adapter/package.json index 3fcbf35f61..13635feb47 100644 --- a/usvm-ts-fast-check/fast-check-adapter/package.json +++ b/usvm-ts-fast-check/fast-check-adapter/package.json @@ -12,6 +12,7 @@ "test:compiled": "node --test dist/test/coverage-branches.test.js dist/test/entry-point.test.js dist/test/execute-property.test.js dist/test/execution-cli.test.js dist/test/js-value.test.js dist/test/observe-object.test.js dist/test/process-group-shutdown.test.js dist/test/process-supervisor.test.js dist/test/project-domain.test.js dist/test/projection-cli.test.js dist/test/source-replay.test.js dist/test/source-captures.test.js dist/test/local-source-closure.test.js dist/test/class-coverage-source.test.js" }, "dependencies": { + "@bcoe/v8-coverage": "1.0.2", "@jridgewell/trace-mapping": "0.3.31", "c8": "10.1.3", "fast-check": "4.9.0", diff --git a/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts b/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts index 67d26044c2..2868ba7f13 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts @@ -353,7 +353,9 @@ function convertIf( if (trueBody === undefined) { return { kind: 'unsupported', reason: 'Empty true arm has no V8 execution point' }; } - if (falseBody === undefined) { + const following = followingStatement(node); + const joinsAfterIf = node.elseStatement === undefined && falseBody === undefined && following !== undefined; + if (falseBody === undefined && !joinsAfterIf) { const reason = node.elseStatement === undefined ? 'An if without else requires a terminating true arm and a following false successor' : 'Empty false arm has no V8 execution point'; @@ -363,7 +365,14 @@ function convertIf( const conditionHits = countAt(node.getStart(ast), ast, coverage, map); const trueHits = countAt(trueBody.getStart(ast), ast, coverage, map); - const falseHits = countAt(falseBody.getStart(ast), ast, coverage, map); + // For a non-terminating true arm, the following statement is a join, not the + // false edge. Conservation is valid only when every condition evaluation + // reaches that join; a throwing condition/arm makes the record ambiguous. + const joinHits = joinsAfterIf ? countAt(following.getStart(ast), ast, coverage, map) : undefined; + const falseHits = joinsAfterIf + ? conditionHits !== undefined && joinHits === conditionHits && trueHits !== undefined + ? conditionHits - trueHits : undefined + : countAt((falseBody as ts.Statement).getStart(ast), ast, coverage, map); if (conditionHits === undefined || trueHits === undefined || falseHits === undefined || trueHits < 0 || falseHits < 0 || trueHits + falseHits !== conditionHits) { @@ -452,6 +461,10 @@ function falseSuccessor(node: ts.IfStatement): ts.Statement | undefined { (ts.isReturnStatement(soleStatement) || ts.isThrowStatement(soleStatement)); if (!terminates) return undefined; + return followingStatement(node); +} + +function followingStatement(node: ts.IfStatement): ts.Statement | undefined { const parent = node.parent; if (!ts.isBlock(parent) && !ts.isSourceFile(parent)) return undefined; diff --git a/usvm-ts-fast-check/fast-check-adapter/src/diagnostics.ts b/usvm-ts-fast-check/fast-check-adapter/src/diagnostics.ts index fba52cc37a..b2fd4351b1 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/diagnostics.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/diagnostics.ts @@ -1,4 +1,4 @@ -export type AdapterDiagnosticKind = 'invalid-request' | 'entry-point'; +export type AdapterDiagnosticKind = 'invalid-request' | 'entry-point' | 'coverage'; export interface AdapterDiagnosticDescriptor { readonly kind: AdapterDiagnosticKind; diff --git a/usvm-ts-fast-check/fast-check-adapter/src/execute-property.ts b/usvm-ts-fast-check/fast-check-adapter/src/execute-property.ts index f4b8714dba..93166119d6 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/execute-property.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/execute-property.ts @@ -1,3 +1,4 @@ +import type { ExecutionCoverage } from './execution-coverage.js'; import { performance } from 'node:perf_hooks'; import fc from 'fast-check'; import type { Parameters, RunDetails } from 'fast-check'; @@ -76,6 +77,8 @@ export interface FastCheckExecutionRequest { timeoutMillis: number; examples: TaggedJsValue[][]; observationRequest?: ObservationRequest; + admittedCoverage?: boolean; + exactReplay?: boolean; } export interface FastCheckFailureDetails { @@ -106,7 +109,7 @@ export interface FastCheckExecutionSuccess { result: FastCheckRunResult; } -export async function executeProperty(requestValue: unknown): Promise { +export async function executeProperty(requestValue: unknown, coverage?: ExecutionCoverage): Promise { const request = validateRequest(requestValue); const startedAt = performance.now(); const recorder = request.observationRequest === undefined @@ -127,12 +130,12 @@ export async function executeProperty(requestValue: unknown): Promise request.replayPath === undefined ? tracker?.phaseOf(values) ?? 'unknown' : 'replay'; - const property = buildProperty(arbitrary, predicate, precondition, contractErrors, recorder, phaseOf); + const property = buildProperty(arbitrary, predicate, precondition, contractErrors, recorder, phaseOf, coverage); const parameters = buildParameters(request, tracker); const details = await checkProperty(property, parameters, request.replayPath); @@ -158,6 +161,7 @@ function buildProperty( contractErrors: ContractErrorState, recorder: ObservationRecorder | undefined, phaseOf: (values: JsConcreteValue[]) => 'generation' | 'explicit' | 'shrink' | 'replay' | 'unknown', + coverage: ExecutionCoverage | undefined, ): fc.IProperty<[JsConcreteValue[]]> | fc.IAsyncProperty<[JsConcreteValue[]]> { const asynchronous = predicate.executionKind === 'async' || precondition?.executionKind === 'async'; let selectedCheckId: string | undefined; @@ -166,7 +170,7 @@ function buildProperty( return fc.asyncProperty(arbitrary, (values: JsConcreteValue[]): Promise => preserveAsyncContractError( contractErrors, async () => { - const result = await invokePropertyOnce(values, predicate, precondition, recorder, phaseOf(values)); + const result = await invokePropertyOnce(values, predicate, precondition, recorder, phaseOf(values), coverage); if (result === null) fc.pre(false); return result; @@ -179,7 +183,7 @@ function buildProperty( () => { let result: boolean | null; try { - result = invokePropertyOnce(values, predicate, precondition, recorder, phaseOf(values)) as boolean | null; + result = invokePropertyOnce(values, predicate, precondition, recorder, phaseOf(values), coverage) as boolean | null; } catch (error: unknown) { if (isNamedCheckFailure(error)) { selectedCheckId ??= error.checkId; @@ -202,6 +206,7 @@ function invokePropertyOnce( precondition: LoadedEntryPoint | undefined, recorder: ObservationRecorder | undefined, phase: 'generation' | 'explicit' | 'shrink' | 'replay' | 'unknown', + coverage?: ExecutionCoverage, ): boolean | null | Promise { const invocationValues = cloneArguments(values); const invoke = (): boolean | null | Promise => { @@ -218,6 +223,9 @@ function invokePropertyOnce( throw error; } + const collect = coverage?.admittedOnly === true && phase !== 'shrink' && phase !== 'unknown'; + if (collect) coverage.reset(); + try { const result = await predicate.invoke(invocationValues); recorder?.finish('admitted', result ? 'holds' : 'false'); @@ -226,6 +234,8 @@ function invokePropertyOnce( } catch (error: unknown) { recorder?.finish('admitted', predicateExceptionOutcome(error)); throw error; + } finally { + if (collect) coverage.admit(); } })(); } @@ -241,6 +251,9 @@ function invokePropertyOnce( throw error; } + const collect = coverage?.admittedOnly === true && phase !== 'shrink' && phase !== 'unknown'; + if (collect) coverage.reset(); + try { const result = predicate.invoke(invocationValues) as boolean; recorder?.finish('admitted', result ? 'holds' : 'false'); @@ -249,6 +262,8 @@ function invokePropertyOnce( } catch (error: unknown) { recorder?.finish('admitted', predicateExceptionOutcome(error)); throw error; + } finally { + if (collect) coverage.admit(); } }; @@ -403,6 +418,10 @@ function buildParameters( if (request.seed !== undefined) parameters.seed = request.seed; if (request.replayPath !== undefined) parameters.path = request.replayPath; + if (request.exactReplay) { + parameters.maxSkipsPerRun = 0; + parameters.endOnFailure = true; + } return parameters; } @@ -582,6 +601,14 @@ function validateRequest(value: unknown): FastCheckExecutionRequest { if (request.tsconfig !== undefined) validated.tsconfig = request.tsconfig as string; if (request.seed !== undefined) validated.seed = request.seed as number; if (request.replayPath !== undefined) validated.replayPath = request.replayPath as string; + if (request.admittedCoverage === true) validated.admittedCoverage = true; + if (request.exactReplay === true) { + if (request.numRuns !== 1 || examples.length !== 1 || request.replayPath !== undefined) { + throw protocolError(adapterDiagnostic.protocolRequestInvalid, 'Exact replay requires one explicit example', 'exactReplay'); + } + validated.exactReplay = true; + } + if (request.observationRequest !== undefined) { validated.observationRequest = validateObservationRequest(request.observationRequest, manifest); } diff --git a/usvm-ts-fast-check/fast-check-adapter/src/execution-cli.ts b/usvm-ts-fast-check/fast-check-adapter/src/execution-cli.ts index f9746a5bf3..a752e13142 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/execution-cli.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/execution-cli.ts @@ -13,13 +13,19 @@ interface FastCheckExecutionFailure { const writeProtocolOutput = process.stdout.write.bind(process.stdout) as typeof process.stdout.write; process.stdout.write = process.stderr.write.bind(process.stderr) as typeof process.stdout.write; -const coverage = await startExecutionCoverage(); +let request: unknown; +let inputError: unknown; +try { request = parseRequest(await readStdin()); } catch (error) { inputError = error; } +const admittedOnly = request !== null && typeof request === 'object' && + (request as Record).admittedCoverage === true; +const timeoutMillis = request !== null && typeof request === 'object' + ? (request as Record).timeoutMillis : undefined; +const coverage = await startExecutionCoverage(admittedOnly, typeof timeoutMillis === 'number' ? timeoutMillis : 60_000); try { let response: FastCheckExecutionSuccess | FastCheckExecutionFailure; try { - const request = parseRequest(await readStdin()); - - response = await executeProperty(request); + if (inputError !== undefined) throw inputError; + response = await executeProperty(request, coverage); } catch (error: unknown) { if (!(error instanceof ProtocolError)) throw error; diff --git a/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage-worker.ts b/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage-worker.ts index 0f821d5e85..0f1ed38a83 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage-worker.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage-worker.ts @@ -1,8 +1,10 @@ import { Session, type Debugger, type Profiler } from 'node:inspector'; -import { parentPort } from 'node:worker_threads'; +import { parentPort, workerData } from 'node:worker_threads'; +import { mergeProcessCovs } from '@bcoe/v8-coverage'; export interface ExecutionCoverageSnapshot { kind: 'coverage'; + admittedInvocations: number; result: Profiler.ScriptCoverage[]; scripts: Record; } @@ -12,13 +14,26 @@ if (port === null) throw new Error('Coverage collector requires a worker'); const session = new Session(); const parsedScripts = new Map(); +const scriptIds = new Map(); session.connectToMainThread(); session.on('Debugger.scriptParsed', ({ params }) => parsedScripts.set(params.scriptId, params)); // Keep the port referenced while inspector replies are pending. A one-shot // listener would let this worker exit before an asynchronous snapshot finishes. -port.on('message', () => { - void takeSnapshot().then(snapshot => port.postMessage(snapshot)).catch(error => { +let admitted: Profiler.ScriptCoverage[] = []; +let admittedInvocations = 0; + +const completion = new Int32Array(workerData.completion); +port.on('message', (command: 'reset' | 'admit' | 'take') => { + void takeSnapshot(command).then(snapshot => { + if (command === 'take') port.postMessage(snapshot); + else { + Atomics.store(completion, 0, 1); + Atomics.notify(completion, 0); + } + }).catch(error => { + Atomics.store(completion, 0, -1); + Atomics.notify(completion, 0); throw error; }); }); @@ -29,16 +44,22 @@ await post('Profiler.startPreciseCoverage', { callCount: true, detailed: true }) port.postMessage('ready'); -async function takeSnapshot(): Promise { +async function takeSnapshot(command: 'reset' | 'admit' | 'take'): Promise { const coverage = await post('Profiler.takePreciseCoverage'); + for (const script of coverage.result) scriptIds.set(script.url, script.scriptId); + if (command === 'admit') { + admitted = mergeProcessCovs([{ result: admitted }, { result: coverage.result }]).result; + admittedInvocations += 1; + } + const result = command === 'take' && workerData.admittedOnly === true ? admitted : coverage.result; const scripts: ExecutionCoverageSnapshot['scripts'] = Object.create(null); - for (const script of coverage.result) { + for (const script of command === 'take' ? result : []) { if (!script.url.startsWith('file:')) continue; const { scriptSource } = await post( - 'Debugger.getScriptSource', { scriptId: script.scriptId }, + 'Debugger.getScriptSource', { scriptId: scriptIds.get(script.url) ?? script.scriptId }, ); - const sourceMapUrl = parsedScripts.get(script.scriptId)?.sourceMapURL; + const sourceMapUrl = parsedScripts.get(scriptIds.get(script.url) ?? script.scriptId)?.sourceMapURL; scripts[script.url] = { // V8 offsets are UTF-16 offsets. Keep CR in a CRLF line, like Node's cache. lineLengths: scriptSource.split(/[\n\u2028\u2029]/).map(line => line.length), @@ -46,10 +67,12 @@ async function takeSnapshot(): Promise { }; } - await post('Profiler.stopPreciseCoverage'); - session.disconnect(); + if (command === 'take') { + await post('Profiler.stopPreciseCoverage'); + session.disconnect(); + } - return { kind: 'coverage', result: coverage.result, scripts }; + return { kind: 'coverage', result, scripts, admittedInvocations }; } function post(method: string, params: object = {}): Promise { diff --git a/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage.ts b/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage.ts index 3931dd187b..ae613942a9 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/execution-coverage.ts @@ -2,23 +2,31 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { findSourceMap } from 'node:module'; import path from 'node:path'; import { Worker } from 'node:worker_threads'; +import { performance } from 'node:perf_hooks'; +import { protocolError } from './js-value.js'; import type { ExecutionCoverageSnapshot } from './execution-coverage-worker.js'; import { stringifyProtocolJson } from './protocol-json.js'; export interface ExecutionCoverage { + readonly admittedOnly: boolean; + reset(): void; + admit(): void; write(): Promise; close(): Promise; } /** Collect coverage without invoking serializers installed by the source under test. */ -export async function startExecutionCoverage(): Promise { +export async function startExecutionCoverage(admittedOnly = false, timeoutMillis = 60_000): Promise { const rawDirectory = process.env.USVM_V8_COVERAGE_DIRECTORY; if (rawDirectory === undefined || rawDirectory.length === 0) return undefined; + const deadline = performance.now() + timeoutMillis; // Inspector's protocol serializer also consults inherited toJSON, so its // session lives in a worker that never executes the source under test. + const completion = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)); const worker = new Worker(new URL('./execution-coverage-worker.js', import.meta.url), { env: { ...process.env, NODE_V8_COVERAGE: '' }, + workerData: { admittedOnly, completion: completion.buffer }, }); let workerFailure: Error | undefined; let workerExited = false; @@ -61,21 +69,42 @@ export async function startExecutionCoverage(): Promise { + const pending = receive(); + worker.postMessage(command); + const report = await pending; + if (report === 'ready') throw new Error('Coverage collector did not produce a snapshot'); + return report; + } + + // A synchronous handshake preserves the uninterrupted precondition/predicate + // call: yielding here could run user microtasks and change its result. + function snapshotSync(command: 'reset' | 'admit'): void { + Atomics.store(completion, 0, 0); + worker.postMessage(command); + const remaining = Math.max(0, deadline - performance.now()); + const outcome = Atomics.wait(completion, 0, 0, remaining); + if (outcome === 'timed-out' || Atomics.load(completion, 0) !== 1) { + throw protocolError({ kind: 'coverage', code: 'coverage.snapshot.failed' }, + 'Admitted coverage snapshot failed or exhausted the run deadline', 'coverage'); + } + } return { + admittedOnly, + reset(): void { snapshotSync('reset'); }, + admit(): void { snapshotSync('admit'); }, async write(): Promise { // Descendants already hold their inherited environment. Clear the main // process before coverage is drained or any shutdown hook can serialize it. delete process.env.NODE_V8_COVERAGE; - const pending = receive(); - worker.postMessage('take'); - const snapshot = await pending; - if (snapshot === 'ready') throw new Error('Coverage collector did not produce a snapshot'); + const reportSnapshot = await snapshot('take'); const sourceMapCache: Record = Object.create(null); - for (const [url, script] of Object.entries(snapshot.scripts)) { + for (const [url, script] of Object.entries(reportSnapshot.scripts)) { const sourceMap = findSourceMap(url); if (sourceMap !== undefined || script.sourceMapUrl !== undefined) { sourceMapCache[url] = { @@ -86,8 +115,11 @@ export async function startExecutionCoverage(): Promise { diff --git a/usvm-ts-fast-check/fast-check-adapter/src/v8-coverage.d.ts b/usvm-ts-fast-check/fast-check-adapter/src/v8-coverage.d.ts new file mode 100644 index 0000000000..e685feb73d --- /dev/null +++ b/usvm-ts-fast-check/fast-check-adapter/src/v8-coverage.d.ts @@ -0,0 +1,7 @@ +declare module '@bcoe/v8-coverage' { + import type { Profiler } from 'node:inspector'; + + export function mergeProcessCovs(reports: { result: Profiler.ScriptCoverage[] }[]): { + result: Profiler.ScriptCoverage[]; + }; +} diff --git a/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts index 612c2e6441..ab8108ef73 100644 --- a/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts +++ b/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts @@ -288,6 +288,51 @@ ${calls} } }); +test('if without else measures the missing edge through conservation at a shared join', async () => { + for (const [calls, expected] of [ + ['subject(1);', [1, 0]], + ['subject(-1);', [0, 1]], + ['subject(1); subject(-1);', [1, 1]], + ] as const) { + const fixture = await runCoverage(` +function subject(value: number): number { + let result = 1; + if (value > 0) { result = 1; } + return result; +} +${calls} +`); + try { + const report = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.deepEqual(report.files[0]?.branches[0]?.arms.map(arm => arm.hits), expected); + assert.deepEqual(report.diagnostics, []); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } + } +}); + +test('a throwing condition cannot create a false-edge count at a shared join', async () => { + const fixture = await runCoverage(` +function gate(value: number): boolean { if (value < 0) throw new Error('expected'); return value > 0; } +function subject(value: number): number { + let result = 1; + if (gate(value)) { result = 1; } + return result; +} +for (const value of [-1, 1]) { try { subject(value); } catch {} } +`); + try { + const report = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.equal(report.files[0]?.branches.some(branch => branch.location.start.line === 5), false); + assert.ok(report.diagnostics.some(entry => entry.code === 'coverage.branch.ambiguous')); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + test('duplicate V8 scripts remain ambiguous', async () => { const fixture = await runCoverage(` function subject(value: number): number { diff --git a/usvm-ts-fast-check/fast-check-adapter/test/execution-cli.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/execution-cli.test.ts index 767df48fe5..def9e9d338 100644 --- a/usvm-ts-fast-check/fast-check-adapter/test/execution-cli.test.ts +++ b/usvm-ts-fast-check/fast-check-adapter/test/execution-cli.test.ts @@ -12,6 +12,129 @@ import { convertV8IfBranches } from '../src/coverage-branches.js'; const cliPath = fileURLToPath(new URL('../src/execution-cli.js', import.meta.url)); const coverageLauncher = fileURLToPath(new URL('../src/coverage-launcher.js', import.meta.url)); +test('admitted coverage excludes rejected preconditions and keeps sync predicate semantics', async () => { + const sourceRoot = await realpath(await mkdtemp(path.join(tmpdir(), 'usvm-admitted-coverage-'))); + const raw = path.join(sourceRoot, 'raw'); + try { + await writeFile(path.join(sourceRoot, 'helper.ts'), ` +export function helper(value: boolean): number { + if (value) { return 1; } + else { return 2; } +} +`); + await writeFile(path.join(sourceRoot, 'property.ts'), ` +import { helper } from './helper'; +let changed = false; +export function precondition(value: boolean): boolean { + helper(!value); + queueMicrotask(() => { changed = true; }); + return value; +} +export function predicate(_value: boolean): boolean { return helper(false) === 2 && !changed; } +`); + const request = executionRequest(sourceRoot); + const manifest = request.manifest as Record; + manifest.inputs = [{ name: 'value', domain: { kind: 'boolean' } }]; + manifest.precondition = { module: 'property.ts', exportName: 'precondition', executionKind: 'sync' }; + request.examples = [[{ kind: 'boolean', value: false }], [{ kind: 'boolean', value: true }]]; + request.admittedCoverage = true; + request.timeoutMillis = 5_000; + + const invocation = await invokeCli(JSON.stringify(request), { coverageDirectory: raw, timeoutMillis: 10_000 }); + const response = JSON.parse(invocation.stdout) as { status: string; result: { status: string; numSkips: number } }; + const branches = await convertV8IfBranches(raw, [sourceRoot]); + + assert.equal(invocation.exitCode, 0, invocation.stderr); + assert.equal(response.status, 'ok'); + assert.equal(response.result.status, 'success'); + assert.equal(response.result.numSkips, 1); + assert.deepEqual(branches.files.find(file => file.path.endsWith('/helper.ts'))?.branches + .map(branch => branch.arms.map(arm => arm.hits)), [[0, 1]]); + assert.equal(await readFile(path.join(raw, 'admitted-count.txt'), 'utf8'), '1'); + } finally { + await rm(sourceRoot, { recursive: true, force: true }); + } +}); + +test('admitted snapshots merge both branch outcomes without counting shrink trials', async () => { + const sourceRoot = await realpath(await mkdtemp(path.join(tmpdir(), 'usvm-admitted-merge-'))); + try { + await writeFile(path.join(sourceRoot, 'helper.ts'), ` +export function helper(value: boolean): number { + if (value) { return 1; } + else { return 2; } +} +`); + await writeFile(path.join(sourceRoot, 'property.ts'), ` +import { helper } from './helper'; +export function predicate(value: boolean): boolean { helper(value); return true; } +export function failing(value: number): boolean { helper(value === 0); return value === 0; } +`); + const request = executionRequest(sourceRoot); + const manifest = request.manifest as Record; + manifest.inputs = [{ name: 'value', domain: { kind: 'boolean' } }]; + request.examples = [[{ kind: 'boolean', value: true }], [{ kind: 'boolean', value: false }]]; + request.admittedCoverage = true; + request.numRuns = 2; + request.timeoutMillis = 5_000; + const raw = path.join(sourceRoot, 'raw'); + + const invocation = await invokeCli(JSON.stringify(request), { coverageDirectory: raw, timeoutMillis: 10_000 }); + const branches = await convertV8IfBranches(raw, [sourceRoot]); + + assert.equal(invocation.exitCode, 0, invocation.stderr); + assert.deepEqual(branches.files.find(file => file.path.endsWith('/helper.ts'))?.branches + .map(branch => branch.arms.map(arm => arm.hits)), [[1, 1]]); + assert.equal(await readFile(path.join(raw, 'admitted-count.txt'), 'utf8'), '2'); + + manifest.inputs = [{ name: 'value', domain: { kind: 'integer', min: 0, max: 10 } }]; + manifest.predicate = { module: 'property.ts', exportName: 'failing', executionKind: 'sync' }; + request.examples = [[{ kind: 'number', value: 'finite', bits: '4024000000000000' }]]; + request.numRuns = 1; + const shrinkRaw = path.join(sourceRoot, 'shrink-raw'); + const shrink = await invokeCli(JSON.stringify(request), { coverageDirectory: shrinkRaw, timeoutMillis: 10_000 }); + const shrinkResponse = JSON.parse(shrink.stdout) as { result: { numShrinks: number } }; + const shrinkBranches = await convertV8IfBranches(shrinkRaw, [sourceRoot]); + + assert.equal(shrink.exitCode, 0, shrink.stderr); + assert.ok(shrinkResponse.result.numShrinks > 0); + assert.deepEqual(shrinkBranches.files.find(file => file.path.endsWith('/helper.ts'))?.branches + .map(branch => branch.arms.map(arm => arm.hits)), [[0, 1]]); + assert.equal(await readFile(path.join(shrinkRaw, 'admitted-count.txt'), 'utf8'), '1'); + } finally { + await rm(sourceRoot, { recursive: true, force: true }); + } +}); + +test('exact replay stops on a rejected example without generated fallback', async () => { + const sourceRoot = await realpath(await mkdtemp(path.join(tmpdir(), 'usvm-exact-coverage-'))); + try { + await writeFile(path.join(sourceRoot, 'property.ts'), ` +export function precondition(value: boolean): boolean { return value; } +export function predicate(_value: boolean): boolean { return false; } +`); + const request = executionRequest(sourceRoot); + const manifest = request.manifest as Record; + manifest.inputs = [{ name: 'value', domain: { kind: 'boolean' } }]; + manifest.precondition = { module: 'property.ts', exportName: 'precondition', executionKind: 'sync' }; + request.examples = [[{ kind: 'boolean', value: false }]]; + request.exactReplay = true; + + const invocation = await invokeCli(JSON.stringify(request)); + const response = JSON.parse(invocation.stdout) as { result: { + numRuns: number; numSkips: number; counterexample: unknown; failure: { kind: string }; + } }; + + assert.equal(invocation.exitCode, 0, invocation.stderr); + assert.equal(response.result.numRuns, 0); + assert.equal(response.result.numSkips, 1); + assert.equal(response.result.counterexample, null); + assert.equal(response.result.failure.kind, 'precondition-exhausted'); + } finally { + await rm(sourceRoot, { recursive: true, force: true }); + } +}); + test('execution CLI emits one successful response document', async () => { const sourceRoot = await realpath(await mkdtemp(path.join(tmpdir(), 'usvm-execution-cli-'))); try { diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt index b0974e53b2..1daae02579 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt @@ -8,6 +8,8 @@ import org.usvm.SolverType import org.usvm.UMachineOptions import org.usvm.ts.pbt.backend.ProjectionCapability import org.usvm.ts.pbt.backend.ProjectionLevel +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.backend.PropertyFailureDetails import org.usvm.ts.pbt.backend.PropertyFailureKind import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunResult @@ -17,12 +19,14 @@ import org.usvm.ts.pbt.manifest.toManifest import org.usvm.ts.pbt.mapping.PropertyEtsMapper import org.usvm.ts.pbt.model.JsConcreteValue import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import org.usvm.ts.pbt.usvm.UsvmCoverageFeedback import org.usvm.ts.pbt.usvm.UsvmPropertySearchResult import org.usvm.ts.pbt.usvm.UsvmPropertySearchStatus import org.usvm.ts.pbt.usvm.UsvmPropertySearcher import java.nio.file.Files import java.nio.file.Path import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.TimeSource /** Selects execution without changing the author property or its JavaScript semantics. */ enum class PropertyEngine { @@ -37,6 +41,9 @@ data class DslPropertyRun( val concrete: PropertyRunResult?, val symbolic: UsvmPropertySearchResult?, val symbolicReplay: PropertyRunResult?, + val elapsedMillis: Long = 0, + val budgetMillis: Long? = null, + val symbolicSkippedDueToDeadline: Boolean = false, ) { fun requireSuccess() { val violation = listOfNotNull(concrete, symbolicReplay).firstOrNull { @@ -47,8 +54,13 @@ data class DslPropertyRun( } val concreteFailure = concrete?.status == PropertyRunStatus.FAILURE val incompleteSearch = symbolic != null && symbolic.status != UsvmPropertySearchStatus.NO_VIOLATION_REACHED - if (concreteFailure || incompleteSearch) { - throw DslPropertyExecutionError(this, "Property run was inconclusive: ${concrete?.failure ?: symbolic}") + if (concreteFailure || incompleteSearch || symbolicSkippedDueToDeadline) { + val message = if (symbolicSkippedDueToDeadline) { + "The total deadline expired before symbolic analysis could start" + } else { + "Property run was inconclusive: ${concrete?.failure ?: symbolic}" + } + throw DslPropertyExecutionError(this, message) } } @@ -101,6 +113,9 @@ class DslPropertyRunner(sourceRoots: List) { engine: PropertyEngine = PropertyEngine.FAST_CHECK, configuration: PropertyRunConfiguration = PropertyRunConfiguration(), ): DslPropertyRun { + val started = TimeSource.Monotonic.markNow() + fun remainingMillis(): Long = configuration.timeoutMillis - started.elapsedNow().inWholeMilliseconds + val directory = Files.createTempDirectory("usvm-kotlin-property-") try { val compiled = PropertyTypeScriptCompiler().compile( @@ -113,26 +128,67 @@ class DslPropertyRunner(sourceRoots: List) { copySources(directory) val sourceRoots = listOf(directory) val backend = FastCheckBackend(sourceRoots = sourceRoots) + val hybridConfiguration = if (engine == PropertyEngine.BOTH) { + val request = configuration.coverageRequest ?: PropertyCoverageRequest() + configuration.copy( + coverageRequest = request.copy(admittedOnly = true), + ) + } else { + configuration + } val concrete = if (engine != PropertyEngine.USVM) { - backend.run(property = compiled.definition, configuration = configuration) + val concreteBudget = if (engine == PropertyEngine.BOTH) { + minOf(remainingMillis(), maxOf(1L, configuration.timeoutMillis / CONCRETE_BUDGET_DIVISOR)) + } else { + remainingMillis() + } + if (concreteBudget <= 0L) { + exhaustedConcreteRun(compiled, configuration, started.elapsedNow().inWholeMilliseconds) + } else { + backend.run( + property = compiled.definition, + configuration = hybridConfiguration.copy(timeoutMillis = concreteBudget), + ) + } } else { null } val symbolic = if (engine != PropertyEngine.FAST_CHECK) { - search(compiled, directory, sourceRoots, configuration) + search( + compiled = compiled, + directory = directory, + sourceRoots = sourceRoots, + configuration = hybridConfiguration, + concrete = concrete, + backend = backend, + remainingMillis = ::remainingMillis, + ) } else { null } val candidate = symbolic?.inputs - val replay = if (candidate != null) { + val replay = if (candidate != null && remainingMillis() > 0) { backend.run( property = compiled.definition, - configuration = configuration.copy(replayPath = null, examples = listOf(candidate), numRuns = 1), + configuration = configuration.copy( + replayPath = null, + examples = listOf(candidate), + numRuns = 1, + timeoutMillis = remainingMillis().coerceAtLeast(1L), + ), ) } else { null } - return DslPropertyRun(compiled, concrete, symbolic, replay) + return DslPropertyRun( + property = compiled, + concrete = concrete, + symbolic = symbolic, + symbolicReplay = replay, + elapsedMillis = started.elapsedNow().inWholeMilliseconds, + budgetMillis = configuration.timeoutMillis, + symbolicSkippedDueToDeadline = engine != PropertyEngine.FAST_CHECK && symbolic == null, + ) } finally { Files.walk(directory).use { paths -> paths.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) @@ -145,22 +201,86 @@ class DslPropertyRunner(sourceRoots: List) { directory: Path, sourceRoots: List, configuration: PropertyRunConfiguration, - ): UsvmPropertySearchResult { + concrete: PropertyRunResult?, + backend: FastCheckBackend, + remainingMillis: () -> Long, + ): UsvmPropertySearchResult? { + if (remainingMillis() <= 0L) return null + val scene = loadEtsProjectAutoConvert(directory, provider = EtsIrProvider.TS_FRONTEND) val manifest = compiled.definition.toManifest() - val mapping = PropertyEtsMapper(scene = scene, sourceRoots = sourceRoots).map(manifest) + val mapping = PropertyEtsMapper(scene = scene, sourceRoots = sourceRoots).map(manifest, concrete?.coverage) + val remainingBudgetMillis = remainingMillis().coerceAtLeast(0L) + val finalReplayReserveMillis = remainingBudgetMillis / FINAL_REPLAY_BUDGET_DIVISOR + val searchBudgetMillis = remainingBudgetMillis - finalReplayReserveMillis + fun symbolicRemainingMillis(): Long = remainingMillis() - finalReplayReserveMillis + val options = UMachineOptions( pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), solverType = SolverType.YICES, - timeout = configuration.timeoutMillis.milliseconds, + timeout = searchBudgetMillis.milliseconds, ) - return UsvmPropertySearcher(scene = scene, machineOptions = options).search( + val replayBudgetMillis = searchBudgetMillis / FEEDBACK_REPLAY_BUDGET_DIVISOR + var spentReplayMillis = 0L + val feedback = if (concrete != null) { + UsvmCoverageFeedback(initialCoverage = concrete.coverage) { inputs -> + val replayRemaining = minOf(symbolicRemainingMillis(), replayBudgetMillis - spentReplayMillis) + if (replayRemaining <= 0L) { + null + } else { + val replayStarted = TimeSource.Monotonic.markNow() + try { + backend.run( + property = compiled.definition, + configuration = configuration.copy( + replayPath = null, + examples = listOf(inputs), + numRuns = 1, + timeoutMillis = replayRemaining, + exactReplay = true, + ), + ) + } finally { + spentReplayMillis += replayStarted.elapsedNow().inWholeMilliseconds + } + } + } + } else { + null + } + return UsvmPropertySearcher( + scene = scene, + machineOptions = options, + deadlineReached = { symbolicRemainingMillis() <= 0L }, + ).search( manifest = manifest, mapping = mapping, concreteCapability = ProjectionCapability(level = ProjectionLevel.EXACT), + coverageFeedback = feedback, ) } + private fun exhaustedConcreteRun( + compiled: CompiledDslProperty, + configuration: PropertyRunConfiguration, + elapsedMillis: Long, + ) = PropertyRunResult( + propertyId = compiled.definition.id, + status = PropertyRunStatus.FAILURE, + seed = configuration.seed ?: 0, + replayPath = null, + counterexample = null, + numRuns = 0, + numSkips = 0, + numShrinks = 0, + failure = PropertyFailureDetails( + kind = PropertyFailureKind.TIMEOUT, + errorName = "HybridDeadlineExceeded", + message = "The total property deadline expired before the concrete run", + ), + executionTimeMillis = elapsedMillis, + ) + private fun resolveModule(module: String, directory: Path): String { val modulePath = Path.of(module) val normalized = !modulePath.isAbsolute && '\\' !in module && @@ -201,6 +321,9 @@ class DslPropertyRunner(sourceRoots: List) { private companion object { const val GENERATED_MODULE = "__usvm_kotlin_property.ts" + const val CONCRETE_BUDGET_DIVISOR = 3L + const val FEEDBACK_REPLAY_BUDGET_DIVISOR = 3L + const val FINAL_REPLAY_BUDGET_DIVISOR = 3L val EXCLUDED_DIRECTORIES = setOf("node_modules", ".git", ".gradle", "build") } } diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt index 3fc53edcbf..7510d63640 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckBackend.kt @@ -1,10 +1,12 @@ package org.usvm.ts.pbt.fastcheck import org.usvm.ts.pbt.FastCheckDiagnosticCode +import org.usvm.ts.pbt.backend.CoverageDiagnostic import org.usvm.ts.pbt.backend.PropertyBasedTestingBackend import org.usvm.ts.pbt.backend.PropertyCoverageCapability import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunResult +import org.usvm.ts.pbt.coverage.captureCoverageIdentity import org.usvm.ts.pbt.manifest.toManifest import org.usvm.ts.pbt.model.JsConcreteValue import org.usvm.ts.pbt.model.PropertyDefinition @@ -44,6 +46,8 @@ class FastCheckBackend( validateConfiguration(property, configuration) val manifest = property.toManifest() + val admittedCoverage = configuration.coverageRequest?.admittedOnly == true + val identity = if (admittedCoverage) captureCoverageIdentity(manifest, sourceRoots) else null val sourceRootPaths = sourceRoots.map(Path::toString) val request = FastCheckExecutionRequest( manifest = manifest, @@ -56,9 +60,24 @@ class FastCheckBackend( examples = configuration.examples, observationRequest = configuration.observationRequest, coverageRequest = configuration.coverageRequest, + admittedCoverage = admittedCoverage, + exactReplay = configuration.exactReplay, ) - return client.check(request) + val result = client.check(request) + val coverage = result.coverage + if (identity == null || coverage == null) return result + + val unchanged = identity == captureCoverageIdentity(manifest, sourceRoots) + val diagnostics = if (unchanged) { + coverage.diagnostics + } else { + coverage.diagnostics + CoverageDiagnostic( + code = "coverage.source.stale", + message = "TypeScript sources changed during the concrete run", + ) + } + return result.copy(coverage = coverage.copy(identity = identity, diagnostics = diagnostics)) } private fun validateConfiguration( diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt index 25fd5c04f1..5a8a8dd095 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt @@ -106,7 +106,13 @@ internal class FastCheckCoverageSession private constructor( ) } - return result.copy(coverage = artifact) + val admittedInvocations = if (coverageRequest.admittedOnly) { + Files.readString(workspace.rawDirectory.resolve("admitted-count.txt")).trim().toInt() + } else { + null + } + + return result.copy(coverage = artifact.copy(admittedInvocations = admittedInvocations)) } private fun propertyEntryPointPaths(): Set { diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt index 53246a08fc..6722f4d231 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckExecutionProtocol.kt @@ -20,6 +20,8 @@ internal data class FastCheckExecutionRequest( val timeoutMillis: Long, val examples: List> = emptyList(), val observationRequest: PropertyObservationRequest? = null, + val admittedCoverage: Boolean = false, + val exactReplay: Boolean = false, @Transient val coverageRequest: PropertyCoverageRequest? = null, ) diff --git a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/PbtEdgeGuidanceTest.kt b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/PbtEdgeGuidanceTest.kt new file mode 100644 index 0000000000..a80dce86d2 --- /dev/null +++ b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/PbtEdgeGuidanceTest.kt @@ -0,0 +1,140 @@ +package org.usvm.ts.pbt.dsl + +import org.junit.jupiter.api.Test +import org.usvm.ts.pbt.backend.PropertyRunConfiguration +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.testResourcesRoot +import org.usvm.ts.pbt.usvm.UsvmEdgeReplayStatus +import org.usvm.ts.pbt.usvm.UsvmPropertySearchStatus +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +class PbtEdgeGuidanceTest { + private val runner = DslPropertyRunner(sourceRoots = listOf(testResourcesRoot().resolve("properties/dsl"))) + private val configuration = PropertyRunConfiguration(seed = 42, numRuns = 1, timeoutMillis = 30_000) + + @Test + fun `real PBT evidence changes the pending state selected in the original property search`() { + val program = positiveResult(exportName = "branch") + + val positive = run(program, examples = listOf(1)) + val negative = run(program, examples = listOf(-1)) + + val positiveTrace = assertNotNull(positive.symbolic?.edgeGuidance) + val negativeTrace = assertNotNull(negative.symbolic?.edgeGuidance) + assertEquals(setOf(true), positiveTrace.initialCovered.map { it.outcome }.toSet()) + assertEquals(setOf(false), positiveTrace.initialGoals.map { it.outcome }.toSet()) + assertEquals(setOf(false), negativeTrace.initialCovered.map { it.outcome }.toSet()) + assertEquals(setOf(true), negativeTrace.initialGoals.map { it.outcome }.toSet()) + val positiveSelection = positiveTrace.selections.first { it.pendingStates > 1 && it.distance == 0 } + val negativeSelection = negativeTrace.selections.first { it.pendingStates > 1 && it.distance == 0 } + assertEquals(false, positiveSelection.goal?.outcome) + assertEquals(true, negativeSelection.goal?.outcome) + assertTrue(positiveSelection.goal in positiveSelection.traversedGoals) + assertTrue(negativeSelection.goal in negativeSelection.traversedGoals) + assertEquals(setOf(false), positiveSelection.traversedGoals.map { it.outcome }.toSet()) + assertEquals(setOf(true), negativeSelection.traversedGoals.map { it.outcome }.toSet()) + val replayConfirmedNewEdge = positiveTrace.replays.any { + it.status == UsvmEdgeReplayStatus.CONFIRMED && it.additions.isNotEmpty() + } + assertTrue(replayConfirmedNewEdge, positiveTrace.toString()) + assertTrue(positiveTrace.remainingGoals.isEmpty(), positiveTrace.toString()) + assertTrue(negativeTrace.remainingGoals.isEmpty(), negativeTrace.toString()) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, positive.symbolic?.status) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, negative.symbolic?.status) + } + + @Test + fun `covered prefixes remain traversable for a nested missing edge and confirmed additions change later choices`() { + val program = positiveResult(exportName = "nested") + + val run = run(program, examples = listOf(1)) + + val trace = assertNotNull(run.symbolic?.edgeGuidance) + val nestedMissing = trace.initialGoals.single { it.outcome } + assertTrue(trace.replays.any { nestedMissing in it.additions }, trace.toString()) + assertTrue(trace.selections.any { it.evidenceRevision > 0 }, trace.toString()) + assertTrue(trace.remainingGoals.isEmpty(), trace.toString()) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, run.symbolic?.status) + } + + @Test + fun `native replay can make a previously unknown nested branch an exact goal in the same machine`() { + val program = positiveResult(exportName = "hidden") + + val run = run(program, examples = listOf(-1)) + + val trace = assertNotNull(run.symbolic?.edgeGuidance) + assertTrue(trace.initialGoals.none { it.method.contains("nested") }) + assertTrue(trace.replays.any { replay -> replay.additions.any { it.method.contains("nested") } }) + val nestedOutcomes = trace.confirmedCovered.filter { it.method.contains("nested") } + .groupBy { it.conditionIndex }.mapValues { (_, edges) -> edges.map { it.outcome }.toSet() } + assertEquals(2, nestedOutcomes.size) + val innerCondition = nestedOutcomes.entries.single { it.value == setOf(true, false) }.key + val selectedInnerGoal = trace.selections.any { + it.evidenceRevision > 0 && it.goal?.method?.contains("nested") == true && + it.goal?.conditionIndex == innerCondition + } + assertTrue(selectedInnerGoal) + // hidden calls nested only for positive inputs: nested's outer false edge remains infeasible here. + val remaining = trace.remainingGoals.single() + assertTrue(remaining.method.contains("nested")) + assertEquals(false, remaining.outcome) + assertEquals(setOf(true), nestedOutcomes.getValue(remaining.conditionIndex)) + assertTrue(remaining !in trace.confirmedCovered) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, run.symbolic?.status) + } + + @Test + fun `if without else retains a missing edge even when every statement was covered`() { + val program = positiveResult(exportName = "withoutElse") + + val run = run(program, examples = listOf(1)) + + val trace = assertNotNull(run.symbolic?.edgeGuidance) + val coverage = assertNotNull(run.concrete?.coverage) + val source = coverage.files.single { it.path.endsWith("edge-guidance.ts") } + val functionStatements = source.statements.filter { it.location.start.line in 23..28 } + assertTrue(functionStatements.isNotEmpty()) + assertTrue(functionStatements.all { it.hits > 0 }, functionStatements.toString()) + val missingOutcomes = trace.initialGoals.filter { it.method.contains("withoutElse") }.map { it.outcome }.toSet() + assertEquals(setOf(false), missingOutcomes) + assertTrue(trace.remainingGoals.isEmpty(), trace.toString()) + } + + @Test + fun `fully covered code still searches for a violation of the original property`() { + val program = defineProperty(id = "edge-guidance.saturated") { + val branch = function<(Double) -> Double>(module = "edge-guidance.ts", exportName = "branch") + val input by int(min = -100, max = 100) + + check(name = "original") { branch(input) notEqualTo input } + } + + val run = run(program, examples = listOf(-1, 3)) + + val trace = assertNotNull(run.symbolic?.edgeGuidance) + assertTrue(trace.initialGoals.isEmpty(), trace.toString()) + assertTrue(trace.selections.isNotEmpty()) + assertTrue(trace.selections.all { it.goal == null && it.originalObjectiveTurn }) + assertEquals(UsvmPropertySearchStatus.VIOLATION_REACHED, run.symbolic?.status) + assertEquals("original", run.symbolicReplay?.failure?.checkId) + } + + private fun positiveResult(exportName: String): DslProperty = defineProperty(id = "edge-guidance.$exportName") { + val function = function<(Double) -> Double>(module = "edge-guidance.ts", exportName = exportName) + val input by int(min = -100, max = 100) + + check(name = "positive") { function(input) greaterThan 0 } + } + + private fun run(program: DslProperty, examples: List): DslPropertyRun = runner.run( + program = program, + engine = PropertyEngine.BOTH, + configuration = configuration.copy( + numRuns = examples.size, + examples = examples.map { listOf(JsConcreteValue.number(it.toDouble())) }, + ), + ) +} diff --git a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt new file mode 100644 index 0000000000..ce3e526c4e --- /dev/null +++ b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt @@ -0,0 +1,212 @@ +package org.usvm.ts.pbt.fastcheck + +import org.jacodb.ets.model.EtsScene +import org.jacodb.ets.utils.EtsIrProvider +import org.jacodb.ets.utils.loadEtsProjectAutoConvert +import org.junit.jupiter.api.Test +import org.usvm.SolverType +import org.usvm.UMachineOptions +import org.usvm.ts.pbt.backend.ProjectionCapability +import org.usvm.ts.pbt.backend.ProjectionLevel +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.backend.PropertyRunConfiguration +import org.usvm.ts.pbt.backend.PropertyRunResult +import org.usvm.ts.pbt.manifest.toManifest +import org.usvm.ts.pbt.mapping.PropertyEtsMapper +import org.usvm.ts.pbt.model.IntegerDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.PropertyDefinition +import org.usvm.ts.pbt.model.PropertyId +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import org.usvm.ts.pbt.testResourcesRoot +import org.usvm.ts.pbt.usvm.UsvmCoverageEvidenceStatus +import org.usvm.ts.pbt.usvm.UsvmCoverageFeedback +import org.usvm.ts.pbt.usvm.UsvmEdgeReplayStatus +import org.usvm.ts.pbt.usvm.UsvmPropertySearchResult +import org.usvm.ts.pbt.usvm.UsvmPropertySearchStatus +import org.usvm.ts.pbt.usvm.UsvmPropertySearcher +import java.util.concurrent.CancellationException +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.seconds + +class UsvmEdgeFeedbackContractTest { + @Test + fun `only concrete evidence changes the selected pending path for the same scene and symbolic query`() { + val property = property() + val positive = backend.run(property, configuration) + val negative = backend.run( + property, + configuration.copy(examples = listOf(listOf(JsConcreteValue.number(-1.0)))), + ) + + val positiveSearch = search(property, positive.coverage) { null } + val negativeSearch = search(property, negative.coverage) { null } + + val positiveSelection = assertNotNull(positiveSearch.edgeGuidance).selections + .first { it.pendingStates > 1 && it.distance == 0 } + val negativeSelection = assertNotNull(negativeSearch.edgeGuidance).selections + .first { it.pendingStates > 1 && it.distance == 0 } + assertEquals(false, positiveSelection.goal?.outcome) + assertEquals(true, negativeSelection.goal?.outcome) + assertTrue(positiveSelection.goal in positiveSelection.traversedGoals) + assertTrue(negativeSelection.goal in negativeSelection.traversedGoals) + assertEquals(setOf(false), positiveSelection.traversedGoals.map { it.outcome }.toSet()) + assertEquals(setOf(true), negativeSelection.traversedGoals.map { it.outcome }.toSet()) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, positiveSearch.status) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, negativeSearch.status) + } + + @Test + fun `native replay cancellation propagates instead of becoming an ordinary replay failure`() { + val property = property() + val initial = backend.run(property, configuration) + + assertFailsWith { + search(property, initial.coverage) { throw CancellationException("cancelled") } + } + } + + @Test + fun `absent stale unsupported and no-admission evidence keep distinct outcomes and original property search`() { + val property = property() + val initial = backend.run(property, configuration) + val coverage = assertNotNull(initial.coverage) + val identity = assertNotNull(coverage.identity) + val stale = coverage.copy(identity = identity.copy(manifestSha256 = "0".repeat(64))) + val unsupported = coverage.copy( + files = coverage.files.map { file -> file.copy(branches = file.branches.filter { it.type != "if" }) }, + ) + val variants = listOf( + null to UsvmCoverageEvidenceStatus.ABSENT, + stale to UsvmCoverageEvidenceStatus.STALE, + coverage.copy(identity = null) to UsvmCoverageEvidenceStatus.UNSUPPORTED, + coverage.copy(admittedInvocations = 0) to UsvmCoverageEvidenceStatus.NO_ADMITTED_INPUTS, + unsupported to UsvmCoverageEvidenceStatus.UNSUPPORTED, + ) + + for ((evidence, status) in variants) { + val result = search(property, evidence) { error("No exact goals should request replay") } + + val trace = assertNotNull(result.edgeGuidance) + assertEquals(status, trace.evidenceStatus) + assertTrue(trace.initialGoals.isEmpty()) + assertTrue(trace.initialCovered.isEmpty()) + assertTrue(trace.selections.isNotEmpty()) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, result.status) + } + } + + @Test + fun `failed replay and wrong edge receipt leave the target concretely uncovered`() { + val property = property() + val initial = backend.run(property, configuration) + val coverage = assertNotNull(initial.coverage) + + val failed = search(property, coverage) { throw IllegalStateException("native failure") } + val mismatched = search(property, coverage) { initial } + + val results = listOf( + failed to UsvmEdgeReplayStatus.REPLAY_FAILED, + mismatched to UsvmEdgeReplayStatus.EDGE_MISMATCH, + ) + for ((result, expectedStatus) in results) { + val trace = assertNotNull(result.edgeGuidance) + assertEquals(trace.initialGoals, trace.remainingGoals) + assertEquals(trace.initialCovered, trace.confirmedCovered) + assertTrue(trace.replays.isNotEmpty(), trace.toString()) + assertTrue(trace.replays.all { it.status == expectedStatus && it.additions.isEmpty() }, trace.toString()) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, result.status) + } + } + + @Test + fun `rejected precondition cannot remove a source edge from the real coverage target queue`() { + val property = property().copy( + predicate = TypeScriptEntryPoint(module = "guidance-property.ts", exportName = "guarded"), + precondition = TypeScriptEntryPoint(module = "guidance-property.ts", exportName = "admission"), + ) + val initial = backend.run( + property, + configuration.copy( + examples = listOf(listOf(JsConcreteValue.number(-1.0)), listOf(JsConcreteValue.number(1.0))), + ), + ) + + val result = search(property, assertNotNull(initial.coverage)) { null } + + val trace = assertNotNull(result.edgeGuidance) + assertEquals(1, initial.numSkips) + assertEquals(setOf(false), trace.initialCovered.map { it.outcome }.toSet()) + assertEquals(setOf(true), trace.initialGoals.map { it.outcome }.toSet()) + assertEquals(trace.initialGoals, trace.remainingGoals) + assertTrue( + trace.replays.isEmpty(), + "The source edge reached only in the guard cannot confirm predicate coverage", + ) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, result.status) + } + + @Test + fun `caller deadline stops before any symbolic queue selection`() { + val property = property() + val manifest = property.toManifest() + val mapping = PropertyEtsMapper(scene = scene, sourceRoots = roots).map(manifest) + val searcher = UsvmPropertySearcher( + scene = scene, + machineOptions = UMachineOptions(solverType = SolverType.YICES, timeout = 10.seconds), + deadlineReached = { true }, + ) + + val result = searcher.search( + manifest = manifest, + mapping = mapping, + concreteCapability = ProjectionCapability(level = ProjectionLevel.EXACT), + coverageFeedback = UsvmCoverageFeedback(initialCoverage = null) { error("Deadline forbids replay") }, + ) + + assertEquals(UsvmPropertySearchStatus.TIMEOUT, result.status) + assertTrue(assertNotNull(result.edgeGuidance).selections.isEmpty()) + } + + private fun search( + property: PropertyDefinition, + coverage: PropertyCoverageArtifact?, + replay: (List) -> PropertyRunResult?, + ): UsvmPropertySearchResult { + val manifest = property.toManifest() + val mapping = PropertyEtsMapper(scene = scene, sourceRoots = roots).map(manifest, coverage) + return UsvmPropertySearcher( + scene = scene, + machineOptions = UMachineOptions(solverType = SolverType.YICES, timeout = 10.seconds), + ).search( + manifest = manifest, + mapping = mapping, + concreteCapability = ProjectionCapability(level = ProjectionLevel.EXACT), + coverageFeedback = UsvmCoverageFeedback(initialCoverage = coverage, replay = replay), + ) + } + + private fun property() = PropertyDefinition( + id = PropertyId("edge-feedback.contract"), + inputs = listOf(PropertyInput(name = "value", domain = IntegerDomain(min = -100, max = 100))), + predicate = TypeScriptEntryPoint(module = "guidance-property.ts", exportName = "valid"), + ) + + private companion object { + val roots = listOf(testResourcesRoot().resolve("properties/dsl")) + val backend = FastCheckBackend(sourceRoots = roots) + val scene: EtsScene by lazy { loadEtsProjectAutoConvert(roots.single(), provider = EtsIrProvider.TS_FRONTEND) } + val configuration = PropertyRunConfiguration( + seed = 42, + numRuns = 1, + timeoutMillis = 10_000, + examples = listOf(listOf(JsConcreteValue.number(1.0))), + coverageRequest = PropertyCoverageRequest(admittedOnly = true), + ) + } +} diff --git a/usvm-ts-fast-check/src/test/resources/properties/dsl/edge-guidance.ts b/usvm-ts-fast-check/src/test/resources/properties/dsl/edge-guidance.ts new file mode 100644 index 0000000000..eedcd3ae9c --- /dev/null +++ b/usvm-ts-fast-check/src/test/resources/properties/dsl/edge-guidance.ts @@ -0,0 +1,35 @@ +export function branch(value: number): number { + if (value > 0) { + return 1; + } else { + return 2; + } +} + +export function nested(value: number): number { + if (value > 0) { + if (value === 42) { + return 3; + } else { + return 1; + } + } else { + return 2; + } +} + +export function withoutElse(value: number): number { + let result = 1; + if (value > 0) { + result = 1; + } + return result; +} + +export function hidden(value: number): number { + if (value > 0) { + return nested(value); + } else { + return 2; + } +} diff --git a/usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts b/usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts new file mode 100644 index 0000000000..aa015a6239 --- /dev/null +++ b/usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts @@ -0,0 +1,14 @@ +import { branch } from './edge-guidance'; + +export function valid(value: number): boolean { + return branch(value) > 0; +} + +export function admission(value: number): boolean { + branch(-value); + return value > 0; +} + +export function guarded(value: number): boolean { + return branch(-value) > 0; +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt index 566ad6afd4..c053c02fd2 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyBasedTestingBackend.kt @@ -29,6 +29,8 @@ data class PropertyRunConfiguration( val examples: List> = emptyList(), val coverageRequest: PropertyCoverageRequest? = null, val observationRequest: PropertyObservationRequest? = null, + /** Execute the single explicit example, without generation after rejection or shrinking. */ + val exactReplay: Boolean = false, ) { init { require(numRuns > 0) { "Number of runs must be positive" } @@ -36,6 +38,9 @@ data class PropertyRunConfiguration( require(timeoutMillis <= MAX_TIMEOUT_MILLIS) { "Timeout exceeds the maximum delay supported by Node timers" } + require(!exactReplay || (numRuns == 1 && examples.size == 1 && replayPath == null)) { + "Exact replay requires one explicit example, one run, and no replay path" + } } companion object { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt index 1b29ef0765..ca0f41346f 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt @@ -119,6 +119,8 @@ data class PropertyCoverageRequest( val includePatterns: List = emptyList(), val excludePatterns: List = emptyList(), val includeUnexecutedSources: Boolean = false, + /** Collect predicate invocations admitted by the precondition, excluding shrinking trials. */ + val admittedOnly: Boolean = false, ) { init { require(scopes.isNotEmpty()) { "At least one coverage scope is required" } @@ -247,8 +249,13 @@ data class PropertyCoverageArtifact( val provenance: CoverageProvenance, val files: List, val diagnostics: List = emptyList(), + val identity: PropertyCoverageIdentity? = null, + val admittedInvocations: Int? = null, ) { init { + require(admittedInvocations == null || admittedInvocations >= 0) { + "Admitted invocation count must not be negative" + } require(backendId.isNotBlank()) { "Coverage backend ID must not be blank" } require(backendVersion.isNotBlank()) { "Coverage backend version must not be blank" } require(files.map(SourceFileCoverage::path).distinct().size == files.size) { @@ -256,3 +263,10 @@ data class PropertyCoverageArtifact( } } } + +/** Exact property and source snapshot used by an admitted-only concrete run. */ +@Serializable +data class PropertyCoverageIdentity( + val manifestSha256: String, + val sourceSha256: Map, +) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt new file mode 100644 index 0000000000..a7945103b4 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt @@ -0,0 +1,37 @@ +package org.usvm.ts.pbt.coverage + +import org.usvm.ts.pbt.backend.PropertyCoverageIdentity +import org.usvm.ts.pbt.manifest.PropertyManifest +import org.usvm.ts.pbt.manifest.PropertyManifestJson +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** Snapshots the declared support/oracle and the local TypeScript files used by the concrete runtime. */ +fun captureCoverageIdentity(manifest: PropertyManifest, sourceRoots: List): PropertyCoverageIdentity { + val sources = sortedMapOf() + for (root in sourceRoots) { + val files = Files.walk(root).use { paths -> paths.filter { isCoverageSource(root, it) }.toList() } + for (path in files) { + sources[path.toRealPath().toString()] = sha256(Files.readAllBytes(path)) + } + } + + val manifestBytes = PropertyManifestJson.encode(manifest).toByteArray(Charsets.UTF_8) + return PropertyCoverageIdentity( + manifestSha256 = sha256(manifestBytes), + sourceSha256 = sources, + ) +} + +private fun isCoverageSource(root: Path, path: Path): Boolean { + val excluded = root.relativize(path).any { it.toString() in EXCLUDED_DIRECTORIES } + val typeScript = path.toString().endsWith(".ts") || path.toString().endsWith(".ets") + return !excluded && typeScript && Files.isRegularFile(path) +} + +private fun sha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256") + .digest(bytes) + .joinToString("") { byte -> "%02x".format(byte) } + +private val EXCLUDED_DIRECTORIES = setOf("node_modules", ".git", ".gradle", "build", "dist") diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmCoverageFeedback.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmCoverageFeedback.kt new file mode 100644 index 0000000000..55fc793b43 --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmCoverageFeedback.kt @@ -0,0 +1,75 @@ +package org.usvm.ts.pbt.usvm + +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.PropertyRunResult +import org.usvm.ts.pbt.model.JsConcreteValue + +/** Concrete feedback for the original property analysis. A replay must execute exactly [inputs]. */ +data class UsvmCoverageFeedback( + val initialCoverage: PropertyCoverageArtifact?, + val replay: (inputs: List) -> PropertyRunResult?, +) + +/** Why a report can or cannot supply exact coverage goals for this property/source snapshot. */ +enum class UsvmCoverageEvidenceStatus { + ABSENT, + STALE, + UNSUPPORTED, + NO_ADMITTED_INPUTS, + EXACT, + PARTIAL, +} + +/** Distinguishes a CFG edge from either of its endpoint statements. */ +data class UsvmCoverageEdge( + val method: String, + val conditionIndex: Int, + val successorIndex: Int, + val outcome: Boolean, +) + +/** A real queue decision made during the original predicate analysis. */ +data class UsvmEdgeSelection( + val evidenceRevision: Int, + val stateId: UInt, + val statement: String, + val pendingStates: Int, + val goal: UsvmCoverageEdge?, + val distance: Int?, + val originalObjectiveTurn: Boolean, + val traversedGoals: Set, +) + +/** Concrete confirmation outcomes; none except CONFIRMED can add edges. */ +enum class UsvmEdgeReplayStatus { + CONFIRMED, + EDGE_MISMATCH, + REJECTED_OR_INCOMPATIBLE, + NOT_RUN, + REPLAY_FAILED, +} + +/** A bounded exact-input native replay and its effect on the shared edge set. */ +data class UsvmEdgeReplay( + val inputs: List, + val status: UsvmEdgeReplayStatus, + val additions: Set, + val remainingGoals: Set, + val elapsedMillis: Long, +) + +/** Bounded history; symbolic reachability never updates [confirmedCovered]. */ +data class UsvmEdgeGuidanceTrace( + val evidenceStatus: UsvmCoverageEvidenceStatus, + val initialCovered: Set, + val initialGoals: Set, + val confirmedCovered: Set, + val remainingGoals: Set, + val attemptedGoals: Set, + val attemptLimitPerGoal: Int, + val selections: List, + val omittedSelections: Int, + val replays: List, + val diagnostics: List, + val elapsedMillis: Long, +) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt new file mode 100644 index 0000000000..9c55cc717a --- /dev/null +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt @@ -0,0 +1,367 @@ +package org.usvm.ts.pbt.usvm + +import org.jacodb.ets.model.EtsMethod +import org.jacodb.ets.model.EtsScene +import org.jacodb.ets.model.EtsStmt +import org.usvm.UPathSelector +import org.usvm.machine.TsConcreteMethodCallStmt +import org.usvm.machine.TsEntryPointGuardResultStmt +import org.usvm.machine.TsGraph +import org.usvm.machine.TsMethodCall +import org.usvm.machine.state.TsState +import org.usvm.ts.pbt.backend.PropertyCoverageArtifact +import org.usvm.ts.pbt.backend.PropertyFailureKind +import org.usvm.ts.pbt.backend.PropertyRunResult +import org.usvm.ts.pbt.backend.PropertyRunStatus +import org.usvm.ts.pbt.coverage.captureCoverageIdentity +import org.usvm.ts.pbt.manifest.PropertyManifest +import org.usvm.ts.pbt.mapping.EtsBranchArmTarget +import org.usvm.ts.pbt.mapping.EtsMappingStatus +import org.usvm.ts.pbt.mapping.PropertyEtsMapper +import org.usvm.ts.pbt.mapping.PropertyEtsMappingArtifact +import org.usvm.ts.pbt.model.JsConcreteValue +import java.nio.file.Path +import java.util.concurrent.CancellationException +import kotlin.time.TimeSource + +/** Owns only exact concrete edge evidence and a deterministic scheduling hint, never path constraints. */ +internal class UsvmEdgeGuidance( + private val scene: EtsScene, + private val manifest: PropertyManifest, + mapping: PropertyEtsMappingArtifact, + private val feedback: UsvmCoverageFeedback, + val selectionSupported: Boolean, +) { + private val roots = mapping.provenance.sourceRoots.map(Path::of) + private val started = TimeSource.Monotonic.markNow() + private val identity = captureCoverageIdentity(manifest, roots) + private val targets = linkedMapOf() + private val covered = linkedSetOf() + private val attempted = linkedSetOf() + private val attemptsByGoal = hashMapOf() + private val replayedInputs = hashSetOf>() + private val selections = mutableListOf() + private val replays = mutableListOf() + private val diagnostics = mutableListOf() + private var revision = 0 + private var omittedSelections = 0 + private val evidenceStatus: UsvmCoverageEvidenceStatus + private val initialCovered: Set + private val initialGoals: Set + + init { + val coverage = feedback.initialCoverage + evidenceStatus = evidenceStatus(coverage) + val usable = evidenceStatus == UsvmCoverageEvidenceStatus.EXACT || + evidenceStatus == UsvmCoverageEvidenceStatus.PARTIAL + if (usable) { + val evidence = exactEdges(requireNotNull(coverage)) + evidence.forEach { (edge, target, hits) -> + targets[edge] = target + if (hits > 0) covered += edge + } + } + initialCovered = covered.toSet() + initialGoals = pendingGoals() + } + + private fun evidenceStatus(coverage: PropertyCoverageArtifact?): UsvmCoverageEvidenceStatus { + if (!selectionSupported) { + diagnostics += "Edge scheduling does not support state merging or selector loop bounds; " + + "original selection retained" + return UsvmCoverageEvidenceStatus.UNSUPPORTED + } + if (coverage == null) return UsvmCoverageEvidenceStatus.ABSENT + if (!coverage.provenance.request.admittedOnly || coverage.identity == null) { + return UsvmCoverageEvidenceStatus.UNSUPPORTED + } + val stale = coverage.identity != identity || coverage.propertyId.value != manifest.propertyId || + coverage.diagnostics.any { it.code == "coverage.source.stale" } + if (stale) return UsvmCoverageEvidenceStatus.STALE + if (coverage.admittedInvocations == null) return UsvmCoverageEvidenceStatus.UNSUPPORTED + if (coverage.admittedInvocations == 0) return UsvmCoverageEvidenceStatus.NO_ADMITTED_INPUTS + + val mapped = mapper().map(manifest, coverage).coverage + val exact = exactEdges(coverage) + val mappingDiagnostics = mapped.diagnostics + mapped.branches.flatMap { branch -> + branch.mapping.diagnostics + branch.arms.flatMap { it.mapping.diagnostics } + } + diagnostics += mappingDiagnostics.map { "${it.code}: ${it.message}" }.distinct() + if (exact.isEmpty()) return UsvmCoverageEvidenceStatus.UNSUPPORTED + + val unknownBranches = mapped.branches.any { it.mapping.status != EtsMappingStatus.EXACT } + return if (unknownBranches || mapped.diagnostics.isNotEmpty()) { + UsvmCoverageEvidenceStatus.PARTIAL + } else { + UsvmCoverageEvidenceStatus.EXACT + } + } + + private fun mapper() = PropertyEtsMapper(scene = scene, sourceRoots = roots) + + private fun exactEdges(coverage: PropertyCoverageArtifact): List { + return mapper().map(manifest, coverage).coverage.branches.flatMap { branch -> + val sourcePath = branch.location?.path + if (sourcePath !in identity.sourceSha256) return@flatMap emptyList() + + branch.arms.flatMap { arm -> + if (arm.mapping.status != EtsMappingStatus.EXACT) return@flatMap emptyList() + + arm.mapping.targets.mapNotNull { target -> + val method = target.condition.location.method + val successors = method.cfg.successors(target.condition) + if (successors.distinct().size != 2) return@mapNotNull null + + val edge = UsvmCoverageEdge( + method = method.signature.toString(), + conditionIndex = method.cfg.stmts.indexOf(target.condition), + successorIndex = method.cfg.stmts.indexOf(target.successor), + outcome = target.outcome, + ) + ObservedEdge(edge = edge, target = target, hits = arm.coverage.hits) + } + } + } + } + + fun selector(initialStates: Map, graph: TsGraph): UPathSelector = + EdgePathSelector(initialStates.values, graph) + + fun confirm(state: TsState, inputs: List, expectsViolation: Boolean) { + val claimed = traversedGoals(state) + if (claimed.isEmpty()) return + + if (!replayedInputs.add(inputs)) return + attempted += claimed + claimed.forEach { goal -> attemptsByGoal[goal] = (attemptsByGoal[goal] ?: 0) + 1 } + + val replayStarted = TimeSource.Monotonic.markNow() + val decision = replay(inputs, claimed, expectsViolation) + replays += UsvmEdgeReplay( + inputs = inputs, + status = decision.status, + additions = decision.additions, + remainingGoals = pendingGoals(), + elapsedMillis = replayStarted.elapsedNow().inWholeMilliseconds, + ) + } + + // Feedback is optional: an arbitrary backend/transport exception must not + // abort the original symbolic property objective or manufacture coverage. + @Suppress("TooGenericExceptionCaught") + private fun replay( + inputs: List, + claimed: Set, + expectsViolation: Boolean, + ): ReplayDecision = try { + val result = feedback.replay(inputs) + if (result == null) { + ReplayDecision(status = UsvmEdgeReplayStatus.NOT_RUN) + } else { + confirmReplay(result, inputs, claimed, expectsViolation) + } + } catch (error: InterruptedException) { + Thread.currentThread().interrupt() + throw error + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + if (Thread.currentThread().isInterrupted) throw error + + diagnostics += "Concrete replay failed: ${error.message}" + ReplayDecision(status = UsvmEdgeReplayStatus.REPLAY_FAILED) + } + + private fun confirmReplay( + result: PropertyRunResult, + inputs: List, + claimed: Set, + expectsViolation: Boolean, + ): ReplayDecision { + val coverage = result.coverage + val compatible = coverage != null && isCompatible(coverage) + val matchesOracle = matchesOracle(result, inputs, expectsViolation) + if (!matchesOracle || !compatible) { + return ReplayDecision(status = UsvmEdgeReplayStatus.REJECTED_OR_INCOMPATIBLE) + } + + val observed = exactEdges(requireNotNull(coverage)) + val confirmed = observed.filter { it.hits > 0 }.mapTo(hashSetOf()) { it.edge } + if (!confirmed.containsAll(claimed)) return ReplayDecision(status = UsvmEdgeReplayStatus.EDGE_MISMATCH) + + observed.forEach { targets.putIfAbsent(it.edge, it.target) } + val additions = confirmed - covered + covered += additions + if (additions.isNotEmpty()) revision += 1 + return ReplayDecision(status = UsvmEdgeReplayStatus.CONFIRMED, additions = additions) + } + + private fun matchesOracle( + result: PropertyRunResult, + inputs: List, + expectsViolation: Boolean, + ): Boolean { + val propertyFailure = result.failure?.kind == PropertyFailureKind.PROPERTY + val completed = result.status == PropertyRunStatus.SUCCESS || + (propertyFailure && result.counterexample == inputs) + val exact = result.numSkips == 0 && result.numRuns == 1 && result.numShrinks == 0 + return exact && completed && propertyFailure == expectsViolation + } + + private fun isCompatible(coverage: PropertyCoverageArtifact): Boolean { + val original = feedback.initialCoverage ?: return false + val sameBackend = coverage.backendId == original.backendId && + coverage.backendVersion == original.backendVersion && coverage.kind == original.kind + val fresh = coverage.identity == identity && captureCoverageIdentity(manifest, roots) == identity && + coverage.diagnostics.none { it.code == "coverage.source.stale" } + val sameRuntime = coverage.provenance == original.provenance && coverage.admittedInvocations == 1 + return sameBackend && fresh && sameRuntime + } + + private fun pendingGoals(): Set = targets.keys - covered + + // Failed attempts remain uncovered. Try at most two distinct concrete inputs + // per edge; property exploration retains half of all scheduling turns. + private fun availableGoals(): Set = pendingGoals().filterTo(linkedSetOf()) { + (attemptsByGoal[it] ?: 0) < MAX_REPLAY_ATTEMPTS_PER_GOAL + } + + private fun traversedGoals(state: TsState): Set { + if (state.entryPointGuardActive) return emptySet() + + val path = state.pathNode.allStatements.takeWhile { it !is TsEntryPointGuardResultStmt } + return availableGoals().filterTo(linkedSetOf()) { edge -> + val target = targets.getValue(edge) + path.zipWithNext().any { (successor, condition) -> + condition == target.condition && successor == target.successor + } + } + } + + fun snapshot() = UsvmEdgeGuidanceTrace( + evidenceStatus = evidenceStatus, + initialCovered = initialCovered, + initialGoals = initialGoals, + confirmedCovered = covered.toSet(), + remainingGoals = pendingGoals(), + attemptedGoals = attempted.toSet(), + attemptLimitPerGoal = MAX_REPLAY_ATTEMPTS_PER_GOAL, + selections = selections.toList(), + omittedSelections = omittedSelections, + replays = replays.toList(), + diagnostics = diagnostics.toList(), + elapsedMillis = started.elapsedNow().inWholeMilliseconds, + ) + + private data class ObservedEdge(val edge: UsvmCoverageEdge, val target: EtsBranchArmTarget, val hits: Long) + private data class GoalDistance(val goal: UsvmCoverageEdge, val distance: Int) + private data class ReplayDecision( + val status: UsvmEdgeReplayStatus, + val additions: Set = emptySet(), + ) + + private inner class EdgePathSelector( + initialStates: Collection, + private val graph: TsGraph, + ) : UPathSelector { + private val states = linkedMapOf() + private var turn = 0L + + init { + add(initialStates) + } + + override fun isEmpty(): Boolean = states.isEmpty() + override fun add(states: Collection) { + states.forEach { state -> this.states[state] = turn } + } + override fun update(state: TsState) = Unit + override fun remove(state: TsState) { + states.remove(state) + } + + override fun peek(): TsState { + val originalTurn = turn % 2L == 1L + val choices = if (originalTurn) { + emptyList() + } else { + states.keys.mapNotNull { state -> nearestGoal(state)?.let { distance -> state to distance } } + } + val priority = compareBy>( + { it.second.distance }, + { states.getValue(it.first) }, + { it.first.id }, + ) + val chosen = choices.minWithOrNull(priority) + val state = chosen?.first ?: states.minBy { it.value }.key + val goal = chosen?.second + states[state] = ++turn + + if (selections.size < MAX_RECORDED_SELECTIONS) { + selections += UsvmEdgeSelection( + evidenceRevision = revision, + stateId = state.id, + statement = state.currentStatement.toString(), + pendingStates = states.size, + goal = goal?.goal, + distance = goal?.distance, + originalObjectiveTurn = originalTurn || chosen == null, + traversedGoals = traversedGoals(state), + ) + } else { + omittedSelections += 1 + } + return state + } + + private fun nearestGoal(state: TsState): GoalDistance? { + val pending = availableGoals() + val crossed = traversedGoals(state).firstOrNull() + if (crossed != null) return GoalDistance(goal = crossed, distance = 0) + if (pending.isEmpty()) return null + + val queue = ArrayDeque>() + val visited = hashSetOf() + val returnSites = hashMapOf>() + state.callStack.forEach { frame -> + val returnSite = frame.returnSite + if (returnSite != null) returnSites.getOrPut(frame.method) { hashSetOf() } += returnSite + } + queue += state.currentStatement to 0 + while (queue.isNotEmpty() && visited.size < MAX_DISTANCE_STATEMENTS) { + val (statement, distance) = queue.removeFirst() + if (!visited.add(statement)) continue + if (statement is TsEntryPointGuardResultStmt) { + queue += statement.entryPoint to distance + 1 + continue + } + + val goal = pending.firstOrNull { targets.getValue(it).condition == statement } + if (goal != null) return GoalDistance(goal = goal, distance = distance + 1) + + graph.successors(statement).forEach { queue += it to distance + 1 } + val callees = when (statement) { + is TsConcreteMethodCallStmt -> sequenceOf(statement.callee) + is TsMethodCall -> graph.callees(statement.returnSite) + else -> graph.callees(statement) + } + callees.forEach { callee -> + returnSites.getOrPut(callee) { hashSetOf() }.addAll(graph.successors(statement)) + graph.entryPoints(callee).forEach { queue += it to distance + 1 } + } + // Returns are an overapproximation for priority only, never a reachability claim. + val method = graph.methodOf(statement) + if (statement in graph.exitPoints(method)) { + returnSites[method].orEmpty().forEach { queue += it to distance + 1 } + } + } + return null + } + } + + private companion object { + const val MAX_RECORDED_SELECTIONS = 4096 + const val MAX_DISTANCE_STATEMENTS = 20_000 + const val MAX_REPLAY_ATTEMPTS_PER_GOAL = 2 + } +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearchModel.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearchModel.kt index 192f8ae6a5..4f88c78aba 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearchModel.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearchModel.kt @@ -32,6 +32,7 @@ data class UsvmPropertySearchResult( val inputs: List?, val capability: UsvmPropertyProjectionCapability, val diagnostics: List, + val edgeGuidance: UsvmEdgeGuidanceTrace? = null, ) { init { when (status) { diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt index 8e2cadda37..a8e38d36af 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt @@ -38,8 +38,13 @@ class UsvmPropertySearcher( ), private val tsOptions: TsOptions = TsOptions(), projectionOptions: UsvmProjectionOptions = UsvmProjectionOptions(), + private val deadlineReached: (() -> Boolean)? = null, ) { - private val machineOptions = machineOptions.copy(stateCollectionStrategy = StateCollectionStrategy.ALL) + private val machineOptions = machineOptions.copy( + stateCollectionStrategy = StateCollectionStrategy.ALL, + // Statement saturation cannot establish the absence of a property violation. + stopOnCoverage = 0, + ) private val projectionOptions = projectionOptions.copy( maxSymbolicCollectionLength = minOf(projectionOptions.maxSymbolicCollectionLength, tsOptions.maxArraySize), maxSymbolicStringLength = minOf(projectionOptions.maxSymbolicStringLength, tsOptions.maxArraySize), @@ -50,6 +55,7 @@ class UsvmPropertySearcher( manifest: PropertyManifest, mapping: PropertyEtsMappingArtifact, concreteCapability: ProjectionCapability, + coverageFeedback: UsvmCoverageFeedback? = null, ): UsvmPropertySearchResult { val capability = capabilityResolver.resolve( manifest = manifest, @@ -77,13 +83,25 @@ class UsvmPropertySearcher( ), ) val precondition = mapping.precondition?.exactTargetOrNull() + val guidedSelectionSupported = !machineOptions.useMerging && !machineOptions.loopIterativeDeepening && + machineOptions.loopIterationLimit == null + val guidance = coverageFeedback?.let { feedback -> + UsvmEdgeGuidance( + scene = scene, + manifest = manifest, + mapping = mapping, + feedback = feedback, + selectionSupported = guidedSelectionSupported, + ) + } return executeSearch( manifest = manifest, predicate = predicate, precondition = precondition, capability = capability, - ) + guidance = guidance, + ).copy(edgeGuidance = guidance?.snapshot()) } private fun executeSearch( @@ -91,10 +109,23 @@ class UsvmPropertySearcher( predicate: EtsEntryPointTarget, precondition: EtsEntryPointTarget?, capability: UsvmPropertyProjectionCapability, + guidance: UsvmEdgeGuidance?, ): UsvmPropertySearchResult { lateinit var projection: UsvmDeclaredDomainProjection val target = UsvmViolationTsTarget() - val observer = UsvmViolationObserver(target) + lateinit var observer: UsvmViolationObserver + observer = UsvmViolationObserver(target) { state -> + if (guidance != null) { + val violation = observer.violationStates.lastOrNull { it.pathNode === state.pathNode } + val replayState = violation ?: state + val inputs = runCatching { + UsvmCandidateInputResolver(projection.objectShapes).resolve( + replayState, manifest.inputs, projection, + ) + }.getOrNull() + if (inputs != null) guidance.confirm(replayState, inputs, expectsViolation = violation != null) + } + } val objectShapes = PlainObjectInputShapes(scene, manifest.inputs.map { it.domain }) val domainProjector = UsvmDomainProjector(projectionOptions, objectShapes) @@ -103,6 +134,8 @@ class UsvmPropertySearcher( options = machineOptions, tsOptions = tsOptions, machineObserver = observer, + deadlineReached = deadlineReached, + pathSelectorFactory = guidance?.takeIf { it.selectionSupported }?.let { it::selector }, ).use { machine -> val analysis = machine.analyzeWithMetadata( methods = listOf(predicate.method), @@ -422,6 +455,7 @@ private class UsvmViolationTsTarget : TsTarget(location = null) private class UsvmViolationObserver( private val target: UsvmViolationTsTarget, + private val onCompletedState: (TsState) -> Unit = {}, ) : UMachineObserver { private val mutableViolationStates = mutableListOf() @@ -461,6 +495,7 @@ private class UsvmViolationObserver( TsMethodResult.NoCall -> Unit } + onCompletedState(state) } private fun recordViolation(state: TsState) { diff --git a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/usvm/UsvmPreconditionSearchTest.kt b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/usvm/UsvmPreconditionSearchTest.kt index 2025445eba..7ae0c2752b 100644 --- a/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/usvm/UsvmPreconditionSearchTest.kt +++ b/usvm-ts-pbt/src/test/kotlin/org/usvm/ts/pbt/usvm/UsvmPreconditionSearchTest.kt @@ -42,7 +42,8 @@ class UsvmPreconditionSearchTest { val unsupported = search(manifest = errorPrecondition, propertySearcher = searcherWithoutModels) assertEquals(UsvmPropertySearchStatus.PRECONDITION_REJECTED, rejected.status) - assertEquals(UsvmPropertySearchStatus.VIOLATION_REACHED, modeled.status) + assertEquals(UsvmPropertySearchStatus.PROPERTY_ERROR, modeled.status) + assertTrue(modeled.diagnostics.any { it.code == "usvm.precondition.threw" }) assertEquals(UsvmPropertyViolationTarget.PREDICATE_FALSE, modeled.target) assertEquals(listOf(JsConcreteValue.number(0.0)), modeled.inputs) assertEquals(UsvmPropertySearchStatus.UNSUPPORTED, unsupported.status) diff --git a/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt b/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt index 39695c703a..0902d42d98 100644 --- a/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt +++ b/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt @@ -94,6 +94,10 @@ class TsMachine( unknownCallModels: TsUnknownCallModelCatalog? = null, /** Creates an observer with access to the analysis queue, including unsupported terminal states. */ private val machineObserverFactory: ((UPathSelector) -> UMachineObserver)? = null, + /** Replaces scheduling for a caller-owned objective without changing the interpreted program. */ + private val pathSelectorFactory: ((Map, TsGraph) -> UPathSelector)? = null, + /** Caller-owned monotonic deadline, checked before setup and between interpreter steps. */ + private val deadlineReached: (() -> Boolean)? = null, ) : UMachine() { init { val hasCompatibleCoverageScope = tsOptions.classCoverageTargets == null || @@ -213,6 +217,7 @@ class TsMachine( coverageDistanceCalculator = null interpreter.resetStepFailure() failureTrackingUnknownCallObserver.reset() + if (deadlineReached?.invoke() == true) return deadlineAnalysisResult() val (initialStates, unsupportedPaths, infeasibleEntryPoints) = createInitialStates(methods, targets, configureInitialState) @@ -236,7 +241,7 @@ class TsMachine( val timeStatistics = createTimeStatistics() - val pathSelector = createPathSelector( + val pathSelector = pathSelectorFactory?.invoke(initialStates, graph) ?: createPathSelector( initialStates = initialStates, options = options, applicationGraph = graph, @@ -272,7 +277,7 @@ class TsMachine( ) override fun shouldStop(): Boolean { - if (options.timeout <= kotlin.time.Duration.ZERO) { + if (options.timeout <= kotlin.time.Duration.ZERO || deadlineReached?.invoke() == true) { timedOut = true return true } @@ -405,6 +410,11 @@ class TsMachine( infeasibleEntryPoints = infeasibleEntryPoints, ) + private fun deadlineAnalysisResult(): TsMachineAnalysisResult = emptyAnalysisResult(emptySet(), emptyList()).copy( + stopReason = TsAnalysisStopReason.STOPPED, + timedOut = true, + ) + private data class InitialStatesResult( val initialStates: Map, val unsupportedPaths: MutableSet, From e8991fcbc4ac79678da2fb2b6652f905a3c7c6b7 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Thu, 15 Oct 2026 18:43:57 +0300 Subject: [PATCH 12/15] [TS PBT] Fix Java 11 CI and uncalled branch coverage Use Java 11-compatible stream collection and restore the public npm artifact URL. Select the innermost V8 function before requiring block coverage, and retain separate regressions for supported nonterminal branches and unsupported logical conditions. --- .../fast-check-adapter/package-lock.json | 2 +- .../src/coverage-branches.ts | 6 +++-- .../test/coverage-branches.test.ts | 23 ++++++++++++++++ .../ts/pbt/fastcheck/FastCheckCoverageTest.kt | 26 +++++++++++++++++-- .../properties/coverage/CoverageProperties.ts | 6 +++++ .../coverage/logical-source-under-test.ts | 7 +++++ .../pbt/coverage/PropertyCoverageIdentity.kt | 5 +++- 7 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 usvm-ts-fast-check/src/test/resources/properties/coverage/logical-source-under-test.ts diff --git a/usvm-ts-fast-check/fast-check-adapter/package-lock.json b/usvm-ts-fast-check/fast-check-adapter/package-lock.json index 3b7a99bb51..146c4e6c08 100644 --- a/usvm-ts-fast-check/fast-check-adapter/package-lock.json +++ b/usvm-ts-fast-check/fast-check-adapter/package-lock.json @@ -24,7 +24,7 @@ }, "node_modules/@bcoe/v8-coverage": { "version": "1.0.2", - "resolved": "https://nexus.vkteam.ru/repository/npm-remote/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", "license": "MIT", "engines": { diff --git a/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts b/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts index 2868ba7f13..936da4f7a6 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/coverage-branches.ts @@ -426,13 +426,15 @@ function countAt( const offset = lineLengths.slice(0, generated.line - 1).reduce((sum, length) => sum + length + 1, 0) + generated.column; - const functions = coverage.script.functions.filter(fn => fn.isBlockCoverage && + const functions = coverage.script.functions.filter(fn => fn.ranges.length > 0 && contains(fn.ranges[0] as V8Range, offset)); const orderedFunctions = functions.sort((left, right) => rangeLength(left.ranges[0] as V8Range) - rangeLength(right.ranges[0] as V8Range), ); const functionCoverage = orderedFunctions[0]; - if (functionCoverage === undefined) return undefined; + // A non-invoked inner function has no block counters. Its containing script's + // counters describe initialization, not execution of that function's body. + if (functionCoverage === undefined || !functionCoverage.isBlockCoverage) return undefined; if (orderedFunctions[1] !== undefined && rangeLength(orderedFunctions[1].ranges[0] as V8Range) === rangeLength(functionCoverage.ranges[0] as V8Range)) { return undefined; diff --git a/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts index ab8108ef73..0c0020a047 100644 --- a/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts +++ b/usvm-ts-fast-check/fast-check-adapter/test/coverage-branches.test.ts @@ -333,6 +333,29 @@ for (const value of [-1, 1]) { try { subject(value); } catch {} } } }); +test('an uncalled function cannot inherit exact if counts from its containing script', async () => { + const fixture = await runCoverage(` +function neverCalled(value: number): number { + let result = 0; + if (value > 0) { result = 1; } + return result; +} +function subject(value: number): number { + if (value > 0) { return 1; } else { return 2; } +} +subject(1); +`); + try { + const report = await convertV8IfBranches(fixture.raw, [fixture.root]); + + assert.deepEqual(report.files[0]?.branches.map(branch => branch.arms.map(arm => arm.hits)), [[1, 0]]); + assert.equal(report.files[0]?.branches.some(branch => branch.location.start.line === 4), false); + assert.ok(report.diagnostics.some(entry => entry.code === 'coverage.branch.ambiguous')); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + test('duplicate V8 scripts remain ambiguous', async () => { const fixture = await runCoverage(` function subject(value: number): number { diff --git a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt index 4c4dee032c..7e6c46b9fa 100644 --- a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt +++ b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageTest.kt @@ -160,7 +160,7 @@ class FastCheckCoverageTest { } @Test - fun `unsupported TypeScript branch retains exact statement coverage`() { + fun `nonterminating if without else retains exact branch counts`() { val result = backend.run( property = property( exportName = "coversNonterminal", @@ -174,9 +174,31 @@ class FastCheckCoverageTest { candidate.path.endsWith("properties/coverage/nonterminal-source-under-test.ts") } + assertTrue(file.statements.any { statement -> statement.hits > 0 }) + assertEquals(listOf(1L, 0L), exactIf(file).arms.map { arm -> arm.hits }) + } + + @Test + fun `unsupported TypeScript branch retains exact statement coverage`() { + val result = backend.run( + property = property( + exportName = "coversLogical", + domain = IntegerDomain(min = 1, max = 1), + ), + configuration = configuration, + ) + + val coverage = assertNotNull(result.coverage) + val file = coverage.files.single { candidate -> + candidate.path.endsWith("properties/coverage/logical-source-under-test.ts") + } + assertTrue(file.statements.any { statement -> statement.hits > 0 }) assertTrue(file.branches.none { branch -> branch.type == "if" }) - assertTrue(coverage.diagnostics.any { diagnostic -> diagnostic.code == "coverage.branch.unsupported" }) + val unsupportedBranch = coverage.diagnostics.any { diagnostic -> + diagnostic.code == "coverage.branch.unsupported" && diagnostic.path == file.path + } + assertTrue(unsupportedBranch) } @Test diff --git a/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts b/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts index 653db26a2d..c32bef35d3 100644 --- a/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts +++ b/usvm-ts-fast-check/src/test/resources/properties/coverage/CoverageProperties.ts @@ -1,6 +1,7 @@ import { classify } from './source-under-test.ts'; import { classifyNested } from './nested-source-under-test.ts'; import { nonterminal } from './nonterminal-source-under-test.ts'; +import { logical } from './logical-source-under-test.ts'; export function coversPositive(value: number): boolean { return classify(value) === 'positive'; @@ -29,3 +30,8 @@ export function coversNonterminal(value: number): boolean { nonterminal(value); return true; } + +export function coversLogical(value: number): boolean { + logical(value); + return true; +} diff --git a/usvm-ts-fast-check/src/test/resources/properties/coverage/logical-source-under-test.ts b/usvm-ts-fast-check/src/test/resources/properties/coverage/logical-source-under-test.ts new file mode 100644 index 0000000000..99e65e4225 --- /dev/null +++ b/usvm-ts-fast-check/src/test/resources/properties/coverage/logical-source-under-test.ts @@ -0,0 +1,7 @@ +export function logical(value: number): number { + let result = 0; + if (value > 0 && value < 10) { + result += 1; + } + return result; +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt index a7945103b4..2ccf7730eb 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/coverage/PropertyCoverageIdentity.kt @@ -6,12 +6,15 @@ import org.usvm.ts.pbt.manifest.PropertyManifestJson import java.nio.file.Files import java.nio.file.Path import java.security.MessageDigest +import java.util.stream.Collectors /** Snapshots the declared support/oracle and the local TypeScript files used by the concrete runtime. */ fun captureCoverageIdentity(manifest: PropertyManifest, sourceRoots: List): PropertyCoverageIdentity { val sources = sortedMapOf() for (root in sourceRoots) { - val files = Files.walk(root).use { paths -> paths.filter { isCoverageSource(root, it) }.toList() } + val files = Files.walk(root).use { paths -> + paths.filter { isCoverageSource(root, it) }.collect(Collectors.toList()) + } for (path in files) { sources[path.toRealPath().toString()] = sha256(Files.readAllBytes(path)) } From 4c241cd66b40d34c94be143794a13d7fab7061c8 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Thu, 15 Oct 2026 19:07:42 +0300 Subject: [PATCH 13/15] Handle exhausted DSL preparation budgets Stop before mapping or constructing a symbolic search after preparation exhausts the shared deadline. Keep functional DSL checks on the public default budget, add stage diagnostics, and verify deadline expiration remains inconclusive. --- .../org/usvm/ts/pbt/dsl/TsPropertyTest.kt | 6 ++- .../usvm/ts/pbt/dsl/KotlinPropertyDslTest.kt | 38 +++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt index 1daae02579..dd9fd60826 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt @@ -208,9 +208,13 @@ class DslPropertyRunner(sourceRoots: List) { if (remainingMillis() <= 0L) return null val scene = loadEtsProjectAutoConvert(directory, provider = EtsIrProvider.TS_FRONTEND) + if (remainingMillis() <= 0L) return null + val manifest = compiled.definition.toManifest() val mapping = PropertyEtsMapper(scene = scene, sourceRoots = sourceRoots).map(manifest, concrete?.coverage) - val remainingBudgetMillis = remainingMillis().coerceAtLeast(0L) + val remainingBudgetMillis = remainingMillis() + if (remainingBudgetMillis <= 0L) return null + val finalReplayReserveMillis = remainingBudgetMillis / FINAL_REPLAY_BUDGET_DIVISOR val searchBudgetMillis = remainingBudgetMillis - finalReplayReserveMillis fun symbolicRemainingMillis(): Long = remainingMillis() - finalReplayReserveMillis diff --git a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/KotlinPropertyDslTest.kt b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/KotlinPropertyDslTest.kt index efa49c7a0b..dea8903b7a 100644 --- a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/KotlinPropertyDslTest.kt +++ b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/dsl/KotlinPropertyDslTest.kt @@ -10,12 +10,15 @@ import org.usvm.ts.pbt.usvm.UsvmPropertySearchStatus import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertNotNull +import kotlin.test.assertNull import kotlin.test.assertTrue class KotlinPropertyDslTest { private val roots = listOf(testResourcesRoot().resolve("properties/dsl")) private val runner = DslPropertyRunner(sourceRoots = roots) - private val configuration = PropertyRunConfiguration(seed = 42, numRuns = 50, timeoutMillis = 10_000) + + // Functional assertions include cold frontend initialization in the runner's shared timeout. + private val configuration = PropertyRunConfiguration(seed = 42, numRuns = 50) @Test fun `calculator runs through existing concrete and symbolic execution`() { @@ -197,12 +200,39 @@ class KotlinPropertyDslTest { val run = runner.run(program = program, engine = PropertyEngine.BOTH, configuration = configuration) - assertEquals("unexpected", run.concrete?.failure?.checkId) - assertEquals("Error", run.concrete?.failure?.errorName) - assertEquals("unexpected", run.symbolicReplay?.failure?.checkId) + val diagnostic = run.executionDiagnostic() + assertEquals("unexpected", run.concrete?.failure?.checkId, diagnostic) + assertEquals("Error", run.concrete?.failure?.errorName, diagnostic) + val replay = assertNotNull(run.symbolicReplay, diagnostic) + assertEquals("unexpected", replay.failure?.checkId, diagnostic) assertFailsWith { run.requireSuccess() } } + @Test + fun `expired total deadline skips symbolic analysis and cannot report success`() { + val program = addition(exportName = "add") + + val run = runner.run( + program = program, + engine = PropertyEngine.USVM, + configuration = configuration.copy(timeoutMillis = 1), + ) + + val diagnostic = run.executionDiagnostic() + assertTrue(run.symbolicSkippedDueToDeadline, diagnostic) + assertNull(run.concrete, diagnostic) + assertNull(run.symbolic, diagnostic) + assertNull(run.symbolicReplay, diagnostic) + assertTrue(run.elapsedMillis >= 1, diagnostic) + assertFailsWith { run.requireSuccess() } + } + + private fun DslPropertyRun.executionDiagnostic(): String = + "elapsedMillis=$elapsedMillis, budgetMillis=$budgetMillis, " + + "symbolicSkippedDueToDeadline=$symbolicSkippedDueToDeadline, symbolicStatus=${symbolic?.status}, " + + "concreteStatus=${concrete?.status}, replayStatus=${symbolicReplay?.status}, " + + "replayFailure=${symbolicReplay?.failure}" + private fun addition(exportName: String): DslProperty = defineProperty(id = "calculator.$exportName") { val sum = function<(Double, Double) -> Double>(module = "calculator.ts", exportName = exportName) val a by int(min = -100, max = 100) From 47b6c27c48ee2dc780cb20a376e57fa661d56f24 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 10:51:10 +0300 Subject: [PATCH 14/15] [TS PBT] Reuse the core coverage-guided selector Delegate directed state priorities and external coverage updates to CoverageGuidedPathSelector. Keep TS edge distances, per-step alternation with BFS, native confirmation, and bounded replay attempts in the PBT consumer. --- .../fastcheck/UsvmEdgeFeedbackContractTest.kt | 23 ++++++ .../properties/dsl/guidance-property.ts | 6 ++ .../org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt | 80 +++++++++++++------ 3 files changed, 86 insertions(+), 23 deletions(-) diff --git a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt index ce3e526c4e..e2c211bd2e 100644 --- a/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt +++ b/usvm-ts-fast-check/src/test/kotlin/org/usvm/ts/pbt/fastcheck/UsvmEdgeFeedbackContractTest.kt @@ -124,6 +124,29 @@ class UsvmEdgeFeedbackContractTest { } } + @Test + fun `exhausted replay attempts stop guiding without confirming coverage or dropping paths`() { + val property = property().copy( + precondition = TypeScriptEntryPoint(module = "guidance-property.ts", exportName = "retryAdmission"), + ) + val initial = backend.run(property, configuration) + + val result = search(property, initial.coverage) { null } + + val trace = assertNotNull(result.edgeGuidance) + assertEquals(1, trace.initialGoals.size) + assertEquals(trace.attemptLimitPerGoal, trace.replays.size) + assertEquals(trace.initialGoals, trace.remainingGoals) + assertEquals(trace.initialCovered, trace.confirmedCovered) + assertTrue(trace.replays.all { it.status == UsvmEdgeReplayStatus.NOT_RUN }, trace.toString()) + assertEquals(trace.replays.size, trace.replays.map { it.inputs }.distinct().size) + assertEquals(null, trace.selections.last().goal) + val originalTurns = trace.selections.filterIndexed { index, _ -> index % 2 == 1 } + assertTrue(originalTurns.isNotEmpty()) + assertTrue(originalTurns.all { it.originalObjectiveTurn }, trace.toString()) + assertEquals(UsvmPropertySearchStatus.NO_VIOLATION_REACHED, result.status) + } + @Test fun `rejected precondition cannot remove a source edge from the real coverage target queue`() { val property = property().copy( diff --git a/usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts b/usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts index aa015a6239..e512753868 100644 --- a/usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts +++ b/usvm-ts-fast-check/src/test/resources/properties/dsl/guidance-property.ts @@ -12,3 +12,9 @@ export function admission(value: number): boolean { export function guarded(value: number): boolean { return branch(-value) > 0; } + +export function retryAdmission(value: number): boolean { + if (value === -1) return true; + if (value === -2) return true; + return true; +} diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt index 9c55cc717a..2cc5a46522 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt @@ -9,6 +9,8 @@ import org.usvm.machine.TsEntryPointGuardResultStmt import org.usvm.machine.TsGraph import org.usvm.machine.TsMethodCall import org.usvm.machine.state.TsState +import org.usvm.ps.BfsPathSelector +import org.usvm.ps.CoverageGuidedPathSelector import org.usvm.ts.pbt.backend.PropertyCoverageArtifact import org.usvm.ts.pbt.backend.PropertyFailureKind import org.usvm.ts.pbt.backend.PropertyRunResult @@ -48,6 +50,7 @@ internal class UsvmEdgeGuidance( private val evidenceStatus: UsvmCoverageEvidenceStatus private val initialCovered: Set private val initialGoals: Set + private var activeSelector: EdgePathSelector? = null init { val coverage = feedback.initialCoverage @@ -124,8 +127,11 @@ internal class UsvmEdgeGuidance( } } - fun selector(initialStates: Map, graph: TsGraph): UPathSelector = - EdgePathSelector(initialStates.values, graph) + fun selector(initialStates: Map, graph: TsGraph): UPathSelector { + val selector = EdgePathSelector(initialStates.values, graph) + activeSelector = selector + return selector + } fun confirm(state: TsState, inputs: List, expectsViolation: Boolean) { val claimed = traversedGoals(state) @@ -137,6 +143,12 @@ internal class UsvmEdgeGuidance( val replayStarted = TimeSource.Monotonic.markNow() val decision = replay(inputs, claimed, expectsViolation) + activeSelector?.updateCoverage() + val exhaustedRetry = claimed.any { attemptsByGoal.getValue(it) == MAX_REPLAY_ATTEMPTS_PER_GOAL } + if (exhaustedRetry && decision.status != UsvmEdgeReplayStatus.CONFIRMED) { + activeSelector?.refreshPriorities() + } + replays += UsvmEdgeReplay( inputs = inputs, status = decision.status, @@ -264,38 +276,57 @@ internal class UsvmEdgeGuidance( initialStates: Collection, private val graph: TsGraph, ) : UPathSelector { - private val states = linkedMapOf() + private val states = linkedSetOf() + private val original = BfsPathSelector() + private val coverage = CoverageGuidedPathSelector( + targets = targets.keys, + distance = { state, goal -> nearestGoal(state, setOf(goal))?.distance?.toUInt() ?: UInt.MAX_VALUE }, + ) private var turn = 0L init { + updateCoverage() add(initialStates) } override fun isEmpty(): Boolean = states.isEmpty() override fun add(states: Collection) { - states.forEach { state -> this.states[state] = turn } + this.states.addAll(states) + original.add(states) + coverage.add(states) } - override fun update(state: TsState) = Unit + override fun update(state: TsState) = coverage.update(state) override fun remove(state: TsState) { states.remove(state) + original.remove(state) + coverage.remove(state) + } + + override fun drain(): Sequence = sequence { + for (state in coverage.drain()) { + states.remove(state) + original.remove(state) + yield(state) + } + } + + fun updateCoverage() { + coverage.addTargets(targets.keys) + coverage.markCovered(covered) + } + + fun refreshPriorities() { + states.forEach(coverage::update) } override fun peek(): TsState { val originalTurn = turn % 2L == 1L - val choices = if (originalTurn) { - emptyList() - } else { - states.keys.mapNotNull { state -> nearestGoal(state)?.let { distance -> state to distance } } - } - val priority = compareBy>( - { it.second.distance }, - { states.getValue(it.first) }, - { it.first.id }, - ) - val chosen = choices.minWithOrNull(priority) - val state = chosen?.first ?: states.minBy { it.value }.key - val goal = chosen?.second - states[state] = ++turn + val guidedState = if (originalTurn) null else coverage.peek() + val goal = guidedState?.let { nearestGoal(it) } + val state = if (goal == null) original.peek() else requireNotNull(guidedState) + original.remove(state) + original.add(listOf(state)) + turn += 1 if (selections.size < MAX_RECORDED_SELECTIONS) { selections += UsvmEdgeSelection( @@ -305,7 +336,7 @@ internal class UsvmEdgeGuidance( pendingStates = states.size, goal = goal?.goal, distance = goal?.distance, - originalObjectiveTurn = originalTurn || chosen == null, + originalObjectiveTurn = originalTurn || goal == null, traversedGoals = traversedGoals(state), ) } else { @@ -314,9 +345,12 @@ internal class UsvmEdgeGuidance( return state } - private fun nearestGoal(state: TsState): GoalDistance? { - val pending = availableGoals() - val crossed = traversedGoals(state).firstOrNull() + private fun nearestGoal( + state: TsState, + candidates: Set = availableGoals(), + ): GoalDistance? { + val pending = availableGoals().intersect(candidates) + val crossed = traversedGoals(state).firstOrNull { it in pending } if (crossed != null) return GoalDistance(goal = crossed, distance = 0) if (pending.isEmpty()) return null From 024804408f2873130b810c24c25911a72aeedca8 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 19:27:08 +0300 Subject: [PATCH 15/15] Use lazy NURS weights for concrete edge guidance --- .../org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt | 41 +++++-------------- .../usvm/ts/pbt/usvm/UsvmPropertySearcher.kt | 2 + 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt index 2cc5a46522..9e12fb632c 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmEdgeGuidance.kt @@ -10,7 +10,7 @@ import org.usvm.machine.TsGraph import org.usvm.machine.TsMethodCall import org.usvm.machine.state.TsState import org.usvm.ps.BfsPathSelector -import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.createClosestToUncoveredRandomPathSelector import org.usvm.ts.pbt.backend.PropertyCoverageArtifact import org.usvm.ts.pbt.backend.PropertyFailureKind import org.usvm.ts.pbt.backend.PropertyRunResult @@ -24,15 +24,17 @@ import org.usvm.ts.pbt.mapping.PropertyEtsMappingArtifact import org.usvm.ts.pbt.model.JsConcreteValue import java.nio.file.Path import java.util.concurrent.CancellationException +import kotlin.random.Random import kotlin.time.TimeSource -/** Owns only exact concrete edge evidence and a deterministic scheduling hint, never path constraints. */ +/** Owns only exact concrete edge evidence and a seeded NURS scheduling hint, never path constraints. */ internal class UsvmEdgeGuidance( private val scene: EtsScene, private val manifest: PropertyManifest, mapping: PropertyEtsMappingArtifact, private val feedback: UsvmCoverageFeedback, val selectionSupported: Boolean, + private val random: Random, ) { private val roots = mapping.provenance.sourceRoots.map(Path::of) private val started = TimeSource.Monotonic.markNow() @@ -50,7 +52,6 @@ internal class UsvmEdgeGuidance( private val evidenceStatus: UsvmCoverageEvidenceStatus private val initialCovered: Set private val initialGoals: Set - private var activeSelector: EdgePathSelector? = null init { val coverage = feedback.initialCoverage @@ -128,9 +129,7 @@ internal class UsvmEdgeGuidance( } fun selector(initialStates: Map, graph: TsGraph): UPathSelector { - val selector = EdgePathSelector(initialStates.values, graph) - activeSelector = selector - return selector + return EdgePathSelector(initialStates.values, graph) } fun confirm(state: TsState, inputs: List, expectsViolation: Boolean) { @@ -143,11 +142,6 @@ internal class UsvmEdgeGuidance( val replayStarted = TimeSource.Monotonic.markNow() val decision = replay(inputs, claimed, expectsViolation) - activeSelector?.updateCoverage() - val exhaustedRetry = claimed.any { attemptsByGoal.getValue(it) == MAX_REPLAY_ATTEMPTS_PER_GOAL } - if (exhaustedRetry && decision.status != UsvmEdgeReplayStatus.CONFIRMED) { - activeSelector?.refreshPriorities() - } replays += UsvmEdgeReplay( inputs = inputs, @@ -276,16 +270,15 @@ internal class UsvmEdgeGuidance( initialStates: Collection, private val graph: TsGraph, ) : UPathSelector { - private val states = linkedSetOf() + private val states = hashSetOf() private val original = BfsPathSelector() - private val coverage = CoverageGuidedPathSelector( - targets = targets.keys, - distance = { state, goal -> nearestGoal(state, setOf(goal))?.distance?.toUInt() ?: UInt.MAX_VALUE }, + private val coverage = createClosestToUncoveredRandomPathSelector( + distance = { state -> nearestGoal(state)?.distance?.toUInt() ?: UInt.MAX_VALUE }, + random = random, ) private var turn = 0L init { - updateCoverage() add(initialStates) } @@ -310,15 +303,6 @@ internal class UsvmEdgeGuidance( } } - fun updateCoverage() { - coverage.addTargets(targets.keys) - coverage.markCovered(covered) - } - - fun refreshPriorities() { - states.forEach(coverage::update) - } - override fun peek(): TsState { val originalTurn = turn % 2L == 1L val guidedState = if (originalTurn) null else coverage.peek() @@ -345,11 +329,8 @@ internal class UsvmEdgeGuidance( return state } - private fun nearestGoal( - state: TsState, - candidates: Set = availableGoals(), - ): GoalDistance? { - val pending = availableGoals().intersect(candidates) + private fun nearestGoal(state: TsState): GoalDistance? { + val pending = availableGoals() val crossed = traversedGoals(state).firstOrNull { it in pending } if (crossed != null) return GoalDistance(goal = crossed, distance = 0) if (pending.isEmpty()) return null diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt index a8e38d36af..9d89581a83 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt @@ -28,6 +28,7 @@ import org.usvm.ts.pbt.manifest.PropertyManifest import org.usvm.ts.pbt.mapping.EtsEntryPointTarget import org.usvm.ts.pbt.mapping.PropertyEtsMappingArtifact import org.usvm.ts.pbt.model.PropertyId +import kotlin.random.Random /** Searches mapped TypeScript predicates for concrete property violations with USVM. */ class UsvmPropertySearcher( @@ -92,6 +93,7 @@ class UsvmPropertySearcher( mapping = mapping, feedback = feedback, selectionSupported = guidedSelectionSupported, + random = Random(seed = machineOptions.randomSeed), ) }