From 825856f7604187f88691b60b80f8b0f63fb0a418 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Thu, 15 Oct 2026 23:39:40 +0300 Subject: [PATCH 01/10] [Core] Accept externally supplied coverage goals Reuse closest-to-uncovered and weighted selectors with revisioned goals supplied by callers. Refresh pending priorities after coverage changes while retaining queued states and existing reporting semantics. --- usvm-core/README.md | 59 ++++ .../org/usvm/ps/CoverageGuidedPathSelector.kt | 30 ++ .../kotlin/org/usvm/ps/PathSelectorFactory.kt | 76 +++-- .../org/usvm/ps/WeightedPathSelector.kt | 27 +- .../org/usvm/statistics/CoverageStatistics.kt | 10 +- .../org/usvm/statistics/CoverageTargets.kt | 43 +++ .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 278 ++++++++++++++++++ .../statistics/ExternalCoverageTargetsTest.kt | 47 +++ 8 files changed, 540 insertions(+), 30 deletions(-) create mode 100644 usvm-core/README.md create mode 100644 usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt create mode 100644 usvm-core/src/main/kotlin/org/usvm/statistics/CoverageTargets.kt create mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt create mode 100644 usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt diff --git a/usvm-core/README.md b/usvm-core/README.md new file mode 100644 index 0000000000..853a55b309 --- /dev/null +++ b/usvm-core/README.md @@ -0,0 +1,59 @@ +# External coverage goals + +`CoverageTargets` supplies the currently uncovered goals and a revision that changes whenever that set changes. +The normal `CoverageStatistics` implements this contract for statements. `ExternalCoverageTargets` lets a caller +register goals and report confirmed coverage between analysis steps. + +The standard factory accepts this source independently of the statistics used for reporting and stopping: + +```kotlin +val coverage = ExternalCoverageTargets(targets = knownStatements) +coverage.markCovered(targets = statementsCoveredByConcreteTests) + +val selector = createPathSelector( + initialState = initialState, + options = options.copy(pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED)), + applicationGraph = graph, + cfgStatisticsFactory = { cfgStatistics }, + coverageTargetsFactory = { coverage }, +) + +// The analysis can continue with the same queue after the caller confirms more coverage. +coverage.markCovered(targets = newlyConfirmedStatements) +``` + +Both `CLOSEST_TO_UNCOVERED` variants use cached CFG distances in the active call stack. Existing entry-point fairness, +exception propagation, loop bounds and selector composition remain controlled by `UMachineOptions`. +External goals do not change reported coverage percentages or coverage-based stop conditions. + +For targets that are not statements, use the same weighted queue with a caller-supplied distance: + +```kotlin +data class Edge(val from: Statement, val to: Statement) + +val coverage = ExternalCoverageTargets(targets = knownEdges) +coverage.markCovered(targets = edgesCoveredByConcreteTests) + +val selector = createCoverageGuidedPathSelector( + coverageTargets = coverage, + distance = ::distanceFromStateToEdge, +) +``` + +The distance function owns the meaning of each target and returns `UInt.MAX_VALUE` when it is unreachable. +An edge must retain both endpoints: reaching its successor through a different predecessor does not cover that edge. +The core does not turn an edge into a statement or infer coverage from the distance function. +Language-specific code can reuse the existing CFG and interprocedural distance calculators in this function. + +Updates are applied on the analysis thread, between machine steps. The next `peek()` refreshes all queued weights +after a revision change; repeated peeks with unchanged evidence reuse the weights. No listener keeps the selector +attached to the external source after the analysis ends. Draining stopped work does not calculate new distances. + +`markCovered` ignores unknown goals, and repeated coverage reports do not change the revision. Use `addTargets` +to register newly identified goals; re-registering a covered goal does not make it uncovered again. Missing records +are not automatically registered as uncovered goals. + +An empty goal set retains every queued state. Covered prefixes remain traversable, and coverage alone does not +establish a property on other inputs. Property search must keep its own stop conditions and exploration policy. +Runtime admission, source identity and concrete replay confirmation belong to the caller; this core API accepts +its coverage decisions without interpreting them. diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt new file mode 100644 index 0000000000..2b266d7bb7 --- /dev/null +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -0,0 +1,30 @@ +package org.usvm.ps + +import org.usvm.UPathSelector +import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.statistics.CoverageTargets + +/** + * Selects the closest uncovered goal without pruning states. Goals may be statements, edges or other caller types. + * [distance] owns their semantics; [UInt.MAX_VALUE] means unreachable. Updates become visible on the next peek. + */ +fun createCoverageGuidedPathSelector( + coverageTargets: CoverageTargets, + distance: (State, Target) -> UInt, +): UPathSelector { + var revision = coverageTargets.revision + var targets = coverageTargets.uncoveredTargets.toList() + + return WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, + weighter = { state -> + if (revision != coverageTargets.revision) { + targets = coverageTargets.uncoveredTargets.toList() + revision = coverageTargets.revision + } + + targets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE + }, + weightRevision = { coverageTargets.revision }, + ) +} diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt index 9a3d0b4c94..e7214bb419 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt @@ -13,6 +13,7 @@ import org.usvm.merging.CloseStatesSearcherImpl import org.usvm.merging.MergingPathSelector import org.usvm.statistics.ApplicationGraph import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.CoverageTargets import org.usvm.statistics.TimeStatistics import org.usvm.statistics.distances.CallGraphStatistics import org.usvm.statistics.distances.CallStackDistanceCalculator @@ -36,6 +37,7 @@ private fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { @@ -66,18 +68,24 @@ private fun createPathSelector( PathSelectionStrategy.FORK_DEPTH -> createForkDepthPathSelector() PathSelectionStrategy.FORK_DEPTH_RANDOM -> createForkDepthPathSelector(random) - PathSelectionStrategy.CLOSEST_TO_UNCOVERED -> createClosestToUncoveredPathSelector( - requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, - requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, - applicationGraph - ) + PathSelectionStrategy.CLOSEST_TO_UNCOVERED, + PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM, + -> { + val coverageTargets = requireNotNull(coverageTargetsFactory()) { + "Coverage targets are required for closest to uncovered path selector" + } + val cfgStatistics = requireNotNull(cfgStatisticsFactory()) { + "CFG statistics is required for closest to uncovered path selector" + } + val selectorRandom = if (strategy == PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM) random else null - PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM -> createClosestToUncoveredPathSelector( - requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, - requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, - applicationGraph, - random - ) + createClosestToUncoveredPathSelector( + coverageTargets = coverageTargets, + cfgStatistics = cfgStatistics, + applicationGraph = applicationGraph, + random = selectorRandom, + ) + } PathSelectionStrategy.TARGETED -> createTargetedPathSelector( requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for targeted path selector" }, @@ -145,6 +153,7 @@ private fun createPathSelector( return selector } +/** [coverageTargetsFactory] overrides closest-to-uncovered goals, leaving reported coverage unchanged. */ fun createPathSelector( initialState: State, options: UMachineOptions, @@ -153,6 +162,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> = createPathSelector( listOf(initialState), @@ -161,9 +171,11 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageTargetsFactory, ) +/** External goals are shared by the entry points; coverage statistics still own reporting and stop conditions. */ fun createPathSelector( initialStates: Map, options: UMachineOptions, @@ -173,6 +185,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { val hasSingleEntryPoint = initialStates.size == 1 val hasUnlimitedScheduling = options.timeout == Duration.INFINITE && @@ -185,7 +198,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageTargetsFactory, ) } @@ -202,7 +216,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageTargetsFactory, ) val coverageStatistics = coverageStatisticsFactory() @@ -312,29 +327,35 @@ private fun > createDepthPathSelector(rando ) } -private fun > createClosestToUncoveredPathSelector( - coverageStatistics: CoverageStatistics, +/** Uses cached CFG distances within the active call stack and refreshes priorities after goal changes. */ +fun > createClosestToUncoveredPathSelector( + coverageTargets: CoverageTargets, cfgStatistics: CfgStatistics, applicationGraph: ApplicationGraph, random: Random? = null, ): UPathSelector { - val distanceCalculator = CallStackDistanceCalculator( - targets = coverageStatistics.getUncoveredStatements(), + fun createDistanceCalculator() = CallStackDistanceCalculator( + targets = coverageTargets.uncoveredTargets, cfgStatistics = cfgStatistics, - applicationGraph + applicationGraph = applicationGraph, ) + var distanceCalculator = createDistanceCalculator() + var revision = coverageTargets.revision - coverageStatistics.addOnCoveredObserver { _, method, statement -> - distanceCalculator.removeTarget( - method, - statement - ) + fun calculateDistance(state: State): UInt { + if (revision != coverageTargets.revision) { + distanceCalculator = createDistanceCalculator() + revision = coverageTargets.revision + } + + return distanceCalculator.calculateDistance(state.currentStatement, state.callStack) } if (random == null) { return WeightedPathSelector( priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = { distanceCalculator.calculateDistance(it.currentStatement, it.callStack) } + weighter = ::calculateDistance, + weightRevision = { coverageTargets.revision }, ) } @@ -342,10 +363,11 @@ private fun ( priorityCollectionFactory: () -> UPriorityCollection, - private val weighter: StateWeighter + private val weighter: StateWeighter, + private val weightRevision: (() -> Long)? = null, ) : UPathSelector { private val priorityCollection = priorityCollectionFactory() + private var currentWeightRevision = weightRevision?.invoke() override fun isEmpty(): Boolean = priorityCollection.count == 0 - override fun peek(): State = priorityCollection.peek() + override fun peek(): State { + val revision = weightRevision?.invoke() + if (revision != currentWeightRevision) { + val states = drainPriorityCollection().toList() + add(states) + currentWeightRevision = revision + } + + return priorityCollection.peek() + } override fun update(state: State) = priorityCollection.update(state, weighter.weight(state)) @@ -30,4 +42,15 @@ open class WeightedPathSelector( } override fun remove(state: State) = priorityCollection.remove(state) + + override fun drain(): Sequence = drainPriorityCollection() + + private fun drainPriorityCollection(): Sequence = sequence { + // Draining stopped work must not recalculate weights or consult external goals. + while (priorityCollection.count > 0) { + val state = priorityCollection.peek() + priorityCollection.remove(state) + yield(state) + } + } } diff --git a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt index df263e6ac1..a4b767d25b 100644 --- a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt +++ b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt @@ -16,7 +16,13 @@ import java.util.concurrent.ConcurrentHashMap class CoverageStatistics>( methods: Set, private val applicationGraph: ApplicationGraph, -) : UMachineObserver { +) : UMachineObserver, CoverageTargets { + + override val uncoveredTargets: Collection + get() = getUncoveredStatements() + + override var revision: Long = 0 + private set private val onStatementCoveredObservers: MutableSet<(State, Method, Statement) -> Unit> = ConcurrentHashMap.newKeySet() @@ -56,6 +62,7 @@ class CoverageStatistics { + val uncoveredTargets: Collection + val revision: Long +} + +/** + * Caller-owned coverage goals, updated on the analysis thread between machine steps. + * Reporting coverage does not remove queued states or establish a property on other inputs. + */ +class ExternalCoverageTargets(targets: Collection) : CoverageTargets { + private val knownTargets = targets.toMutableSet() + private val pendingTargets = targets.toMutableSet() + + override val uncoveredTargets: Set + get() = pendingTargets.toSet() + + override var revision: Long = 0 + private set + + /** Registers newly identified goals; previously covered goals stay covered. */ + fun addTargets(targets: Collection): Boolean { + val additions = targets.filter { knownTargets.add(it) } + if (additions.isEmpty()) return false + + pendingTargets.addAll(additions) + revision++ + return true + } + + /** Accepts caller-confirmed coverage of known goals. Unknown goals are ignored. */ + fun markCovered(targets: Collection): Boolean { + val changed = pendingTargets.removeAll(targets.toSet()) + if (changed) revision++ + + return changed + } +} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt new file mode 100644 index 0000000000..515fc8561e --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt @@ -0,0 +1,278 @@ +package org.usvm.ps + +import io.mockk.every +import org.junit.jupiter.api.Test +import org.usvm.PathNode +import org.usvm.PathSelectionStrategy +import org.usvm.TestInstruction +import org.usvm.TestState +import org.usvm.TestTarget +import org.usvm.UMachineOptions +import org.usvm.UPathSelector +import org.usvm.appGraph +import org.usvm.mockState +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.ExternalCoverageTargets +import org.usvm.statistics.TimeStatistics +import org.usvm.statistics.distances.CfgStatisticsImpl +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class CoverageGuidedPathSelectorTest { + @Test + fun `external coverage reprioritizes every pending state without state updates`() { + val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + val selector = selector(targets) + selector.add(states = listOf("left", "right")) + + targets.markCovered(targets = listOf("left")) + assertEquals(expected = "right", actual = selector.peek()) + + targets.addTargets(targets = listOf("new-left")) + targets.markCovered(targets = listOf("right")) + assertEquals(expected = "left", actual = selector.peek()) + assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) + } + + @Test + fun `stable evidence does not recalculate queued weights`() { + val targets = ExternalCoverageTargets(targets = listOf("goal")) + var evaluations = 0 + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { _, _ -> + evaluations++ + 0u + }, + ) + selector.add(states = listOf("first", "second")) + val afterAdd = evaluations + + selector.peek() + selector.peek() + assertFalse(targets.markCovered(targets = listOf("unknown"))) + selector.peek() + + assertEquals(expected = afterAdd, actual = evaluations) + } + + @Test + fun `saturated coverage retains all queued states and accepts later work`() { + val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + val selector = selector(targets) + selector.add(states = listOf("left", "right")) + targets.markCovered(targets = listOf("left", "right")) + + selector.peek() + val drained = selector.drain().toList() + selector.add(states = listOf("later")) + + assertEquals(expected = setOf("left", "right"), actual = drained.toSet()) + assertEquals(expected = 2, actual = drained.size) + assertEquals(expected = "later", actual = selector.peek()) + } + + @Test + fun `edges sharing a successor remain distinct coverage goals`() { + val firstEdge = Edge(from = 1, to = 3) + val secondEdge = Edge(from = 2, to = 3) + val targets = ExternalCoverageTargets(targets = listOf(firstEdge, secondEdge)) + val firstPath = setOf(firstEdge) + val secondPath = setOf(secondEdge) + val selector = createCoverageGuidedPathSelector, Edge>( + coverageTargets = targets, + distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, + ) + selector.add(states = listOf(firstPath, secondPath)) + + targets.markCovered(targets = listOf(firstEdge)) + + assertSame(expected = secondPath, actual = selector.peek()) + assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) + } + + @Test + fun `empty initial evidence does not invoke the distance function or remove states`() { + val targets = ExternalCoverageTargets(targets = emptyList()) + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { _, _ -> error("There is no known goal") }, + ) + selector.add(states = listOf("first", "second")) + + selector.peek() + + assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) + assertTrue(selector.isEmpty()) + } + + @Test + fun `draining stopped work does not refresh weights after new evidence`() { + val targets = ExternalCoverageTargets(targets = listOf("left")) + var stopped = false + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { _, _ -> + check(!stopped) + 0u + }, + ) + selector.add(states = listOf("first", "second")) + stopped = true + targets.addTargets(targets = listOf("right")) + + val drained = selector.drain().toList() + + assertEquals(expected = setOf("first", "second"), actual = drained.toSet()) + assertTrue(selector.isEmpty()) + } + + @Test + fun `state updates additions and removals work across coverage revisions`() { + val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + val distances = mutableMapOf("first" to 0u, "second" to 2u, "third" to 1u) + val selector = createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { state, _ -> distances.getValue(state) }, + ) + selector.add(states = listOf("first", "second")) + assertEquals(expected = "first", actual = selector.peek()) + + distances["first"] = 3u + selector.update(state = "first") + selector.add(states = listOf("third")) + targets.markCovered(targets = listOf("left")) + + assertEquals(expected = "third", actual = selector.peek()) + selector.remove(state = "third") + assertEquals(expected = "second", actual = selector.peek()) + assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) + } + + @Test + fun `normal factory accepts external goals without replacing machine statistics`() { + val statistics = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 2), instruction(offset = 4))) + targets.markCovered(targets = listOf(instruction(offset = 2))) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createPathSelector( + initialState = left, + options = options(), + applicationGraph = graph, + coverageStatisticsFactory = { statistics }, + cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, + coverageTargetsFactory = { targets }, + ) + selector.add(states = listOf(right)) + assertSame(expected = right, actual = selector.peek()) + + targets.markCovered(targets = listOf(instruction(offset = 4))) + targets.addTargets(targets = listOf(instruction(offset = 1))) + + assertSame(expected = left, actual = selector.peek()) + assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) + assertEquals(expected = 0, actual = statistics.getTotalCoveredStatements()) + } + + @Test + fun `map factory forwards external goals to the standard selector`() { + val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createPathSelector( + initialStates = mapOf("method" to left), + options = options(), + applicationGraph = graph, + timeStatistics = TimeStatistics(), + cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, + coverageTargetsFactory = { targets }, + ) + selector.add(states = listOf(right)) + + assertSame(expected = right, actual = selector.peek()) + } + + @Test + fun `default statistics refresh weights for states that were not stepped`() { + val statistics = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val completed = mockState(id = 3u, startMethod = "method", startInstruction = 2) + every { completed.pathNode } returns PathNode.root() + + instruction(offset = 0) + instruction(offset = 1) + instruction(offset = 2) + val selector = createPathSelector( + initialState = left, + options = options(), + applicationGraph = graph, + coverageStatisticsFactory = { statistics }, + cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, + ) + selector.add(states = listOf(right)) + selector.peek() + + statistics.onStateTerminated(state = completed, stateReachable = true) + + assertSame(expected = right, actual = selector.peek()) + assertEquals(expected = 3, actual = statistics.getTotalCoveredStatements()) + } + + @Test + fun `randomized closest selector also refreshes all pending weights`() { + val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createClosestToUncoveredPathSelector( + coverageTargets = targets, + cfgStatistics = CfgStatisticsImpl(graph), + applicationGraph = graph, + random = Random(seed = 42), + ) + selector.add(states = listOf(left, right)) + assertSame(expected = right, actual = selector.peek()) + + targets.markCovered(targets = listOf(instruction(offset = 4))) + targets.addTargets(targets = listOf(instruction(offset = 1))) + + assertSame(expected = left, actual = selector.peek()) + } + + private fun selector(targets: ExternalCoverageTargets): UPathSelector = + createCoverageGuidedPathSelector( + coverageTargets = targets, + distance = { state, target -> + val destination = if (target == "new-left") "left" else target + if (state == destination) 0u else 10u + }, + ) + + private fun instruction(offset: Int) = TestInstruction(method = "method", offset = offset) + + private fun options() = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED), + exceptionsPropagation = false, + ) + + private val graph = appGraph { + method(name = "method", instructionsCount = 5) { + entryPoint(offset = 0) + edge(from = 0, to = 1) + edge(from = 0, to = 3) + edge(from = 1, to = 2) + edge(from = 3, to = 4) + exitPoint(offset = 2) + exitPoint(offset = 4) + } + } + + private data class Edge(val from: Int, val to: Int) +} diff --git a/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt b/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt new file mode 100644 index 0000000000..dbf0c03795 --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt @@ -0,0 +1,47 @@ +package org.usvm.statistics + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ExternalCoverageTargetsTest { + @Test + fun `coverage only changes registered goals and repeated reports are idempotent`() { + val targets = ExternalCoverageTargets(targets = listOf("first", "second")) + + assertFalse(targets.markCovered(targets = listOf("unknown"))) + assertEquals(expected = 0L, actual = targets.revision) + assertTrue(targets.markCovered(targets = listOf("first"))) + assertFalse(targets.markCovered(targets = listOf("first"))) + + assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) + assertEquals(expected = 1L, actual = targets.revision) + } + + @Test + fun `new goals can be registered without resurrecting covered goals`() { + val targets = ExternalCoverageTargets(targets = listOf("first")) + targets.markCovered(targets = listOf("first")) + + assertTrue(targets.addTargets(targets = listOf("first", "second", "second"))) + assertFalse(targets.addTargets(targets = listOf("first", "second"))) + + assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) + assertEquals(expected = 2L, actual = targets.revision) + } + + @Test + fun `goal snapshots and constructor inputs do not share mutable storage`() { + val initial = mutableListOf("first", "second") + val targets = ExternalCoverageTargets(targets = initial) + val snapshot = targets.uncoveredTargets + + initial.clear() + targets.markCovered(targets = listOf("first", "second")) + + assertEquals(expected = setOf("first", "second"), actual = snapshot) + assertTrue(targets.uncoveredTargets.isEmpty()) + assertEquals(expected = 1L, actual = targets.revision) + } +} From 362d2fc17645a5000b378dc2988d1af05ef7d937 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 00:25:38 +0300 Subject: [PATCH 02/10] [Core] Isolate external coverage updates in one selector Restore existing statistics and selector APIs. Keep goal updates and lazy priority refresh inside CoverageGuidedPathSelector, and remove the unsolicited README. --- usvm-core/README.md | 59 ---- .../org/usvm/ps/CoverageGuidedPathSelector.kt | 73 +++-- .../kotlin/org/usvm/ps/PathSelectorFactory.kt | 76 ++--- .../org/usvm/ps/WeightedPathSelector.kt | 27 +- .../org/usvm/statistics/CoverageStatistics.kt | 10 +- .../org/usvm/statistics/CoverageTargets.kt | 43 --- .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 265 +++++++++--------- .../statistics/ExternalCoverageTargetsTest.kt | 47 ---- 8 files changed, 209 insertions(+), 391 deletions(-) delete mode 100644 usvm-core/README.md delete mode 100644 usvm-core/src/main/kotlin/org/usvm/statistics/CoverageTargets.kt delete mode 100644 usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt diff --git a/usvm-core/README.md b/usvm-core/README.md deleted file mode 100644 index 853a55b309..0000000000 --- a/usvm-core/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# External coverage goals - -`CoverageTargets` supplies the currently uncovered goals and a revision that changes whenever that set changes. -The normal `CoverageStatistics` implements this contract for statements. `ExternalCoverageTargets` lets a caller -register goals and report confirmed coverage between analysis steps. - -The standard factory accepts this source independently of the statistics used for reporting and stopping: - -```kotlin -val coverage = ExternalCoverageTargets(targets = knownStatements) -coverage.markCovered(targets = statementsCoveredByConcreteTests) - -val selector = createPathSelector( - initialState = initialState, - options = options.copy(pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED)), - applicationGraph = graph, - cfgStatisticsFactory = { cfgStatistics }, - coverageTargetsFactory = { coverage }, -) - -// The analysis can continue with the same queue after the caller confirms more coverage. -coverage.markCovered(targets = newlyConfirmedStatements) -``` - -Both `CLOSEST_TO_UNCOVERED` variants use cached CFG distances in the active call stack. Existing entry-point fairness, -exception propagation, loop bounds and selector composition remain controlled by `UMachineOptions`. -External goals do not change reported coverage percentages or coverage-based stop conditions. - -For targets that are not statements, use the same weighted queue with a caller-supplied distance: - -```kotlin -data class Edge(val from: Statement, val to: Statement) - -val coverage = ExternalCoverageTargets(targets = knownEdges) -coverage.markCovered(targets = edgesCoveredByConcreteTests) - -val selector = createCoverageGuidedPathSelector( - coverageTargets = coverage, - distance = ::distanceFromStateToEdge, -) -``` - -The distance function owns the meaning of each target and returns `UInt.MAX_VALUE` when it is unreachable. -An edge must retain both endpoints: reaching its successor through a different predecessor does not cover that edge. -The core does not turn an edge into a statement or infer coverage from the distance function. -Language-specific code can reuse the existing CFG and interprocedural distance calculators in this function. - -Updates are applied on the analysis thread, between machine steps. The next `peek()` refreshes all queued weights -after a revision change; repeated peeks with unchanged evidence reuse the weights. No listener keeps the selector -attached to the external source after the analysis ends. Draining stopped work does not calculate new distances. - -`markCovered` ignores unknown goals, and repeated coverage reports do not change the revision. Use `addTargets` -to register newly identified goals; re-registering a covered goal does not make it uncovered again. Missing records -are not automatically registered as uncovered goals. - -An empty goal set retains every queued state. Covered prefixes remain traversable, and coverage alone does not -establish a property on other inputs. Property search must keep its own stop conditions and exploration policy. -Runtime admission, source identity and concrete replay confirmation belong to the caller; this core API accepts -its coverage decisions without interpreting them. diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt index 2b266d7bb7..df0f0efe87 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -2,29 +2,62 @@ package org.usvm.ps import org.usvm.UPathSelector import org.usvm.algorithms.DeterministicPriorityCollection -import org.usvm.statistics.CoverageTargets /** - * Selects the closest uncovered goal without pruning states. Goals may be statements, edges or other caller types. - * [distance] owns their semantics; [UInt.MAX_VALUE] means unreachable. Updates become visible on the next peek. + * Caller-confirmed coverage changes priorities, never removes states. Updates belong on the analysis thread. + * [distance] defines statement, edge or other goal semantics; [UInt.MAX_VALUE] means unreachable. */ -fun createCoverageGuidedPathSelector( - coverageTargets: CoverageTargets, +class CoverageGuidedPathSelector( + targets: Collection, distance: (State, Target) -> UInt, -): UPathSelector { - var revision = coverageTargets.revision - var targets = coverageTargets.uncoveredTargets.toList() - - return WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = { state -> - if (revision != coverageTargets.revision) { - targets = coverageTargets.uncoveredTargets.toList() - revision = coverageTargets.revision - } - - targets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE - }, - weightRevision = { coverageTargets.revision }, +) : UPathSelector { + private val knownTargets = targets.toMutableSet() + private val pendingTargets = targets.toMutableSet() + private var prioritiesChanged = false + private val selector = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, + weighter = { state -> pendingTargets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE }, ) + + val uncoveredTargets: Set + get() = pendingTargets.toSet() + + /** Registers newly identified goals; previously covered goals stay covered. */ + fun addTargets(targets: Collection): Boolean { + val additions = targets.filter { knownTargets.add(it) } + if (additions.isEmpty()) return false + + pendingTargets.addAll(additions) + prioritiesChanged = true + return true + } + + /** Unknown goals are ignored. Changed priorities become visible on the next [peek]. */ + fun markCovered(targets: Collection): Boolean { + val changed = pendingTargets.removeAll(targets.toSet()) + if (changed) prioritiesChanged = true + + return changed + } + + override fun isEmpty(): Boolean = selector.isEmpty() + + override fun peek(): State { + if (prioritiesChanged) { + val states = selector.drain().toList() + selector.add(states) + prioritiesChanged = false + } + + return selector.peek() + } + + override fun add(states: Collection) = selector.add(states) + + override fun update(state: State) = selector.update(state) + + override fun remove(state: State) = selector.remove(state) + + // Bypass pending priority updates when collecting stopped work. + override fun drain(): Sequence = selector.drain() } diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt index e7214bb419..9a3d0b4c94 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt @@ -13,7 +13,6 @@ import org.usvm.merging.CloseStatesSearcherImpl import org.usvm.merging.MergingPathSelector import org.usvm.statistics.ApplicationGraph import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.CoverageTargets import org.usvm.statistics.TimeStatistics import org.usvm.statistics.distances.CallGraphStatistics import org.usvm.statistics.distances.CallStackDistanceCalculator @@ -37,7 +36,6 @@ private fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, - coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { @@ -68,24 +66,18 @@ private fun createPathSelector( PathSelectionStrategy.FORK_DEPTH -> createForkDepthPathSelector() PathSelectionStrategy.FORK_DEPTH_RANDOM -> createForkDepthPathSelector(random) - PathSelectionStrategy.CLOSEST_TO_UNCOVERED, - PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM, - -> { - val coverageTargets = requireNotNull(coverageTargetsFactory()) { - "Coverage targets are required for closest to uncovered path selector" - } - val cfgStatistics = requireNotNull(cfgStatisticsFactory()) { - "CFG statistics is required for closest to uncovered path selector" - } - val selectorRandom = if (strategy == PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM) random else null + PathSelectionStrategy.CLOSEST_TO_UNCOVERED -> createClosestToUncoveredPathSelector( + requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, + requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, + applicationGraph + ) - createClosestToUncoveredPathSelector( - coverageTargets = coverageTargets, - cfgStatistics = cfgStatistics, - applicationGraph = applicationGraph, - random = selectorRandom, - ) - } + PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM -> createClosestToUncoveredPathSelector( + requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, + requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, + applicationGraph, + random + ) PathSelectionStrategy.TARGETED -> createTargetedPathSelector( requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for targeted path selector" }, @@ -153,7 +145,6 @@ private fun createPathSelector( return selector } -/** [coverageTargetsFactory] overrides closest-to-uncovered goals, leaving reported coverage unchanged. */ fun createPathSelector( initialState: State, options: UMachineOptions, @@ -162,7 +153,6 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, - coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> = createPathSelector( listOf(initialState), @@ -171,11 +161,9 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory, - coverageTargetsFactory, + loopStatisticFactory ) -/** External goals are shared by the entry points; coverage statistics still own reporting and stop conditions. */ fun createPathSelector( initialStates: Map, options: UMachineOptions, @@ -185,7 +173,6 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, - coverageTargetsFactory: () -> CoverageTargets? = coverageStatisticsFactory, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { val hasSingleEntryPoint = initialStates.size == 1 val hasUnlimitedScheduling = options.timeout == Duration.INFINITE && @@ -198,8 +185,7 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory, - coverageTargetsFactory, + loopStatisticFactory ) } @@ -216,8 +202,7 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory, - coverageTargetsFactory, + loopStatisticFactory ) val coverageStatistics = coverageStatisticsFactory() @@ -327,35 +312,29 @@ private fun > createDepthPathSelector(rando ) } -/** Uses cached CFG distances within the active call stack and refreshes priorities after goal changes. */ -fun > createClosestToUncoveredPathSelector( - coverageTargets: CoverageTargets, +private fun > createClosestToUncoveredPathSelector( + coverageStatistics: CoverageStatistics, cfgStatistics: CfgStatistics, applicationGraph: ApplicationGraph, random: Random? = null, ): UPathSelector { - fun createDistanceCalculator() = CallStackDistanceCalculator( - targets = coverageTargets.uncoveredTargets, + val distanceCalculator = CallStackDistanceCalculator( + targets = coverageStatistics.getUncoveredStatements(), cfgStatistics = cfgStatistics, - applicationGraph = applicationGraph, + applicationGraph ) - var distanceCalculator = createDistanceCalculator() - var revision = coverageTargets.revision - - fun calculateDistance(state: State): UInt { - if (revision != coverageTargets.revision) { - distanceCalculator = createDistanceCalculator() - revision = coverageTargets.revision - } - return distanceCalculator.calculateDistance(state.currentStatement, state.callStack) + coverageStatistics.addOnCoveredObserver { _, method, statement -> + distanceCalculator.removeTarget( + method, + statement + ) } if (random == null) { return WeightedPathSelector( priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = ::calculateDistance, - weightRevision = { coverageTargets.revision }, + weighter = { distanceCalculator.calculateDistance(it.currentStatement, it.callStack) } ) } @@ -363,11 +342,10 @@ fun > creat priorityCollectionFactory = { RandomizedPriorityCollection(compareById()) { random.nextDouble() } }, weighter = { 1.0 / max( - calculateDistance(it).toDouble(), + distanceCalculator.calculateDistance(it.currentStatement, it.callStack).toDouble(), 1.0 ) - }, - weightRevision = { coverageTargets.revision }, + } ) } diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt index 0b29f65294..7844295c55 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt @@ -9,29 +9,17 @@ import org.usvm.algorithms.UPriorityCollection * * @param priorityCollectionFactory function to create a priority collection for states. * @param weighter [StateWeighter] used to get states' weight which is used as priority in collection. - * @param weightRevision invalidates all queued weights when shared guidance changes between steps. */ open class WeightedPathSelector( priorityCollectionFactory: () -> UPriorityCollection, - private val weighter: StateWeighter, - private val weightRevision: (() -> Long)? = null, + private val weighter: StateWeighter ) : UPathSelector { private val priorityCollection = priorityCollectionFactory() - private var currentWeightRevision = weightRevision?.invoke() override fun isEmpty(): Boolean = priorityCollection.count == 0 - override fun peek(): State { - val revision = weightRevision?.invoke() - if (revision != currentWeightRevision) { - val states = drainPriorityCollection().toList() - add(states) - currentWeightRevision = revision - } - - return priorityCollection.peek() - } + override fun peek(): State = priorityCollection.peek() override fun update(state: State) = priorityCollection.update(state, weighter.weight(state)) @@ -42,15 +30,4 @@ open class WeightedPathSelector( } override fun remove(state: State) = priorityCollection.remove(state) - - override fun drain(): Sequence = drainPriorityCollection() - - private fun drainPriorityCollection(): Sequence = sequence { - // Draining stopped work must not recalculate weights or consult external goals. - while (priorityCollection.count > 0) { - val state = priorityCollection.peek() - priorityCollection.remove(state) - yield(state) - } - } } diff --git a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt index a4b767d25b..df263e6ac1 100644 --- a/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt +++ b/usvm-core/src/main/kotlin/org/usvm/statistics/CoverageStatistics.kt @@ -16,13 +16,7 @@ import java.util.concurrent.ConcurrentHashMap class CoverageStatistics>( methods: Set, private val applicationGraph: ApplicationGraph, -) : UMachineObserver, CoverageTargets { - - override val uncoveredTargets: Collection - get() = getUncoveredStatements() - - override var revision: Long = 0 - private set +) : UMachineObserver { private val onStatementCoveredObservers: MutableSet<(State, Method, Statement) -> Unit> = ConcurrentHashMap.newKeySet() @@ -62,7 +56,6 @@ class CoverageStatistics { - val uncoveredTargets: Collection - val revision: Long -} - -/** - * Caller-owned coverage goals, updated on the analysis thread between machine steps. - * Reporting coverage does not remove queued states or establish a property on other inputs. - */ -class ExternalCoverageTargets(targets: Collection) : CoverageTargets { - private val knownTargets = targets.toMutableSet() - private val pendingTargets = targets.toMutableSet() - - override val uncoveredTargets: Set - get() = pendingTargets.toSet() - - override var revision: Long = 0 - private set - - /** Registers newly identified goals; previously covered goals stay covered. */ - fun addTargets(targets: Collection): Boolean { - val additions = targets.filter { knownTargets.add(it) } - if (additions.isEmpty()) return false - - pendingTargets.addAll(additions) - revision++ - return true - } - - /** Accepts caller-confirmed coverage of known goals. Unknown goals are ignored. */ - fun markCovered(targets: Collection): Boolean { - val changed = pendingTargets.removeAll(targets.toSet()) - if (changed) revision++ - - return changed - } -} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt index 515fc8561e..84c2a8c88e 100644 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt @@ -1,21 +1,13 @@ package org.usvm.ps -import io.mockk.every import org.junit.jupiter.api.Test -import org.usvm.PathNode -import org.usvm.PathSelectionStrategy import org.usvm.TestInstruction import org.usvm.TestState -import org.usvm.TestTarget -import org.usvm.UMachineOptions -import org.usvm.UPathSelector import org.usvm.appGraph import org.usvm.mockState -import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.ExternalCoverageTargets -import org.usvm.statistics.TimeStatistics +import org.usvm.statistics.distances.CallStackDistanceCalculator import org.usvm.statistics.distances.CfgStatisticsImpl -import kotlin.random.Random +import org.usvm.statistics.distances.MultiTargetDistanceCalculator import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertSame @@ -23,26 +15,24 @@ import kotlin.test.assertTrue class CoverageGuidedPathSelectorTest { @Test - fun `external coverage reprioritizes every pending state without state updates`() { - val targets = ExternalCoverageTargets(targets = listOf("left", "right")) - val selector = selector(targets) + fun `reported coverage reprioritizes every pending state without state updates`() { + val selector = selector(targets = listOf("left", "right")) selector.add(states = listOf("left", "right")) - targets.markCovered(targets = listOf("left")) + selector.markCovered(targets = listOf("left")) assertEquals(expected = "right", actual = selector.peek()) - targets.addTargets(targets = listOf("new-left")) - targets.markCovered(targets = listOf("right")) + selector.addTargets(targets = listOf("new-left")) + selector.markCovered(targets = listOf("right")) assertEquals(expected = "left", actual = selector.peek()) assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) } @Test - fun `stable evidence does not recalculate queued weights`() { - val targets = ExternalCoverageTargets(targets = listOf("goal")) + fun `unchanged coverage does not recalculate queued weights`() { var evaluations = 0 - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector( + targets = listOf("goal"), distance = { _, _ -> evaluations++ 0u @@ -53,18 +43,43 @@ class CoverageGuidedPathSelectorTest { selector.peek() selector.peek() - assertFalse(targets.markCovered(targets = listOf("unknown"))) + assertFalse(selector.markCovered(targets = listOf("unknown"))) + assertFalse(selector.addTargets(targets = listOf("goal"))) selector.peek() assertEquals(expected = afterAdd, actual = evaluations) } + @Test + fun `multiple reports are applied together before the next selection`() { + var refreshing = false + var evaluations = 0 + val selector = CoverageGuidedPathSelector( + targets = listOf("first", "second", "third"), + distance = { state, target -> + if (refreshing) { + check(target == "third") + evaluations++ + } + + if (state == target) 0u else 10u + }, + ) + selector.add(states = listOf("first", "third")) + refreshing = true + + selector.markCovered(targets = listOf("first")) + selector.markCovered(targets = listOf("second")) + + assertEquals(expected = "third", actual = selector.peek()) + assertEquals(expected = 2, actual = evaluations) + } + @Test fun `saturated coverage retains all queued states and accepts later work`() { - val targets = ExternalCoverageTargets(targets = listOf("left", "right")) - val selector = selector(targets) + val selector = selector(targets = listOf("left", "right")) selector.add(states = listOf("left", "right")) - targets.markCovered(targets = listOf("left", "right")) + selector.markCovered(targets = listOf("left", "right")) selector.peek() val drained = selector.drain().toList() @@ -79,26 +94,24 @@ class CoverageGuidedPathSelectorTest { fun `edges sharing a successor remain distinct coverage goals`() { val firstEdge = Edge(from = 1, to = 3) val secondEdge = Edge(from = 2, to = 3) - val targets = ExternalCoverageTargets(targets = listOf(firstEdge, secondEdge)) val firstPath = setOf(firstEdge) val secondPath = setOf(secondEdge) - val selector = createCoverageGuidedPathSelector, Edge>( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector, Edge>( + targets = listOf(firstEdge, secondEdge), distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, ) selector.add(states = listOf(firstPath, secondPath)) - targets.markCovered(targets = listOf(firstEdge)) + selector.markCovered(targets = listOf(firstEdge)) assertSame(expected = secondPath, actual = selector.peek()) assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) } @Test - fun `empty initial evidence does not invoke the distance function or remove states`() { - val targets = ExternalCoverageTargets(targets = emptyList()) - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + fun `empty goals do not invoke the distance function or remove states`() { + val selector = CoverageGuidedPathSelector( + targets = emptyList(), distance = { _, _ -> error("There is no known goal") }, ) selector.add(states = listOf("first", "second")) @@ -110,11 +123,10 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `draining stopped work does not refresh weights after new evidence`() { - val targets = ExternalCoverageTargets(targets = listOf("left")) + fun `draining stopped work does not refresh weights after coverage changes`() { var stopped = false - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector( + targets = listOf("left"), distance = { _, _ -> check(!stopped) 0u @@ -122,7 +134,7 @@ class CoverageGuidedPathSelectorTest { ) selector.add(states = listOf("first", "second")) stopped = true - targets.addTargets(targets = listOf("right")) + selector.addTargets(targets = listOf("right")) val drained = selector.drain().toList() @@ -131,11 +143,10 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `state updates additions and removals work across coverage revisions`() { - val targets = ExternalCoverageTargets(targets = listOf("left", "right")) + fun `state updates additions and removals work after coverage changes`() { val distances = mutableMapOf("first" to 0u, "second" to 2u, "third" to 1u) - val selector = createCoverageGuidedPathSelector( - coverageTargets = targets, + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), distance = { state, _ -> distances.getValue(state) }, ) selector.add(states = listOf("first", "second")) @@ -144,7 +155,7 @@ class CoverageGuidedPathSelectorTest { distances["first"] = 3u selector.update(state = "first") selector.add(states = listOf("third")) - targets.markCovered(targets = listOf("left")) + selector.markCovered(targets = listOf("left")) assertEquals(expected = "third", actual = selector.peek()) selector.remove(state = "third") @@ -153,126 +164,102 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `normal factory accepts external goals without replacing machine statistics`() { - val statistics = CoverageStatistics( - methods = setOf("method"), - applicationGraph = graph, - ) - val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 2), instruction(offset = 4))) - targets.markCovered(targets = listOf(instruction(offset = 2))) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val selector = createPathSelector( - initialState = left, - options = options(), - applicationGraph = graph, - coverageStatisticsFactory = { statistics }, - cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, - coverageTargetsFactory = { targets }, - ) - selector.add(states = listOf(right)) - assertSame(expected = right, actual = selector.peek()) + fun `coverage reports are idempotent and unknown goals are ignored`() { + val selector = selector(targets = listOf("first", "second")) - targets.markCovered(targets = listOf(instruction(offset = 4))) - targets.addTargets(targets = listOf(instruction(offset = 1))) + assertFalse(selector.markCovered(targets = listOf("unknown"))) + assertTrue(selector.markCovered(targets = listOf("first"))) + assertFalse(selector.markCovered(targets = listOf("first"))) - assertSame(expected = left, actual = selector.peek()) - assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) - assertEquals(expected = 0, actual = statistics.getTotalCoveredStatements()) + assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) } @Test - fun `map factory forwards external goals to the standard selector`() { - val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val selector = createPathSelector( - initialStates = mapOf("method" to left), - options = options(), - applicationGraph = graph, - timeStatistics = TimeStatistics(), - cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, - coverageTargetsFactory = { targets }, - ) - selector.add(states = listOf(right)) + fun `new goals can be registered without resurrecting covered goals`() { + val selector = selector(targets = listOf("first")) + selector.markCovered(targets = listOf("first")) - assertSame(expected = right, actual = selector.peek()) + assertTrue(selector.addTargets(targets = listOf("first", "second", "second"))) + assertFalse(selector.addTargets(targets = listOf("first", "second"))) + + assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) } @Test - fun `default statistics refresh weights for states that were not stepped`() { - val statistics = CoverageStatistics( - methods = setOf("method"), - applicationGraph = graph, - ) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val completed = mockState(id = 3u, startMethod = "method", startInstruction = 2) - every { completed.pathNode } returns PathNode.root() + - instruction(offset = 0) + instruction(offset = 1) + instruction(offset = 2) - val selector = createPathSelector( - initialState = left, - options = options(), - applicationGraph = graph, - coverageStatisticsFactory = { statistics }, - cfgStatisticsFactory = { CfgStatisticsImpl(graph) }, - ) - selector.add(states = listOf(right)) - selector.peek() + fun `goal snapshots and constructor inputs do not share mutable storage`() { + val initial = mutableListOf("first", "second") + val selector = selector(targets = initial) + val snapshot = selector.uncoveredTargets - statistics.onStateTerminated(state = completed, stateReachable = true) + initial.clear() + selector.markCovered(targets = listOf("first", "second")) - assertSame(expected = right, actual = selector.peek()) - assertEquals(expected = 3, actual = statistics.getTotalCoveredStatements()) + assertEquals(expected = setOf("first", "second"), actual = snapshot) + assertTrue(selector.uncoveredTargets.isEmpty()) } @Test - fun `randomized closest selector also refreshes all pending weights`() { - val targets = ExternalCoverageTargets(targets = listOf(instruction(offset = 4))) + fun `caller can reuse existing CFG distance calculators`() { + val graph = appGraph { + method(name = "method", instructionsCount = 5) { + entryPoint(offset = 0) + edge(from = 0, to = 1) + edge(from = 0, to = 3) + edge(from = 1, to = 2) + edge(from = 3, to = 4) + exitPoint(offset = 2) + exitPoint(offset = 4) + } + } + val cfgStatistics = CfgStatisticsImpl(graph) + val distances = MultiTargetDistanceCalculator { target -> + CallStackDistanceCalculator( + targets = listOf(target), + cfgStatistics = cfgStatistics, + applicationGraph = graph, + ) + } + val firstTarget = TestInstruction(method = "method", offset = 2) + val secondTarget = TestInstruction(method = "method", offset = 4) + val selector = CoverageGuidedPathSelector( + targets = listOf(firstTarget, secondTarget), + distance = { state, target -> + distances.calculateDistance(state.currentStatement, state.callStack, target) + }, + ) val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - val selector = createClosestToUncoveredPathSelector( - coverageTargets = targets, - cfgStatistics = CfgStatisticsImpl(graph), - applicationGraph = graph, - random = Random(seed = 42), - ) selector.add(states = listOf(left, right)) - assertSame(expected = right, actual = selector.peek()) - targets.markCovered(targets = listOf(instruction(offset = 4))) - targets.addTargets(targets = listOf(instruction(offset = 1))) + selector.markCovered(targets = listOf(firstTarget)) - assertSame(expected = left, actual = selector.peek()) + assertSame(expected = right, actual = selector.peek()) + assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) } - private fun selector(targets: ExternalCoverageTargets): UPathSelector = - createCoverageGuidedPathSelector( - coverageTargets = targets, - distance = { state, target -> - val destination = if (target == "new-left") "left" else target - if (state == destination) 0u else 10u - }, - ) - - private fun instruction(offset: Int) = TestInstruction(method = "method", offset = offset) + @Test + fun `coverage guidance composes with a standard selector on shared states`() { + val coverage = selector(targets = listOf("left", "right")) + val combined = InterleavedPathSelector(selectors = listOf(coverage, BfsPathSelector())) + combined.add(states = listOf("left", "right")) + assertEquals(expected = "left", actual = combined.peek()) + + coverage.markCovered(targets = listOf("left")) + combined.add(states = listOf("fork")) + + assertEquals(expected = "right", actual = combined.peek()) + combined.remove(state = "right") + assertEquals(expected = setOf("left", "fork"), actual = combined.drain().toSet()) + assertTrue(combined.isEmpty()) + } - private fun options() = UMachineOptions( - pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED), - exceptionsPropagation = false, + private fun selector(targets: Collection) = CoverageGuidedPathSelector( + targets = targets, + distance = { state, target -> + val destination = if (target == "new-left") "left" else target + if (state == destination) 0u else 10u + }, ) - private val graph = appGraph { - method(name = "method", instructionsCount = 5) { - entryPoint(offset = 0) - edge(from = 0, to = 1) - edge(from = 0, to = 3) - edge(from = 1, to = 2) - edge(from = 3, to = 4) - exitPoint(offset = 2) - exitPoint(offset = 4) - } - } - private data class Edge(val from: Int, val to: Int) } diff --git a/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt b/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt deleted file mode 100644 index dbf0c03795..0000000000 --- a/usvm-core/src/test/kotlin/org/usvm/statistics/ExternalCoverageTargetsTest.kt +++ /dev/null @@ -1,47 +0,0 @@ -package org.usvm.statistics - -import org.junit.jupiter.api.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class ExternalCoverageTargetsTest { - @Test - fun `coverage only changes registered goals and repeated reports are idempotent`() { - val targets = ExternalCoverageTargets(targets = listOf("first", "second")) - - assertFalse(targets.markCovered(targets = listOf("unknown"))) - assertEquals(expected = 0L, actual = targets.revision) - assertTrue(targets.markCovered(targets = listOf("first"))) - assertFalse(targets.markCovered(targets = listOf("first"))) - - assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) - assertEquals(expected = 1L, actual = targets.revision) - } - - @Test - fun `new goals can be registered without resurrecting covered goals`() { - val targets = ExternalCoverageTargets(targets = listOf("first")) - targets.markCovered(targets = listOf("first")) - - assertTrue(targets.addTargets(targets = listOf("first", "second", "second"))) - assertFalse(targets.addTargets(targets = listOf("first", "second"))) - - assertEquals(expected = setOf("second"), actual = targets.uncoveredTargets) - assertEquals(expected = 2L, actual = targets.revision) - } - - @Test - fun `goal snapshots and constructor inputs do not share mutable storage`() { - val initial = mutableListOf("first", "second") - val targets = ExternalCoverageTargets(targets = initial) - val snapshot = targets.uncoveredTargets - - initial.clear() - targets.markCovered(targets = listOf("first", "second")) - - assertEquals(expected = setOf("first", "second"), actual = snapshot) - assertTrue(targets.uncoveredTargets.isEmpty()) - assertEquals(expected = 1L, actual = targets.revision) - } -} From 2d96bc063930691c301b44aaf952240fad94b057 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 00:35:23 +0300 Subject: [PATCH 03/10] Fix cached priority ordering after insertion --- .../DeterministicPriorityCollection.kt | 2 +- .../DeterministicPriorityCollectionTest.kt | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt diff --git a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt index af9eb77830..844a16487c 100644 --- a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt +++ b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt @@ -44,7 +44,7 @@ class DeterministicPriorityCollection(private val comparator: Compa assert({ !priorityQueue.any { (e, _) -> e == element } }) { "Element already exists in priority queue" } val currentTop = topElement - if (currentTop != null && comparator.compare(currentTop.second, priority) < 0) { + if (currentTop != null && comparator.compare(currentTop.second, priority) > 0) { topElement = element to priority priorityQueue.add(currentTop) return diff --git a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt new file mode 100644 index 0000000000..c5d182a099 --- /dev/null +++ b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionTest.kt @@ -0,0 +1,62 @@ +package org.usvm.algorithms + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals + +internal class DeterministicPriorityCollectionTest { + @Test + fun `adding a worse priority keeps the cached minimum`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + collection.add(element = "first", priority = 1) + assertEquals("first", collection.peek()) + + collection.add(element = "second", priority = 2) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } + + @Test + fun `adding a better priority replaces the cached minimum`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + collection.add(element = "second", priority = 2) + assertEquals("second", collection.peek()) + + collection.add(element = "first", priority = 1) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } + + @Test + fun `adding a worse priority keeps the cached maximum with reverse order`() { + val collection = DeterministicPriorityCollection(reverseOrder()) + collection.add(element = "first", priority = 2) + assertEquals("first", collection.peek()) + + collection.add(element = "second", priority = 1) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } + + @Test + fun `adding a better priority replaces the cached maximum with reverse order`() { + val collection = DeterministicPriorityCollection(reverseOrder()) + collection.add(element = "second", priority = 1) + assertEquals("second", collection.peek()) + + collection.add(element = "first", priority = 2) + + assertEquals("first", collection.peek()) + collection.remove("first") + assertEquals("second", collection.peek()) + assertEquals(1, collection.count) + } +} From 841e15b5d557da44bcebf924e1426da58ec2fef0 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 16:15:19 +0300 Subject: [PATCH 04/10] Batch coverage priority refreshes and heapify updated weights --- .../org/usvm/ps/CoverageGuidedPathSelector.kt | 80 +++++- .../CoverageGuidedPathSelectorRefreshTest.kt | 234 ++++++++++++++++++ .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 27 +- .../DeterministicPriorityCollection.kt | 50 +++- ...erministicPriorityCollectionRefreshTest.kt | 108 ++++++++ 5 files changed, 470 insertions(+), 29 deletions(-) create mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt create mode 100644 usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt index df0f0efe87..6dc86cdfcf 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -6,17 +6,33 @@ import org.usvm.algorithms.DeterministicPriorityCollection /** * Caller-confirmed coverage changes priorities, never removes states. Updates belong on the analysis thread. * [distance] defines statement, edge or other goal semantics; [UInt.MAX_VALUE] means unreachable. + * + * Weights use a snapshot of uncovered goals. Changes are applied before selection after at least + * [priorityRefreshInterval] completed selections since the previous priority refresh. A selection completes when + * its state is updated or removed; repeated [peek] calls do not advance the interval. + * The default interval is a tuning starting point, not a measured optimum. */ class CoverageGuidedPathSelector( targets: Collection, distance: (State, Target) -> UInt, + private val priorityRefreshInterval: Int = DEFAULT_PRIORITY_REFRESH_INTERVAL, ) : UPathSelector { + init { + require(priorityRefreshInterval > 0) { "Priority refresh interval must be positive" } + } + private val knownTargets = targets.toMutableSet() private val pendingTargets = targets.toMutableSet() + private var priorityTargets: Set = targets.toSet() + private val distanceToTarget = distance private var prioritiesChanged = false + private var completedSelections = 0 + private var selectionPending = false + private var selectedState: State? = null + private val priorities = DeterministicPriorityCollection(Comparator.naturalOrder()) private val selector = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(Comparator.naturalOrder()) }, - weighter = { state -> pendingTargets.minOfOrNull { target -> distance(state, target) } ?: UInt.MAX_VALUE }, + priorityCollectionFactory = { priorities }, + weighter = { state -> weight(state, priorityTargets) }, ) val uncoveredTargets: Set @@ -32,10 +48,16 @@ class CoverageGuidedPathSelector( return true } - /** Unknown goals are ignored. Changed priorities become visible on the next [peek]. */ + /** Unknown goals are ignored. Changed priorities are applied at the next eligible selection. */ fun markCovered(targets: Collection): Boolean { val changed = pendingTargets.removeAll(targets.toSet()) - if (changed) prioritiesChanged = true + if (changed) { + prioritiesChanged = pendingTargets.isNotEmpty() + if (!prioritiesChanged) { + // With no goals, ordering is irrelevant and new work needs no distance calculations. + priorityTargets = emptySet() + } + } return changed } @@ -43,21 +65,55 @@ class CoverageGuidedPathSelector( override fun isEmpty(): Boolean = selector.isEmpty() override fun peek(): State { - if (prioritiesChanged) { - val states = selector.drain().toList() - selector.add(states) + val refreshDue = prioritiesChanged && completedSelections >= priorityRefreshInterval + if (refreshDue) { + val refreshedTargets = pendingTargets.toSet() + priorities.updatePriorities { state -> weight(state, refreshedTargets) } + priorityTargets = refreshedTargets prioritiesChanged = false + completedSelections = 0 } - return selector.peek() + val state = selector.peek() + selectedState = state + selectionPending = true + return state } override fun add(states: Collection) = selector.add(states) - override fun update(state: State) = selector.update(state) + override fun update(state: State) { + selector.update(state) + completeSelection(state) + } - override fun remove(state: State) = selector.remove(state) + override fun remove(state: State) { + selector.remove(state) + completeSelection(state) + } + + // Bypass pending priority updates and selection accounting when collecting stopped work. + override fun drain(): Sequence { + selectionPending = false + selectedState = null + return selector.drain() + } - // Bypass pending priority updates when collecting stopped work. - override fun drain(): Sequence = selector.drain() + private fun weight(state: State, targets: Set): UInt = + targets.minOfOrNull { target -> distanceToTarget(state, target) } ?: UInt.MAX_VALUE + + private fun completeSelection(state: State) { + val completesSelection = selectionPending && selectedState == state + if (!completesSelection) return + + if (completedSelections < priorityRefreshInterval) { + completedSelections++ + } + selectionPending = false + selectedState = null + } + + private companion object { + const val DEFAULT_PRIORITY_REFRESH_INTERVAL = 128 + } } diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt new file mode 100644 index 0000000000..62b2200096 --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt @@ -0,0 +1,234 @@ +package org.usvm.ps + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class CoverageGuidedPathSelectorRefreshTest { + @Test + fun `default interval keeps weights stale for 128 completed selections`() { + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + ) + selector.add(states = listOf("left", "right")) + selector.markCovered(targets = listOf("left")) + + repeat(127) { + assertEquals(expected = "left", actual = selector.peek()) + evaluatedTargets.clear() + selector.update(state = "left") + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + } + evaluatedTargets.clear() + repeat(256) { + assertEquals(expected = "left", actual = selector.peek()) + } + assertTrue(evaluatedTargets.isEmpty()) + + selector.update(state = "left") + evaluatedTargets.clear() + + assertEquals(expected = "right", actual = selector.peek()) + assertEquals(expected = listOf("right", "right"), actual = evaluatedTargets) + evaluatedTargets.clear() + selector.peek() + assertTrue(evaluatedTargets.isEmpty()) + } + + @Test + fun `new and updated states use the old snapshot until the interval expires`() { + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("old"), + distance = { state, target -> + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("old")) + selector.peek() + + selector.addTargets(targets = listOf("new")) + selector.markCovered(targets = listOf("old")) + evaluatedTargets.clear() + selector.add(states = listOf("new")) + selector.update(state = "old") + + assertEquals(expected = listOf("old", "old"), actual = evaluatedTargets) + assertEquals(expected = setOf("new"), actual = selector.uncoveredTargets) + assertEquals(expected = "old", actual = selector.peek()) + + selector.update(state = "old") + evaluatedTargets.clear() + + assertEquals(expected = "new", actual = selector.peek()) + assertEquals(expected = listOf("new", "new"), actual = evaluatedTargets) + } + + @Test + fun `only one completion of the selected state advances the interval`() { + var evaluations = 0 + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + evaluations++ + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("left", "right", "other")) + selector.peek() + selector.markCovered(targets = listOf("left")) + + selector.update(state = "other") + selector.remove(state = "other") + selector.update(state = "left") + selector.update(state = "left") + val beforePeek = evaluations + + assertEquals(expected = "left", actual = selector.peek()) + assertEquals(expected = beforePeek, actual = evaluations) + + selector.remove(state = "left") + val beforeRefresh = evaluations + + assertEquals(expected = "right", actual = selector.peek()) + assertEquals(expected = beforeRefresh + 1, actual = evaluations) + } + + @Test + fun `each refresh starts a new interval including newly registered goals`() { + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + val destination = if (target == "left-again") "left" else target + if (state == destination) 0u else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("left", "right")) + selector.markCovered(targets = listOf("left")) + repeat(2) { selector.update(state = selector.peek()) } + assertEquals(expected = "right", actual = selector.peek()) + + selector.addTargets(targets = listOf("left-again")) + selector.markCovered(targets = listOf("right")) + selector.update(state = "right") + + assertEquals(expected = "right", actual = selector.peek()) + + selector.update(state = "right") + + assertEquals(expected = "left", actual = selector.peek()) + assertEquals(expected = setOf("left-again"), actual = selector.uncoveredTargets) + } + + @Test + fun `unchanged goals do not trigger refresh even after the interval`() { + var evaluations = 0 + val selector = CoverageGuidedPathSelector( + targets = listOf("goal"), + distance = { _, _ -> + evaluations++ + 0u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("first", "second")) + + repeat(8) { selector.update(state = selector.peek()) } + val beforePeek = evaluations + selector.markCovered(targets = listOf("unknown")) + selector.addTargets(targets = listOf("goal")) + selector.peek() + + assertEquals(expected = 2 + 8, actual = evaluations) + assertEquals(expected = beforePeek, actual = evaluations) + } + + @Test + fun `saturation avoids distance work and later goals resume guidance`() { + var distancesAllowed = true + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("old"), + distance = { state, target -> + check(distancesAllowed) + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 2, + ) + selector.add(states = listOf("old", "new")) + distancesAllowed = false + + selector.markCovered(targets = listOf("old")) + repeat(4) { selector.update(state = selector.peek()) } + selector.add(states = listOf("later")) + + assertTrue(selector.uncoveredTargets.isEmpty()) + + selector.addTargets(targets = listOf("new")) + distancesAllowed = true + evaluatedTargets.clear() + + assertEquals(expected = "new", actual = selector.peek()) + assertEquals(expected = listOf("new", "new", "new"), actual = evaluatedTargets) + assertEquals(expected = setOf("old", "new", "later"), actual = selector.drain().toSet()) + } + + @Test + fun `failed refresh preserves the snapshot and can be retried`() { + var failRefresh = false + val evaluatedTargets = mutableListOf() + val selector = CoverageGuidedPathSelector( + targets = listOf("left", "right"), + distance = { state, target -> + check(!failRefresh) + evaluatedTargets.add(target) + val matchingDistance = if (state == "left") 0u else 1u + if (state == target) matchingDistance else 10u + }, + priorityRefreshInterval = 1, + ) + selector.add(states = listOf("left", "right")) + selector.peek() + selector.markCovered(targets = listOf("left")) + selector.update(state = "left") + failRefresh = true + + assertFailsWith { selector.peek() } + + failRefresh = false + evaluatedTargets.clear() + selector.add(states = listOf("other")) + + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + assertEquals(expected = "right", actual = selector.peek()) + assertEquals(expected = setOf("left", "right", "other"), actual = selector.drain().toSet()) + } + + @Test + fun `refresh interval must be positive`() { + for (interval in listOf(0, -1)) { + assertFailsWith { + CoverageGuidedPathSelector( + targets = emptyList(), + distance = { _, _ -> 0u }, + priorityRefreshInterval = interval, + ) + } + } + } +} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt index 84c2a8c88e..825c93380e 100644 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt @@ -15,15 +15,21 @@ import kotlin.test.assertTrue class CoverageGuidedPathSelectorTest { @Test - fun `reported coverage reprioritizes every pending state without state updates`() { + fun `reported coverage reprioritizes every pending state after a completed selection`() { val selector = selector(targets = listOf("left", "right")) selector.add(states = listOf("left", "right")) + assertEquals(expected = "left", actual = selector.peek()) + selector.markCovered(targets = listOf("left")) + selector.update(state = "left") + assertEquals(expected = "right", actual = selector.peek()) selector.addTargets(targets = listOf("new-left")) selector.markCovered(targets = listOf("right")) + selector.update(state = "right") + assertEquals(expected = "left", actual = selector.peek()) assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) } @@ -51,7 +57,7 @@ class CoverageGuidedPathSelectorTest { } @Test - fun `multiple reports are applied together before the next selection`() { + fun `multiple reports are applied together at the refresh boundary`() { var refreshing = false var evaluations = 0 val selector = CoverageGuidedPathSelector( @@ -64,12 +70,15 @@ class CoverageGuidedPathSelectorTest { if (state == target) 0u else 10u }, + priorityRefreshInterval = 1, ) selector.add(states = listOf("first", "third")) - refreshing = true + val selected = selector.peek() selector.markCovered(targets = listOf("first")) selector.markCovered(targets = listOf("second")) + selector.update(state = selected) + refreshing = true assertEquals(expected = "third", actual = selector.peek()) assertEquals(expected = 2, actual = evaluations) @@ -99,10 +108,13 @@ class CoverageGuidedPathSelectorTest { val selector = CoverageGuidedPathSelector, Edge>( targets = listOf(firstEdge, secondEdge), distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, + priorityRefreshInterval = 1, ) selector.add(states = listOf(firstPath, secondPath)) + val selected = selector.peek() selector.markCovered(targets = listOf(firstEdge)) + selector.update(state = selected) assertSame(expected = secondPath, actual = selector.peek()) assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) @@ -226,12 +238,15 @@ class CoverageGuidedPathSelectorTest { distance = { state, target -> distances.calculateDistance(state.currentStatement, state.callStack, target) }, + priorityRefreshInterval = 1, ) val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) selector.add(states = listOf(left, right)) + val selected = selector.peek() selector.markCovered(targets = listOf(firstTarget)) + selector.update(state = selected) assertSame(expected = right, actual = selector.peek()) assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) @@ -240,14 +255,15 @@ class CoverageGuidedPathSelectorTest { @Test fun `coverage guidance composes with a standard selector on shared states`() { val coverage = selector(targets = listOf("left", "right")) - val combined = InterleavedPathSelector(selectors = listOf(coverage, BfsPathSelector())) + val combined = InterleavedPathSelector(selectors = listOf(BfsPathSelector(), coverage)) combined.add(states = listOf("left", "right")) assertEquals(expected = "left", actual = combined.peek()) coverage.markCovered(targets = listOf("left")) - combined.add(states = listOf("fork")) + combined.update(state = "left") assertEquals(expected = "right", actual = combined.peek()) + combined.add(states = listOf("fork")) combined.remove(state = "right") assertEquals(expected = setOf("left", "fork"), actual = combined.drain().toSet()) assertTrue(combined.isEmpty()) @@ -259,6 +275,7 @@ class CoverageGuidedPathSelectorTest { val destination = if (target == "new-left") "left" else target if (state == destination) 0u else 10u }, + priorityRefreshInterval = 1, ) private data class Edge(val from: Int, val to: Int) diff --git a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt index 844a16487c..1c5248ffef 100644 --- a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt +++ b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt @@ -11,16 +11,22 @@ import kotlin.NoSuchElementException // TODO: what to do if elements have same priority? class DeterministicPriorityCollection(private val comparator: Comparator) : UPriorityCollection { - private var topElement: Pair? = null - private val priorityQueue = PriorityQueue> { (_, p1), (_, p2) -> comparator.compare(p1, p2) } + private inner class Entry(val element: T, val priority: Priority) : Comparable { + override fun compareTo(other: Entry): Int = comparator.compare(priority, other.priority) + } + + private var topElement: Entry? = null + private var priorityQueue = PriorityQueue() override val count: Int get() = priorityQueue.size + if (topElement == null) 0 else 1 override fun peek(): T { - if (topElement == null) { - topElement = priorityQueue.remove() - } - return topElement!!.first + val currentTop = topElement + if (currentTop != null) return currentTop.element + + val next = priorityQueue.remove() + topElement = next + return next.element } override fun update(element: T, priority: Priority) { @@ -30,25 +36,45 @@ class DeterministicPriorityCollection(private val comparator: Compa override fun remove(element: T) { // Don't traverse the whole queue if we remove the top element - if (topElement != null && topElement?.first == element) { + if (topElement != null && topElement?.element == element) { topElement = null return } - if (!priorityQueue.removeIf { (e, _) -> e == element }) { + if (!priorityQueue.removeIf { it.element == element }) { throw NoSuchElementException("Element not found in priority queue") } } override fun add(element: T, priority: Priority) { - assert({ !priorityQueue.any { (e, _) -> e == element } }) { "Element already exists in priority queue" } + assert({ !priorityQueue.any { it.element == element } }) { "Element already exists in priority queue" } val currentTop = topElement - if (currentTop != null && comparator.compare(currentTop.second, priority) > 0) { - topElement = element to priority + if (currentTop != null && comparator.compare(currentTop.priority, priority) > 0) { + topElement = Entry(element, priority) priorityQueue.add(currentTop) return } - priorityQueue.add(element to priority) + priorityQueue.add(Entry(element, priority)) + } + + /** + * Recalculates each priority once and rebuilds the heap in linear time, without removing and reinserting elements. + * If [priority] throws, the collection keeps its previous priorities and cached selection. + */ + fun updatePriorities(priority: (T) -> Priority) { + val entries = ArrayList(count) + val currentTop = topElement + if (currentTop != null) { + entries.add(Entry(currentTop.element, priority(currentTop.element))) + } + for (entry in priorityQueue) { + entries.add(Entry(entry.element, priority(entry.element))) + } + + // The collection constructor heapifies Comparable entries; addAll would perform individual insertions. + val refreshedQueue = PriorityQueue(entries) + priorityQueue = refreshedQueue + topElement = null } } diff --git a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt new file mode 100644 index 0000000000..2537c630db --- /dev/null +++ b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt @@ -0,0 +1,108 @@ +package org.usvm.algorithms + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +internal class DeterministicPriorityCollectionRefreshTest { + @Test + fun `refresh reorders every element including the cached selection`() { + for (comparator in listOf(naturalOrder(), reverseOrder())) { + val collection = DeterministicPriorityCollection(comparator) + collection.add(element = "first", priority = 1) + collection.add(element = "second", priority = 2) + collection.add(element = "third", priority = 3) + collection.peek() + val evaluations = mutableListOf() + val priorities = mapOf("first" to 3, "second" to 1, "third" to 2) + + collection.updatePriorities { element -> + evaluations.add(element) + priorities.getValue(element) + } + + assertEquals(expected = 3, actual = collection.count) + assertEquals(expected = 3, actual = evaluations.size) + assertEquals(expected = priorities.keys, actual = evaluations.toSet()) + val expected = priorities.keys.sortedWith { left, right -> + comparator.compare(priorities.getValue(left), priorities.getValue(right)) + } + assertEquals(expected = expected, actual = drain(collection)) + } + } + + @Test + fun `heap rebuilding uses a linear number of comparisons`() { + var comparisons = 0 + val comparator = Comparator { left, right -> + comparisons++ + left.compareTo(right) + } + val collection = DeterministicPriorityCollection(comparator) + val size = 1024 + repeat(size) { element -> collection.add(element = element, priority = element) } + collection.peek() + comparisons = 0 + + collection.updatePriorities { element -> size - element } + + assertTrue(comparisons <= 2 * size, "Heapify used $comparisons comparisons for $size elements") + assertEquals(expected = (0 until size).reversed().toList(), actual = drain(collection)) + } + + @Test + fun `equal refreshed priorities retain every element`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + repeat(8) { element -> collection.add(element = element, priority = element) } + collection.peek() + + collection.updatePriorities { 0 } + + val elements = drain(collection) + assertEquals(expected = 8, actual = elements.size) + assertEquals(expected = (0 until 8).toSet(), actual = elements.toSet()) + } + + @Test + fun `failed refresh leaves old ordering and all elements intact`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + collection.add(element = "first", priority = 1) + collection.add(element = "second", priority = 2) + collection.peek() + + assertFailsWith { + collection.updatePriorities { element -> + check(element != "second") + 100 + } + } + + assertEquals(expected = "first", actual = collection.peek()) + assertEquals(expected = listOf("first", "second"), actual = drain(collection)) + } + + @Test + fun `empty and singleton collections can be refreshed repeatedly`() { + val collection = DeterministicPriorityCollection(naturalOrder()) + + collection.updatePriorities { error("Empty collection") } + collection.add(element = "only", priority = 1) + collection.peek() + repeat(2) { collection.updatePriorities { 2 } } + + assertEquals(expected = listOf("only"), actual = drain(collection)) + collection.updatePriorities { error("Empty collection") } + assertEquals(expected = 0, actual = collection.count) + } + + private fun drain(collection: DeterministicPriorityCollection): List { + val elements = mutableListOf() + while (collection.count > 0) { + val element = collection.peek() + collection.remove(element) + elements.add(element) + } + return elements + } +} From be64b3d0113598209cb4fc537b1bc2b2b9438aba Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 17:09:50 +0300 Subject: [PATCH 05/10] Use unordered sets for coverage goals and snapshots --- .../kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt | 12 ++++++------ .../usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt index 6dc86cdfcf..416043bf11 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt @@ -21,9 +21,9 @@ class CoverageGuidedPathSelector( require(priorityRefreshInterval > 0) { "Priority refresh interval must be positive" } } - private val knownTargets = targets.toMutableSet() - private val pendingTargets = targets.toMutableSet() - private var priorityTargets: Set = targets.toSet() + private val knownTargets = targets.toHashSet() + private val pendingTargets = targets.toHashSet() + private var priorityTargets: Set = targets.toHashSet() private val distanceToTarget = distance private var prioritiesChanged = false private var completedSelections = 0 @@ -36,7 +36,7 @@ class CoverageGuidedPathSelector( ) val uncoveredTargets: Set - get() = pendingTargets.toSet() + get() = pendingTargets.toHashSet() /** Registers newly identified goals; previously covered goals stay covered. */ fun addTargets(targets: Collection): Boolean { @@ -50,7 +50,7 @@ class CoverageGuidedPathSelector( /** Unknown goals are ignored. Changed priorities are applied at the next eligible selection. */ fun markCovered(targets: Collection): Boolean { - val changed = pendingTargets.removeAll(targets.toSet()) + val changed = pendingTargets.removeAll(targets.toHashSet()) if (changed) { prioritiesChanged = pendingTargets.isNotEmpty() if (!prioritiesChanged) { @@ -67,7 +67,7 @@ class CoverageGuidedPathSelector( override fun peek(): State { val refreshDue = prioritiesChanged && completedSelections >= priorityRefreshInterval if (refreshDue) { - val refreshedTargets = pendingTargets.toSet() + val refreshedTargets = pendingTargets.toHashSet() priorities.updatePriorities { state -> weight(state, refreshedTargets) } priorityTargets = refreshedTargets prioritiesChanged = false diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt index 62b2200096..9421b89c4c 100644 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt +++ b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt @@ -24,7 +24,7 @@ class CoverageGuidedPathSelectorRefreshTest { assertEquals(expected = "left", actual = selector.peek()) evaluatedTargets.clear() selector.update(state = "left") - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) } evaluatedTargets.clear() repeat(256) { @@ -214,7 +214,7 @@ class CoverageGuidedPathSelectorRefreshTest { evaluatedTargets.clear() selector.add(states = listOf("other")) - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets) + assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) assertEquals(expected = "right", actual = selector.peek()) assertEquals(expected = setOf("left", "right", "other"), actual = selector.drain().toSet()) } From 2ae0f29baa885135103b8c2c6d84d66595a10e58 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 17:14:50 +0300 Subject: [PATCH 06/10] Measure coverage selector costs on real JVM programs --- .../coverage-selector-performance/README.md | 91 +++++ .../coverage-selector-performance/logback.xml | 3 + .../manifest.json | 36 ++ .../performance.gradle | 19 + .../results.jsonl | 16 + .../coverage-selector-performance/run.py | 83 +++++ .../summarize.py | 39 ++ .../coverage-selector-performance/summary.csv | 13 + .../CoverageSelectorPerformanceTest.kt | 336 ++++++++++++++++++ 9 files changed, 636 insertions(+) create mode 100644 docs/experiments/coverage-selector-performance/README.md create mode 100644 docs/experiments/coverage-selector-performance/logback.xml create mode 100644 docs/experiments/coverage-selector-performance/manifest.json create mode 100644 docs/experiments/coverage-selector-performance/performance.gradle create mode 100644 docs/experiments/coverage-selector-performance/results.jsonl create mode 100644 docs/experiments/coverage-selector-performance/run.py create mode 100644 docs/experiments/coverage-selector-performance/summarize.py create mode 100644 docs/experiments/coverage-selector-performance/summary.csv create mode 100644 usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md new file mode 100644 index 0000000000..bb43ada212 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/README.md @@ -0,0 +1,91 @@ +# Coverage selector performance on real JVM bytecode + +This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. + +## Recorded results (2026-10-16) + +The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. + +The retained dataset contains 16 runs and 1,328,129 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. + +| Program | Mode | Steps | Stop | Analysis seconds | Selector seconds | Selector share | Refresh ms | +| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | +| Codec | existing | 100,000 | steps | 17.145 | 0.134 | 0.78% | 0 | +| Codec | eager | 100,000 | steps | 15.300 | 4.284 | 27.96% | 13.86 | +| Codec | 128 | 100,000 | steps | 21.468 | 6.319 | 29.48% | 90.32 | +| Codec | 512 | 100,000 | steps | 19.110 | 5.842 | 30.61% | 146.87 | +| Guava gcd | existing | 61,538 | timeout | 120.167 | 0.235 | 0.20% | 0 | +| Guava gcd | eager | 57,141 | timeout | 120.024 | 10.822 | 9.02% | 5.36 | +| Guava gcd | 128 | 63,808 | timeout | 120.189 | 14.249 | 11.86% | 5.73 | +| Guava gcd | 512 | 51,566 | timeout | 120.269 | 8.584 | 7.14% | 3.48 | +| LoanExam | existing | 73,519 | exhausted | 2.227 | 0.069 | 3.08% | 0 | +| LoanExam | eager | 73,519 | exhausted | 2.003 | 0.248 | 12.38% | 29.85 | +| LoanExam | 128 | 73,519 | exhausted | 2.333 | 0.266 | 11.39% | 27.08 | +| LoanExam | 512 | 73,519 | exhausted | 2.319 | 0.244 | 10.51% | 16.97 | + +For interval 128: + +| Program | Initial goals | Covered goals | Peak queued states | Per-target distance calls | Analysis-thread allocated GB (decimal) | +| --- | ---: | ---: | ---: | ---: | ---: | +| Codec | 273 | 74 | 3,364 | 23,826,680 | 5.952 | +| Guava gcd | 673 | 17 | 3,022 | 43,862,759 | 10.762 | +| LoanExam | 126 | 123 | 4,378 | 886,279 | 0.717 | + +All modes reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. + +Codec whole-analysis times varied substantially between repetitions: interval 128 took 15.007–27.930 seconds, and `existing` took 14.066–20.224 seconds. Its per-target callback count was identical in both repetitions, and its selector share was 29.33–29.62%. Do not treat the median wall-time ratio as a stable speedup estimate. + +Coverage-guided runs produced only 1–26 peeks with distance refresh work per analysis. They do not establish the worst-case cost of continuously discovering new goals or changing coverage while the queue is large. Initial goals are static; dynamic `addTargets` performance is not exercised. + +A useful next change is a caller-supplied batched minimum-distance function over the snapshot, allowing domain-specific caches such as the aggregate `CallStackDistanceCalculator`. A generic per-target callback may depend on arbitrary mutable state, so the selector cannot safely introduce a statement/call-stack-only cache for every caller. The proposed batched API itself has not been implemented or benchmarked here. + +The production selector files match the recorded source hashes throughout the retained runs. The manifest records the final reproducible harness; setup-helper extraction and report formatting were refined between exploratory runs and the repeated Codec runs, without changes to the measured execution loop, selector algorithms or settings. + +Raw data: [results.jsonl](results.jsonl). Full median summary: [summary.csv](summary.csv). Source/settings checksums: [manifest.json](manifest.json). + +Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 16 retained experiment processes completed successfully; Guava analyses ended at the explicit time limit. The experiment is excluded from ordinary JVM test runs. + +## Programs and setup + +- Apache Commons Codec 1.16.0: `org.apache.commons.codec.binary.BinaryCodec.toAsciiBytes(byte[])`. +- Guava 32.0.1-jre: `com.google.common.math.IntMath.gcd(int, int)`. +- Repository sample: `org.usvm.samples.psbenchmarks.LoanExam.getCreditPercent(CreditRequest)`. + +Coverage goals are the statements of all declared, non-abstract, non-native, non-constructor methods in the entry point's class, including static initialization. Coverage is reported by the standard `CoverageStatistics` observer for terminated reachable symbolic states. Class-wide goals intentionally include methods not visited by a particular entry point. + +Each analysis allows 100,000 interpreter steps or 120 seconds, whichever comes first. The timeout is checked between steps, so an in-flight solver operation may overshoot. Reaching the time limit is recorded as `stop=timeout`, not treated as exhaustive analysis. There is no early stop for coverage saturation or coverage stagnation, and no loop-iteration cap. + +Settings: Yices, 250 ms solver/type-operation timeout, implicit exceptions enabled, 1,500-element array bound, soft constraints disabled, 4 GiB Java heap, assertions disabled. Each run uses a fresh test JVM. JacoDB/classpath setup and Gradle startup are outside the analysis timer; initial queue insertion and the complete execution loop are inside it. These are cold-start analysis measurements, not JMH steady-state measurements. + +The recorded host was an Apple M3 Pro (arm64), 18 GiB RAM, Zulu JDK 11.0.32.1+1. Runs were sequential; other host activity was not controlled. + +## Selectors + +- `existing`: the aggregate `CallStackDistanceCalculator` / `WeightedPathSelector` strategy used by `CLOSEST_TO_UNCOVERED`. Coverage reports remove goals from that calculator. Queued state weights are refreshed on normal updates, without a global rebuild. +- `eager`: the original #526 algorithm, refreshing via `drain().toList()` and re-adding states after changed coverage. It uses the same cached-ordering fix and queue implementation as the periodic variants. +- `128` and `512`: the new `CoverageGuidedPathSelector`, using `HashSet` snapshots and the respective refresh interval. + +The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. + +`selectorMs` measures add/update/remove/peek and coverage-report handling, including distance evaluation. `refreshMs` identifies distance work inside `peek` and includes heap rebuilding plus selecting its root. `distanceCalls` counts per-target callbacks in eager/periodic modes, but aggregate calculator calls in `existing`; those are different units of work. `allocatedBytes` is all allocation on the analysis thread, not retained heap, native memory, or allocation attributed specifically to the selector. + +## Reproduction + +Use JDK 11 and cached project dependencies. The runner locates the two library jars in the Gradle cache, or accepts explicit `--codec-jar` and `--guava-jar` paths. It runs Gradle offline and retains per-run logs, raw JSONL measurements and process outcomes. + +```sh +JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ + --cases codec,gcd,loan --modes existing,128,eager,512 \ + --steps 100000 --timeout-ms 120000 --repetitions 1 \ + --output build/coverage-selector-performance + +python3 docs/experiments/coverage-selector-performance/summarize.py \ + build/coverage-selector-performance/results.jsonl +``` + +The experiment is tagged `manual` and excluded from ordinary test runs. The runner's dedicated Gradle task selects only this experiment and disables assertions, avoiding the debug duplicate-element scan in queue insertion. + +Library SHA-256: + +- Commons Codec: `56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d`. +- Guava: `bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5`. diff --git a/docs/experiments/coverage-selector-performance/logback.xml b/docs/experiments/coverage-selector-performance/logback.xml new file mode 100644 index 0000000000..463a09ebff --- /dev/null +++ b/docs/experiments/coverage-selector-performance/logback.xml @@ -0,0 +1,3 @@ + + + diff --git a/docs/experiments/coverage-selector-performance/manifest.json b/docs/experiments/coverage-selector-performance/manifest.json new file mode 100644 index 0000000000..8d44a5cd67 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/manifest.json @@ -0,0 +1,36 @@ +{ + "recordedDate": "2026-10-16", + "productionRevision": "deca5d3bc", + "cpu": "Apple M3 Pro", + "architecture": "arm64", + "memoryBytes": 19327352832, + "java": "Zulu 11.0.32.1+1", + "testHeapBytes": 4294967296, + "settings": { + "stepLimit": 100000, + "timeoutMillis": 120000, + "solver": "YICES", + "solverTimeoutMillis": 250, + "typeTimeoutMillis": 250, + "arrayMaxSize": 1500, + "useSoftConstraints": false, + "assertions": false, + "dynamicGoals": false + }, + "counts": { + "runs": 16, + "interpreterSteps": 1328129 + }, + "librarySha256": { + "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", + "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" + }, + "sourceSha256": { + "usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt": "5905b62b890e94b7390058e46c07fbd9dbb7285521ccc3c65abfceb458ce6b1e", + "usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt": "e55fd73657139ec726574f5ddfca43eccfe8d615520b5c30111146a65019bc32", + "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "602ecb571d50078a628fcc0b8d05ee7259e1a67949040b752db52dcae4b890cd", + "docs/experiments/coverage-selector-performance/run.py": "fa4dfa0bc6a328daf8bcdb736c08a719581bdf8bd46f29e388f03e77a74da0f5", + "docs/experiments/coverage-selector-performance/performance.gradle": "119f882bfba0d7222daaab9a844e7f7498b5a4b82600d9fd108ffe7ecab2bfbf" + }, + "harnessProvenance": "Final reproducible harness; setup-helper extraction and report-formatting refinements occurred during exploratory runs. Production selector sources, measured execution loop and settings were unchanged." +} diff --git a/docs/experiments/coverage-selector-performance/performance.gradle b/docs/experiments/coverage-selector-performance/performance.gradle new file mode 100644 index 0000000000..adc379a57e --- /dev/null +++ b/docs/experiments/coverage-selector-performance/performance.gradle @@ -0,0 +1,19 @@ +gradle.projectsEvaluated { + def jvm = gradle.rootProject.findProject(':usvm-jvm') + if (jvm == null) return + jvm.tasks.register('coverageSelectorPerformance', Test) { + testClassesDirs = jvm.sourceSets.test.output.classesDirs + classpath = jvm.sourceSets.test.runtimeClasspath + dependsOn(jvm.tasks.named('testClasses')) + useJUnitPlatform { includeTags 'manual' } + filter { includeTestsMatching 'org.usvm.experiments.CoverageSelectorPerformanceTest' } + enableAssertions = false + maxHeapSize = '4g' + def logging = jvm.rootProject.file('docs/experiments/coverage-selector-performance/logback.xml') + jvmArgs "-Dlogback.configurationFile=${logging.absolutePath}" + ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition'].each { key -> + systemProperty("coverage.perf.${key}", jvm.findProperty("coveragePerf${key.capitalize()}")) + } + outputs.upToDateWhen { false } + } +} diff --git a/docs/experiments/coverage-selector-performance/results.jsonl b/docs/experiments/coverage-selector-performance/results.jsonl new file mode 100644 index 0000000000..ff93c66817 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/results.jsonl @@ -0,0 +1,16 @@ +{"program":"gcd","mode":"existing","steps":61538,"stop":"timeout","elapsedMs":120167.436333,"selectorMs":235.437487,"refreshMs":0.0,"refreshes":0,"distanceCalls":64765,"peakQueue":3227,"pending":3227,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":856651016,"gcCount":5,"gcMs":449,"repetition":1} +{"program":"gcd","mode":"128","steps":63808,"stop":"timeout","elapsedMs":120189.039625,"selectorMs":14249.244039,"refreshMs":5.728084,"refreshes":2,"distanceCalls":43862759,"peakQueue":3022,"pending":3022,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":10761566008,"gcCount":10,"gcMs":304,"repetition":1} +{"program":"gcd","mode":"eager","repetition":1,"steps":57141,"stop":"timeout","elapsedMs":120024.317916,"selectorMs":10822.395495,"refreshMs":5.358375,"refreshes":3,"distanceCalls":39557171,"peakQueue":3124,"pending":3124,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":9568604488,"gcCount":12,"gcMs":435} +{"program":"gcd","mode":"512","repetition":1,"steps":51566,"stop":"timeout","elapsedMs":120269.191542,"selectorMs":8584.144519,"refreshMs":3.480708,"refreshes":1,"distanceCalls":35593159,"peakQueue":2655,"pending":2655,"goals":673,"covered":17,"coverageReports":17,"terminated":5,"allocatedBytes":8698358712,"gcCount":9,"gcMs":175} +{"program":"loan","mode":"existing","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2227.168709,"selectorMs":68.547639,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":513509984,"gcCount":2,"gcMs":26} +{"program":"loan","mode":"128","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2332.535792,"selectorMs":265.641903,"refreshMs":27.08429,"refreshes":18,"distanceCalls":886279,"peakQueue":4378,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":716986848,"gcCount":2,"gcMs":24} +{"program":"loan","mode":"eager","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2003.228,"selectorMs":248.054217,"refreshMs":29.854126,"refreshes":26,"distanceCalls":944047,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":734125672,"gcCount":2,"gcMs":36} +{"program":"loan","mode":"512","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2319.290542,"selectorMs":243.802455,"refreshMs":16.965501,"refreshes":12,"distanceCalls":760175,"peakQueue":4360,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":701963040,"gcCount":2,"gcMs":19} +{"program":"codec","mode":"existing","repetition":1,"steps":100000,"stop":"steps","elapsedMs":14065.793084,"selectorMs":103.386872,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2071408672,"gcCount":5,"gcMs":327} +{"program":"codec","mode":"128","repetition":1,"steps":100000,"stop":"steps","elapsedMs":27929.542917,"selectorMs":8192.190751,"refreshMs":93.001749,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952381656,"gcCount":10,"gcMs":367} +{"program":"codec","mode":"eager","repetition":1,"steps":100000,"stop":"steps","elapsedMs":17828.648083,"selectorMs":5027.923518,"refreshMs":13.558208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5478178352,"gcCount":12,"gcMs":287} +{"program":"codec","mode":"512","repetition":1,"steps":100000,"stop":"steps","elapsedMs":18655.336042,"selectorMs":6038.100888,"refreshMs":193.471708,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":6332651112,"gcCount":9,"gcMs":260} +{"program":"codec","mode":"512","repetition":2,"steps":100000,"stop":"steps","elapsedMs":19563.939083,"selectorMs":5645.342032,"refreshMs":100.26425,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5946781568,"gcCount":8,"gcMs":210} +{"program":"codec","mode":"eager","repetition":2,"steps":100000,"stop":"steps","elapsedMs":12771.742584,"selectorMs":3540.902094,"refreshMs":14.152208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5479713280,"gcCount":8,"gcMs":186} +{"program":"codec","mode":"128","repetition":2,"steps":100000,"stop":"steps","elapsedMs":15007.014042,"selectorMs":4445.708042,"refreshMs":87.639833,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952287184,"gcCount":7,"gcMs":173} +{"program":"codec","mode":"existing","repetition":2,"steps":100000,"stop":"steps","elapsedMs":20224.028875,"selectorMs":165.022394,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2078604128,"gcCount":6,"gcMs":202} diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py new file mode 100644 index 0000000000..fea8bdeb59 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/run.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Run isolated JVM analyses; keep successes, failures and timeouts in the output directory.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import time + + +def cached_jar(group, artifact, version): + cache = Path.home() / '.gradle/caches/modules-2/files-2.1' + matches = list((cache / group / artifact / version).glob(f'*/{artifact}-{version}.jar')) + if len(matches) != 1: + raise RuntimeError(f'Pass the jar explicitly: {group}:{artifact}:{version}') + return matches[0] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cases', default='codec,gcd,loan') + parser.add_argument('--modes', default='existing,128,eager,512') + parser.add_argument('--steps', type=int, default=100000) + parser.add_argument('--timeout-ms', type=int, default=120000) + parser.add_argument('--repetitions', type=int, default=2) + parser.add_argument('--output', type=Path) + parser.add_argument('--codec-jar', type=Path) + parser.add_argument('--guava-jar', type=Path) + args = parser.parse_args() + + experiment = Path(__file__).resolve().parent + root = experiment.parents[2] + output = (args.output or root / 'build/coverage-selector-performance').resolve() + output.mkdir(parents=True, exist_ok=True) + codec = args.codec_jar or cached_jar('commons-codec', 'commons-codec', '1.16.0') + guava = args.guava_jar or cached_jar('com.google.guava', 'guava', '32.0.1-jre') + jars = os.pathsep.join(str(path.resolve()) for path in [codec, guava]) + cases = args.cases.split(',') + modes = args.modes.split(',') + results_file = output / 'results.jsonl' + outcomes_file = output / 'outcomes.jsonl' + + for repetition in range(1, args.repetitions + 1): + # Alternate mode order to reduce a fixed ordering effect. Runs remain sequential. + run_modes = modes if repetition % 2 else list(reversed(modes)) + for case in cases: + for mode in run_modes: + label = f'{case}-{mode}-r{repetition}' + print(f'START {label}', flush=True) + log_file = output / f'{label}.log' + command = [ + str(root / 'gradlew'), ':usvm-jvm:coverageSelectorPerformance', + '-I', str(experiment / 'performance.gradle'), + f'-PcoveragePerfCase={case}', f'-PcoveragePerfMode={mode}', + f'-PcoveragePerfSteps={args.steps}', + f'-PcoveragePerfTimeoutMillis={args.timeout_ms}', + f'-PcoveragePerfRepetition={repetition}', + f'-PcoveragePerfResults={results_file}', f'-PcoveragePerfJars={jars}', + '--offline', '--console=plain', + ] + started = time.monotonic() + outcome = dict(program=case, mode=mode, repetition=repetition, log=log_file.name) + try: + with log_file.open('w') as log: + process = subprocess.run( + command, cwd=root, env=os.environ.copy(), stdout=log, + stderr=subprocess.STDOUT, timeout=args.timeout_ms / 1000 + 180, + ) + outcome['exitCode'] = process.returncode + outcome['status'] = 'completed' if process.returncode == 0 else 'failed' + except subprocess.TimeoutExpired: + outcome['status'] = 'process-timeout' + outcome['wallSeconds'] = round(time.monotonic() - started, 3) + with outcomes_file.open('a') as outcomes: + outcomes.write(json.dumps(outcome) + '\n') + print(json.dumps(outcome), flush=True) + if outcome['status'] != 'completed': + print(log_file.read_text()[-4000:], flush=True) + + +if __name__ == '__main__': + main() diff --git a/docs/experiments/coverage-selector-performance/summarize.py b/docs/experiments/coverage-selector-performance/summarize.py new file mode 100644 index 0000000000..6b0a63e766 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/summarize.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Write per-program/per-mode medians from benchmark JSONL to CSV.""" + +import argparse +import csv +import json +from collections import defaultdict +from pathlib import Path +from statistics import median +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('results', type=Path) + args = parser.parse_args() + groups = defaultdict(list) + for line in args.results.read_text().splitlines(): + row = json.loads(line) + groups[row['program'], row['mode']].append(row) + + fields = ['program', 'mode', 'runs', 'stops', 'steps', 'elapsedMs', 'selectorMs', + 'selectorPercent', 'refreshMs', 'refreshes', 'distanceCalls', 'peakQueue', + 'covered', 'goals', 'allocatedBytes', 'gcMs'] + writer = csv.DictWriter(sys.stdout, fieldnames=fields, lineterminator='\n') + writer.writeheader() + for (program, mode), rows in sorted(groups.items()): + summary = dict(program=program, mode=mode, runs=len(rows)) + summary['stops'] = ','.join(sorted({row['stop'] for row in rows})) + for field in fields[4:]: + if field == 'selectorPercent': + summary[field] = median(100 * row['selectorMs'] / row['elapsedMs'] for row in rows) + else: + summary[field] = median(row[field] for row in rows) + writer.writerow(summary) + + +if __name__ == '__main__': + main() diff --git a/docs/experiments/coverage-selector-performance/summary.csv b/docs/experiments/coverage-selector-performance/summary.csv new file mode 100644 index 0000000000..0baf0df1c0 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/summary.csv @@ -0,0 +1,13 @@ +program,mode,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs +codec,128,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 +codec,512,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 +codec,eager,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 +codec,existing,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 +gcd,128,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 +gcd,512,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 +gcd,eager,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 +gcd,existing,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 +loan,128,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 +loan,512,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 +loan,eager,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 +loan,existing,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt new file mode 100644 index 0000000000..829671d918 --- /dev/null +++ b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt @@ -0,0 +1,336 @@ +package org.usvm.experiments + +import kotlinx.coroutines.runBlocking +import org.jacodb.api.jvm.JcClasspath +import org.jacodb.api.jvm.JcMethod +import org.jacodb.api.jvm.cfg.JcInst +import org.jacodb.api.jvm.ext.findClass +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test +import org.usvm.SolverType +import org.usvm.UMachine +import org.usvm.UMachineOptions +import org.usvm.UPathSelector +import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.machine.JcApplicationGraph +import org.usvm.machine.JcComponents +import org.usvm.machine.JcContext +import org.usvm.machine.JcMachineOptions +import org.usvm.machine.JcTypeSystem +import org.usvm.machine.interpreter.JcInterpreter +import org.usvm.machine.state.JcState +import org.usvm.ps.BfsPathSelector +import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.WeightedPathSelector +import org.usvm.samples.JacoDBContainer +import org.usvm.statistics.CompositeUMachineObserver +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.UMachineObserver +import org.usvm.statistics.distances.CallStackDistanceCalculator +import org.usvm.statistics.distances.CfgStatisticsImpl +import org.usvm.statistics.distances.MultiTargetDistanceCalculator +import org.usvm.stopstrategies.StopStrategy +import org.usvm.util.originalInst +import java.io.File +import java.lang.management.ManagementFactory +import kotlin.time.Duration.Companion.milliseconds + +/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ +@Tag("manual") +class CoverageSelectorPerformanceTest { + @Test + fun runProgram() { + val program = System.getProperty("coverage.perf.case", "codec") + val mode = System.getProperty("coverage.perf.mode", "128") + val steps = System.getProperty("coverage.perf.steps", "100000").toLong() + val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() + val resultFile = File(System.getProperty("coverage.perf.results")) + val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) + val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) + val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) + try { + val (className, methodName) = when (program) { + "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" + "gcd" -> "com.google.common.math.IntMath" to "gcd" + "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" + else -> error("Unknown program: $program") + } + val cls = container.cp.findClass(className) + val method = cls.declaredMethods.single { it.name == methodName } + val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } + PerformanceMachine(container.cp).use { machine -> + val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) + resultFile.parentFile.mkdirs() + resultFile.appendText(result + "\n") + } + } finally { + runBlocking { + container.cp.close() + container.db.close() + } + } + } +} + +private class PerformanceMachine(cp: JcClasspath) : UMachine() { + override val options = UMachineOptions( + solverType = SolverType.YICES, + solverTimeout = 250.milliseconds, + typeOperationsTimeout = 250.milliseconds, + throwExceptionOnStepFailure = true, + useSoftConstraints = false, + ) + private val graph = JcApplicationGraph(cp) + private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) + private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) + + fun measure( + program: String, + mode: String, + method: JcMethod, + methodsToCover: List, + stepLimit: Long, + timeoutMillis: Long, + ): String { + val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) + val goals = coverage.getUncoveredStatements() + val metrics = Metrics() + val (base, markCovered) = createGuidance(mode, goals, metrics) + val selector = MeasuredSelector(base, metrics) + coverage.addOnCoveredObserver { _, _, statement -> + val started = System.nanoTime() + markCovered(statement) + metrics.selectorNanos += System.nanoTime() - started + metrics.coverageReports++ + } + val observer = metrics.observer() + val initial = interpreter.getInitialState(method) + val gcCountBefore = gcCount() + val gcMillisBefore = gcMillis() + val allocatedBefore = allocatedBytes() + val started = System.nanoTime() + var stop = "exhausted" + selector.add(states = listOf(initial)) + run( + interpreter = interpreter, + pathSelector = selector, + observer = CompositeUMachineObserver(coverage, observer), + isStateTerminated = { state -> state.callStack.isEmpty() }, + stopStrategy = StopStrategy { + val elapsedMillis = (System.nanoTime() - started) / 1_000_000 + when { + metrics.steps >= stepLimit -> { + stop = "steps" + true + } + elapsedMillis >= timeoutMillis -> { + stop = "timeout" + true + } + else -> { + false + } + } + }, + ) + val elapsedNanos = System.nanoTime() - started + val allocated = allocatedBytes() - allocatedBefore + val totals = AnalysisTotals( + elapsedNanos = elapsedNanos, + allocatedBytes = allocated, + gcCount = gcCount() - gcCountBefore, + gcMillis = gcMillis() - gcMillisBefore, + ) + return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) + } + + private fun createGuidance( + mode: String, + goals: Collection, + metrics: Metrics, + ): Pair, (JcInst) -> Unit> { + val cfg = CfgStatisticsImpl(graph) + val calculators = MultiTargetDistanceCalculator { target -> + CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) + } + val distance: (JcState, JcInst) -> UInt = { state, target -> + metrics.distanceCalls++ + calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) + } + val markCovered: (JcInst) -> Unit + val base: UPathSelector + if (mode == "eager") { + val eager = EagerCoverageSelector(goals, distance) + base = eager + markCovered = { eager.markCovered(it) } + } else if (mode == "existing") { + val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) + base = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state: JcState -> + metrics.distanceCalls++ + aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) + }, + ) + markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } + } else if (mode == "bfs") { + base = BfsPathSelector() + markCovered = {} + } else { + val periodic = CoverageGuidedPathSelector( + targets = goals, + distance = distance, + priorityRefreshInterval = mode.toInt(), + ) + base = periodic + markCovered = { periodic.markCovered(targets = listOf(it)) } + } + return base to markCovered + } + + override fun close() = components.close() +} + +private class Metrics { + var steps = 0L + var terminated = 0L + var selectorNanos = 0L + var refreshNanos = 0L + var refreshes = 0L + var distanceCalls = 0L + var coverageReports = 0L + var queueSize = 0 + var peakQueue = 0 +} + +private class MeasuredSelector( + private val delegate: UPathSelector, + private val metrics: Metrics, +) : UPathSelector { + override fun isEmpty(): Boolean = delegate.isEmpty() + + override fun peek(): JcState { + val beforeCalls = metrics.distanceCalls + val started = System.nanoTime() + val result = delegate.peek() + val elapsed = System.nanoTime() - started + metrics.selectorNanos += elapsed + if (metrics.distanceCalls > beforeCalls) { + metrics.refreshes++ + metrics.refreshNanos += elapsed + } + return result + } + + override fun add(states: Collection) { + val started = System.nanoTime() + delegate.add(states) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize += states.size + metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) + } + + override fun update(state: JcState) { + val started = System.nanoTime() + delegate.update(state) + metrics.selectorNanos += System.nanoTime() - started + } + + override fun remove(state: JcState) { + val started = System.nanoTime() + delegate.remove(state) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize-- + } +} + +/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ +private class EagerCoverageSelector( + targets: Collection, + distance: (JcState, JcInst) -> UInt, +) : UPathSelector { + private val pending = targets.toMutableSet() + private var dirty = false + private val delegate = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, + ) + + fun markCovered(target: JcInst) { + if (pending.remove(target)) dirty = true + } + + override fun isEmpty(): Boolean = delegate.isEmpty() + override fun peek(): JcState { + if (dirty) { + val states = delegate.drain().toList() + delegate.add(states) + dirty = false + } + return delegate.peek() + } + override fun add(states: Collection) = delegate.add(states) + override fun update(state: JcState) = delegate.update(state) + override fun remove(state: JcState) = delegate.remove(state) +} + +private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionCount.coerceAtLeast(minimumValue = 0) +} + +private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionTime.coerceAtLeast(minimumValue = 0) +} +private fun allocatedBytes(): Long { + val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean + return bean.getThreadAllocatedBytes(Thread.currentThread().id) +} + +private data class AnalysisTotals( + val elapsedNanos: Long, + val allocatedBytes: Long, + val gcCount: Long, + val gcMillis: Long, +) + +private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { + override fun onStatePeeked(state: JcState) { + steps++ + } + + override fun onStateTerminated(state: JcState, stateReachable: Boolean) { + terminated++ + } +} + +private fun Metrics.report( + program: String, + mode: String, + goals: Int, + covered: Int, + stop: String, + totals: AnalysisTotals, +): String { + val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() + return """ + {"program":"$program", + "mode":"$mode", + "repetition":$repetition, + "steps":$steps, + "stop":"$stop", + "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, + "selectorMs":${selectorNanos / 1_000_000.0}, + "refreshMs":${refreshNanos / 1_000_000.0}, + "refreshes":$refreshes, + "distanceCalls":$distanceCalls, + "peakQueue":$peakQueue, + "pending":$queueSize, + "goals":$goals, + "covered":$covered, + "coverageReports":$coverageReports, + "terminated":$terminated, + "allocatedBytes":${totals.allocatedBytes}, + "gcCount":${totals.gcCount}, + "gcMs":${totals.gcMillis}} + """.trimIndent().replace("\n", "") +} From 0719f4796112dfbe124be735b206af5d4eb8af5f Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 17:25:35 +0300 Subject: [PATCH 07/10] Compare coverage guidance with random selection baselines --- .../coverage-selector-performance/README.md | 45 +++++++++++++++++-- .../manifest.json | 18 +++++--- .../performance.gradle | 2 +- .../results.jsonl | 12 +++++ .../coverage-selector-performance/run.py | 7 +-- .../summarize.py | 16 ++++--- .../coverage-selector-performance/summary.csv | 38 ++++++++++------ .../CoverageSelectorPerformanceTest.kt | 34 +++++++++++++- 8 files changed, 136 insertions(+), 36 deletions(-) diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md index bb43ada212..fa9cbe8ac9 100644 --- a/docs/experiments/coverage-selector-performance/README.md +++ b/docs/experiments/coverage-selector-performance/README.md @@ -6,7 +6,7 @@ This experiment drives the production `JcInterpreter` through `UMachine.run`, wi The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. -The retained dataset contains 16 runs and 1,328,129 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. +The distance-guidance comparison below contains 16 runs and 1,328,129 interpreter steps. The random comparison adds 12 runs; the complete retained dataset contains 28 runs and 2,422,205 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. | Program | Mode | Steps | Stop | Analysis seconds | Selector seconds | Selector share | Refresh ms | | --- | --- | ---: | --- | ---: | ---: | ---: | ---: | @@ -31,7 +31,7 @@ For interval 128: | Guava gcd | 673 | 17 | 3,022 | 43,862,759 | 10.762 | | LoanExam | 126 | 123 | 4,378 | 886,279 | 0.717 | -All modes reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. +All distance-guidance modes in the preceding table reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. Codec whole-analysis times varied substantially between repetitions: interval 128 took 15.007–27.930 seconds, and `existing` took 14.066–20.224 seconds. Its per-target callback count was identical in both repetitions, and its selector share was 29.33–29.62%. Do not treat the median wall-time ratio as a stable speedup estimate. @@ -43,7 +43,36 @@ The production selector files match the recorded source hashes throughout the re Raw data: [results.jsonl](results.jsonl). Full median summary: [summary.csv](summary.csv). Source/settings checksums: [manifest.json](manifest.json). -Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 16 retained experiment processes completed successfully; Guava analyses ended at the explicit time limit. The experiment is excluded from ordinary JVM test runs. +Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 28 retained experiment processes completed successfully. The four distance-guidance Guava analyses ended at the explicit time limit; the random Guava runs reached 100,000 steps. The experiment is excluded from ordinary JVM test runs. + + +## Random baselines + +Two additional strategies were measured once for each selector RNG seed, 0 and 1, with the same step/time limits and production interpreter: + +- `random-path`: the actual `RandomTreePathSelector.fromRoot` used by `RANDOM_PATH`; it samples branches of the execution tree and traverses that tree on selection. +- `uniform`: the existing `RandomizedPriorityCollection` with constant weight 1.0 in a `WeightedPathSelector`, giving uniform random selection among queued states. It performs no goal-distance calculations and does not traverse the execution tree. + +| Program | Mode | Seed | Steps | Analysis seconds | Selector seconds | Covered goals | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| codec | random-path | 0 | 100,000 | 13.359 | 6.585 | 74 | +| codec | random-path | 1 | 100,000 | 7.283 | 3.991 | 74 | +| codec | uniform | 0 | 100,000 | 3.937 | 0.339 | 74 | +| codec | uniform | 1 | 100,000 | 3.430 | 0.315 | 74 | +| gcd | random-path | 0 | 100,000 | 19.171 | 3.411 | 28 | +| gcd | random-path | 1 | 100,000 | 18.281 | 3.186 | 28 | +| gcd | uniform | 0 | 100,000 | 10.153 | 0.572 | 28 | +| gcd | uniform | 1 | 100,000 | 7.328 | 0.351 | 28 | +| loan | random-path | 0 | 73,519 | 3.868 | 0.883 | 123 | +| loan | random-path | 1 | 73,519 | 5.003 | 1.155 | 123 | +| loan | uniform | 0 | 73,519 | 3.046 | 0.254 | 123 | +| loan | uniform | 1 | 73,519 | 2.745 | 0.224 | 123 | + +Uniform random was a strong inexpensive baseline on both library methods in these runs. Codec reached the same covered-goal count (74) in 3.430–3.937 seconds, versus a 21.468-second median for interval 128. Guava uniform random reached 100,000 steps and 28 covered goals in 7.328–10.153 seconds; interval 128 reached 63,808 steps and 17 covered goals at the 120-second timeout. LoanExam is a counterexample: uniform random took 2.745–3.046 seconds, versus 2.333 seconds for interval 128, with the same 73,519 steps and 123 covered goals. + +The standard `RANDOM_PATH` strategy was itself relatively expensive on Codec: 3.991–6.585 seconds spent in its selector, versus 0.315–0.339 seconds for uniform random. A random strategy's data structure and selection distribution matter. + +Different exploration order changes work outside the selector as well. Guava's wall-time difference cannot be explained by selector overhead alone; its interpreter/solver cost has not been separately profiled. Two seeds and three programs support using cheap uniform random as a baseline, not a universal claim that it dominates all coverage guidance. Goal counts do not establish equality of covered-goal sets or concrete replay coverage. ## Programs and setup @@ -65,7 +94,7 @@ The recorded host was an Apple M3 Pro (arm64), 18 GiB RAM, Zulu JDK 11.0.32.1+1. - `eager`: the original #526 algorithm, refreshing via `drain().toList()` and re-adding states after changed coverage. It uses the same cached-ordering fix and queue implementation as the periodic variants. - `128` and `512`: the new `CoverageGuidedPathSelector`, using `HashSet` snapshots and the respective refresh interval. -The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. +For the random strategies, the selector RNG seed is recorded separately for every row and CSV group. The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. `selectorMs` measures add/update/remove/peek and coverage-report handling, including distance evaluation. `refreshMs` identifies distance work inside `peek` and includes heap rebuilding plus selecting its root. `distanceCalls` counts per-target callbacks in eager/periodic modes, but aggregate calculator calls in `existing`; those are different units of work. `allocatedBytes` is all allocation on the analysis thread, not retained heap, native memory, or allocation attributed specifically to the selector. @@ -79,6 +108,14 @@ JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/ --steps 100000 --timeout-ms 120000 --repetitions 1 \ --output build/coverage-selector-performance +JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ + --cases codec,gcd,loan --modes random-path,uniform --seed 0 --repetitions 1 \ + --output build/coverage-selector-performance + +JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ + --cases codec,gcd,loan --modes random-path,uniform --seed 1 --repetitions 1 \ + --output build/coverage-selector-performance + python3 docs/experiments/coverage-selector-performance/summarize.py \ build/coverage-selector-performance/results.jsonl ``` diff --git a/docs/experiments/coverage-selector-performance/manifest.json b/docs/experiments/coverage-selector-performance/manifest.json index 8d44a5cd67..1e41f1c164 100644 --- a/docs/experiments/coverage-selector-performance/manifest.json +++ b/docs/experiments/coverage-selector-performance/manifest.json @@ -15,11 +15,16 @@ "arrayMaxSize": 1500, "useSoftConstraints": false, "assertions": false, - "dynamicGoals": false + "dynamicGoals": false, + "randomSeeds": [ + 0, + 1 + ] }, "counts": { - "runs": 16, - "interpreterSteps": 1328129 + "runs": 28, + "interpreterSteps": 2422205, + "peakQueue": 4921 }, "librarySha256": { "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", @@ -28,9 +33,10 @@ "sourceSha256": { "usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt": "5905b62b890e94b7390058e46c07fbd9dbb7285521ccc3c65abfceb458ce6b1e", "usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt": "e55fd73657139ec726574f5ddfca43eccfe8d615520b5c30111146a65019bc32", - "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "602ecb571d50078a628fcc0b8d05ee7259e1a67949040b752db52dcae4b890cd", - "docs/experiments/coverage-selector-performance/run.py": "fa4dfa0bc6a328daf8bcdb736c08a719581bdf8bd46f29e388f03e77a74da0f5", - "docs/experiments/coverage-selector-performance/performance.gradle": "119f882bfba0d7222daaab9a844e7f7498b5a4b82600d9fd108ffe7ecab2bfbf" + "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "42c2c518e157ff8edca81044c7561c6832a9868eef0ef0514c223c9ee05209a2", + "docs/experiments/coverage-selector-performance/run.py": "9c6affa7309e0a6bbacfcf6fd5fdb005ee63817aa3d1a8103a3caf635c667a50", + "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0", + "docs/experiments/coverage-selector-performance/summarize.py": "67b695a12348503a3700d26b3fc938337ff9215a392353060ece05713dbf2fa2" }, "harnessProvenance": "Final reproducible harness; setup-helper extraction and report-formatting refinements occurred during exploratory runs. Production selector sources, measured execution loop and settings were unchanged." } diff --git a/docs/experiments/coverage-selector-performance/performance.gradle b/docs/experiments/coverage-selector-performance/performance.gradle index adc379a57e..8a301e7f07 100644 --- a/docs/experiments/coverage-selector-performance/performance.gradle +++ b/docs/experiments/coverage-selector-performance/performance.gradle @@ -11,7 +11,7 @@ gradle.projectsEvaluated { maxHeapSize = '4g' def logging = jvm.rootProject.file('docs/experiments/coverage-selector-performance/logback.xml') jvmArgs "-Dlogback.configurationFile=${logging.absolutePath}" - ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition'].each { key -> + ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition', 'seed'].each { key -> systemProperty("coverage.perf.${key}", jvm.findProperty("coveragePerf${key.capitalize()}")) } outputs.upToDateWhen { false } diff --git a/docs/experiments/coverage-selector-performance/results.jsonl b/docs/experiments/coverage-selector-performance/results.jsonl index ff93c66817..1fe7bae02f 100644 --- a/docs/experiments/coverage-selector-performance/results.jsonl +++ b/docs/experiments/coverage-selector-performance/results.jsonl @@ -14,3 +14,15 @@ {"program":"codec","mode":"eager","repetition":2,"steps":100000,"stop":"steps","elapsedMs":12771.742584,"selectorMs":3540.902094,"refreshMs":14.152208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5479713280,"gcCount":8,"gcMs":186} {"program":"codec","mode":"128","repetition":2,"steps":100000,"stop":"steps","elapsedMs":15007.014042,"selectorMs":4445.708042,"refreshMs":87.639833,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952287184,"gcCount":7,"gcMs":173} {"program":"codec","mode":"existing","repetition":2,"steps":100000,"stop":"steps","elapsedMs":20224.028875,"selectorMs":165.022394,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2078604128,"gcCount":6,"gcMs":202} +{"program":"codec","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":13359.287291,"selectorMs":6585.102705,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4615,"pending":4615,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513024960,"gcCount":7,"gcMs":539} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3936.713375,"selectorMs":338.87798,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1080632408,"gcCount":3,"gcMs":62} +{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":19170.84275,"selectorMs":3410.622702,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4921,"pending":4920,"goals":673,"covered":28,"coverageReports":28,"terminated":895,"allocatedBytes":2143921128,"gcCount":3,"gcMs":41} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10152.718417,"selectorMs":572.30041,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1025193416,"gcCount":5,"gcMs":409} +{"program":"loan","mode":"random-path","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3868.197,"selectorMs":882.931681,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1527,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":881496912,"gcCount":2,"gcMs":50} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3045.632291,"selectorMs":253.601349,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743016424,"gcCount":3,"gcMs":98} +{"program":"codec","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7282.688708,"selectorMs":3990.851612,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4562,"pending":4562,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513769624,"gcCount":3,"gcMs":111} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":3429.624834,"selectorMs":314.7182,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4780,"pending":4780,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1087211024,"gcCount":4,"gcMs":122} +{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":18280.874209,"selectorMs":3186.388872,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4898,"pending":4897,"goals":673,"covered":28,"coverageReports":28,"terminated":911,"allocatedBytes":2151045424,"gcCount":3,"gcMs":32} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7327.864792,"selectorMs":351.211791,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4444,"pending":4440,"goals":673,"covered":28,"coverageReports":28,"terminated":1521,"allocatedBytes":1029454208,"gcCount":4,"gcMs":259} +{"program":"loan","mode":"random-path","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":5003.377792,"selectorMs":1155.413364,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1513,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":878046240,"gcCount":2,"gcMs":39} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":2744.655583,"selectorMs":224.198052,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":2902,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":747822944,"gcCount":2,"gcMs":26} diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py index fea8bdeb59..590a47f71e 100644 --- a/docs/experiments/coverage-selector-performance/run.py +++ b/docs/experiments/coverage-selector-performance/run.py @@ -24,6 +24,7 @@ def main(): parser.add_argument('--steps', type=int, default=100000) parser.add_argument('--timeout-ms', type=int, default=120000) parser.add_argument('--repetitions', type=int, default=2) + parser.add_argument('--seed', type=int, default=0) parser.add_argument('--output', type=Path) parser.add_argument('--codec-jar', type=Path) parser.add_argument('--guava-jar', type=Path) @@ -46,7 +47,7 @@ def main(): run_modes = modes if repetition % 2 else list(reversed(modes)) for case in cases: for mode in run_modes: - label = f'{case}-{mode}-r{repetition}' + label = f'{case}-{mode}-s{args.seed}-r{repetition}' print(f'START {label}', flush=True) log_file = output / f'{label}.log' command = [ @@ -55,12 +56,12 @@ def main(): f'-PcoveragePerfCase={case}', f'-PcoveragePerfMode={mode}', f'-PcoveragePerfSteps={args.steps}', f'-PcoveragePerfTimeoutMillis={args.timeout_ms}', - f'-PcoveragePerfRepetition={repetition}', + f'-PcoveragePerfRepetition={repetition}', f'-PcoveragePerfSeed={args.seed}', f'-PcoveragePerfResults={results_file}', f'-PcoveragePerfJars={jars}', '--offline', '--console=plain', ] started = time.monotonic() - outcome = dict(program=case, mode=mode, repetition=repetition, log=log_file.name) + outcome = dict(program=case, mode=mode, repetition=repetition, randomSeed=args.seed, log=log_file.name) try: with log_file.open('w') as log: process = subprocess.run( diff --git a/docs/experiments/coverage-selector-performance/summarize.py b/docs/experiments/coverage-selector-performance/summarize.py index 6b0a63e766..5ba0af99d9 100644 --- a/docs/experiments/coverage-selector-performance/summarize.py +++ b/docs/experiments/coverage-selector-performance/summarize.py @@ -17,17 +17,19 @@ def main(): groups = defaultdict(list) for line in args.results.read_text().splitlines(): row = json.loads(line) - groups[row['program'], row['mode']].append(row) + seed = row.get('randomSeed') if row['mode'] in ['random-path', 'uniform'] else None + groups[row['program'], row['mode'], seed].append(row) - fields = ['program', 'mode', 'runs', 'stops', 'steps', 'elapsedMs', 'selectorMs', - 'selectorPercent', 'refreshMs', 'refreshes', 'distanceCalls', 'peakQueue', - 'covered', 'goals', 'allocatedBytes', 'gcMs'] + metrics = ['steps', 'elapsedMs', 'selectorMs', 'selectorPercent', 'refreshMs', + 'refreshes', 'distanceCalls', 'peakQueue', 'covered', 'goals', + 'allocatedBytes', 'gcMs'] + fields = ['program', 'mode', 'randomSeed', 'runs', 'stops', *metrics] writer = csv.DictWriter(sys.stdout, fieldnames=fields, lineterminator='\n') writer.writeheader() - for (program, mode), rows in sorted(groups.items()): - summary = dict(program=program, mode=mode, runs=len(rows)) + for (program, mode, seed), rows in sorted(groups.items()): + summary = dict(program=program, mode=mode, randomSeed=seed, runs=len(rows)) summary['stops'] = ','.join(sorted({row['stop'] for row in rows})) - for field in fields[4:]: + for field in metrics: if field == 'selectorPercent': summary[field] = median(100 * row['selectorMs'] / row['elapsedMs'] for row in rows) else: diff --git a/docs/experiments/coverage-selector-performance/summary.csv b/docs/experiments/coverage-selector-performance/summary.csv index 0baf0df1c0..fc5876db60 100644 --- a/docs/experiments/coverage-selector-performance/summary.csv +++ b/docs/experiments/coverage-selector-performance/summary.csv @@ -1,13 +1,25 @@ -program,mode,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs -codec,128,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 -codec,512,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 -codec,eager,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 -codec,existing,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 -gcd,128,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 -gcd,512,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 -gcd,eager,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 -gcd,existing,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 -loan,128,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 -loan,512,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 -loan,eager,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 -loan,existing,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 +program,mode,randomSeed,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs +codec,128,,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 +codec,512,,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 +codec,eager,,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 +codec,existing,,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 +codec,random-path,0,1,steps,100000,13359.287291,6585.102705,49.29232047757748,0.0,0,0,4615,74,273,2513024960,539 +codec,random-path,1,1,steps,100000,7282.688708,3990.851612,54.79915141252801,0.0,0,0,4562,74,273,2513769624,111 +codec,uniform,0,1,steps,100000,3936.713375,338.87798,8.608144604888842,0.0,0,0,4790,74,273,1080632408,62 +codec,uniform,1,1,steps,100000,3429.624834,314.7182,9.176461427500849,0.0,0,0,4780,74,273,1087211024,122 +gcd,128,,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 +gcd,512,,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 +gcd,eager,,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 +gcd,existing,,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 +gcd,random-path,0,1,steps,100000,19170.84275,3410.622702,17.79067694872204,0.0,0,0,4921,28,673,2143921128,41 +gcd,random-path,1,1,steps,100000,18280.874209,3186.388872,17.430177767052758,0.0,0,0,4898,28,673,2151045424,32 +gcd,uniform,0,1,steps,100000,10152.718417,572.30041,5.636917980919514,0.0,0,0,4338,28,673,1025193416,409 +gcd,uniform,1,1,steps,100000,7327.864792,351.211791,4.7928257544192965,0.0,0,0,4444,28,673,1029454208,259 +loan,128,,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 +loan,512,,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 +loan,eager,,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 +loan,existing,,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 +loan,random-path,0,1,exhausted,73519,3868.197,882.931681,22.82540628101413,0.0,0,0,1527,123,126,881496912,50 +loan,random-path,1,1,exhausted,73519,5003.377792,1155.413364,23.09266683494125,0.0,0,0,1513,123,126,878046240,39 +loan,uniform,0,1,exhausted,73519,3045.632291,253.601349,8.326722492055428,0.0,0,0,3049,123,126,743016424,98 +loan,uniform,1,1,exhausted,73519,2744.655583,224.198052,8.168531359222275,0.0,0,0,2902,123,126,747822944,26 diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt index 829671d918..df0da45000 100644 --- a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt +++ b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt @@ -12,6 +12,7 @@ import org.usvm.UMachine import org.usvm.UMachineOptions import org.usvm.UPathSelector import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.algorithms.RandomizedPriorityCollection import org.usvm.machine.JcApplicationGraph import org.usvm.machine.JcComponents import org.usvm.machine.JcContext @@ -21,6 +22,7 @@ import org.usvm.machine.interpreter.JcInterpreter import org.usvm.machine.state.JcState import org.usvm.ps.BfsPathSelector import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.RandomTreePathSelector import org.usvm.ps.WeightedPathSelector import org.usvm.samples.JacoDBContainer import org.usvm.statistics.CompositeUMachineObserver @@ -33,6 +35,7 @@ import org.usvm.stopstrategies.StopStrategy import org.usvm.util.originalInst import java.io.File import java.lang.management.ManagementFactory +import kotlin.random.Random import kotlin.time.Duration.Companion.milliseconds /** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ @@ -74,6 +77,7 @@ class CoverageSelectorPerformanceTest { private class PerformanceMachine(cp: JcClasspath) : UMachine() { override val options = UMachineOptions( + randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), solverType = SolverType.YICES, solverTimeout = 250.milliseconds, typeOperationsTimeout = 250.milliseconds, @@ -95,7 +99,8 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) val goals = coverage.getUncoveredStatements() val metrics = Metrics() - val (base, markCovered) = createGuidance(mode, goals, metrics) + val initial = interpreter.getInitialState(method) + val (base, markCovered) = createGuidance(mode, goals, metrics, initial) val selector = MeasuredSelector(base, metrics) coverage.addOnCoveredObserver { _, _, statement -> val started = System.nanoTime() @@ -104,7 +109,6 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { metrics.coverageReports++ } val observer = metrics.observer() - val initial = interpreter.getInitialState(method) val gcCountBefore = gcCount() val gcMillisBefore = gcMillis() val allocatedBefore = allocatedBytes() @@ -148,6 +152,7 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { mode: String, goals: Collection, metrics: Metrics, + initial: JcState, ): Pair, (JcInst) -> Unit> { val cfg = CfgStatisticsImpl(graph) val calculators = MultiTargetDistanceCalculator { target -> @@ -173,6 +178,9 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { }, ) markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } + } else if (mode == "random-path" || mode == "uniform") { + base = createRandomSelector(mode, initial) + markCovered = {} } else if (mode == "bfs") { base = BfsPathSelector() markCovered = {} @@ -188,6 +196,26 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { return base to markCovered } + private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { + val random = Random(seed = options.randomSeed) + if (mode == "uniform") { + return WeightedPathSelector( + priorityCollectionFactory = { + RandomizedPriorityCollection( + comparator = compareBy { it.id }, + unitIntervalRandom = { random.nextDouble() }, + ) + }, + weighter = { 1.0 }, + ) + } + + return RandomTreePathSelector.fromRoot( + root = initial.pathNode, + randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, + ) + } + override fun close() = components.close() } @@ -312,10 +340,12 @@ private fun Metrics.report( totals: AnalysisTotals, ): String { val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() + val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() return """ {"program":"$program", "mode":"$mode", "repetition":$repetition, + "randomSeed":$randomSeed, "steps":$steps, "stop":"$stop", "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, From 5ddaa1c461024685ba69f9875c28cf88d9d83b65 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 19:22:06 +0300 Subject: [PATCH 08/10] Reuse lazy NURS guidance for external TypeScript coverage --- .../CoverageSelectorPerformanceTest.kt | 366 ++++++++++++++++++ .../coverage-selector-performance/README.md | 2 + .../coverage-selector-performance/run.py | 9 +- .../org/usvm/ps/CoverageGuidedPathSelector.kt | 119 ------ .../kotlin/org/usvm/ps/PathSelectorFactory.kt | 45 ++- ...losestToUncoveredRandomPathSelectorTest.kt | 149 +++++++ .../CoverageGuidedPathSelectorRefreshTest.kt | 234 ----------- .../usvm/ps/CoverageGuidedPathSelectorTest.kt | 282 -------------- .../CoverageSelectorPerformanceTest.kt | 87 ++--- .../org/usvm/ts/calls/CallsClassStartSmoke.kt | 1 + .../usvm/ts/pbt/usvm/UsvmPropertySearcher.kt | 6 +- .../main/kotlin/org/usvm/machine/TsMachine.kt | 47 ++- .../machine/TsMachineExternalCoverageTest.kt | 103 +++++ .../DeterministicPriorityCollection.kt | 44 +-- ...erministicPriorityCollectionRefreshTest.kt | 108 ------ 15 files changed, 736 insertions(+), 866 deletions(-) create mode 100644 docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt delete mode 100644 usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt create mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt delete mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt delete mode 100644 usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt create mode 100644 usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt delete mode 100644 usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt diff --git a/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt b/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt new file mode 100644 index 0000000000..df0da45000 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt @@ -0,0 +1,366 @@ +package org.usvm.experiments + +import kotlinx.coroutines.runBlocking +import org.jacodb.api.jvm.JcClasspath +import org.jacodb.api.jvm.JcMethod +import org.jacodb.api.jvm.cfg.JcInst +import org.jacodb.api.jvm.ext.findClass +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test +import org.usvm.SolverType +import org.usvm.UMachine +import org.usvm.UMachineOptions +import org.usvm.UPathSelector +import org.usvm.algorithms.DeterministicPriorityCollection +import org.usvm.algorithms.RandomizedPriorityCollection +import org.usvm.machine.JcApplicationGraph +import org.usvm.machine.JcComponents +import org.usvm.machine.JcContext +import org.usvm.machine.JcMachineOptions +import org.usvm.machine.JcTypeSystem +import org.usvm.machine.interpreter.JcInterpreter +import org.usvm.machine.state.JcState +import org.usvm.ps.BfsPathSelector +import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.RandomTreePathSelector +import org.usvm.ps.WeightedPathSelector +import org.usvm.samples.JacoDBContainer +import org.usvm.statistics.CompositeUMachineObserver +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.UMachineObserver +import org.usvm.statistics.distances.CallStackDistanceCalculator +import org.usvm.statistics.distances.CfgStatisticsImpl +import org.usvm.statistics.distances.MultiTargetDistanceCalculator +import org.usvm.stopstrategies.StopStrategy +import org.usvm.util.originalInst +import java.io.File +import java.lang.management.ManagementFactory +import kotlin.random.Random +import kotlin.time.Duration.Companion.milliseconds + +/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ +@Tag("manual") +class CoverageSelectorPerformanceTest { + @Test + fun runProgram() { + val program = System.getProperty("coverage.perf.case", "codec") + val mode = System.getProperty("coverage.perf.mode", "128") + val steps = System.getProperty("coverage.perf.steps", "100000").toLong() + val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() + val resultFile = File(System.getProperty("coverage.perf.results")) + val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) + val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) + val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) + try { + val (className, methodName) = when (program) { + "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" + "gcd" -> "com.google.common.math.IntMath" to "gcd" + "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" + else -> error("Unknown program: $program") + } + val cls = container.cp.findClass(className) + val method = cls.declaredMethods.single { it.name == methodName } + val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } + PerformanceMachine(container.cp).use { machine -> + val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) + resultFile.parentFile.mkdirs() + resultFile.appendText(result + "\n") + } + } finally { + runBlocking { + container.cp.close() + container.db.close() + } + } + } +} + +private class PerformanceMachine(cp: JcClasspath) : UMachine() { + override val options = UMachineOptions( + randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), + solverType = SolverType.YICES, + solverTimeout = 250.milliseconds, + typeOperationsTimeout = 250.milliseconds, + throwExceptionOnStepFailure = true, + useSoftConstraints = false, + ) + private val graph = JcApplicationGraph(cp) + private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) + private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) + + fun measure( + program: String, + mode: String, + method: JcMethod, + methodsToCover: List, + stepLimit: Long, + timeoutMillis: Long, + ): String { + val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) + val goals = coverage.getUncoveredStatements() + val metrics = Metrics() + val initial = interpreter.getInitialState(method) + val (base, markCovered) = createGuidance(mode, goals, metrics, initial) + val selector = MeasuredSelector(base, metrics) + coverage.addOnCoveredObserver { _, _, statement -> + val started = System.nanoTime() + markCovered(statement) + metrics.selectorNanos += System.nanoTime() - started + metrics.coverageReports++ + } + val observer = metrics.observer() + val gcCountBefore = gcCount() + val gcMillisBefore = gcMillis() + val allocatedBefore = allocatedBytes() + val started = System.nanoTime() + var stop = "exhausted" + selector.add(states = listOf(initial)) + run( + interpreter = interpreter, + pathSelector = selector, + observer = CompositeUMachineObserver(coverage, observer), + isStateTerminated = { state -> state.callStack.isEmpty() }, + stopStrategy = StopStrategy { + val elapsedMillis = (System.nanoTime() - started) / 1_000_000 + when { + metrics.steps >= stepLimit -> { + stop = "steps" + true + } + elapsedMillis >= timeoutMillis -> { + stop = "timeout" + true + } + else -> { + false + } + } + }, + ) + val elapsedNanos = System.nanoTime() - started + val allocated = allocatedBytes() - allocatedBefore + val totals = AnalysisTotals( + elapsedNanos = elapsedNanos, + allocatedBytes = allocated, + gcCount = gcCount() - gcCountBefore, + gcMillis = gcMillis() - gcMillisBefore, + ) + return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) + } + + private fun createGuidance( + mode: String, + goals: Collection, + metrics: Metrics, + initial: JcState, + ): Pair, (JcInst) -> Unit> { + val cfg = CfgStatisticsImpl(graph) + val calculators = MultiTargetDistanceCalculator { target -> + CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) + } + val distance: (JcState, JcInst) -> UInt = { state, target -> + metrics.distanceCalls++ + calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) + } + val markCovered: (JcInst) -> Unit + val base: UPathSelector + if (mode == "eager") { + val eager = EagerCoverageSelector(goals, distance) + base = eager + markCovered = { eager.markCovered(it) } + } else if (mode == "existing") { + val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) + base = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state: JcState -> + metrics.distanceCalls++ + aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) + }, + ) + markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } + } else if (mode == "random-path" || mode == "uniform") { + base = createRandomSelector(mode, initial) + markCovered = {} + } else if (mode == "bfs") { + base = BfsPathSelector() + markCovered = {} + } else { + val periodic = CoverageGuidedPathSelector( + targets = goals, + distance = distance, + priorityRefreshInterval = mode.toInt(), + ) + base = periodic + markCovered = { periodic.markCovered(targets = listOf(it)) } + } + return base to markCovered + } + + private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { + val random = Random(seed = options.randomSeed) + if (mode == "uniform") { + return WeightedPathSelector( + priorityCollectionFactory = { + RandomizedPriorityCollection( + comparator = compareBy { it.id }, + unitIntervalRandom = { random.nextDouble() }, + ) + }, + weighter = { 1.0 }, + ) + } + + return RandomTreePathSelector.fromRoot( + root = initial.pathNode, + randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, + ) + } + + override fun close() = components.close() +} + +private class Metrics { + var steps = 0L + var terminated = 0L + var selectorNanos = 0L + var refreshNanos = 0L + var refreshes = 0L + var distanceCalls = 0L + var coverageReports = 0L + var queueSize = 0 + var peakQueue = 0 +} + +private class MeasuredSelector( + private val delegate: UPathSelector, + private val metrics: Metrics, +) : UPathSelector { + override fun isEmpty(): Boolean = delegate.isEmpty() + + override fun peek(): JcState { + val beforeCalls = metrics.distanceCalls + val started = System.nanoTime() + val result = delegate.peek() + val elapsed = System.nanoTime() - started + metrics.selectorNanos += elapsed + if (metrics.distanceCalls > beforeCalls) { + metrics.refreshes++ + metrics.refreshNanos += elapsed + } + return result + } + + override fun add(states: Collection) { + val started = System.nanoTime() + delegate.add(states) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize += states.size + metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) + } + + override fun update(state: JcState) { + val started = System.nanoTime() + delegate.update(state) + metrics.selectorNanos += System.nanoTime() - started + } + + override fun remove(state: JcState) { + val started = System.nanoTime() + delegate.remove(state) + metrics.selectorNanos += System.nanoTime() - started + metrics.queueSize-- + } +} + +/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ +private class EagerCoverageSelector( + targets: Collection, + distance: (JcState, JcInst) -> UInt, +) : UPathSelector { + private val pending = targets.toMutableSet() + private var dirty = false + private val delegate = WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, + ) + + fun markCovered(target: JcInst) { + if (pending.remove(target)) dirty = true + } + + override fun isEmpty(): Boolean = delegate.isEmpty() + override fun peek(): JcState { + if (dirty) { + val states = delegate.drain().toList() + delegate.add(states) + dirty = false + } + return delegate.peek() + } + override fun add(states: Collection) = delegate.add(states) + override fun update(state: JcState) = delegate.update(state) + override fun remove(state: JcState) = delegate.remove(state) +} + +private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionCount.coerceAtLeast(minimumValue = 0) +} + +private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { + it.collectionTime.coerceAtLeast(minimumValue = 0) +} +private fun allocatedBytes(): Long { + val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean + return bean.getThreadAllocatedBytes(Thread.currentThread().id) +} + +private data class AnalysisTotals( + val elapsedNanos: Long, + val allocatedBytes: Long, + val gcCount: Long, + val gcMillis: Long, +) + +private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { + override fun onStatePeeked(state: JcState) { + steps++ + } + + override fun onStateTerminated(state: JcState, stateReachable: Boolean) { + terminated++ + } +} + +private fun Metrics.report( + program: String, + mode: String, + goals: Int, + covered: Int, + stop: String, + totals: AnalysisTotals, +): String { + val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() + val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() + return """ + {"program":"$program", + "mode":"$mode", + "repetition":$repetition, + "randomSeed":$randomSeed, + "steps":$steps, + "stop":"$stop", + "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, + "selectorMs":${selectorNanos / 1_000_000.0}, + "refreshMs":${refreshNanos / 1_000_000.0}, + "refreshes":$refreshes, + "distanceCalls":$distanceCalls, + "peakQueue":$peakQueue, + "pending":$queueSize, + "goals":$goals, + "covered":$covered, + "coverageReports":$coverageReports, + "terminated":$terminated, + "allocatedBytes":${totals.allocatedBytes}, + "gcCount":${totals.gcCount}, + "gcMs":${totals.gcMillis}} + """.trimIndent().replace("\n", "") +} diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md index fa9cbe8ac9..5df8bce855 100644 --- a/docs/experiments/coverage-selector-performance/README.md +++ b/docs/experiments/coverage-selector-performance/README.md @@ -1,5 +1,7 @@ # Coverage selector performance on real JVM bytecode +These are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). + This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. ## Recorded results (2026-10-16) diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py index 590a47f71e..c1e9ef53d9 100644 --- a/docs/experiments/coverage-selector-performance/run.py +++ b/docs/experiments/coverage-selector-performance/run.py @@ -20,7 +20,7 @@ def cached_jar(group, artifact, version): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--cases', default='codec,gcd,loan') - parser.add_argument('--modes', default='existing,128,eager,512') + parser.add_argument('--modes', default='nurs,uniform') parser.add_argument('--steps', type=int, default=100000) parser.add_argument('--timeout-ms', type=int, default=120000) parser.add_argument('--repetitions', type=int, default=2) @@ -32,6 +32,13 @@ def main(): experiment = Path(__file__).resolve().parent root = experiment.parents[2] + revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() + historical_revision = '44869512bca6b1ce89511b9f6799f4475efdf972' + historical_modes = {'128', '512', 'eager'} + uses_historical_modes = bool(set(args.modes.split(',')) & historical_modes) + if uses_historical_modes and revision != historical_revision: + parser.error(f'Historical experiment: run this script from a checkout of {historical_revision}') + output = (args.output or root / 'build/coverage-selector-performance').resolve() output.mkdir(parents=True, exist_ok=True) codec = args.codec_jar or cached_jar('commons-codec', 'commons-codec', '1.16.0') diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt b/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt deleted file mode 100644 index 416043bf11..0000000000 --- a/usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt +++ /dev/null @@ -1,119 +0,0 @@ -package org.usvm.ps - -import org.usvm.UPathSelector -import org.usvm.algorithms.DeterministicPriorityCollection - -/** - * Caller-confirmed coverage changes priorities, never removes states. Updates belong on the analysis thread. - * [distance] defines statement, edge or other goal semantics; [UInt.MAX_VALUE] means unreachable. - * - * Weights use a snapshot of uncovered goals. Changes are applied before selection after at least - * [priorityRefreshInterval] completed selections since the previous priority refresh. A selection completes when - * its state is updated or removed; repeated [peek] calls do not advance the interval. - * The default interval is a tuning starting point, not a measured optimum. - */ -class CoverageGuidedPathSelector( - targets: Collection, - distance: (State, Target) -> UInt, - private val priorityRefreshInterval: Int = DEFAULT_PRIORITY_REFRESH_INTERVAL, -) : UPathSelector { - init { - require(priorityRefreshInterval > 0) { "Priority refresh interval must be positive" } - } - - private val knownTargets = targets.toHashSet() - private val pendingTargets = targets.toHashSet() - private var priorityTargets: Set = targets.toHashSet() - private val distanceToTarget = distance - private var prioritiesChanged = false - private var completedSelections = 0 - private var selectionPending = false - private var selectedState: State? = null - private val priorities = DeterministicPriorityCollection(Comparator.naturalOrder()) - private val selector = WeightedPathSelector( - priorityCollectionFactory = { priorities }, - weighter = { state -> weight(state, priorityTargets) }, - ) - - val uncoveredTargets: Set - get() = pendingTargets.toHashSet() - - /** Registers newly identified goals; previously covered goals stay covered. */ - fun addTargets(targets: Collection): Boolean { - val additions = targets.filter { knownTargets.add(it) } - if (additions.isEmpty()) return false - - pendingTargets.addAll(additions) - prioritiesChanged = true - return true - } - - /** Unknown goals are ignored. Changed priorities are applied at the next eligible selection. */ - fun markCovered(targets: Collection): Boolean { - val changed = pendingTargets.removeAll(targets.toHashSet()) - if (changed) { - prioritiesChanged = pendingTargets.isNotEmpty() - if (!prioritiesChanged) { - // With no goals, ordering is irrelevant and new work needs no distance calculations. - priorityTargets = emptySet() - } - } - - return changed - } - - override fun isEmpty(): Boolean = selector.isEmpty() - - override fun peek(): State { - val refreshDue = prioritiesChanged && completedSelections >= priorityRefreshInterval - if (refreshDue) { - val refreshedTargets = pendingTargets.toHashSet() - priorities.updatePriorities { state -> weight(state, refreshedTargets) } - priorityTargets = refreshedTargets - prioritiesChanged = false - completedSelections = 0 - } - - val state = selector.peek() - selectedState = state - selectionPending = true - return state - } - - override fun add(states: Collection) = selector.add(states) - - override fun update(state: State) { - selector.update(state) - completeSelection(state) - } - - override fun remove(state: State) { - selector.remove(state) - completeSelection(state) - } - - // Bypass pending priority updates and selection accounting when collecting stopped work. - override fun drain(): Sequence { - selectionPending = false - selectedState = null - return selector.drain() - } - - private fun weight(state: State, targets: Set): UInt = - targets.minOfOrNull { target -> distanceToTarget(state, target) } ?: UInt.MAX_VALUE - - private fun completeSelection(state: State) { - val completesSelection = selectionPending && selectedState == state - if (!completesSelection) return - - if (completedSelections < priorityRefreshInterval) { - completedSelections++ - } - selectionPending = false - selectedState = null - } - - private companion object { - const val DEFAULT_PRIORITY_REFRESH_INTERVAL = 128 - } -} diff --git a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt index 9a3d0b4c94..60173bd83b 100644 --- a/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt +++ b/usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt @@ -36,6 +36,7 @@ private fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageDistanceCalculatorFactory: () -> CallStackDistanceCalculator? = { null }, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { @@ -69,14 +70,16 @@ private fun createPathSelector( PathSelectionStrategy.CLOSEST_TO_UNCOVERED -> createClosestToUncoveredPathSelector( requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, - applicationGraph + applicationGraph, + distanceCalculator = coverageDistanceCalculatorFactory(), ) PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM -> createClosestToUncoveredPathSelector( requireNotNull(coverageStatisticsFactory()) { "Coverage statistics is required for closest to uncovered path selector" }, requireNotNull(cfgStatisticsFactory()) { "CFG statistics is required for closest to uncovered path selector" }, applicationGraph, - random + random, + distanceCalculator = coverageDistanceCalculatorFactory(), ) PathSelectionStrategy.TARGETED -> createTargetedPathSelector( @@ -153,6 +156,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageDistanceCalculatorFactory: () -> CallStackDistanceCalculator? = { null }, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> = createPathSelector( listOf(initialState), @@ -161,7 +165,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageDistanceCalculatorFactory, ) fun createPathSelector( @@ -173,6 +178,7 @@ fun createPathSelector( cfgStatisticsFactory: () -> CfgStatistics? = { null }, callGraphStatisticsFactory: () -> CallGraphStatistics? = { null }, loopStatisticFactory: () -> StateLoopTracker<*, Statement, State>? = { null }, + coverageDistanceCalculatorFactory: () -> CallStackDistanceCalculator? = { null }, ): UPathSelector where Target : UTarget, State : UState<*, Method, Statement, *, Target, State> { val hasSingleEntryPoint = initialStates.size == 1 val hasUnlimitedScheduling = options.timeout == Duration.INFINITE && @@ -185,7 +191,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageDistanceCalculatorFactory, ) } @@ -202,7 +209,8 @@ fun createPathSelector( coverageStatisticsFactory, cfgStatisticsFactory, callGraphStatisticsFactory, - loopStatisticFactory + loopStatisticFactory, + coverageDistanceCalculatorFactory, ) val coverageStatistics = coverageStatisticsFactory() @@ -317,15 +325,16 @@ private fun , applicationGraph: ApplicationGraph, random: Random? = null, + distanceCalculator: CallStackDistanceCalculator? = null, ): UPathSelector { - val distanceCalculator = CallStackDistanceCalculator( + val calculator = distanceCalculator ?: CallStackDistanceCalculator( targets = coverageStatistics.getUncoveredStatements(), cfgStatistics = cfgStatistics, applicationGraph ) coverageStatistics.addOnCoveredObserver { _, method, statement -> - distanceCalculator.removeTarget( + calculator.removeTarget( method, statement ) @@ -334,21 +343,25 @@ private fun > createClosestToUncoveredRandomPathSelector( + distance: StateWeighter, + random: Random, +): UPathSelector = WeightedPathSelector( + priorityCollectionFactory = { RandomizedPriorityCollection(compareById()) { random.nextDouble() } }, + weighter = { 1.0 / max(distance.weight(it).toDouble(), 1.0) }, +) + fun > createForkDepthPathSelector( random: Random? = null, ): UPathSelector { diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt new file mode 100644 index 0000000000..a7365dde31 --- /dev/null +++ b/usvm-core/src/test/kotlin/org/usvm/ps/ClosestToUncoveredRandomPathSelectorTest.kt @@ -0,0 +1,149 @@ +package org.usvm.ps + +import io.mockk.every +import org.junit.jupiter.api.Test +import org.usvm.PathNode +import org.usvm.PathSelectionStrategy +import org.usvm.TestInstruction +import org.usvm.TestState +import org.usvm.UMachineOptions +import org.usvm.appGraph +import org.usvm.mockState +import org.usvm.statistics.CoverageStatistics +import org.usvm.statistics.distances.CallStackDistanceCalculator +import org.usvm.statistics.distances.CfgStatistics +import org.usvm.statistics.distances.CfgStatisticsImpl +import kotlin.test.assertEquals +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class ClosestToUncoveredRandomPathSelectorTest { + private val graph = appGraph { + method(name = "method", instructionsCount = 7) { + entryPoint(offset = 0) + edge(from = 0, to = 1) + edge(from = 1, to = 2) + edge(from = 0, to = 3) + edge(from = 3, to = 4) + edge(from = 4, to = 5) + edge(from = 5, to = 6) + exitPoint(offset = 2) + exitPoint(offset = 6) + } + } + + @Test + fun `coverage and peek never recalculate queued weights while update touches only its state`() { + val cfg = CfgStatisticsImpl(graph) + val evaluated = mutableListOf() + val countedCfg = object : CfgStatistics by cfg { + override fun getShortestDistanceToExit(method: String, stmtFrom: TestInstruction): UInt { + evaluated.add(stmtFrom) + return cfg.getShortestDistanceToExit(method, stmtFrom) + } + } + val leftGoal = TestInstruction(method = "method", offset = 1) + val rightGoal = TestInstruction(method = "method", offset = 6) + val calculator = CallStackDistanceCalculator( + targets = listOf(leftGoal, rightGoal), + cfgStatistics = countedCfg, + applicationGraph = graph, + ) + val nativeCoverage = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) + val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) + val selector = createPathSelector( + initialState = left, + options = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + randomSeed = 0, + ), + applicationGraph = graph, + coverageStatisticsFactory = { nativeCoverage }, + cfgStatisticsFactory = { countedCfg }, + coverageDistanceCalculatorFactory = { calculator }, + ) + selector.add(states = listOf(right)) + assertEquals(expected = listOf(left.currentStatement, right.currentStatement), actual = evaluated) + assertSame(expected = left, actual = selector.peek()) + evaluated.clear() + + assertTrue(calculator.removeTarget(method = "method", statement = leftGoal)) + repeat(256) { selector.peek() } + + assertTrue(evaluated.isEmpty()) + assertEquals(expected = 0, actual = nativeCoverage.getTotalCoveredStatements()) + + selector.update(state = left) + + assertEquals(expected = listOf(left.currentStatement), actual = evaluated) + assertSame(expected = right, actual = selector.peek()) + evaluated.clear() + + val fork = mockState(id = 3u, startMethod = "method", startInstruction = 2) + selector.add(states = listOf(fork)) + + assertEquals(expected = listOf(fork.currentStatement), actual = evaluated) + assertSame(expected = right, actual = selector.peek()) + assertEquals(expected = setOf(left, right, fork), actual = selector.drain().toHashSet()) + } + + @Test + fun `native coverage still removes targets from the supplied calculator`() { + val cfg = CfgStatisticsImpl(graph) + val goal = TestInstruction(method = "method", offset = 1) + val calculator = CallStackDistanceCalculator( + targets = listOf(goal), + cfgStatistics = cfg, + applicationGraph = graph, + ) + val state = mockState(id = 1u, startMethod = "method", startInstruction = 1) + every { state.pathNode } returns PathNode.root() + goal + val coverage = CoverageStatistics( + methods = setOf("method"), + applicationGraph = graph, + ) + val selector = createPathSelector( + initialState = state, + options = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + ), + applicationGraph = graph, + coverageStatisticsFactory = { coverage }, + cfgStatisticsFactory = { cfg }, + coverageDistanceCalculatorFactory = { calculator }, + ) + + coverage.onStateTerminated(state = state, stateReachable = true) + selector.update(state = state) + + assertEquals(expected = 1, actual = coverage.getTotalCoveredStatements()) + assertEquals( + expected = UInt.MAX_VALUE, + actual = calculator.calculateDistance(state.currentStatement, state.callStack), + ) + assertSame(expected = state, actual = selector.peek()) + assertEquals(expected = listOf(state), actual = selector.drain().toList()) + } + + @Test + fun `covered prefixes can still lead to uncovered successors`() { + val goal = TestInstruction(method = "method", offset = 6) + val prefix = mockState(id = 1u, startMethod = "method", startInstruction = 3) + val calculator = CallStackDistanceCalculator( + targets = listOf(prefix.currentStatement, goal), + cfgStatistics = CfgStatisticsImpl(graph), + applicationGraph = graph, + ) + + calculator.removeTarget(method = "method", statement = prefix.currentStatement) + + assertEquals( + expected = 3u, + actual = calculator.calculateDistance(prefix.currentStatement, prefix.callStack), + ) + } +} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt deleted file mode 100644 index 9421b89c4c..0000000000 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorRefreshTest.kt +++ /dev/null @@ -1,234 +0,0 @@ -package org.usvm.ps - -import org.junit.jupiter.api.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertTrue - -class CoverageGuidedPathSelectorRefreshTest { - @Test - fun `default interval keeps weights stale for 128 completed selections`() { - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - ) - selector.add(states = listOf("left", "right")) - selector.markCovered(targets = listOf("left")) - - repeat(127) { - assertEquals(expected = "left", actual = selector.peek()) - evaluatedTargets.clear() - selector.update(state = "left") - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) - } - evaluatedTargets.clear() - repeat(256) { - assertEquals(expected = "left", actual = selector.peek()) - } - assertTrue(evaluatedTargets.isEmpty()) - - selector.update(state = "left") - evaluatedTargets.clear() - - assertEquals(expected = "right", actual = selector.peek()) - assertEquals(expected = listOf("right", "right"), actual = evaluatedTargets) - evaluatedTargets.clear() - selector.peek() - assertTrue(evaluatedTargets.isEmpty()) - } - - @Test - fun `new and updated states use the old snapshot until the interval expires`() { - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("old"), - distance = { state, target -> - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("old")) - selector.peek() - - selector.addTargets(targets = listOf("new")) - selector.markCovered(targets = listOf("old")) - evaluatedTargets.clear() - selector.add(states = listOf("new")) - selector.update(state = "old") - - assertEquals(expected = listOf("old", "old"), actual = evaluatedTargets) - assertEquals(expected = setOf("new"), actual = selector.uncoveredTargets) - assertEquals(expected = "old", actual = selector.peek()) - - selector.update(state = "old") - evaluatedTargets.clear() - - assertEquals(expected = "new", actual = selector.peek()) - assertEquals(expected = listOf("new", "new"), actual = evaluatedTargets) - } - - @Test - fun `only one completion of the selected state advances the interval`() { - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - evaluations++ - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("left", "right", "other")) - selector.peek() - selector.markCovered(targets = listOf("left")) - - selector.update(state = "other") - selector.remove(state = "other") - selector.update(state = "left") - selector.update(state = "left") - val beforePeek = evaluations - - assertEquals(expected = "left", actual = selector.peek()) - assertEquals(expected = beforePeek, actual = evaluations) - - selector.remove(state = "left") - val beforeRefresh = evaluations - - assertEquals(expected = "right", actual = selector.peek()) - assertEquals(expected = beforeRefresh + 1, actual = evaluations) - } - - @Test - fun `each refresh starts a new interval including newly registered goals`() { - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - val destination = if (target == "left-again") "left" else target - if (state == destination) 0u else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("left", "right")) - selector.markCovered(targets = listOf("left")) - repeat(2) { selector.update(state = selector.peek()) } - assertEquals(expected = "right", actual = selector.peek()) - - selector.addTargets(targets = listOf("left-again")) - selector.markCovered(targets = listOf("right")) - selector.update(state = "right") - - assertEquals(expected = "right", actual = selector.peek()) - - selector.update(state = "right") - - assertEquals(expected = "left", actual = selector.peek()) - assertEquals(expected = setOf("left-again"), actual = selector.uncoveredTargets) - } - - @Test - fun `unchanged goals do not trigger refresh even after the interval`() { - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("goal"), - distance = { _, _ -> - evaluations++ - 0u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("first", "second")) - - repeat(8) { selector.update(state = selector.peek()) } - val beforePeek = evaluations - selector.markCovered(targets = listOf("unknown")) - selector.addTargets(targets = listOf("goal")) - selector.peek() - - assertEquals(expected = 2 + 8, actual = evaluations) - assertEquals(expected = beforePeek, actual = evaluations) - } - - @Test - fun `saturation avoids distance work and later goals resume guidance`() { - var distancesAllowed = true - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("old"), - distance = { state, target -> - check(distancesAllowed) - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 2, - ) - selector.add(states = listOf("old", "new")) - distancesAllowed = false - - selector.markCovered(targets = listOf("old")) - repeat(4) { selector.update(state = selector.peek()) } - selector.add(states = listOf("later")) - - assertTrue(selector.uncoveredTargets.isEmpty()) - - selector.addTargets(targets = listOf("new")) - distancesAllowed = true - evaluatedTargets.clear() - - assertEquals(expected = "new", actual = selector.peek()) - assertEquals(expected = listOf("new", "new", "new"), actual = evaluatedTargets) - assertEquals(expected = setOf("old", "new", "later"), actual = selector.drain().toSet()) - } - - @Test - fun `failed refresh preserves the snapshot and can be retried`() { - var failRefresh = false - val evaluatedTargets = mutableListOf() - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, target -> - check(!failRefresh) - evaluatedTargets.add(target) - val matchingDistance = if (state == "left") 0u else 1u - if (state == target) matchingDistance else 10u - }, - priorityRefreshInterval = 1, - ) - selector.add(states = listOf("left", "right")) - selector.peek() - selector.markCovered(targets = listOf("left")) - selector.update(state = "left") - failRefresh = true - - assertFailsWith { selector.peek() } - - failRefresh = false - evaluatedTargets.clear() - selector.add(states = listOf("other")) - - assertEquals(expected = listOf("left", "right"), actual = evaluatedTargets.sorted()) - assertEquals(expected = "right", actual = selector.peek()) - assertEquals(expected = setOf("left", "right", "other"), actual = selector.drain().toSet()) - } - - @Test - fun `refresh interval must be positive`() { - for (interval in listOf(0, -1)) { - assertFailsWith { - CoverageGuidedPathSelector( - targets = emptyList(), - distance = { _, _ -> 0u }, - priorityRefreshInterval = interval, - ) - } - } - } -} diff --git a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt b/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt deleted file mode 100644 index 825c93380e..0000000000 --- a/usvm-core/src/test/kotlin/org/usvm/ps/CoverageGuidedPathSelectorTest.kt +++ /dev/null @@ -1,282 +0,0 @@ -package org.usvm.ps - -import org.junit.jupiter.api.Test -import org.usvm.TestInstruction -import org.usvm.TestState -import org.usvm.appGraph -import org.usvm.mockState -import org.usvm.statistics.distances.CallStackDistanceCalculator -import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.statistics.distances.MultiTargetDistanceCalculator -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertSame -import kotlin.test.assertTrue - -class CoverageGuidedPathSelectorTest { - @Test - fun `reported coverage reprioritizes every pending state after a completed selection`() { - val selector = selector(targets = listOf("left", "right")) - selector.add(states = listOf("left", "right")) - - assertEquals(expected = "left", actual = selector.peek()) - - selector.markCovered(targets = listOf("left")) - selector.update(state = "left") - - assertEquals(expected = "right", actual = selector.peek()) - - selector.addTargets(targets = listOf("new-left")) - selector.markCovered(targets = listOf("right")) - selector.update(state = "right") - - assertEquals(expected = "left", actual = selector.peek()) - assertEquals(expected = setOf("left", "right"), actual = selector.drain().toSet()) - } - - @Test - fun `unchanged coverage does not recalculate queued weights`() { - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("goal"), - distance = { _, _ -> - evaluations++ - 0u - }, - ) - selector.add(states = listOf("first", "second")) - val afterAdd = evaluations - - selector.peek() - selector.peek() - assertFalse(selector.markCovered(targets = listOf("unknown"))) - assertFalse(selector.addTargets(targets = listOf("goal"))) - selector.peek() - - assertEquals(expected = afterAdd, actual = evaluations) - } - - @Test - fun `multiple reports are applied together at the refresh boundary`() { - var refreshing = false - var evaluations = 0 - val selector = CoverageGuidedPathSelector( - targets = listOf("first", "second", "third"), - distance = { state, target -> - if (refreshing) { - check(target == "third") - evaluations++ - } - - if (state == target) 0u else 10u - }, - priorityRefreshInterval = 1, - ) - selector.add(states = listOf("first", "third")) - val selected = selector.peek() - - selector.markCovered(targets = listOf("first")) - selector.markCovered(targets = listOf("second")) - selector.update(state = selected) - refreshing = true - - assertEquals(expected = "third", actual = selector.peek()) - assertEquals(expected = 2, actual = evaluations) - } - - @Test - fun `saturated coverage retains all queued states and accepts later work`() { - val selector = selector(targets = listOf("left", "right")) - selector.add(states = listOf("left", "right")) - selector.markCovered(targets = listOf("left", "right")) - - selector.peek() - val drained = selector.drain().toList() - selector.add(states = listOf("later")) - - assertEquals(expected = setOf("left", "right"), actual = drained.toSet()) - assertEquals(expected = 2, actual = drained.size) - assertEquals(expected = "later", actual = selector.peek()) - } - - @Test - fun `edges sharing a successor remain distinct coverage goals`() { - val firstEdge = Edge(from = 1, to = 3) - val secondEdge = Edge(from = 2, to = 3) - val firstPath = setOf(firstEdge) - val secondPath = setOf(secondEdge) - val selector = CoverageGuidedPathSelector, Edge>( - targets = listOf(firstEdge, secondEdge), - distance = { path, edge -> if (edge in path) 0u else UInt.MAX_VALUE }, - priorityRefreshInterval = 1, - ) - selector.add(states = listOf(firstPath, secondPath)) - val selected = selector.peek() - - selector.markCovered(targets = listOf(firstEdge)) - selector.update(state = selected) - - assertSame(expected = secondPath, actual = selector.peek()) - assertEquals(expected = setOf(firstPath, secondPath), actual = selector.drain().toSet()) - } - - @Test - fun `empty goals do not invoke the distance function or remove states`() { - val selector = CoverageGuidedPathSelector( - targets = emptyList(), - distance = { _, _ -> error("There is no known goal") }, - ) - selector.add(states = listOf("first", "second")) - - selector.peek() - - assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) - assertTrue(selector.isEmpty()) - } - - @Test - fun `draining stopped work does not refresh weights after coverage changes`() { - var stopped = false - val selector = CoverageGuidedPathSelector( - targets = listOf("left"), - distance = { _, _ -> - check(!stopped) - 0u - }, - ) - selector.add(states = listOf("first", "second")) - stopped = true - selector.addTargets(targets = listOf("right")) - - val drained = selector.drain().toList() - - assertEquals(expected = setOf("first", "second"), actual = drained.toSet()) - assertTrue(selector.isEmpty()) - } - - @Test - fun `state updates additions and removals work after coverage changes`() { - val distances = mutableMapOf("first" to 0u, "second" to 2u, "third" to 1u) - val selector = CoverageGuidedPathSelector( - targets = listOf("left", "right"), - distance = { state, _ -> distances.getValue(state) }, - ) - selector.add(states = listOf("first", "second")) - assertEquals(expected = "first", actual = selector.peek()) - - distances["first"] = 3u - selector.update(state = "first") - selector.add(states = listOf("third")) - selector.markCovered(targets = listOf("left")) - - assertEquals(expected = "third", actual = selector.peek()) - selector.remove(state = "third") - assertEquals(expected = "second", actual = selector.peek()) - assertEquals(expected = setOf("first", "second"), actual = selector.drain().toSet()) - } - - @Test - fun `coverage reports are idempotent and unknown goals are ignored`() { - val selector = selector(targets = listOf("first", "second")) - - assertFalse(selector.markCovered(targets = listOf("unknown"))) - assertTrue(selector.markCovered(targets = listOf("first"))) - assertFalse(selector.markCovered(targets = listOf("first"))) - - assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) - } - - @Test - fun `new goals can be registered without resurrecting covered goals`() { - val selector = selector(targets = listOf("first")) - selector.markCovered(targets = listOf("first")) - - assertTrue(selector.addTargets(targets = listOf("first", "second", "second"))) - assertFalse(selector.addTargets(targets = listOf("first", "second"))) - - assertEquals(expected = setOf("second"), actual = selector.uncoveredTargets) - } - - @Test - fun `goal snapshots and constructor inputs do not share mutable storage`() { - val initial = mutableListOf("first", "second") - val selector = selector(targets = initial) - val snapshot = selector.uncoveredTargets - - initial.clear() - selector.markCovered(targets = listOf("first", "second")) - - assertEquals(expected = setOf("first", "second"), actual = snapshot) - assertTrue(selector.uncoveredTargets.isEmpty()) - } - - @Test - fun `caller can reuse existing CFG distance calculators`() { - val graph = appGraph { - method(name = "method", instructionsCount = 5) { - entryPoint(offset = 0) - edge(from = 0, to = 1) - edge(from = 0, to = 3) - edge(from = 1, to = 2) - edge(from = 3, to = 4) - exitPoint(offset = 2) - exitPoint(offset = 4) - } - } - val cfgStatistics = CfgStatisticsImpl(graph) - val distances = MultiTargetDistanceCalculator { target -> - CallStackDistanceCalculator( - targets = listOf(target), - cfgStatistics = cfgStatistics, - applicationGraph = graph, - ) - } - val firstTarget = TestInstruction(method = "method", offset = 2) - val secondTarget = TestInstruction(method = "method", offset = 4) - val selector = CoverageGuidedPathSelector( - targets = listOf(firstTarget, secondTarget), - distance = { state, target -> - distances.calculateDistance(state.currentStatement, state.callStack, target) - }, - priorityRefreshInterval = 1, - ) - val left = mockState(id = 1u, startMethod = "method", startInstruction = 1) - val right = mockState(id = 2u, startMethod = "method", startInstruction = 3) - selector.add(states = listOf(left, right)) - val selected = selector.peek() - - selector.markCovered(targets = listOf(firstTarget)) - selector.update(state = selected) - - assertSame(expected = right, actual = selector.peek()) - assertEquals(expected = setOf(left, right), actual = selector.drain().toSet()) - } - - @Test - fun `coverage guidance composes with a standard selector on shared states`() { - val coverage = selector(targets = listOf("left", "right")) - val combined = InterleavedPathSelector(selectors = listOf(BfsPathSelector(), coverage)) - combined.add(states = listOf("left", "right")) - assertEquals(expected = "left", actual = combined.peek()) - - coverage.markCovered(targets = listOf("left")) - combined.update(state = "left") - - assertEquals(expected = "right", actual = combined.peek()) - combined.add(states = listOf("fork")) - combined.remove(state = "right") - assertEquals(expected = setOf("left", "fork"), actual = combined.drain().toSet()) - assertTrue(combined.isEmpty()) - } - - private fun selector(targets: Collection) = CoverageGuidedPathSelector( - targets = targets, - distance = { state, target -> - val destination = if (target == "new-left") "left" else target - if (state == destination) 0u else 10u - }, - priorityRefreshInterval = 1, - ) - - private data class Edge(val from: Int, val to: Int) -} diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt index df0da45000..4f41716094 100644 --- a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt +++ b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt @@ -21,7 +21,7 @@ import org.usvm.machine.JcTypeSystem import org.usvm.machine.interpreter.JcInterpreter import org.usvm.machine.state.JcState import org.usvm.ps.BfsPathSelector -import org.usvm.ps.CoverageGuidedPathSelector +import org.usvm.ps.createClosestToUncoveredRandomPathSelector import org.usvm.ps.RandomTreePathSelector import org.usvm.ps.WeightedPathSelector import org.usvm.samples.JacoDBContainer @@ -30,7 +30,6 @@ import org.usvm.statistics.CoverageStatistics import org.usvm.statistics.UMachineObserver import org.usvm.statistics.distances.CallStackDistanceCalculator import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.statistics.distances.MultiTargetDistanceCalculator import org.usvm.stopstrategies.StopStrategy import org.usvm.util.originalInst import java.io.File @@ -44,7 +43,7 @@ class CoverageSelectorPerformanceTest { @Test fun runProgram() { val program = System.getProperty("coverage.perf.case", "codec") - val mode = System.getProperty("coverage.perf.mode", "128") + val mode = System.getProperty("coverage.perf.mode", "nurs") val steps = System.getProperty("coverage.perf.steps", "100000").toLong() val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() val resultFile = File(System.getProperty("coverage.perf.results")) @@ -154,45 +153,33 @@ private class PerformanceMachine(cp: JcClasspath) : UMachine() { metrics: Metrics, initial: JcState, ): Pair, (JcInst) -> Unit> { - val cfg = CfgStatisticsImpl(graph) - val calculators = MultiTargetDistanceCalculator { target -> - CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) + if (mode == "random-path" || mode == "uniform") { + return createRandomSelector(mode, initial) to {} } - val distance: (JcState, JcInst) -> UInt = { state, target -> + if (mode == "bfs") return BfsPathSelector() to {} + require(mode == "nurs" || mode == "existing") { "Historical modes require checkout 44869512b" } + + val aggregate = CallStackDistanceCalculator( + targets = goals, + cfgStatistics = CfgStatisticsImpl(graph), + applicationGraph = graph, + ) + val distance: (JcState) -> UInt = { state -> metrics.distanceCalls++ - calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) + aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) } - val markCovered: (JcInst) -> Unit - val base: UPathSelector - if (mode == "eager") { - val eager = EagerCoverageSelector(goals, distance) - base = eager - markCovered = { eager.markCovered(it) } - } else if (mode == "existing") { - val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) - base = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state: JcState -> - metrics.distanceCalls++ - aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) - }, + val base = if (mode == "nurs") { + createClosestToUncoveredRandomPathSelector( + distance = distance, + random = Random(seed = options.randomSeed), ) - markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } - } else if (mode == "random-path" || mode == "uniform") { - base = createRandomSelector(mode, initial) - markCovered = {} - } else if (mode == "bfs") { - base = BfsPathSelector() - markCovered = {} } else { - val periodic = CoverageGuidedPathSelector( - targets = goals, - distance = distance, - priorityRefreshInterval = mode.toInt(), + WeightedPathSelector( + priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, + weighter = distance, ) - base = periodic - markCovered = { periodic.markCovered(targets = listOf(it)) } } + val markCovered: (JcInst) -> Unit = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } return base to markCovered } @@ -272,36 +259,6 @@ private class MeasuredSelector( } } -/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ -private class EagerCoverageSelector( - targets: Collection, - distance: (JcState, JcInst) -> UInt, -) : UPathSelector { - private val pending = targets.toMutableSet() - private var dirty = false - private val delegate = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, - ) - - fun markCovered(target: JcInst) { - if (pending.remove(target)) dirty = true - } - - override fun isEmpty(): Boolean = delegate.isEmpty() - override fun peek(): JcState { - if (dirty) { - val states = delegate.drain().toList() - delegate.add(states) - dirty = false - } - return delegate.peek() - } - override fun add(states: Collection) = delegate.add(states) - override fun update(state: JcState) = delegate.update(state) - override fun remove(state: JcState) = delegate.remove(state) -} - private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { it.collectionCount.coerceAtLeast(minimumValue = 0) } diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt index 7c03bb6e5c..d244d948c9 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassStartSmoke.kt @@ -572,6 +572,7 @@ private fun runRawClassAnalysis(args: List, exploratory: Boolean) { TsMachine( scene = scene, options = UMachineOptions( + pathSelectionStrategies = listOf(CALLS_PATH_SELECTION_STRATEGY), stateCollectionStrategy = StateCollectionStrategy.ALL, randomSeed = manifest.seed, collectedStatesLimit = manifest.collectedStatesLimit, diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt index 10bf82289f..8e2cadda37 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/usvm/UsvmPropertySearcher.kt @@ -3,6 +3,7 @@ package org.usvm.ts.pbt.usvm import io.ksmt.utils.asExpr import org.jacodb.ets.model.EtsScene import org.jacodb.ets.model.EtsType +import org.usvm.PathSelectionStrategy import org.usvm.SolverType import org.usvm.StateCollectionStrategy import org.usvm.UMachineOptions @@ -31,7 +32,10 @@ import org.usvm.ts.pbt.model.PropertyId /** Searches mapped TypeScript predicates for concrete property violations with USVM. */ class UsvmPropertySearcher( private val scene: EtsScene, - machineOptions: UMachineOptions = UMachineOptions(solverType = SolverType.YICES), + machineOptions: UMachineOptions = UMachineOptions( + solverType = SolverType.YICES, + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + ), private val tsOptions: TsOptions = TsOptions(), projectionOptions: UsvmProjectionOptions = UsvmProjectionOptions(), ) { diff --git a/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt b/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt index bdbbfa40fd..afbeb7e848 100644 --- a/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt +++ b/usvm-ts/src/main/kotlin/org/usvm/machine/TsMachine.kt @@ -5,6 +5,7 @@ import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsScene import org.jacodb.ets.model.EtsStmt import org.usvm.CoverageZone +import org.usvm.PathSelectionStrategy import org.usvm.SolverType import org.usvm.StateCollectionStrategy import org.usvm.UMachine @@ -37,6 +38,7 @@ import org.usvm.statistics.collectors.CoveredNewStatesCollector import org.usvm.statistics.collectors.StatesCollector import org.usvm.statistics.collectors.TargetsReachedStatesCollector import org.usvm.statistics.constraints.SoftConstraintsObserver +import org.usvm.statistics.distances.CallStackDistanceCalculator import org.usvm.statistics.distances.CfgStatisticsImpl import org.usvm.statistics.distances.PlainCallGraphStatistics import org.usvm.stopstrategies.StopStrategy @@ -78,9 +80,13 @@ data class TsMachineAnalysisResult( val runtimeLimited: Boolean get() = unsupportedPaths.isNotEmpty() } +/** Symbolically executes TypeScript entry points with shared coverage guidance. */ class TsMachine( scene: EtsScene, - override val options: UMachineOptions = UMachineOptions(solverType = SolverType.YICES), + override val options: UMachineOptions = UMachineOptions( + solverType = SolverType.YICES, + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + ), private val tsOptions: TsOptions, private val machineObserver: UMachineObserver? = null, private val observer: TsInterpreterObserver? = null, @@ -149,6 +155,24 @@ class TsMachine( throwExceptionOnStepFailure = options.throwExceptionOnStepFailure, ) private val cfgStatistics = CfgStatisticsImpl(graph) + private val externallyCoveredStatements = hashSetOf() + private var coverageDistanceCalculator: CallStackDistanceCalculator? = null + + /** + * Reports caller-confirmed coverage on the analysis thread, including before analysis starts. + * This removes distance goals without changing native coverage counters or refreshing queued weights. + * Reports are retained across analyses on this machine. + */ + fun markCovered(statements: Collection): Boolean { + var changed = false + for (statement in statements) { + if (externallyCoveredStatements.add(statement)) { + coverageDistanceCalculator?.removeTarget(graph.methodOf(statement), statement) + changed = true + } + } + return changed + } /** Returns supported states only. Call [analyzeWithOutcome] to inspect excluded unsupported paths. */ fun analyze( @@ -192,6 +216,7 @@ class TsMachine( targets: List = emptyList(), configureInitialState: (EtsMethod, TsState) -> Unit = { _, _ -> }, ): TsMachineAnalysisResult { + coverageDistanceCalculator = null interpreter.resetStepFailure() failureTrackingUnknownCallObserver.reset() @@ -213,11 +238,7 @@ class TsMachine( applicationGraph = graph, ) - val callGraphStatistics: PlainCallGraphStatistics = - when (options.targetSearchDepth) { - 0u -> PlainCallGraphStatistics() - else -> TODO("Unsupported yet") - } + val callGraphStatistics = createCallGraphStatistics() val timeStatistics = createTimeStatistics() @@ -229,6 +250,7 @@ class TsMachine( coverageStatisticsFactory = { coverageStatistics }, cfgStatisticsFactory = { cfgStatistics }, callGraphStatisticsFactory = { callGraphStatistics }, + coverageDistanceCalculatorFactory = { getCoverageDistanceCalculator(coverageStatistics) }, ) val statesCollector = createStatesCollector(coverageStatistics) @@ -325,6 +347,19 @@ class TsMachine( ) } + private fun createCallGraphStatistics(): PlainCallGraphStatistics = when (options.targetSearchDepth) { + 0u -> PlainCallGraphStatistics() + else -> TODO("Unsupported yet") + } + + private fun getCoverageDistanceCalculator( + coverageStatistics: CoverageStatistics, + ): CallStackDistanceCalculator = coverageDistanceCalculator ?: CallStackDistanceCalculator( + targets = coverageStatistics.getUncoveredStatements().filterNot { it in externallyCoveredStatements }, + cfgStatistics = cfgStatistics, + applicationGraph = graph, + ).also { coverageDistanceCalculator = it } + private fun createStatesCollector( coverageStatistics: CoverageStatistics, ): StatesCollector = when (options.stateCollectionStrategy) { diff --git a/usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt b/usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt new file mode 100644 index 0000000000..d2ce585a46 --- /dev/null +++ b/usvm-ts/src/test/kotlin/org/usvm/machine/TsMachineExternalCoverageTest.kt @@ -0,0 +1,103 @@ +package org.usvm.machine + +import org.jacodb.ets.model.EtsScene +import org.jacodb.ets.utils.EtsIrProvider +import org.jacodb.ets.utils.loadEtsFileAutoConvert +import org.junit.jupiter.api.Test +import org.usvm.CoverageZone +import org.usvm.PathSelectionStrategy +import org.usvm.StateCollectionStrategy +import org.usvm.machine.state.TsState +import org.usvm.statistics.UMachineObserver +import org.usvm.util.TsMethodTestRunner +import org.usvm.util.getResourcePath +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration + +/** External coverage affects scheduling only, independently of native collection and stopping. */ +class TsMachineExternalCoverageTest : TsMethodTestRunner() { + override val scene = EtsScene( + projectFiles = listOf( + loadEtsFileAutoConvert( + getResourcePath("/samples/coverage/ClassCoverage.ts"), + provider = EtsIrProvider.TS_FRONTEND, + ), + ), + ) + + @Test + fun `TypeScript defaults to NURS and respects an explicit strategy`() { + TsMachine(scene = scene, tsOptions = TsOptions()).use { machine -> + assertEquals( + expected = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + actual = machine.options.pathSelectionStrategies, + ) + } + val explicitOptions = options.copy(pathSelectionStrategies = listOf(PathSelectionStrategy.BFS)) + + TsMachine(scene = scene, options = explicitOptions, tsOptions = TsOptions()).use { machine -> + assertEquals(expected = listOf(PathSelectionStrategy.BFS), actual = machine.options.pathSelectionStrategies) + } + } + + @Test + fun `external saturation before analysis neither stops execution nor suppresses native witnesses`() { + val method = getMethod(methodName = "entry", className = "HelperCoverage") + val statements = scene.projectClasses.flatMap { owner -> owner.methods.flatMap { it.cfg.stmts } } + val analysisOptions = options.copy( + coverageZone = CoverageZone.CLASS, + stateCollectionStrategy = StateCollectionStrategy.COVERED_NEW, + stopOnCoverage = 100, + timeout = Duration.INFINITE, + ) + + TsMachine(scene = scene, options = analysisOptions, tsOptions = TsOptions()).use { machine -> + assertTrue(machine.markCovered(statements)) + assertFalse(machine.markCovered(statements)) + + repeat(2) { + val result = machine.analyzeWithMetadata(methods = listOf(method)) + + assertEquals(expected = TsAnalysisStopReason.EXHAUSTED, actual = result.stopReason) + assertEquals(expected = 2, actual = result.states.size) + assertFalse(result.engineFailed) + } + } + } + + @Test + fun `external coverage can arrive on the analysis thread while paths are queued`() { + val method = getMethod(methodName = "entry", className = "HelperCoverage") + val statements = scene.projectClasses.flatMap { owner -> owner.methods.flatMap { it.cfg.stmts } } + lateinit var machine: TsMachine + var reported = false + val observer = object : UMachineObserver { + override fun onStatePeeked(state: TsState) { + if (!reported) { + reported = machine.markCovered(statements) + } + } + } + val analysisOptions = options.copy( + coverageZone = CoverageZone.CLASS, + stateCollectionStrategy = StateCollectionStrategy.COVERED_NEW, + stopOnCoverage = 0, + timeout = Duration.INFINITE, + ) + machine = TsMachine( + scene = scene, + options = analysisOptions, + tsOptions = TsOptions(), + machineObserver = observer, + ) + + val result = machine.use { it.analyzeWithMetadata(methods = listOf(method)) } + + assertTrue(reported) + assertEquals(expected = TsAnalysisStopReason.EXHAUSTED, actual = result.stopReason) + assertEquals(expected = 2, actual = result.states.size) + assertFalse(result.engineFailed) + } +} diff --git a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt index 1c5248ffef..a2b866e182 100644 --- a/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt +++ b/usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt @@ -11,22 +11,18 @@ import kotlin.NoSuchElementException // TODO: what to do if elements have same priority? class DeterministicPriorityCollection(private val comparator: Comparator) : UPriorityCollection { - private inner class Entry(val element: T, val priority: Priority) : Comparable { - override fun compareTo(other: Entry): Int = comparator.compare(priority, other.priority) - } - - private var topElement: Entry? = null - private var priorityQueue = PriorityQueue() + private var topElement: Pair? = null + private val priorityQueue = PriorityQueue> { (_, p1), (_, p2) -> comparator.compare(p1, p2) } override val count: Int get() = priorityQueue.size + if (topElement == null) 0 else 1 override fun peek(): T { val currentTop = topElement - if (currentTop != null) return currentTop.element + if (currentTop != null) return currentTop.first val next = priorityQueue.remove() topElement = next - return next.element + return next.first } override fun update(element: T, priority: Priority) { @@ -36,45 +32,25 @@ class DeterministicPriorityCollection(private val comparator: Compa override fun remove(element: T) { // Don't traverse the whole queue if we remove the top element - if (topElement != null && topElement?.element == element) { + if (topElement != null && topElement?.first == element) { topElement = null return } - if (!priorityQueue.removeIf { it.element == element }) { + if (!priorityQueue.removeIf { (e, _) -> e == element }) { throw NoSuchElementException("Element not found in priority queue") } } override fun add(element: T, priority: Priority) { - assert({ !priorityQueue.any { it.element == element } }) { "Element already exists in priority queue" } + assert({ !priorityQueue.any { (e, _) -> e == element } }) { "Element already exists in priority queue" } val currentTop = topElement - if (currentTop != null && comparator.compare(currentTop.priority, priority) > 0) { - topElement = Entry(element, priority) + if (currentTop != null && comparator.compare(currentTop.second, priority) > 0) { + topElement = element to priority priorityQueue.add(currentTop) return } - priorityQueue.add(Entry(element, priority)) - } - - /** - * Recalculates each priority once and rebuilds the heap in linear time, without removing and reinserting elements. - * If [priority] throws, the collection keeps its previous priorities and cached selection. - */ - fun updatePriorities(priority: (T) -> Priority) { - val entries = ArrayList(count) - val currentTop = topElement - if (currentTop != null) { - entries.add(Entry(currentTop.element, priority(currentTop.element))) - } - for (entry in priorityQueue) { - entries.add(Entry(entry.element, priority(entry.element))) - } - - // The collection constructor heapifies Comparable entries; addAll would perform individual insertions. - val refreshedQueue = PriorityQueue(entries) - priorityQueue = refreshedQueue - topElement = null + priorityQueue.add(element to priority) } } diff --git a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt b/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt deleted file mode 100644 index 2537c630db..0000000000 --- a/usvm-util/src/test/kotlin/org/usvm/algorithms/DeterministicPriorityCollectionRefreshTest.kt +++ /dev/null @@ -1,108 +0,0 @@ -package org.usvm.algorithms - -import org.junit.jupiter.api.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertTrue - -internal class DeterministicPriorityCollectionRefreshTest { - @Test - fun `refresh reorders every element including the cached selection`() { - for (comparator in listOf(naturalOrder(), reverseOrder())) { - val collection = DeterministicPriorityCollection(comparator) - collection.add(element = "first", priority = 1) - collection.add(element = "second", priority = 2) - collection.add(element = "third", priority = 3) - collection.peek() - val evaluations = mutableListOf() - val priorities = mapOf("first" to 3, "second" to 1, "third" to 2) - - collection.updatePriorities { element -> - evaluations.add(element) - priorities.getValue(element) - } - - assertEquals(expected = 3, actual = collection.count) - assertEquals(expected = 3, actual = evaluations.size) - assertEquals(expected = priorities.keys, actual = evaluations.toSet()) - val expected = priorities.keys.sortedWith { left, right -> - comparator.compare(priorities.getValue(left), priorities.getValue(right)) - } - assertEquals(expected = expected, actual = drain(collection)) - } - } - - @Test - fun `heap rebuilding uses a linear number of comparisons`() { - var comparisons = 0 - val comparator = Comparator { left, right -> - comparisons++ - left.compareTo(right) - } - val collection = DeterministicPriorityCollection(comparator) - val size = 1024 - repeat(size) { element -> collection.add(element = element, priority = element) } - collection.peek() - comparisons = 0 - - collection.updatePriorities { element -> size - element } - - assertTrue(comparisons <= 2 * size, "Heapify used $comparisons comparisons for $size elements") - assertEquals(expected = (0 until size).reversed().toList(), actual = drain(collection)) - } - - @Test - fun `equal refreshed priorities retain every element`() { - val collection = DeterministicPriorityCollection(naturalOrder()) - repeat(8) { element -> collection.add(element = element, priority = element) } - collection.peek() - - collection.updatePriorities { 0 } - - val elements = drain(collection) - assertEquals(expected = 8, actual = elements.size) - assertEquals(expected = (0 until 8).toSet(), actual = elements.toSet()) - } - - @Test - fun `failed refresh leaves old ordering and all elements intact`() { - val collection = DeterministicPriorityCollection(naturalOrder()) - collection.add(element = "first", priority = 1) - collection.add(element = "second", priority = 2) - collection.peek() - - assertFailsWith { - collection.updatePriorities { element -> - check(element != "second") - 100 - } - } - - assertEquals(expected = "first", actual = collection.peek()) - assertEquals(expected = listOf("first", "second"), actual = drain(collection)) - } - - @Test - fun `empty and singleton collections can be refreshed repeatedly`() { - val collection = DeterministicPriorityCollection(naturalOrder()) - - collection.updatePriorities { error("Empty collection") } - collection.add(element = "only", priority = 1) - collection.peek() - repeat(2) { collection.updatePriorities { 2 } } - - assertEquals(expected = listOf("only"), actual = drain(collection)) - collection.updatePriorities { error("Empty collection") } - assertEquals(expected = 0, actual = collection.count) - } - - private fun drain(collection: DeterministicPriorityCollection): List { - val elements = mutableListOf() - while (collection.count > 0) { - val element = collection.peek() - collection.remove(element) - elements.add(element) - } - return elements - } -} From 9677d6e2ca13f64294684e4d96b257b8bee5c19e Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 19:29:53 +0300 Subject: [PATCH 09/10] Use NURS in the TS DSL and retain lazy guidance measurements --- .../coverage-selector-performance/README.md | 23 +++++++++- .../lazy-nurs/exploratory-outcomes.jsonl | 6 +++ .../lazy-nurs/exploratory-results.jsonl | 6 +++ .../lazy-nurs/manifest.json | 46 +++++++++++++++++++ .../lazy-nurs/sequential-outcomes.jsonl | 6 +++ .../lazy-nurs/sequential-results.jsonl | 6 +++ .../org/usvm/ts/pbt/dsl/TsPropertyTest.kt | 2 + 7 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl create mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md index 5df8bce855..e944031cf8 100644 --- a/docs/experiments/coverage-selector-performance/README.md +++ b/docs/experiments/coverage-selector-performance/README.md @@ -1,10 +1,29 @@ # Coverage selector performance on real JVM bytecode -These are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). +The original 28 runs below are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. -## Recorded results (2026-10-16) +## Lazy NURS follow-up (2026-10-16) + +The current harness uses the production `createClosestToUncoveredRandomPathSelector` with one aggregate `CallStackDistanceCalculator`. Coverage reports remove goals; normal add/update operations recompute only the affected state's weight. No queue-wide refresh or per-target calculator is used. This checks core NURS on real JVM bytecode, not TS-machine performance or concrete replay coverage. + +Two batches contain 12 successful runs and 1,094,076 interpreter steps, with seed 0 and the same limits as above. The first batch partly overlapped a dependent PBT build; its raw measurements are retained separately. The table uses only the subsequent sequential batch, which ran after that build finished. Other host activity was not controlled. + +| Program | Mode | Steps | Analysis seconds | Selector seconds | Covered goals | Global refreshes | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| codec | nurs | 100,000 | 11.829 | 1.277 | 74 | 0 | +| codec | uniform | 100,000 | 10.851 | 1.029 | 74 | 0 | +| gcd | nurs | 100,000 | 9.247 | 0.508 | 28 | 0 | +| gcd | uniform | 100,000 | 9.641 | 0.608 | 28 | 0 | +| loan | nurs | 73,519 | 3.344 | 0.328 | 123 | 0 | +| loan | uniform | 73,519 | 7.063 | 0.682 | 123 | 0 | + +Both strategies reached the step limit for Codec and Guava and exhausted LoanExam. NURS made 104,817 / 104,581 / 73,519 aggregate distance calls respectively, only through state add/update. All runs recorded zero distance work in peek and zero global refreshes. Goal counts match the uniform baseline in these runs; equal counts do not prove equal covered sets. Timing varied considerably between batches, so these short runs do not establish a stable speedup or a TS performance conclusion. + +Raw measurements, process outcomes, source hashes and batch conditions are in [`lazy-nurs/`](lazy-nurs/). The old results and their manifest are unchanged. To reproduce the current comparison, use the current checkout with `--modes nurs,uniform --seed 0 --repetitions 1`; to reproduce the original periodic modes, use the historical commit noted above. + +## Original recorded results (2026-10-16) The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl new file mode 100644 index 0000000000..b8bbbb4988 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl @@ -0,0 +1,6 @@ +{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 34.428} +{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.777} +{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 23.742} +{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 37.788} +{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.323} +{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 67.84} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl new file mode 100644 index 0000000000..ca88bc184c --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl @@ -0,0 +1,6 @@ +{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":8079.354458,"selectorMs":672.783741,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1092642416,"gcCount":7,"gcMs":502} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3722.539417,"selectorMs":318.351377,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1086488832,"gcCount":3,"gcMs":98} +{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":17174.910917,"selectorMs":1584.688466,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1053153056,"gcCount":6,"gcMs":735} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":21399.094625,"selectorMs":1286.346016,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1029830680,"gcCount":4,"gcMs":265} +{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":4847.856875,"selectorMs":534.124959,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":755228528,"gcCount":2,"gcMs":115} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":17242.797666,"selectorMs":1517.318704,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":737997944,"gcCount":5,"gcMs":343} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json b/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json new file mode 100644 index 0000000000..54358caa3d --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json @@ -0,0 +1,46 @@ +{ + "recordedDate": "2026-10-16", + "productionRevision": "465c303de7f1968a85511fd12d6b7d651ec29b97", + "cpu": "Apple M3 Pro", + "architecture": "arm64", + "memoryBytes": 19327352832, + "java": "Zulu 11.0.32.1+1", + "testHeapBytes": 4294967296, + "settings": { + "stepLimit": 100000, + "timeoutMillis": 120000, + "solver": "YICES", + "solverTimeoutMillis": 250, + "typeTimeoutMillis": 250, + "arrayMaxSize": 1500, + "useSoftConstraints": false, + "assertions": false, + "dynamicGoals": false, + "randomSeeds": [ + 0 + ] + }, + "counts": { + "runs": 12, + "interpreterSteps": 1094076, + "peakQueue": 4817 + }, + "librarySha256": { + "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", + "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" + }, + "sourceSha256": { + "usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt": "6fe66b001e94c4fca877626f109c79eb02b9a3e65c4e75af2f6c33f77fffb379", + "usvm-core/src/main/kotlin/org/usvm/statistics/distances/CallStackDistanceCalculator.kt": "7121a8c6152361afe49ef7b92d10d36ce1a9a76a226c03d0fd206acc5751b770", + "usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt": "674c113ed05d6ec26f7dbe2e608415cf8893f8b765bac58c9e26ac660fab3b6c", + "usvm-util/src/main/kotlin/org/usvm/algorithms/RandomizedPriorityCollection.kt": "f4b859ccef3f263d70853c6148c6809a205734d75197a7eccf41287fe569c9b8", + "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "be70752c25cae757597770a35731ebc9e675d427efaa9d5eb9a92448f67625ac", + "docs/experiments/coverage-selector-performance/run.py": "316e4eed00fd3f0c7cb4808df9401c714d760597931b0ccefbe089792aa708ab", + "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0" + }, + "harnessProvenance": "Current production NURS factory and aggregate CallStackDistanceCalculator; external coverage-report callback removes targets on the analysis thread. No concrete replay or public TsMachine run.", + "batches": { + "exploratory": "Some runs overlapped compilation and tests of the dependent PBT checkout. Retained for transparency; do not interpret timing differences as selector effects.", + "sequential": "Fresh test JVM per run, sequential runs after the dependent build finished. No concurrent build or test was launched by this chat; other host activity was not controlled." + } +} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl new file mode 100644 index 0000000000..2560960892 --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl @@ -0,0 +1,6 @@ +{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 22.268} +{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 19.444} +{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 15.125} +{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 16.442} +{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 10.253} +{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 13.169} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl new file mode 100644 index 0000000000..8fdf8de67f --- /dev/null +++ b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl @@ -0,0 +1,6 @@ +{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":11829.20275,"selectorMs":1276.802646,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1098245584,"gcCount":5,"gcMs":356} +{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10851.263541,"selectorMs":1029.121608,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1076156496,"gcCount":6,"gcMs":263} +{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9246.969,"selectorMs":508.044296,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1047281200,"gcCount":2,"gcMs":112} +{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9640.541209,"selectorMs":608.421144,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1026080192,"gcCount":6,"gcMs":162} +{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3343.874542,"selectorMs":327.584665,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":749417280,"gcCount":2,"gcMs":88} +{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":7062.51475,"selectorMs":681.600274,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743486632,"gcCount":3,"gcMs":577} diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt index 9b32e22127..b0974e53b2 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/dsl/TsPropertyTest.kt @@ -3,6 +3,7 @@ package org.usvm.ts.pbt.dsl import kotlinx.serialization.json.Json import org.jacodb.ets.utils.EtsIrProvider import org.jacodb.ets.utils.loadEtsProjectAutoConvert +import org.usvm.PathSelectionStrategy import org.usvm.SolverType import org.usvm.UMachineOptions import org.usvm.ts.pbt.backend.ProjectionCapability @@ -149,6 +150,7 @@ class DslPropertyRunner(sourceRoots: List) { val manifest = compiled.definition.toManifest() val mapping = PropertyEtsMapper(scene = scene, sourceRoots = sourceRoots).map(manifest) val options = UMachineOptions( + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), solverType = SolverType.YICES, timeout = configuration.timeoutMillis.milliseconds, ) From 42cc2c9338c66b6a32d0090f0943aa7913556058 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Fri, 16 Oct 2026 19:35:51 +0300 Subject: [PATCH 10/10] Remove coverage selector benchmark artifacts --- .../CoverageSelectorPerformanceTest.kt | 366 ------------------ .../coverage-selector-performance/README.md | 149 ------- .../lazy-nurs/exploratory-outcomes.jsonl | 6 - .../lazy-nurs/exploratory-results.jsonl | 6 - .../lazy-nurs/manifest.json | 46 --- .../lazy-nurs/sequential-outcomes.jsonl | 6 - .../lazy-nurs/sequential-results.jsonl | 6 - .../coverage-selector-performance/logback.xml | 3 - .../manifest.json | 42 -- .../performance.gradle | 19 - .../results.jsonl | 28 -- .../coverage-selector-performance/run.py | 91 ----- .../summarize.py | 41 -- .../coverage-selector-performance/summary.csv | 25 -- .../CoverageSelectorPerformanceTest.kt | 323 ---------------- 15 files changed, 1157 deletions(-) delete mode 100644 docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt delete mode 100644 docs/experiments/coverage-selector-performance/README.md delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/logback.xml delete mode 100644 docs/experiments/coverage-selector-performance/manifest.json delete mode 100644 docs/experiments/coverage-selector-performance/performance.gradle delete mode 100644 docs/experiments/coverage-selector-performance/results.jsonl delete mode 100644 docs/experiments/coverage-selector-performance/run.py delete mode 100644 docs/experiments/coverage-selector-performance/summarize.py delete mode 100644 docs/experiments/coverage-selector-performance/summary.csv delete mode 100644 usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt diff --git a/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt b/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt deleted file mode 100644 index df0da45000..0000000000 --- a/docs/experiments/coverage-selector-performance/CoverageSelectorPerformanceTest.kt +++ /dev/null @@ -1,366 +0,0 @@ -package org.usvm.experiments - -import kotlinx.coroutines.runBlocking -import org.jacodb.api.jvm.JcClasspath -import org.jacodb.api.jvm.JcMethod -import org.jacodb.api.jvm.cfg.JcInst -import org.jacodb.api.jvm.ext.findClass -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.usvm.SolverType -import org.usvm.UMachine -import org.usvm.UMachineOptions -import org.usvm.UPathSelector -import org.usvm.algorithms.DeterministicPriorityCollection -import org.usvm.algorithms.RandomizedPriorityCollection -import org.usvm.machine.JcApplicationGraph -import org.usvm.machine.JcComponents -import org.usvm.machine.JcContext -import org.usvm.machine.JcMachineOptions -import org.usvm.machine.JcTypeSystem -import org.usvm.machine.interpreter.JcInterpreter -import org.usvm.machine.state.JcState -import org.usvm.ps.BfsPathSelector -import org.usvm.ps.CoverageGuidedPathSelector -import org.usvm.ps.RandomTreePathSelector -import org.usvm.ps.WeightedPathSelector -import org.usvm.samples.JacoDBContainer -import org.usvm.statistics.CompositeUMachineObserver -import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.UMachineObserver -import org.usvm.statistics.distances.CallStackDistanceCalculator -import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.statistics.distances.MultiTargetDistanceCalculator -import org.usvm.stopstrategies.StopStrategy -import org.usvm.util.originalInst -import java.io.File -import java.lang.management.ManagementFactory -import kotlin.random.Random -import kotlin.time.Duration.Companion.milliseconds - -/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ -@Tag("manual") -class CoverageSelectorPerformanceTest { - @Test - fun runProgram() { - val program = System.getProperty("coverage.perf.case", "codec") - val mode = System.getProperty("coverage.perf.mode", "128") - val steps = System.getProperty("coverage.perf.steps", "100000").toLong() - val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() - val resultFile = File(System.getProperty("coverage.perf.results")) - val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) - val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) - val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) - try { - val (className, methodName) = when (program) { - "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" - "gcd" -> "com.google.common.math.IntMath" to "gcd" - "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" - else -> error("Unknown program: $program") - } - val cls = container.cp.findClass(className) - val method = cls.declaredMethods.single { it.name == methodName } - val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } - PerformanceMachine(container.cp).use { machine -> - val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) - resultFile.parentFile.mkdirs() - resultFile.appendText(result + "\n") - } - } finally { - runBlocking { - container.cp.close() - container.db.close() - } - } - } -} - -private class PerformanceMachine(cp: JcClasspath) : UMachine() { - override val options = UMachineOptions( - randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), - solverType = SolverType.YICES, - solverTimeout = 250.milliseconds, - typeOperationsTimeout = 250.milliseconds, - throwExceptionOnStepFailure = true, - useSoftConstraints = false, - ) - private val graph = JcApplicationGraph(cp) - private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) - private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) - - fun measure( - program: String, - mode: String, - method: JcMethod, - methodsToCover: List, - stepLimit: Long, - timeoutMillis: Long, - ): String { - val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) - val goals = coverage.getUncoveredStatements() - val metrics = Metrics() - val initial = interpreter.getInitialState(method) - val (base, markCovered) = createGuidance(mode, goals, metrics, initial) - val selector = MeasuredSelector(base, metrics) - coverage.addOnCoveredObserver { _, _, statement -> - val started = System.nanoTime() - markCovered(statement) - metrics.selectorNanos += System.nanoTime() - started - metrics.coverageReports++ - } - val observer = metrics.observer() - val gcCountBefore = gcCount() - val gcMillisBefore = gcMillis() - val allocatedBefore = allocatedBytes() - val started = System.nanoTime() - var stop = "exhausted" - selector.add(states = listOf(initial)) - run( - interpreter = interpreter, - pathSelector = selector, - observer = CompositeUMachineObserver(coverage, observer), - isStateTerminated = { state -> state.callStack.isEmpty() }, - stopStrategy = StopStrategy { - val elapsedMillis = (System.nanoTime() - started) / 1_000_000 - when { - metrics.steps >= stepLimit -> { - stop = "steps" - true - } - elapsedMillis >= timeoutMillis -> { - stop = "timeout" - true - } - else -> { - false - } - } - }, - ) - val elapsedNanos = System.nanoTime() - started - val allocated = allocatedBytes() - allocatedBefore - val totals = AnalysisTotals( - elapsedNanos = elapsedNanos, - allocatedBytes = allocated, - gcCount = gcCount() - gcCountBefore, - gcMillis = gcMillis() - gcMillisBefore, - ) - return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) - } - - private fun createGuidance( - mode: String, - goals: Collection, - metrics: Metrics, - initial: JcState, - ): Pair, (JcInst) -> Unit> { - val cfg = CfgStatisticsImpl(graph) - val calculators = MultiTargetDistanceCalculator { target -> - CallStackDistanceCalculator(targets = listOf(target), cfgStatistics = cfg, applicationGraph = graph) - } - val distance: (JcState, JcInst) -> UInt = { state, target -> - metrics.distanceCalls++ - calculators.calculateDistance(state.currentStatement.originalInst(), state.callStack, target) - } - val markCovered: (JcInst) -> Unit - val base: UPathSelector - if (mode == "eager") { - val eager = EagerCoverageSelector(goals, distance) - base = eager - markCovered = { eager.markCovered(it) } - } else if (mode == "existing") { - val aggregate = CallStackDistanceCalculator(targets = goals, cfgStatistics = cfg, applicationGraph = graph) - base = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state: JcState -> - metrics.distanceCalls++ - aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) - }, - ) - markCovered = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } - } else if (mode == "random-path" || mode == "uniform") { - base = createRandomSelector(mode, initial) - markCovered = {} - } else if (mode == "bfs") { - base = BfsPathSelector() - markCovered = {} - } else { - val periodic = CoverageGuidedPathSelector( - targets = goals, - distance = distance, - priorityRefreshInterval = mode.toInt(), - ) - base = periodic - markCovered = { periodic.markCovered(targets = listOf(it)) } - } - return base to markCovered - } - - private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { - val random = Random(seed = options.randomSeed) - if (mode == "uniform") { - return WeightedPathSelector( - priorityCollectionFactory = { - RandomizedPriorityCollection( - comparator = compareBy { it.id }, - unitIntervalRandom = { random.nextDouble() }, - ) - }, - weighter = { 1.0 }, - ) - } - - return RandomTreePathSelector.fromRoot( - root = initial.pathNode, - randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, - ) - } - - override fun close() = components.close() -} - -private class Metrics { - var steps = 0L - var terminated = 0L - var selectorNanos = 0L - var refreshNanos = 0L - var refreshes = 0L - var distanceCalls = 0L - var coverageReports = 0L - var queueSize = 0 - var peakQueue = 0 -} - -private class MeasuredSelector( - private val delegate: UPathSelector, - private val metrics: Metrics, -) : UPathSelector { - override fun isEmpty(): Boolean = delegate.isEmpty() - - override fun peek(): JcState { - val beforeCalls = metrics.distanceCalls - val started = System.nanoTime() - val result = delegate.peek() - val elapsed = System.nanoTime() - started - metrics.selectorNanos += elapsed - if (metrics.distanceCalls > beforeCalls) { - metrics.refreshes++ - metrics.refreshNanos += elapsed - } - return result - } - - override fun add(states: Collection) { - val started = System.nanoTime() - delegate.add(states) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize += states.size - metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) - } - - override fun update(state: JcState) { - val started = System.nanoTime() - delegate.update(state) - metrics.selectorNanos += System.nanoTime() - started - } - - override fun remove(state: JcState) { - val started = System.nanoTime() - delegate.remove(state) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize-- - } -} - -/** Original PR algorithm, using the same queue ordering fix as the periodic implementation. */ -private class EagerCoverageSelector( - targets: Collection, - distance: (JcState, JcInst) -> UInt, -) : UPathSelector { - private val pending = targets.toMutableSet() - private var dirty = false - private val delegate = WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = { state -> pending.minOfOrNull { distance(state, it) } ?: UInt.MAX_VALUE }, - ) - - fun markCovered(target: JcInst) { - if (pending.remove(target)) dirty = true - } - - override fun isEmpty(): Boolean = delegate.isEmpty() - override fun peek(): JcState { - if (dirty) { - val states = delegate.drain().toList() - delegate.add(states) - dirty = false - } - return delegate.peek() - } - override fun add(states: Collection) = delegate.add(states) - override fun update(state: JcState) = delegate.update(state) - override fun remove(state: JcState) = delegate.remove(state) -} - -private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionCount.coerceAtLeast(minimumValue = 0) -} - -private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionTime.coerceAtLeast(minimumValue = 0) -} -private fun allocatedBytes(): Long { - val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean - return bean.getThreadAllocatedBytes(Thread.currentThread().id) -} - -private data class AnalysisTotals( - val elapsedNanos: Long, - val allocatedBytes: Long, - val gcCount: Long, - val gcMillis: Long, -) - -private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { - override fun onStatePeeked(state: JcState) { - steps++ - } - - override fun onStateTerminated(state: JcState, stateReachable: Boolean) { - terminated++ - } -} - -private fun Metrics.report( - program: String, - mode: String, - goals: Int, - covered: Int, - stop: String, - totals: AnalysisTotals, -): String { - val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() - val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() - return """ - {"program":"$program", - "mode":"$mode", - "repetition":$repetition, - "randomSeed":$randomSeed, - "steps":$steps, - "stop":"$stop", - "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, - "selectorMs":${selectorNanos / 1_000_000.0}, - "refreshMs":${refreshNanos / 1_000_000.0}, - "refreshes":$refreshes, - "distanceCalls":$distanceCalls, - "peakQueue":$peakQueue, - "pending":$queueSize, - "goals":$goals, - "covered":$covered, - "coverageReports":$coverageReports, - "terminated":$terminated, - "allocatedBytes":${totals.allocatedBytes}, - "gcCount":${totals.gcCount}, - "gcMs":${totals.gcMillis}} - """.trimIndent().replace("\n", "") -} diff --git a/docs/experiments/coverage-selector-performance/README.md b/docs/experiments/coverage-selector-performance/README.md deleted file mode 100644 index e944031cf8..0000000000 --- a/docs/experiments/coverage-selector-performance/README.md +++ /dev/null @@ -1,149 +0,0 @@ -# Coverage selector performance on real JVM bytecode - -The original 28 runs below are historical results for the superseded periodic selector. Reproduce them at commit `44869512bca6b1ce89511b9f6799f4475efdf972`, where the harness and old selector are compiled together. The preserved harness in this directory is evidence, not part of current test sources. The current compiled harness supports `nurs`, `existing`, `uniform`, and `random-path`; the runner defaults to `nurs,uniform`. None of the original 28 measurements benchmarks `CLOSEST_TO_UNCOVERED_RANDOM` (NURS). - -This experiment drives the production `JcInterpreter` through `UMachine.run`, with the coverage-guided selector installed directly in the execution loop. It does not use mock states. It does not call the public `JcMachine.analyze` wrapper or concretely replay generated inputs. - -## Lazy NURS follow-up (2026-10-16) - -The current harness uses the production `createClosestToUncoveredRandomPathSelector` with one aggregate `CallStackDistanceCalculator`. Coverage reports remove goals; normal add/update operations recompute only the affected state's weight. No queue-wide refresh or per-target calculator is used. This checks core NURS on real JVM bytecode, not TS-machine performance or concrete replay coverage. - -Two batches contain 12 successful runs and 1,094,076 interpreter steps, with seed 0 and the same limits as above. The first batch partly overlapped a dependent PBT build; its raw measurements are retained separately. The table uses only the subsequent sequential batch, which ran after that build finished. Other host activity was not controlled. - -| Program | Mode | Steps | Analysis seconds | Selector seconds | Covered goals | Global refreshes | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| codec | nurs | 100,000 | 11.829 | 1.277 | 74 | 0 | -| codec | uniform | 100,000 | 10.851 | 1.029 | 74 | 0 | -| gcd | nurs | 100,000 | 9.247 | 0.508 | 28 | 0 | -| gcd | uniform | 100,000 | 9.641 | 0.608 | 28 | 0 | -| loan | nurs | 73,519 | 3.344 | 0.328 | 123 | 0 | -| loan | uniform | 73,519 | 7.063 | 0.682 | 123 | 0 | - -Both strategies reached the step limit for Codec and Guava and exhausted LoanExam. NURS made 104,817 / 104,581 / 73,519 aggregate distance calls respectively, only through state add/update. All runs recorded zero distance work in peek and zero global refreshes. Goal counts match the uniform baseline in these runs; equal counts do not prove equal covered sets. Timing varied considerably between batches, so these short runs do not establish a stable speedup or a TS performance conclusion. - -Raw measurements, process outcomes, source hashes and batch conditions are in [`lazy-nurs/`](lazy-nurs/). The old results and their manifest are unchanged. To reproduce the current comparison, use the current checkout with `--modes nurs,uniform --seed 0 --repetitions 1`; to reproduce the original periodic modes, use the historical commit noted above. - -## Original recorded results (2026-10-16) - -The periodic selector remains expensive on these programs. Most measured selector time is outside full refreshes: every ordinary state update still scans the entire goal snapshot through the per-target distance callback. Increasing the interval to 512 does not remove that cost. - -The distance-guidance comparison below contains 16 runs and 1,328,129 interpreter steps. The random comparison adds 12 runs; the complete retained dataset contains 28 runs and 2,422,205 interpreter steps. Commons Codec has two fresh-process repetitions per mode; Guava and LoanExam have one run per mode. Numeric entries below are per-metric medians where there are two runs. - -| Program | Mode | Steps | Stop | Analysis seconds | Selector seconds | Selector share | Refresh ms | -| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | -| Codec | existing | 100,000 | steps | 17.145 | 0.134 | 0.78% | 0 | -| Codec | eager | 100,000 | steps | 15.300 | 4.284 | 27.96% | 13.86 | -| Codec | 128 | 100,000 | steps | 21.468 | 6.319 | 29.48% | 90.32 | -| Codec | 512 | 100,000 | steps | 19.110 | 5.842 | 30.61% | 146.87 | -| Guava gcd | existing | 61,538 | timeout | 120.167 | 0.235 | 0.20% | 0 | -| Guava gcd | eager | 57,141 | timeout | 120.024 | 10.822 | 9.02% | 5.36 | -| Guava gcd | 128 | 63,808 | timeout | 120.189 | 14.249 | 11.86% | 5.73 | -| Guava gcd | 512 | 51,566 | timeout | 120.269 | 8.584 | 7.14% | 3.48 | -| LoanExam | existing | 73,519 | exhausted | 2.227 | 0.069 | 3.08% | 0 | -| LoanExam | eager | 73,519 | exhausted | 2.003 | 0.248 | 12.38% | 29.85 | -| LoanExam | 128 | 73,519 | exhausted | 2.333 | 0.266 | 11.39% | 27.08 | -| LoanExam | 512 | 73,519 | exhausted | 2.319 | 0.244 | 10.51% | 16.97 | - -For interval 128: - -| Program | Initial goals | Covered goals | Peak queued states | Per-target distance calls | Analysis-thread allocated GB (decimal) | -| --- | ---: | ---: | ---: | ---: | ---: | -| Codec | 273 | 74 | 3,364 | 23,826,680 | 5.952 | -| Guava gcd | 673 | 17 | 3,022 | 43,862,759 | 10.762 | -| LoanExam | 126 | 123 | 4,378 | 886,279 | 0.717 | - -All distance-guidance modes in the preceding table reached the same final covered-goal count within each program in these runs. Queue order and solver work still differed. For example, Guava interval 128 made more steps than `existing` within the same time budget despite spending much more time in the selector. The measurements do not establish that the complete analysis is always slower. - -Codec whole-analysis times varied substantially between repetitions: interval 128 took 15.007–27.930 seconds, and `existing` took 14.066–20.224 seconds. Its per-target callback count was identical in both repetitions, and its selector share was 29.33–29.62%. Do not treat the median wall-time ratio as a stable speedup estimate. - -Coverage-guided runs produced only 1–26 peeks with distance refresh work per analysis. They do not establish the worst-case cost of continuously discovering new goals or changing coverage while the queue is large. Initial goals are static; dynamic `addTargets` performance is not exercised. - -A useful next change is a caller-supplied batched minimum-distance function over the snapshot, allowing domain-specific caches such as the aggregate `CallStackDistanceCalculator`. A generic per-target callback may depend on arbitrary mutable state, so the selector cannot safely introduce a statement/call-stack-only cache for every caller. The proposed batched API itself has not been implemented or benchmarked here. - -The production selector files match the recorded source hashes throughout the retained runs. The manifest records the final reproducible harness; setup-helper extraction and report formatting were refined between exploratory runs and the repeated Codec runs, without changes to the measured execution loop, selector algorithms or settings. - -Raw data: [results.jsonl](results.jsonl). Full median summary: [summary.csv](summary.csv). Source/settings checksums: [manifest.json](manifest.json). - -Validation after the HashSet change: `:usvm-core:check` passed (274 tests passed, one existing skipped test). Strict Detekt on the changed core main/tests and the manual JVM experiment reported no findings. All 28 retained experiment processes completed successfully. The four distance-guidance Guava analyses ended at the explicit time limit; the random Guava runs reached 100,000 steps. The experiment is excluded from ordinary JVM test runs. - - -## Random baselines - -Two additional strategies were measured once for each selector RNG seed, 0 and 1, with the same step/time limits and production interpreter: - -- `random-path`: the actual `RandomTreePathSelector.fromRoot` used by `RANDOM_PATH`; it samples branches of the execution tree and traverses that tree on selection. -- `uniform`: the existing `RandomizedPriorityCollection` with constant weight 1.0 in a `WeightedPathSelector`, giving uniform random selection among queued states. It performs no goal-distance calculations and does not traverse the execution tree. - -| Program | Mode | Seed | Steps | Analysis seconds | Selector seconds | Covered goals | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| codec | random-path | 0 | 100,000 | 13.359 | 6.585 | 74 | -| codec | random-path | 1 | 100,000 | 7.283 | 3.991 | 74 | -| codec | uniform | 0 | 100,000 | 3.937 | 0.339 | 74 | -| codec | uniform | 1 | 100,000 | 3.430 | 0.315 | 74 | -| gcd | random-path | 0 | 100,000 | 19.171 | 3.411 | 28 | -| gcd | random-path | 1 | 100,000 | 18.281 | 3.186 | 28 | -| gcd | uniform | 0 | 100,000 | 10.153 | 0.572 | 28 | -| gcd | uniform | 1 | 100,000 | 7.328 | 0.351 | 28 | -| loan | random-path | 0 | 73,519 | 3.868 | 0.883 | 123 | -| loan | random-path | 1 | 73,519 | 5.003 | 1.155 | 123 | -| loan | uniform | 0 | 73,519 | 3.046 | 0.254 | 123 | -| loan | uniform | 1 | 73,519 | 2.745 | 0.224 | 123 | - -Uniform random was a strong inexpensive baseline on both library methods in these runs. Codec reached the same covered-goal count (74) in 3.430–3.937 seconds, versus a 21.468-second median for interval 128. Guava uniform random reached 100,000 steps and 28 covered goals in 7.328–10.153 seconds; interval 128 reached 63,808 steps and 17 covered goals at the 120-second timeout. LoanExam is a counterexample: uniform random took 2.745–3.046 seconds, versus 2.333 seconds for interval 128, with the same 73,519 steps and 123 covered goals. - -The standard `RANDOM_PATH` strategy was itself relatively expensive on Codec: 3.991–6.585 seconds spent in its selector, versus 0.315–0.339 seconds for uniform random. A random strategy's data structure and selection distribution matter. - -Different exploration order changes work outside the selector as well. Guava's wall-time difference cannot be explained by selector overhead alone; its interpreter/solver cost has not been separately profiled. Two seeds and three programs support using cheap uniform random as a baseline, not a universal claim that it dominates all coverage guidance. Goal counts do not establish equality of covered-goal sets or concrete replay coverage. - -## Programs and setup - -- Apache Commons Codec 1.16.0: `org.apache.commons.codec.binary.BinaryCodec.toAsciiBytes(byte[])`. -- Guava 32.0.1-jre: `com.google.common.math.IntMath.gcd(int, int)`. -- Repository sample: `org.usvm.samples.psbenchmarks.LoanExam.getCreditPercent(CreditRequest)`. - -Coverage goals are the statements of all declared, non-abstract, non-native, non-constructor methods in the entry point's class, including static initialization. Coverage is reported by the standard `CoverageStatistics` observer for terminated reachable symbolic states. Class-wide goals intentionally include methods not visited by a particular entry point. - -Each analysis allows 100,000 interpreter steps or 120 seconds, whichever comes first. The timeout is checked between steps, so an in-flight solver operation may overshoot. Reaching the time limit is recorded as `stop=timeout`, not treated as exhaustive analysis. There is no early stop for coverage saturation or coverage stagnation, and no loop-iteration cap. - -Settings: Yices, 250 ms solver/type-operation timeout, implicit exceptions enabled, 1,500-element array bound, soft constraints disabled, 4 GiB Java heap, assertions disabled. Each run uses a fresh test JVM. JacoDB/classpath setup and Gradle startup are outside the analysis timer; initial queue insertion and the complete execution loop are inside it. These are cold-start analysis measurements, not JMH steady-state measurements. - -The recorded host was an Apple M3 Pro (arm64), 18 GiB RAM, Zulu JDK 11.0.32.1+1. Runs were sequential; other host activity was not controlled. - -## Selectors - -- `existing`: the aggregate `CallStackDistanceCalculator` / `WeightedPathSelector` strategy used by `CLOSEST_TO_UNCOVERED`. Coverage reports remove goals from that calculator. Queued state weights are refreshed on normal updates, without a global rebuild. -- `eager`: the original #526 algorithm, refreshing via `drain().toList()` and re-adding states after changed coverage. It uses the same cached-ordering fix and queue implementation as the periodic variants. -- `128` and `512`: the new `CoverageGuidedPathSelector`, using `HashSet` snapshots and the respective refresh interval. - -For the random strategies, the selector RNG seed is recorded separately for every row and CSV group. The variants may explore different paths. Whole-analysis times, allocations and step counts therefore include differences in solver work and exploration order; they are not isolated measurements of the queue implementation. - -`selectorMs` measures add/update/remove/peek and coverage-report handling, including distance evaluation. `refreshMs` identifies distance work inside `peek` and includes heap rebuilding plus selecting its root. `distanceCalls` counts per-target callbacks in eager/periodic modes, but aggregate calculator calls in `existing`; those are different units of work. `allocatedBytes` is all allocation on the analysis thread, not retained heap, native memory, or allocation attributed specifically to the selector. - -## Reproduction - -Use JDK 11 and cached project dependencies. The runner locates the two library jars in the Gradle cache, or accepts explicit `--codec-jar` and `--guava-jar` paths. It runs Gradle offline and retains per-run logs, raw JSONL measurements and process outcomes. - -```sh -JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ - --cases codec,gcd,loan --modes existing,128,eager,512 \ - --steps 100000 --timeout-ms 120000 --repetitions 1 \ - --output build/coverage-selector-performance - -JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ - --cases codec,gcd,loan --modes random-path,uniform --seed 0 --repetitions 1 \ - --output build/coverage-selector-performance - -JAVA_HOME=/path/to/jdk11 python3 docs/experiments/coverage-selector-performance/run.py \ - --cases codec,gcd,loan --modes random-path,uniform --seed 1 --repetitions 1 \ - --output build/coverage-selector-performance - -python3 docs/experiments/coverage-selector-performance/summarize.py \ - build/coverage-selector-performance/results.jsonl -``` - -The experiment is tagged `manual` and excluded from ordinary test runs. The runner's dedicated Gradle task selects only this experiment and disables assertions, avoiding the debug duplicate-element scan in queue insertion. - -Library SHA-256: - -- Commons Codec: `56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d`. -- Guava: `bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5`. diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl deleted file mode 100644 index b8bbbb4988..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-outcomes.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 34.428} -{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.777} -{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 23.742} -{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 37.788} -{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 14.323} -{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 67.84} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl deleted file mode 100644 index ca88bc184c..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/exploratory-results.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":8079.354458,"selectorMs":672.783741,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1092642416,"gcCount":7,"gcMs":502} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3722.539417,"selectorMs":318.351377,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1086488832,"gcCount":3,"gcMs":98} -{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":17174.910917,"selectorMs":1584.688466,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1053153056,"gcCount":6,"gcMs":735} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":21399.094625,"selectorMs":1286.346016,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1029830680,"gcCount":4,"gcMs":265} -{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":4847.856875,"selectorMs":534.124959,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":755228528,"gcCount":2,"gcMs":115} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":17242.797666,"selectorMs":1517.318704,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":737997944,"gcCount":5,"gcMs":343} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json b/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json deleted file mode 100644 index 54358caa3d..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/manifest.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "recordedDate": "2026-10-16", - "productionRevision": "465c303de7f1968a85511fd12d6b7d651ec29b97", - "cpu": "Apple M3 Pro", - "architecture": "arm64", - "memoryBytes": 19327352832, - "java": "Zulu 11.0.32.1+1", - "testHeapBytes": 4294967296, - "settings": { - "stepLimit": 100000, - "timeoutMillis": 120000, - "solver": "YICES", - "solverTimeoutMillis": 250, - "typeTimeoutMillis": 250, - "arrayMaxSize": 1500, - "useSoftConstraints": false, - "assertions": false, - "dynamicGoals": false, - "randomSeeds": [ - 0 - ] - }, - "counts": { - "runs": 12, - "interpreterSteps": 1094076, - "peakQueue": 4817 - }, - "librarySha256": { - "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", - "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" - }, - "sourceSha256": { - "usvm-core/src/main/kotlin/org/usvm/ps/PathSelectorFactory.kt": "6fe66b001e94c4fca877626f109c79eb02b9a3e65c4e75af2f6c33f77fffb379", - "usvm-core/src/main/kotlin/org/usvm/statistics/distances/CallStackDistanceCalculator.kt": "7121a8c6152361afe49ef7b92d10d36ce1a9a76a226c03d0fd206acc5751b770", - "usvm-core/src/main/kotlin/org/usvm/ps/WeightedPathSelector.kt": "674c113ed05d6ec26f7dbe2e608415cf8893f8b765bac58c9e26ac660fab3b6c", - "usvm-util/src/main/kotlin/org/usvm/algorithms/RandomizedPriorityCollection.kt": "f4b859ccef3f263d70853c6148c6809a205734d75197a7eccf41287fe569c9b8", - "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "be70752c25cae757597770a35731ebc9e675d427efaa9d5eb9a92448f67625ac", - "docs/experiments/coverage-selector-performance/run.py": "316e4eed00fd3f0c7cb4808df9401c714d760597931b0ccefbe089792aa708ab", - "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0" - }, - "harnessProvenance": "Current production NURS factory and aggregate CallStackDistanceCalculator; external coverage-report callback removes targets on the analysis thread. No concrete replay or public TsMachine run.", - "batches": { - "exploratory": "Some runs overlapped compilation and tests of the dependent PBT checkout. Retained for transparency; do not interpret timing differences as selector effects.", - "sequential": "Fresh test JVM per run, sequential runs after the dependent build finished. No concurrent build or test was launched by this chat; other host activity was not controlled." - } -} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl deleted file mode 100644 index 2560960892..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-outcomes.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program": "codec", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "codec-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 22.268} -{"program": "codec", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "codec-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 19.444} -{"program": "gcd", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "gcd-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 15.125} -{"program": "gcd", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "gcd-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 16.442} -{"program": "loan", "mode": "nurs", "repetition": 1, "randomSeed": 0, "log": "loan-nurs-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 10.253} -{"program": "loan", "mode": "uniform", "repetition": 1, "randomSeed": 0, "log": "loan-uniform-s0-r1.log", "exitCode": 0, "status": "completed", "wallSeconds": 13.169} diff --git a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl b/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl deleted file mode 100644 index 8fdf8de67f..0000000000 --- a/docs/experiments/coverage-selector-performance/lazy-nurs/sequential-results.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"program":"codec","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":11829.20275,"selectorMs":1276.802646,"refreshMs":0.0,"refreshes":0,"distanceCalls":104817,"peakQueue":4817,"pending":4817,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1098245584,"gcCount":5,"gcMs":356} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10851.263541,"selectorMs":1029.121608,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1076156496,"gcCount":6,"gcMs":263} -{"program":"gcd","mode":"nurs","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9246.969,"selectorMs":508.044296,"refreshMs":0.0,"refreshes":0,"distanceCalls":104581,"peakQueue":4582,"pending":4581,"goals":673,"covered":28,"coverageReports":28,"terminated":1417,"allocatedBytes":1047281200,"gcCount":2,"gcMs":112} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":9640.541209,"selectorMs":608.421144,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1026080192,"gcCount":6,"gcMs":162} -{"program":"loan","mode":"nurs","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3343.874542,"selectorMs":327.584665,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":3087,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":749417280,"gcCount":2,"gcMs":88} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":7062.51475,"selectorMs":681.600274,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743486632,"gcCount":3,"gcMs":577} diff --git a/docs/experiments/coverage-selector-performance/logback.xml b/docs/experiments/coverage-selector-performance/logback.xml deleted file mode 100644 index 463a09ebff..0000000000 --- a/docs/experiments/coverage-selector-performance/logback.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/docs/experiments/coverage-selector-performance/manifest.json b/docs/experiments/coverage-selector-performance/manifest.json deleted file mode 100644 index 1e41f1c164..0000000000 --- a/docs/experiments/coverage-selector-performance/manifest.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "recordedDate": "2026-10-16", - "productionRevision": "deca5d3bc", - "cpu": "Apple M3 Pro", - "architecture": "arm64", - "memoryBytes": 19327352832, - "java": "Zulu 11.0.32.1+1", - "testHeapBytes": 4294967296, - "settings": { - "stepLimit": 100000, - "timeoutMillis": 120000, - "solver": "YICES", - "solverTimeoutMillis": 250, - "typeTimeoutMillis": 250, - "arrayMaxSize": 1500, - "useSoftConstraints": false, - "assertions": false, - "dynamicGoals": false, - "randomSeeds": [ - 0, - 1 - ] - }, - "counts": { - "runs": 28, - "interpreterSteps": 2422205, - "peakQueue": 4921 - }, - "librarySha256": { - "commons-codec-1.16.0.jar": "56595fb20b0b85bc91d0d503dad50bb7f1b9afc0eed5dffa6cbb25929000484d", - "guava-32.0.1-jre.jar": "bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" - }, - "sourceSha256": { - "usvm-core/src/main/kotlin/org/usvm/ps/CoverageGuidedPathSelector.kt": "5905b62b890e94b7390058e46c07fbd9dbb7285521ccc3c65abfceb458ce6b1e", - "usvm-util/src/main/kotlin/org/usvm/algorithms/DeterministicPriorityCollection.kt": "e55fd73657139ec726574f5ddfca43eccfe8d615520b5c30111146a65019bc32", - "usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt": "42c2c518e157ff8edca81044c7561c6832a9868eef0ef0514c223c9ee05209a2", - "docs/experiments/coverage-selector-performance/run.py": "9c6affa7309e0a6bbacfcf6fd5fdb005ee63817aa3d1a8103a3caf635c667a50", - "docs/experiments/coverage-selector-performance/performance.gradle": "0e83e296ce4fe65b0c6caa5afff775ae2be5a29e5e5d81823336b5c44d2e40d0", - "docs/experiments/coverage-selector-performance/summarize.py": "67b695a12348503a3700d26b3fc938337ff9215a392353060ece05713dbf2fa2" - }, - "harnessProvenance": "Final reproducible harness; setup-helper extraction and report-formatting refinements occurred during exploratory runs. Production selector sources, measured execution loop and settings were unchanged." -} diff --git a/docs/experiments/coverage-selector-performance/performance.gradle b/docs/experiments/coverage-selector-performance/performance.gradle deleted file mode 100644 index 8a301e7f07..0000000000 --- a/docs/experiments/coverage-selector-performance/performance.gradle +++ /dev/null @@ -1,19 +0,0 @@ -gradle.projectsEvaluated { - def jvm = gradle.rootProject.findProject(':usvm-jvm') - if (jvm == null) return - jvm.tasks.register('coverageSelectorPerformance', Test) { - testClassesDirs = jvm.sourceSets.test.output.classesDirs - classpath = jvm.sourceSets.test.runtimeClasspath - dependsOn(jvm.tasks.named('testClasses')) - useJUnitPlatform { includeTags 'manual' } - filter { includeTestsMatching 'org.usvm.experiments.CoverageSelectorPerformanceTest' } - enableAssertions = false - maxHeapSize = '4g' - def logging = jvm.rootProject.file('docs/experiments/coverage-selector-performance/logback.xml') - jvmArgs "-Dlogback.configurationFile=${logging.absolutePath}" - ['case', 'mode', 'steps', 'timeoutMillis', 'results', 'jars', 'repetition', 'seed'].each { key -> - systemProperty("coverage.perf.${key}", jvm.findProperty("coveragePerf${key.capitalize()}")) - } - outputs.upToDateWhen { false } - } -} diff --git a/docs/experiments/coverage-selector-performance/results.jsonl b/docs/experiments/coverage-selector-performance/results.jsonl deleted file mode 100644 index 1fe7bae02f..0000000000 --- a/docs/experiments/coverage-selector-performance/results.jsonl +++ /dev/null @@ -1,28 +0,0 @@ -{"program":"gcd","mode":"existing","steps":61538,"stop":"timeout","elapsedMs":120167.436333,"selectorMs":235.437487,"refreshMs":0.0,"refreshes":0,"distanceCalls":64765,"peakQueue":3227,"pending":3227,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":856651016,"gcCount":5,"gcMs":449,"repetition":1} -{"program":"gcd","mode":"128","steps":63808,"stop":"timeout","elapsedMs":120189.039625,"selectorMs":14249.244039,"refreshMs":5.728084,"refreshes":2,"distanceCalls":43862759,"peakQueue":3022,"pending":3022,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":10761566008,"gcCount":10,"gcMs":304,"repetition":1} -{"program":"gcd","mode":"eager","repetition":1,"steps":57141,"stop":"timeout","elapsedMs":120024.317916,"selectorMs":10822.395495,"refreshMs":5.358375,"refreshes":3,"distanceCalls":39557171,"peakQueue":3124,"pending":3124,"goals":673,"covered":17,"coverageReports":17,"terminated":3,"allocatedBytes":9568604488,"gcCount":12,"gcMs":435} -{"program":"gcd","mode":"512","repetition":1,"steps":51566,"stop":"timeout","elapsedMs":120269.191542,"selectorMs":8584.144519,"refreshMs":3.480708,"refreshes":1,"distanceCalls":35593159,"peakQueue":2655,"pending":2655,"goals":673,"covered":17,"coverageReports":17,"terminated":5,"allocatedBytes":8698358712,"gcCount":9,"gcMs":175} -{"program":"loan","mode":"existing","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2227.168709,"selectorMs":68.547639,"refreshMs":0.0,"refreshes":0,"distanceCalls":73519,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":513509984,"gcCount":2,"gcMs":26} -{"program":"loan","mode":"128","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2332.535792,"selectorMs":265.641903,"refreshMs":27.08429,"refreshes":18,"distanceCalls":886279,"peakQueue":4378,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":716986848,"gcCount":2,"gcMs":24} -{"program":"loan","mode":"eager","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2003.228,"selectorMs":248.054217,"refreshMs":29.854126,"refreshes":26,"distanceCalls":944047,"peakQueue":4379,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":734125672,"gcCount":2,"gcMs":36} -{"program":"loan","mode":"512","repetition":1,"steps":73519,"stop":"exhausted","elapsedMs":2319.290542,"selectorMs":243.802455,"refreshMs":16.965501,"refreshes":12,"distanceCalls":760175,"peakQueue":4360,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":701963040,"gcCount":2,"gcMs":19} -{"program":"codec","mode":"existing","repetition":1,"steps":100000,"stop":"steps","elapsedMs":14065.793084,"selectorMs":103.386872,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2071408672,"gcCount":5,"gcMs":327} -{"program":"codec","mode":"128","repetition":1,"steps":100000,"stop":"steps","elapsedMs":27929.542917,"selectorMs":8192.190751,"refreshMs":93.001749,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952381656,"gcCount":10,"gcMs":367} -{"program":"codec","mode":"eager","repetition":1,"steps":100000,"stop":"steps","elapsedMs":17828.648083,"selectorMs":5027.923518,"refreshMs":13.558208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5478178352,"gcCount":12,"gcMs":287} -{"program":"codec","mode":"512","repetition":1,"steps":100000,"stop":"steps","elapsedMs":18655.336042,"selectorMs":6038.100888,"refreshMs":193.471708,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":6332651112,"gcCount":9,"gcMs":260} -{"program":"codec","mode":"512","repetition":2,"steps":100000,"stop":"steps","elapsedMs":19563.939083,"selectorMs":5645.342032,"refreshMs":100.26425,"refreshes":2,"distanceCalls":23843924,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5946781568,"gcCount":8,"gcMs":210} -{"program":"codec","mode":"eager","repetition":2,"steps":100000,"stop":"steps","elapsedMs":12771.742584,"selectorMs":3540.902094,"refreshMs":14.152208,"refreshes":2,"distanceCalls":21185015,"peakQueue":3868,"pending":3858,"goals":273,"covered":74,"coverageReports":74,"terminated":1316,"allocatedBytes":5479713280,"gcCount":8,"gcMs":186} -{"program":"codec","mode":"128","repetition":2,"steps":100000,"stop":"steps","elapsedMs":15007.014042,"selectorMs":4445.708042,"refreshMs":87.639833,"refreshes":2,"distanceCalls":23826680,"peakQueue":3364,"pending":3348,"goals":273,"covered":74,"coverageReports":74,"terminated":1544,"allocatedBytes":5952287184,"gcCount":7,"gcMs":173} -{"program":"codec","mode":"existing","repetition":2,"steps":100000,"stop":"steps","elapsedMs":20224.028875,"selectorMs":165.022394,"refreshMs":0.0,"refreshes":0,"distanceCalls":103547,"peakQueue":3564,"pending":3547,"goals":273,"covered":74,"coverageReports":74,"terminated":1469,"allocatedBytes":2078604128,"gcCount":6,"gcMs":202} -{"program":"codec","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":13359.287291,"selectorMs":6585.102705,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4615,"pending":4615,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513024960,"gcCount":7,"gcMs":539} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":3936.713375,"selectorMs":338.87798,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4790,"pending":4790,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1080632408,"gcCount":3,"gcMs":62} -{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":19170.84275,"selectorMs":3410.622702,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4921,"pending":4920,"goals":673,"covered":28,"coverageReports":28,"terminated":895,"allocatedBytes":2143921128,"gcCount":3,"gcMs":41} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":0,"steps":100000,"stop":"steps","elapsedMs":10152.718417,"selectorMs":572.30041,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4338,"pending":4338,"goals":673,"covered":28,"coverageReports":28,"terminated":1570,"allocatedBytes":1025193416,"gcCount":5,"gcMs":409} -{"program":"loan","mode":"random-path","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3868.197,"selectorMs":882.931681,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1527,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":881496912,"gcCount":2,"gcMs":50} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":0,"steps":73519,"stop":"exhausted","elapsedMs":3045.632291,"selectorMs":253.601349,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":3049,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":743016424,"gcCount":3,"gcMs":98} -{"program":"codec","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7282.688708,"selectorMs":3990.851612,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4562,"pending":4562,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":2513769624,"gcCount":3,"gcMs":111} -{"program":"codec","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":3429.624834,"selectorMs":314.7182,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4780,"pending":4780,"goals":273,"covered":74,"coverageReports":74,"terminated":258,"allocatedBytes":1087211024,"gcCount":4,"gcMs":122} -{"program":"gcd","mode":"random-path","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":18280.874209,"selectorMs":3186.388872,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4898,"pending":4897,"goals":673,"covered":28,"coverageReports":28,"terminated":911,"allocatedBytes":2151045424,"gcCount":3,"gcMs":32} -{"program":"gcd","mode":"uniform","repetition":1,"randomSeed":1,"steps":100000,"stop":"steps","elapsedMs":7327.864792,"selectorMs":351.211791,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":4444,"pending":4440,"goals":673,"covered":28,"coverageReports":28,"terminated":1521,"allocatedBytes":1029454208,"gcCount":4,"gcMs":259} -{"program":"loan","mode":"random-path","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":5003.377792,"selectorMs":1155.413364,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":1513,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":878046240,"gcCount":2,"gcMs":39} -{"program":"loan","mode":"uniform","repetition":1,"randomSeed":1,"steps":73519,"stop":"exhausted","elapsedMs":2744.655583,"selectorMs":224.198052,"refreshMs":0.0,"refreshes":0,"distanceCalls":0,"peakQueue":2902,"pending":0,"goals":126,"covered":123,"coverageReports":123,"terminated":5339,"allocatedBytes":747822944,"gcCount":2,"gcMs":26} diff --git a/docs/experiments/coverage-selector-performance/run.py b/docs/experiments/coverage-selector-performance/run.py deleted file mode 100644 index c1e9ef53d9..0000000000 --- a/docs/experiments/coverage-selector-performance/run.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Run isolated JVM analyses; keep successes, failures and timeouts in the output directory.""" - -import argparse -import json -import os -from pathlib import Path -import subprocess -import time - - -def cached_jar(group, artifact, version): - cache = Path.home() / '.gradle/caches/modules-2/files-2.1' - matches = list((cache / group / artifact / version).glob(f'*/{artifact}-{version}.jar')) - if len(matches) != 1: - raise RuntimeError(f'Pass the jar explicitly: {group}:{artifact}:{version}') - return matches[0] - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--cases', default='codec,gcd,loan') - parser.add_argument('--modes', default='nurs,uniform') - parser.add_argument('--steps', type=int, default=100000) - parser.add_argument('--timeout-ms', type=int, default=120000) - parser.add_argument('--repetitions', type=int, default=2) - parser.add_argument('--seed', type=int, default=0) - parser.add_argument('--output', type=Path) - parser.add_argument('--codec-jar', type=Path) - parser.add_argument('--guava-jar', type=Path) - args = parser.parse_args() - - experiment = Path(__file__).resolve().parent - root = experiment.parents[2] - revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() - historical_revision = '44869512bca6b1ce89511b9f6799f4475efdf972' - historical_modes = {'128', '512', 'eager'} - uses_historical_modes = bool(set(args.modes.split(',')) & historical_modes) - if uses_historical_modes and revision != historical_revision: - parser.error(f'Historical experiment: run this script from a checkout of {historical_revision}') - - output = (args.output or root / 'build/coverage-selector-performance').resolve() - output.mkdir(parents=True, exist_ok=True) - codec = args.codec_jar or cached_jar('commons-codec', 'commons-codec', '1.16.0') - guava = args.guava_jar or cached_jar('com.google.guava', 'guava', '32.0.1-jre') - jars = os.pathsep.join(str(path.resolve()) for path in [codec, guava]) - cases = args.cases.split(',') - modes = args.modes.split(',') - results_file = output / 'results.jsonl' - outcomes_file = output / 'outcomes.jsonl' - - for repetition in range(1, args.repetitions + 1): - # Alternate mode order to reduce a fixed ordering effect. Runs remain sequential. - run_modes = modes if repetition % 2 else list(reversed(modes)) - for case in cases: - for mode in run_modes: - label = f'{case}-{mode}-s{args.seed}-r{repetition}' - print(f'START {label}', flush=True) - log_file = output / f'{label}.log' - command = [ - str(root / 'gradlew'), ':usvm-jvm:coverageSelectorPerformance', - '-I', str(experiment / 'performance.gradle'), - f'-PcoveragePerfCase={case}', f'-PcoveragePerfMode={mode}', - f'-PcoveragePerfSteps={args.steps}', - f'-PcoveragePerfTimeoutMillis={args.timeout_ms}', - f'-PcoveragePerfRepetition={repetition}', f'-PcoveragePerfSeed={args.seed}', - f'-PcoveragePerfResults={results_file}', f'-PcoveragePerfJars={jars}', - '--offline', '--console=plain', - ] - started = time.monotonic() - outcome = dict(program=case, mode=mode, repetition=repetition, randomSeed=args.seed, log=log_file.name) - try: - with log_file.open('w') as log: - process = subprocess.run( - command, cwd=root, env=os.environ.copy(), stdout=log, - stderr=subprocess.STDOUT, timeout=args.timeout_ms / 1000 + 180, - ) - outcome['exitCode'] = process.returncode - outcome['status'] = 'completed' if process.returncode == 0 else 'failed' - except subprocess.TimeoutExpired: - outcome['status'] = 'process-timeout' - outcome['wallSeconds'] = round(time.monotonic() - started, 3) - with outcomes_file.open('a') as outcomes: - outcomes.write(json.dumps(outcome) + '\n') - print(json.dumps(outcome), flush=True) - if outcome['status'] != 'completed': - print(log_file.read_text()[-4000:], flush=True) - - -if __name__ == '__main__': - main() diff --git a/docs/experiments/coverage-selector-performance/summarize.py b/docs/experiments/coverage-selector-performance/summarize.py deleted file mode 100644 index 5ba0af99d9..0000000000 --- a/docs/experiments/coverage-selector-performance/summarize.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -"""Write per-program/per-mode medians from benchmark JSONL to CSV.""" - -import argparse -import csv -import json -from collections import defaultdict -from pathlib import Path -from statistics import median -import sys - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('results', type=Path) - args = parser.parse_args() - groups = defaultdict(list) - for line in args.results.read_text().splitlines(): - row = json.loads(line) - seed = row.get('randomSeed') if row['mode'] in ['random-path', 'uniform'] else None - groups[row['program'], row['mode'], seed].append(row) - - metrics = ['steps', 'elapsedMs', 'selectorMs', 'selectorPercent', 'refreshMs', - 'refreshes', 'distanceCalls', 'peakQueue', 'covered', 'goals', - 'allocatedBytes', 'gcMs'] - fields = ['program', 'mode', 'randomSeed', 'runs', 'stops', *metrics] - writer = csv.DictWriter(sys.stdout, fieldnames=fields, lineterminator='\n') - writer.writeheader() - for (program, mode, seed), rows in sorted(groups.items()): - summary = dict(program=program, mode=mode, randomSeed=seed, runs=len(rows)) - summary['stops'] = ','.join(sorted({row['stop'] for row in rows})) - for field in metrics: - if field == 'selectorPercent': - summary[field] = median(100 * row['selectorMs'] / row['elapsedMs'] for row in rows) - else: - summary[field] = median(row[field] for row in rows) - writer.writerow(summary) - - -if __name__ == '__main__': - main() diff --git a/docs/experiments/coverage-selector-performance/summary.csv b/docs/experiments/coverage-selector-performance/summary.csv deleted file mode 100644 index fc5876db60..0000000000 --- a/docs/experiments/coverage-selector-performance/summary.csv +++ /dev/null @@ -1,25 +0,0 @@ -program,mode,randomSeed,runs,stops,steps,elapsedMs,selectorMs,selectorPercent,refreshMs,refreshes,distanceCalls,peakQueue,covered,goals,allocatedBytes,gcMs -codec,128,,2,steps,100000.0,21468.2784795,6318.9493965,29.477916632853606,90.320791,2.0,23826680.0,3364.0,74.0,273.0,5952334420.0,270.0 -codec,512,,2,steps,100000.0,19109.6375625,5841.72146,30.611235605310384,146.867979,2.0,23843924.0,3364.0,74.0,273.0,6139716340.0,235.0 -codec,eager,,2,steps,100000.0,15300.1953335,4284.412806,27.962937677238415,13.855208000000001,2.0,21185015.0,3868.0,74.0,273.0,5478945816.0,236.5 -codec,existing,,2,steps,100000.0,17144.9109795,134.204633,0.7754976584015962,0.0,0.0,103547.0,3564.0,74.0,273.0,2075006400.0,264.5 -codec,random-path,0,1,steps,100000,13359.287291,6585.102705,49.29232047757748,0.0,0,0,4615,74,273,2513024960,539 -codec,random-path,1,1,steps,100000,7282.688708,3990.851612,54.79915141252801,0.0,0,0,4562,74,273,2513769624,111 -codec,uniform,0,1,steps,100000,3936.713375,338.87798,8.608144604888842,0.0,0,0,4790,74,273,1080632408,62 -codec,uniform,1,1,steps,100000,3429.624834,314.7182,9.176461427500849,0.0,0,0,4780,74,273,1087211024,122 -gcd,128,,1,timeout,63808,120189.039625,14249.244039,11.855693400545382,5.728084,2,43862759,3022,17,673,10761566008,304 -gcd,512,,1,timeout,51566,120269.191542,8584.144519,7.137442605991306,3.480708,1,35593159,2655,17,673,8698358712,175 -gcd,eager,,1,timeout,57141,120024.317916,10822.395495,9.01683565706588,5.358375,3,39557171,3124,17,673,9568604488,435 -gcd,existing,,1,timeout,61538,120167.436333,235.437487,0.19592453179043556,0.0,0,64765,3227,17,673,856651016,449 -gcd,random-path,0,1,steps,100000,19170.84275,3410.622702,17.79067694872204,0.0,0,0,4921,28,673,2143921128,41 -gcd,random-path,1,1,steps,100000,18280.874209,3186.388872,17.430177767052758,0.0,0,0,4898,28,673,2151045424,32 -gcd,uniform,0,1,steps,100000,10152.718417,572.30041,5.636917980919514,0.0,0,0,4338,28,673,1025193416,409 -gcd,uniform,1,1,steps,100000,7327.864792,351.211791,4.7928257544192965,0.0,0,0,4444,28,673,1029454208,259 -loan,128,,1,exhausted,73519,2332.535792,265.641903,11.388545629656944,27.08429,18,886279,4378,123,126,716986848,24 -loan,512,,1,exhausted,73519,2319.290542,243.802455,10.511941069261688,16.965501,12,760175,4360,123,126,701963040,19 -loan,eager,,1,exhausted,73519,2003.228,248.054217,12.382725131637535,29.854126,26,944047,4379,123,126,734125672,36 -loan,existing,,1,exhausted,73519,2227.168709,68.547639,3.0777928372915193,0.0,0,73519,4379,123,126,513509984,26 -loan,random-path,0,1,exhausted,73519,3868.197,882.931681,22.82540628101413,0.0,0,0,1527,123,126,881496912,50 -loan,random-path,1,1,exhausted,73519,5003.377792,1155.413364,23.09266683494125,0.0,0,0,1513,123,126,878046240,39 -loan,uniform,0,1,exhausted,73519,3045.632291,253.601349,8.326722492055428,0.0,0,0,3049,123,126,743016424,98 -loan,uniform,1,1,exhausted,73519,2744.655583,224.198052,8.168531359222275,0.0,0,0,2902,123,126,747822944,26 diff --git a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt b/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt deleted file mode 100644 index 4f41716094..0000000000 --- a/usvm-jvm/src/test/kotlin/org/usvm/experiments/CoverageSelectorPerformanceTest.kt +++ /dev/null @@ -1,323 +0,0 @@ -package org.usvm.experiments - -import kotlinx.coroutines.runBlocking -import org.jacodb.api.jvm.JcClasspath -import org.jacodb.api.jvm.JcMethod -import org.jacodb.api.jvm.cfg.JcInst -import org.jacodb.api.jvm.ext.findClass -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.usvm.SolverType -import org.usvm.UMachine -import org.usvm.UMachineOptions -import org.usvm.UPathSelector -import org.usvm.algorithms.DeterministicPriorityCollection -import org.usvm.algorithms.RandomizedPriorityCollection -import org.usvm.machine.JcApplicationGraph -import org.usvm.machine.JcComponents -import org.usvm.machine.JcContext -import org.usvm.machine.JcMachineOptions -import org.usvm.machine.JcTypeSystem -import org.usvm.machine.interpreter.JcInterpreter -import org.usvm.machine.state.JcState -import org.usvm.ps.BfsPathSelector -import org.usvm.ps.createClosestToUncoveredRandomPathSelector -import org.usvm.ps.RandomTreePathSelector -import org.usvm.ps.WeightedPathSelector -import org.usvm.samples.JacoDBContainer -import org.usvm.statistics.CompositeUMachineObserver -import org.usvm.statistics.CoverageStatistics -import org.usvm.statistics.UMachineObserver -import org.usvm.statistics.distances.CallStackDistanceCalculator -import org.usvm.statistics.distances.CfgStatisticsImpl -import org.usvm.stopstrategies.StopStrategy -import org.usvm.util.originalInst -import java.io.File -import java.lang.management.ManagementFactory -import kotlin.random.Random -import kotlin.time.Duration.Companion.milliseconds - -/** Manual end-to-end runs over real JVM bytecode; not a microbenchmark or concrete replay. */ -@Tag("manual") -class CoverageSelectorPerformanceTest { - @Test - fun runProgram() { - val program = System.getProperty("coverage.perf.case", "codec") - val mode = System.getProperty("coverage.perf.mode", "nurs") - val steps = System.getProperty("coverage.perf.steps", "100000").toLong() - val timeoutMillis = System.getProperty("coverage.perf.timeoutMillis", "120000").toLong() - val resultFile = File(System.getProperty("coverage.perf.results")) - val libraries = System.getProperty("coverage.perf.jars").split(File.pathSeparatorChar).map(::File) - val samplePaths = System.getenv("usvm.jvm.test.samples").split(File.pathSeparatorChar).map(::File) - val container = JacoDBContainer(key = "coverage-performance", classpath = samplePaths + libraries) - try { - val (className, methodName) = when (program) { - "codec" -> "org.apache.commons.codec.binary.BinaryCodec" to "toAsciiBytes" - "gcd" -> "com.google.common.math.IntMath" to "gcd" - "loan" -> "org.usvm.samples.psbenchmarks.LoanExam" to "getCreditPercent" - else -> error("Unknown program: $program") - } - val cls = container.cp.findClass(className) - val method = cls.declaredMethods.single { it.name == methodName } - val goals = cls.declaredMethods.filter { !it.isAbstract && !it.isNative && !it.isConstructor } - PerformanceMachine(container.cp).use { machine -> - val result = machine.measure(program, mode, method, goals, steps, timeoutMillis) - resultFile.parentFile.mkdirs() - resultFile.appendText(result + "\n") - } - } finally { - runBlocking { - container.cp.close() - container.db.close() - } - } - } -} - -private class PerformanceMachine(cp: JcClasspath) : UMachine() { - override val options = UMachineOptions( - randomSeed = System.getProperty("coverage.perf.seed", "0").toLong(), - solverType = SolverType.YICES, - solverTimeout = 250.milliseconds, - typeOperationsTimeout = 250.milliseconds, - throwExceptionOnStepFailure = true, - useSoftConstraints = false, - ) - private val graph = JcApplicationGraph(cp) - private val components = JcComponents(JcTypeSystem(cp, options.typeOperationsTimeout), options) - private val interpreter = JcInterpreter(JcContext(cp, components), graph, JcMachineOptions(arrayMaxSize = 1500)) - - fun measure( - program: String, - mode: String, - method: JcMethod, - methodsToCover: List, - stepLimit: Long, - timeoutMillis: Long, - ): String { - val coverage = CoverageStatistics(methodsToCover.toHashSet(), graph) - val goals = coverage.getUncoveredStatements() - val metrics = Metrics() - val initial = interpreter.getInitialState(method) - val (base, markCovered) = createGuidance(mode, goals, metrics, initial) - val selector = MeasuredSelector(base, metrics) - coverage.addOnCoveredObserver { _, _, statement -> - val started = System.nanoTime() - markCovered(statement) - metrics.selectorNanos += System.nanoTime() - started - metrics.coverageReports++ - } - val observer = metrics.observer() - val gcCountBefore = gcCount() - val gcMillisBefore = gcMillis() - val allocatedBefore = allocatedBytes() - val started = System.nanoTime() - var stop = "exhausted" - selector.add(states = listOf(initial)) - run( - interpreter = interpreter, - pathSelector = selector, - observer = CompositeUMachineObserver(coverage, observer), - isStateTerminated = { state -> state.callStack.isEmpty() }, - stopStrategy = StopStrategy { - val elapsedMillis = (System.nanoTime() - started) / 1_000_000 - when { - metrics.steps >= stepLimit -> { - stop = "steps" - true - } - elapsedMillis >= timeoutMillis -> { - stop = "timeout" - true - } - else -> { - false - } - } - }, - ) - val elapsedNanos = System.nanoTime() - started - val allocated = allocatedBytes() - allocatedBefore - val totals = AnalysisTotals( - elapsedNanos = elapsedNanos, - allocatedBytes = allocated, - gcCount = gcCount() - gcCountBefore, - gcMillis = gcMillis() - gcMillisBefore, - ) - return metrics.report(program, mode, goals.size, coverage.getTotalCoveredStatements(), stop, totals) - } - - private fun createGuidance( - mode: String, - goals: Collection, - metrics: Metrics, - initial: JcState, - ): Pair, (JcInst) -> Unit> { - if (mode == "random-path" || mode == "uniform") { - return createRandomSelector(mode, initial) to {} - } - if (mode == "bfs") return BfsPathSelector() to {} - require(mode == "nurs" || mode == "existing") { "Historical modes require checkout 44869512b" } - - val aggregate = CallStackDistanceCalculator( - targets = goals, - cfgStatistics = CfgStatisticsImpl(graph), - applicationGraph = graph, - ) - val distance: (JcState) -> UInt = { state -> - metrics.distanceCalls++ - aggregate.calculateDistance(state.currentStatement.originalInst(), state.callStack) - } - val base = if (mode == "nurs") { - createClosestToUncoveredRandomPathSelector( - distance = distance, - random = Random(seed = options.randomSeed), - ) - } else { - WeightedPathSelector( - priorityCollectionFactory = { DeterministicPriorityCollection(naturalOrder()) }, - weighter = distance, - ) - } - val markCovered: (JcInst) -> Unit = { statement -> aggregate.removeTarget(graph.methodOf(statement), statement) } - return base to markCovered - } - - private fun createRandomSelector(mode: String, initial: JcState): UPathSelector { - val random = Random(seed = options.randomSeed) - if (mode == "uniform") { - return WeightedPathSelector( - priorityCollectionFactory = { - RandomizedPriorityCollection( - comparator = compareBy { it.id }, - unitIntervalRandom = { random.nextDouble() }, - ) - }, - weighter = { 1.0 }, - ) - } - - return RandomTreePathSelector.fromRoot( - root = initial.pathNode, - randomNonNegativeInt = { bound -> random.nextInt(from = 0, until = bound) }, - ) - } - - override fun close() = components.close() -} - -private class Metrics { - var steps = 0L - var terminated = 0L - var selectorNanos = 0L - var refreshNanos = 0L - var refreshes = 0L - var distanceCalls = 0L - var coverageReports = 0L - var queueSize = 0 - var peakQueue = 0 -} - -private class MeasuredSelector( - private val delegate: UPathSelector, - private val metrics: Metrics, -) : UPathSelector { - override fun isEmpty(): Boolean = delegate.isEmpty() - - override fun peek(): JcState { - val beforeCalls = metrics.distanceCalls - val started = System.nanoTime() - val result = delegate.peek() - val elapsed = System.nanoTime() - started - metrics.selectorNanos += elapsed - if (metrics.distanceCalls > beforeCalls) { - metrics.refreshes++ - metrics.refreshNanos += elapsed - } - return result - } - - override fun add(states: Collection) { - val started = System.nanoTime() - delegate.add(states) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize += states.size - metrics.peakQueue = maxOf(metrics.peakQueue, metrics.queueSize) - } - - override fun update(state: JcState) { - val started = System.nanoTime() - delegate.update(state) - metrics.selectorNanos += System.nanoTime() - started - } - - override fun remove(state: JcState) { - val started = System.nanoTime() - delegate.remove(state) - metrics.selectorNanos += System.nanoTime() - started - metrics.queueSize-- - } -} - -private fun gcCount(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionCount.coerceAtLeast(minimumValue = 0) -} - -private fun gcMillis(): Long = ManagementFactory.getGarbageCollectorMXBeans().sumOf { - it.collectionTime.coerceAtLeast(minimumValue = 0) -} -private fun allocatedBytes(): Long { - val bean = ManagementFactory.getThreadMXBean() as com.sun.management.ThreadMXBean - return bean.getThreadAllocatedBytes(Thread.currentThread().id) -} - -private data class AnalysisTotals( - val elapsedNanos: Long, - val allocatedBytes: Long, - val gcCount: Long, - val gcMillis: Long, -) - -private fun Metrics.observer(): UMachineObserver = object : UMachineObserver { - override fun onStatePeeked(state: JcState) { - steps++ - } - - override fun onStateTerminated(state: JcState, stateReachable: Boolean) { - terminated++ - } -} - -private fun Metrics.report( - program: String, - mode: String, - goals: Int, - covered: Int, - stop: String, - totals: AnalysisTotals, -): String { - val repetition = System.getProperty("coverage.perf.repetition", "1").toInt() - val randomSeed = System.getProperty("coverage.perf.seed", "0").toLong() - return """ - {"program":"$program", - "mode":"$mode", - "repetition":$repetition, - "randomSeed":$randomSeed, - "steps":$steps, - "stop":"$stop", - "elapsedMs":${totals.elapsedNanos / 1_000_000.0}, - "selectorMs":${selectorNanos / 1_000_000.0}, - "refreshMs":${refreshNanos / 1_000_000.0}, - "refreshes":$refreshes, - "distanceCalls":$distanceCalls, - "peakQueue":$peakQueue, - "pending":$queueSize, - "goals":$goals, - "covered":$covered, - "coverageReports":$coverageReports, - "terminated":$terminated, - "allocatedBytes":${totals.allocatedBytes}, - "gcCount":${totals.gcCount}, - "gcMs":${totals.gcMillis}} - """.trimIndent().replace("\n", "") -}