From 0eab090414de605a4350f328068dc3759d4a4ceb Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Tue, 13 Oct 2026 02:09:30 +0300 Subject: [PATCH 1/2] Collect and replay coverage candidates through current Calls preparation --- .../org/usvm/ts/calls/CallsCoverageCell.kt | 202 +++++++++++++ .../org/usvm/ts/calls/CallsCoverageReplay.kt | 250 ++++++++++++++++ .../org/usvm/ts/calls/CallsCoverageSearch.kt | 267 ++++++++++++++++++ .../org/usvm/ts/calls/CallsExperimentCli.kt | 2 + .../ts/calls/CurrentTsCallsSymbolicEngine.kt | 100 +++++-- .../ts/calls/CallsCoverageExperimentTest.kt | 162 +++++++++++ 6 files changed, 962 insertions(+), 21 deletions(-) create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt new file mode 100644 index 0000000000..2a04d11c7f --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt @@ -0,0 +1,202 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import java.nio.ByteBuffer +import java.nio.channels.FileChannel +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.nio.file.StandardOpenOption +import kotlin.time.Duration.Companion.milliseconds + +@Serializable +internal data class CallsCoverageCellIdentity( + val schemaVersion: Int, + val toolRevision: String, + val nativeFrontendRevision: String, + val sourceManifest: String, + val projectIndex: Int, + val functionIndex: Int, + val profile: CallsExperimentProfile, + val seed: Long, + val budgetMillis: Long, + val solverLimitMillis: Long, + val candidateCap: Int, +) + +@Serializable +internal data class CallsCoverageReplayEvent( + val index: Int, + val replay: CallsCoverageReplayResult? = null, + val error: String? = null, +) + +@Serializable +internal data class CallsCoverageCheckpoint( + val searchBudgetMillis: Long, + val emittedCandidates: Int, + val replayedCandidates: Int, + val coveredStatements: Int, + val coveredIfArms: Int, +) + +@Serializable +internal data class CallsCoverageCellResult( + val identity: CallsCoverageCellIdentity, + val search: CallsCoverageSearchResult, + val replayErrors: Int, + val completionMismatches: Int, + val checkpoints: List, +) + +internal fun runCoverageCell(args: List) { + require(args.size == 9) { "coverage-cell requires 9 arguments" } + val manifestPath = Path.of(args[0]).toRealPath() + val manifest = CallsExperimentJson.decodeManifest(Files.readString(manifestPath)) + val projectIndex = args[1].toInt() + val functionIndex = args[2].toInt() + val profile = CallsExperimentProfile.valueOf(args[3]) + val seed = args[4].toLong() + val output = Path.of(args[5]).toAbsolutePath().normalize() + val budgetMillis = args[6].toLong() + val solverLimitMillis = args[7].toLong() + val candidateCap = args[8].toInt() + val project = manifest.projects[projectIndex] + val function = project.functions[functionIndex] + require(seed in manifest.seeds) { "Seed is outside the frozen source manifest" } + require(budgetMillis > 0 && solverLimitMillis in 1..budgetMillis && candidateCap > 0) + require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" } + + val identity = CallsCoverageCellIdentity( + schemaVersion = 1, + toolRevision = CallsBuildIdentity.toolRevision, + nativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + sourceManifest = manifestPath.toString(), + projectIndex = projectIndex, + functionIndex = functionIndex, + profile = profile, + seed = seed, + budgetMillis = budgetMillis, + solverLimitMillis = solverLimitMillis, + candidateCap = candidateCap, + ) + require(Regex("[0-9a-f]{40}").matches(identity.toolRevision)) { "Coverage build must be clean" } + + Files.createDirectories(output) + val identityPath = output.resolve("identity.json") + val expectedIdentity = encode(identity) + if (Files.exists(identityPath)) { + require(Files.readString(identityPath).trim() == expectedIdentity) { "Cell identity changed" } + } else { + require(Files.list(output).use { entries -> entries.findAny().isEmpty }) { "Output directory is not empty" } + atomicWrite(identityPath, expectedIdentity) + } + require(!Files.exists(output.resolve("result.json"))) { "Cell already completed" } + + val candidatePath = output.resolve("candidates.jsonl") + val searchPath = output.resolve("search.json") + val search = if (Files.exists(searchPath)) { + CallsExperimentJson.json.decodeFromString(Files.readString(searchPath)) + } else { + require(!Files.exists(candidatePath)) { "Interrupted search requires investigation before restart" } + val request = CallsCoverageSearchRequest( + sourceRoot = manifestPath.parent.resolve(project.sourceRoot).toRealPath(), + project = project, + function = function, + profile = profile, + frozenModelIds = manifest.modelSet.ids, + expectedNativeFrontendRevision = identity.nativeFrontendRevision, + seed = seed, + budget = budgetMillis.milliseconds, + solverQueryLimit = solverLimitMillis.milliseconds, + candidateCap = candidateCap, + onCandidate = { candidate -> appendDurably(candidatePath, encode(candidate)) }, + ) + val result = CurrentTsCallsCoverageEngine().search(request) + val persistedCandidates = readLines(candidatePath) + require(result.candidates == persistedCandidates) { "Persisted candidate journal differs from search result" } + atomicWrite(searchPath, encode(result)) + result + } + + val replayPath = output.resolve("replays.jsonl") + val events = readLines(replayPath).toMutableList() + require(events.map(CallsCoverageReplayEvent::index) == events.indices.toList()) { + "Replay journal has missing or duplicate indices" + } + if (search.candidates.isNotEmpty()) { + OriginalTypeScriptCoverageReplayer( + sourceRoot = manifestPath.parent.resolve(project.sourceRoot), + function = function, + ).use { replayer -> + search.candidates.indices.drop(events.size).forEach { index -> + val candidate = search.candidates[index] + val replay = runCatching { + replayer.replay(inputs = candidate.inputs, timeoutMillis = REPLAY_TIMEOUT_MILLIS) + } + val event = CallsCoverageReplayEvent( + index = index, + replay = replay.getOrNull(), + error = replay.exceptionOrNull()?.let { error -> error.message ?: error::class.java.name }, + ) + appendDurably(replayPath, encode(event)) + events += event + } + } + } + + val checkpoints = listOf(30_000L, 60_000L, 120_000L).filter { it <= budgetMillis }.map { checkpoint -> + val indices = search.candidates.indices.filter { index -> + search.candidates[index].emittedAtMillis <= checkpoint + } + val replayed = indices.mapNotNull { index -> events[index].replay } + CallsCoverageCheckpoint( + searchBudgetMillis = checkpoint, + emittedCandidates = indices.size, + replayedCandidates = replayed.size, + coveredStatements = replayed.flatMap { it.coveredStatementKeys }.toSet().size, + coveredIfArms = replayed.flatMap { it.coveredIfArmKeys }.toSet().size, + ) + } + val result = CallsCoverageCellResult( + identity = identity, + search = search, + replayErrors = events.count { it.error != null }, + completionMismatches = events.indices.count { index -> + events[index].replay?.completion != null && + events[index].replay?.completion != search.candidates[index].completion + }, + checkpoints = checkpoints, + ) + atomicWrite(output.resolve("result.json"), encode(result)) + println(encode(result)) +} + +private inline fun readLines(path: Path): List = if (Files.exists(path)) { + Files.readAllLines(path).filter(String::isNotBlank).map { line -> + CallsExperimentJson.json.decodeFromString(line) + } +} else { + emptyList() +} + +private inline fun encode(value: T): String = CallsExperimentJson.json.encodeToUtf8SafeString(value) + +private fun appendDurably(path: Path, line: String) { + val bytes = ByteBuffer.wrap((line + "\n").toByteArray(Charsets.UTF_8)) + FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND).use { channel -> + while (bytes.hasRemaining()) channel.write(bytes) + channel.force(true) + } +} + +private fun atomicWrite(path: Path, contents: String) { + val temporary = path.resolveSibling("${path.fileName}.tmp") + Files.writeString(temporary, contents + "\n") + FileChannel.open(temporary, StandardOpenOption.READ).use { it.force(true) } + Files.move(temporary, path, StandardCopyOption.ATOMIC_MOVE) +} + +private const val REPLAY_TIMEOUT_MILLIS = 20_000L diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt new file mode 100644 index 0000000000..01d5b983bf --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt @@ -0,0 +1,250 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import org.usvm.ts.pbt.backend.CoverageScope +import org.usvm.ts.pbt.backend.PropertyCoverageRequest +import org.usvm.ts.pbt.backend.PropertyRunConfiguration +import org.usvm.ts.pbt.backend.PropertyRunStatus +import org.usvm.ts.pbt.backend.SourceFileCoverage +import org.usvm.ts.pbt.fastcheck.FastCheckBackend +import org.usvm.ts.pbt.fastcheck.TypeScriptSourceInspector +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.PropertyDefinition +import org.usvm.ts.pbt.model.PropertyId +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import kotlin.time.TimeSource + +@Serializable +internal data class CallsCoverageReplayObservation( + val inputs: List, + val completion: CallsCoverageCompletion, + val errorName: String? = null, +) + +@Serializable +internal data class CallsCoverageReplayResult( + val coveredStatementKeys: Set, + val coveredIfArmKeys: Set, + val supportedIfArmKeys: Set, + val completion: CallsCoverageCompletion, + val diagnostics: List, + val replayElapsedMillis: Long, +) + +/** Replays one generated input in a fresh Node.js process and checks its exact transport value. */ +internal class OriginalTypeScriptCoverageReplayer( + sourceRoot: Path, + private val function: CallsFunctionCase, +) : AutoCloseable { + private val sourceRoot = sourceRoot.toRealPath() + private val workspace = Files.createTempDirectory("usvm-ts-calls-coverage-") + private val overlayRoot = workspace.resolve("source-overlay") + private val observationPath = workspace.resolve("observation.json") + private val wrapperName = ".usvm-coverage-${UUID.randomUUID()}.ts" + private val usesSentinel = function.inputs.isEmpty() + private val allowedFiles: Set + private val backend: FastCheckBackend + + init { + try { + require(function.entryPoint.executionKind == org.usvm.ts.pbt.model.ExecutionKind.SYNC) { + "Coverage replay requires a synchronous entry point" + } + val source = this.sourceRoot.resolve(function.sourceFile).normalize() + require(source.startsWith(this.sourceRoot) && Files.isRegularFile(source)) { + "Coverage entry source is outside its frozen root" + } + val closure = TypeScriptSourceInspector.localSourceClosure(sourceRoot = this.sourceRoot, source = source) + require(closure.reasonCode == null) { closure.diagnostic ?: "Unsupported local source closure" } + allowedFiles = closure.files.toSet() + + Files.createDirectory(overlayRoot) + Files.newDirectoryStream(this.sourceRoot).use { entries -> + entries.forEach { entry -> + Files.createSymbolicLink(overlayRoot.resolve(entry.fileName), entry.toAbsolutePath()) + } + } + Files.writeString( + overlayRoot.resolve(wrapperName), + coverageWrapper( + sourceFile = function.sourceFile, + exportName = function.entryPoint.exportName, + observationPath = observationPath, + dropSentinel = usesSentinel, + ), + ) + backend = FastCheckBackend(sourceRoots = listOf(overlayRoot, this.sourceRoot)) + } catch (error: Throwable) { + deleteWorkspace() + throw error + } + } + + fun replay(inputs: List, timeoutMillis: Long): CallsCoverageReplayResult { + require(timeoutMillis > 0) + require(inputs.size == function.inputs.size) + Files.deleteIfExists(observationPath) + + val started = TimeSource.Monotonic.markNow() + val result = backend.run( + property = PropertyDefinition( + id = PropertyId("calls.coverage.replay"), + inputs = if (usesSentinel) { + listOf(PropertyInput(name = "__usvmCoverageSentinel", domain = BooleanDomain)) + } else { + function.inputs + }, + predicate = TypeScriptEntryPoint(module = wrapperName, exportName = "coverageProbe"), + ), + configuration = PropertyRunConfiguration( + seed = 0, + numRuns = 1, + timeoutMillis = timeoutMillis, + examples = listOf(if (usesSentinel) listOf(JsConcreteValue.Boolean(true)) else inputs), + coverageRequest = PropertyCoverageRequest(scopes = setOf(CoverageScope.SOURCE_UNDER_TEST)), + ), + ) + val elapsedMillis = started.elapsedNow().inWholeMilliseconds + require(result.status == PropertyRunStatus.SUCCESS && result.numRuns == 1) { + "Exact coverage replay did not complete once: $result" + } + val observation = CallsExperimentJson.json.decodeFromString( + Files.readString(observationPath), + ) + require(observation.inputs == inputs) { + "Concrete replay received a different tagged input than the symbolic candidate" + } + val coverage = requireNotNull(result.coverage) { "Concrete replay produced no c8 coverage artifact" } + val statementKeys = linkedSetOf() + val ifArmKeys = linkedSetOf() + val supportedIfArmKeys = linkedSetOf() + + coverage.files.forEach { file -> + val relative = relativeSourcePath(file.path) ?: return@forEach + if (relative !in allowedFiles) return@forEach + + collectHits( + relative = relative, + file = file, + statementKeys = statementKeys, + ifArmKeys = ifArmKeys, + supportedIfArmKeys = supportedIfArmKeys, + ) + } + + return CallsCoverageReplayResult( + coveredStatementKeys = statementKeys, + coveredIfArmKeys = ifArmKeys, + supportedIfArmKeys = supportedIfArmKeys, + completion = observation.completion, + diagnostics = coverage.diagnostics.map { diagnostic -> "${diagnostic.code}: ${diagnostic.message}" }, + replayElapsedMillis = elapsedMillis, + ) + } + + private fun relativeSourcePath(path: String): String? { + val sourcePath = Path.of(path).toAbsolutePath().normalize() + val root = when { + sourcePath.startsWith(overlayRoot) -> overlayRoot + sourcePath.startsWith(sourceRoot) -> sourceRoot + else -> return null + } + + return root.relativize(sourcePath).toString().replace('\\', '/') + } + + override fun close() = deleteWorkspace() + + private fun deleteWorkspace() { + Files.walk(workspace).use { paths -> + paths.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } +} + +private fun collectHits( + relative: String, + file: SourceFileCoverage, + statementKeys: MutableSet, + ifArmKeys: MutableSet, + supportedIfArmKeys: MutableSet, +) { + file.statements.filter { statement -> statement.hits > 0 }.forEach { statement -> + statementKeys += "$relative:statement:${statement.location.stableKey()}" + } + + file.branches.filter { branch -> branch.type == "if" && branch.arms.size == 2 }.forEach { branch -> + branch.arms.forEachIndexed { index, arm -> + val key = "$relative:if:${branch.location.stableKey()}:$index" + supportedIfArmKeys += key + if (arm.hits > 0) ifArmKeys += key + } + } +} + +private fun org.usvm.ts.pbt.backend.SourceRange.stableKey(): String = + "${start.line}:${start.column}-${end.line}:${end.column}" + +private fun coverageWrapper( + sourceFile: String, + exportName: String, + observationPath: Path, + dropSentinel: Boolean, +): String { + val moduleName = CallsExperimentJson.json.encodeToUtf8SafeString("./$sourceFile") + val exportKey = CallsExperimentJson.json.encodeToUtf8SafeString(exportName) + val outputPath = CallsExperimentJson.json.encodeToUtf8SafeString(observationPath.toString()) + val sentinelExpression = if (dropSentinel) "args.slice(1)" else "args" + + return """ + import { writeFileSync } from 'node:fs'; + import * as targetModule from $moduleName; + + const callable = targetModule[$exportKey]; + if (typeof callable !== 'function') throw new Error('Coverage export is not a function'); + + function encodeValue(value: unknown): unknown { + if (value === undefined) return { kind: 'undefined' }; + if (value === null) return { kind: 'null' }; + if (typeof value === 'boolean') return { kind: 'boolean', value }; + if (typeof value === 'string') return { kind: 'string', value }; + if (typeof value === 'number') { + if (Number.isNaN(value)) return { kind: 'number', value: 'nan' }; + if (value === Infinity) return { kind: 'number', value: 'positive-infinity' }; + if (value === -Infinity) return { kind: 'number', value: 'negative-infinity' }; + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value, false); + const bits = view.getBigUint64(0, false).toString(16).padStart(16, '0'); + return { kind: 'number', value: 'finite', bits }; + } + if (Array.isArray(value)) return { kind: 'array', elements: value.map(encodeValue) }; + throw new Error('Unsupported replay argument type'); + } + + export function coverageProbe(...args: unknown[]): boolean { + const actualArgs = $sentinelExpression; + let completion: 'RETURNED' | 'THREW' = 'RETURNED'; + let errorName: string | undefined; + try { + const result = callable(...actualArgs); + if (result !== null && (typeof result === 'object' || typeof result === 'function') + && typeof (result as { then?: unknown }).then === 'function') { + throw new Error('Synchronous coverage export returned an awaitable value'); + } + } catch (error: unknown) { + completion = 'THREW'; + errorName = error instanceof Error ? error.name : typeof error; + } + writeFileSync($outputPath, JSON.stringify({ inputs: actualArgs.map(encodeValue), completion, errorName }), 'utf8'); + + return true; + } + """.trimIndent() + "\n" +} diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt new file mode 100644 index 0000000000..d47c253cc9 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt @@ -0,0 +1,267 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import org.jacodb.ets.model.EtsMethod +import org.jacodb.ets.model.EtsStmt +import org.usvm.SolverType +import org.usvm.StateCollectionStrategy +import org.usvm.UMachineOptions +import org.usvm.machine.TsAnalysisStopReason +import org.usvm.machine.TsMachine +import org.usvm.machine.TsOptions +import org.usvm.machine.call.TsUnknownCallModelSelection +import org.usvm.machine.interpreter.TsInitialStateSolverUnknownException +import org.usvm.machine.state.TsMethodResult +import org.usvm.machine.state.TsState +import org.usvm.statistics.UMachineObserver +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.contains +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import kotlin.time.Duration +import kotlin.time.TimeSource + +@Serializable +internal data class CallsCoverageCandidate( + val emittedAtMillis: Long, + val inputs: List, + val newSymbolicStatements: Int, + val completion: CallsCoverageCompletion, +) + +@Serializable +internal enum class CallsCoverageCompletion { + RETURNED, + THREW, +} + +@Serializable +internal enum class CallsCoverageSearchStatus { + EXHAUSTED, + TIMEOUT, + STOPPED, + UNSUPPORTED, + UNMAPPED, + AMBIGUOUS, + UNKNOWN, + TOOL_ERROR, +} + +@Serializable +internal data class CallsCoverageSearchResult( + val status: CallsCoverageSearchStatus, + val candidates: List, + val selectedStates: Int, + val extractionFailures: List, + val candidateCapReached: Boolean, + val preparationElapsedMillis: Long, + val searchElapsedMillis: Long, + val unsupportedCall: Boolean = false, + val engineFailed: Boolean = false, + val runtimeLimited: Boolean = false, + val diagnostic: String? = null, +) + +internal data class CallsCoverageSearchRequest( + val sourceRoot: java.nio.file.Path, + val project: CallsProjectCase, + val function: CallsFunctionCase, + val profile: CallsExperimentProfile, + val frozenModelIds: Set, + val expectedNativeFrontendRevision: String, + val seed: Long, + val budget: Duration, + val solverQueryLimit: Duration, + val candidateCap: Int, + val onCandidate: (CallsCoverageCandidate) -> Unit = {}, +) { + init { + require(budget > Duration.ZERO) + require(solverQueryLimit > Duration.ZERO && solverQueryLimit <= budget) + require(candidateCap > 0) + } +} + +/** Runs one function without a return target and records completed coverage-novel paths. */ +internal class CurrentTsCallsCoverageEngine( + private val sourceEngine: CurrentTsCallsSymbolicEngine = CurrentTsCallsSymbolicEngine(), +) { + fun search(request: CallsCoverageSearchRequest): CallsCoverageSearchResult { + val preparationStarted = TimeSource.Monotonic.markNow() + val preparation = sourceEngine.prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = request.sourceRoot, + project = request.project, + function = request.function, + expectedNativeFrontendRevision = request.expectedNativeFrontendRevision, + ), + ) + val preparationElapsedMillis = preparationStarted.elapsedNow().inWholeMilliseconds + if (preparation is CallsFunctionPreparation.Rejected) { + return CallsCoverageSearchResult( + status = preparation.status.toCoverageStatus(), + candidates = emptyList(), + selectedStates = 0, + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = preparationElapsedMillis, + searchElapsedMillis = 0, + diagnostic = "${preparation.reasonCode}: ${preparation.diagnostic}", + ) + } + + preparation as CallsFunctionPreparation.Eligible + val symbolicInputs = CallsSymbolicInputs( + inputs = request.function.inputs, + bindings = preparation.inputBindings, + lexicalEnvironment = preparation.lexicalEnvironment, + captures = preparation.captures, + ) + val candidates = mutableListOf() + val extractionFailures = mutableListOf() + val searchStarted = TimeSource.Monotonic.markNow() + val observer = CompletedCoverageCandidateObserver( + method = preparation.method, + inputDomains = request.function.inputs, + symbolicInputs = symbolicInputs, + candidateCap = request.candidateCap, + searchStarted = searchStarted, + onCandidate = { candidate -> + request.onCandidate(candidate) + candidates += candidate + }, + onExtractionFailure = extractionFailures::add, + ) + val modelSelection = if (request.profile.usesFrozenModels) { + TsUnknownCallModelSelection.Only(request.frozenModelIds) + } else { + TsUnknownCallModelSelection.Only(emptySet()) + } + val machineOptions = UMachineOptions( + pathSelectionStrategies = listOf(CALLS_PATH_SELECTION_STRATEGY), + stateCollectionStrategy = StateCollectionStrategy.COVERED_NEW, + randomSeed = request.seed, + timeout = request.budget, + solverTimeout = request.solverQueryLimit, + solverType = SolverType.Z3, + stopOnCoverage = CALLS_STOP_ON_COVERAGE, + stopOnTargetsReached = false, + throwExceptionOnStepFailure = true, + ) + val machineResult = runCatching { + TsMachine( + scene = preparation.scene, + options = machineOptions, + tsOptions = TsOptions( + unknownCallModelSelection = modelSelection, + unknownCallFallback = request.profile.fallback, + ), + machineObserver = observer, + ).use { machine -> + machine.analyzeWithMetadata( + methods = listOf(preparation.method), + configureInitialState = { _, state -> symbolicInputs.initialize(state) }, + ) + } + } + val searchElapsedMillis = searchStarted.elapsedNow().inWholeMilliseconds + val outcome = machineResult.getOrNull() + val status = when { + machineResult.exceptionOrNull() is TsInitialStateSolverUnknownException -> CallsCoverageSearchStatus.UNKNOWN + machineResult.isFailure -> CallsCoverageSearchStatus.TOOL_ERROR + outcome?.timedOut == true -> CallsCoverageSearchStatus.TIMEOUT + outcome?.stopReason == TsAnalysisStopReason.EXHAUSTED -> CallsCoverageSearchStatus.EXHAUSTED + else -> CallsCoverageSearchStatus.STOPPED + } + + return CallsCoverageSearchResult( + status = status, + candidates = candidates, + selectedStates = observer.selectedStates, + extractionFailures = extractionFailures, + candidateCapReached = observer.capReached, + preparationElapsedMillis = preparationElapsedMillis, + searchElapsedMillis = searchElapsedMillis, + unsupportedCall = outcome?.unsupportedCall == true, + engineFailed = outcome?.engineFailed == true, + runtimeLimited = outcome?.runtimeLimited == true, + diagnostic = machineResult.exceptionOrNull()?.let { error -> error.message ?: error::class.java.name }, + ) + } +} + +private class CompletedCoverageCandidateObserver( + private val method: EtsMethod, + private val inputDomains: List, + private val symbolicInputs: CallsSymbolicInputs, + private val candidateCap: Int, + private val searchStarted: TimeSource.Monotonic.ValueTimeMark, + private val onCandidate: (CallsCoverageCandidate) -> Unit, + private val onExtractionFailure: (String) -> Unit, +) : UMachineObserver { + private val entryStatements: Set = method.cfg.stmts.toSet() + private val coveredStatements = hashSetOf() + private val inputKeys = hashSetOf() + + var selectedStates: Int = 0 + private set + + var capReached: Boolean = false + private set + + override fun onStateTerminated(state: TsState, stateReachable: Boolean) { + if (!stateReachable) return + + val completion = when (val result = state.methodResult) { + is TsMethodResult.Success.RegularCall -> { + if (result.method != method) return + CallsCoverageCompletion.RETURNED + } + is TsMethodResult.TsException -> CallsCoverageCompletion.THREW + else -> return + } + val pathStatements = state.pathNode.allStatements.filterTo(hashSetOf()) { statement -> + statement in entryStatements + } + val newStatements = pathStatements.count { statement -> statement !in coveredStatements } + if (newStatements == 0) return + + if (selectedStates >= candidateCap) { + capReached = true + return + } + selectedStates++ + val candidate = runCatching { + val inputs = symbolicInputs.resolve(state).firstOrNull { candidate -> + candidate.size == inputDomains.size && + candidate.zip(inputDomains).all { (value, input) -> value in input.domain } + } + requireNotNull(inputs) { "No extracted input is inside every frozen domain" } + val key = CallsExperimentJson.json.encodeToUtf8SafeString(inputs) + if (inputKeys.add(key)) { + CallsCoverageCandidate( + emittedAtMillis = searchStarted.elapsedNow().inWholeMilliseconds, + inputs = inputs, + newSymbolicStatements = newStatements, + completion = completion, + ) + } else { + null + } + }.getOrElse { error -> + onExtractionFailure(error.message ?: error::class.java.name) + null + } + + if (candidate != null) { + onCandidate(candidate) + coveredStatements += pathStatements + } + } +} + +private fun CallsSymbolicStatus.toCoverageStatus(): CallsCoverageSearchStatus = when (this) { + CallsSymbolicStatus.UNSUPPORTED, CallsSymbolicStatus.RUNTIME_LIMITATION -> CallsCoverageSearchStatus.UNSUPPORTED + CallsSymbolicStatus.UNMAPPED -> CallsCoverageSearchStatus.UNMAPPED + CallsSymbolicStatus.AMBIGUOUS -> CallsCoverageSearchStatus.AMBIGUOUS + else -> CallsCoverageSearchStatus.TOOL_ERROR +} diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt index 2dc0a6448d..1229d14d8d 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt @@ -10,6 +10,7 @@ fun main(args: Array) { when (args.first()) { "run" -> runExperiment(args.drop(1)) + "coverage-cell" -> runCoverageCell(args.drop(1)) "replay-witness" -> replayWitness(args.drop(1)) "summarize" -> summarize(args.drop(1)) else -> error(usage()) @@ -88,6 +89,7 @@ private fun summarize(args: List) { private fun usage(): String = """ Usage: calls run + calls coverage-cell calls replay-witness calls summarize """.trimIndent() diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt index 6bb8b72162..f2d88b3e91 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt @@ -43,6 +43,29 @@ internal data class CallsSymbolicPreflightRequest( val expectedNativeFrontendRevision: String, ) +internal data class CallsFunctionPreflightRequest( + val sourceRoot: Path, + val project: CallsProjectCase, + val function: CallsFunctionCase, + val expectedNativeFrontendRevision: String, +) + +internal sealed interface CallsFunctionPreparation { + data class Eligible( + val scene: EtsScene, + val method: EtsMethod, + val inputBindings: List, + val lexicalEnvironment: EtsLexicalEnvironmentBinding?, + val captures: List, + ) : CallsFunctionPreparation + + data class Rejected( + val status: CallsSymbolicStatus, + val reasonCode: CallsSymbolicPreflightReasonCode, + val diagnostic: String, + ) : CallsFunctionPreparation +} + internal enum class CallsSymbolicPreflightStatus { ELIGIBLE, UNSUPPORTED, @@ -263,10 +286,19 @@ internal class CurrentTsCallsSymbolicEngine( return preparation } + internal fun prepareFunction(request: CallsFunctionPreflightRequest): CallsFunctionPreparation = + runCatching { prepareFunctionSupported(request) }.getOrElse { error -> + CallsFunctionPreparation.Rejected( + status = CallsSymbolicStatus.TOOL_ERROR, + reasonCode = CallsSymbolicPreflightReasonCode.FRONTEND_OR_PREFLIGHT_ERROR, + diagnostic = error.message ?: error::class.java.name, + ) + } + @Suppress("LongMethod") - private fun prepare(request: CallsSymbolicPreflightRequest): CallsTargetPreparation { + private fun prepareFunctionSupported(request: CallsFunctionPreflightRequest): CallsFunctionPreparation { callsSymbolicInputPreflight(request.function.inputs)?.let { diagnostic -> - return CallsTargetPreparation.Rejected( + return CallsFunctionPreparation.Rejected( status = CallsSymbolicStatus.UNSUPPORTED, reasonCode = CallsSymbolicPreflightReasonCode.INPUT_DOMAIN_UNSUPPORTED, diagnostic = diagnostic, @@ -282,18 +314,11 @@ internal class CurrentTsCallsSymbolicEngine( val source = request.sourceRoot.resolve(request.function.sourceFile).normalize() require(source.startsWith(request.sourceRoot)) { "Function source escapes its frozen source root" } - val sourceText = java.nio.file.Files.readString(source) - callsTargetCoordinateDiagnostic(source = sourceText, target = request.target)?.let { diagnostic -> - return CallsTargetPreparation.Rejected( - status = CallsSymbolicStatus.UNMAPPED, - reasonCode = CallsSymbolicPreflightReasonCode.TARGET_ORIGIN_UNMAPPED, - diagnostic = diagnostic, - ) - } - val loaded = when (val project = loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source)) { + val project = loadCallsSourceProject(sourceRoot = request.sourceRoot, source = source) + val loaded = when (project) { is CallsSourceProject.Loaded -> project - is CallsSourceProject.Unsupported -> return CallsTargetPreparation.Rejected( + is CallsSourceProject.Unsupported -> return CallsFunctionPreparation.Rejected( status = CallsSymbolicStatus.UNSUPPORTED, reasonCode = project.issue.reasonCode, diagnostic = project.issue.diagnostic, @@ -308,7 +333,7 @@ internal class CurrentTsCallsSymbolicEngine( ) val mapping = PropertyEtsMapper(scene = scene, sourceRoots = listOf(request.sourceRoot)).map(propertyManifest) if (mapping.predicate.status != EtsMappingStatus.EXACT) { - return CallsTargetPreparation.Rejected( + return CallsFunctionPreparation.Rejected( status = mapping.predicate.status.toSymbolicStatus(), reasonCode = mapping.predicate.status.toPreflightReasonCode(), diagnostic = mapping.predicate.diagnostics.joinToString { diagnostic -> diagnostic.message }, @@ -323,7 +348,7 @@ internal class CurrentTsCallsSymbolicEngine( method = method, ) if (inputTypeLimitation != null) { - return CallsTargetPreparation.Rejected( + return CallsFunctionPreparation.Rejected( status = CallsSymbolicStatus.UNSUPPORTED, reasonCode = CallsSymbolicPreflightReasonCode.INPUT_DOMAIN_UNSUPPORTED, diagnostic = inputTypeLimitation, @@ -332,7 +357,7 @@ internal class CurrentTsCallsSymbolicEngine( val lexicalEnvironment = mappedTarget.bindings.lexicalEnvironment if (lexicalEnvironment != null && lexicalEnvironment.parameter.type !is EtsLexicalEnvType) { - return CallsTargetPreparation.Rejected( + return CallsFunctionPreparation.Rejected( status = CallsSymbolicStatus.UNSUPPORTED, reasonCode = CallsSymbolicPreflightReasonCode.LEXICAL_CAPTURE_UNSUPPORTED, diagnostic = "Mapped lexical environment does not have a lexical-environment type", @@ -353,7 +378,7 @@ internal class CurrentTsCallsSymbolicEngine( callsCapturePreflight(scene, binding, sourceCaptures) } if (captureLimitation != null) { - return CallsTargetPreparation.Rejected( + return CallsFunctionPreparation.Rejected( status = CallsSymbolicStatus.UNSUPPORTED, reasonCode = CallsSymbolicPreflightReasonCode.LEXICAL_CAPTURE_UNSUPPORTED, diagnostic = captureLimitation, @@ -366,13 +391,46 @@ internal class CurrentTsCallsSymbolicEngine( sourceByFile = loaded.sourceByFile, ) if (readinessIssue != null) { - return CallsTargetPreparation.Rejected( + return CallsFunctionPreparation.Rejected( status = CallsSymbolicStatus.UNSUPPORTED, reasonCode = readinessIssue.reasonCode, diagnostic = readinessIssue.diagnostic, ) } + return CallsFunctionPreparation.Eligible( + scene = scene, + method = method, + inputBindings = mappedTarget.bindings.inputs, + lexicalEnvironment = lexicalEnvironment, + captures = sourceCaptures, + ) + } + + private fun prepare(request: CallsSymbolicPreflightRequest): CallsTargetPreparation { + val function = prepareFunction( + CallsFunctionPreflightRequest( + sourceRoot = request.sourceRoot, + project = request.project, + function = request.function, + expectedNativeFrontendRevision = request.expectedNativeFrontendRevision, + ), + ) + if (function is CallsFunctionPreparation.Rejected) { + return CallsTargetPreparation.Rejected(function.status, function.reasonCode, function.diagnostic) + } + function as CallsFunctionPreparation.Eligible + val method = function.method + val source = request.sourceRoot.resolve(request.function.sourceFile).normalize() + val sourceText = java.nio.file.Files.readString(source) + callsTargetCoordinateDiagnostic(source = sourceText, target = request.target)?.let { diagnostic -> + return CallsTargetPreparation.Rejected( + status = CallsSymbolicStatus.UNMAPPED, + reasonCode = CallsSymbolicPreflightReasonCode.TARGET_ORIGIN_UNMAPPED, + diagnostic = diagnostic, + ) + } + val exactTargetCandidates = exactTargetCandidates(method, request.target) val targetObservation = when (request.target.mode) { CallsSourceTargetMode.ENTRY -> { @@ -437,11 +495,11 @@ internal class CurrentTsCallsSymbolicEngine( } return CallsTargetPreparation.Eligible( - scene = scene, + scene = function.scene, method = method, - inputBindings = mappedTarget.bindings.inputs, - lexicalEnvironment = lexicalEnvironment, - captures = sourceCaptures, + inputBindings = function.inputBindings, + lexicalEnvironment = function.lexicalEnvironment, + captures = function.captures, targetObservation = targetObservation, exactTargetCandidateCount = exactTargetCandidates.size, ) diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt new file mode 100644 index 0000000000..1a545e3288 --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt @@ -0,0 +1,162 @@ +package org.usvm.ts.calls + +import org.junit.jupiter.api.io.TempDir +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.JsConcreteValue +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.StringDomain +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.seconds + +class CallsCoverageExperimentTest { + @TempDir + lateinit var directory: Path + + @Test + fun `two symbolic branches replay to distinct original source if arms`() { + val fixture = fixture( + source = """ + export function classify(flag: boolean): boolean { + if (flag) { + return true; + } else { + return false; + } + } + """.trimIndent(), + exportName = "classify", + inputs = listOf(PropertyInput(name = "flag", domain = BooleanDomain)), + ) + val emitted = mutableListOf() + + val search = CurrentTsCallsCoverageEngine(testSourceEngine()).search( + request = coverageRequest(fixture, onCandidate = emitted::add), + ) + + assertEquals(CallsCoverageSearchStatus.EXHAUSTED, search.status, search.toString()) + assertEquals(search.candidates, emitted) + assertEquals(2, search.candidates.map { it.inputs }.distinct().size, search.toString()) + assertTrue(search.candidates.all { candidate -> candidate.emittedAtMillis <= search.searchElapsedMillis }) + + val replayed = OriginalTypeScriptCoverageReplayer( + sourceRoot = fixture.sourceRoot, + function = fixture.function, + ).use { replayer -> + search.candidates.map { candidate -> + replayer.replay(inputs = candidate.inputs, timeoutMillis = 20_000L) + } + } + assertEquals(2, replayed.flatMap { it.coveredIfArmKeys }.toSet().size, replayed.toString()) + assertTrue(replayed.all { replay -> replay.coveredStatementKeys.isNotEmpty() }) + } + + @Test + fun `coverage replay preserves an isolated UTF-16 surrogate`() { + val fixture = fixture( + source = "export function lengthOf(value: string): number { return value.length; }", + exportName = "lengthOf", + inputs = listOf(PropertyInput(name = "value", domain = StringDomain(maxLength = 1))), + ) + + val replay = OriginalTypeScriptCoverageReplayer( + sourceRoot = fixture.sourceRoot, + function = fixture.function, + ).use { replayer -> + replayer.replay( + inputs = listOf(JsConcreteValue.String("\uD800")), + timeoutMillis = 20_000L, + ) + } + + assertEquals(CallsCoverageCompletion.RETURNED, replay.completion) + assertTrue(replay.coveredStatementKeys.isNotEmpty()) + } + + @Test + fun `coverage replay accepts an entry function with no inputs`() { + val fixture = fixture( + source = "export function constant(): number { return 42; }", + exportName = "constant", + inputs = emptyList(), + ) + + val replay = OriginalTypeScriptCoverageReplayer( + sourceRoot = fixture.sourceRoot, + function = fixture.function, + ).use { replayer -> + replayer.replay(inputs = emptyList(), timeoutMillis = 20_000L) + } + + assertEquals(CallsCoverageCompletion.RETURNED, replay.completion) + assertTrue(replay.coveredStatementKeys.isNotEmpty()) + } + + private fun coverageRequest( + fixture: CoverageFixture, + onCandidate: (CallsCoverageCandidate) -> Unit, + ) = CallsCoverageSearchRequest( + sourceRoot = fixture.sourceRoot, + project = fixture.project, + function = fixture.function, + profile = CallsExperimentProfile.EMPTY_STOP, + frozenModelIds = emptySet(), + expectedNativeFrontendRevision = "bundled:test", + seed = 17, + budget = 10.seconds, + solverQueryLimit = 2.seconds, + candidateCap = 8, + onCandidate = onCandidate, + ) + + private fun testSourceEngine() = CurrentTsCallsSymbolicEngine( + environment = emptyMap()::get, + bundledNativeFrontendRevision = "bundled:test", + ) + + private fun fixture(source: String, exportName: String, inputs: List): CoverageFixture { + val sourceRoot = Files.createDirectory(directory.resolve(exportName)) + Files.writeString(sourceRoot.resolve("Fixture.ts"), source) + runGit(sourceRoot, "init") + runGit(sourceRoot, "config", "user.name", "USVM Tests") + runGit(sourceRoot, "config", "user.email", "usvm@example.test") + runGit(sourceRoot, "add", ".") + runGit(sourceRoot, "commit", "-m", "fixture") + val revision = runGit(sourceRoot, "rev-parse", "HEAD").trim() + val function = CallsFunctionCase( + functionId = exportName, + sourceFile = "Fixture.ts", + entryPoint = TypeScriptEntryPoint(module = "Fixture.ts", exportName = exportName), + inputs = inputs, + targets = emptyList(), + ) + val project = CallsProjectCase( + projectId = exportName, + revision = revision, + sourceRoot = ".", + development = true, + functions = listOf(function), + ) + + return CoverageFixture(sourceRoot = sourceRoot, project = project, function = function) + } + + private fun runGit(directory: Path, vararg arguments: String): String { + val process = ProcessBuilder(listOf("git", "-C", directory.toString()) + arguments) + .redirectErrorStream(true) + .start() + val output = process.inputStream.bufferedReader().use { it.readText() } + check(process.waitFor() == 0) { "Git ${arguments.joinToString()} failed: $output" } + return output + } + + private data class CoverageFixture( + val sourceRoot: Path, + val project: CallsProjectCase, + val function: CallsFunctionCase, + ) +} From 19b118465c9cce85cab9db16f83b4ee286973c89 Mon Sep 17 00:00:00 2001 From: Aleksei Menshutin Date: Tue, 13 Oct 2026 01:10:50 +0300 Subject: [PATCH 2/2] Bind coverage replay to frozen source and pre-call inputs --- .../org/usvm/ts/calls/CallsCoverageCell.kt | 36 ++- .../org/usvm/ts/calls/CallsCoverageReplay.kt | 8 +- .../org/usvm/ts/calls/CallsCoverageSource.kt | 64 +++++ .../ts/calls/CurrentTsCallsSymbolicEngine.kt | 2 +- .../ts/calls/CallsCoverageExperimentTest.kt | 57 +++++ .../usvm/ts/calls/CallsCoverageSourceTest.kt | 230 ++++++++++++++++++ .../usvm/ts/pbt/fastcheck/FastCheckRuntime.kt | 13 +- 7 files changed, 392 insertions(+), 18 deletions(-) create mode 100644 usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSource.kt create mode 100644 usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageSourceTest.kt diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt index 2a04d11c7f..d91a348784 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt @@ -17,6 +17,7 @@ internal data class CallsCoverageCellIdentity( val toolRevision: String, val nativeFrontendRevision: String, val sourceManifest: String, + val sourceIdentity: CallsCoverageSourceIdentity, val projectIndex: Int, val functionIndex: Int, val profile: CallsExperimentProfile, @@ -51,29 +52,36 @@ internal data class CallsCoverageCellResult( val checkpoints: List, ) -internal fun runCoverageCell(args: List) { +internal fun runCoverageCell( + args: List, + runtimeToolRevision: String = CallsBuildIdentity.toolRevision, + runtimeNativeFrontendRevision: String = CallsBuildIdentity.nativeFrontendRevision, + searchFunction: (CallsCoverageSearchRequest) -> CallsCoverageSearchResult = CurrentTsCallsCoverageEngine()::search, +) { require(args.size == 9) { "coverage-cell requires 9 arguments" } - val manifestPath = Path.of(args[0]).toRealPath() - val manifest = CallsExperimentJson.decodeManifest(Files.readString(manifestPath)) val projectIndex = args[1].toInt() val functionIndex = args[2].toInt() + val source = CallsCoverageSource.load(Path.of(args[0]), projectIndex, functionIndex) + val manifestPath = source.manifestPath + val manifest = source.manifest val profile = CallsExperimentProfile.valueOf(args[3]) val seed = args[4].toLong() val output = Path.of(args[5]).toAbsolutePath().normalize() val budgetMillis = args[6].toLong() val solverLimitMillis = args[7].toLong() val candidateCap = args[8].toInt() - val project = manifest.projects[projectIndex] - val function = project.functions[functionIndex] + val project = source.project + val function = source.function require(seed in manifest.seeds) { "Seed is outside the frozen source manifest" } require(budgetMillis > 0 && solverLimitMillis in 1..budgetMillis && candidateCap > 0) require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" } val identity = CallsCoverageCellIdentity( - schemaVersion = 1, - toolRevision = CallsBuildIdentity.toolRevision, - nativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + schemaVersion = 2, + toolRevision = runtimeToolRevision, + nativeFrontendRevision = runtimeNativeFrontendRevision, sourceManifest = manifestPath.toString(), + sourceIdentity = source.identity, projectIndex = projectIndex, functionIndex = functionIndex, profile = profile, @@ -101,8 +109,9 @@ internal fun runCoverageCell(args: List) { CallsExperimentJson.json.decodeFromString(Files.readString(searchPath)) } else { require(!Files.exists(candidatePath)) { "Interrupted search requires investigation before restart" } + source.verifyUnchanged() val request = CallsCoverageSearchRequest( - sourceRoot = manifestPath.parent.resolve(project.sourceRoot).toRealPath(), + sourceRoot = source.sourceRoot, project = project, function = function, profile = profile, @@ -114,7 +123,8 @@ internal fun runCoverageCell(args: List) { candidateCap = candidateCap, onCandidate = { candidate -> appendDurably(candidatePath, encode(candidate)) }, ) - val result = CurrentTsCallsCoverageEngine().search(request) + val result = searchFunction(request) + source.verifyUnchanged() val persistedCandidates = readLines(candidatePath) require(result.candidates == persistedCandidates) { "Persisted candidate journal differs from search result" } atomicWrite(searchPath, encode(result)) @@ -126,16 +136,19 @@ internal fun runCoverageCell(args: List) { require(events.map(CallsCoverageReplayEvent::index) == events.indices.toList()) { "Replay journal has missing or duplicate indices" } + source.verifyUnchanged() if (search.candidates.isNotEmpty()) { OriginalTypeScriptCoverageReplayer( - sourceRoot = manifestPath.parent.resolve(project.sourceRoot), + sourceRoot = source.sourceRoot, function = function, ).use { replayer -> search.candidates.indices.drop(events.size).forEach { index -> val candidate = search.candidates[index] + source.verifyUnchanged() val replay = runCatching { replayer.replay(inputs = candidate.inputs, timeoutMillis = REPLAY_TIMEOUT_MILLIS) } + source.verifyUnchanged() val event = CallsCoverageReplayEvent( index = index, replay = replay.getOrNull(), @@ -147,6 +160,7 @@ internal fun runCoverageCell(args: List) { } } + source.verifyUnchanged() val checkpoints = listOf(30_000L, 60_000L, 120_000L).filter { it <= budgetMillis }.map { checkpoint -> val indices = search.candidates.indices.filter { index -> search.candidates[index].emittedAtMillis <= checkpoint diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt index 01d5b983bf..be888d1564 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt @@ -8,6 +8,7 @@ import org.usvm.ts.pbt.backend.PropertyRunConfiguration import org.usvm.ts.pbt.backend.PropertyRunStatus import org.usvm.ts.pbt.backend.SourceFileCoverage import org.usvm.ts.pbt.fastcheck.FastCheckBackend +import org.usvm.ts.pbt.fastcheck.FastCheckRuntime import org.usvm.ts.pbt.fastcheck.TypeScriptSourceInspector import org.usvm.ts.pbt.model.JsConcreteValue import org.usvm.ts.pbt.model.BooleanDomain @@ -198,6 +199,9 @@ private fun coverageWrapper( observationPath: Path, dropSentinel: Boolean, ): String { + val protocolModule = CallsExperimentJson.json.encodeToUtf8SafeString( + FastCheckRuntime.protocolJsonModule().toUri().toString(), + ) val moduleName = CallsExperimentJson.json.encodeToUtf8SafeString("./$sourceFile") val exportKey = CallsExperimentJson.json.encodeToUtf8SafeString(exportName) val outputPath = CallsExperimentJson.json.encodeToUtf8SafeString(observationPath.toString()) @@ -205,6 +209,7 @@ private fun coverageWrapper( return """ import { writeFileSync } from 'node:fs'; + import { stringifyProtocolJson } from $protocolModule; import * as targetModule from $moduleName; const callable = targetModule[$exportKey]; @@ -230,6 +235,7 @@ private fun coverageWrapper( export function coverageProbe(...args: unknown[]): boolean { const actualArgs = $sentinelExpression; + const encodedInputs = actualArgs.map(encodeValue); let completion: 'RETURNED' | 'THREW' = 'RETURNED'; let errorName: string | undefined; try { @@ -242,7 +248,7 @@ private fun coverageWrapper( completion = 'THREW'; errorName = error instanceof Error ? error.name : typeof error; } - writeFileSync($outputPath, JSON.stringify({ inputs: actualArgs.map(encodeValue), completion, errorName }), 'utf8'); + writeFileSync($outputPath, stringifyProtocolJson({ inputs: encodedInputs, completion, errorName }), 'utf8'); return true; } diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSource.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSource.kt new file mode 100644 index 0000000000..7f9ba56ecb --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSource.kt @@ -0,0 +1,64 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.Serializable +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** The same frozen source identity is used by search journals, replay and coverage universes. */ +@Serializable +internal data class CallsCoverageSourceIdentity( + val manifestSha256: String, + val sourceRoot: String, + val revision: String, +) + +internal class CallsCoverageSource private constructor( + val manifestPath: Path, + val manifest: CallsExperimentManifest, + val project: CallsProjectCase, + val function: CallsFunctionCase, + val sourceRoot: Path, + val identity: CallsCoverageSourceIdentity, +) { + /** Never use the symbolic engine's verify-once cache at a persisted experiment boundary. */ + fun verifyUnchanged() { + val manifestSha256 = sha256(Files.readAllBytes(manifestPath)) + require(manifestSha256 == identity.manifestSha256) { "Coverage source manifest changed" } + val currentRoot = manifestPath.parent.resolve(project.sourceRoot).toRealPath() + require(currentRoot == sourceRoot) { "Coverage source root changed" } + + verifyCallsGitCheckout(sourceRoot, identity.revision) + val checkoutStatus = runCallsGit(sourceRoot, "status", "--porcelain", "--untracked-files=all") + require(checkoutStatus.isBlank()) { "Coverage source checkout must be clean: $checkoutStatus" } + } + + companion object { + fun load(manifestPath: Path, projectIndex: Int, functionIndex: Int): CallsCoverageSource { + val realManifestPath = manifestPath.toRealPath() + val manifestBytes = Files.readAllBytes(realManifestPath) + val manifest = CallsExperimentJson.decodeManifest(manifestBytes.toString(Charsets.UTF_8)) + val project = manifest.projects[projectIndex] + val function = project.functions[functionIndex] + val sourceRoot = realManifestPath.parent.resolve(project.sourceRoot).toRealPath() + val source = CallsCoverageSource( + manifestPath = realManifestPath, + manifest = manifest, + project = project, + function = function, + sourceRoot = sourceRoot, + identity = CallsCoverageSourceIdentity( + manifestSha256 = sha256(manifestBytes), + sourceRoot = sourceRoot.toString(), + revision = project.revision, + ), + ) + source.verifyUnchanged() + + return source + } + + private fun sha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes) + .joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and 0xff) } + } +} diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt index f2d88b3e91..4d919a200b 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CurrentTsCallsSymbolicEngine.kt @@ -819,7 +819,7 @@ internal fun verifyCallsGitCheckout(checkout: Path, expectedRevision: String) { runCallsGit(checkout, "diff", "--quiet", "HEAD", "--") } -private fun runCallsGit(checkout: Path, vararg arguments: String): String { +internal fun runCallsGit(checkout: Path, vararg arguments: String): String { val process = ProcessBuilder(listOf("git", "-C", checkout.toString()) + arguments) .redirectErrorStream(true) .start() diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt index 1a545e3288..724e81eb38 100644 --- a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt @@ -2,6 +2,8 @@ package org.usvm.ts.calls import org.junit.jupiter.api.io.TempDir import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.ArrayDomain +import org.usvm.ts.pbt.model.NumberDomain import org.usvm.ts.pbt.model.JsConcreteValue import org.usvm.ts.pbt.model.PropertyInput import org.usvm.ts.pbt.model.StringDomain @@ -77,6 +79,61 @@ class CallsCoverageExperimentTest { assertTrue(replay.coveredStatementKeys.isNotEmpty()) } + @Test + fun `coverage replay records array input before mutation`() { + val fixture = fixture( + source = "export function replace(values: number[]): number { values[0] = 2; return values[0]; }", + exportName = "replace", + inputs = listOf(PropertyInput( + name = "values", + domain = ArrayDomain(element = NumberDomain(), minLength = 1, maxLength = 1), + )), + ) + val inputs = listOf(JsConcreteValue.Array(listOf(JsConcreteValue.number(1.0)))) + + val replay = OriginalTypeScriptCoverageReplayer( + sourceRoot = fixture.sourceRoot, + function = fixture.function, + ).use { replayer -> replayer.replay(inputs = inputs, timeoutMillis = 20_000L) } + + assertEquals(CallsCoverageCompletion.RETURNED, replay.completion) + assertTrue(replay.coveredStatementKeys.isNotEmpty()) + } + + @Test + fun `coverage observation does not invoke inherited protocol serializers`() { + val fixture = fixture( + source = """ + export function replace(values: number[]): number { + Object.defineProperty(Object.prototype, 'toJSON', { + configurable: true, + value() { throw new Error('Object serializer must not run'); }, + }); + Object.defineProperty(Array.prototype, 'toJSON', { + configurable: true, + value() { throw new Error('Array serializer must not run'); }, + }); + values[0] = 2; + return values[0]; + } + """.trimIndent(), + exportName = "replace", + inputs = listOf(PropertyInput( + name = "values", + domain = ArrayDomain(element = NumberDomain(), minLength = 1, maxLength = 1), + )), + ) + val inputs = listOf(JsConcreteValue.Array(listOf(JsConcreteValue.number(1.0)))) + + val replay = OriginalTypeScriptCoverageReplayer( + sourceRoot = fixture.sourceRoot, + function = fixture.function, + ).use { replayer -> replayer.replay(inputs = inputs, timeoutMillis = 20_000L) } + + assertEquals(CallsCoverageCompletion.RETURNED, replay.completion) + assertTrue(replay.coveredStatementKeys.isNotEmpty()) + } + @Test fun `coverage replay accepts an entry function with no inputs`() { val fixture = fixture( diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageSourceTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageSourceTest.kt new file mode 100644 index 0000000000..f04103e0af --- /dev/null +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageSourceTest.kt @@ -0,0 +1,230 @@ +package org.usvm.ts.calls + +import kotlinx.serialization.decodeFromString +import org.junit.jupiter.api.io.TempDir +import org.usvm.ts.pbt.model.BooleanDomain +import org.usvm.ts.pbt.model.NumberDomain +import org.usvm.ts.pbt.model.PropertyInput +import org.usvm.ts.pbt.model.TypeScriptEntryPoint +import org.usvm.ts.pbt.model.encodeToUtf8SafeString +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class CallsCoverageSourceTest { + @TempDir + lateinit var directory: Path + + @Test + fun `unchanged source resumes the exact persisted search without rerunning it`() { + val fixture = fixture() + fixture.persistSearch() + + fixture.runCell() + + val result = CallsExperimentJson.json.decodeFromString( + Files.readString(fixture.output.resolve("result.json")), + ) + assertEquals(emptySearch, result.search) + assertEquals(fixture.source.identity, result.identity.sourceIdentity) + } + + @Test + fun `resumed search rejects changed export domain and frozen models at the same manifest path`() { + val fixture = fixture() + fixture.persistSearch() + val original = fixture.source.manifest + val project = fixture.source.project + val function = fixture.source.function + val mutations = listOf( + original.copy(projects = listOf(project.copy(functions = listOf( + function.copy(entryPoint = function.entryPoint.copy(exportName = "second")), + )))), + original.copy(projects = listOf(project.copy(functions = listOf( + function.copy(inputs = listOf(PropertyInput(name = "flag", domain = NumberDomain()))), + )))), + original.copy(modelSet = original.modelSet.copy(ids = setOf("ts.array.includes"))), + ) + + for (changed in mutations) { + Files.writeString(fixture.manifestPath, CallsExperimentJson.encodeManifest(changed)) + + val failure = assertFailsWith { fixture.runCell() } + + assertEquals("Cell identity changed", failure.message) + fixture.assertNoReplayOrCompletion() + } + } + + @Test + fun `resumed search rejects a changed checkout revision even with an updated manifest`() { + val fixture = fixture() + fixture.persistSearch() + Files.writeString(fixture.sourceFile, "export function first(flag: boolean): boolean { return !flag; }") + runCallsGit(fixture.source.sourceRoot, "add", "Fixture.ts") + runCallsGit(fixture.source.sourceRoot, "commit", "-m", "Change source") + val newRevision = runCallsGit(fixture.source.sourceRoot, "rev-parse", "HEAD").trim() + val changed = fixture.source.manifest.copy(projects = listOf(fixture.source.project.copy(revision = newRevision))) + Files.writeString(fixture.manifestPath, CallsExperimentJson.encodeManifest(changed)) + + val failure = assertFailsWith { fixture.runCell() } + + assertEquals("Cell identity changed", failure.message) + fixture.assertNoReplayOrCompletion() + } + + @Test + fun `resumed search rejects tracked source edits before executing target code`() { + val fixture = fixture() + fixture.persistSearch() + Files.writeString(fixture.sourceFile, "export function first(): never { throw new Error('changed source'); }") + + assertFailsWith { fixture.runCell() } + + fixture.assertNoReplayOrCompletion() + } + + @Test + fun `untracked source cannot silently change a resumed program`() { + val fixture = fixture() + fixture.persistSearch() + Files.writeString(fixture.source.sourceRoot.resolve("Added.ts"), "export const added = true;") + + val failure = assertFailsWith { fixture.runCell() } + + assertTrue(failure.message.orEmpty().contains("must be clean")) + fixture.assertNoReplayOrCompletion() + } + + @Test + fun `source edit during search aborts before persisting search or replay`() { + val fixture = fixture() + + assertFailsWith { + fixture.runCell(searchFunction = { + Files.writeString(fixture.sourceFile, "export function first(flag: boolean): boolean { return !flag; }") + emptySearch + }) + } + + assertFalse(Files.exists(fixture.output.resolve("search.json"))) + fixture.assertNoReplayOrCompletion() + } + + @Test + fun `manifest edit between search and replay is detected by the frozen source guard`() { + val fixture = fixture() + fixture.persistSearch() + val changed = fixture.source.manifest.copy(modelSet = fixture.source.manifest.modelSet.copy(ids = setOf("other"))) + Files.writeString(fixture.manifestPath, CallsExperimentJson.encodeManifest(changed)) + + val failure = assertFailsWith { fixture.source.verifyUnchanged() } + + assertEquals("Coverage source manifest changed", failure.message) + fixture.assertNoReplayOrCompletion() + } + + private fun fixture(): Fixture { + val sourceRoot = Files.createDirectory(directory.resolve("source")) + val sourceFile = sourceRoot.resolve("Fixture.ts") + Files.writeString(sourceFile, "export function first(flag: boolean): boolean { return flag; }") + runCallsGit(sourceRoot, "init") + runCallsGit(sourceRoot, "config", "user.name", "USVM Tests") + runCallsGit(sourceRoot, "config", "user.email", "usvm@example.test") + runCallsGit(sourceRoot, "add", ".") + runCallsGit(sourceRoot, "commit", "-m", "Create fixture") + val revision = runCallsGit(sourceRoot, "rev-parse", "HEAD").trim() + val function = CallsFunctionCase( + functionId = "first", + sourceFile = "Fixture.ts", + entryPoint = TypeScriptEntryPoint(module = "Fixture.ts", exportName = "first"), + inputs = listOf(PropertyInput(name = "flag", domain = BooleanDomain)), + targets = emptyList(), + ) + val project = CallsProjectCase( + projectId = "fixture", + revision = revision, + sourceRoot = "source", + development = true, + functions = listOf(function), + ) + val manifest = CallsExperimentManifest( + schemaVersion = 1, + experimentId = "coverage-identity", + toolRevision = TOOL_REVISION, + nativeFrontendRevision = FRONTEND_REVISION, + solver = "Z3", + searchPolicy = CALLS_PATH_SELECTION_STRATEGY.name, + modelSet = CallsModelSetIdentity(ids = emptySet(), toolRevision = TOOL_REVISION), + seeds = listOf(17L), + perTargetBudgetMillis = 30_000L, + projects = listOf(project), + ) + val manifestPath = directory.resolve("manifest.json") + Files.writeString(manifestPath, CallsExperimentJson.encodeManifest(manifest)) + val source = CallsCoverageSource.load(manifestPath, projectIndex = 0, functionIndex = 0) + val output = Files.createDirectory(directory.resolve("output")) + + return Fixture(source, manifestPath, sourceFile, output) + } + + private data class Fixture( + val source: CallsCoverageSource, + val manifestPath: Path, + val sourceFile: Path, + val output: Path, + ) { + fun persistSearch() { + val identity = CallsCoverageCellIdentity( + schemaVersion = 2, + toolRevision = TOOL_REVISION, + nativeFrontendRevision = FRONTEND_REVISION, + sourceManifest = manifestPath.toRealPath().toString(), + sourceIdentity = source.identity, + projectIndex = 0, + functionIndex = 0, + profile = CallsExperimentProfile.EMPTY_STOP, + seed = 17L, + budgetMillis = 30_000L, + solverLimitMillis = 1_000L, + candidateCap = 8, + ) + Files.writeString(output.resolve("identity.json"), CallsExperimentJson.json.encodeToUtf8SafeString(identity)) + Files.writeString(output.resolve("search.json"), CallsExperimentJson.json.encodeToUtf8SafeString(emptySearch)) + } + + fun runCell( + searchFunction: (CallsCoverageSearchRequest) -> CallsCoverageSearchResult = { error("Must resume persisted search") }, + ) { + runCoverageCell( + args = listOf(manifestPath.toString(), "0", "0", "EMPTY_STOP", "17", output.toString(), "30000", "1000", "8"), + runtimeToolRevision = TOOL_REVISION, + runtimeNativeFrontendRevision = FRONTEND_REVISION, + searchFunction = searchFunction, + ) + } + + fun assertNoReplayOrCompletion() { + assertFalse(Files.exists(output.resolve("replays.jsonl"))) + assertFalse(Files.exists(output.resolve("result.json"))) + } + } + + companion object { + private const val TOOL_REVISION = "1111111111111111111111111111111111111111" + private const val FRONTEND_REVISION = "bundled:test" + private val emptySearch = CallsCoverageSearchResult( + status = CallsCoverageSearchStatus.EXHAUSTED, + candidates = emptyList(), + selectedStates = 0, + extractionFailures = emptyList(), + candidateCapReached = false, + preparationElapsedMillis = 0, + searchElapsedMillis = 1, + ) + } +} diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntime.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntime.kt index bd2fc4f135..51d4514aa0 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntime.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckRuntime.kt @@ -5,14 +5,17 @@ import java.nio.file.Files import java.nio.file.Path /** Locates the private fast-check runtime in development and installed distributions. */ -internal object FastCheckRuntime { - fun sourceInspectorEntryPoint(): Path = locateEntryPoint(SOURCE_INSPECTOR_CLI) +object FastCheckRuntime { + /** Passive JSON serializer shared by generated replay wrappers and the adapter protocol. */ + fun protocolJsonModule(): Path = locateEntryPoint("protocol-json.js") - fun executionEntryPoint(): Path = locateEntryPoint(EXECUTION_CLI) + internal fun executionEntryPoint(): Path = locateEntryPoint(EXECUTION_CLI) - fun projectionEntryPoint(): Path = locateEntryPoint(PROJECTION_CLI) + internal fun projectionEntryPoint(): Path = locateEntryPoint(PROJECTION_CLI) - fun processSupervisorEntryPoint(): Path = locateEntryPoint(PROCESS_SUPERVISOR) + internal fun processSupervisorEntryPoint(): Path = locateEntryPoint(PROCESS_SUPERVISOR) + + internal fun sourceInspectorEntryPoint(): Path = locateEntryPoint(SOURCE_INSPECTOR_CLI) private fun locateEntryPoint(fileName: String): Path { val candidates = runtimeDirectories().map { runtimeDirectory ->