Preserve Calls source semantics and bounded native iterators

buildSrc/src/main/kotlin/Dependencies.kt

@@ -6,7 +6,7 @@ object Versions {
     const val clikt = "5.0.0"
     const val detekt = "1.23.7"
     const val ini4j = "0.5.4"
-    const val jacodb = "7596ae5834"
+    const val jacodb = "55a0990fa1"
     const val juliet = "1.3.2"
     const val junit = "5.9.3"
     const val kotlin = "2.1.0"

usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperiment.kt

@@ -6,6 +6,7 @@ import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import org.usvm.PathSelectionStrategy
+import org.usvm.machine.TsRuntimeFeatureLimitationEvent
 import org.usvm.machine.call.TsResidualCallPolicy
 import org.usvm.machine.call.TsUnknownCallEvent
 import org.usvm.ts.pbt.model.JsConcreteValue
@@ -116,6 +117,7 @@ internal enum class CallsSymbolicStatus {
     UNREACHED,
     UNREPRESENTABLE,
     UNSUPPORTED,
+    RUNTIME_LIMITATION,
     TIMEOUT,
     UNKNOWN,
     TOOL_ERROR,
@@ -134,6 +136,7 @@ internal data class CallsSymbolicSearchRequest(
     val seed: Long,
     val budget: Duration,
     val unknownCallEventSink: ((TsUnknownCallEvent) -> Unit)? = null,
+    val runtimeLimitationEventSink: ((TsRuntimeFeatureLimitationEvent) -> Unit)? = null,
 )
 
 internal data class CallsSymbolicSearchResult(
@@ -406,7 +409,7 @@ internal class CallsExperimentRunner(
         target: CallsSourceTarget,
         seed: Long,
         profile: CallsExperimentProfile,
-        appendUnknownCall: (CallsUnknownCallRecord) -> Unit,
+        appendUnknownCall: (CallsRawRecord) -> Unit,
     ): CallsTargetResult {
         val request = CallsSymbolicSearchRequest(
             sourceRoot = sourceRoot,
@@ -424,9 +427,14 @@ internal class CallsExperimentRunner(
             cell = request.cellIdentity(experimentId = manifest.experimentId),
             appendAndFlush = appendUnknownCall,
         )
+        val runtimeLimitationSink = callsRuntimeLimitationEventSink(
+            cell = request.cellIdentity(experimentId = manifest.experimentId),
+            appendAndFlush = appendUnknownCall,
+        )
         val symbolic = symbolicEngine.search(
             request.copy(
                 unknownCallEventSink = eventWrites.track(unknownCallSink),
+                runtimeLimitationEventSink = eventWrites.track(runtimeLimitationSink),
             ),
         )
         eventWrites.requireComplete()
@@ -600,7 +608,8 @@ internal object CallsExperimentAggregator {
                 replayConfirmed = rows.count { result -> result.replayStatus == CallsReplayStatus.CONFIRMED },
                 replayRejected = rows.count { result -> result.replayStatus == CallsReplayStatus.REJECTED },
                 unsupported = rows.count { result ->
-                    result.symbolicStatus == CallsSymbolicStatus.UNSUPPORTED
+                    result.symbolicStatus == CallsSymbolicStatus.UNSUPPORTED ||
+                        result.symbolicStatus == CallsSymbolicStatus.RUNTIME_LIMITATION
                 },
                 timeouts = rows.count { result ->
                     result.symbolicStatus == CallsSymbolicStatus.TIMEOUT ||

usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsInputSnapshot.kt

@@ -17,6 +17,7 @@ import org.usvm.machine.state.TsState
 import org.usvm.model.UModelBase
 import org.usvm.ts.pbt.model.JsConcreteValue
 import org.usvm.util.componentType
+import org.usvm.util.markDenseArrayOwnSlots
 import org.usvm.util.markDenseInputArray
 
 internal sealed interface CallsInputSnapshot {
@@ -65,6 +66,8 @@ internal data class ArrayInputSnapshot(
 ) : CallsInputSnapshot {
     override fun markDenseInputs(state: TsState) {
         elements.forEach { element -> element.markDenseInputs(state) }
+        state.markDenseArrayOwnSlots(array = reference, length = length, capacity = elements.size)
+
         val elementSort = state.ctx.typeToSort(runtimeType.componentType)
         if (elementSort !is TsUnresolvedSort) {
             state.markDenseInputArray(array = reference, type = runtimeType)

usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsIrReadinessIssue.kt

@@ -0,0 +1,200 @@
+package org.usvm.ts.calls
+
+import org.jacodb.ets.model.EtsClassType
+import org.jacodb.ets.model.EtsEntity
+import org.jacodb.ets.model.EtsFileSignature
+import org.jacodb.ets.model.EtsFunctionType
+import org.jacodb.ets.model.EtsMethod
+import org.jacodb.ets.model.EtsNewExpr
+import org.jacodb.ets.model.EtsPtrCallExpr
+import org.jacodb.ets.model.EtsRawEntity
+import org.jacodb.ets.model.EtsRawStmt
+import org.jacodb.ets.model.EtsStaticFieldRef
+import org.jacodb.ets.model.EtsStmt
+import org.jacodb.ets.utils.DEFAULT_ARK_CLASS_NAME
+import org.jacodb.ets.utils.DEFAULT_ARK_METHOD_NAME
+import org.jacodb.ets.utils.STATIC_INIT_METHOD_NAME
+import org.jacodb.ets.utils.getOperands
+import org.usvm.machine.TsGraph
+
+internal data class CallsIrReadinessIssue(
+    val reasonCode: CallsSymbolicPreflightReasonCode,
+    val diagnostic: String,
+)
+
+internal fun callsIrReadinessIssue(
+    method: EtsMethod,
+    graph: TsGraph,
+    sourceByFile: Map<EtsFileSignature, String>,
+): CallsIrReadinessIssue? {
+    val pending = ArrayDeque<EtsMethod>().apply { addLast(method) }
+    val visited = hashSetOf<EtsMethod>()
+
+    while (pending.isNotEmpty()) {
+        val currentMethod = pending.removeFirst()
+        if (!visited.add(currentMethod)) continue
+
+        currentMethod.irReadinessIssue(
+            source = sourceByFile.getValue(currentMethod.signature.enclosingClass.file),
+            graph = graph,
+        )?.let { issue -> return issue }
+
+        currentMethod.projectCallees(graph = graph, sourceFiles = sourceByFile.keys)
+            .forEach(pending::addLast)
+        currentMethod.projectStaticInitializers(graph = graph, sourceFiles = sourceByFile.keys)
+            .forEach(pending::addLast)
+    }
+
+    return null
+}
+
+private fun EtsMethod.projectCallees(
+    graph: TsGraph,
+    sourceFiles: Set<EtsFileSignature>,
+): Sequence<EtsMethod> = cfg.stmts.asSequence()
+    .flatMap { statement ->
+        val directCallees = runCatching { graph.callees(statement).toList() }.getOrDefault(emptyList())
+        val pointerCallees = statement.walkEntities().filterIsInstance<EtsPtrCallExpr>().flatMap { call ->
+            val signature = (call.ptr.type as? EtsFunctionType)?.signature ?: return@flatMap emptyList()
+            graph.cp.projectClasses.filter { owner -> owner.signature == signature.enclosingClass }
+                .flatMap { owner -> owner.methods }
+                .filter { callee -> callee.signature == signature }
+        }
+        (directCallees + pointerCallees).asSequence()
+    }
+    .filter { callee -> callee.signature.enclosingClass.file in sourceFiles }
+
+private fun EtsMethod.projectStaticInitializers(
+    graph: TsGraph,
+    sourceFiles: Set<EtsFileSignature>,
+): Sequence<EtsMethod> = cfg.stmts.asSequence()
+    .flatMap { it.walkEntities().asSequence() }
+    .filterIsInstance<EtsStaticFieldRef>()
+    .map { it.field.enclosingClass }
+    .filter { it.file in sourceFiles }
+    .distinct()
+    .mapNotNull { signature ->
+        val owner = graph.cp.projectClasses.singleOrNull { it.signature == signature } ?: return@mapNotNull null
+        owner.methods.singleOrNull { it.name == STATIC_INIT_METHOD_NAME }
+            ?: owner.takeIf { it.name == DEFAULT_ARK_CLASS_NAME }
+                ?.methods
+                ?.singleOrNull { it.name == DEFAULT_ARK_METHOD_NAME }
+    }
+
+internal fun callsIrReadinessIssue(
+    method: EtsMethod,
+    graph: TsGraph,
+    source: String,
+): CallsIrReadinessIssue? = callsIrReadinessIssue(
+    method = method,
+    graph = graph,
+    sourceByFile = mapOf(method.signature.enclosingClass.file to source),
+)
+
+private fun EtsMethod.irReadinessIssue(
+    source: String,
+    graph: TsGraph,
+): CallsIrReadinessIssue? {
+    val statements = cfg.stmts
+    val entitiesByStatement = statements.associateWith { statement -> statement.walkEntities() }
+    val intrinsicRegexAllocation = entitiesByStatement.values.asSequence().flatten()
+        .filterIsInstance<EtsNewExpr>()
+        .any { allocation ->
+            val type = allocation.type as? EtsClassType ?: return@any false
+            val intrinsicOwner = type.signature.file == EtsFileSignature.UNKNOWN ||
+                graph.cp.sdkClasses.any { owner -> owner.signature == type.signature }
+            type.signature.name == "RegExp" && intrinsicOwner
+        }
+    if (intrinsicRegexAllocation) {
+        return issue(
+            reasonCode = CallsSymbolicPreflightReasonCode.REGEX_LITERAL_UNSUPPORTED,
+            diagnostic = "Reachable EtsIR allocates an intrinsic RegExp without a runtime model in $name",
+        )
+    }
+
+    val rawNodes = buildList {
+        entitiesByStatement.forEach { (statement, statementEntities) ->
+            val sourceSnippet = statement.sourceSnippet(source)
+            if (statement is EtsRawStmt) {
+                add(RawNode(kind = statement.kind, extra = statement.extra, sourceSnippet = sourceSnippet))
+            }
+            statementEntities.filterIsInstance<EtsRawEntity>().forEach { entity ->
+                add(RawNode(kind = entity.kind, extra = entity.extra, sourceSnippet = sourceSnippet))
+            }
+        }
+    }
+    rawNodes.firstOrNull(RawNode::isDestructuring)?.let { raw ->
+        return raw.issue(CallsSymbolicPreflightReasonCode.DESTRUCTURING_UNSUPPORTED, name)
+    }
+    rawNodes.firstOrNull(RawNode::isSpread)?.let { raw ->
+        return raw.issue(CallsSymbolicPreflightReasonCode.SPREAD_UNSUPPORTED, name)
+    }
+    rawNodes.firstOrNull(RawNode::isRegexLiteral)?.let { raw ->
+        return raw.issue(CallsSymbolicPreflightReasonCode.REGEX_LITERAL_UNSUPPORTED, name)
+    }
+    rawNodes.firstOrNull()?.let { raw ->
+        return raw.issue(CallsSymbolicPreflightReasonCode.RAW_ENTITY_UNSUPPORTED, name)
+    }
+
+    return null
+}
+
+private fun EtsStmt.sourceSnippet(source: String): String {
+    val origin = location.origin ?: return ""
+    if (origin.startOffset !in 0..source.length || origin.endOffset !in origin.startOffset..source.length) return ""
+
+    return source.substring(origin.startOffset, origin.endOffset)
+}
+
+private fun EtsStmt.walkEntities(): List<EtsEntity> {
+    val result = mutableListOf<EtsEntity>()
+    val pending = ArrayDeque<EtsEntity>()
+    getOperands().forEach(pending::addLast)
+    while (pending.isNotEmpty()) {
+        val entity = pending.removeFirst()
+        result += entity
+        entity.getOperands().forEach(pending::addLast)
+    }
+
+    return result
+}
+
+private data class RawNode(
+    val kind: String,
+    val extra: Map<String, Any>,
+    val sourceSnippet: String,
+) {
+    private val normalizedDescription: String = sequenceOf(kind)
+        .plus(extra.entries.sortedBy { (key, _) -> key }.flatMap { (key, value) -> sequenceOf(key, value.toString()) })
+        .plus(sourceSnippet)
+        .joinToString(separator = " ")
+        .lowercase()
+
+    fun matches(vararg markers: String): Boolean = markers.any(normalizedDescription::contains)
+
+    fun isDestructuring(): Boolean = matches("destruct", "bindingpattern", "binding pattern") ||
+        DESTRUCTURING_DECLARATION.containsMatchIn(sourceSnippet) ||
+        DESTRUCTURING_ASSIGNMENT.containsMatchIn(sourceSnippet)
+
+    fun isSpread(): Boolean = matches("spread") || sourceSnippet.contains("...")
+
+    fun isRegexLiteral(): Boolean = matches("regex", "regexp", "regularexpression", "regular expression") ||
+        REGEX_LITERAL.containsMatchIn(sourceSnippet)
+
+    fun issue(reasonCode: CallsSymbolicPreflightReasonCode, methodName: String): CallsIrReadinessIssue = issue(
+        reasonCode = reasonCode,
+        diagnostic = "Reachable EtsIR contains unsupported raw node kind $kind in $methodName",
+    )
+}
+
+private val DESTRUCTURING_DECLARATION = Regex("""\b(?:const|let|var)\s*[\[{]""")
+private val DESTRUCTURING_ASSIGNMENT = Regex("""(?:^|[;{}])\s*[\[{][^=]*[\]}]\s*=""")
+private val REGEX_LITERAL = Regex("""/(?:\\.|[^/\r\n])+/[a-z]*""")
+
+private fun issue(
+    reasonCode: CallsSymbolicPreflightReasonCode,
+    diagnostic: String,
+): CallsIrReadinessIssue = CallsIrReadinessIssue(
+    reasonCode = reasonCode,
+    diagnostic = diagnostic,
+)

usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsSourceProject.kt

@@ -0,0 +1,107 @@
+package org.usvm.ts.calls
+
+import org.jacodb.ets.dto.EtsFileDto
+import org.jacodb.ets.dto.toEtsFile
+import org.jacodb.ets.model.EtsFile
+import org.jacodb.ets.model.EtsFileSignature
+import org.jacodb.ets.model.EtsScene
+import org.jacodb.ets.utils.EtsIrProvider
+import org.jacodb.ets.utils.generateEtsIR
+import org.jacodb.ets.utils.loadEtsProjectFromIR
+import org.usvm.ts.pbt.fastcheck.TypeScriptSourceInspector
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.time.Duration.Companion.seconds
+
+internal sealed interface CallsSourceProject {
+    data class Loaded(
+        val scene: EtsScene,
+        val entryModule: String,
+        val sourceByFile: Map<EtsFileSignature, String>,
+    ) : CallsSourceProject
+
+    data class Unsupported(val issue: CallsIrReadinessIssue) : CallsSourceProject
+}
+
+internal fun loadCallsSourceProject(sourceRoot: Path, source: Path): CallsSourceProject {
+    val closure = TypeScriptSourceInspector.localSourceClosure(sourceRoot = sourceRoot, source = source)
+    if (closure.reasonCode != null) {
+        return CallsSourceProject.Unsupported(
+            issue = CallsIrReadinessIssue(
+                reasonCode = CallsSymbolicPreflightReasonCode.valueOf(requireNotNull(closure.reasonCode)),
+                diagnostic = requireNotNull(closure.diagnostic),
+            ),
+        )
+    }
+
+    val entryModule = sourceRoot.relativize(source).toString()
+    require(entryModule in closure.files) { "Dependency closure does not contain its entry source" }
+    if (closure.files.size == 1) {
+        val entry = loadSingleCallsSource(source)
+        return CallsSourceProject.Loaded(
+            scene = EtsScene(projectFiles = listOf(entry)),
+            entryModule = source.fileName.toString(),
+            sourceByFile = mapOf(entry.signature to Files.readString(source)),
+        )
+    }
+
+    val sources = closure.files.associateWith { relative ->
+        val path = sourceRoot.resolve(relative).normalize()
+        require(path.startsWith(sourceRoot) && path.toRealPath().startsWith(sourceRoot.toRealPath())) {
+            "Dependency source escapes its frozen root: $relative"
+        }
+        Files.readString(path)
+    }
+    val temporaryProject = Files.createTempDirectory("usvm-calls-source-closure-").toRealPath()
+    try {
+        for (relative in closure.files) {
+            val copy = temporaryProject.resolve(relative)
+            Files.createDirectories(copy.parent)
+            Files.copy(sourceRoot.resolve(relative), copy)
+        }
+        val generatedIr = generateEtsIR(
+            keepPartialOutputOnFailure = false,
+            projectPath = temporaryProject,
+            isProject = true,
+            loadEntrypoints = false,
+            timeout = 30.seconds,
+            provider = EtsIrProvider.TS_FRONTEND,
+        )
+        val scene = try {
+            loadEtsProjectFromIR(projectFilesPath = generatedIr, sdkFilesPath = null)
+        } finally {
+            generatedIr.toFile().deleteRecursively()
+        }
+        val sourceByFile = scene.projectFiles.associate { file ->
+            val filePath = Path.of(file.signature.fileName)
+            val relative = if (filePath.isAbsolute) {
+                temporaryProject.relativize(filePath).toString()
+            } else {
+                filePath.toString()
+            }
+            file.signature to requireNotNull(sources[relative]) {
+                "Frontend source identity is outside the inspected closure: ${file.signature}"
+            }
+        }
+        require(sourceByFile.size == closure.files.size) { "Frontend omitted or duplicated a dependency source" }
+
+        return CallsSourceProject.Loaded(scene = scene, entryModule = entryModule, sourceByFile = sourceByFile)
+    } finally {
+        temporaryProject.toFile().deleteRecursively()
+    }
+}
+
+private fun loadSingleCallsSource(source: Path): EtsFile {
+    val generatedIr = generateEtsIR(
+        keepPartialOutputOnFailure = false,
+        projectPath = source,
+        useArkAnalyzerTypeInference = 1,
+        provider = EtsIrProvider.TS_FRONTEND,
+    )
+
+    return try {
+        Files.newInputStream(generatedIr).use { stream -> EtsFileDto.loadFromJson(stream).toEtsFile() }
+    } finally {
+        Files.deleteIfExists(generatedIr)
+    }
+}