From 02cb363d396ca8799b34ad4a0c25d5619890049c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:21 +0000 Subject: [PATCH 01/10] fix(backend): update cached relay locations after the families are computed Families are computed asynchronously, so the cache was refreshed or evicted before the new family IDs were stored and kept the previous families. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- .../service/DescriptorCoordinationService.kt | 8 +- .../service/RelayDetailsUpdateService.kt | 17 +- .../DescriptorCoordinationServiceTest.kt | 152 ++++++++++++++++-- ...elayDetailsUpdateServiceIntegrationTest.kt | 120 ++++++++++++++ .../service/RelayDetailsUpdateServiceTest.kt | 69 ++++++++ 5 files changed, 346 insertions(+), 20 deletions(-) create mode 100644 backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceIntegrationTest.kt create mode 100644 backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceTest.kt diff --git a/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt b/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt index 8b348726..74daca78 100644 --- a/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt +++ b/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt @@ -124,10 +124,14 @@ class DescriptorCoordinationService( private fun computeRelayDetailsAndCaches(descriptorType: DescriptorType, processedMonths: Set) { if (descriptorType.isRelayServerType()) { - relayDetailsUpdateService.computeFamilies(processedMonths) relayDetailsUpdateService.lookupMissingAutonomousSystems(processedMonths) + // The cached relay locations contain the family IDs, so they are only updated once the families are computed + relayDetailsUpdateService.computeFamilies(processedMonths).whenComplete { _, _ -> + updateCaches(descriptorType, processedMonths) + } + } else { + updateCaches(descriptorType, processedMonths) } - updateCaches(descriptorType, processedMonths) } private fun updateCaches(descriptorType: DescriptorType, processedMonths: Set) { diff --git a/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt b/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt index 1963fd41..f7fd920f 100644 --- a/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt +++ b/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt @@ -10,6 +10,7 @@ import org.tormap.util.addFamilyMember import org.tormap.util.commaSeparatedToList import org.tormap.util.getFamilyMember import org.tormap.util.logger +import java.util.concurrent.CompletableFuture import javax.sql.DataSource /** @@ -76,23 +77,29 @@ class RelayDetailsUpdateService( val monthFamilyMemberCount = relayDetailsRepositoryImpl.findDistinctMonthFamilyMemberCount().filter { it.count == 0L } val monthsToProcess = monthFamilyMemberCount.map { it.month }.toSet() - computeFamilies(monthsToProcess) - cacheService.evictRelayLocationsPerDay(monthsToProcess) + // The families are computed asynchronously, evicting earlier would let requests cache the old families again + computeFamilies(monthsToProcess).whenComplete { _, _ -> + cacheService.evictRelayLocationsPerDay(monthsToProcess) + } } /** - * Updates [RelayDetails.familyId] for all entities of the requested [months]. + * Updates [RelayDetails.familyId] for all entities of the requested [months] asynchronously. + * @return a future that completes once the families of all [months] are computed (also if some failed) */ - fun computeFamilies(months: Set) { + fun computeFamilies(months: Set): CompletableFuture { logger.info("... Updating relay families for months: {}", months.joinToString(", ")) - months.forEach { month -> + val futures = months.map { month -> coalesceService.submitAsync("computeFamilies-$month") { transactionTemplate.executeWithoutResult { computeFamiliesForMonth(month) } } + // Failures are logged by the CoalesceService, a failed month must not hide the others + .exceptionally { null } } + return CompletableFuture.allOf(*futures.toTypedArray()) } /** diff --git a/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt index 4a9020ee..f531199b 100644 --- a/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt @@ -10,25 +10,48 @@ import org.tormap.database.repository.RelayDetailsRepository import org.tormap.database.repository.RelayLocationRepository import org.torproject.descriptor.Descriptor import java.io.File +import java.nio.file.Files +import java.util.concurrent.CompletableFuture class DescriptorCoordinationServiceTest : StringSpec({ - "processLocalDescriptorFiles saves the final processed descriptor file" { + lateinit var relayDetailsUpdateService: RelayDetailsUpdateService + lateinit var descriptorFileService: DescriptorFileService + lateinit var descriptorProcessingService: DescriptorProcessingService + lateinit var cacheService: CacheService + lateinit var service: DescriptorCoordinationService + lateinit var relayLocationRepository: RelayLocationRepository + val downloadDirectory = Files.createTempDirectory("tormap-collect").toFile() + afterSpec { downloadDirectory.deleteRecursively() } + + beforeEach { + relayDetailsUpdateService = mockk(relaxed = true) + descriptorFileService = mockk(relaxed = true) + descriptorProcessingService = mockk() + cacheService = mockk(relaxed = true) + relayLocationRepository = mockk(relaxed = true) val descriptorConfig = mockk(relaxed = true) - val relayDetailsUpdateService = mockk(relaxed = true) - val descriptorFileService = mockk(relaxed = true) - val descriptorProcessingService = mockk() - val relayDetailsRepository = mockk(relaxed = true) - val relayLocationRepository = mockk(relaxed = true) - val cacheService = mockk(relaxed = true) - val service = DescriptorCoordinationService( + // Nothing listens on port 1, so collecting fails right away without network access + every { descriptorConfig.apiBaseURL } returns "http://127.0.0.1:1" + every { descriptorConfig.localDownloadDirectory } returns downloadDirectory.absolutePath + service = DescriptorCoordinationService( descriptorConfig, relayDetailsUpdateService, descriptorFileService, descriptorProcessingService, - relayDetailsRepository, + mockk(relaxed = true), relayLocationRepository, cacheService, ) + } + + fun processLocalDescriptorFiles(apiPath: String, descriptorType: DescriptorType) { + DescriptorCoordinationService::class.java + .getDeclaredMethod("processLocalDescriptorFiles", String::class.java, DescriptorType::class.java) + .apply { isAccessible = true } + .invoke(service, apiPath, descriptorType) + } + + "processLocalDescriptorFiles saves the final processed descriptor file" { val firstFile = File("first-descriptors.tar.xz") val finalFile = File("final-descriptors.tar.xz") val descriptors = mutableListOf( @@ -42,10 +65,7 @@ class DescriptorCoordinationServiceTest : StringSpec({ } returns descriptors every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-01") - DescriptorCoordinationService::class.java - .getDeclaredMethod("processLocalDescriptorFiles", String::class.java, DescriptorType::class.java) - .apply { isAccessible = true } - .invoke(service, "/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) verify(exactly = 1) { descriptorFileService.saveProcessedFileReference(firstFile, DescriptorType.ARCHIVE_RELAY_CONSENSUS) @@ -53,6 +73,112 @@ class DescriptorCoordinationServiceTest : StringSpec({ verify(exactly = 1) { descriptorFileService.saveProcessedFileReference(finalFile, DescriptorType.ARCHIVE_RELAY_CONSENSUS) } + verify(exactly = 2) { cacheService.cacheRelayLocationDistinctDays() } + verify(exactly = 2) { cacheService.cacheRelayDailyCounts() } + verify(exactly = 2) { cacheService.evictRelayLocationsPerDay(setOf("2024-01")) } + verify(exactly = 0) { relayDetailsUpdateService.computeFamilies(any()) } + } + + "files with errors are not saved as processed" { + val file = File("broken-descriptors.tar.xz") + every { + descriptorFileService.getDescriptorDiskReader("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file), descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returnsMany listOf( + ProcessedDescriptorInfo("2024-01"), + ProcessedDescriptorInfo(error = "unparsable"), + ) + + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + + verify(exactly = 0) { descriptorFileService.saveProcessedFileReference(any(), any()) } + } + + "server descriptors only update the cached relay locations once the families are computed" { + val file = File("server-descriptors.tar.xz") + val families = CompletableFuture() + every { + descriptorFileService.getDescriptorDiskReader("/archive", DescriptorType.ARCHIVE_RELAY_SERVER) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-02") + every { relayDetailsUpdateService.computeFamilies(setOf("2024-02")) } returns families + + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_SERVER) + + verify { relayDetailsUpdateService.lookupMissingAutonomousSystems(setOf("2024-02")) } + verify(exactly = 0) { cacheService.evictRelayLocationsPerDay(any()) } + + families.complete(null) + + verify(exactly = 1) { cacheService.evictRelayLocationsPerDay(setOf("2024-02")) } + // Server descriptors do not change the days or daily counts + verify(exactly = 0) { cacheService.cacheRelayLocationDistinctDays() } + verify(exactly = 0) { cacheService.cacheRelayDailyCounts() } + } + + "recent server descriptors refresh the cached relay locations after the families, also if they failed" { + val file = File("recent-server-descriptors") + val families = CompletableFuture() + every { + descriptorFileService.getDescriptorDiskReader("/recent", DescriptorType.RECENT_RELAY_SERVER) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-03") + every { relayDetailsUpdateService.computeFamilies(setOf("2024-03")) } returns families + + processLocalDescriptorFiles("/recent", DescriptorType.RECENT_RELAY_SERVER) + + // Once for the finished file + verify(exactly = 1) { cacheService.cacheRelayLocationsPerDay(setOf("2024-03")) } + + families.completeExceptionally(IllegalStateException("failed")) + + verify(exactly = 2) { cacheService.cacheRelayLocationsPerDay(setOf("2024-03")) } + } + + "recent consensuses cache the relay locations without computing families" { + val file = File("recent-consensuses") + every { + descriptorFileService.getDescriptorDiskReader("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-04") + + processLocalDescriptorFiles("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + + verify(exactly = 2) { cacheService.cacheRelayLocationsPerDay(setOf("2024-04")) } + verify(exactly = 2) { cacheService.cacheRelayDailyCounts() } + verify(exactly = 0) { relayDetailsUpdateService.computeFamilies(any()) } + } + + "collectAndProcessDescriptors processes local files also if the remote server is not reachable" { + val file = File("recent-consensuses") + every { + descriptorFileService.getDescriptorDiskReader("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-04") + + service.collectAndProcessDescriptors("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + + verify { relayLocationRepository.flush() } + verify { descriptorFileService.saveProcessedFileReference(file, DescriptorType.RECENT_RELAY_CONSENSUS) } + } + + "collectAndProcessDescriptors does not throw if processing fails" { + every { descriptorFileService.getDescriptorDiskReader(any(), any()) } throws IllegalStateException("disk") + service.collectAndProcessDescriptors("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + verify(exactly = 0) { cacheService.cacheRelayDailyCounts() } + } + + "a failing flush does not stop the processing" { + val file = File("archive-consensuses") + every { + descriptorFileService.getDescriptorDiskReader("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-05") + every { relayLocationRepository.flush() } throws IllegalStateException("database gone") + + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + + verify { descriptorFileService.saveProcessedFileReference(file, DescriptorType.ARCHIVE_RELAY_CONSENSUS) } } }) diff --git a/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceIntegrationTest.kt b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceIntegrationTest.kt new file mode 100644 index 00000000..8af07b7e --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceIntegrationTest.kt @@ -0,0 +1,120 @@ +package org.tormap.service + +import io.kotest.assertions.nondeterministic.eventually +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.nulls.shouldBeNull +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.tormap.database.repository.RelayDetailsRepositoryImpl +import org.tormap.mockRelayDetails +import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.seconds + +@SpringBootTest +@ActiveProfiles("test") +class RelayDetailsUpdateServiceIntegrationTest( + private val relayDetailsUpdateService: RelayDetailsUpdateService, + private val relayDetailsRepository: RelayDetailsRepositoryImpl, +) : StringSpec({ + val month = "2023-05" + val fingerprint = { letter: Char -> letter.toString().repeat(40) } + + beforeEach { + relayDetailsRepository.deleteAll() + } + + fun relay(letter: Char, familyEntries: String?, address: String = "172.104.114.113") = + mockRelayDetails(letter).apply { + this.month = month + this.familyEntries = familyEntries + this.address = address + } + + fun byNickname() = relayDetailsRepository.findAll().associateBy { it.nickname } + + "relays that list each other form a family" { + relayDetailsRepository.saveAll( + listOf( + relay('A', "\$${fingerprint('B')}, \$${fingerprint('C')}"), + relay('B', "\$${fingerprint('A')}"), + // C does not confirm A + relay('C', "\$${fingerprint('D')}"), + // D and E form a second family, the invalid entry is skipped + relay('D', "\$${fingerprint('E')}, not a valid entry!"), + relay('E', "\$${fingerprint('D')}"), + relay('F', null), + ) + ) + + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + + val relays = byNickname() + relays.getValue("A").familyId.shouldNotBeNull() + relays.getValue("A").familyId shouldBe relays.getValue("B").familyId + relays.getValue("D").familyId.shouldNotBeNull() + relays.getValue("D").familyId shouldBe relays.getValue("E").familyId + relays.getValue("A").familyId shouldNotBe relays.getValue("D").familyId + relays.getValue("C").familyId.shouldBeNull() + relays.getValue("F").familyId.shouldBeNull() + } + + "families that are connected through a member are merged" { + relayDetailsRepository.saveAll( + listOf( + relay('A', "\$${fingerprint('B')}"), + relay('B', "\$${fingerprint('A')}, \$${fingerprint('C')}"), + relay('C', "\$${fingerprint('B')}"), + ) + ) + + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + + byNickname().values.map { it.familyId }.toSet().single().shouldNotBeNull() + } + + "computing the families again replaces the previous families" { + relayDetailsRepository.saveAll(listOf(relay('A', "\$${fingerprint('B')}"), relay('B', "\$${fingerprint('A')}"))) + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + val firstFamilyId = byNickname().getValue("A").familyId + + val relayA = byNickname().getValue("A") + relayA.familyEntries = null + relayDetailsRepository.save(relayA) + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + + firstFamilyId.shouldNotBeNull() + byNickname().getValue("A").familyId.shouldBeNull() + byNickname().getValue("B").familyId.shouldBeNull() + } + + "computeAllMissingFamiliesAndEvictCache computes families of months without any" { + relayDetailsRepository.saveAll(listOf(relay('A', "\$${fingerprint('B')}"), relay('B', "\$${fingerprint('A')}"))) + + relayDetailsUpdateService.computeAllMissingFamiliesAndEvictCache() + + eventually(10.seconds) { + byNickname().getValue("A").familyId.shouldNotBeNull() + } + } + + "lookupAllMissingAutonomousSystems fills in the autonomous system" { + relayDetailsRepository.saveAll( + listOf( + relay('A', null, address = "94.130.58.99"), + // Private addresses have no autonomous system + relay('B', null, address = "10.0.0.1"), + ) + ) + + relayDetailsUpdateService.lookupAllMissingAutonomousSystems() + + eventually(10.seconds) { + byNickname().getValue("A").autonomousSystemNumber shouldBe 24940 + } + byNickname().getValue("A").autonomousSystemName.shouldNotBeNull() + byNickname().getValue("B").autonomousSystemNumber.shouldBeNull() + } +}) diff --git a/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceTest.kt new file mode 100644 index 00000000..5a5110f8 --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceTest.kt @@ -0,0 +1,69 @@ +package org.tormap.service + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.booleans.shouldBeTrue +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.springframework.transaction.support.TransactionTemplate +import org.tormap.database.repository.MonthFamilyMembersCount +import org.tormap.database.repository.RelayDetailsRepositoryImpl +import java.util.concurrent.CompletableFuture +import javax.sql.DataSource + +class RelayDetailsUpdateServiceTest : StringSpec({ + "computeAllMissingFamiliesAndEvictCache evicts the cache only after the families are computed" { + val repository = mockk(relaxed = true) + val cacheService = mockk(relaxed = true) + val coalesceService = mockk() + val familiesOfMonth = mapOf("2024-01" to CompletableFuture(), "2024-02" to CompletableFuture()) + every { repository.findDistinctMonthFamilyMemberCount() } returns listOf( + MonthFamilyMembersCount("2024-01", 0), + MonthFamilyMembersCount("2024-02", 0), + // Months that already have families are skipped + MonthFamilyMembersCount("2024-03", 5), + ) + every { coalesceService.submitAsync(any(), any()) } answers { + familiesOfMonth.getValue(firstArg().removePrefix("computeFamilies-")) + } + val service = RelayDetailsUpdateService( + repository, + mockk(relaxed = true), + cacheService, + coalesceService, + mockk(relaxed = true), + mockk(relaxed = true), + ) + + service.computeAllMissingFamiliesAndEvictCache() + + verify(exactly = 1) { coalesceService.submitAsync("computeFamilies-2024-01", any()) } + verify(exactly = 1) { coalesceService.submitAsync("computeFamilies-2024-02", any()) } + verify(exactly = 0) { coalesceService.submitAsync("computeFamilies-2024-03", any()) } + familiesOfMonth.getValue("2024-01").complete(null) + verify(exactly = 0) { cacheService.evictRelayLocationsPerDay(any()) } + + // A failed month does not prevent the eviction + familiesOfMonth.getValue("2024-02").completeExceptionally(IllegalStateException("failed")) + verify(exactly = 1) { cacheService.evictRelayLocationsPerDay(setOf("2024-01", "2024-02")) } + } + + "computeFamilies completes once all months are done" { + val coalesceService = mockk() + val future = CompletableFuture() + every { coalesceService.submitAsync(any(), any()) } returns future + val service = RelayDetailsUpdateService( + mockk(relaxed = true), + mockk(relaxed = true), + mockk(relaxed = true), + coalesceService, + mockk(relaxed = true), + mockk(relaxed = true), + ) + + val result = service.computeFamilies(setOf("2024-01")) + future.complete(null) + result.isDone.shouldBeTrue() + service.computeFamilies(emptySet()).isDone.shouldBeTrue() + } +}) From d037e865bf378b06a4b39f731052831e4ee0ff66 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:21 +0000 Subject: [PATCH 02/10] fix(backend): only give relays with the Guard flag a guard weight Clients only pick Guard flagged relays as entry guards, and a BadExit does not count as exit (Wgd), as in Tor's compute_weighted_bandwidths. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- .../kotlin/org/tormap/util/PathSelection.kt | 14 ++-- .../DescriptorProcessingServiceTest.kt | 76 ++++++++++++++++++- .../org/tormap/util/PathSelectionTest.kt | 13 +++- 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/backend/src/main/kotlin/org/tormap/util/PathSelection.kt b/backend/src/main/kotlin/org/tormap/util/PathSelection.kt index 83a2256d..a99032ea 100644 --- a/backend/src/main/kotlin/org/tormap/util/PathSelection.kt +++ b/backend/src/main/kotlin/org/tormap/util/PathSelection.kt @@ -13,9 +13,11 @@ object PathSelection { /** * Returns the share of a relay's consensus weight that clients use for choosing it as entry guard, so the * relay's probability of being picked as guard is `guardWeight / sum of the guard weights of all relays`. + * It follows the guard position weights of Tor clients (`compute_weighted_bandwidths` in Tor's node_select.c): * * - A relay that is not `Running` and `Valid` can not be picked, which results in 0. - * - Guard flagged relays use `Wgg`, relays that are also exits use `Wgd`, all other relays `Wgm`. + * - Clients only pick relays with the `Guard` flag as entry guards, all other relays result in 0. + * - Guard flagged relays that are also exits use `Wgd`, all other guards `Wgg`. A `BadExit` does not count as exit. * * @param consensusWeight the `w Bandwidth=` value of the relay's consensus entry * @param flags the names of the flags the relay has in the consensus @@ -23,13 +25,9 @@ object PathSelection { * @return null, if the consensus does not contain the required weight */ fun guardWeight(consensusWeight: Long, flags: Set, bandwidthWeights: Map?): Long? { - if (!flags.contains("Running") || !flags.contains("Valid")) return 0 - val weightKey = when { - flags.contains("Guard") && flags.contains("Exit") -> "Wgd" - flags.contains("Guard") -> "Wgg" - else -> "Wgm" - } - val weight = bandwidthWeights?.get(weightKey) ?: return null + if (!flags.contains("Running") || !flags.contains("Valid") || !flags.contains("Guard")) return 0 + val isExit = flags.contains("Exit") && !flags.contains("BadExit") + val weight = bandwidthWeights?.get(if (isExit) "Wgd" else "Wgg") ?: return null return Math.round(consensusWeight.toDouble() * weight / BANDWIDTH_WEIGHT_SCALE) } } diff --git a/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt index 2f046d95..3b177992 100644 --- a/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt @@ -6,7 +6,13 @@ import io.mockk.every import io.mockk.mockk import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles +import org.tormap.database.repository.RelayDetailsRepositoryImpl import org.tormap.database.repository.RelayLocationRepositoryImpl +import org.torproject.descriptor.Descriptor +import org.torproject.descriptor.DescriptorParseException +import org.torproject.descriptor.ServerDescriptor +import org.torproject.descriptor.UnparseableDescriptor +import java.io.File import org.torproject.descriptor.NetworkStatusEntry import org.torproject.descriptor.RelayNetworkStatusConsensus import java.time.LocalDate @@ -19,13 +25,16 @@ import java.util.TreeMap class DescriptorProcessingServiceTest( private val descriptorProcessingService: DescriptorProcessingService, private val relayLocationRepository: RelayLocationRepositoryImpl, + private val relayDetailsRepository: RelayDetailsRepositoryImpl, ) : StringSpec({ beforeEach { relayLocationRepository.deleteAll() + relayDetailsRepository.deleteAll() } val day = LocalDate.of(2024, 3, 5) - val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 0) + // Wgm is not 0, so a relay without the Guard flag shows that only guards get a guard weight + val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 5000) fun entry(letter: Char, flags: Set, bandwidth: Long): NetworkStatusEntry { val entry = mockk() @@ -92,4 +101,69 @@ class DescriptorProcessingServiceTest( descriptorProcessingService.processDescriptor(consensus(entry('A', setOf("Running", "Valid", "Guard"), 9000))) relayLocationRepository.findAll().single().consensusWeight shouldBe 4000 } + + fun serverDescriptor(nickname: String, published: LocalDate, familyEntries: List? = null): ServerDescriptor { + val descriptor = mockk(relaxed = true) + every { descriptor.fingerprint } returns "F".repeat(40) + every { descriptor.nickname } returns nickname + every { descriptor.address } returns "94.130.58.99" + every { descriptor.publishedMillis } returns published.atTime(8, 0).toInstant(ZoneOffset.UTC).toEpochMilli() + every { descriptor.platform } returns "Tor 0.4.8.12 on Linux" + every { descriptor.contact } returns "x".repeat(300) + every { descriptor.protocols } returns sortedMapOf("Link" to sortedSetOf(4, 5)) + every { descriptor.familyEntries } returns familyEntries + every { descriptor.linkProtocolVersions } returns null + every { descriptor.circuitProtocolVersions } returns null + return descriptor + } + + "server descriptors store the details of the relay per month" { + val result = descriptorProcessingService.processDescriptor( + serverDescriptor("first", day, listOf("\$${"A".repeat(40)}", "friend")) + ) + + result.yearMonth shouldBe "2024-03" + result.error shouldBe null + val relay = relayDetailsRepository.findAll().single() + relay.month shouldBe "2024-03" + relay.day shouldBe day + relay.nickname shouldBe "first" + relay.autonomousSystemNumber shouldBe 24940 + relay.familyEntries shouldBe "\$${"A".repeat(40)}, friend" + relay.protocols shouldBe "Link (4, 5)" + // Long untrusted values are truncated to fit the column + relay.contact!!.length shouldBe 255 + } + + "a newer server descriptor of the month replaces the older one, an older one is ignored" { + descriptorProcessingService.processDescriptor(serverDescriptor("first", day)) + descriptorProcessingService.processDescriptor(serverDescriptor("newer", day.plusDays(2))) + descriptorProcessingService.processDescriptor(serverDescriptor("older", day.plusDays(1))) + + val relay = relayDetailsRepository.findAll().single() + relay.nickname shouldBe "newer" + relay.day shouldBe day.plusDays(2) + + // Another month gets its own entry + descriptorProcessingService.processDescriptor(serverDescriptor("april", LocalDate.of(2024, 4, 1))) + relayDetailsRepository.findAll().map { it.month }.toSet() shouldBe setOf("2024-03", "2024-04") + } + + "unparsable descriptors are skipped without an error" { + val descriptor = mockk() + every { descriptor.descriptorFile } returns File("broken") + every { descriptor.descriptorParseException } returns DescriptorParseException("broken") + + val result = descriptorProcessingService.processDescriptor(descriptor) + result.yearMonth shouldBe null + result.error shouldBe null + } + + "unsupported descriptor types result in an error" { + val descriptor = mockk() + every { descriptor.descriptorFile } returns File("other") + + descriptorProcessingService.processDescriptor(descriptor).error shouldBe + "Descriptor type ${descriptor.javaClass.name} is not yet supported!" + } }) diff --git a/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt b/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt index b727ac96..8cb467f0 100644 --- a/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt +++ b/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt @@ -4,7 +4,7 @@ import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe class PathSelectionTest : StringSpec({ - val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 0) + val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 5000) val running = setOf("Running", "Valid") "guard relays use Wgg" { @@ -15,9 +15,15 @@ class PathSelectionTest : StringSpec({ PathSelection.guardWeight(1000, running + "Guard" + "Exit", weights) shouldBe 250 } - "relays without guard flag use Wgm" { + "guard relays with the BadExit flag do not count as exits" { + PathSelection.guardWeight(1000, running + "Guard" + "Exit" + "BadExit", weights) shouldBe 500 + } + + "relays without guard flag are never picked as guard" { PathSelection.guardWeight(1000, running + "Exit", weights) shouldBe 0 - PathSelection.guardWeight(1000, running, mapOf("Wgm" to 1000)) shouldBe 100 + PathSelection.guardWeight(1000, running + "Exit" + "BadExit", weights) shouldBe 0 + PathSelection.guardWeight(1000, running + "Fast" + "Stable", weights) shouldBe 0 + PathSelection.guardWeight(1000, running, null) shouldBe 0 } "relays that are not running or valid can not be picked" { @@ -32,5 +38,6 @@ class PathSelectionTest : StringSpec({ "missing weights result in null" { PathSelection.guardWeight(1000, running + "Guard", null) shouldBe null PathSelection.guardWeight(1000, running + "Guard", mapOf("Wmm" to 10000)) shouldBe null + PathSelection.guardWeight(1000, running + "Guard" + "Exit", mapOf("Wgg" to 10000)) shouldBe null } }) From be6c266a5d8beb2a7f7f2ea13784a1901e9e82a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:21 +0000 Subject: [PATCH 03/10] fix(backend): keep the known flags of relays that also have unknown flags A flag such as MiddleOnly dropped all flags of the relay. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- .../tormap/database/entity/RelayLocation.kt | 9 ++-- .../database/entity/RelayLocationTest.kt | 50 +++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 backend/src/test/kotlin/org/tormap/database/entity/RelayLocationTest.kt diff --git a/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt b/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt index 9c89c9a0..09aad537 100644 --- a/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt +++ b/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt @@ -37,11 +37,10 @@ class RelayLocation( @Column(length = 40, columnDefinition = "bpchar(40)") var fingerprint: String = networkStatusEntry.fingerprint - var flags: String? = try { - networkStatusEntry.flags.map { TorRelayFlag.valueOf(it).ordinal }.joinToString(", ") - } catch (exception: Exception) { - null - } + // Flags this version does not know (e.g. MiddleOnly) are skipped, so the known flags of the relay are kept + var flags: String? = networkStatusEntry.flags + .mapNotNull { flag -> TorRelayFlag.entries.find { it.name == flag }?.ordinal } + .joinToString(", ") /** * The `w Bandwidth=` value of the consensus entry, which clients use for path selection. diff --git a/backend/src/test/kotlin/org/tormap/database/entity/RelayLocationTest.kt b/backend/src/test/kotlin/org/tormap/database/entity/RelayLocationTest.kt new file mode 100644 index 00000000..395f284a --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/database/entity/RelayLocationTest.kt @@ -0,0 +1,50 @@ +package org.tormap.database.entity + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import org.tormap.adapter.dto.RelayLocationDto +import org.torproject.descriptor.NetworkStatusEntry +import java.math.BigDecimal +import java.time.LocalDate + +class RelayLocationTest : StringSpec({ + fun relayLocation(flags: Set, bandwidth: Long = 1000, bandwidthWeights: Map? = null): RelayLocation { + val entry = mockk() + every { entry.fingerprint } returns "A".repeat(40) + every { entry.flags } returns flags.toSortedSet() + every { entry.bandwidth } returns bandwidth + return RelayLocation(entry, LocalDate.of(2024, 3, 5), BigDecimal.ONE, BigDecimal.ONE, "DE", bandwidthWeights) + } + + "flags are stored as ordinals" { + relayLocation(setOf("Exit", "Guard", "Valid")).flags shouldBe "5, 7, 0" + } + + "flags unknown to TorMap are skipped instead of dropping all flags" { + val location = relayLocation(setOf("MiddleOnly", "Running", "Valid")) + location.flags shouldBe "3, 0" + RelayLocationDto(BigDecimal.ONE, BigDecimal.ONE, "DE", location.flags, null, null, null, null) + .flags shouldBe listOf(3, 0) + } + + "a relay without flags has none" { + val location = relayLocation(emptySet()) + location.flags shouldBe "" + RelayLocationDto(BigDecimal.ONE, BigDecimal.ONE, "DE", location.flags, null, null, null, null) + .flags shouldBe null + } + + "the weights are computed from the consensus entry" { + val location = relayLocation(setOf("Guard", "Running", "Valid"), 3000, mapOf("Wgg" to 5000)) + location.consensusWeight shouldBe 3000 + location.guardWeight shouldBe 1500 + } + + "a consensus entry without bandwidth has no weights" { + val location = relayLocation(setOf("Guard", "Running", "Valid"), -1, mapOf("Wgg" to 5000)) + location.consensusWeight shouldBe null + location.guardWeight shouldBe null + } +}) From 2afc829484cabeb9e4d47f0a42dec1b46fc96d54 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:21 +0000 Subject: [PATCH 04/10] fix(backend): report an unknown request character encoding as unsupported encoding Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- .../tormap/config/RequestBodyLimitConfig.kt | 8 +- .../config/RequestBodyLimitFilterTest.kt | 83 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 backend/src/test/kotlin/org/tormap/config/RequestBodyLimitFilterTest.kt diff --git a/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt b/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt index 25f3464f..472c2ce8 100644 --- a/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt +++ b/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt @@ -14,6 +14,7 @@ import org.springframework.web.filter.OncePerRequestFilter import org.springframework.web.server.ContentTooLargeException import java.io.BufferedReader import java.io.InputStreamReader +import java.io.UnsupportedEncodingException import java.nio.charset.Charset /** @@ -57,7 +58,12 @@ class RequestBodyLimitFilter(private val maxBodyBytes: Long) : OncePerRequestFil override fun getInputStream(): ServletInputStream = inputStream override fun getReader(): BufferedReader { - val charset = characterEncoding?.let { Charset.forName(it) } ?: Charsets.ISO_8859_1 + val charset = try { + characterEncoding?.let { Charset.forName(it) } ?: Charsets.ISO_8859_1 + } catch (_: IllegalArgumentException) { + // The encoding comes from the request, the servlet API reports an unknown one like this + throw UnsupportedEncodingException(characterEncoding) + } return BufferedReader(InputStreamReader(inputStream, charset)) } } diff --git a/backend/src/test/kotlin/org/tormap/config/RequestBodyLimitFilterTest.kt b/backend/src/test/kotlin/org/tormap/config/RequestBodyLimitFilterTest.kt new file mode 100644 index 00000000..ccfe39c5 --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/config/RequestBodyLimitFilterTest.kt @@ -0,0 +1,83 @@ +package org.tormap.config + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import jakarta.servlet.FilterChain +import jakarta.servlet.ReadListener +import jakarta.servlet.ServletRequest +import jakarta.servlet.http.HttpServletRequest +import org.springframework.mock.web.MockHttpServletRequest +import org.springframework.mock.web.MockHttpServletResponse +import org.springframework.web.server.ContentTooLargeException +import java.io.UnsupportedEncodingException + +class RequestBodyLimitFilterTest : StringSpec({ + val filter = RequestBodyLimitFilter(maxBodyBytes = 10) + + fun request(body: String, encoding: String? = null) = MockHttpServletRequest("POST", "/relay/details/relay/identifiers").apply { + setContent(body.toByteArray()) + if (encoding != null) characterEncoding = encoding + } + + fun filterWith(request: HttpServletRequest, read: (ServletRequest) -> Unit): MockHttpServletResponse { + val response = MockHttpServletResponse() + filter.doFilter(request, response, FilterChain { wrapped, _ -> read(wrapped) }) + return response + } + + "a declared Content-Length above the limit is rejected before the body is read" { + var called = false + val response = filterWith(request("x".repeat(11))) { called = true } + response.status shouldBe 413 + called shouldBe false + } + + "a body within the limit can be read as stream and as text" { + filterWith(request("0123456789")) { it.inputStream.readAllBytes().decodeToString() shouldBe "0123456789" } + filterWith(request("äöü", "UTF-8")) { it.reader.readText() shouldBe "äöü" } + // Without an encoding the servlet default ISO-8859-1 applies + filterWith(request("abc")) { it.reader.readText() shouldBe "abc" } + } + + "reading beyond the limit fails, byte by byte as well as in blocks" { + // Without a known length, as with chunked transfer encoding + fun unknownLength(body: String) = object : MockHttpServletRequest("POST", "/") { + override fun getContentLengthLong() = -1L + }.apply { setContent(body.toByteArray()) } + + filterWith(unknownLength("x".repeat(20))) { request -> + shouldThrow { request.inputStream.readAllBytes() } + } + filterWith(unknownLength("x".repeat(11))) { request -> + val stream = request.inputStream + repeat(10) { stream.read() shouldBe 'x'.code } + shouldThrow { stream.read() } + } + filterWith(unknownLength("")) { it.inputStream.read() shouldBe -1 } + } + + "an unknown character encoding is reported like by the servlet API" { + filterWith(request("abc", "no-such-charset")) { + shouldThrow { it.reader } + } + } + + "the stream state and read listener are those of the request" { + filterWith(request("abc")) { + val stream = it.inputStream + stream.isFinished shouldBe false + stream.isReady shouldBe true + stream.readAllBytes() + stream.isFinished shouldBe true + // MockHttpServletRequest does not support async IO, which shows the call is delegated + shouldThrow { + stream.setReadListener(object : ReadListener { + override fun onDataAvailable() = Unit + override fun onAllDataRead() = Unit + override fun onError(t: Throwable) = Unit + }) + } + } + } +}) From 4f987a646d4f5582c396f1c03dc1d14343feb17f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:21 +0000 Subject: [PATCH 05/10] fix(frontend): do not scroll the page when selecting a timeline day with space Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- .../date/RelayCountTimeline.test.tsx | 53 +++++++++++++++++++ .../components/date/RelayCountTimeline.tsx | 6 ++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/date/RelayCountTimeline.test.tsx b/frontend/src/components/date/RelayCountTimeline.test.tsx index 41ebeacf..705dacaa 100644 --- a/frontend/src/components/date/RelayCountTimeline.test.tsx +++ b/frontend/src/components/date/RelayCountTimeline.test.tsx @@ -79,4 +79,57 @@ describe("RelayCountTimeline", () => { await renderTimeline() await expect.element(page.getByRole("img", {name: /Relays per day/})).not.toBeInTheDocument() }) + + it("selects an outlier day with the keyboard", async () => { + mockBackend.onGet("/relay/statistics/daily-counts", counts) + await renderTimeline() + + const outlier = page.getByTestId("relay-count-outlier") + await expect.element(outlier).toBeVisible() + outlier.element().dispatchEvent(new KeyboardEvent("keydown", {key: "Tab", bubbles: true})) + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[39]) + + const space = new KeyboardEvent("keydown", {key: " ", bubbles: true, cancelable: true}) + outlier.element().dispatchEvent(space) + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[20]) + expect(space.defaultPrevented).toBe(true) + + }) + + it("selects an outlier day with Enter", async () => { + mockBackend.onGet("/relay/statistics/daily-counts", counts) + await renderTimeline() + + const outlier = page.getByTestId("relay-count-outlier") + await expect.element(outlier).toBeVisible() + outlier.element().dispatchEvent(new KeyboardEvent("keydown", {key: "Enter", bubbles: true})) + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[20]) + }) + + it("describes the outlier with its change and marks the selected day", async () => { + const spike = counts.map((count, i) => i === 10 ? {...count, total: 9000} : count) + mockBackend.onGet("/relay/statistics/daily-counts", spike) + await renderTimeline() + + await expect.element(page.getByRole("button", {name: new RegExp(`Select ${days[10]}: 9000 relays \\(\\+`)})).toBeVisible() + await expect.element(page.getByRole("button", {name: new RegExp(`Select ${days[20]}: 2500 relays \\(-`)})).toBeVisible() + // The selected (latest) day is marked with a vertical line at the right end + expect(document.querySelector("svg line")?.getAttribute("x1")).toBe("1000") + }) + + it("breaks the line at days without counts", async () => { + const withGap = counts.filter((_, i) => i < 15 || i > 17) + mockBackend.onGet("/relay/statistics/daily-counts", withGap) + await renderTimeline() + + await expect.element(page.getByRole("img", {name: /Relays per day/})).toBeVisible() + expect(document.querySelectorAll("svg polyline")).toHaveLength(2) + }) + + it("renders nothing if no count matches an available day", async () => { + mockBackend.onGet("/relay/statistics/daily-counts", [{day: "2001-01-01", total: 1, exit: 0, guard: 0, other: 1}]) + await renderTimeline() + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[39]) + await expect.element(page.getByRole("img", {name: /Relays per day/})).not.toBeInTheDocument() + }) }) diff --git a/frontend/src/components/date/RelayCountTimeline.tsx b/frontend/src/components/date/RelayCountTimeline.tsx index e5b77b15..506ec545 100644 --- a/frontend/src/components/date/RelayCountTimeline.tsx +++ b/frontend/src/components/date/RelayCountTimeline.tsx @@ -116,7 +116,11 @@ export const RelayCountTimeline: FunctionComponent = () => { style={{cursor: "pointer"}} onClick={() => setSelectedDate(outlier.day)} onKeyDown={event => { - if (event.key === "Enter" || event.key === " ") setSelectedDate(outlier.day) + if (event.key === "Enter" || event.key === " ") { + // Space would otherwise scroll the page, like for a native button + event.preventDefault() + setSelectedDate(outlier.day) + } }} > {outlier.label} From 9df2bc7db397458bccf271e8d8d80e7d34f8813e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:37 +0000 Subject: [PATCH 06/10] chore(backend): add JaCoCo coverage report Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- AGENTS.md | 2 +- backend/README.md | 2 +- backend/build.gradle.kts | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b08e9089..1657ba80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ Rules for AI coding agents (and a quick reference for humans) working in this re - `.github/workflows/`: CI. Actions are pinned by commit SHA with a version comment. ## Build and test -- Backend: `cd backend && ./gradlew test` (needs a JDK matching `sourceCompatibility` in `backend/build.gradle.kts` and a running Docker daemon for Testcontainers). +- Backend: `cd backend && ./gradlew test` (needs a JDK matching `sourceCompatibility` in `backend/build.gradle.kts` and a running Docker daemon for Testcontainers). Add `jacocoTestReport` for coverage (`build/reports/jacoco/test/html/index.html`, vendored `org.torproject` code is excluded). - Local backend image: `./gradlew jibDockerBuild -Djib.to.image=tormap/backend:local -Djib.to.tags=local`. - Frontend: `cd frontend && yarn install --immutable && yarn build && yarn lint`. - Frontend tests: `cd frontend && yarn test:run` (Vitest). `*.unit.test.ts` run in plain Node, all other `*.test.ts(x)` run in headless Chromium (browser mode), so run `yarn playwright install chromium` once. Use `yarn test` for watch mode and `yarn test:coverage` for coverage. The sandbox blocks Chromium, so browser tests must run outside it. diff --git a/backend/README.md b/backend/README.md index 23826678..e02cc24a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -50,7 +50,7 @@ less reliable. `SPRING_PROFILES_ACTIVE` is set. 4. Backend should be available at http://localhost:8080 -Run `./gradlew test`, to only build & tests. +Run `./gradlew test`, to only build & tests. Run `./gradlew test jacocoTestReport` for a coverage report in `build/reports/jacoco/test/html/index.html`. In a freshly cloned repository the backend will start to download an archive `>33 GB` in size from [TorProject](https://metrics.torproject.org/collector/archive/). The total time for the whole archive will depend diff --git a/backend/build.gradle.kts b/backend/build.gradle.kts index a614bae9..37d20727 100644 --- a/backend/build.gradle.kts +++ b/backend/build.gradle.kts @@ -17,6 +17,9 @@ plugins { // Build and push docker images id("com.google.cloud.tools.jib") version "3.5.4" + + // Test coverage, run with ./gradlew test jacocoTestReport + jacoco } repositories { @@ -152,6 +155,22 @@ tasks.withType { useJUnitPlatform() } +jacoco { + // Java 25 class files need JaCoCo 0.8.14 or later + toolVersion = "0.8.15" +} + +// Writes build/reports/jacoco/test/html/index.html and an XML report for tools +tasks.jacocoTestReport { + dependsOn(tasks.test) + reports { + xml.required = true + html.required = true + } + // The vendored metrics-lib (org.torproject) is mostly unused by TorMap, so it would dilute the numbers + classDirectories.setFrom(classDirectories.files.map { fileTree(it) { exclude("org/torproject/**") } }) +} + // The local development server uses the dev profile unless SPRING_PROFILES_ACTIVE is set tasks.bootRun { if (providers.environmentVariable("SPRING_PROFILES_ACTIVE").orNull == null) { From d417896c5e3ef7b1dc45ca255784e03517a9b02f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:37 +0000 Subject: [PATCH 07/10] test(backend): cover families, descriptor files and processing, admin user, statistics endpoint and lookups Moves the admin user creation and the public address check into functions that tests can call, without changing their behavior. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- .../org/tormap/config/SecurityConfig.kt | 17 ++- .../tormap/service/ReverseDnsLookupService.kt | 36 +++--- .../kotlin/org/tormap/config/AdminUserTest.kt | 51 +++++++++ .../RelayStatisticsControllerTest.kt | 71 ++++++++++++ .../tormap/service/CountryUsageServiceTest.kt | 26 ++++- .../service/DescriptorFileServiceTest.kt | 105 ++++++++++++++++++ .../org/tormap/service/IpLookupServiceTest.kt | 24 +++- .../service/PubliclyRoutableAddressTest.kt | 38 +++++++ .../service/ReverseDnsLookupServiceTest.kt | 32 ++++++ .../tormap/service/SchedulingServiceTest.kt | 36 ++++++ 10 files changed, 414 insertions(+), 22 deletions(-) create mode 100644 backend/src/test/kotlin/org/tormap/config/AdminUserTest.kt create mode 100644 backend/src/test/kotlin/org/tormap/controller/RelayStatisticsControllerTest.kt create mode 100644 backend/src/test/kotlin/org/tormap/service/DescriptorFileServiceTest.kt create mode 100644 backend/src/test/kotlin/org/tormap/service/PubliclyRoutableAddressTest.kt diff --git a/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt b/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt index 3174b829..81c9db80 100644 --- a/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt +++ b/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt @@ -52,12 +52,23 @@ class SecurityConfig( fun passwordEncoder(): PasswordEncoder = BCryptPasswordEncoder() @Bean - fun userDetailsService(passwordEncoder: PasswordEncoder): UserDetailsService { - if (!System.getenv(ENV_REMOVED_ADMIN_PASSWORD_BCRYPT).isNullOrBlank()) { + fun userDetailsService(passwordEncoder: PasswordEncoder): UserDetailsService = + buildUserDetailsService(passwordEncoder, System.getenv(ENV_ADMIN_PASSWORD), System.getenv(ENV_REMOVED_ADMIN_PASSWORD_BCRYPT)) + + /** + * Creates the admin user from the values of [ENV_ADMIN_PASSWORD] and [ENV_REMOVED_ADMIN_PASSWORD_BCRYPT]. + * Without a password there is no user, so the actuator denies every request. + */ + internal fun buildUserDetailsService( + passwordEncoder: PasswordEncoder, + adminPassword: String?, + removedAdminPasswordBcrypt: String?, + ): UserDetailsService { + if (!removedAdminPasswordBcrypt.isNullOrBlank()) { logger.warn("$ENV_REMOVED_ADMIN_PASSWORD_BCRYPT is no longer supported and is ignored. Use $ENV_ADMIN_PASSWORD instead.") } - val password = System.getenv(ENV_ADMIN_PASSWORD)?.trim().orEmpty() + val password = adminPassword?.trim().orEmpty() if (password.isBlank()) { logger.warn("------------------------------------------------------") diff --git a/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt b/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt index aa5caf5c..6f624c10 100644 --- a/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt +++ b/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt @@ -85,24 +85,28 @@ class DNSJavaReverseDnsResolver : ReverseDnsResolver { private fun lookup(name: Name, type: Int): List = Lookup(name, type).apply { setResolver(dnsResolver) }.run()?.toList() ?: emptyList() - - private fun InetAddress.isULA(): Boolean = - this is Inet6Address && - (address[0].toInt() and 0xFE) == 0xFC // fc00::/7 - - private fun InetAddress.isPubliclyRoutable(): Boolean = - !isLoopbackAddress && - !isLinkLocalAddress && - !isSiteLocalAddress && // reliable for IPv4 - !isULA() && // fills the IPv6 gap isSiteLocalAddress misses - !isMulticastAddress && - !isAnyLocalAddress && - address.first() != 0.toByte() && - !(address.size == 4 && - address[0] == 100.toByte() && - (address[1].toInt() and 0xFF) in 64..127) } +private fun InetAddress.isULA(): Boolean = + this is Inet6Address && + (address[0].toInt() and 0xFE) == 0xFC // fc00::/7 + +/** + * Whether the address can be reached on the public internet. Forward lookups only verify a PTR name with such + * addresses, so that a name pointing to an internal address is never reported as verified. + */ +internal fun InetAddress.isPubliclyRoutable(): Boolean = + !isLoopbackAddress && + !isLinkLocalAddress && + !isSiteLocalAddress && // reliable for IPv4 + !isULA() && // fills the IPv6 gap isSiteLocalAddress misses + !isMulticastAddress && + !isAnyLocalAddress && + address.first() != 0.toByte() && + !(address.size == 4 && + address[0] == 100.toByte() && + (address[1].toInt() and 0xFF) in 64..127) + @Service class ReverseDnsLookupService( private val reverseDnsResolver: ReverseDnsResolver, diff --git a/backend/src/test/kotlin/org/tormap/config/AdminUserTest.kt b/backend/src/test/kotlin/org/tormap/config/AdminUserTest.kt new file mode 100644 index 00000000..ec9d76ae --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/config/AdminUserTest.kt @@ -0,0 +1,51 @@ +package org.tormap.config + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import org.springframework.security.core.userdetails.UsernameNotFoundException +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder +import org.springframework.web.cors.CorsConfiguration +import org.springframework.mock.web.MockHttpServletRequest + +class AdminUserTest : StringSpec({ + val passwordEncoder = BCryptPasswordEncoder(4) + fun securityConfig(allowedOrigins: String = "https://tormap.org") = + SecurityConfig("admin", "/actuator", allowedOrigins, false) + + "an admin password creates the admin user with a hashed password" { + val users = securityConfig().buildUserDetailsService(passwordEncoder, " a-long-generated-password ", null) + val admin = users.loadUserByUsername("admin") + admin.authorities.map { it.authority } shouldBe listOf("ROLE_ADMIN") + // Surrounding whitespace, e.g. from an env file, is not part of the password + passwordEncoder.matches("a-long-generated-password", admin.password) shouldBe true + admin.password!!.startsWith("{") shouldBe false + } + + "a short password still creates the admin user" { + val users = securityConfig().buildUserDetailsService(passwordEncoder, "short", null) + passwordEncoder.matches("short", users.loadUserByUsername("admin").password) shouldBe true + } + + "without a password there is no user" { + listOf(null, "", " ").forEach { password -> + val users = securityConfig().buildUserDetailsService(passwordEncoder, password, null) + shouldThrow { users.loadUserByUsername("admin") } + } + } + + "the removed bcrypt variable does not create a user" { + val hash = passwordEncoder.encode("a-long-generated-password") + val users = securityConfig().buildUserDetailsService(passwordEncoder, null, hash) + shouldThrow { users.loadUserByUsername("admin") } + } + + "allowed CORS origins are read from a comma separated list" { + val source = securityConfig(" https://a.example , ,https://*.b.example").corsConfigurationSource() + val cors: CorsConfiguration = source.getCorsConfiguration(MockHttpServletRequest("GET", "/relay/location/days"))!! + cors.allowedOriginPatterns shouldBe listOf("https://a.example", "https://*.b.example") + cors.checkOrigin("https://x.b.example") shouldBe "https://x.b.example" + cors.checkOrigin("https://evil.example") shouldBe null + cors.allowCredentials shouldBe false + } +}) diff --git a/backend/src/test/kotlin/org/tormap/controller/RelayStatisticsControllerTest.kt b/backend/src/test/kotlin/org/tormap/controller/RelayStatisticsControllerTest.kt new file mode 100644 index 00000000..691e73be --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/controller/RelayStatisticsControllerTest.kt @@ -0,0 +1,71 @@ +package org.tormap.controller + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldStartWith +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc +import org.springframework.cache.CacheManager +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.web.servlet.MockMvc +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import org.tormap.config.CacheConfig +import org.tormap.database.entity.TorRelayFlag +import org.tormap.database.repository.RelayLocationRepositoryImpl +import org.tormap.mockRelayLocation +import org.tormap.service.CacheService +import java.time.LocalDate + +@SpringBootTest +@AutoConfigureMockMvc +@ActiveProfiles("test") +class RelayStatisticsControllerTest( + private val mockMvc: MockMvc, + private val relayLocationRepository: RelayLocationRepositoryImpl, + private val cacheManager: CacheManager, + private val cacheService: CacheService, +) : StringSpec({ + val day = LocalDate.of(2024, 3, 5) + + beforeEach { + relayLocationRepository.deleteAll() + cacheManager.getCache(CacheConfig.RELAY_DAILY_COUNTS)?.clear() + } + + "returns the relay counts per day" { + relayLocationRepository.saveAll( + listOf( + mockRelayLocation(day, 'A', setOf(TorRelayFlag.Exit)), + mockRelayLocation(day, 'B', setOf(TorRelayFlag.Guard)), + mockRelayLocation(day, 'C'), + ) + ) + mockMvc.perform(get("/relay/statistics/daily-counts")) + .andExpect(status().isOk) + .andExpect(jsonPath("$.length()").value(1)) + .andExpect(jsonPath("$[0].day").value("2024-03-05")) + .andExpect(jsonPath("$[0].total").value(3)) + .andExpect(jsonPath("$[0].exit").value(1)) + .andExpect(jsonPath("$[0].guard").value(1)) + .andExpect(jsonPath("$[0].other").value(1)) + } + + "serves the cached counts until the cache is refreshed" { + mockMvc.perform(get("/relay/statistics/daily-counts")).andExpect(jsonPath("$.length()").value(0)) + relayLocationRepository.save(mockRelayLocation(day, 'A')) + mockMvc.perform(get("/relay/statistics/daily-counts")).andExpect(jsonPath("$.length()").value(0)) + + cacheService.cacheRelayDailyCounts().get() + + mockMvc.perform(get("/relay/statistics/daily-counts")).andExpect(jsonPath("$.length()").value(1)) + } + + "GET / names the application" { + val response = mockMvc.perform(get("/")).andExpect(status().isOk).andReturn().response + response.contentAsString shouldStartWith "TorMap " + // Swagger is disabled by default, so the docs are not linked + response.contentAsString.contains("Docs:") shouldBe false + } +}) diff --git a/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt index ce0ff49f..5e92392d 100644 --- a/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt @@ -3,6 +3,7 @@ package org.tormap.service import com.sun.net.httpserver.HttpServer import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe import org.springframework.boot.test.context.SpringBootTest import org.springframework.cache.CacheManager import org.springframework.jdbc.core.JdbcTemplate @@ -46,9 +47,9 @@ class CountryUsageServiceTest( } val baseUrl = "http://127.0.0.1:${server.address.port}" - fun service(maxDownloadBytes: Long = 10_000_000) = CountryUsageService( + fun service(maxDownloadBytes: Long = 10_000_000, url: String = baseUrl) = CountryUsageService( UsageStatisticsConfig( - baseUrl = baseUrl, + baseUrl = url, relayUsersPath = "/userstats-relay-country.csv", maxDownloadBytes = maxDownloadBytes, connectTimeout = Duration.ofSeconds(2), @@ -119,4 +120,25 @@ class CountryUsageServiceTest( service().refresh() shouldBe 1 countryUsageRepository.findLatestDay() shouldBe LocalDate.of(2024, 3, 5) } + + "malformed lines are skipped and the valid ones are stored" { + body = "date,country,users,lower,upper,frac\n2024-03-05,de,2,,,50\n2024-03-05,fr,many,,,50\n" + + "2024-03-05,it,3,,,50,extra\n2024-13-01,es,4,,,50\n" + service().refresh() shouldBe 1 + countryUsageRepository.findAllByDay(LocalDate.of(2024, 3, 5)).map { it.country } shouldBe listOf("DE") + } + + "an import without rows keeps the cache" { + val cache = cacheManager.getCache(CacheConfig.COUNTRY_USAGE_PER_DAY)!! + cache.put("2024-03-05", listOf()) + body = "date,country,users,lower,upper,frac\n" + service().refresh() shouldBe 0 + cache.get("2024-03-05") shouldNotBe null + } + + "only http and https URLs are requested" { + service(url = "file:///etc").refresh() shouldBe 0 + service(url = "http://").refresh() shouldBe 0 + requestedUris shouldBe emptyList() + } }) diff --git a/backend/src/test/kotlin/org/tormap/service/DescriptorFileServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/DescriptorFileServiceTest.kt new file mode 100644 index 00000000..10b3087a --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/DescriptorFileServiceTest.kt @@ -0,0 +1,105 @@ +package org.tormap.service + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.shouldBe +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.tormap.config.value.DescriptorConfig +import org.tormap.database.entity.DescriptorFileId +import org.tormap.database.entity.DescriptorType +import org.tormap.database.entity.ProcessedFile +import org.tormap.database.repository.ProcessedFileRepository +import java.io.File +import java.nio.file.Files +import java.time.Duration +import java.time.Instant + +@SpringBootTest +@ActiveProfiles("test") +class DescriptorFileServiceTest( + private val descriptorConfig: DescriptorConfig, + private val processedFileRepository: ProcessedFileRepository, +) : StringSpec({ + val downloadDirectory = Files.createTempDirectory("tormap-descriptors").toFile() + val service = DescriptorFileService( + descriptorConfig.copy(localDownloadDirectory = downloadDirectory.absolutePath), + processedFileRepository, + ) + val apiPath = "/recent/relay-descriptors/server-descriptors" + + beforeEach { + processedFileRepository.deleteAll() + downloadDirectory.listFiles()?.forEach { it.deleteRecursively() } + } + + afterSpec { + downloadDirectory.deleteRecursively() + } + + fun writeDescriptorFile(name: String): File { + val directory = File(downloadDirectory, apiPath).apply { mkdirs() } + // The type annotation lets the reader pick the parser, the content itself does not parse + return File(directory, name).apply { writeText("@type server-descriptor 1.0\nrouter broken\n") } + } + + fun descriptorFileNames(type: DescriptorType = DescriptorType.RECENT_RELAY_SERVER) = + service.getDescriptorDiskReader(apiPath, type).map { it.descriptorFile.name } + + "reads the descriptors of files that were not processed yet" { + writeDescriptorFile("2024-03-05-00-00-00-server-descriptors") + descriptorFileNames() shouldBe listOf("2024-03-05-00-00-00-server-descriptors") + } + + "skips files that were processed unless they changed since" { + val file = writeDescriptorFile("2024-03-05-00-00-00-server-descriptors") + service.saveProcessedFileReference(file, DescriptorType.RECENT_RELAY_SERVER) + descriptorFileNames().shouldBeEmpty() + // Another descriptor type has its own processed files + descriptorFileNames(DescriptorType.RECENT_RELAY_CONSENSUS) shouldBe listOf(file.name) + + file.setLastModified(file.lastModified() + 60_000) + descriptorFileNames() shouldBe listOf(file.name) + } + + "saving a processed file again updates its reference" { + val file = writeDescriptorFile("2024-03-05-00-00-00-server-descriptors") + service.saveProcessedFileReference(file, DescriptorType.ARCHIVE_RELAY_SERVER) + val firstProcessedAt = processedFileRepository.findAll().single().processedAt + Thread.sleep(5) + service.saveProcessedFileReference(file, DescriptorType.ARCHIVE_RELAY_SERVER) + val stored = processedFileRepository.findAll().single() + stored.id shouldBe DescriptorFileId(DescriptorType.ARCHIVE_RELAY_SERVER, file.name) + (stored.processedAt > firstProcessedAt) shouldBe true + } + + "deletes only old references of recent files" { + val old = Instant.now().minus(Duration.ofDays(DAYS_TO_KEEP_RECENT_FILES + 1)).toEpochMilli() + val fresh = Instant.now().toEpochMilli() + processedFileRepository.saveAll( + listOf( + ProcessedFile(DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "old-recent"), old), + ProcessedFile(DescriptorFileId(DescriptorType.RECENT_RELAY_CONSENSUS, "old-recent"), old), + ProcessedFile(DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "fresh-recent"), fresh), + ProcessedFile(DescriptorFileId(DescriptorType.ARCHIVE_RELAY_SERVER, "old-archive"), old), + ) + ) + + service.deleteRecentFileReferences() + + processedFileRepository.findAll().map { it.id.type to it.id.filename }.toSet() shouldBe setOf( + DescriptorType.RECENT_RELAY_SERVER to "fresh-recent", + DescriptorType.ARCHIVE_RELAY_SERVER to "old-archive", + ) + } + + "file ids are equal by type and name" { + val id = DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "a") + id shouldBe DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "a") + id.hashCode() shouldBe DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "a").hashCode() + (id == DescriptorFileId(DescriptorType.ARCHIVE_RELAY_SERVER, "a")) shouldBe false + (id == DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "b")) shouldBe false + id.equals("a") shouldBe false + } + +}) diff --git a/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt index b1b0283c..f87ebb30 100644 --- a/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt @@ -1,6 +1,7 @@ package org.tormap.service import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.shouldBe import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -28,6 +29,27 @@ class IpLookupServiceTest( } "returns null for private IP address" { - ipLookupService.lookupLocation("10.0.0.1") shouldBe null + listOf("10.0.0.1", "172.16.0.1", "172.31.255.255", "192.168.1.1").forEach { + ipLookupService.lookupLocation(it) shouldBe null + ipLookupService.lookupAutonomousSystem(it) shouldBe null + } + } + + "neighbours of private ranges are looked up" { + // Hetzner and Google, right next to 172.16.0.0/12 and 192.168.0.0/16 + ipLookupService.lookupLocation("172.15.255.255").shouldNotBeNull() + ipLookupService.lookupLocation("172.32.0.1").shouldNotBeNull() + ipLookupService.lookupLocation("192.169.0.1").shouldNotBeNull() + } + + "returns null for addresses without a database entry" { + listOf("127.0.0.1", "0.0.0.1").forEach { + ipLookupService.lookupLocation(it) shouldBe null + ipLookupService.lookupAutonomousSystem(it) shouldBe null + } + } + + "IPv6 addresses are not looked up" { + ipLookupService.lookupLocation("2001:4860:4860::8888") shouldBe null } }) diff --git a/backend/src/test/kotlin/org/tormap/service/PubliclyRoutableAddressTest.kt b/backend/src/test/kotlin/org/tormap/service/PubliclyRoutableAddressTest.kt new file mode 100644 index 00000000..5a19f03a --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/PubliclyRoutableAddressTest.kt @@ -0,0 +1,38 @@ +package org.tormap.service + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import java.net.InetAddress + +class PubliclyRoutableAddressTest : StringSpec({ + // Literal addresses, so no DNS lookup takes place + fun routable(address: String) = InetAddress.getByName(address).isPubliclyRoutable() + + "public addresses are routable" { + listOf("1.1.1.1", "94.130.58.99", "100.63.255.255", "100.128.0.1", "2001:4860:4860::8888").forEach { + routable(it) shouldBe true + } + } + + "internal and special addresses are not routable" { + listOf( + "127.0.0.1", "::1", // loopback + "169.254.1.1", "fe80::1", // link local + "10.0.0.1", "172.16.0.1", "192.168.1.1", // private + "fc00::1", "fd12:3456::1", // unique local + "224.0.0.1", "ff02::1", // multicast + "0.0.0.0", "::", // any + "0.1.2.3", // "this" network + "100.64.0.1", "100.127.255.255", // carrier-grade NAT + ).forEach { + withClue(it) { routable(it) shouldBe false } + } + } + + "DNSJava resolver drops unresolvable names without throwing" { + val resolver = DNSJavaReverseDnsResolver() + resolver.lookupAddresses("not a valid host name..") shouldBe emptyList() + resolver.lookupPtrRecords("not an address") shouldBe emptyList() + } +}) diff --git a/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt index d57065ef..2f887911 100644 --- a/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt @@ -126,6 +126,38 @@ class ReverseDnsLookupServiceTest : StringSpec({ reverseDnsLookupService.destroy() } } + + "lookupHostNames returns what is known when a lookup fails" { + val failingResolver = object : ReverseDnsResolver { + override fun lookupPtrRecords(ipAddress: String) = listOf("host.example.") + override fun lookupAddresses(hostName: String): List = throw IllegalStateException("resolver broke") + } + + val result = ReverseDnsLookupService(failingResolver).lookupHostNames("1.2.3.4") + + result shouldBe ReverseDnsLookupResult(unverifiedHostNames = listOf("host.example")) + } + + "lookupHostNames keeps the interrupt of the calling thread" { + val reverseDnsResolver = LookupServiceFakeReverseDnsResolver( + ptrRecordsByIpAddress = mapOf("1.2.3.4" to listOf("slow.example.")), + ptrLookupDelay = Duration.ofSeconds(5), + ) + val service = ReverseDnsLookupService(reverseDnsResolver, lookupTimeout = Duration.ofSeconds(10)) + var result: ReverseDnsLookupResult? = null + var interrupted = false + val caller = thread { + result = service.lookupHostNames("1.2.3.4") + interrupted = Thread.currentThread().isInterrupted + } + Thread.sleep(200) + caller.interrupt() + caller.join(5_000) + + result shouldBe ReverseDnsLookupResult() + interrupted shouldBe true + service.destroy() + } }) private class LookupServiceFakeReverseDnsResolver( diff --git a/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt index e819e3e5..7b5a4a50 100644 --- a/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt @@ -3,7 +3,12 @@ package org.tormap.service import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.shouldBeEmpty import io.kotest.matchers.collections.shouldNotBeEmpty +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify import org.springframework.scheduling.annotation.Scheduled +import org.tormap.config.value.DescriptorConfig +import org.tormap.database.entity.DescriptorType import java.util.concurrent.Future class SchedulingServiceTest : StringSpec({ @@ -18,4 +23,35 @@ class SchedulingServiceTest : StringSpec({ .map { "${it.name}: ${it.returnType}" } .shouldBeEmpty() } + + "each scheduled method starts its job" { + val descriptorConfig = mockk() + every { descriptorConfig.recentRelayConsensuses } returns "/recent/consensuses" + every { descriptorConfig.recentRelayServers } returns "/recent/servers" + every { descriptorConfig.archiveRelayConsensuses } returns "/archive/consensuses" + every { descriptorConfig.archiveRelayServers } returns "/archive/servers" + val coordination = mockk(relaxed = true) + val relayDetails = mockk(relaxed = true) + val files = mockk(relaxed = true) + val countryUsage = mockk(relaxed = true) + val service = SchedulingService(descriptorConfig, coordination, relayDetails, files, countryUsage) + + service.recentConsensousDescriptors() + service.recentServerDescriptors() + service.archiveRelayConsensuses() + service.archiveRelayServers() + service.refreshCountryUsage() + service.updateRelayFamilies() + service.updateRelayAutonomousSystems() + service.deleteRecentFileReference() + + verify { coordination.collectAndProcessDescriptors("/recent/consensuses", DescriptorType.RECENT_RELAY_CONSENSUS) } + verify { coordination.collectAndProcessDescriptors("/recent/servers", DescriptorType.RECENT_RELAY_SERVER) } + verify { coordination.collectAndProcessDescriptors("/archive/consensuses", DescriptorType.ARCHIVE_RELAY_CONSENSUS) } + verify { coordination.collectAndProcessDescriptors("/archive/servers", DescriptorType.ARCHIVE_RELAY_SERVER) } + verify { countryUsage.refresh() } + verify { relayDetails.computeAllMissingFamiliesAndEvictCache() } + verify { relayDetails.lookupAllMissingAutonomousSystems() } + verify { files.deleteRecentFileReferences() } + } }) From a479b55d963f147cdb9a372fc8eb703b2694c97e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:37 +0000 Subject: [PATCH 08/10] test(frontend): cover the date picker, date slider, timeline, heat layer and country usage layer Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- .../src/components/date/DateSlider.test.tsx | 26 +++++- .../date/ResponsiveDatePicker.test.tsx | 85 +++++++++++++++++++ .../leaflet/CountryUsageLayer.test.tsx | 18 ++++ frontend/src/util/heat-layer.test.ts | 54 +++++++++++- 4 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 frontend/src/components/date/ResponsiveDatePicker.test.tsx diff --git a/frontend/src/components/date/DateSlider.test.tsx b/frontend/src/components/date/DateSlider.test.tsx index 0e103e90..d81135aa 100644 --- a/frontend/src/components/date/DateSlider.test.tsx +++ b/frontend/src/components/date/DateSlider.test.tsx @@ -1,6 +1,6 @@ import {FunctionComponent, useEffect, useRef} from "react"; import {describe, expect, it} from "vitest"; -import {page} from "vitest/browser"; +import {page, userEvent} from "vitest/browser"; import {useDate} from "../../context/date-context"; import {renderWithProviders} from "../../test/render"; @@ -9,14 +9,14 @@ import {DateSlider} from "./DateSlider"; const days = ["2024-02-01", "2024-02-02", "2024-02-03"] const Seed: FunctionComponent = () => { - const {setAvailableDays} = useDate() + const {setAvailableDays, selectedDate} = useDate() const seeded = useRef(false) useEffect(() => { if (seeded.current) return seeded.current = true setAvailableDays(days) }, [setAvailableDays]) - return null + return {selectedDate} } describe("DateSlider", () => { @@ -28,4 +28,24 @@ describe("DateSlider", () => { // The last available day is selected by default await expect.element(slider).toHaveAttribute("aria-valuetext", "2024-02-03") }) + + it("selects another day with the keyboard", async () => { + await renderWithProviders(<>) + const slider = page.getByRole("slider", {name: "Selected day"}) + await expect.element(slider).toHaveAttribute("aria-valuetext", "2024-02-03") + + // The range input is visually hidden, so it is focused instead of clicked + const input = slider.element() as HTMLInputElement + input.focus() + await userEvent.keyboard("{ArrowLeft}") + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-02") + await userEvent.keyboard("{Home}") + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-01") + await expect.element(slider).toHaveAttribute("aria-valuetext", "2024-02-01") + }) + + it("is disabled until days are available", async () => { + await renderWithProviders() + await expect.element(page.getByRole("slider", {name: "Selected day"})).toBeDisabled() + }) }) diff --git a/frontend/src/components/date/ResponsiveDatePicker.test.tsx b/frontend/src/components/date/ResponsiveDatePicker.test.tsx new file mode 100644 index 00000000..3dcac147 --- /dev/null +++ b/frontend/src/components/date/ResponsiveDatePicker.test.tsx @@ -0,0 +1,85 @@ +import {FunctionComponent, useEffect, useRef} from "react"; +import {describe, expect, it} from "vitest"; +import {page, userEvent} from "vitest/browser"; + +import {useDate} from "../../context/date-context"; +import {renderWithProviders} from "../../test/render"; +import {ResponsiveDatePicker} from "./ResponsiveDatePicker"; + +// 2024-02-03 is missing, e.g. because no consensus was archived that day +const days = ["2024-02-01", "2024-02-02", "2024-02-04", "2024-02-05"] + +const Seed: FunctionComponent = () => { + const {setAvailableDays, selectedDate, setSelectedDate} = useDate() + const seeded = useRef(false) + useEffect(() => { + if (seeded.current) return + seeded.current = true + setAvailableDays(days) + }, [setAvailableDays]) + return ( + <> + {selectedDate} + + + ) +} + +const renderPicker = (largeScreen = true) => renderWithProviders(<>) + +const field = () => document.querySelector("input.MuiPickersInputBase-input")! + +describe("ResponsiveDatePicker", () => { + it("shows the selected day", async () => { + await renderPicker() + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-05") + await expect.poll(() => field().value).toBe("2024-02-05") + }) + + it("selects an available day from the calendar", async () => { + await renderPicker() + await page.getByRole("button", {name: /Choose date/}).click() + await page.getByRole("gridcell", {name: "2", exact: true}).click() + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-02") + await expect.poll(() => field().value).toBe("2024-02-02") + }) + + it("disables days without data", async () => { + await renderPicker(false) + await page.getByRole("button", {name: /Choose date/}).click() + await expect.element(page.getByRole("gridcell", {name: "3", exact: true})).toBeDisabled() + await expect.element(page.getByRole("gridcell", {name: "4", exact: true})).toBeEnabled() + // Days outside the available range are disabled as well + await expect.element(page.getByRole("gridcell", {name: "6", exact: true})).toBeDisabled() + }) + + it("follows a day selected elsewhere, e.g. on the slider", async () => { + await renderPicker() + await page.getByRole("button", {name: "select first day"}).click() + await expect.poll(() => field().value).toBe("2024-02-01") + }) + + it("keeps a typed day that is not available out of the selection and resets the field on blur", async () => { + await renderPicker() + await expect.poll(() => field().value).toBe("2024-02-05") + await page.getByRole("spinbutton", {name: "Day"}).click() + await userEvent.keyboard("03") + await expect.poll(() => field().value).toBe("2024-02-03") + // 2024-02-03 has no data, so the loaded day stays selected + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-05") + + await page.getByRole("button", {name: "select first day"}).click() + await expect.poll(() => field().value).toBe("2024-02-01") + await page.getByRole("spinbutton", {name: "Day"}).click() + await userEvent.keyboard("03") + await userEvent.click(page.getByTestId("selected-date")) + await expect.poll(() => field().value).toBe("2024-02-01") + }) + + it("selects a typed day that is available", async () => { + await renderPicker() + await page.getByRole("spinbutton", {name: "Day"}).click() + await userEvent.keyboard("04") + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-04") + }) +}) diff --git a/frontend/src/components/leaflet/CountryUsageLayer.test.tsx b/frontend/src/components/leaflet/CountryUsageLayer.test.tsx index c7ddf6ea..95cb8c71 100644 --- a/frontend/src/components/leaflet/CountryUsageLayer.test.tsx +++ b/frontend/src/components/leaflet/CountryUsageLayer.test.tsx @@ -75,4 +75,22 @@ describe("CountryUsageLayer", () => { await expect.poll(() => mockBackend.requested.length).toBe(1) await expect.element(page.getByRole("status", {name: "usage"})).toHaveTextContent("none") }) + + it("colors the countries with an estimate on the map", async () => { + mockBackend.onGet("/relay/country-usage/day/2024-03-05", usage) + await renderLayer(true) + await expect.element(page.getByRole("status", {name: "usage"})).toHaveTextContent("2024-03-05:2") + // Only the two countries with an estimate are drawn, in their own pane below the markers + await expect.poll(() => document.querySelectorAll(".leaflet-countryUsage-pane path").length).toBe(2) + const pane = document.querySelector(".leaflet-countryUsage-pane")! + expect(pane.style.zIndex).toBe("350") + }) + + it("removes the colors when the estimates are switched off", async () => { + mockBackend.onGet("/relay/country-usage/day/2024-03-05", usage) + const screen = await renderLayer(true) + await expect.poll(() => document.querySelectorAll(".leaflet-countryUsage-pane path").length).toBe(2) + await screen.unmount() + expect(document.querySelectorAll(".leaflet-countryUsage-pane path").length).toBe(0) + }) }) diff --git a/frontend/src/util/heat-layer.test.ts b/frontend/src/util/heat-layer.test.ts index 8da09bf5..319fadfa 100644 --- a/frontend/src/util/heat-layer.test.ts +++ b/frontend/src/util/heat-layer.test.ts @@ -14,12 +14,12 @@ describe("HeatLayer", () => { container = undefined }) - const createMap = () => { + const createMap = (zoomAnimation = false) => { container = document.createElement("div") container.style.width = "400px" container.style.height = "300px" document.body.appendChild(container) - map = L.map(container, {zoomAnimation: false}).setView([0, 0], 2) + map = L.map(container, {zoomAnimation}).setView([0, 0], 2) return map } @@ -61,4 +61,54 @@ describe("HeatLayer", () => { expect(heatLayer([])).toBeInstanceOf(L.Layer) expect(heatLayer([])).toBeInstanceOf(HeatLayer) }) + + it("uses the default radius and gradient without options", () => { + const leafletMap = createMap() + heatLayer([[0, 0, 1]]).addTo(leafletMap) + const center = leafletMap.latLngToContainerPoint([0, 0]) + expect(pixelAt(canvasOf(), Math.round(center.x), Math.round(center.y))[3]).toBeGreaterThan(0) + expect(canvasOf().classList).toContain("leaflet-zoom-hide") + }) + + it("skips points outside of the visible map", () => { + const leafletMap = createMap() + leafletMap.setView([0, 0], 8) + heatLayer([[60, 120, 1]], {radius: 10, blur: 5}).addTo(leafletMap) + const canvas = canvasOf() + const data = canvas.getContext("2d")!.getImageData(0, 0, canvas.width, canvas.height).data + expect(data.some((value, index) => index % 4 === 3 && value > 0)).toBe(false) + }) + + it("follows the size of the map", () => { + const leafletMap = createMap() + heatLayer([[0, 0, 1]], {maxZoom: 10}).addTo(leafletMap) + container!.style.width = "200px" + container!.style.height = "100px" + leafletMap.invalidateSize() + leafletMap.fire("moveend") + expect(canvasOf().width).toBe(200) + expect(canvasOf().height).toBe(100) + }) + + it("can be added again after it was removed", () => { + const leafletMap = createMap() + const layer = heatLayer([[0, 0, 1]]).addTo(leafletMap) + const canvas = canvasOf() + leafletMap.removeLayer(layer) + layer.addTo(leafletMap) + expect(canvasOf()).toBe(canvas) + }) + + it("scales the canvas while the map animates a zoom", () => { + const leafletMap = createMap(true) + const layer = heatLayer([[0, 0, 1]]).addTo(leafletMap) + expect(canvasOf().classList).toContain("leaflet-zoom-animated") + + leafletMap.fire("zoomanim", {center: leafletMap.getCenter(), zoom: 3, noUpdate: false}) + expect(canvasOf().style.transform).toContain("scale(2)") + + leafletMap.removeLayer(layer) + expect(leafletMap.listens("zoomanim", (layer as unknown as {animateZoom: L.LeafletEventHandlerFn}).animateZoom, layer)) + .toBe(false) + }) }) From e1a408e499ee1af578757ba736ed43cbf2eb0954 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:59 +0000 Subject: [PATCH 09/10] chore: remove committed .DS_Store files Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- backend/.DS_Store | Bin 6148 -> 0 bytes backend/src/.DS_Store | Bin 6148 -> 0 bytes backend/src/main/.DS_Store | Bin 6148 -> 0 bytes backend/src/test/.DS_Store | Bin 6148 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/.DS_Store delete mode 100644 backend/src/.DS_Store delete mode 100644 backend/src/main/.DS_Store delete mode 100644 backend/src/test/.DS_Store diff --git a/backend/.DS_Store b/backend/.DS_Store deleted file mode 100644 index 87ceaaf86bce98dca557fa51d5e90556da196832..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHLK}#D!6n+z9np(shtkm0HdbHp_*h;*WqU2T(Y_c_xBy2F`C|xLmcq<-Dp?I~F z(jIaWMbP{KFIq}(JzK#;ZhmiONM78Hcq$U+g_&=5-kW*few&@igoxBaE0`zB5m6R} zF+G7Q!Pw6wW+~hA45;)R0oCYTtvp{Sg|TSY;S_KRJTwLP-F-was7xF5mc;v8Z)mDd z-!I1_>R~=#tc68*vLD7PuYUdRyiAG;?h)0f@M>YR&FG}#RHPQIV=W2HT4KD@!Oufv z5njpT;`!94t8>j`AfnG9IYQ>2-nm2q$_!P5_W#}XOAg;U*RyXnPgskAIqG4b#jqI3 zgXjoT&|_%OpgHIe;!}kt4XH<}`?;DNB{{WUr#`OjX-*0ANoIFPN`9&C7v`7N%*xc1 ztmv$HAbpa{?(yHP$v^rT^~OpDNq(7^&W=T2ob_zHJoB}-sczy6t7=4S8LA`XK(nkPb8r|B^&T#|i zS@bPN2H}AzGZbisDtp9GW;psi8|Pb$44UDj?BPS%%*vinl+KR%JsnQUH|T1ofKy;l zfiZKLjrPd d{J(-W_&ja^eT$Jn^uXK?0WE{8oC5z}H6F$CuL)(U6F83_%v9N5tIgTC-qrvg|QC9!(CLdfv0i zZ#BuYXH?Sxozi*w{DZyg*VI=xWm(Od3buUf>gMJC?s2>8pML9~pDei^E2u>}BTFh_wdy@cch!@{stga^W!3e;4#7K1e%_F!>^VXLU= z#MXSURsL*VI9A90A%+tdiat97&On!ebzKhS{@=kT)7#{CQ+(wNI0Ju-0nW-vImS!b z-MaF6a@PioeGCzaYej)T@B9Q{A?L_ubgDmyj<~|GRg_)C9@ByTBM=Gk$r<S5T1=Cq*SCwy#=4Z;tRwQ&lY+RJlQ||YfMq=K@jp5K8Xh}-t+~00pCGI z5Cp&3UF)uCdlC_uf!S{}Kf9AJA(JH{)g1R5M0FyHpo~Ek!xUjZYe@oYSp+IxW4PC7 z_wsC1iWI|NWq_Yug$}7h=hX3@Uv2x^^<{aQc87Tvw&>yQWo75`Zo~KY=lVzc&bzVn zh*S#ch;llkAsx`Mw>Lene)qfkUkqM9_xIMw@~;|Z#dviDYg|W_PG~@iwiE0LA6b6I=VPL`l ze;;BfV`8x|s6QPT^br79fmsXYe3syx*kWR_FbEGsnN*-jRrZLXOgikbjY}*R22DCC zy))LaJ1cuaQF?dSV;xQ^F({)jAPo2nESSqW_y5h==YKy*u7m+$;9oJIDrq}y;+AZ0 xE!`aV+8DYEW#PEO;3NeT-HH**t#}!#1@@RHz{Fx<5Eh912xuB)5C(pgfe*BwbO- zU@jY2bFp{cri<>V?K#DddQKhVmGNt4KSws_C*OyQGvEw31AD^&YPQJWNYPtoz!`7` zY6j&05TJsYVWpUU9ca=K0N95+3GB320tRsaGs8*|9tdkHP*d5C7_8~=2aC%LD@9Ew zPItyQc4xK|3foSNLkcI(6uor@oPj0-2YNY@`hWiY`M;UuSI&SluvZLlKcD6kJd)Mc w*277y4bUs7h{RQjbqb7BiV-WN_!yc5{vZZ0GprQhf%rxs(cq0U@TUxX0{GNZ`v3p{ From 3fd0c474192cc1eb8c69c3d3afa069e5136849af Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 09:54:59 +0000 Subject: [PATCH 10/10] chore(release): 4.0.0 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011RFTfTnzwCcx8wGbtQvdbF --- backend/build.gradle.kts | 2 +- frontend/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/build.gradle.kts b/backend/build.gradle.kts index 37d20727..1e213f99 100644 --- a/backend/build.gradle.kts +++ b/backend/build.gradle.kts @@ -2,7 +2,7 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinCompile group = "org.tormap" -version = "3.0.1" +version = "4.0.0" java.sourceCompatibility = JavaVersion.VERSION_25 plugins { diff --git a/frontend/package.json b/frontend/package.json index 27b6c487..597dd68a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tormap", - "version": "3.0.1", + "version": "4.0.0", "private": true, "type": "module", "scripts": {