diff --git a/AGENTS.md b/AGENTS.md index b08e9089..1657ba80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ Rules for AI coding agents (and a quick reference for humans) working in this re - `.github/workflows/`: CI. Actions are pinned by commit SHA with a version comment. ## Build and test -- Backend: `cd backend && ./gradlew test` (needs a JDK matching `sourceCompatibility` in `backend/build.gradle.kts` and a running Docker daemon for Testcontainers). +- Backend: `cd backend && ./gradlew test` (needs a JDK matching `sourceCompatibility` in `backend/build.gradle.kts` and a running Docker daemon for Testcontainers). Add `jacocoTestReport` for coverage (`build/reports/jacoco/test/html/index.html`, vendored `org.torproject` code is excluded). - Local backend image: `./gradlew jibDockerBuild -Djib.to.image=tormap/backend:local -Djib.to.tags=local`. - Frontend: `cd frontend && yarn install --immutable && yarn build && yarn lint`. - Frontend tests: `cd frontend && yarn test:run` (Vitest). `*.unit.test.ts` run in plain Node, all other `*.test.ts(x)` run in headless Chromium (browser mode), so run `yarn playwright install chromium` once. Use `yarn test` for watch mode and `yarn test:coverage` for coverage. The sandbox blocks Chromium, so browser tests must run outside it. diff --git a/backend/.DS_Store b/backend/.DS_Store deleted file mode 100644 index 87ceaaf8..00000000 Binary files a/backend/.DS_Store and /dev/null differ diff --git a/backend/README.md b/backend/README.md index 23826678..e02cc24a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -50,7 +50,7 @@ less reliable. `SPRING_PROFILES_ACTIVE` is set. 4. Backend should be available at http://localhost:8080 -Run `./gradlew test`, to only build & tests. +Run `./gradlew test`, to only build & tests. Run `./gradlew test jacocoTestReport` for a coverage report in `build/reports/jacoco/test/html/index.html`. In a freshly cloned repository the backend will start to download an archive `>33 GB` in size from [TorProject](https://metrics.torproject.org/collector/archive/). The total time for the whole archive will depend diff --git a/backend/build.gradle.kts b/backend/build.gradle.kts index a614bae9..1e213f99 100644 --- a/backend/build.gradle.kts +++ b/backend/build.gradle.kts @@ -2,7 +2,7 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinCompile group = "org.tormap" -version = "3.0.1" +version = "4.0.0" java.sourceCompatibility = JavaVersion.VERSION_25 plugins { @@ -17,6 +17,9 @@ plugins { // Build and push docker images id("com.google.cloud.tools.jib") version "3.5.4" + + // Test coverage, run with ./gradlew test jacocoTestReport + jacoco } repositories { @@ -152,6 +155,22 @@ tasks.withType { useJUnitPlatform() } +jacoco { + // Java 25 class files need JaCoCo 0.8.14 or later + toolVersion = "0.8.15" +} + +// Writes build/reports/jacoco/test/html/index.html and an XML report for tools +tasks.jacocoTestReport { + dependsOn(tasks.test) + reports { + xml.required = true + html.required = true + } + // The vendored metrics-lib (org.torproject) is mostly unused by TorMap, so it would dilute the numbers + classDirectories.setFrom(classDirectories.files.map { fileTree(it) { exclude("org/torproject/**") } }) +} + // The local development server uses the dev profile unless SPRING_PROFILES_ACTIVE is set tasks.bootRun { if (providers.environmentVariable("SPRING_PROFILES_ACTIVE").orNull == null) { diff --git a/backend/src/.DS_Store b/backend/src/.DS_Store deleted file mode 100644 index a37f8a45..00000000 Binary files a/backend/src/.DS_Store and /dev/null differ diff --git a/backend/src/main/.DS_Store b/backend/src/main/.DS_Store deleted file mode 100644 index 42e2b0f1..00000000 Binary files a/backend/src/main/.DS_Store and /dev/null differ diff --git a/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt b/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt index 25f3464f..472c2ce8 100644 --- a/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt +++ b/backend/src/main/kotlin/org/tormap/config/RequestBodyLimitConfig.kt @@ -14,6 +14,7 @@ import org.springframework.web.filter.OncePerRequestFilter import org.springframework.web.server.ContentTooLargeException import java.io.BufferedReader import java.io.InputStreamReader +import java.io.UnsupportedEncodingException import java.nio.charset.Charset /** @@ -57,7 +58,12 @@ class RequestBodyLimitFilter(private val maxBodyBytes: Long) : OncePerRequestFil override fun getInputStream(): ServletInputStream = inputStream override fun getReader(): BufferedReader { - val charset = characterEncoding?.let { Charset.forName(it) } ?: Charsets.ISO_8859_1 + val charset = try { + characterEncoding?.let { Charset.forName(it) } ?: Charsets.ISO_8859_1 + } catch (_: IllegalArgumentException) { + // The encoding comes from the request, the servlet API reports an unknown one like this + throw UnsupportedEncodingException(characterEncoding) + } return BufferedReader(InputStreamReader(inputStream, charset)) } } diff --git a/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt b/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt index 3174b829..81c9db80 100644 --- a/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt +++ b/backend/src/main/kotlin/org/tormap/config/SecurityConfig.kt @@ -52,12 +52,23 @@ class SecurityConfig( fun passwordEncoder(): PasswordEncoder = BCryptPasswordEncoder() @Bean - fun userDetailsService(passwordEncoder: PasswordEncoder): UserDetailsService { - if (!System.getenv(ENV_REMOVED_ADMIN_PASSWORD_BCRYPT).isNullOrBlank()) { + fun userDetailsService(passwordEncoder: PasswordEncoder): UserDetailsService = + buildUserDetailsService(passwordEncoder, System.getenv(ENV_ADMIN_PASSWORD), System.getenv(ENV_REMOVED_ADMIN_PASSWORD_BCRYPT)) + + /** + * Creates the admin user from the values of [ENV_ADMIN_PASSWORD] and [ENV_REMOVED_ADMIN_PASSWORD_BCRYPT]. + * Without a password there is no user, so the actuator denies every request. + */ + internal fun buildUserDetailsService( + passwordEncoder: PasswordEncoder, + adminPassword: String?, + removedAdminPasswordBcrypt: String?, + ): UserDetailsService { + if (!removedAdminPasswordBcrypt.isNullOrBlank()) { logger.warn("$ENV_REMOVED_ADMIN_PASSWORD_BCRYPT is no longer supported and is ignored. Use $ENV_ADMIN_PASSWORD instead.") } - val password = System.getenv(ENV_ADMIN_PASSWORD)?.trim().orEmpty() + val password = adminPassword?.trim().orEmpty() if (password.isBlank()) { logger.warn("------------------------------------------------------") diff --git a/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt b/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt index 9c89c9a0..09aad537 100644 --- a/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt +++ b/backend/src/main/kotlin/org/tormap/database/entity/RelayLocation.kt @@ -37,11 +37,10 @@ class RelayLocation( @Column(length = 40, columnDefinition = "bpchar(40)") var fingerprint: String = networkStatusEntry.fingerprint - var flags: String? = try { - networkStatusEntry.flags.map { TorRelayFlag.valueOf(it).ordinal }.joinToString(", ") - } catch (exception: Exception) { - null - } + // Flags this version does not know (e.g. MiddleOnly) are skipped, so the known flags of the relay are kept + var flags: String? = networkStatusEntry.flags + .mapNotNull { flag -> TorRelayFlag.entries.find { it.name == flag }?.ordinal } + .joinToString(", ") /** * The `w Bandwidth=` value of the consensus entry, which clients use for path selection. diff --git a/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt b/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt index 8b348726..74daca78 100644 --- a/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt +++ b/backend/src/main/kotlin/org/tormap/service/DescriptorCoordinationService.kt @@ -124,10 +124,14 @@ class DescriptorCoordinationService( private fun computeRelayDetailsAndCaches(descriptorType: DescriptorType, processedMonths: Set) { if (descriptorType.isRelayServerType()) { - relayDetailsUpdateService.computeFamilies(processedMonths) relayDetailsUpdateService.lookupMissingAutonomousSystems(processedMonths) + // The cached relay locations contain the family IDs, so they are only updated once the families are computed + relayDetailsUpdateService.computeFamilies(processedMonths).whenComplete { _, _ -> + updateCaches(descriptorType, processedMonths) + } + } else { + updateCaches(descriptorType, processedMonths) } - updateCaches(descriptorType, processedMonths) } private fun updateCaches(descriptorType: DescriptorType, processedMonths: Set) { diff --git a/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt b/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt index 1963fd41..f7fd920f 100644 --- a/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt +++ b/backend/src/main/kotlin/org/tormap/service/RelayDetailsUpdateService.kt @@ -10,6 +10,7 @@ import org.tormap.util.addFamilyMember import org.tormap.util.commaSeparatedToList import org.tormap.util.getFamilyMember import org.tormap.util.logger +import java.util.concurrent.CompletableFuture import javax.sql.DataSource /** @@ -76,23 +77,29 @@ class RelayDetailsUpdateService( val monthFamilyMemberCount = relayDetailsRepositoryImpl.findDistinctMonthFamilyMemberCount().filter { it.count == 0L } val monthsToProcess = monthFamilyMemberCount.map { it.month }.toSet() - computeFamilies(monthsToProcess) - cacheService.evictRelayLocationsPerDay(monthsToProcess) + // The families are computed asynchronously, evicting earlier would let requests cache the old families again + computeFamilies(monthsToProcess).whenComplete { _, _ -> + cacheService.evictRelayLocationsPerDay(monthsToProcess) + } } /** - * Updates [RelayDetails.familyId] for all entities of the requested [months]. + * Updates [RelayDetails.familyId] for all entities of the requested [months] asynchronously. + * @return a future that completes once the families of all [months] are computed (also if some failed) */ - fun computeFamilies(months: Set) { + fun computeFamilies(months: Set): CompletableFuture { logger.info("... Updating relay families for months: {}", months.joinToString(", ")) - months.forEach { month -> + val futures = months.map { month -> coalesceService.submitAsync("computeFamilies-$month") { transactionTemplate.executeWithoutResult { computeFamiliesForMonth(month) } } + // Failures are logged by the CoalesceService, a failed month must not hide the others + .exceptionally { null } } + return CompletableFuture.allOf(*futures.toTypedArray()) } /** diff --git a/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt b/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt index aa5caf5c..6f624c10 100644 --- a/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt +++ b/backend/src/main/kotlin/org/tormap/service/ReverseDnsLookupService.kt @@ -85,24 +85,28 @@ class DNSJavaReverseDnsResolver : ReverseDnsResolver { private fun lookup(name: Name, type: Int): List = Lookup(name, type).apply { setResolver(dnsResolver) }.run()?.toList() ?: emptyList() - - private fun InetAddress.isULA(): Boolean = - this is Inet6Address && - (address[0].toInt() and 0xFE) == 0xFC // fc00::/7 - - private fun InetAddress.isPubliclyRoutable(): Boolean = - !isLoopbackAddress && - !isLinkLocalAddress && - !isSiteLocalAddress && // reliable for IPv4 - !isULA() && // fills the IPv6 gap isSiteLocalAddress misses - !isMulticastAddress && - !isAnyLocalAddress && - address.first() != 0.toByte() && - !(address.size == 4 && - address[0] == 100.toByte() && - (address[1].toInt() and 0xFF) in 64..127) } +private fun InetAddress.isULA(): Boolean = + this is Inet6Address && + (address[0].toInt() and 0xFE) == 0xFC // fc00::/7 + +/** + * Whether the address can be reached on the public internet. Forward lookups only verify a PTR name with such + * addresses, so that a name pointing to an internal address is never reported as verified. + */ +internal fun InetAddress.isPubliclyRoutable(): Boolean = + !isLoopbackAddress && + !isLinkLocalAddress && + !isSiteLocalAddress && // reliable for IPv4 + !isULA() && // fills the IPv6 gap isSiteLocalAddress misses + !isMulticastAddress && + !isAnyLocalAddress && + address.first() != 0.toByte() && + !(address.size == 4 && + address[0] == 100.toByte() && + (address[1].toInt() and 0xFF) in 64..127) + @Service class ReverseDnsLookupService( private val reverseDnsResolver: ReverseDnsResolver, diff --git a/backend/src/main/kotlin/org/tormap/util/PathSelection.kt b/backend/src/main/kotlin/org/tormap/util/PathSelection.kt index 83a2256d..a99032ea 100644 --- a/backend/src/main/kotlin/org/tormap/util/PathSelection.kt +++ b/backend/src/main/kotlin/org/tormap/util/PathSelection.kt @@ -13,9 +13,11 @@ object PathSelection { /** * Returns the share of a relay's consensus weight that clients use for choosing it as entry guard, so the * relay's probability of being picked as guard is `guardWeight / sum of the guard weights of all relays`. + * It follows the guard position weights of Tor clients (`compute_weighted_bandwidths` in Tor's node_select.c): * * - A relay that is not `Running` and `Valid` can not be picked, which results in 0. - * - Guard flagged relays use `Wgg`, relays that are also exits use `Wgd`, all other relays `Wgm`. + * - Clients only pick relays with the `Guard` flag as entry guards, all other relays result in 0. + * - Guard flagged relays that are also exits use `Wgd`, all other guards `Wgg`. A `BadExit` does not count as exit. * * @param consensusWeight the `w Bandwidth=` value of the relay's consensus entry * @param flags the names of the flags the relay has in the consensus @@ -23,13 +25,9 @@ object PathSelection { * @return null, if the consensus does not contain the required weight */ fun guardWeight(consensusWeight: Long, flags: Set, bandwidthWeights: Map?): Long? { - if (!flags.contains("Running") || !flags.contains("Valid")) return 0 - val weightKey = when { - flags.contains("Guard") && flags.contains("Exit") -> "Wgd" - flags.contains("Guard") -> "Wgg" - else -> "Wgm" - } - val weight = bandwidthWeights?.get(weightKey) ?: return null + if (!flags.contains("Running") || !flags.contains("Valid") || !flags.contains("Guard")) return 0 + val isExit = flags.contains("Exit") && !flags.contains("BadExit") + val weight = bandwidthWeights?.get(if (isExit) "Wgd" else "Wgg") ?: return null return Math.round(consensusWeight.toDouble() * weight / BANDWIDTH_WEIGHT_SCALE) } } diff --git a/backend/src/test/.DS_Store b/backend/src/test/.DS_Store deleted file mode 100644 index 01fac528..00000000 Binary files a/backend/src/test/.DS_Store and /dev/null differ diff --git a/backend/src/test/kotlin/org/tormap/config/AdminUserTest.kt b/backend/src/test/kotlin/org/tormap/config/AdminUserTest.kt new file mode 100644 index 00000000..ec9d76ae --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/config/AdminUserTest.kt @@ -0,0 +1,51 @@ +package org.tormap.config + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import org.springframework.security.core.userdetails.UsernameNotFoundException +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder +import org.springframework.web.cors.CorsConfiguration +import org.springframework.mock.web.MockHttpServletRequest + +class AdminUserTest : StringSpec({ + val passwordEncoder = BCryptPasswordEncoder(4) + fun securityConfig(allowedOrigins: String = "https://tormap.org") = + SecurityConfig("admin", "/actuator", allowedOrigins, false) + + "an admin password creates the admin user with a hashed password" { + val users = securityConfig().buildUserDetailsService(passwordEncoder, " a-long-generated-password ", null) + val admin = users.loadUserByUsername("admin") + admin.authorities.map { it.authority } shouldBe listOf("ROLE_ADMIN") + // Surrounding whitespace, e.g. from an env file, is not part of the password + passwordEncoder.matches("a-long-generated-password", admin.password) shouldBe true + admin.password!!.startsWith("{") shouldBe false + } + + "a short password still creates the admin user" { + val users = securityConfig().buildUserDetailsService(passwordEncoder, "short", null) + passwordEncoder.matches("short", users.loadUserByUsername("admin").password) shouldBe true + } + + "without a password there is no user" { + listOf(null, "", " ").forEach { password -> + val users = securityConfig().buildUserDetailsService(passwordEncoder, password, null) + shouldThrow { users.loadUserByUsername("admin") } + } + } + + "the removed bcrypt variable does not create a user" { + val hash = passwordEncoder.encode("a-long-generated-password") + val users = securityConfig().buildUserDetailsService(passwordEncoder, null, hash) + shouldThrow { users.loadUserByUsername("admin") } + } + + "allowed CORS origins are read from a comma separated list" { + val source = securityConfig(" https://a.example , ,https://*.b.example").corsConfigurationSource() + val cors: CorsConfiguration = source.getCorsConfiguration(MockHttpServletRequest("GET", "/relay/location/days"))!! + cors.allowedOriginPatterns shouldBe listOf("https://a.example", "https://*.b.example") + cors.checkOrigin("https://x.b.example") shouldBe "https://x.b.example" + cors.checkOrigin("https://evil.example") shouldBe null + cors.allowCredentials shouldBe false + } +}) diff --git a/backend/src/test/kotlin/org/tormap/config/RequestBodyLimitFilterTest.kt b/backend/src/test/kotlin/org/tormap/config/RequestBodyLimitFilterTest.kt new file mode 100644 index 00000000..ccfe39c5 --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/config/RequestBodyLimitFilterTest.kt @@ -0,0 +1,83 @@ +package org.tormap.config + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import jakarta.servlet.FilterChain +import jakarta.servlet.ReadListener +import jakarta.servlet.ServletRequest +import jakarta.servlet.http.HttpServletRequest +import org.springframework.mock.web.MockHttpServletRequest +import org.springframework.mock.web.MockHttpServletResponse +import org.springframework.web.server.ContentTooLargeException +import java.io.UnsupportedEncodingException + +class RequestBodyLimitFilterTest : StringSpec({ + val filter = RequestBodyLimitFilter(maxBodyBytes = 10) + + fun request(body: String, encoding: String? = null) = MockHttpServletRequest("POST", "/relay/details/relay/identifiers").apply { + setContent(body.toByteArray()) + if (encoding != null) characterEncoding = encoding + } + + fun filterWith(request: HttpServletRequest, read: (ServletRequest) -> Unit): MockHttpServletResponse { + val response = MockHttpServletResponse() + filter.doFilter(request, response, FilterChain { wrapped, _ -> read(wrapped) }) + return response + } + + "a declared Content-Length above the limit is rejected before the body is read" { + var called = false + val response = filterWith(request("x".repeat(11))) { called = true } + response.status shouldBe 413 + called shouldBe false + } + + "a body within the limit can be read as stream and as text" { + filterWith(request("0123456789")) { it.inputStream.readAllBytes().decodeToString() shouldBe "0123456789" } + filterWith(request("äöü", "UTF-8")) { it.reader.readText() shouldBe "äöü" } + // Without an encoding the servlet default ISO-8859-1 applies + filterWith(request("abc")) { it.reader.readText() shouldBe "abc" } + } + + "reading beyond the limit fails, byte by byte as well as in blocks" { + // Without a known length, as with chunked transfer encoding + fun unknownLength(body: String) = object : MockHttpServletRequest("POST", "/") { + override fun getContentLengthLong() = -1L + }.apply { setContent(body.toByteArray()) } + + filterWith(unknownLength("x".repeat(20))) { request -> + shouldThrow { request.inputStream.readAllBytes() } + } + filterWith(unknownLength("x".repeat(11))) { request -> + val stream = request.inputStream + repeat(10) { stream.read() shouldBe 'x'.code } + shouldThrow { stream.read() } + } + filterWith(unknownLength("")) { it.inputStream.read() shouldBe -1 } + } + + "an unknown character encoding is reported like by the servlet API" { + filterWith(request("abc", "no-such-charset")) { + shouldThrow { it.reader } + } + } + + "the stream state and read listener are those of the request" { + filterWith(request("abc")) { + val stream = it.inputStream + stream.isFinished shouldBe false + stream.isReady shouldBe true + stream.readAllBytes() + stream.isFinished shouldBe true + // MockHttpServletRequest does not support async IO, which shows the call is delegated + shouldThrow { + stream.setReadListener(object : ReadListener { + override fun onDataAvailable() = Unit + override fun onAllDataRead() = Unit + override fun onError(t: Throwable) = Unit + }) + } + } + } +}) diff --git a/backend/src/test/kotlin/org/tormap/controller/RelayStatisticsControllerTest.kt b/backend/src/test/kotlin/org/tormap/controller/RelayStatisticsControllerTest.kt new file mode 100644 index 00000000..691e73be --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/controller/RelayStatisticsControllerTest.kt @@ -0,0 +1,71 @@ +package org.tormap.controller + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldStartWith +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc +import org.springframework.cache.CacheManager +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.web.servlet.MockMvc +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import org.tormap.config.CacheConfig +import org.tormap.database.entity.TorRelayFlag +import org.tormap.database.repository.RelayLocationRepositoryImpl +import org.tormap.mockRelayLocation +import org.tormap.service.CacheService +import java.time.LocalDate + +@SpringBootTest +@AutoConfigureMockMvc +@ActiveProfiles("test") +class RelayStatisticsControllerTest( + private val mockMvc: MockMvc, + private val relayLocationRepository: RelayLocationRepositoryImpl, + private val cacheManager: CacheManager, + private val cacheService: CacheService, +) : StringSpec({ + val day = LocalDate.of(2024, 3, 5) + + beforeEach { + relayLocationRepository.deleteAll() + cacheManager.getCache(CacheConfig.RELAY_DAILY_COUNTS)?.clear() + } + + "returns the relay counts per day" { + relayLocationRepository.saveAll( + listOf( + mockRelayLocation(day, 'A', setOf(TorRelayFlag.Exit)), + mockRelayLocation(day, 'B', setOf(TorRelayFlag.Guard)), + mockRelayLocation(day, 'C'), + ) + ) + mockMvc.perform(get("/relay/statistics/daily-counts")) + .andExpect(status().isOk) + .andExpect(jsonPath("$.length()").value(1)) + .andExpect(jsonPath("$[0].day").value("2024-03-05")) + .andExpect(jsonPath("$[0].total").value(3)) + .andExpect(jsonPath("$[0].exit").value(1)) + .andExpect(jsonPath("$[0].guard").value(1)) + .andExpect(jsonPath("$[0].other").value(1)) + } + + "serves the cached counts until the cache is refreshed" { + mockMvc.perform(get("/relay/statistics/daily-counts")).andExpect(jsonPath("$.length()").value(0)) + relayLocationRepository.save(mockRelayLocation(day, 'A')) + mockMvc.perform(get("/relay/statistics/daily-counts")).andExpect(jsonPath("$.length()").value(0)) + + cacheService.cacheRelayDailyCounts().get() + + mockMvc.perform(get("/relay/statistics/daily-counts")).andExpect(jsonPath("$.length()").value(1)) + } + + "GET / names the application" { + val response = mockMvc.perform(get("/")).andExpect(status().isOk).andReturn().response + response.contentAsString shouldStartWith "TorMap " + // Swagger is disabled by default, so the docs are not linked + response.contentAsString.contains("Docs:") shouldBe false + } +}) diff --git a/backend/src/test/kotlin/org/tormap/database/entity/RelayLocationTest.kt b/backend/src/test/kotlin/org/tormap/database/entity/RelayLocationTest.kt new file mode 100644 index 00000000..395f284a --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/database/entity/RelayLocationTest.kt @@ -0,0 +1,50 @@ +package org.tormap.database.entity + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import org.tormap.adapter.dto.RelayLocationDto +import org.torproject.descriptor.NetworkStatusEntry +import java.math.BigDecimal +import java.time.LocalDate + +class RelayLocationTest : StringSpec({ + fun relayLocation(flags: Set, bandwidth: Long = 1000, bandwidthWeights: Map? = null): RelayLocation { + val entry = mockk() + every { entry.fingerprint } returns "A".repeat(40) + every { entry.flags } returns flags.toSortedSet() + every { entry.bandwidth } returns bandwidth + return RelayLocation(entry, LocalDate.of(2024, 3, 5), BigDecimal.ONE, BigDecimal.ONE, "DE", bandwidthWeights) + } + + "flags are stored as ordinals" { + relayLocation(setOf("Exit", "Guard", "Valid")).flags shouldBe "5, 7, 0" + } + + "flags unknown to TorMap are skipped instead of dropping all flags" { + val location = relayLocation(setOf("MiddleOnly", "Running", "Valid")) + location.flags shouldBe "3, 0" + RelayLocationDto(BigDecimal.ONE, BigDecimal.ONE, "DE", location.flags, null, null, null, null) + .flags shouldBe listOf(3, 0) + } + + "a relay without flags has none" { + val location = relayLocation(emptySet()) + location.flags shouldBe "" + RelayLocationDto(BigDecimal.ONE, BigDecimal.ONE, "DE", location.flags, null, null, null, null) + .flags shouldBe null + } + + "the weights are computed from the consensus entry" { + val location = relayLocation(setOf("Guard", "Running", "Valid"), 3000, mapOf("Wgg" to 5000)) + location.consensusWeight shouldBe 3000 + location.guardWeight shouldBe 1500 + } + + "a consensus entry without bandwidth has no weights" { + val location = relayLocation(setOf("Guard", "Running", "Valid"), -1, mapOf("Wgg" to 5000)) + location.consensusWeight shouldBe null + location.guardWeight shouldBe null + } +}) diff --git a/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt index ce0ff49f..5e92392d 100644 --- a/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/CountryUsageServiceTest.kt @@ -3,6 +3,7 @@ package org.tormap.service import com.sun.net.httpserver.HttpServer import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe import org.springframework.boot.test.context.SpringBootTest import org.springframework.cache.CacheManager import org.springframework.jdbc.core.JdbcTemplate @@ -46,9 +47,9 @@ class CountryUsageServiceTest( } val baseUrl = "http://127.0.0.1:${server.address.port}" - fun service(maxDownloadBytes: Long = 10_000_000) = CountryUsageService( + fun service(maxDownloadBytes: Long = 10_000_000, url: String = baseUrl) = CountryUsageService( UsageStatisticsConfig( - baseUrl = baseUrl, + baseUrl = url, relayUsersPath = "/userstats-relay-country.csv", maxDownloadBytes = maxDownloadBytes, connectTimeout = Duration.ofSeconds(2), @@ -119,4 +120,25 @@ class CountryUsageServiceTest( service().refresh() shouldBe 1 countryUsageRepository.findLatestDay() shouldBe LocalDate.of(2024, 3, 5) } + + "malformed lines are skipped and the valid ones are stored" { + body = "date,country,users,lower,upper,frac\n2024-03-05,de,2,,,50\n2024-03-05,fr,many,,,50\n" + + "2024-03-05,it,3,,,50,extra\n2024-13-01,es,4,,,50\n" + service().refresh() shouldBe 1 + countryUsageRepository.findAllByDay(LocalDate.of(2024, 3, 5)).map { it.country } shouldBe listOf("DE") + } + + "an import without rows keeps the cache" { + val cache = cacheManager.getCache(CacheConfig.COUNTRY_USAGE_PER_DAY)!! + cache.put("2024-03-05", listOf()) + body = "date,country,users,lower,upper,frac\n" + service().refresh() shouldBe 0 + cache.get("2024-03-05") shouldNotBe null + } + + "only http and https URLs are requested" { + service(url = "file:///etc").refresh() shouldBe 0 + service(url = "http://").refresh() shouldBe 0 + requestedUris shouldBe emptyList() + } }) diff --git a/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt index 4a9020ee..f531199b 100644 --- a/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/DescriptorCoordinationServiceTest.kt @@ -10,25 +10,48 @@ import org.tormap.database.repository.RelayDetailsRepository import org.tormap.database.repository.RelayLocationRepository import org.torproject.descriptor.Descriptor import java.io.File +import java.nio.file.Files +import java.util.concurrent.CompletableFuture class DescriptorCoordinationServiceTest : StringSpec({ - "processLocalDescriptorFiles saves the final processed descriptor file" { + lateinit var relayDetailsUpdateService: RelayDetailsUpdateService + lateinit var descriptorFileService: DescriptorFileService + lateinit var descriptorProcessingService: DescriptorProcessingService + lateinit var cacheService: CacheService + lateinit var service: DescriptorCoordinationService + lateinit var relayLocationRepository: RelayLocationRepository + val downloadDirectory = Files.createTempDirectory("tormap-collect").toFile() + afterSpec { downloadDirectory.deleteRecursively() } + + beforeEach { + relayDetailsUpdateService = mockk(relaxed = true) + descriptorFileService = mockk(relaxed = true) + descriptorProcessingService = mockk() + cacheService = mockk(relaxed = true) + relayLocationRepository = mockk(relaxed = true) val descriptorConfig = mockk(relaxed = true) - val relayDetailsUpdateService = mockk(relaxed = true) - val descriptorFileService = mockk(relaxed = true) - val descriptorProcessingService = mockk() - val relayDetailsRepository = mockk(relaxed = true) - val relayLocationRepository = mockk(relaxed = true) - val cacheService = mockk(relaxed = true) - val service = DescriptorCoordinationService( + // Nothing listens on port 1, so collecting fails right away without network access + every { descriptorConfig.apiBaseURL } returns "http://127.0.0.1:1" + every { descriptorConfig.localDownloadDirectory } returns downloadDirectory.absolutePath + service = DescriptorCoordinationService( descriptorConfig, relayDetailsUpdateService, descriptorFileService, descriptorProcessingService, - relayDetailsRepository, + mockk(relaxed = true), relayLocationRepository, cacheService, ) + } + + fun processLocalDescriptorFiles(apiPath: String, descriptorType: DescriptorType) { + DescriptorCoordinationService::class.java + .getDeclaredMethod("processLocalDescriptorFiles", String::class.java, DescriptorType::class.java) + .apply { isAccessible = true } + .invoke(service, apiPath, descriptorType) + } + + "processLocalDescriptorFiles saves the final processed descriptor file" { val firstFile = File("first-descriptors.tar.xz") val finalFile = File("final-descriptors.tar.xz") val descriptors = mutableListOf( @@ -42,10 +65,7 @@ class DescriptorCoordinationServiceTest : StringSpec({ } returns descriptors every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-01") - DescriptorCoordinationService::class.java - .getDeclaredMethod("processLocalDescriptorFiles", String::class.java, DescriptorType::class.java) - .apply { isAccessible = true } - .invoke(service, "/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) verify(exactly = 1) { descriptorFileService.saveProcessedFileReference(firstFile, DescriptorType.ARCHIVE_RELAY_CONSENSUS) @@ -53,6 +73,112 @@ class DescriptorCoordinationServiceTest : StringSpec({ verify(exactly = 1) { descriptorFileService.saveProcessedFileReference(finalFile, DescriptorType.ARCHIVE_RELAY_CONSENSUS) } + verify(exactly = 2) { cacheService.cacheRelayLocationDistinctDays() } + verify(exactly = 2) { cacheService.cacheRelayDailyCounts() } + verify(exactly = 2) { cacheService.evictRelayLocationsPerDay(setOf("2024-01")) } + verify(exactly = 0) { relayDetailsUpdateService.computeFamilies(any()) } + } + + "files with errors are not saved as processed" { + val file = File("broken-descriptors.tar.xz") + every { + descriptorFileService.getDescriptorDiskReader("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file), descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returnsMany listOf( + ProcessedDescriptorInfo("2024-01"), + ProcessedDescriptorInfo(error = "unparsable"), + ) + + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + + verify(exactly = 0) { descriptorFileService.saveProcessedFileReference(any(), any()) } + } + + "server descriptors only update the cached relay locations once the families are computed" { + val file = File("server-descriptors.tar.xz") + val families = CompletableFuture() + every { + descriptorFileService.getDescriptorDiskReader("/archive", DescriptorType.ARCHIVE_RELAY_SERVER) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-02") + every { relayDetailsUpdateService.computeFamilies(setOf("2024-02")) } returns families + + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_SERVER) + + verify { relayDetailsUpdateService.lookupMissingAutonomousSystems(setOf("2024-02")) } + verify(exactly = 0) { cacheService.evictRelayLocationsPerDay(any()) } + + families.complete(null) + + verify(exactly = 1) { cacheService.evictRelayLocationsPerDay(setOf("2024-02")) } + // Server descriptors do not change the days or daily counts + verify(exactly = 0) { cacheService.cacheRelayLocationDistinctDays() } + verify(exactly = 0) { cacheService.cacheRelayDailyCounts() } + } + + "recent server descriptors refresh the cached relay locations after the families, also if they failed" { + val file = File("recent-server-descriptors") + val families = CompletableFuture() + every { + descriptorFileService.getDescriptorDiskReader("/recent", DescriptorType.RECENT_RELAY_SERVER) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-03") + every { relayDetailsUpdateService.computeFamilies(setOf("2024-03")) } returns families + + processLocalDescriptorFiles("/recent", DescriptorType.RECENT_RELAY_SERVER) + + // Once for the finished file + verify(exactly = 1) { cacheService.cacheRelayLocationsPerDay(setOf("2024-03")) } + + families.completeExceptionally(IllegalStateException("failed")) + + verify(exactly = 2) { cacheService.cacheRelayLocationsPerDay(setOf("2024-03")) } + } + + "recent consensuses cache the relay locations without computing families" { + val file = File("recent-consensuses") + every { + descriptorFileService.getDescriptorDiskReader("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-04") + + processLocalDescriptorFiles("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + + verify(exactly = 2) { cacheService.cacheRelayLocationsPerDay(setOf("2024-04")) } + verify(exactly = 2) { cacheService.cacheRelayDailyCounts() } + verify(exactly = 0) { relayDetailsUpdateService.computeFamilies(any()) } + } + + "collectAndProcessDescriptors processes local files also if the remote server is not reachable" { + val file = File("recent-consensuses") + every { + descriptorFileService.getDescriptorDiskReader("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-04") + + service.collectAndProcessDescriptors("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + + verify { relayLocationRepository.flush() } + verify { descriptorFileService.saveProcessedFileReference(file, DescriptorType.RECENT_RELAY_CONSENSUS) } + } + + "collectAndProcessDescriptors does not throw if processing fails" { + every { descriptorFileService.getDescriptorDiskReader(any(), any()) } throws IllegalStateException("disk") + service.collectAndProcessDescriptors("/recent", DescriptorType.RECENT_RELAY_CONSENSUS) + verify(exactly = 0) { cacheService.cacheRelayDailyCounts() } + } + + "a failing flush does not stop the processing" { + val file = File("archive-consensuses") + every { + descriptorFileService.getDescriptorDiskReader("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + } returns mutableListOf(descriptorFrom(file)) + every { descriptorProcessingService.processDescriptor(any()) } returns ProcessedDescriptorInfo("2024-05") + every { relayLocationRepository.flush() } throws IllegalStateException("database gone") + + processLocalDescriptorFiles("/archive", DescriptorType.ARCHIVE_RELAY_CONSENSUS) + + verify { descriptorFileService.saveProcessedFileReference(file, DescriptorType.ARCHIVE_RELAY_CONSENSUS) } } }) diff --git a/backend/src/test/kotlin/org/tormap/service/DescriptorFileServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/DescriptorFileServiceTest.kt new file mode 100644 index 00000000..10b3087a --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/DescriptorFileServiceTest.kt @@ -0,0 +1,105 @@ +package org.tormap.service + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.shouldBe +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.tormap.config.value.DescriptorConfig +import org.tormap.database.entity.DescriptorFileId +import org.tormap.database.entity.DescriptorType +import org.tormap.database.entity.ProcessedFile +import org.tormap.database.repository.ProcessedFileRepository +import java.io.File +import java.nio.file.Files +import java.time.Duration +import java.time.Instant + +@SpringBootTest +@ActiveProfiles("test") +class DescriptorFileServiceTest( + private val descriptorConfig: DescriptorConfig, + private val processedFileRepository: ProcessedFileRepository, +) : StringSpec({ + val downloadDirectory = Files.createTempDirectory("tormap-descriptors").toFile() + val service = DescriptorFileService( + descriptorConfig.copy(localDownloadDirectory = downloadDirectory.absolutePath), + processedFileRepository, + ) + val apiPath = "/recent/relay-descriptors/server-descriptors" + + beforeEach { + processedFileRepository.deleteAll() + downloadDirectory.listFiles()?.forEach { it.deleteRecursively() } + } + + afterSpec { + downloadDirectory.deleteRecursively() + } + + fun writeDescriptorFile(name: String): File { + val directory = File(downloadDirectory, apiPath).apply { mkdirs() } + // The type annotation lets the reader pick the parser, the content itself does not parse + return File(directory, name).apply { writeText("@type server-descriptor 1.0\nrouter broken\n") } + } + + fun descriptorFileNames(type: DescriptorType = DescriptorType.RECENT_RELAY_SERVER) = + service.getDescriptorDiskReader(apiPath, type).map { it.descriptorFile.name } + + "reads the descriptors of files that were not processed yet" { + writeDescriptorFile("2024-03-05-00-00-00-server-descriptors") + descriptorFileNames() shouldBe listOf("2024-03-05-00-00-00-server-descriptors") + } + + "skips files that were processed unless they changed since" { + val file = writeDescriptorFile("2024-03-05-00-00-00-server-descriptors") + service.saveProcessedFileReference(file, DescriptorType.RECENT_RELAY_SERVER) + descriptorFileNames().shouldBeEmpty() + // Another descriptor type has its own processed files + descriptorFileNames(DescriptorType.RECENT_RELAY_CONSENSUS) shouldBe listOf(file.name) + + file.setLastModified(file.lastModified() + 60_000) + descriptorFileNames() shouldBe listOf(file.name) + } + + "saving a processed file again updates its reference" { + val file = writeDescriptorFile("2024-03-05-00-00-00-server-descriptors") + service.saveProcessedFileReference(file, DescriptorType.ARCHIVE_RELAY_SERVER) + val firstProcessedAt = processedFileRepository.findAll().single().processedAt + Thread.sleep(5) + service.saveProcessedFileReference(file, DescriptorType.ARCHIVE_RELAY_SERVER) + val stored = processedFileRepository.findAll().single() + stored.id shouldBe DescriptorFileId(DescriptorType.ARCHIVE_RELAY_SERVER, file.name) + (stored.processedAt > firstProcessedAt) shouldBe true + } + + "deletes only old references of recent files" { + val old = Instant.now().minus(Duration.ofDays(DAYS_TO_KEEP_RECENT_FILES + 1)).toEpochMilli() + val fresh = Instant.now().toEpochMilli() + processedFileRepository.saveAll( + listOf( + ProcessedFile(DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "old-recent"), old), + ProcessedFile(DescriptorFileId(DescriptorType.RECENT_RELAY_CONSENSUS, "old-recent"), old), + ProcessedFile(DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "fresh-recent"), fresh), + ProcessedFile(DescriptorFileId(DescriptorType.ARCHIVE_RELAY_SERVER, "old-archive"), old), + ) + ) + + service.deleteRecentFileReferences() + + processedFileRepository.findAll().map { it.id.type to it.id.filename }.toSet() shouldBe setOf( + DescriptorType.RECENT_RELAY_SERVER to "fresh-recent", + DescriptorType.ARCHIVE_RELAY_SERVER to "old-archive", + ) + } + + "file ids are equal by type and name" { + val id = DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "a") + id shouldBe DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "a") + id.hashCode() shouldBe DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "a").hashCode() + (id == DescriptorFileId(DescriptorType.ARCHIVE_RELAY_SERVER, "a")) shouldBe false + (id == DescriptorFileId(DescriptorType.RECENT_RELAY_SERVER, "b")) shouldBe false + id.equals("a") shouldBe false + } + +}) diff --git a/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt index 2f046d95..3b177992 100644 --- a/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/DescriptorProcessingServiceTest.kt @@ -6,7 +6,13 @@ import io.mockk.every import io.mockk.mockk import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles +import org.tormap.database.repository.RelayDetailsRepositoryImpl import org.tormap.database.repository.RelayLocationRepositoryImpl +import org.torproject.descriptor.Descriptor +import org.torproject.descriptor.DescriptorParseException +import org.torproject.descriptor.ServerDescriptor +import org.torproject.descriptor.UnparseableDescriptor +import java.io.File import org.torproject.descriptor.NetworkStatusEntry import org.torproject.descriptor.RelayNetworkStatusConsensus import java.time.LocalDate @@ -19,13 +25,16 @@ import java.util.TreeMap class DescriptorProcessingServiceTest( private val descriptorProcessingService: DescriptorProcessingService, private val relayLocationRepository: RelayLocationRepositoryImpl, + private val relayDetailsRepository: RelayDetailsRepositoryImpl, ) : StringSpec({ beforeEach { relayLocationRepository.deleteAll() + relayDetailsRepository.deleteAll() } val day = LocalDate.of(2024, 3, 5) - val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 0) + // Wgm is not 0, so a relay without the Guard flag shows that only guards get a guard weight + val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 5000) fun entry(letter: Char, flags: Set, bandwidth: Long): NetworkStatusEntry { val entry = mockk() @@ -92,4 +101,69 @@ class DescriptorProcessingServiceTest( descriptorProcessingService.processDescriptor(consensus(entry('A', setOf("Running", "Valid", "Guard"), 9000))) relayLocationRepository.findAll().single().consensusWeight shouldBe 4000 } + + fun serverDescriptor(nickname: String, published: LocalDate, familyEntries: List? = null): ServerDescriptor { + val descriptor = mockk(relaxed = true) + every { descriptor.fingerprint } returns "F".repeat(40) + every { descriptor.nickname } returns nickname + every { descriptor.address } returns "94.130.58.99" + every { descriptor.publishedMillis } returns published.atTime(8, 0).toInstant(ZoneOffset.UTC).toEpochMilli() + every { descriptor.platform } returns "Tor 0.4.8.12 on Linux" + every { descriptor.contact } returns "x".repeat(300) + every { descriptor.protocols } returns sortedMapOf("Link" to sortedSetOf(4, 5)) + every { descriptor.familyEntries } returns familyEntries + every { descriptor.linkProtocolVersions } returns null + every { descriptor.circuitProtocolVersions } returns null + return descriptor + } + + "server descriptors store the details of the relay per month" { + val result = descriptorProcessingService.processDescriptor( + serverDescriptor("first", day, listOf("\$${"A".repeat(40)}", "friend")) + ) + + result.yearMonth shouldBe "2024-03" + result.error shouldBe null + val relay = relayDetailsRepository.findAll().single() + relay.month shouldBe "2024-03" + relay.day shouldBe day + relay.nickname shouldBe "first" + relay.autonomousSystemNumber shouldBe 24940 + relay.familyEntries shouldBe "\$${"A".repeat(40)}, friend" + relay.protocols shouldBe "Link (4, 5)" + // Long untrusted values are truncated to fit the column + relay.contact!!.length shouldBe 255 + } + + "a newer server descriptor of the month replaces the older one, an older one is ignored" { + descriptorProcessingService.processDescriptor(serverDescriptor("first", day)) + descriptorProcessingService.processDescriptor(serverDescriptor("newer", day.plusDays(2))) + descriptorProcessingService.processDescriptor(serverDescriptor("older", day.plusDays(1))) + + val relay = relayDetailsRepository.findAll().single() + relay.nickname shouldBe "newer" + relay.day shouldBe day.plusDays(2) + + // Another month gets its own entry + descriptorProcessingService.processDescriptor(serverDescriptor("april", LocalDate.of(2024, 4, 1))) + relayDetailsRepository.findAll().map { it.month }.toSet() shouldBe setOf("2024-03", "2024-04") + } + + "unparsable descriptors are skipped without an error" { + val descriptor = mockk() + every { descriptor.descriptorFile } returns File("broken") + every { descriptor.descriptorParseException } returns DescriptorParseException("broken") + + val result = descriptorProcessingService.processDescriptor(descriptor) + result.yearMonth shouldBe null + result.error shouldBe null + } + + "unsupported descriptor types result in an error" { + val descriptor = mockk() + every { descriptor.descriptorFile } returns File("other") + + descriptorProcessingService.processDescriptor(descriptor).error shouldBe + "Descriptor type ${descriptor.javaClass.name} is not yet supported!" + } }) diff --git a/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt index b1b0283c..f87ebb30 100644 --- a/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/IpLookupServiceTest.kt @@ -1,6 +1,7 @@ package org.tormap.service import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.shouldBe import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -28,6 +29,27 @@ class IpLookupServiceTest( } "returns null for private IP address" { - ipLookupService.lookupLocation("10.0.0.1") shouldBe null + listOf("10.0.0.1", "172.16.0.1", "172.31.255.255", "192.168.1.1").forEach { + ipLookupService.lookupLocation(it) shouldBe null + ipLookupService.lookupAutonomousSystem(it) shouldBe null + } + } + + "neighbours of private ranges are looked up" { + // Hetzner and Google, right next to 172.16.0.0/12 and 192.168.0.0/16 + ipLookupService.lookupLocation("172.15.255.255").shouldNotBeNull() + ipLookupService.lookupLocation("172.32.0.1").shouldNotBeNull() + ipLookupService.lookupLocation("192.169.0.1").shouldNotBeNull() + } + + "returns null for addresses without a database entry" { + listOf("127.0.0.1", "0.0.0.1").forEach { + ipLookupService.lookupLocation(it) shouldBe null + ipLookupService.lookupAutonomousSystem(it) shouldBe null + } + } + + "IPv6 addresses are not looked up" { + ipLookupService.lookupLocation("2001:4860:4860::8888") shouldBe null } }) diff --git a/backend/src/test/kotlin/org/tormap/service/PubliclyRoutableAddressTest.kt b/backend/src/test/kotlin/org/tormap/service/PubliclyRoutableAddressTest.kt new file mode 100644 index 00000000..5a19f03a --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/PubliclyRoutableAddressTest.kt @@ -0,0 +1,38 @@ +package org.tormap.service + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import java.net.InetAddress + +class PubliclyRoutableAddressTest : StringSpec({ + // Literal addresses, so no DNS lookup takes place + fun routable(address: String) = InetAddress.getByName(address).isPubliclyRoutable() + + "public addresses are routable" { + listOf("1.1.1.1", "94.130.58.99", "100.63.255.255", "100.128.0.1", "2001:4860:4860::8888").forEach { + routable(it) shouldBe true + } + } + + "internal and special addresses are not routable" { + listOf( + "127.0.0.1", "::1", // loopback + "169.254.1.1", "fe80::1", // link local + "10.0.0.1", "172.16.0.1", "192.168.1.1", // private + "fc00::1", "fd12:3456::1", // unique local + "224.0.0.1", "ff02::1", // multicast + "0.0.0.0", "::", // any + "0.1.2.3", // "this" network + "100.64.0.1", "100.127.255.255", // carrier-grade NAT + ).forEach { + withClue(it) { routable(it) shouldBe false } + } + } + + "DNSJava resolver drops unresolvable names without throwing" { + val resolver = DNSJavaReverseDnsResolver() + resolver.lookupAddresses("not a valid host name..") shouldBe emptyList() + resolver.lookupPtrRecords("not an address") shouldBe emptyList() + } +}) diff --git a/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceIntegrationTest.kt b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceIntegrationTest.kt new file mode 100644 index 00000000..8af07b7e --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceIntegrationTest.kt @@ -0,0 +1,120 @@ +package org.tormap.service + +import io.kotest.assertions.nondeterministic.eventually +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.nulls.shouldBeNull +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.tormap.database.repository.RelayDetailsRepositoryImpl +import org.tormap.mockRelayDetails +import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.seconds + +@SpringBootTest +@ActiveProfiles("test") +class RelayDetailsUpdateServiceIntegrationTest( + private val relayDetailsUpdateService: RelayDetailsUpdateService, + private val relayDetailsRepository: RelayDetailsRepositoryImpl, +) : StringSpec({ + val month = "2023-05" + val fingerprint = { letter: Char -> letter.toString().repeat(40) } + + beforeEach { + relayDetailsRepository.deleteAll() + } + + fun relay(letter: Char, familyEntries: String?, address: String = "172.104.114.113") = + mockRelayDetails(letter).apply { + this.month = month + this.familyEntries = familyEntries + this.address = address + } + + fun byNickname() = relayDetailsRepository.findAll().associateBy { it.nickname } + + "relays that list each other form a family" { + relayDetailsRepository.saveAll( + listOf( + relay('A', "\$${fingerprint('B')}, \$${fingerprint('C')}"), + relay('B', "\$${fingerprint('A')}"), + // C does not confirm A + relay('C', "\$${fingerprint('D')}"), + // D and E form a second family, the invalid entry is skipped + relay('D', "\$${fingerprint('E')}, not a valid entry!"), + relay('E', "\$${fingerprint('D')}"), + relay('F', null), + ) + ) + + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + + val relays = byNickname() + relays.getValue("A").familyId.shouldNotBeNull() + relays.getValue("A").familyId shouldBe relays.getValue("B").familyId + relays.getValue("D").familyId.shouldNotBeNull() + relays.getValue("D").familyId shouldBe relays.getValue("E").familyId + relays.getValue("A").familyId shouldNotBe relays.getValue("D").familyId + relays.getValue("C").familyId.shouldBeNull() + relays.getValue("F").familyId.shouldBeNull() + } + + "families that are connected through a member are merged" { + relayDetailsRepository.saveAll( + listOf( + relay('A', "\$${fingerprint('B')}"), + relay('B', "\$${fingerprint('A')}, \$${fingerprint('C')}"), + relay('C', "\$${fingerprint('B')}"), + ) + ) + + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + + byNickname().values.map { it.familyId }.toSet().single().shouldNotBeNull() + } + + "computing the families again replaces the previous families" { + relayDetailsRepository.saveAll(listOf(relay('A', "\$${fingerprint('B')}"), relay('B', "\$${fingerprint('A')}"))) + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + val firstFamilyId = byNickname().getValue("A").familyId + + val relayA = byNickname().getValue("A") + relayA.familyEntries = null + relayDetailsRepository.save(relayA) + relayDetailsUpdateService.computeFamilies(setOf(month)).get(30, TimeUnit.SECONDS) + + firstFamilyId.shouldNotBeNull() + byNickname().getValue("A").familyId.shouldBeNull() + byNickname().getValue("B").familyId.shouldBeNull() + } + + "computeAllMissingFamiliesAndEvictCache computes families of months without any" { + relayDetailsRepository.saveAll(listOf(relay('A', "\$${fingerprint('B')}"), relay('B', "\$${fingerprint('A')}"))) + + relayDetailsUpdateService.computeAllMissingFamiliesAndEvictCache() + + eventually(10.seconds) { + byNickname().getValue("A").familyId.shouldNotBeNull() + } + } + + "lookupAllMissingAutonomousSystems fills in the autonomous system" { + relayDetailsRepository.saveAll( + listOf( + relay('A', null, address = "94.130.58.99"), + // Private addresses have no autonomous system + relay('B', null, address = "10.0.0.1"), + ) + ) + + relayDetailsUpdateService.lookupAllMissingAutonomousSystems() + + eventually(10.seconds) { + byNickname().getValue("A").autonomousSystemNumber shouldBe 24940 + } + byNickname().getValue("A").autonomousSystemName.shouldNotBeNull() + byNickname().getValue("B").autonomousSystemNumber.shouldBeNull() + } +}) diff --git a/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceTest.kt new file mode 100644 index 00000000..5a5110f8 --- /dev/null +++ b/backend/src/test/kotlin/org/tormap/service/RelayDetailsUpdateServiceTest.kt @@ -0,0 +1,69 @@ +package org.tormap.service + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.booleans.shouldBeTrue +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.springframework.transaction.support.TransactionTemplate +import org.tormap.database.repository.MonthFamilyMembersCount +import org.tormap.database.repository.RelayDetailsRepositoryImpl +import java.util.concurrent.CompletableFuture +import javax.sql.DataSource + +class RelayDetailsUpdateServiceTest : StringSpec({ + "computeAllMissingFamiliesAndEvictCache evicts the cache only after the families are computed" { + val repository = mockk(relaxed = true) + val cacheService = mockk(relaxed = true) + val coalesceService = mockk() + val familiesOfMonth = mapOf("2024-01" to CompletableFuture(), "2024-02" to CompletableFuture()) + every { repository.findDistinctMonthFamilyMemberCount() } returns listOf( + MonthFamilyMembersCount("2024-01", 0), + MonthFamilyMembersCount("2024-02", 0), + // Months that already have families are skipped + MonthFamilyMembersCount("2024-03", 5), + ) + every { coalesceService.submitAsync(any(), any()) } answers { + familiesOfMonth.getValue(firstArg().removePrefix("computeFamilies-")) + } + val service = RelayDetailsUpdateService( + repository, + mockk(relaxed = true), + cacheService, + coalesceService, + mockk(relaxed = true), + mockk(relaxed = true), + ) + + service.computeAllMissingFamiliesAndEvictCache() + + verify(exactly = 1) { coalesceService.submitAsync("computeFamilies-2024-01", any()) } + verify(exactly = 1) { coalesceService.submitAsync("computeFamilies-2024-02", any()) } + verify(exactly = 0) { coalesceService.submitAsync("computeFamilies-2024-03", any()) } + familiesOfMonth.getValue("2024-01").complete(null) + verify(exactly = 0) { cacheService.evictRelayLocationsPerDay(any()) } + + // A failed month does not prevent the eviction + familiesOfMonth.getValue("2024-02").completeExceptionally(IllegalStateException("failed")) + verify(exactly = 1) { cacheService.evictRelayLocationsPerDay(setOf("2024-01", "2024-02")) } + } + + "computeFamilies completes once all months are done" { + val coalesceService = mockk() + val future = CompletableFuture() + every { coalesceService.submitAsync(any(), any()) } returns future + val service = RelayDetailsUpdateService( + mockk(relaxed = true), + mockk(relaxed = true), + mockk(relaxed = true), + coalesceService, + mockk(relaxed = true), + mockk(relaxed = true), + ) + + val result = service.computeFamilies(setOf("2024-01")) + future.complete(null) + result.isDone.shouldBeTrue() + service.computeFamilies(emptySet()).isDone.shouldBeTrue() + } +}) diff --git a/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt index d57065ef..2f887911 100644 --- a/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/ReverseDnsLookupServiceTest.kt @@ -126,6 +126,38 @@ class ReverseDnsLookupServiceTest : StringSpec({ reverseDnsLookupService.destroy() } } + + "lookupHostNames returns what is known when a lookup fails" { + val failingResolver = object : ReverseDnsResolver { + override fun lookupPtrRecords(ipAddress: String) = listOf("host.example.") + override fun lookupAddresses(hostName: String): List = throw IllegalStateException("resolver broke") + } + + val result = ReverseDnsLookupService(failingResolver).lookupHostNames("1.2.3.4") + + result shouldBe ReverseDnsLookupResult(unverifiedHostNames = listOf("host.example")) + } + + "lookupHostNames keeps the interrupt of the calling thread" { + val reverseDnsResolver = LookupServiceFakeReverseDnsResolver( + ptrRecordsByIpAddress = mapOf("1.2.3.4" to listOf("slow.example.")), + ptrLookupDelay = Duration.ofSeconds(5), + ) + val service = ReverseDnsLookupService(reverseDnsResolver, lookupTimeout = Duration.ofSeconds(10)) + var result: ReverseDnsLookupResult? = null + var interrupted = false + val caller = thread { + result = service.lookupHostNames("1.2.3.4") + interrupted = Thread.currentThread().isInterrupted + } + Thread.sleep(200) + caller.interrupt() + caller.join(5_000) + + result shouldBe ReverseDnsLookupResult() + interrupted shouldBe true + service.destroy() + } }) private class LookupServiceFakeReverseDnsResolver( diff --git a/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt b/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt index e819e3e5..7b5a4a50 100644 --- a/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt +++ b/backend/src/test/kotlin/org/tormap/service/SchedulingServiceTest.kt @@ -3,7 +3,12 @@ package org.tormap.service import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.shouldBeEmpty import io.kotest.matchers.collections.shouldNotBeEmpty +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify import org.springframework.scheduling.annotation.Scheduled +import org.tormap.config.value.DescriptorConfig +import org.tormap.database.entity.DescriptorType import java.util.concurrent.Future class SchedulingServiceTest : StringSpec({ @@ -18,4 +23,35 @@ class SchedulingServiceTest : StringSpec({ .map { "${it.name}: ${it.returnType}" } .shouldBeEmpty() } + + "each scheduled method starts its job" { + val descriptorConfig = mockk() + every { descriptorConfig.recentRelayConsensuses } returns "/recent/consensuses" + every { descriptorConfig.recentRelayServers } returns "/recent/servers" + every { descriptorConfig.archiveRelayConsensuses } returns "/archive/consensuses" + every { descriptorConfig.archiveRelayServers } returns "/archive/servers" + val coordination = mockk(relaxed = true) + val relayDetails = mockk(relaxed = true) + val files = mockk(relaxed = true) + val countryUsage = mockk(relaxed = true) + val service = SchedulingService(descriptorConfig, coordination, relayDetails, files, countryUsage) + + service.recentConsensousDescriptors() + service.recentServerDescriptors() + service.archiveRelayConsensuses() + service.archiveRelayServers() + service.refreshCountryUsage() + service.updateRelayFamilies() + service.updateRelayAutonomousSystems() + service.deleteRecentFileReference() + + verify { coordination.collectAndProcessDescriptors("/recent/consensuses", DescriptorType.RECENT_RELAY_CONSENSUS) } + verify { coordination.collectAndProcessDescriptors("/recent/servers", DescriptorType.RECENT_RELAY_SERVER) } + verify { coordination.collectAndProcessDescriptors("/archive/consensuses", DescriptorType.ARCHIVE_RELAY_CONSENSUS) } + verify { coordination.collectAndProcessDescriptors("/archive/servers", DescriptorType.ARCHIVE_RELAY_SERVER) } + verify { countryUsage.refresh() } + verify { relayDetails.computeAllMissingFamiliesAndEvictCache() } + verify { relayDetails.lookupAllMissingAutonomousSystems() } + verify { files.deleteRecentFileReferences() } + } }) diff --git a/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt b/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt index b727ac96..8cb467f0 100644 --- a/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt +++ b/backend/src/test/kotlin/org/tormap/util/PathSelectionTest.kt @@ -4,7 +4,7 @@ import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe class PathSelectionTest : StringSpec({ - val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 0) + val weights = mapOf("Wgg" to 5000, "Wgd" to 2500, "Wgm" to 5000) val running = setOf("Running", "Valid") "guard relays use Wgg" { @@ -15,9 +15,15 @@ class PathSelectionTest : StringSpec({ PathSelection.guardWeight(1000, running + "Guard" + "Exit", weights) shouldBe 250 } - "relays without guard flag use Wgm" { + "guard relays with the BadExit flag do not count as exits" { + PathSelection.guardWeight(1000, running + "Guard" + "Exit" + "BadExit", weights) shouldBe 500 + } + + "relays without guard flag are never picked as guard" { PathSelection.guardWeight(1000, running + "Exit", weights) shouldBe 0 - PathSelection.guardWeight(1000, running, mapOf("Wgm" to 1000)) shouldBe 100 + PathSelection.guardWeight(1000, running + "Exit" + "BadExit", weights) shouldBe 0 + PathSelection.guardWeight(1000, running + "Fast" + "Stable", weights) shouldBe 0 + PathSelection.guardWeight(1000, running, null) shouldBe 0 } "relays that are not running or valid can not be picked" { @@ -32,5 +38,6 @@ class PathSelectionTest : StringSpec({ "missing weights result in null" { PathSelection.guardWeight(1000, running + "Guard", null) shouldBe null PathSelection.guardWeight(1000, running + "Guard", mapOf("Wmm" to 10000)) shouldBe null + PathSelection.guardWeight(1000, running + "Guard" + "Exit", mapOf("Wgg" to 10000)) shouldBe null } }) diff --git a/frontend/package.json b/frontend/package.json index 27b6c487..597dd68a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tormap", - "version": "3.0.1", + "version": "4.0.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/components/date/DateSlider.test.tsx b/frontend/src/components/date/DateSlider.test.tsx index 0e103e90..d81135aa 100644 --- a/frontend/src/components/date/DateSlider.test.tsx +++ b/frontend/src/components/date/DateSlider.test.tsx @@ -1,6 +1,6 @@ import {FunctionComponent, useEffect, useRef} from "react"; import {describe, expect, it} from "vitest"; -import {page} from "vitest/browser"; +import {page, userEvent} from "vitest/browser"; import {useDate} from "../../context/date-context"; import {renderWithProviders} from "../../test/render"; @@ -9,14 +9,14 @@ import {DateSlider} from "./DateSlider"; const days = ["2024-02-01", "2024-02-02", "2024-02-03"] const Seed: FunctionComponent = () => { - const {setAvailableDays} = useDate() + const {setAvailableDays, selectedDate} = useDate() const seeded = useRef(false) useEffect(() => { if (seeded.current) return seeded.current = true setAvailableDays(days) }, [setAvailableDays]) - return null + return {selectedDate} } describe("DateSlider", () => { @@ -28,4 +28,24 @@ describe("DateSlider", () => { // The last available day is selected by default await expect.element(slider).toHaveAttribute("aria-valuetext", "2024-02-03") }) + + it("selects another day with the keyboard", async () => { + await renderWithProviders(<>) + const slider = page.getByRole("slider", {name: "Selected day"}) + await expect.element(slider).toHaveAttribute("aria-valuetext", "2024-02-03") + + // The range input is visually hidden, so it is focused instead of clicked + const input = slider.element() as HTMLInputElement + input.focus() + await userEvent.keyboard("{ArrowLeft}") + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-02") + await userEvent.keyboard("{Home}") + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-01") + await expect.element(slider).toHaveAttribute("aria-valuetext", "2024-02-01") + }) + + it("is disabled until days are available", async () => { + await renderWithProviders() + await expect.element(page.getByRole("slider", {name: "Selected day"})).toBeDisabled() + }) }) diff --git a/frontend/src/components/date/RelayCountTimeline.test.tsx b/frontend/src/components/date/RelayCountTimeline.test.tsx index 41ebeacf..705dacaa 100644 --- a/frontend/src/components/date/RelayCountTimeline.test.tsx +++ b/frontend/src/components/date/RelayCountTimeline.test.tsx @@ -79,4 +79,57 @@ describe("RelayCountTimeline", () => { await renderTimeline() await expect.element(page.getByRole("img", {name: /Relays per day/})).not.toBeInTheDocument() }) + + it("selects an outlier day with the keyboard", async () => { + mockBackend.onGet("/relay/statistics/daily-counts", counts) + await renderTimeline() + + const outlier = page.getByTestId("relay-count-outlier") + await expect.element(outlier).toBeVisible() + outlier.element().dispatchEvent(new KeyboardEvent("keydown", {key: "Tab", bubbles: true})) + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[39]) + + const space = new KeyboardEvent("keydown", {key: " ", bubbles: true, cancelable: true}) + outlier.element().dispatchEvent(space) + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[20]) + expect(space.defaultPrevented).toBe(true) + + }) + + it("selects an outlier day with Enter", async () => { + mockBackend.onGet("/relay/statistics/daily-counts", counts) + await renderTimeline() + + const outlier = page.getByTestId("relay-count-outlier") + await expect.element(outlier).toBeVisible() + outlier.element().dispatchEvent(new KeyboardEvent("keydown", {key: "Enter", bubbles: true})) + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[20]) + }) + + it("describes the outlier with its change and marks the selected day", async () => { + const spike = counts.map((count, i) => i === 10 ? {...count, total: 9000} : count) + mockBackend.onGet("/relay/statistics/daily-counts", spike) + await renderTimeline() + + await expect.element(page.getByRole("button", {name: new RegExp(`Select ${days[10]}: 9000 relays \\(\\+`)})).toBeVisible() + await expect.element(page.getByRole("button", {name: new RegExp(`Select ${days[20]}: 2500 relays \\(-`)})).toBeVisible() + // The selected (latest) day is marked with a vertical line at the right end + expect(document.querySelector("svg line")?.getAttribute("x1")).toBe("1000") + }) + + it("breaks the line at days without counts", async () => { + const withGap = counts.filter((_, i) => i < 15 || i > 17) + mockBackend.onGet("/relay/statistics/daily-counts", withGap) + await renderTimeline() + + await expect.element(page.getByRole("img", {name: /Relays per day/})).toBeVisible() + expect(document.querySelectorAll("svg polyline")).toHaveLength(2) + }) + + it("renders nothing if no count matches an available day", async () => { + mockBackend.onGet("/relay/statistics/daily-counts", [{day: "2001-01-01", total: 1, exit: 0, guard: 0, other: 1}]) + await renderTimeline() + await expect.element(page.getByTestId("selected-date")).toHaveTextContent(days[39]) + await expect.element(page.getByRole("img", {name: /Relays per day/})).not.toBeInTheDocument() + }) }) diff --git a/frontend/src/components/date/RelayCountTimeline.tsx b/frontend/src/components/date/RelayCountTimeline.tsx index e5b77b15..506ec545 100644 --- a/frontend/src/components/date/RelayCountTimeline.tsx +++ b/frontend/src/components/date/RelayCountTimeline.tsx @@ -116,7 +116,11 @@ export const RelayCountTimeline: FunctionComponent = () => { style={{cursor: "pointer"}} onClick={() => setSelectedDate(outlier.day)} onKeyDown={event => { - if (event.key === "Enter" || event.key === " ") setSelectedDate(outlier.day) + if (event.key === "Enter" || event.key === " ") { + // Space would otherwise scroll the page, like for a native button + event.preventDefault() + setSelectedDate(outlier.day) + } }} > {outlier.label} diff --git a/frontend/src/components/date/ResponsiveDatePicker.test.tsx b/frontend/src/components/date/ResponsiveDatePicker.test.tsx new file mode 100644 index 00000000..3dcac147 --- /dev/null +++ b/frontend/src/components/date/ResponsiveDatePicker.test.tsx @@ -0,0 +1,85 @@ +import {FunctionComponent, useEffect, useRef} from "react"; +import {describe, expect, it} from "vitest"; +import {page, userEvent} from "vitest/browser"; + +import {useDate} from "../../context/date-context"; +import {renderWithProviders} from "../../test/render"; +import {ResponsiveDatePicker} from "./ResponsiveDatePicker"; + +// 2024-02-03 is missing, e.g. because no consensus was archived that day +const days = ["2024-02-01", "2024-02-02", "2024-02-04", "2024-02-05"] + +const Seed: FunctionComponent = () => { + const {setAvailableDays, selectedDate, setSelectedDate} = useDate() + const seeded = useRef(false) + useEffect(() => { + if (seeded.current) return + seeded.current = true + setAvailableDays(days) + }, [setAvailableDays]) + return ( + <> + {selectedDate} + + + ) +} + +const renderPicker = (largeScreen = true) => renderWithProviders(<>) + +const field = () => document.querySelector("input.MuiPickersInputBase-input")! + +describe("ResponsiveDatePicker", () => { + it("shows the selected day", async () => { + await renderPicker() + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-05") + await expect.poll(() => field().value).toBe("2024-02-05") + }) + + it("selects an available day from the calendar", async () => { + await renderPicker() + await page.getByRole("button", {name: /Choose date/}).click() + await page.getByRole("gridcell", {name: "2", exact: true}).click() + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-02") + await expect.poll(() => field().value).toBe("2024-02-02") + }) + + it("disables days without data", async () => { + await renderPicker(false) + await page.getByRole("button", {name: /Choose date/}).click() + await expect.element(page.getByRole("gridcell", {name: "3", exact: true})).toBeDisabled() + await expect.element(page.getByRole("gridcell", {name: "4", exact: true})).toBeEnabled() + // Days outside the available range are disabled as well + await expect.element(page.getByRole("gridcell", {name: "6", exact: true})).toBeDisabled() + }) + + it("follows a day selected elsewhere, e.g. on the slider", async () => { + await renderPicker() + await page.getByRole("button", {name: "select first day"}).click() + await expect.poll(() => field().value).toBe("2024-02-01") + }) + + it("keeps a typed day that is not available out of the selection and resets the field on blur", async () => { + await renderPicker() + await expect.poll(() => field().value).toBe("2024-02-05") + await page.getByRole("spinbutton", {name: "Day"}).click() + await userEvent.keyboard("03") + await expect.poll(() => field().value).toBe("2024-02-03") + // 2024-02-03 has no data, so the loaded day stays selected + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-05") + + await page.getByRole("button", {name: "select first day"}).click() + await expect.poll(() => field().value).toBe("2024-02-01") + await page.getByRole("spinbutton", {name: "Day"}).click() + await userEvent.keyboard("03") + await userEvent.click(page.getByTestId("selected-date")) + await expect.poll(() => field().value).toBe("2024-02-01") + }) + + it("selects a typed day that is available", async () => { + await renderPicker() + await page.getByRole("spinbutton", {name: "Day"}).click() + await userEvent.keyboard("04") + await expect.element(page.getByTestId("selected-date")).toHaveTextContent("2024-02-04") + }) +}) diff --git a/frontend/src/components/leaflet/CountryUsageLayer.test.tsx b/frontend/src/components/leaflet/CountryUsageLayer.test.tsx index c7ddf6ea..95cb8c71 100644 --- a/frontend/src/components/leaflet/CountryUsageLayer.test.tsx +++ b/frontend/src/components/leaflet/CountryUsageLayer.test.tsx @@ -75,4 +75,22 @@ describe("CountryUsageLayer", () => { await expect.poll(() => mockBackend.requested.length).toBe(1) await expect.element(page.getByRole("status", {name: "usage"})).toHaveTextContent("none") }) + + it("colors the countries with an estimate on the map", async () => { + mockBackend.onGet("/relay/country-usage/day/2024-03-05", usage) + await renderLayer(true) + await expect.element(page.getByRole("status", {name: "usage"})).toHaveTextContent("2024-03-05:2") + // Only the two countries with an estimate are drawn, in their own pane below the markers + await expect.poll(() => document.querySelectorAll(".leaflet-countryUsage-pane path").length).toBe(2) + const pane = document.querySelector(".leaflet-countryUsage-pane")! + expect(pane.style.zIndex).toBe("350") + }) + + it("removes the colors when the estimates are switched off", async () => { + mockBackend.onGet("/relay/country-usage/day/2024-03-05", usage) + const screen = await renderLayer(true) + await expect.poll(() => document.querySelectorAll(".leaflet-countryUsage-pane path").length).toBe(2) + await screen.unmount() + expect(document.querySelectorAll(".leaflet-countryUsage-pane path").length).toBe(0) + }) }) diff --git a/frontend/src/util/heat-layer.test.ts b/frontend/src/util/heat-layer.test.ts index 8da09bf5..319fadfa 100644 --- a/frontend/src/util/heat-layer.test.ts +++ b/frontend/src/util/heat-layer.test.ts @@ -14,12 +14,12 @@ describe("HeatLayer", () => { container = undefined }) - const createMap = () => { + const createMap = (zoomAnimation = false) => { container = document.createElement("div") container.style.width = "400px" container.style.height = "300px" document.body.appendChild(container) - map = L.map(container, {zoomAnimation: false}).setView([0, 0], 2) + map = L.map(container, {zoomAnimation}).setView([0, 0], 2) return map } @@ -61,4 +61,54 @@ describe("HeatLayer", () => { expect(heatLayer([])).toBeInstanceOf(L.Layer) expect(heatLayer([])).toBeInstanceOf(HeatLayer) }) + + it("uses the default radius and gradient without options", () => { + const leafletMap = createMap() + heatLayer([[0, 0, 1]]).addTo(leafletMap) + const center = leafletMap.latLngToContainerPoint([0, 0]) + expect(pixelAt(canvasOf(), Math.round(center.x), Math.round(center.y))[3]).toBeGreaterThan(0) + expect(canvasOf().classList).toContain("leaflet-zoom-hide") + }) + + it("skips points outside of the visible map", () => { + const leafletMap = createMap() + leafletMap.setView([0, 0], 8) + heatLayer([[60, 120, 1]], {radius: 10, blur: 5}).addTo(leafletMap) + const canvas = canvasOf() + const data = canvas.getContext("2d")!.getImageData(0, 0, canvas.width, canvas.height).data + expect(data.some((value, index) => index % 4 === 3 && value > 0)).toBe(false) + }) + + it("follows the size of the map", () => { + const leafletMap = createMap() + heatLayer([[0, 0, 1]], {maxZoom: 10}).addTo(leafletMap) + container!.style.width = "200px" + container!.style.height = "100px" + leafletMap.invalidateSize() + leafletMap.fire("moveend") + expect(canvasOf().width).toBe(200) + expect(canvasOf().height).toBe(100) + }) + + it("can be added again after it was removed", () => { + const leafletMap = createMap() + const layer = heatLayer([[0, 0, 1]]).addTo(leafletMap) + const canvas = canvasOf() + leafletMap.removeLayer(layer) + layer.addTo(leafletMap) + expect(canvasOf()).toBe(canvas) + }) + + it("scales the canvas while the map animates a zoom", () => { + const leafletMap = createMap(true) + const layer = heatLayer([[0, 0, 1]]).addTo(leafletMap) + expect(canvasOf().classList).toContain("leaflet-zoom-animated") + + leafletMap.fire("zoomanim", {center: leafletMap.getCenter(), zoom: 3, noUpdate: false}) + expect(canvasOf().style.transform).toContain("scale(2)") + + leafletMap.removeLayer(layer) + expect(leafletMap.listens("zoomanim", (layer as unknown as {animateZoom: L.LeafletEventHandlerFn}).animateZoom, layer)) + .toBe(false) + }) })