feat: paginate subscription group memberships

openapi.yaml

@@ -47,6 +47,8 @@ paths:
   /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist }
   /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions }
   /subscriptions/group-memberships: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupMemberships }
+  /subscriptions/group-memberships/page: { $ref: ./openapi/paths/subscription-membership-pages.yaml#/Page }
+  /subscriptions/group-memberships/lookup: { $ref: ./openapi/paths/subscription-membership-pages.yaml#/Lookup }
   /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups }
   /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup }
   /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels }

openapi/components/subscriptions.yaml

@@ -18,6 +18,19 @@ SubscriptionGroupMembershipItem:
       type: array
       uniqueItems: true
       items: { type: string, format: uuid }
+SubscriptionMembershipPage:
+  type: object
+  required: [items, total, totalSubscriptions, ungroupedCount, page, limit]
+  properties:
+    items:
+      type: array
+      maxItems: 100
+      items: { $ref: '#/SubscriptionGroupMembershipItem' }
+    total: { type: integer, format: int64, minimum: 0, description: Number of subscriptions matching all filters. }
+    totalSubscriptions: { type: integer, format: int64, minimum: 0, description: Unfiltered account subscription count. }
+    ungroupedCount: { type: integer, format: int64, minimum: 0, description: Unfiltered account count with no memberships. }
+    page: { type: integer, minimum: 0, maximum: 1000000 }
+    limit: { type: integer, minimum: 1, maximum: 100 }
 SubscriptionCreateRequest:
   type: object
   required: [channelUrl, name, avatarUrl]

openapi/paths/subscription-membership-pages.yaml

@@ -0,0 +1,72 @@
+Page:
+  get:
+    tags: [user-data]
+    summary: Read a bounded page of subscriptions with group memberships
+    description: |
+      Filtering, counting and pagination run in the database before channel rows and their
+      memberships are loaded. Rows are ordered by lowercase name and then canonical channel URL.
+      The zero-based page is an offset into current results, not a snapshot; changes between
+      requests may move rows. A page beyond the last returns empty items with the current total.
+      Group IDs and results are scoped to the authenticated profile. Unknown or inaccessible
+      groupId returns subscription_group_not_found. Omit groupId and ungrouped for all subscriptions.
+    parameters:
+      - name: page
+        in: query
+        schema: { type: integer, minimum: 0, maximum: 1000000, default: 0 }
+      - name: limit
+        in: query
+        schema: { type: integer, minimum: 1, maximum: 100, default: 20 }
+      - name: search
+        in: query
+        description: Case-insensitive literal substring of channel name or URL. Whitespace is trimmed; SQL wildcard characters are literal.
+        schema: { type: string, maxLength: 200 }
+      - name: groupId
+        in: query
+        schema: { type: string, format: uuid }
+      - name: ungrouped
+        in: query
+        description: Select channels in no group. Cannot be combined with groupId.
+        schema: { type: boolean, default: false }
+      - name: excluded
+        in: query
+        description: Select channels outside groupId, including channels in other groups. Requires groupId.
+        schema: { type: boolean, default: false }
+    responses:
+      '200':
+        description: Requested page and counts, including all memberships for each returned channel.
+        content:
+          application/json:
+            schema: { $ref: ../components/subscriptions.yaml#/SubscriptionMembershipPage }
+      '400': { $ref: ../components/common.yaml#/JsonError }
+      '401': { $ref: ../components/common.yaml#/JsonError }
+      '404': { $ref: ../components/common.yaml#/JsonError }
+Lookup:
+  post:
+    tags: [user-data]
+    summary: Refresh memberships for explicitly selected subscriptions
+    description: |
+      Read-only lookup for selections retained across pages. Accepts one channelUrl or up to
+      500 channelUrls, with a 2048-character URL limit and a 1 MiB request-body limit.
+      URLs are canonicalized and deduplicated. Returns only existing subscriptions owned by
+      the authenticated profile, with all their memberships; missing channels are omitted.
+      Results have no guaranteed order. This endpoint does not change subscriptions or groups.
+    requestBody:
+      required: true
+      content:
+        application/json:
+          schema:
+            oneOf:
+              - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest }
+              - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest }
+    responses:
+      '200':
+        description: Current memberships for the selected channels still subscribed to.
+        content:
+          application/json:
+            schema:
+              type: array
+              maxItems: 500
+              items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipItem }
+      '400': { $ref: ../components/common.yaml#/JsonError }
+      '401': { $ref: ../components/common.yaml#/JsonError }
+      '413': { $ref: ../components/common.yaml#/JsonError }

server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt

@@ -0,0 +1,13 @@
+package dev.typetype.server.models
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class SubscriptionMembershipPage(
+    val items: List<SubscriptionGroupMembershipItem>,
+    val total: Long,
+    val totalSubscriptions: Long,
+    val ungroupedCount: Long,
+    val page: Int,
+    val limit: Int,
+)

server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt

@@ -8,6 +8,8 @@ object DatabaseIndexMigrations {
         exec("CREATE INDEX IF NOT EXISTS idx_history_user_watched_id ON history (user_id, watched_at DESC, id DESC)")
         exec("CREATE INDEX IF NOT EXISTS idx_playlist_videos_user_playlist_position ON playlist_videos (user_id, playlist_id, position)")
         exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_user_subscribed_at ON subscriptions (user_id, subscribed_at DESC)")
+        exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_name_trgm ON subscriptions USING gin (lower(name) gin_trgm_ops)")
+        exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_channel_url_trgm ON subscriptions USING gin (lower(channel_url) gin_trgm_ops)")
         exec("CREATE INDEX IF NOT EXISTS idx_favorites_user_favorited_at ON favorites (user_id, favorited_at DESC)")
         exec("CREATE INDEX IF NOT EXISTS idx_watch_later_user_added_at ON watch_later (user_id, added_at DESC)")
         exec("CREATE INDEX IF NOT EXISTS idx_search_history_user_searched_at ON search_history (user_id, searched_at DESC)")

server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt

@@ -75,7 +75,14 @@ object TestDatabase {
         if (initialized) return
         synchronized(this) {
             if (initialized) return
-            val (baseUrl, user, password) = runCatching {
+            val externalUrl = System.getenv("TEST_DATABASE_URL")?.takeIf(String::isNotBlank)
+            val (baseUrl, user, password) = if (externalUrl != null) {
+                Triple(
+                    externalUrl,
+                    firstNonBlank(System.getenv("TEST_DATABASE_USER"), "typetype"),
+                    firstNonBlank(System.getenv("TEST_DATABASE_PASSWORD"), "typetype"),
+                )
+            } else runCatching {
                 val c = container
                 Triple(c.jdbcUrl, c.username, c.password)
             }.getOrElse {
@@ -96,6 +103,7 @@ object TestDatabase {
     private fun ensureSchemaExists(baseUrl: String, user: String, password: String, schema: String): Unit {
         DriverManager.getConnection(baseUrl, user, password).use { connection ->
             connection.createStatement().use { statement ->
+                statement.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public")
                 statement.execute("CREATE SCHEMA IF NOT EXISTS \"$schema\"")
             }
         }

server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt

@@ -10,6 +10,13 @@ object SubscriptionMutationLock {
         )
     }
 
+    fun acquireRead(userId: String) {
+        val userKey = userId.hashCode() and Int.MAX_VALUE
+        TransactionManager.current().exec(
+            "SELECT pg_advisory_xact_lock_shared($LOCK_NAMESPACE, $userKey)",
+        )
+    }
+
     // Precomputed PostgreSQL hashtext('subscriptions').
     private const val LOCK_NAMESPACE = 720_815_616
 }

server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt

@@ -0,0 +1,56 @@
+package dev.typetype.server.routes
+
+import dev.typetype.server.models.ErrorResponse
+import dev.typetype.server.services.AuthService
+import dev.typetype.server.services.SubscriptionGroupsService
+import dev.typetype.server.services.SubscriptionMembershipFilter
+import dev.typetype.server.services.SubscriptionMembershipPageService
+import io.ktor.http.HttpStatusCode
+import io.ktor.server.application.ApplicationCall
+import io.ktor.server.response.respond
+import io.ktor.server.routing.Route
+import io.ktor.server.routing.get
+import io.ktor.server.routing.post
+
+internal fun Route.subscriptionMembershipPageRoutes(auth: AuthService, groups: SubscriptionGroupsService) {
+    val service = SubscriptionMembershipPageService()
+    get("/subscriptions/group-memberships/page") {
+        call.withJwtAuth(auth) { userId ->
+            val filter = call.membershipFilter() ?: return@withJwtAuth call.respond(
+                HttpStatusCode.BadRequest, ErrorResponse("Invalid membership page filter", "subscription_group_invalid_filter"),
+            )
+            val groupId = filter.groupId
+            if (groupId != null && !groups.exists(userId, groupId)) {
+                return@withJwtAuth call.respond(
+                    HttpStatusCode.NotFound, ErrorResponse("Subscription group not found", "subscription_group_not_found"),
+                )
+            }
+            call.respond(service.getPage(userId, filter))
+        }
+    }
+    post("/subscriptions/group-memberships/lookup") {
+        call.withJwtAuth(auth) { userId ->
+            val body = call.receiveMembershipBody() ?: return@withJwtAuth
+            val urls = when (val request = call.receiveMembershipChannels(body) ?: return@withJwtAuth) {
+                is MembershipChannels.Single -> listOf(request.channelUrl)
+                is MembershipChannels.Batch -> request.channelUrls
+            }
+            call.respond(service.lookup(userId, urls))
+        }
+    }
+}
+
+private fun ApplicationCall.membershipFilter(): SubscriptionMembershipFilter? {
+    val params = request.queryParameters
+    val page = params["page"]?.toIntOrNull() ?: if (params["page"] == null) 0 else return null
+    val limit = params["limit"]?.toIntOrNull() ?: if (params["limit"] == null) 20 else return null
+    val search = params["search"]?.trim().orEmpty()
+    val group = params["groupId"]
+    val ungrouped = params["ungrouped"]?.toBooleanStrictOrNull()
+        ?: if (params["ungrouped"] == null) false else return null
+    val excluded = params["excluded"]?.toBooleanStrictOrNull()
+        ?: if (params["excluded"] == null) false else return null
+    if (page !in 0..1_000_000 || limit !in 1..100 || search.length > 200) return null
+    if (group != null && group.isBlank() || group != null && ungrouped || excluded && group == null) return null
+    return SubscriptionMembershipFilter(page, limit, search, group, ungrouped, excluded)
+}

server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt

@@ -29,6 +29,7 @@ internal fun Route.subscriptionsRoutes(
     groupsService: SubscriptionGroupsService = SubscriptionGroupsService(),
     pushNotificationService: PushNotificationService? = null,
 ) {
+    subscriptionMembershipPageRoutes(authService, groupsService)
     get("/subscriptions/group-memberships") {
         call.withJwtAuth(authService) { userId ->
             call.respond(subscriptionsService.getAllWithGroupMemberships(userId))

server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt

@@ -0,0 +1,109 @@
+package dev.typetype.server
+
+import dev.typetype.server.models.SubscriptionItem
+import dev.typetype.server.models.SubscriptionMembershipPage
+import dev.typetype.server.routes.subscriptionsRoutes
+import dev.typetype.server.services.AuthService
+import dev.typetype.server.services.SubscriptionGroupsService
+import dev.typetype.server.services.SubscriptionGroupWriteResult
+import dev.typetype.server.services.SubscriptionsService
+import io.ktor.client.request.get
+import io.ktor.client.request.header
+import io.ktor.client.request.post
+import io.ktor.client.request.setBody
+import io.ktor.client.statement.bodyAsText
+import io.ktor.http.HttpHeaders
+import io.ktor.http.HttpStatusCode
+import io.ktor.serialization.kotlinx.json.json
+import io.ktor.server.application.install
+import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
+import io.ktor.server.routing.routing
+import io.ktor.server.testing.ApplicationTestBuilder
+import io.ktor.server.testing.testApplication
+import kotlinx.serialization.json.Json
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+
+class SubscriptionMembershipPageRoutesTest {
+    private val subscriptions = SubscriptionsService()
+    private val groups = SubscriptionGroupsService()
+    private val path = "/subscriptions/group-memberships"
+
+    companion object {
+        @BeforeAll
+        @JvmStatic
+        fun initDb(): Unit = TestDatabase.setup()
+    }
+
+    @BeforeEach
+    fun clean(): Unit = TestDatabase.truncateAll()
+
+    private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit): Unit = testApplication {
+        application {
+            install(ContentNegotiation) { json() }
+            routing { subscriptionsRoutes(subscriptions, AuthService.fixed(TEST_USER_ID), groupsService = groups) }
+        }
+        block()
+    }
+
+    @Test
+    fun `page and lookup require authentication`() = withApp {
+        assertEquals(HttpStatusCode.Unauthorized, client.get("$path/page").status)
+        assertEquals(HttpStatusCode.Unauthorized, client.post("$path/lookup").status)
+    }
+
+    @Test
+    fun `page validates limits filters and group ownership`() = withApp {
+        for (query in listOf("page=-1", "page=x", "page=1000001", "limit=0", "limit=101", "limit=x",
+            "groupId=", "excluded=true", "ungrouped=maybe", "excluded=maybe", "groupId=x&ungrouped=true",
+            "search=${"a".repeat(201)}")) {
+            assertEquals(HttpStatusCode.BadRequest, get("$path/page?$query").status, query)
+        }
+        val foreign = (groups.create("foreign", "Hidden") as SubscriptionGroupWriteResult.Success).group
+        for (id in listOf("missing", foreign.id)) {
+            val response = get("$path/page?groupId=$id")
+            assertEquals(HttpStatusCode.NotFound, response.status)
+            assertTrue(response.bodyAsText().contains("subscription_group_not_found"))
+        }
+    }
+
+    @Test
+    fun `page contract returns requested rows and accurate counts without changing legacy reads`() = withApp {
+        for (index in 0..2) subscriptions.add(TEST_USER_ID, SubscriptionItem("https://example.com/channel/$index", "Name $index", "avatar"))
+        val response = get("$path/page?page=1&limit=2&search=NAME")
+        assertEquals(HttpStatusCode.OK, response.status)
+        val page = Json.decodeFromString<SubscriptionMembershipPage>(response.bodyAsText())
+        assertEquals(1, page.page)
+        assertEquals(2, page.limit)
+        assertEquals(3L, page.total)
+        assertEquals(3L, page.totalSubscriptions)
+        assertEquals(3L, page.ungroupedCount)
+        assertEquals("Name 2", page.items.single().name)
+        assertTrue(get(path).bodyAsText().startsWith("["))
+    }
+
+    @Test
+    fun `lookup enforces bounded bodies and returns only current owned subscriptions`() = withApp {
+        subscriptions.add(TEST_USER_ID, SubscriptionItem("https://example.com/channel/one", "One", "avatar"))
+        val valid = lookup("""{"channelUrls":["https://example.com/channel/one","missing"]}""")
+        assertEquals(HttpStatusCode.OK, valid.status)
+        assertTrue(valid.bodyAsText().contains("\"name\":\"One\""))
+        assertEquals(HttpStatusCode.BadRequest, lookup("""{"channelUrls":[]}""").status)
+        assertEquals(HttpStatusCode.BadRequest, lookup("""{"channelUrls":["${"x".repeat(2049)}"]}""").status)
+        assertEquals(HttpStatusCode.BadRequest, lookup("{\"channelUrls\":[${List(501) { "\"x\"" }.joinToString(",")}]}").status)
+        assertEquals(HttpStatusCode.PayloadTooLarge, lookup("x".repeat(1024 * 1024 + 1)).status)
+    }
+
+    private suspend fun ApplicationTestBuilder.get(url: String): io.ktor.client.statement.HttpResponse = client.get(url) {
+        header(HttpHeaders.Authorization, "Bearer test-jwt")
+    }
+
+    private suspend fun ApplicationTestBuilder.lookup(body: String): io.ktor.client.statement.HttpResponse = client.post("$path/lookup") {
+        header(HttpHeaders.Authorization, "Bearer test-jwt")
+        header(HttpHeaders.ContentType, "application/json")
+        setBody(body)
+    }
+}

server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt

@@ -18,19 +18,29 @@ import org.jetbrains.exposed.v1.jdbc.update
 
 object SubscriptionAvatarRepairer {
     fun repair(userId: String, items: List<SubscriptionItem>): List<SubscriptionItem> {
+        val avatars = recoverableAvatars(userId, items)
+        avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) }
+        return items.withAvatars(avatars)
+    }
+
+    fun resolve(userId: String, items: List<SubscriptionItem>): List<SubscriptionItem> =
+        items.withAvatars(recoverableAvatars(userId, items))
+
+    private fun recoverableAvatars(userId: String, items: List<SubscriptionItem>): Map<String, String> {
         val candidateUrls = items.filter { it.avatarUrl.isBlank() }
             .map { it.channelUrl }
             .distinct()
-        if (candidateUrls.isEmpty()) return items
+        if (candidateUrls.isEmpty()) return emptyMap()
         val avatars = knownAvatars(userId = userId, channelUrls = candidateUrls)
             .entries
             .take(MAX_AVATAR_REPAIR_PER_REQUEST)
             .associate { it.toPair() }
-        if (avatars.isEmpty()) return items
-        avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) }
-        return items.map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item }
+        return avatars
     }
 
+    private fun List<SubscriptionItem>.withAvatars(avatars: Map<String, String>): List<SubscriptionItem> =
+        if (avatars.isEmpty()) this else map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item }
+
     private fun knownAvatars(userId: String, channelUrls: List<String>): Map<String, String> {
         val avatars = linkedMapOf<String, String>()
         historyAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) }