Cohorts from long id lists, locked cohorts, and saving cohorts by name

CLAUDE.md

@@ -200,6 +200,22 @@ ClickHouse SQL; clients never send SQL. Live at https://cbiolab.vercel.app (UI a
   enabled only where the deployment signs people in, each dataset its saver's. A document pins
   what it uses as `uploads: [{as, dataset, version}]` (added by the page on run); requests send
   `customRefs` and the server loads the pinned, immutable version.
+- **Stored id sets** — a cohort given as a long list of ids (`sampleList`/`patientList` with
+  `"set": "<id>"` instead of `"ids"`). **`lib/id-set-store.ts`** is the only module touching its
+  tables (`db/migrations/0006_id_sets.sql`): owned and shared by `lib/access.ts`, members
+  immutable (trigger), soft delete, every read logged. `lib/id-set-import.ts` matches a pasted list
+  to the studies' cases before saving (`POST /api/id-sets`; UI: "+ Add cohort" → "From a list of
+  ids"). Every engine route passes its body as `uses` to `withRequestCustomData`, which reads the
+  sets it names (`lib/id-sets-server.ts`) into **`lib/id-sets.ts`**'s request context; the compiler
+  then names one ClickHouse external table per set and study, and `lib/ch.ts` uploads it only to
+  the patient-level connection holding that study. A set is not read without a signed-in request
+  (MCP, scripts), and an answer that reads one bypasses the result cache (`hasIdSets()`). Inline
+  lists are refused by name past `INLINE_ID_SQL_LIMIT` (`lib/cohort2.ts`). In the web app every
+  list becomes a set where the reader is signed in (`app/lab/id-sets-client.ts`): oncoprint
+  selections and Cell Explorer imports as they enter the document, and any list still inline when a
+  cohort or analysis is saved (the page's `persist`/`persistSpaces` wrappers, which give the open
+  document the same sets so a re-save stores nothing new). Inline lists stay valid DSL for MCP,
+  scripts and signed-out readers.
 - **Signing in** (`lib/auth.ts`, the header's Sign in) guards custom data, saved
   cohorts/analyses (`/api/artifacts`) and campaign changes; exploring, analyses, `?doc=` links and
   MCP need none. A deployment has ONE way in: an OpenID Connect provider (`lib/oidc.ts`;
@@ -311,7 +327,8 @@ ClickHouse SQL; clients never send SQL. Live at https://cbiolab.vercel.app (UI a
   Read before claiming this project is "fast": on a cached single chart the portal is ~20x
   quicker (those measurements predate the result cache of invariant 11, which answers only
   exact repeats), and our advantage is data movement (36x less to the
-  browser), not speed. Also records the ~20,000-id ceiling on explicit patient lists.
+  browser), not speed. Also records the ceiling on inline id lists (~12,000 sample ids) and the
+  stored id sets that remove it.
 
 ## Invariants — do not break these
 

app/api/analysis/method/route.ts

@@ -12,7 +12,7 @@ export const maxDuration = 60;
 export async function POST(req: Request) {
   try {
     const { customData, customRefs, ...doc } = parseBody(AnalysisRequest, await req.json());
-    const result = await withRequestCustomData(customData, () => cachedAnalysis(doc as unknown as AnalysisDoc, { oncoprint: "spec" }), { refs: customRefs, req });
+    const result = await withRequestCustomData(customData, () => cachedAnalysis(doc as unknown as AnalysisDoc, { oncoprint: "spec" }), { refs: customRefs, req, uses: doc });
     const view = result.views[0];
     if (!view || view.error) return Response.json({ error: view ? view.error : "the document has no view" }, { status: 422 });
     return Response.json({ python: view.method?.python ?? null });

app/api/analysis/notebook/route.ts

@@ -31,7 +31,7 @@ export async function POST(req: Request) {
         view, method: view.method, files, missing, doc, viewIndex: notebookView.index,
         origin: process.env.APP_BASE_URL || new URL(req.url).origin,
       });
-    }, { refs: customRefs, req });
+    }, { refs: customRefs, req, uses: doc });
     const name = slug(notebookView.kind + "-" + String((doc as { study?: string }).study || "")) + ".ipynb";
     return new Response(JSON.stringify(nb), {
       headers: { "Content-Type": "application/x-ipynb+json; charset=utf-8", "Content-Disposition": `attachment; filename="${name}"` },

app/api/analysis/route.ts

@@ -14,7 +14,7 @@ export async function POST(req: Request) {
     const url = new URL(req.url);
     // the web app draws its own oncoprints; ?oncoprint=spec skips rendering them here
     const mode = url.searchParams.get("oncoprint") === "spec" ? "spec" as const : "svg" as const;
-    const result = await withRequestCustomData(customData, () => cachedAnalysis(doc as unknown as AnalysisDoc, { oncoprint: mode }), { refs: customRefs, req });
+    const result = await withRequestCustomData(customData, () => cachedAnalysis(doc as unknown as AnalysisDoc, { oncoprint: mode }), { refs: customRefs, req, uses: doc });
     // ?figures=svg: every view drawn by the shared server renderer, for a client that cannot draw
     if (url.searchParams.get("figures") === "svg")
       return Response.json({ ...result, views: result.views.map((v) => v.svg || v.error ? v : { ...v, svg: viewToSvg(v) || undefined }) });

app/api/cell-explorer/route.ts

@@ -11,7 +11,7 @@ export async function POST(req: Request) {
     const body = parseBody(CellExplorerRequest, await req.json());
     if (body.mode === "import") return Response.json(await cohortFromCells(body));
     const { customData, customRefs, doc, cohort } = body;
-    return Response.json(await withRequestCustomData(customData, () => cohortCellsLink({ doc, cohort }), { refs: customRefs, req }));
+    return Response.json(await withRequestCustomData(customData, () => cohortCellsLink({ doc, cohort }), { refs: customRefs, req, uses: doc }));
   } catch (e) {
     return Response.json({ error: String((e as Error)?.message || e) }, { status: 400 });
   }

app/api/custom-data/members/route.ts

@@ -10,7 +10,7 @@ export const maxDuration = 60;
 export async function POST(req: Request) {
   try {
     const { customData, customRefs, def, studies } = parseBody(MembersRequest, await req.json());
-    return Response.json({ studies: await withRequestCustomData(customData, () => cohortMembership(def, studies.map(String)), { refs: customRefs, req }) });
+    return Response.json({ studies: await withRequestCustomData(customData, () => cohortMembership(def, studies.map(String)), { refs: customRefs, req, uses: def }) });
   } catch (e) {
     return Response.json({ error: String((e as Error)?.message || e) }, { status: 400 });
   }

app/api/explore/route.ts

@@ -10,6 +10,7 @@ import { studyProfiles, resolveStudy } from "../../../lib/analysis.ts";
 import { chQuery } from "../../../lib/ch.ts";
 import { withRequestCustomData } from "../../../lib/custom-data-server.ts";
 import { assertCustomDataPresent } from "../../../lib/custom-data.ts";
+import { idSetRefsIn } from "../../../lib/id-sets.ts";
 import type { DerivedRequest } from "../../../lib/types.ts";
 
 export const maxDuration = 60;
@@ -32,9 +33,9 @@ export async function POST(req: Request) {
 async function handle(req: Request) {
   try {
     const { customData, customRefs, ...body } = parseBody(ExploreRequest, await req.json());
-    // facets over uploaded data are the uploader's alone: answered directly, never cached
-    if ((Array.isArray(customData) && customData.length) || (Array.isArray(customRefs) && customRefs.length))
-      return Response.json(await withRequestCustomData(customData, () => ensureChContext(() => exploreFor(body)), { refs: customRefs, req }));
+    // facets over uploaded data or a stored id set are their reader's alone: answered directly, never cached
+    if ((Array.isArray(customData) && customData.length) || (Array.isArray(customRefs) && customRefs.length) || idSetRefsIn(body.draft).length)
+      return Response.json(await withRequestCustomData(customData, () => ensureChContext(() => exploreFor(body)), { refs: customRefs, req, uses: body.draft }));
     return Response.json(await throughCache(async () => exploreEntry(body, await dataVersion()), () => ensureChContext(() => exploreFor(body))));
   } catch (e) {
     // The reader superseded this request: not a failure, and there is nobody left to tell.

app/api/id-sets/[id]/route.ts

@@ -0,0 +1,25 @@
+import { requirePrincipal, AccessError } from "../../../../lib/auth.ts";
+import { getIdSet, readIdSet, deleteIdSet } from "../../../../lib/id-set-store.ts";
+
+const fail = (e: unknown) => Response.json({ error: String((e as Error)?.message || e) }, { status: e instanceof AccessError ? e.status : 400 });
+
+/** One set's description; with `?ids=1`, its ids too, one "<study>:<id>" per line (that read is logged). */
+export async function GET(req: Request, { params }: { params: Promise<{ id: string }> }) {
+  try {
+    const { id } = await params;
+    const principal = requirePrincipal(req, "use stored id sets");
+    if (new URL(req.url).searchParams.get("ids") !== "1") return Response.json({ set: await getIdSet(principal, id) });
+    const { perStudy } = await readIdSet(principal, id);
+    const text = Object.entries(perStudy).flatMap(([study, ids]) => ids.map((x) => study + ":" + x)).join("\n") + "\n";
+    return new Response(text, { headers: { "Content-Type": "text/plain; charset=utf-8", "Cache-Control": "private, no-store" } });
+  } catch (e) { return fail(e); }
+}
+
+/** Delete (its owner only): documents naming it are refused from then on, saying when it went. */
+export async function DELETE(req: Request, { params }: { params: Promise<{ id: string }> }) {
+  try {
+    const { id } = await params;
+    await deleteIdSet(requirePrincipal(req, "use stored id sets"), id);
+    return Response.json({ deleted: true });
+  } catch (e) { return fail(e); }
+}

app/api/id-sets/[id]/sharing/route.ts

@@ -0,0 +1,13 @@
+import { requirePrincipal, AccessError } from "../../../../../lib/auth.ts";
+import { shareIdSet } from "../../../../../lib/id-set-store.ts";
+
+// Who a stored id set is shared with (its owner only; lib/access.ts). Body as for
+// /api/artifacts/[id]/sharing. Its ids are patient-level: nothing shares them but this.
+export async function PUT(req: Request, { params }: { params: Promise<{ id: string }> }) {
+  try {
+    const { id } = await params;
+    return Response.json({ set: await shareIdSet(requirePrincipal(req, "use stored id sets"), id, await req.json().catch(() => null)) });
+  } catch (e) {
+    return Response.json({ error: String((e as Error)?.message || e) }, { status: e instanceof AccessError ? e.status : 400 });
+  }
+}

app/api/id-sets/route.ts

@@ -0,0 +1,27 @@
+import { connection } from "next/server";
+import { requirePrincipal, AccessError } from "../../../lib/auth.ts";
+import { listIdSets } from "../../../lib/id-set-store.ts";
+import { createIdSet } from "../../../lib/id-set-import.ts";
+
+export const maxDuration = 60;
+
+const fail = (e: unknown) => Response.json({ error: String((e as Error)?.message || e) }, { status: e instanceof AccessError ? e.status : 400 });
+
+/** The stored id sets this reader may use, without their ids. */
+export async function GET(req: Request) {
+  // rendered per request: without it the build prerenders whatever the login check answers then
+  await connection();
+  try { return Response.json({ sets: await listIdSets(requirePrincipal(req, "use stored id sets")) }); }
+  catch (e) { return fail(e); }
+}
+
+/**
+ * Save a list of ids as a stored set: `{ name, level, studies, ids }`. The ids are matched to the
+ * studies' own samples or patients first; the answer is the set and what did not match.
+ */
+export async function POST(req: Request) {
+  try {
+    const principal = requirePrincipal(req, "save stored id sets");
+    return Response.json(await createIdSet(principal, await req.json()), { status: 201 });
+  } catch (e) { return fail(e); }
+}

app/api/lock/route.ts

@@ -0,0 +1,22 @@
+import { parseBody, LockRequest } from "../../../lib/api-schemas.ts";
+import { requirePrincipal, AccessError } from "../../../lib/auth.ts";
+import { withRequestCustomData } from "../../../lib/custom-data-server.ts";
+import { lockCohorts } from "../../../lib/cohort-lock-server.ts";
+import { dataVersion } from "../../../lib/next-cache.ts";
+
+export const maxDuration = 60;
+
+/**
+ * Lock cohorts of a document (lib/cohort-lock.ts): `{ doc, cohorts? }`, every cohort when none are
+ * named. Answers the moment, the data version, the OncoKB release and each cohort's lock, for the
+ * page to write into the document. The members are saved as the reader's stored id sets.
+ */
+export async function POST(req: Request) {
+  try {
+    const principal = requirePrincipal(req, "lock cohorts");
+    const { customData, customRefs, doc, cohorts } = parseBody(LockRequest, await req.json());
+    return Response.json(await withRequestCustomData(customData, () => lockCohorts(principal, { doc, cohorts }, dataVersion), { refs: customRefs, req, uses: doc }));
+  } catch (e) {
+    return Response.json({ error: String((e as Error)?.message || e) }, { status: e instanceof AccessError ? e.status : 400 });
+  }
+}

app/api/oncoprint/route.ts

@@ -19,7 +19,7 @@ export async function POST(req: Request) {
     const sp = new URL(req.url).searchParams;
     // a provenance request is answered with the queries that built the (cached) contract
     const provenance = sp.get("provenance") === "1";
-    const contract = await withRequestCustomData(customData, () => cachedContract(spec, { provenance }), { refs: customRefs, req });
+    const contract = await withRequestCustomData(customData, () => cachedContract(spec, { provenance }), { refs: customRefs, req, uses: spec });
     const engineMs = Date.now() - t0;
     const fmt = sp.get("format");
     if (fmt === "svg" || fmt === "tsv") {

app/api/partition/route.ts

@@ -9,7 +9,7 @@ export const maxDuration = 60;
 export async function POST(req: Request) {
   try {
     const { customData, customRefs, ...body } = parseBody(PartitionRequest, await req.json());
-    return Response.json(await withRequestCustomData(customData, () => proposePartition(body as unknown as Parameters<typeof proposePartition>[0]), { refs: customRefs, req }));
+    return Response.json(await withRequestCustomData(customData, () => proposePartition(body as unknown as Parameters<typeof proposePartition>[0]), { refs: customRefs, req, uses: body }));
   } catch (e) {
     return Response.json({ error: String((e as Error)?.message || e) }, { status: 400 });
   }

app/api/table/route.ts

@@ -33,7 +33,7 @@ export async function POST(req: Request) {
   } catch (e) {
     return new Response("cannot download: " + String((e as Error)?.message || e) + "\n", { status: 400, headers: { "content-type": "text/plain; charset=utf-8" } });
   }
-  const run = <T,>(fn: () => Promise<T>) => withRequestCustomData(body.customData, fn, { refs: body.customRefs, req });
+  const run = <T,>(fn: () => Promise<T>) => withRequestCustomData(body.customData, fn, { refs: body.customRefs, req, uses: body });
   const table = typeof body.table === "string" ? body.table : undefined;
   const first = await run(() => runAnalysis(doc, { table })).catch((e) => ({ views: [{ kind: "", cohorts: [], error: String((e as Error)?.message || e) } as ViewResult] }));
   const v = first.views[0];

app/api/view-page/route.ts

@@ -23,7 +23,7 @@ export async function POST(req: Request) {
     const view = Array.isArray(doc.views) ? doc.views[vi] : null;
     if (!view || !PAGED_KINDS.includes(view.kind)) throw new Error("view " + body.view + " is not a table view (" + PAGED_KINDS.join(", ") + ")");
     const one = { ...doc, views: [view] };
-    const run = (opts: Parameters<typeof runAnalysis>[1]) => withRequestCustomData(customData, () => runAnalysis(one, opts), { refs: customRefs, req });
+    const run = (opts: Parameters<typeof runAnalysis>[1]) => withRequestCustomData(customData, () => runAnalysis(one, opts), { refs: customRefs, req, uses: one });
     const pick = (r: Awaited<ReturnType<typeof runAnalysis>>) => {
       const v = r.views[0];
       const t = pagedTable(v, body.table);