From 1ef2d608cd13d0fb6ce6e6545b00da241c26ed26 Mon Sep 17 00:00:00 2001 From: ael-dev3 <183124839+ael-dev3@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:15:59 +0100 Subject: [PATCH 1/3] feat(library): sync reading history and saved posts --- .github/workflows/ci.yml | 2 +- AGENTS.md | 2 +- CHANGELOG.md | 5 + README.md | 7 +- crates/allthing-core/src/api.rs | 126 +- crates/allthing-core/src/following.rs | 27 + crates/allthing-core/src/lib.rs | 1 + crates/allthing-core/src/library.rs | 589 ++++++ crates/allthing-core/src/seen.rs | 10 + crates/allthing-data/src/hypersnap.rs | 134 +- crates/allthing-data/src/lib.rs | 53 + crates/allthing-data/src/pages.rs | 266 ++- crates/allthing-web/Cargo.toml | 2 + .../allthing-web/src/components/bookmark.rs | 76 + crates/allthing-web/src/components/cast.rs | 49 +- crates/allthing-web/src/components/chips.rs | 8 +- crates/allthing-web/src/components/layout.rs | 11 +- crates/allthing-web/src/components/mod.rs | 1 + crates/allthing-web/src/components/signin.rs | 1 + crates/allthing-web/src/lib.rs | 7 + crates/allthing-web/src/library.rs | 1657 +++++++++++++++++ crates/allthing-web/src/library_disk.rs | 617 ++++++ crates/allthing-web/src/notification_sync.rs | 73 +- crates/allthing-web/src/pages/home.rs | 20 +- crates/allthing-web/src/pages/library.rs | 585 ++++++ crates/allthing-web/src/pages/mod.rs | 1 + crates/allthing-web/src/seen.rs | 246 ++- crates/allthing-web/src/split.rs | 6 +- crates/allthing-web/src/util.rs | 5 + crates/allthing-worker/src/lib.rs | 4 +- docs/ARCHITECTURE.md | 18 +- docs/BUDGET.md | 6 +- e2e/early-paint.fixture.spec.ts | 70 +- e2e/feed-tabs.fixture.spec.ts | 12 +- e2e/fixtures/library-sync-wire.ts | 52 + e2e/following-feed.fixture.spec.ts | 27 +- e2e/following-navigation.fixture.spec.ts | 7 +- e2e/helpers/app.ts | 3 +- e2e/library-sync.fixture.spec.ts | 616 ++++++ e2e/library.fixture.spec.ts | 218 +++ e2e/like-recast.fixture.spec.ts | 2 +- e2e/presence.fixture.spec.ts | 2 + e2e/stdb-attest.fixture.spec.ts | 2 +- e2e/tap-targets.fixture.spec.ts | 5 +- spacetime/HANDOFF.md | 13 +- spacetime/spacetimedb/src/lib.rs | 1 + spacetime/spacetimedb/src/library.rs | 549 ++++++ style/allthing.css | 115 ++ tests/stdb-bridge.test.mjs | 114 ++ web/_headers | 2 +- web/index.html | 4 +- web/paint.js | 6 +- web/stdb/bridge.ts | 66 + .../module_bindings/clear_seen_reducer.ts | 17 + .../module_bindings/forget_seen_reducer.ts | 18 + web/stdb/module_bindings/index.ts | 54 + .../mark_seen_batch_reducer.ts | 23 + .../my_library_membership_table.ts | 25 + .../module_bindings/my_library_page_table.ts | 25 + .../module_bindings/my_library_state_table.ts | 22 + .../module_bindings/set_bookmark_reducer.ts | 18 + web/stdb/module_bindings/types.ts | 48 + web/stdb/module_bindings/types/reducers.ts | 8 + xtask/src/main.rs | 2 +- 64 files changed, 6647 insertions(+), 114 deletions(-) create mode 100644 crates/allthing-core/src/library.rs create mode 100644 crates/allthing-web/src/components/bookmark.rs create mode 100644 crates/allthing-web/src/library.rs create mode 100644 crates/allthing-web/src/library_disk.rs create mode 100644 crates/allthing-web/src/pages/library.rs create mode 100644 e2e/fixtures/library-sync-wire.ts create mode 100644 e2e/library-sync.fixture.spec.ts create mode 100644 e2e/library.fixture.spec.ts create mode 100644 spacetime/spacetimedb/src/library.rs create mode 100644 tests/stdb-bridge.test.mjs create mode 100644 web/stdb/module_bindings/clear_seen_reducer.ts create mode 100644 web/stdb/module_bindings/forget_seen_reducer.ts create mode 100644 web/stdb/module_bindings/mark_seen_batch_reducer.ts create mode 100644 web/stdb/module_bindings/my_library_membership_table.ts create mode 100644 web/stdb/module_bindings/my_library_page_table.ts create mode 100644 web/stdb/module_bindings/my_library_state_table.ts create mode 100644 web/stdb/module_bindings/set_bookmark_reducer.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6957dc4f..5a95334f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,7 +159,7 @@ jobs: - name: Test the static copy run: | bash scripts/build-pages.sh - E2E_PAGES=1 npx playwright test e2e/static-body-limit.fixture.spec.ts e2e/static-metadata.fixture.spec.ts e2e/static-navigation.fixture.spec.ts e2e/static-follows.fixture.spec.ts e2e/preloaded-replies.fixture.spec.ts e2e/poll.fixture.spec.ts --output=test-results/static + E2E_PAGES=1 npx playwright test e2e/static-body-limit.fixture.spec.ts e2e/static-metadata.fixture.spec.ts e2e/static-navigation.fixture.spec.ts e2e/static-follows.fixture.spec.ts e2e/preloaded-replies.fixture.spec.ts e2e/poll.fixture.spec.ts e2e/library.fixture.spec.ts --output=test-results/static env: PLAYWRIGHT_HTML_OUTPUT_DIR: playwright-report/static - name: Upload browser failure reports diff --git a/AGENTS.md b/AGENTS.md index 054e9f14..ef19748b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Allthing is a Farcaster client for reading, written in Rust: a Leptos WebAssembl - Payloads go through `cache::serve`. - Never await another request's I/O; see the coalescing notes in `cache.rs`. - Update `docs/BUDGET.md` when costs change. -- **Links.** The app's routes are `/`, `/c/`, `/p/`, `/u/`, `/u//followers`, `/u//following`, `/channels`, `/search?q=`, `/dau`, `/scheduled`, `/messages` (`?c=`, `?g=`, `?new=1`) and `/notifications` (`?tab=mentions` or `replies`). Build them with the helpers in `crates/allthing-web/src/util.rs` (`href`, `channel_href`, …), never as literal strings, and go to one from code with `util::use_go`, not the router's `use_navigate`, which puts the base before it again: the GitHub Pages build lives under `/Allthing`. +- **Links.** The app's routes are `/`, `/c/`, `/p/`, `/u/`, `/u//followers`, `/u//following`, `/channels`, `/search?q=`, `/dau`, `/scheduled`, `/messages` (`?c=`, `?g=`, `?new=1`), `/notifications` (`?tab=mentions` or `replies`), and `/library` (`?tab=bookmarks|likes|recasts|seen`, optional `q` filters loaded posts). Build them with the helpers in `crates/allthing-web/src/util.rs` (`href`, `channel_href`, …), never as literal strings, and go to one from code with `util::use_go`, not the router's `use_navigate`, which puts the base before it again: the GitHub Pages build lives under `/Allthing`. - **Licensing.** Licensing is AGPL-3.0-or-later. Start new files with an SPDX header. New dependencies must pass `cargo deny check licenses`. - **Secrets.** Never commit secrets, tokens, `.dev.vars` or `.env` files, or session URLs. The Worker's secrets (`SESSION_SECRET`, `SOLANA_RPC_URL`) are set in Cloudflare; see `docs/DEPLOY.md#secrets`. - **Checks.** diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bbb6e2d..30ea982a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,15 @@ All notable changes to Allthing are recorded here. The format follows [Keep a Ch ### Added +- A private Library for bookmarks and seen history synced across signed-in devices through SpacetimeDB, plus paginated Farcaster likes and recasts. Seen history no longer expires after 30 days or 2,000 posts; fresh Home candidates wait for synced history, while already drawn cards keep their place. Library includes loaded-post filtering, activity times, retry, Back restoration, Mark unread and Clear history controls. - Sync notification read progress across signed-in sessions with a private, attested SpacetimeDB watermark. The existing local cache works offline; a delayed background connection uses separate credentials without creating chat presence or DM keys. - Videos up to 250 MB upload in 8 MB chunks. Retry continues from the last acknowledged chunk; Remove cancels the pending upload. Preparation and finishing have visible states, and the cast stays disabled until its attachments are ready. - Compatible MP4 and MOV videos are prepared for HLS from bounded metadata and file slices, preserving their frames and sound without copying the entire video into WebAssembly. Existing image limits and storage safeguards remain. +### Changed + +- Following opens with New selected; For you still opens with Hot. Explicit sort choices remain selected when changing the time range. + ### Fixed - After Escape closes a hover card, layout changes cannot reopen it under a stationary pointer. Moving away and back still opens it normally. diff --git a/README.md b/README.md index e7312fdf..c3432555 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ Allthing is an independent Farcaster client, written in Rust, that shows trendin ### Reading, no account needed -- **Feeds.** A front page sorted **hot**, **new** or **top**, with a time filter in every sort. Signed in, **For you** lifts casts from recently active accounts you follow, while **Following** shows their casts only. New stays chronological. Channel feeds, a channel directory and one search box cover casts, people and channels; the client bar filters casts by the app they came from. -- **Seen posts.** Front-page feeds leave out posts you already read on a later visit, with Show seen posts and Reset seen posts to bring them back. History stays in this browser for 30 days, separate for each reader; cards stay put while you read. +- **Feeds.** A front page sorted **hot**, **new** or **top**, with a time filter in every sort. Signed in, **For you** defaults to **hot** and lifts casts from recently active accounts you follow, while **Following** defaults to **new** and shows their casts only. New stays chronological. Channel feeds, a channel directory and one search box cover casts, people and channels; the client bar filters casts by the app they came from. +- **Seen posts.** Home leaves out posts you already read on future visits. Signed-in history stays private and syncs across devices through SpacetimeDB without a 30-day expiry or 2,000-post limit. Cards stay put while you read; anonymous history stays in this browser for 30 days. - **Who is here.** An online count beside Home, including on phones. Signed-in readers can choose whether their name appears. - **Posts and threads.** A post with its parent, quoted casts (inline up to three levels) and threaded replies that collapse and expand. - **Honest engagement.** Likes, recasts and replies, the only counts Farcaster has. Hovering or tapping a count lists who liked, recasted or replied. @@ -50,6 +50,7 @@ Allthing is an independent Farcaster client, written in Rust, that shows trendin - **Sign in by one QR code.** Approve Allthing once in the Farcaster app (a QR code on a computer, a button on a phone). That one approval signs you in and gives your browser a key to write with; the key never leaves it. - **Write from Allthing.** Cast, reply, quote, like, recast, follow, delete your own casts and [schedule casts](docs/ARCHITECTURE.md#scheduled-casts) (listed at /scheduled), shown "via Allthing". Each is a Farcaster message signed in your browser and sent to Hypersnap; Allthing asks for keys as its own account, @allthingapp. - **A composer like the Farcaster app's.** Images and videos (PNG, JPEG, GIF, WebP; MP4, MOV, WebM) by the attach button, paste or drop, with photo location data stripped in the browser; @mention autocomplete; a Pepe emoji picker, animated Pepe stickers and GIFs (from [KLIPY](https://klipy.com)); drafts that last; a sheet on the keyboard on a phone. +- **Library.** Bookmarks, Likes, Recasts and Seen, with paginated history, activity times, a filter for loaded posts and Back restoration. Save posts privately without a Farcaster signing key; mark posts unread or clear reading history across devices while keeping bookmarks. - **Notifications.** A bell with what's new, and /notifications with likes, recasts, replies, mentions, quotes and follows, grouped as the Farcaster app groups them. - **Messages.** One inbox on SpacetimeDB at /messages: Allthing's public `#general` room and end-to-end encrypted conversations ([docs/ARCHITECTURE.md](docs/ARCHITECTURE.md#messages-on-spacetimedb)). - **Signed out**, a write opens the cast, the profile or a ready draft in the Farcaster app instead. @@ -57,7 +58,7 @@ Allthing is an independent Farcaster client, written in Rust, that shows trendin ### Fast and frugal - **One API call per page**, cached at the edge, so the whole site runs on the Cloudflare Workers Free plan. -- **Early paint.** A small JavaScript renderer draws the front page's first cards from its existing response while the Rust app downloads. It stays in JavaScript because it beat the Rust prototype in the [paired cold-load comparison](docs/PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer). +- **Early paint.** For anonymous readers, a small JavaScript renderer draws the front page's first cards from its existing response while the Rust app downloads. Signed-in Home waits for private history membership to avoid flashing previously read posts. It stays in JavaScript because it beat the Rust prototype in the [paired cold-load comparison](docs/PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer). - **Split WebAssembly.** Pages, the composer, writing, sign-in and the DAU dashboard are code of their own, fetched when used, so every visit downloads less. - **Instant navigation.** Posts open from the feed's own data, cards are prefetched within a capped budget, and Back returns to the same place. CI holds smoothness to budgets ([docs/PERFORMANCE.md](docs/PERFORMANCE.md)). - **Accessible.** Keyboard throughout, alt text, focus kept in dialogs, reduced motion respected, and it fits a phone. diff --git a/crates/allthing-core/src/api.rs b/crates/allthing-core/src/api.rs index 808f39c9..c408ec90 100644 --- a/crates/allthing-core/src/api.rs +++ b/crates/allthing-core/src/api.rs @@ -15,7 +15,7 @@ use crate::latest::Cursor; use crate::model::{Cast, Channel, ThreadNode, User, UserSummary}; use crate::params::{channel_key_from_param, hash_from_param, percent_decode, percent_encode, username_from_param}; use crate::profile::UserCard; -use crate::provenance::{Backend, Origin}; +use crate::provenance::{Backend, Origin, base64_decode}; use crate::ranking::{Sort, TimeRange, TrendingWindow}; use crate::website::Website; @@ -242,6 +242,100 @@ impl ReactionKind { } } +/// Library reads are small batches, independent of the account's history size. +pub const LIBRARY_PAGE_SIZE: usize = 25; +pub const LIBRARY_PATH: &str = "/api/v1/library"; +pub const LIBRARY_CASTS_PATH: &str = "/api/v1/library/casts"; + +/// An active like/recast and the cast it targets. `at` is the reaction's +/// Unix time, not the cast's publication time. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LibraryEntry { + pub hash: String, + pub author_fid: u64, + pub at: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LibraryPayload { + pub fid: u64, + pub kind: ReactionKind, + pub entries: Vec, + /// Available casts in reaction order, with mini apps marked for settings. + pub pool: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, + /// Omitted by a successful provider lookup; this does not assert deletion. + #[serde(default)] + pub unavailable: Vec, + /// Hydrated, but omitted by the app's content rules, not by the provider. + #[serde(default)] + pub filtered: Vec, + #[serde(default)] + pub notes: Annotations, + pub meta: Meta, +} + +/// Public cast hydration for a private list. This endpoint knows only the +/// requested public cast hashes, not whether they were bookmarked or seen. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LibraryCastsPayload { + pub pool: Vec, + #[serde(default)] + pub unavailable: Vec, + #[serde(default)] + pub filtered: Vec, + #[serde(default)] + pub notes: Annotations, + pub meta: Meta, +} + +/// An opaque hub key: validate its bounded encoding, never reinterpret it as +/// a multi-shard cursor. An account's outgoing reactions live in one shard. +pub fn library_cursor(value: &str) -> Option { + if value.is_empty() + || value.len() > 1024 + || !value.bytes().all(|b| b.is_ascii_alphanumeric() || b"+/=".contains(&b)) + { + return None; + } + let bytes = base64_decode(value)?; + (!bytes.is_empty() && bytes.len() <= 512).then(|| value.to_owned()) +} + +/// Canonical, unique hashes in the caller's order. Never silently truncate a +/// request: the limit applies to its raw slots too, keeping URLs bounded. +pub fn parse_library_hashes(value: Option<&str>) -> Option> { + let value = value?; + if value.is_empty() || value.len() > LIBRARY_PAGE_SIZE * 45 { + return None; + } + let mut hashes = Vec::new(); + let mut seen = HashSet::new(); + for (index, value) in value.split(',').enumerate() { + if index >= LIBRARY_PAGE_SIZE { + return None; + } + let hash = hash_from_param(value)?; + if seen.insert(hash.clone()) { + hashes.push(hash); + } + } + (!hashes.is_empty()).then_some(hashes) +} + +pub fn library_path(fid: u64, kind: ReactionKind, cursor: Option<&str>) -> String { + let mut path = format!("{LIBRARY_PATH}?fid={fid}&kind={}", kind.as_str()); + if let Some(cursor) = cursor { + path.push_str(&format!("&cursor={}", percent_encode(cursor))); + } + path +} + +pub fn library_casts_path(hashes: &[String]) -> String { + format!("{LIBRARY_CASTS_PATH}?hashes={}", hashes.join(",")) +} + /// Who liked or recasted a cast, newest first. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ReactionsPayload { @@ -978,6 +1072,8 @@ pub const WALLET_RETRY_CACHE: CachePolicy = CachePolicy { fresh: 300, stale: 600 pub const CHANNELS_CACHE: CachePolicy = CachePolicy { fresh: 3600, stale: 86_400, keep: 172_800, browser: 600 }; pub const SEARCH_CACHE: CachePolicy = CachePolicy { fresh: 300, stale: 1800, keep: 86_400, browser: 120 }; pub const REACTIONS_CACHE: CachePolicy = CachePolicy { fresh: 300, stale: 3600, keep: 86_400, browser: 300 }; +/// Short freshness for lists changed by writes in other Farcaster clients. +pub const LIBRARY_CACHE: CachePolicy = CachePolicy { fresh: 60, stale: 300, keep: 3600, browser: 0 }; /// A batch of like, recast and replies previews: as fresh as a like or /// recast preview, since that is what most of them become in the browser. /// Its replies previews may be a few minutes older than `/replies`' own, @@ -1011,6 +1107,34 @@ pub const MISS_CACHE: CachePolicy = CachePolicy { fresh: 300, stale: 300, keep: mod tests { use super::*; + #[test] + fn library_batches_preserve_order_without_silent_truncation() { + let (a, b) = (format!("0x{:040x}", 1), format!("0x{:040x}", 2)); + assert_eq!( + parse_library_hashes(Some(&format!("{b},{},{b}", a.to_uppercase()))), + Some(vec![b.clone(), a.clone()]) + ); + let batch = vec![a; LIBRARY_PAGE_SIZE]; + assert!(parse_library_hashes(Some(&batch.join(","))).is_some()); + assert!(parse_library_hashes(Some(&[batch, vec![b]].concat().join(","))).is_none()); + for value in [None, Some(""), Some("0x12"), Some("https://example.org"), Some(",")] { + assert!(parse_library_hashes(value).is_none()); + } + } + + #[test] + fn library_cursors_are_bounded_opaque_bytes_and_escaped_once() { + let token = "+/8="; + assert_eq!(library_cursor(token).as_deref(), Some(token)); + assert_eq!( + library_path(7, ReactionKind::Likes, Some(token)), + "/api/v1/library?fid=7&kind=likes&cursor=%2B%2F8%3D" + ); + for token in ["", "%%%", "a b", "\n", &"A".repeat(1025)] { + assert!(library_cursor(token).is_none()); + } + } + #[test] fn pages_read_what_they_ask_for() { let hash = format!("0x{}", "ab".repeat(20)); diff --git a/crates/allthing-core/src/following.rs b/crates/allthing-core/src/following.rs index 001ec7ca..c6970aa3 100644 --- a/crates/allthing-core/src/following.rs +++ b/crates/allthing-core/src/following.rs @@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize}; use crate::api::{API_PREFIX, Annotations, CachePolicy, Meta}; use crate::model::Cast; +use crate::ranking::Sort; /// Where the app reads it. pub const FOLLOWING_PATH: &str = "/api/v1/following"; @@ -36,6 +37,19 @@ pub enum Feed { } impl Feed { + /// Following starts chronologically; For you starts with its hot ranking. + pub fn default_sort(self) -> Sort { + match self { + Feed::ForYou => Sort::Hot, + Feed::Following => Sort::New, + } + } + + /// An explicit supported sort always wins over the tab's default. + pub fn sort(self, param: Option<&str>) -> Sort { + Sort::parse(param, self.default_sort()) + } + /// The tab a reader sees for the page's `feed` query: Following only /// signed in and asked for; signed out, For you is the only one. pub fn of(signed_in: bool, param: Option<&str>) -> Feed { @@ -162,6 +176,19 @@ mod tests { assert_eq!(Feed::ForYou.param(), None); } + #[test] + fn following_starts_new_while_explicit_sorts_and_signed_out_defaults_are_preserved() { + assert_eq!(Feed::Following.sort(None), Sort::New); + assert_eq!(Feed::Following.sort(Some("unsupported")), Sort::New); + assert_eq!(Feed::ForYou.sort(None), Sort::Hot); + assert_eq!(Feed::of(false, Some(FOLLOWING)).sort(None), Sort::Hot); + for feed in [Feed::ForYou, Feed::Following] { + for sort in Sort::ALL { + assert_eq!(feed.sort(Some(sort.as_str())), sort); + } + } + } + #[test] fn followed_authors_are_the_pools() { let mut b = cast("0xb", None); diff --git a/crates/allthing-core/src/lib.rs b/crates/allthing-core/src/lib.rs index e4024af4..2f08f106 100644 --- a/crates/allthing-core/src/lib.rs +++ b/crates/allthing-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod guard; pub mod hls; pub mod index; pub mod latest; +pub mod library; pub mod listing; pub mod media; pub mod message; diff --git a/crates/allthing-core/src/library.rs b/crates/allthing-core/src/library.rs new file mode 100644 index 00000000..07019f9a --- /dev/null +++ b/crates/allthing-core/src/library.rs @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Private Library state and durable offline intent. The server owns revisions; +//! a disconnected reader cannot turn an old bookmark/read back on after a reset. + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Account { + pub fid: u64, + pub generation: u64, + pub revision: u64, + pub seen_count: u64, + pub bookmark_count: u64, + pub cleared_at: i64, + pub last_clear_id: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Item { + pub fid: u64, + pub generation: u64, + pub hash: String, + pub seen: bool, + pub seen_at: i64, + pub seen_revision: u64, + pub bookmarked: bool, + pub bookmark_at: i64, + pub bookmark_revision: u64, + #[serde(default)] + pub cursor: String, +} + +pub fn canonical_hash(value: &str) -> Option { + (value.len() == 42 && value.starts_with("0x") && value[2..].bytes().all(|b| b.is_ascii_hexdigit())) + .then(|| value.to_ascii_lowercase()) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum ReadIntent { + Seen { at: i64 }, + Unread, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PendingRead { + pub generation: u64, + pub expected_revision: u64, + pub intent: ReadIntent, + /// Distinguishes a newer local action from the write being acknowledged. + pub sequence: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PendingBookmark { + pub desired: bool, + pub expected_revision: u64, + pub sequence: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct CachedItem { + pub server: Item, + /// False until a caller-scoped membership/page response established this row. + pub known: bool, + pub read: Option, + pub bookmark: Option, + #[serde(default)] + pub read_settled: u64, + #[serde(default)] + pub bookmark_settled: u64, +} + +impl CachedItem { + fn accepts(&self, row: &Item, account: &Account) -> bool { + row.fid == account.fid + && canonical_hash(&row.hash).as_ref() == Some(&row.hash) + && ((!self.known && self.server.hash.is_empty()) + || (self.server.fid == row.fid && self.server.hash == row.hash)) + } + + pub fn seen(&self, generation: u64) -> bool { + match self.read.as_ref().filter(|read| read.generation == generation) { + Some(PendingRead { intent: ReadIntent::Seen { .. }, .. }) => true, + Some(PendingRead { intent: ReadIntent::Unread, .. }) => false, + None => self.known && self.server.generation == generation && self.server.seen, + } + } + + pub fn bookmarked(&self) -> Option { + self.bookmark.as_ref().map(|intent| intent.desired).or_else(|| self.known.then_some(self.server.bookmarked)) + } + + /// Merge only newer authoritative state, independently for read and bookmark + /// revisions. View caches can temporarily contain overlapping argument sets. + pub fn observe(&mut self, row: Item, account: &Account) { + if !self.accepts(&row, account) { + return; + } + if !self.known { + self.server = row.clone(); + self.known = true; + } + if row.bookmark_revision >= self.server.bookmark_revision { + self.server.bookmarked = row.bookmarked; + self.server.bookmark_at = row.bookmark_at; + self.server.bookmark_revision = row.bookmark_revision; + } + if row.generation == account.generation + && row.generation >= self.server.generation + && (self.server.generation != row.generation || row.seen_revision >= self.server.seen_revision) + { + self.server.generation = row.generation; + self.server.seen = row.seen; + self.server.seen_at = row.seen_at; + self.server.seen_revision = row.seen_revision; + } + self.reconcile(account); + } + + /// Apply a fresh row only after this exact sent operation succeeded. A + /// newer local action may have replaced it while the reducer was in flight. + /// Rebase that action only across our own single revision, never across an + /// unrelated server change or a clear. The controller must defer ordinary + /// observations for an in-flight item until its outcome is known; otherwise + /// `observe` correctly treats an unexplained revision as a remote conflict. + pub fn accepted_read(&mut self, sent: &PendingRead, row: Item, account: &Account) { + if !self.accepts(&row, account) { + return; + } + let own_revision = row.seen_revision == sent.expected_revision + || sent.expected_revision.checked_add(1) == Some(row.seen_revision); + let matches = match sent.intent { + ReadIntent::Seen { at } => row.seen && (row.seen_revision == sent.expected_revision || row.seen_at <= at), + ReadIntent::Unread => !row.seen, + }; + if own_revision + && matches + && sent.generation == account.generation + && row.generation == account.generation + && let Some(pending) = self.read.as_mut() + && pending.sequence > sent.sequence + && pending.generation == sent.generation + && pending.expected_revision == sent.expected_revision + { + pending.expected_revision = row.seen_revision; + } + self.acknowledged_read(sent.sequence); + self.observe(row, account); + } + + /// Bookmark counterpart of `accepted_read`. A successful same-value no-op + /// is safe too, provided no intervening revision has changed the row. + pub fn accepted_bookmark(&mut self, sent: &PendingBookmark, row: Item, account: &Account) { + if !self.accepts(&row, account) { + return; + } + let own_revision = row.bookmark_revision == sent.expected_revision + || sent.expected_revision.checked_add(1) == Some(row.bookmark_revision); + if own_revision + && row.bookmarked == sent.desired + && let Some(pending) = self.bookmark.as_mut() + && pending.sequence > sent.sequence + && pending.expected_revision == sent.expected_revision + { + pending.expected_revision = row.bookmark_revision; + } + self.acknowledged_bookmark(sent.sequence); + self.observe(row, account); + } + + pub fn reconcile(&mut self, account: &Account) { + let generation = if self.known { account.generation.max(self.server.generation) } else { account.generation }; + if self.read.as_ref().is_some_and(|read| { + read.generation != generation + || (self.known + && self.server.generation == generation + && self.server.seen_revision > read.expected_revision) + || (account.generation == generation + && matches!(read.intent, ReadIntent::Seen { at } if at <= account.cleared_at)) + }) && let Some(read) = self.read.take() + { + self.read_settled = self.read_settled.max(read.sequence); + } + // A later server operation wins over disconnected intent. An identical + // value is already complete; a different one is a conflict, never replayed. + if self.bookmark.as_ref().is_some_and(|intent| { + self.known + && (self.server.bookmark_revision > intent.expected_revision + || self.server.bookmarked == intent.desired) + }) && let Some(intent) = self.bookmark.take() + { + self.bookmark_settled = self.bookmark_settled.max(intent.sequence); + } + } + + pub fn acknowledged_read(&mut self, sequence: u64) { + self.read_settled = self.read_settled.max(sequence); + if self.read.as_ref().is_some_and(|read| read.sequence <= sequence) { + self.read = None; + } + } + + pub fn acknowledged_bookmark(&mut self, sequence: u64) { + self.bookmark_settled = self.bookmark_settled.max(sequence); + if self.bookmark.as_ref().is_some_and(|intent| intent.sequence <= sequence) { + self.bookmark = None; + } + } + + pub fn pending(&self) -> bool { + self.read.is_some() || self.bookmark.is_some() + } + + /// Merge inside an IndexedDB read/write transaction. This also protects + /// against another tab committing an older snapshot after an acknowledgement. + pub fn merge_disk(&mut self, incoming: &Self) { + self.merge_disk_preserving_inflight(incoming, None, None); + } + + /// Preserve a queued successor while an older operation is still in flight. + /// The sender must finish with `accepted_read`/`accepted_bookmark` and a fresh + /// row, or observe the authoritative failure state. This protection never + /// overrides explicit settlement or a history-generation reset. + pub fn merge_disk_preserving_inflight( + &mut self, + incoming: &Self, + read_sequence: Option, + bookmark_sequence: Option, + ) { + if canonical_hash(&incoming.server.hash).as_ref() != Some(&incoming.server.hash) { + return; + } + if self.server.hash.is_empty() { + self.server = incoming.server.clone(); + } + if self.server.fid != incoming.server.fid || self.server.hash != incoming.server.hash { + return; + } + if incoming.known { + if !self.known || incoming.server.bookmark_revision >= self.server.bookmark_revision { + self.server.bookmarked = incoming.server.bookmarked; + self.server.bookmark_at = incoming.server.bookmark_at; + self.server.bookmark_revision = incoming.server.bookmark_revision; + } + if !self.known + || incoming.server.generation > self.server.generation + || (incoming.server.generation == self.server.generation + && incoming.server.seen_revision >= self.server.seen_revision) + { + self.server.generation = incoming.server.generation; + self.server.seen = incoming.server.seen; + self.server.seen_at = incoming.server.seen_at; + self.server.seen_revision = incoming.server.seen_revision; + } + self.known = true; + } + self.read_settled = self.read_settled.max(incoming.read_settled); + self.bookmark_settled = self.bookmark_settled.max(incoming.bookmark_settled); + self.read = self + .read + .iter() + .chain(incoming.read.iter()) + .filter(|intent| intent.sequence > self.read_settled) + .max_by_key(|intent| (intent.sequence, intent.expected_revision)) + .cloned(); + self.bookmark = self + .bookmark + .iter() + .chain(incoming.bookmark.iter()) + .filter(|intent| intent.sequence > self.bookmark_settled) + .max_by_key(|intent| (intent.sequence, intent.expected_revision)) + .cloned(); + if self.known { + if let Some(read) = self.read.as_ref() + && (read.generation < self.server.generation + || (read.generation == self.server.generation + && read.expected_revision < self.server.seen_revision + && !read_sequence.is_some_and(|sent| read.sequence >= sent))) + { + self.acknowledged_read(read.sequence); + } + if let Some(bookmark) = self.bookmark.as_ref() + && bookmark.expected_revision < self.server.bookmark_revision + && !bookmark_sequence.is_some_and(|sent| bookmark.sequence >= sent) + { + self.acknowledged_bookmark(bookmark.sequence); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn item() -> Item { + Item { fid: 3, hash: format!("0x{:040x}", 9), ..Item::default() } + } + + fn account() -> Account { + Account { fid: 3, ..Account::default() } + } + + #[test] + fn old_membership_cannot_undo_newer_bookmark_or_read_state() { + let mut cache = CachedItem::default(); + let row = Item { + seen: true, + seen_at: 200, + seen_revision: 5, + bookmarked: true, + bookmark_at: 201, + bookmark_revision: 9, + ..item() + }; + cache.observe(row.clone(), &account()); + cache.observe(item(), &account()); + assert_eq!(cache.server, row); + assert_eq!(cache.bookmarked(), Some(true)); + assert!(cache.seen(0)); + } + + #[test] + fn a_remote_clear_fences_all_older_offline_observations() { + let mut cache = CachedItem { + known: true, + server: item(), + read: Some(PendingRead { + generation: 0, + expected_revision: 0, + intent: ReadIntent::Seen { at: 150 }, + sequence: 1, + }), + ..CachedItem::default() + }; + let account = Account { generation: 1, cleared_at: 200, ..account() }; + cache.reconcile(&account); + assert!(cache.read.is_none()); + assert!(!cache.seen(1)); + } + + #[test] + fn per_item_tombstones_reject_stale_offline_reads_and_bookmarks() { + let mut cache = CachedItem { + read: Some(PendingRead { + generation: 0, + expected_revision: 2, + intent: ReadIntent::Seen { at: 150 }, + sequence: 1, + }), + bookmark: Some(PendingBookmark { desired: true, expected_revision: 4, sequence: 2 }), + ..CachedItem::default() + }; + cache.observe(Item { seen_revision: 3, bookmark_revision: 5, ..item() }, &account()); + assert!(!cache.pending()); + assert!(!cache.seen(0)); + assert_eq!(cache.bookmarked(), Some(false)); + } + + #[test] + fn a_lost_acknowledgement_never_removes_a_newer_local_action() { + let mut cache = CachedItem { + read: Some(PendingRead { generation: 0, expected_revision: 0, intent: ReadIntent::Unread, sequence: 2 }), + bookmark: Some(PendingBookmark { desired: false, expected_revision: 0, sequence: 3 }), + ..CachedItem::default() + }; + cache.acknowledged_read(1); + cache.acknowledged_bookmark(2); + assert!(cache.pending()); + cache.acknowledged_read(2); + cache.acknowledged_bookmark(3); + assert!(!cache.pending()); + } + + #[test] + fn initial_unknown_state_does_not_claim_a_post_is_unbookmarked() { + let mut cache = CachedItem::default(); + assert_eq!(cache.bookmarked(), None); + cache.observe(item(), &account()); + assert_eq!(cache.bookmarked(), Some(false)); + assert!(!cache.pending()); + } + + #[test] + fn other_accounts_and_noncanonical_rows_never_enter_the_cache() { + let mut cache = CachedItem::default(); + cache.observe(Item { fid: 4, ..item() }, &account()); + cache.observe(Item { hash: "bad".into(), ..item() }, &account()); + assert!(!cache.known); + assert_eq!(canonical_hash(&format!("0x{:040X}", 10)), Some(format!("0x{:040x}", 10))); + assert_eq!(canonical_hash("x"), None); + } + + #[test] + fn late_disk_snapshots_cannot_resurrect_settled_or_replace_newer_intent() { + let mut old = CachedItem { + server: item(), + bookmark: Some(PendingBookmark { desired: true, expected_revision: 0, sequence: 1 }), + ..CachedItem::default() + }; + let mut newer = old.clone(); + newer.bookmark = Some(PendingBookmark { desired: false, expected_revision: 0, sequence: 2 }); + old.merge_disk(&newer); + assert_eq!(old.bookmark.as_ref().unwrap().sequence, 2); + let stale = old.clone(); + old.acknowledged_bookmark(2); + old.merge_disk(&stale); + assert!(old.bookmark.is_none()); + assert_eq!(old.bookmark_settled, 2); + } + + #[test] + fn persisted_remote_tombstones_also_fence_another_tabs_outbox() { + let mut pending = CachedItem { + server: item(), + read: Some(PendingRead { + generation: 0, + expected_revision: 0, + intent: ReadIntent::Seen { at: 10 }, + sequence: 1, + }), + ..CachedItem::default() + }; + let remote = CachedItem { known: true, server: Item { seen_revision: 2, ..item() }, ..CachedItem::default() }; + pending.merge_disk(&remote); + assert!(pending.read.is_none()); + assert_eq!(pending.read_settled, 1); + } + + #[test] + fn accepted_add_preserves_a_newer_remove_through_observe_and_disk_merge() { + let sent = PendingBookmark { desired: true, expected_revision: 0, sequence: 1 }; + let mut cache = CachedItem { + known: true, + server: item(), + bookmark: Some(PendingBookmark { desired: false, expected_revision: 0, sequence: 2 }), + ..CachedItem::default() + }; + let stale_disk = cache.clone(); + cache.accepted_bookmark( + &sent, + Item { bookmarked: true, bookmark_at: 150, bookmark_revision: 1, ..item() }, + &account(), + ); + assert_eq!(cache.bookmarked(), Some(false)); + assert_eq!(cache.bookmark.as_ref().unwrap().expected_revision, 1); + cache.merge_disk(&stale_disk); + assert_eq!(cache.bookmark.as_ref().unwrap().expected_revision, 1); + assert_eq!(cache.bookmark.as_ref().unwrap().sequence, 2); + cache.observe(Item { bookmark_revision: 2, ..item() }, &account()); + assert!(cache.bookmark.is_none()); + assert_eq!(cache.bookmark_settled, 2); + } + + #[test] + fn accepted_seen_preserves_a_newer_unread_through_observe_and_disk_merge() { + let sent = + PendingRead { generation: 0, expected_revision: 0, intent: ReadIntent::Seen { at: 150 }, sequence: 1 }; + let mut cache = CachedItem { + known: true, + server: item(), + read: Some(PendingRead { generation: 0, expected_revision: 0, intent: ReadIntent::Unread, sequence: 2 }), + ..CachedItem::default() + }; + let stale_disk = cache.clone(); + cache.accepted_read(&sent, Item { seen: true, seen_at: 150, seen_revision: 1, ..item() }, &account()); + assert!(!cache.seen(0)); + assert_eq!(cache.read.as_ref().unwrap().expected_revision, 1); + cache.merge_disk(&stale_disk); + assert_eq!(cache.read.as_ref().unwrap().expected_revision, 1); + cache.observe(Item { seen_revision: 2, ..item() }, &account()); + assert!(cache.read.is_none()); + assert_eq!(cache.read_settled, 2); + } + + #[test] + fn accepted_same_value_noop_does_not_drop_a_newer_action() { + let mut cache = CachedItem { + known: true, + server: Item { bookmarked: true, bookmark_revision: 4, ..item() }, + bookmark: Some(PendingBookmark { desired: false, expected_revision: 4, sequence: 2 }), + ..CachedItem::default() + }; + let row = cache.server.clone(); + cache.accepted_bookmark(&PendingBookmark { desired: true, expected_revision: 4, sequence: 1 }, row, &account()); + assert_eq!(cache.bookmark.as_ref().unwrap().sequence, 2); + assert_eq!(cache.bookmark_settled, 1); + } + + #[test] + fn successful_old_calls_never_rebase_across_foreign_changes_or_clears() { + let sent = PendingBookmark { desired: true, expected_revision: 0, sequence: 1 }; + let mut cache = CachedItem { + known: true, + server: item(), + bookmark: Some(PendingBookmark { desired: true, expected_revision: 0, sequence: 2 }), + ..CachedItem::default() + }; + cache.accepted_bookmark(&sent, Item { bookmark_revision: 2, ..item() }, &account()); + assert!(cache.bookmark.is_none()); + assert_eq!(cache.bookmarked(), Some(false)); + cache.read = Some(PendingRead { + generation: 0, + expected_revision: 0, + intent: ReadIntent::Seen { at: 150 }, + sequence: 4, + }); + cache.accepted_read( + &PendingRead { generation: 0, expected_revision: 0, intent: ReadIntent::Seen { at: 100 }, sequence: 3 }, + Item { generation: 1, ..item() }, + &Account { generation: 1, cleared_at: 200, ..account() }, + ); + assert!(cache.read.is_none()); + assert!(!cache.seen(1)); + } + + #[test] + fn existing_rows_cannot_mix_accounts_even_when_the_cast_hash_matches() { + for known in [false, true] { + let mut cache = CachedItem { known, server: item(), ..CachedItem::default() }; + cache.observe( + Item { fid: 4, bookmarked: true, bookmark_revision: 9, ..item() }, + &Account { fid: 4, ..account() }, + ); + assert_eq!(cache.server, item()); + assert_eq!(cache.known, known); + } + } + + #[test] + fn cached_equal_value_is_not_an_ack_for_a_queued_inverse() { + let mut cache = CachedItem { + known: true, + server: item(), + bookmark: Some(PendingBookmark { desired: false, expected_revision: 0, sequence: 2 }), + ..CachedItem::default() + }; + cache.merge_disk(&CachedItem { known: true, server: item(), ..CachedItem::default() }); + assert_eq!(cache.bookmark.as_ref().unwrap().sequence, 2); + } + + #[test] + fn inflight_disk_merge_waits_for_provenance_but_never_defeats_a_clear() { + let sent = + PendingRead { generation: 0, expected_revision: 0, intent: ReadIntent::Seen { at: 150 }, sequence: 1 }; + let mut cache = CachedItem { + known: true, + server: item(), + read: Some(PendingRead { generation: 0, expected_revision: 0, intent: ReadIntent::Unread, sequence: 2 }), + ..CachedItem::default() + }; + let fresh = Item { seen: true, seen_at: 150, seen_revision: 1, ..item() }; + cache.merge_disk_preserving_inflight( + &CachedItem { known: true, server: fresh.clone(), ..CachedItem::default() }, + Some(1), + None, + ); + assert_eq!(cache.read.as_ref().unwrap().expected_revision, 0); + cache.accepted_read(&sent, fresh, &account()); + assert_eq!(cache.read.as_ref().unwrap().expected_revision, 1); + cache.merge_disk_preserving_inflight( + &CachedItem { known: true, server: Item { generation: 1, ..item() }, ..CachedItem::default() }, + Some(1), + None, + ); + assert!(cache.read.is_none()); + } + + #[test] + fn stale_account_and_membership_cannot_roll_back_a_known_reset() { + let mut cache = CachedItem { + known: true, + server: Item { generation: 1, ..item() }, + read: Some(PendingRead { + generation: 1, + expected_revision: 0, + intent: ReadIntent::Seen { at: 250 }, + sequence: 2, + }), + ..CachedItem::default() + }; + cache.observe(Item { seen: true, seen_at: 150, seen_revision: 10, ..item() }, &account()); + assert_eq!(cache.server.generation, 1); + assert!(!cache.server.seen); + assert_eq!(cache.read.as_ref().unwrap().generation, 1); + cache.reconcile(&Account { generation: 1, cleared_at: 200, ..account() }); + assert!(cache.seen(1)); + } +} diff --git a/crates/allthing-core/src/seen.rs b/crates/allthing-core/src/seen.rs index 1423a554..bcf4e91d 100644 --- a/crates/allthing-core/src/seen.rs +++ b/crates/allthing-core/src/seen.rs @@ -79,6 +79,16 @@ impl History { self.scopes.iter().filter(|s| s.name == scope).flat_map(|s| s.posts.iter().map(|p| p.hash.clone())).collect() } + /// The existing, bounded browser cache, for a one-time private server + /// migration. The server, rather than this cache, owns durable history. + pub fn entries(&self, scope: &str) -> Vec<(String, i64)> { + self.scopes + .iter() + .filter(|s| s.name == scope) + .flat_map(|s| s.posts.iter().map(|p| (p.hash.clone(), p.at))) + .collect() + } + pub fn record(&mut self, name: &str, value: &str, now: i64) { self.record_at(name, value, now, now); } diff --git a/crates/allthing-data/src/hypersnap.rs b/crates/allthing-data/src/hypersnap.rs index 5c75184c..7fed9f7a 100644 --- a/crates/allthing-data/src/hypersnap.rs +++ b/crates/allthing-data/src/hypersnap.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; -use allthing_core::api::FollowsDir; +use allthing_core::api::{FollowsDir, LIBRARY_PAGE_SIZE, LibraryEntry, ReactionKind, library_cursor}; use allthing_core::channels::{ canonical_parent_url, channel_id_for_url, channel_url_for_id, legacy_url, parent_url_variants, short_parent_url_label, @@ -125,6 +125,138 @@ pub async fn casts_bulk(inv: &Invocation, hashes: &[String]) -> Result Result, UpstreamError> { + #[derive(Deserialize)] + struct Found { + casts: Vec, + } + if hashes.is_empty() { + return Ok(Vec::new()); + } + if hashes.len() > LIBRARY_PAGE_SIZE { + return Err(UpstreamError::Parse("too many Library casts".into())); + } + let list = hashes.join(","); + let found: Found = get_json(inv, Upstream::Hypersnap, &v2("casts", &[("casts", &list)]), false) + .await? + .ok_or(UpstreamError::Status(404))?; + if found.casts.len() > LIBRARY_PAGE_SIZE + || found.casts.iter().any(|cast| { + allthing_core::params::hash_from_param(&cast.hash).is_none() + || cast.author.as_ref().is_none_or(|author| author.fid == 0) + }) + { + return Err(UpstreamError::Parse("invalid Library cast lookup".into())); + } + Ok(found.casts) +} + +#[derive(Deserialize)] +struct LibraryReactionPage { + messages: Vec, + #[serde(rename = "nextPageToken")] + next: String, +} +#[derive(Deserialize)] +struct LibraryReactionMessage { + data: LibraryReactionData, +} +#[derive(Deserialize)] +struct LibraryReactionData { + fid: u64, + timestamp: u64, + #[serde(rename = "type")] + kind: serde_json::Value, + #[serde(rename = "reactionBody")] + reaction: LibraryReactionBody, +} +#[derive(Deserialize)] +struct LibraryReactionBody { + #[serde(rename = "type")] + kind: serde_json::Value, + #[serde(rename = "targetCastId")] + target: Option, + #[serde(rename = "targetUrl")] + target_url: Option, +} +#[derive(Deserialize)] +struct LibraryTarget { + fid: u64, + hash: String, +} + +/// A page of the account's retained active reactions. Follow the opaque +/// `next` key to the end; unlike a target's reactors this is a single shard. +pub async fn library_reactions( + inv: &Invocation, + fid: u64, + kind: ReactionKind, + cursor: Option<&str>, +) -> Result<(Vec, Option), UpstreamError> { + let fid_text = fid.to_string(); + let size = LIBRARY_PAGE_SIZE.to_string(); + let (number, name, query_kind) = match kind { + ReactionKind::Likes => (1, "REACTION_TYPE_LIKE", "Like"), + ReactionKind::Recasts => (2, "REACTION_TYPE_RECAST", "Recast"), + }; + let mut query = vec![ + ("fid", fid_text.as_str()), + ("reaction_type", query_kind), + ("pageSize", size.as_str()), + ("reverse", "true"), + ]; + if let Some(cursor) = cursor { + query.push(("pageToken", cursor)); + } + let page: LibraryReactionPage = get_json(inv, Upstream::Hypersnap, &v1("reactionsByFid", &query), false) + .await? + .ok_or(UpstreamError::Status(404))?; + let invalid = || UpstreamError::Parse("invalid Library reaction page".into()); + if page.messages.len() > LIBRARY_PAGE_SIZE { + return Err(invalid()); + } + let mut seen = std::collections::HashSet::new(); + let mut entries = Vec::new(); + for message in page.messages { + let data = message.data; + let matches = + |value: &serde_json::Value, number, name| value.as_u64() == Some(number) || value.as_str() == Some(name); + if data.fid != fid + || !matches(&data.kind, 3, "MESSAGE_TYPE_REACTION_ADD") + || !matches(&data.reaction.kind, number, name) + { + return Err(invalid()); + } + // A reaction to a URL is valid network history, but is not a cast. + let Some(target) = data.reaction.target else { + if data.reaction.target_url.as_deref().is_some_and(|url| !url.trim().is_empty()) { + continue; + } + return Err(invalid()); + }; + let hash = allthing_core::params::hash_from_param(&target.hash).ok_or_else(invalid)?; + let at = i64::try_from(data.timestamp) + .ok() + .and_then(|at| at.checked_add(allthing_core::message::EPOCH)) + .ok_or_else(invalid)?; + if target.fid == 0 { + return Err(invalid()); + } + if seen.insert(hash.clone()) { + entries.push(LibraryEntry { hash, author_fid: target.fid, at }); + } + } + let next = (!page.next.is_empty()).then_some(page.next); + if let Some(next) = &next + && (library_cursor(next).is_none() || Some(next.as_str()) == cursor) + { + return Err(invalid()); + } + Ok((entries, next)) +} + /// Who liked (`likes`) or recasted (`recasts`) a cast, at most 100 /// (Hypersnap does not page further), as Hypersnap lists them: each shard's /// reactions oldest first, one shard after another ([`normalize::reactors`] diff --git a/crates/allthing-data/src/lib.rs b/crates/allthing-data/src/lib.rs index f01c5f34..ef118355 100644 --- a/crates/allthing-data/src/lib.rs +++ b/crates/allthing-data/src/lib.rs @@ -217,6 +217,10 @@ pub enum Endpoint { Wallets(u64), Channels(Option), Search(String), + /// One page of an account's retained active likes or recasts. + Library(u64, ReactionKind, Option), + /// Public cast hydration for an ordered private Library page. + LibraryCasts(Vec), Reactions(String, ReactionKind, u32), /// An account's followers or following, by name (`fid:` for later /// pages), from the start or from a cursor. @@ -260,6 +264,25 @@ impl Endpoint { } "/latest" => Ok(Endpoint::Latest), "/activity" => Ok(Endpoint::Activity), + "/library" => { + let fid = query("fid") + .and_then(|v| v.parse::().ok()) + .filter(|fid| *fid > 0) + .ok_or(bad("Expected ?fid="))?; + let kind = + query("kind").and_then(|v| ReactionKind::parse(&v)).ok_or(bad("Expected ?kind=likes|recasts"))?; + let cursor = match query("cursor") { + None => None, + Some(cursor) => Some( + allthing_core::api::library_cursor(&cursor) + .ok_or(bad("Expected ?cursor="))?, + ), + }; + Ok(Endpoint::Library(fid, kind, cursor)) + } + "/library/casts" => allthing_core::api::parse_library_hashes(query("hashes").as_deref()) + .map(Endpoint::LibraryCasts) + .ok_or(bad("Expected ?hashes=")), "/reactions" => { let hash = query("hash").and_then(|v| hash_from_param(&v)).ok_or(bad("Expected ?hash="))?; let kind = @@ -365,6 +388,14 @@ impl Endpoint { } Endpoint::Latest => "latest".into(), Endpoint::Activity => "activity".into(), + Endpoint::Library(fid, kind, cursor) => allthing_core::api::library_path(*fid, *kind, cursor.as_deref()) + .trim_start_matches(API_PREFIX) + .trim_start_matches('/') + .to_owned(), + Endpoint::LibraryCasts(hashes) => allthing_core::api::library_casts_path(hashes) + .trim_start_matches(API_PREFIX) + .trim_start_matches('/') + .to_owned(), Endpoint::Reactions(hash, kind, limit) => { format!("reactions?hash={hash}&type={}&limit={limit}", kind.as_str()) } @@ -396,6 +427,8 @@ impl Endpoint { Endpoint::Home(offset, window) => pages::home(inv, offset, window, previous).await, Endpoint::Latest => pages::latest(inv, previous).await, Endpoint::Activity => pages::activity(inv).await, + Endpoint::Library(fid, kind, cursor) => pages::library(inv, fid, kind, cursor, previous).await, + Endpoint::LibraryCasts(hashes) => pages::library_casts(inv, hashes, previous).await, Endpoint::Reactions(hash, kind, limit) => pages::reactions(inv, hash, kind, limit).await, Endpoint::Engagement(request) => pages::engagement(inv, request).await, Endpoint::Replies(hash) => pages::replies(inv, hash).await, @@ -416,6 +449,26 @@ impl Endpoint { mod tests { use super::*; + #[test] + fn library_routes_scope_and_escape_pages_and_keep_hydration_order() { + let page = Endpoint::from_path("/api/v1/library?fid=3&kind=recasts&cursor=%2B%2F8%3D").unwrap(); + assert_eq!(page, Endpoint::Library(3, ReactionKind::Recasts, Some("+/8=".into()))); + assert_eq!(page.cache_name(), "library?fid=3&kind=recasts&cursor=%2B%2F8%3D"); + let (a, b) = (format!("0x{:040x}", 1), format!("0x{:040x}", 2)); + let batch = Endpoint::from_path(&format!("/api/v1/library/casts?hashes={b},{},{b}", a.to_uppercase())).unwrap(); + assert_eq!(batch, Endpoint::LibraryCasts(vec![b.clone(), a.clone()])); + assert_eq!(batch.cache_name(), format!("library/casts?hashes={b},{a}")); + for path in [ + "/api/v1/library", + "/api/v1/library?fid=0&kind=likes", + "/api/v1/library?fid=3&kind=other", + "/api/v1/library?fid=3&kind=likes&cursor=bad!", + "/api/v1/library/casts?hashes=0x12", + ] { + assert_eq!(Endpoint::from_path(path).map_err(|error| error.status), Err(400)); + } + } + #[test] fn counts_share_one_entry_whatever_the_order() { let (a, b) = (format!("0x{:040x}", 1), format!("0x{:040x}", 2)); diff --git a/crates/allthing-data/src/pages.rs b/crates/allthing-data/src/pages.rs index 3418eb9d..b2764e59 100644 --- a/crates/allthing-data/src/pages.rs +++ b/crates/allthing-data/src/pages.rs @@ -7,8 +7,8 @@ use std::rc::Rc; use allthing_core::activity::AppsWindow; use allthing_core::api::{ self, Annotations, CastPayload, ChannelPayload, ChannelsPayload, EngagementPayload, EngagementRequest, FollowsDir, - FollowsPayload, HomePayload, IndexCoverage, LatestPayload, Meta, QuotedCast, ReactionKind, ReactionsPayload, - RepliesPayload, SearchPayload, UserPayload, + FollowsPayload, HomePayload, IndexCoverage, LatestPayload, LibraryCastsPayload, LibraryPayload, Meta, QuotedCast, + ReactionKind, ReactionsPayload, RepliesPayload, SearchPayload, UserPayload, }; use allthing_core::channels::{channel_id_for_url, channel_url_for_id, is_channel_id}; use allthing_core::embeds::link_key; @@ -533,6 +533,268 @@ pub async fn following( Built::json(&payload, following::FOLLOWING_CACHE) } +/// Hydrate a bounded Library page in the caller's order. The provider's bulk +/// order is not the list's order. Errors retain the caller's cursor for Retry; +/// successful omissions and content filtering have separate classifications. +async fn hydrate_library( + inv: &Invocation, + hashes: &[String], + previous: Option>>, +) -> Result { + let raw = hypersnap::library_casts(inv, hashes).await.map_err(Failure::upstream)?; + let mut by_hash: HashMap = + raw.iter().map(normalize::cast).map(|cast| (cast.hash.clone(), cast)).collect(); + let unavailable = hashes.iter().filter(|hash| !by_hash.contains_key(*hash)).cloned().collect(); + let ordered: Vec = hashes.iter().filter_map(|hash| by_hash.remove(hash)).collect(); + let filtered = ordered.iter().filter(|cast| cast.spam).map(|cast| cast.hash.clone()).collect(); + let mut notes = carried(previous.as_deref().map(Vec::as_slice)); + let known_links = std::mem::take(&mut notes.links); + let pool = drop_hidden(inv, ordered, &mut notes, &known_links, true).await; + let quoted = annotate_listing(inv, &mut notes, &pool, ORIGIN_AUTHORS, &known_links).await; + notes.keep_users(read_cards(&raw, quoted), listed_accounts(&pool, ¬es.quotes)); + keep_shown(&mut notes, &pool); + Ok(LibraryCastsPayload { pool, unavailable, filtered, notes, meta: meta(inv) }) +} + +/// The account's retained active likes or recasts, newest reaction first, +/// including older pages until the hub has no continuation token. The +/// browser's recent-reaction cache does not limit this history. +pub async fn library( + inv: Rc, + fid: u64, + kind: ReactionKind, + cursor: Option, + previous: Option>>, +) -> Result { + let (entries, next) = + hypersnap::library_reactions(&inv, fid, kind, cursor.as_deref()).await.map_err(Failure::upstream)?; + let hashes: Vec = entries.iter().map(|entry| entry.hash.clone()).collect(); + let LibraryCastsPayload { pool, unavailable, filtered, notes, meta } = + hydrate_library(&inv, &hashes, previous).await?; + Built::json( + &LibraryPayload { fid, kind, entries, pool, next, unavailable, filtered, notes, meta }, + api::LIBRARY_CACHE, + ) +} + +/// Public cast data for bookmarks/seen metadata supplied by the browser, at +/// most 25 hashes. This does not read or expose any private Library records. +pub async fn library_casts( + inv: Rc, + hashes: Vec, + previous: Option>>, +) -> Result { + Built::json(&hydrate_library(&inv, &hashes, previous).await?, api::LIBRARY_CACHE) +} + +#[cfg(test)] +mod library_tests { + use super::*; + use crate::{FetchError, Host, HttpRequest, HttpResponse}; + use allthing_core::provenance::{base64_decode, base64_encode}; + use futures::future::{FutureExt, LocalBoxFuture}; + use serde_json::json; + use std::cell::{Cell, RefCell}; + + struct Hub { + total: usize, + missing: HashSet, + reaction_status: u16, + reaction_body: Option, + bulk_status: u16, + bulk_body: Option, + timeout: bool, + clock: Cell, + requests: RefCell>, + } + impl Hub { + fn new(total: usize) -> Self { + Self { + total, + missing: HashSet::new(), + reaction_status: 200, + reaction_body: None, + bulk_status: 200, + bulk_body: None, + timeout: false, + clock: Cell::new(1_800_000_000_000.0), + requests: RefCell::new(Vec::new()), + } + } + } + fn hash(n: usize) -> String { + format!("0x{n:040x}") + } + fn query(url: &str, name: &str) -> Option { + url.split_once('?')?.1.split('&').find_map(|part| { + let (key, value) = part.split_once('=')?; + (key == name).then(|| allthing_core::params::percent_decode(value)).flatten() + }) + } + impl Host for Hub { + fn now(&self) -> f64 { + // Independent invocations can refill the upstream test bucket; + // no timing or performance behavior is being measured here. + let now = self.clock.get(); + self.clock.set(now + 10_000.0); + now + } + fn fetch(&self, request: HttpRequest) -> LocalBoxFuture<'static, Result> { + let url = request.url; + assert!(url.starts_with(hypersnap::BASE), "Library never switches read providers: {url}"); + self.requests.borrow_mut().push(url.clone()); + let answer = |status, body: String| { + Ok(HttpResponse { status, retry_after: None, body: (status == 200).then(|| body.into_bytes()) }) + }; + let response = if url.contains("/v1/reactionsByFid?") { + if self.timeout { + return async { Err(FetchError::Timeout) }.boxed_local(); + } + assert_eq!(query(&url, "fid").as_deref(), Some("3")); + assert_eq!(query(&url, "pageSize").as_deref(), Some("25")); + assert_eq!(query(&url, "reverse").as_deref(), Some("true")); + let offset = query(&url, "pageToken") + .map(|token| String::from_utf8(base64_decode(&token).unwrap()).unwrap().parse::().unwrap()) + .unwrap_or(0); + let end = (offset + api::LIBRARY_PAGE_SIZE).min(self.total); + let kind = if query(&url, "reaction_type").as_deref() == Some("Recast") { + "REACTION_TYPE_RECAST" + } else { + "REACTION_TYPE_LIKE" + }; + let messages: Vec<_> = (offset..end) + .map(|n| { + json!({"data": { + "fid":3,"type":"MESSAGE_TYPE_REACTION_ADD","timestamp":1_000_000 - n, + "reactionBody":{"type":kind,"targetCastId":{"fid":n+1000,"hash":hash(n)}} + }}) + }) + .collect(); + let next = if end < self.total { base64_encode(end.to_string().as_bytes()) } else { String::new() }; + answer( + self.reaction_status, + self.reaction_body + .clone() + .unwrap_or_else(|| json!({"messages":messages,"nextPageToken":next}).to_string()), + ) + } else if url.contains("/v2/farcaster/casts?") { + let hashes = query(&url, "casts").unwrap(); + let casts: Vec<_> = hashes.split(',').rev().filter_map(|h| { + let n = usize::from_str_radix(h.trim_start_matches("0x"),16).unwrap(); + if self.missing.contains(&n) { return None; } + let text = if n == 2 { "Just unlocked my DEGEN rewards" } else { "A saved cast" }; + Some(json!({"hash":h,"author":{"fid":n+1000,"username":format!("author{n}"),"follower_count":1},"text":text, + "timestamp":"2026-01-01T00:00:00Z","embeds": if n == 3 { vec![json!({"url":"https://farcaster.xyz/miniapps/abc/game"})] } else { vec![] }})) + }).collect(); + answer(self.bulk_status, self.bulk_body.clone().unwrap_or_else(|| json!({"casts":casts}).to_string())) + } else { + answer(404, String::new()) + }; + async move { response }.boxed_local() + } + fn spawn(&self, _: LocalBoxFuture<'static, ()>) {} + } + fn read(hub: Rc, kind: ReactionKind, cursor: Option) -> Result { + let built = library(Rc::new(Invocation::new(hub, 43)), 3, kind, cursor, None).now_or_never().unwrap()?; + Ok(serde_json::from_slice(&built.body).unwrap()) + } + + #[test] + fn library_pages_continue_past_recent_cache_limits_to_the_actual_end() { + for kind in [ReactionKind::Likes, ReactionKind::Recasts] { + let hub = Rc::new(Hub::new(131)); + let (mut cursor, mut entries, mut pages) = (None, Vec::new(), 0); + loop { + let page = read(hub.clone(), kind, cursor).unwrap(); + assert_eq!((page.fid, page.kind), (3, kind)); + entries.extend(page.entries); + pages += 1; + cursor = page.next; + if cursor.is_none() { + break; + } + assert!(pages < 10, "cursor must finish"); + } + assert_eq!(entries.len(), 131); + assert_eq!( + entries.iter().map(|entry| entry.hash.clone()).collect::>(), + (0..131).map(hash).collect::>() + ); + assert!(entries.windows(2).all(|rows| rows[0].at > rows[1].at)); + assert_eq!(pages, 6); + assert_eq!(hub.requests.borrow().iter().filter(|url| url.contains("/v1/reactionsByFid?")).count(), 6); + assert_eq!(hub.requests.borrow().iter().filter(|url| url.contains("/v2/farcaster/casts?")).count(), 6); + } + } + + #[test] + fn library_hydration_keeps_list_order_and_separates_unavailable_from_filtered() { + let mut hub = Hub::new(0); + hub.missing.insert(1); + let hub = Rc::new(hub); + let ordered = [3, 1, 2, 4].map(hash).to_vec(); + let built = library_casts(Rc::new(Invocation::new(hub, 43)), ordered, None).now_or_never().unwrap().unwrap(); + let page: LibraryCastsPayload = serde_json::from_slice(&built.body).unwrap(); + assert_eq!(page.pool.iter().map(|cast| cast.hash.clone()).collect::>(), [hash(3), hash(4)]); + assert_eq!(page.unavailable, [hash(1)]); + assert_eq!(page.filtered, [hash(2)]); + assert!(page.pool[0].mini_app, "settings, not server hydration, decide whether mini apps show"); + assert!(page.notes.users.contains_key(&1003)); + assert!(!page.notes.users.contains_key(&1002)); + } + + #[test] + fn library_source_failures_and_bad_shapes_never_become_empty_success() { + for status in [400, 404, 500] { + let mut hub = Hub::new(1); + hub.reaction_status = status; + assert!(read(Rc::new(hub), ReactionKind::Likes, None).is_err()); + let mut hub = Hub::new(1); + hub.bulk_status = status; + assert!(read(Rc::new(hub), ReactionKind::Likes, None).is_err()); + } + let mut hub = Hub::new(1); + hub.timeout = true; + assert!(read(Rc::new(hub), ReactionKind::Likes, None).is_err()); + for body in ["{}", r#"{"messages":[]}"#, r#"{"messages":[],"nextPageToken":"invalid!"}"#] { + let mut hub = Hub::new(1); + hub.reaction_body = Some(body.into()); + assert!(read(Rc::new(hub), ReactionKind::Likes, None).is_err()); + } + let mut hub = Hub::new(1); + hub.bulk_body = Some("{}".into()); + assert!(read(Rc::new(hub), ReactionKind::Likes, None).is_err()); + let mut hub = Hub::new(1); + hub.reaction_body = Some(json!({"messages":[{"data":{"fid":3,"timestamp":1,"type":"MESSAGE_TYPE_REACTION_ADD","reactionBody":{"type":"REACTION_TYPE_LIKE"}}}],"nextPageToken":""}).to_string()); + assert!(read(Rc::new(hub), ReactionKind::Likes, None).is_err()); + let mut hub = Hub::new(1); + hub.reaction_body = Some(json!({"messages":[{"data":{"fid":3,"timestamp":1,"type":"MESSAGE_TYPE_REACTION_ADD","reactionBody":{"type":"REACTION_TYPE_LIKE","targetUrl":"https://example.org"}}}],"nextPageToken":""}).to_string()); + assert!(read(Rc::new(hub), ReactionKind::Likes, None).unwrap().entries.is_empty()); + } + + #[test] + fn library_empty_history_is_explicit_and_does_not_fetch_casts() { + let hub = Rc::new(Hub::new(0)); + let page = read(hub.clone(), ReactionKind::Likes, None).unwrap(); + assert!(page.entries.is_empty() && page.pool.is_empty() && page.unavailable.is_empty() && page.next.is_none()); + assert_eq!(hub.requests.borrow().len(), 1); + } + + #[test] + fn library_annotations_stay_within_the_shared_request_budget() { + let hub = Rc::new(Hub::new(25)); + let page = read(hub.clone(), ReactionKind::Recasts, None).unwrap(); + assert_eq!(page.entries.len(), 25); + let urls = hub.requests.borrow(); + assert!(urls.len() <= 43, "the cache reserves its two calls from the45 budget: {}", urls.len()); + assert_eq!(urls.iter().filter(|url| url.contains("/v2/farcaster/casts?")).count(), 1); + assert!( + urls.iter().filter(|url| url.contains("/v1/castById?")).count() <= ORIGIN_AUTHORS, + "only the existing bounded provenance annotator may read individual casts" + ); + } +} + /// How many voted each way in Allthing's poll, so a reader who voted sees /// them with no request of their own: one D1 read of the two counts its /// votes keep, given an index and a subrequest for it. `None` before anyone diff --git a/crates/allthing-web/Cargo.toml b/crates/allthing-web/Cargo.toml index b6013a04..f898e48f 100644 --- a/crates/allthing-web/Cargo.toml +++ b/crates/allthing-web/Cargo.toml @@ -110,6 +110,8 @@ features = [ "CryptoKeyPair", "SubtleCrypto", "IdbDatabase", + "IdbKeyRange", + "DomException", "IdbFactory", "IdbObjectStore", "IdbObjectStoreParameters", diff --git a/crates/allthing-web/src/components/bookmark.rs b/crates/allthing-web/src/components/bookmark.rs new file mode 100644 index 00000000..18188ae3 --- /dev/null +++ b/crates/allthing-web/src/components/bookmark.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! A private saved-post toggle. It needs a session, never a Farcaster signer. + +use leptos::prelude::*; + +#[cfg(not(feature = "static-host"))] +#[component] +pub fn BookmarkButton(hash: String) -> impl IntoView { + use crate::components::signin::use_session; + use crate::library::{SyncStatus, use_library}; + + let library = use_library(); + let session = use_session(); + // Split pages can construct a view under a short-lived async owner before + // mounting it. Follow the mounted control's effect lifetime, as CastCard's + // viewport observer does, so Back retains a live membership subscription. + let node = NodeRef::::new(); + let watched = hash.clone(); + Effect::new(move |_| { + if node.get().is_some() { + library.watch(&watched); + } + }); + let key = hash.clone(); + let saved = Memo::new(move |_| library.bookmarked(&key)); + let signed_in = Memo::new(move |_| session.user.with(Option::is_some)); + let waiting = Memo::new(move |_| signed_in.get() && saved.get().is_none()); + let message = RwSignal::new(None::); + Effect::new(move |_| { + session.user.with(|user| user.as_ref().map(|user| user.fid)); + message.set(None); + }); + let label = move || if saved.get() == Some(true) { "Remove bookmark" } else { "Save post" }; + view! { + + + + {move || message.get()} + + + } +} + +/// The public-only static copy has no private account state or save action. +#[cfg(feature = "static-host")] +#[component] +pub fn BookmarkButton(hash: String) -> impl IntoView { + let _ = hash; +} diff --git a/crates/allthing-web/src/components/cast.rs b/crates/allthing-web/src/components/cast.rs index 1865e2f4..e5c6865c 100644 --- a/crates/allthing-web/src/components/cast.rs +++ b/crates/allthing-web/src/components/cast.rs @@ -45,6 +45,9 @@ pub fn CastCard( #[prop(optional)] index: usize, #[prop(optional)] show_parent: bool, #[prop(optional)] track_seen: bool, + /// A Library activity line/action, inside the existing card element. + #[prop(optional_no_strip)] + row_footer: Option>, ) -> impl IntoView { let href = cast_href(&cast.hash); // Quoted casts the payload carries show as cards; others as links. @@ -64,13 +67,18 @@ pub fn CastCard( let reactions = crate::components::reactions::card_reactions(&cast, ¬es); #[cfg(feature = "static-host")] let reactions = (); + let extra = row_footer.map(|footer| footer.run(cast.clone())); let gone = crate::components::deleted(&cast.hash); // Drawn as Back or Forward puts the reader's place back, a card is where // it was: it doesn't rise into place again. let entrance = if crate::nav::restoring() { "; animation: none" } else { "" }; let node = NodeRef::::new(); let seen = crate::seen::use_seen(); + let read_mark = track_seen.then(|| seen.read_mark(cast.hash.clone())); Effect::new(move |_| { + if let Some(read_mark) = read_mark { + read_mark.track(); + } if let Some(element) = node.get() { if track_seen { crate::seen::watch(&element, seen); @@ -133,6 +141,7 @@ pub fn CastCard( {reactions} + {extra} } } @@ -166,14 +175,29 @@ pub fn CastList( /// and keeping its place as they reorder: the front page's poll. #[prop(default = None)] inset: Option AnyView>, + /// Optional per-cast Library metadata, without wrapping card elements. + #[prop(optional_no_strip)] + row_footer: Option>, ) -> impl IntoView { // Mini app casts, which payloads keep marked, are left out unless the // reader chose to see them in the Settings panel (`crate::miniapps`). let all = casts; - let seen_feed = crate::seen::Feed::new(hide_seen); + let candidates = Signal::derive(move || { + let show = crate::miniapps::shown(); + all.with(|casts| { + casts + .iter() + .filter(|cast| !cast.hidden_for(show)) + .filter_map(|cast| allthing_core::library::canonical_hash(&cast.hash)) + .collect() + }) + }); + let seen_feed = crate::seen::Feed::new(hide_seen, candidates); let hidden_seen = Memo::new(move |_| { let show_apps = crate::miniapps::shown(); - all.with(|casts| casts.iter().filter(|cast| !cast.hidden_for(show_apps) && seen_feed.hides(&cast.hash)).count()) + all.with(|casts| { + casts.iter().filter(|cast| !cast.hidden_for(show_apps) && seen_feed.hidden(&cast.hash)).count() + }) }); let casts: Signal> = Memo::new(move |_| { let show = crate::miniapps::shown(); @@ -277,6 +301,9 @@ pub fn CastList( } }); let grow = move || { + if seen_feed.pending() { + return; + } // Still drawing what is shown: its end isn't the list's end yet. if drawn.with_value(|(count, _)| *count) < shown.get_untracked().min(casts.with_untracked(Vec::len)) { return; @@ -298,7 +325,7 @@ pub fn CastList( // A short or filtered page can leave the sentinel inside the observer's // margin, so there is no new intersection event to continue the feed. Effect::new(move |_| { - let _ = (casts.with(Vec::len), shown.get(), drew_more.get(), page_cursor.get()); + let _ = (casts.with(Vec::len), shown.get(), drew_more.get(), page_cursor.get(), seen_feed.pending()); request_animation_frame(move || { let Some(element) = sentinel.try_get_untracked().flatten() else { return }; if !element.is_connected() { @@ -320,8 +347,13 @@ pub fn CastList( // fewer, as while the first card waits for its image's size, it was at // the list's top, and was pushed down as they came. let items = move || { - let mut items: Vec<(String, Option<(usize, Cast)>)> = - visible().into_iter().map(|(index, cast)| (cast.hash.clone(), Some((index, cast)))).collect(); + let mut items: Vec<(String, Option<(usize, Cast)>)> = visible() + .into_iter() + .map(|(index, cast)| { + seen_feed.preserve(&cast.hash); + (cast.hash.clone(), Some((index, cast))) + }) + .collect(); let all = casts.with(Vec::len); if inset.is_some() && !items.is_empty() && items.len() >= INSET_AFTER.min(all) { let at = INSET_AFTER.min(items.len()); @@ -332,13 +364,16 @@ pub fn CastList( }; view! { - "Nothing here yet."

}> + }.into_any() } + else { view! {

"Nothing here yet."

}.into_any() } + }>
view! { }.into_any(), + Some((index, cast)) => view! { }.into_any(), None => inset.map(|inset| inset()).into_any(), } /> diff --git a/crates/allthing-web/src/components/chips.rs b/crates/allthing-web/src/components/chips.rs index 95b5e6e8..5e4aac0c 100644 --- a/crates/allthing-web/src/components/chips.rs +++ b/crates/allthing-web/src/components/chips.rs @@ -181,7 +181,13 @@ pub fn Engagement( }; // The ranking note is for screen readers: a native tooltip on the row // would sit on top of the counts' own popovers. - view! { } + view! { + + } } /// The reply count of the cast `hash`, `read` as the page has it, with the diff --git a/crates/allthing-web/src/components/layout.rs b/crates/allthing-web/src/components/layout.rs index 875dbc78..f50d621a 100644 --- a/crates/allthing-web/src/components/layout.rs +++ b/crates/allthing-web/src/components/layout.rs @@ -31,6 +31,7 @@ pub fn SiteHeader() -> impl IntoView {
{presence()}
@@ -164,14 +165,20 @@ fn offered(range: TimeRange) -> bool { /// Reddit's time filter: how far back the front page reaches, kept in the /// URL as `t` next to the sort. #[component] -pub fn TimeFilter(path: String, sort: Sort, current: TimeRange) -> impl IntoView { +pub fn TimeFilter( + path: String, + sort: Sort, + current: TimeRange, + #[prop(optional)] default_sort: Option, +) -> impl IntoView { + let default_sort = default_sort.unwrap_or(Sort::Hot); let navigate = use_go(); let change = move |ev: leptos::ev::Event| { let range = TimeRange::parse(Some(&event_target_value(&ev))); let href = with_query( &path, &[ - ("sort", (sort != Sort::Hot).then(|| sort.as_str().to_owned())), + ("sort", (sort != default_sort).then(|| sort.as_str().to_owned())), ("t", (range != TimeRange::default()).then(|| range.as_str().to_owned())), ], ); diff --git a/crates/allthing-web/src/components/mod.rs b/crates/allthing-web/src/components/mod.rs index 010f2699..1fe29972 100644 --- a/crates/allthing-web/src/components/mod.rs +++ b/crates/allthing-web/src/components/mod.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "static-host"))] pub mod attach; +pub mod bookmark; pub mod cast; pub mod chips; pub mod clients; diff --git a/crates/allthing-web/src/components/signin.rs b/crates/allthing-web/src/components/signin.rs index 6762111b..0b7a005d 100644 --- a/crates/allthing-web/src/components/signin.rs +++ b/crates/allthing-web/src/components/signin.rs @@ -650,6 +650,7 @@ pub fn Account() -> impl IntoView { "@"{username.clone()}

+ diff --git a/crates/allthing-web/src/lib.rs b/crates/allthing-web/src/lib.rs index 817f2227..d8849913 100644 --- a/crates/allthing-web/src/lib.rs +++ b/crates/allthing-web/src/lib.rs @@ -13,6 +13,10 @@ mod kept; #[cfg(not(feature = "static-host"))] mod keys; mod leanings; +#[cfg(not(feature = "static-host"))] +mod library; +#[cfg(not(feature = "static-host"))] +mod library_disk; mod miniapps; mod nav; #[cfg(not(feature = "static-host"))] @@ -79,6 +83,8 @@ fn App() -> impl IntoView { #[cfg(not(feature = "static-host"))] provide_context(components::signin::Session::new()); #[cfg(not(feature = "static-host"))] + provide_context(library::Library::new()); + #[cfg(not(feature = "static-host"))] provide_context(components::notifications::Notifications::new()); #[cfg(not(feature = "static-host"))] provide_context(components::compose::Composer::new()); @@ -112,6 +118,7 @@ fn App() -> impl IntoView { + diff --git a/crates/allthing-web/src/library.rs b/crates/allthing-web/src/library.rs new file mode 100644 index 00000000..4dbc6699 --- /dev/null +++ b/crates/allthing-web/src/library.rs @@ -0,0 +1,1657 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Private Library state, caller-scoped subscriptions and a committed offline outbox. +//! Shares the verified notification connection; never touches chat or public presence. + +use std::cell::{Cell, RefCell}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::rc::Rc; + +use allthing_core::library::{Account, CachedItem, Item, PendingBookmark, PendingRead, ReadIntent, canonical_hash}; +use allthing_core::stdb::Called; +use js_sys::Array; +use leptos::prelude::*; +use serde::{Deserialize, Serialize}; +use wasm_bindgen::JsValue; + +use crate::library_disk; +use crate::stdb_bridge::{method, parse}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SyncStatus { + SignedOut, + Connecting, + Ready, + Offline, + Failed(String), +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PrivateKind { + Seen, + Bookmarks, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PrivateCursor { + pub generation: u64, + pub after: String, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PrivateEntry { + pub hash: String, + pub at: i64, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PrivatePage { + pub entries: Vec, + pub next: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct PendingClear { + operation: String, + expected_generation: u64, + sequence: u64, +} +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub(crate) struct Metadata { + pub(crate) fid: u64, + server: Account, + known: bool, + pending_clear: Option, + sequence: u64, + clear_settled: u64, + legacy_imported: bool, +} +impl Metadata { + pub(crate) fn merge_disk(&mut self, incoming: &Self) { + if self.fid != incoming.fid { + return; + } + if incoming.known + && (!self.known + || (incoming.server.generation, incoming.server.revision) + >= (self.server.generation, self.server.revision)) + { + self.server = incoming.server.clone(); + self.known = true; + } + self.sequence = self.sequence.max(incoming.sequence); + self.clear_settled = self.clear_settled.max(incoming.clear_settled); + self.legacy_imported |= incoming.legacy_imported; + self.pending_clear = self + .pending_clear + .iter() + .chain(incoming.pending_clear.iter()) + .filter(|p| p.sequence > self.clear_settled) + .max_by_key(|p| p.sequence) + .cloned(); + if self.pending_clear.as_ref().is_some_and(|p| self.known && self.server.generation > p.expected_generation) + && let Some(p) = self.pending_clear.take() + { + self.clear_settled = self.clear_settled.max(p.sequence); + } + } + fn effective(&self) -> Account { + let mut account = self.server.clone(); + account.fid = self.fid; + if let Some(clear) = &self.pending_clear { + account.generation = clear.expected_generation.saturating_add(1); + } + account + } +} + +#[derive(Clone)] +struct State { + fid: Option, + capability: Option, + status: SyncStatus, + revision: u64, + metadata: Metadata, + items: BTreeMap, + checked: BTreeSet, + loaded: BTreeSet, +} +impl Default for State { + fn default() -> Self { + Self { + fid: None, + capability: None, + status: SyncStatus::SignedOut, + revision: 0, + metadata: Metadata::default(), + items: BTreeMap::new(), + checked: BTreeSet::new(), + loaded: BTreeSet::new(), + } + } +} +#[derive(Clone, Copy)] +pub struct Library(RwSignal, crate::components::signin::Session); +thread_local! { + static HANDLE: Cell> = const { Cell::new(None) }; + static CURRENT: RefCell> = const { RefCell::new(None) }; + static WATCHES: RefCell>> = const { RefCell::new(BTreeMap::new()) }; + static IDS: Cell = const { Cell::new(0) }; +} +pub fn use_library() -> Library { + use_context::().expect("Library context") +} +fn handle() -> Option { + HANDLE.with(Cell::get) +} +fn next_id() -> u64 { + IDS.with(|id| { + let n = id.get().wrapping_add(1); + id.set(n); + n + }) +} +fn canonical(hash: &str) -> Result { + canonical_hash(hash).ok_or_else(|| "Invalid post hash".into()) +} +fn now() -> i64 { + (js_sys::Date::now() / 1000.0) as i64 +} +fn all_watches() -> BTreeSet { + WATCHES.with(|all| all.borrow().values().flat_map(|v| v.iter().cloned()).collect()) +} + +impl Library { + pub fn new() -> Self { + let session = crate::components::signin::use_session(); + let fid = session.user.with_untracked(|u| u.as_ref().map(|u| u.fid).filter(|fid| *fid > 0)); + let state = State { + fid, + status: if fid.is_some() { SyncStatus::Connecting } else { SyncStatus::SignedOut }, + metadata: Metadata { fid: fid.unwrap_or(0), ..Metadata::default() }, + ..State::default() + }; + let library = Self(RwSignal::new(state), session); + HANDLE.with(|h| h.set(Some(library))); + library + } + fn fid(self) -> Option { + self.1.user.with(|u| u.as_ref().map(|u| u.fid).filter(|fid| *fid > 0)) + } + fn same(self) -> bool { + let fid = self.fid(); + self.0.with(|s| s.fid == fid) + } + pub fn status(self) -> SyncStatus { + if !self.same() { + return if self.fid().is_some() { SyncStatus::Connecting } else { SyncStatus::SignedOut }; + } + self.0.with(|s| s.status.clone()) + } + pub fn revision(self) -> u64 { + self.0.with(|s| s.revision) + } + pub fn generation(self) -> u64 { + self.0.with(|s| s.metadata.effective().generation) + } + pub fn remote_required(self) -> bool { + let fid = self.fid(); + fid.is_some() && self.0.with(|s| s.fid != fid || s.capability != Some(false)) + } + pub fn membership_ready(self, hashes: &[String]) -> bool { + if !self.same() { + return false; + } + self.0.with(|s| { + s.capability == Some(false) + || (s.status == SyncStatus::Ready + && s.metadata.legacy_imported + && hashes.iter().all(|h| s.checked.contains(h) && s.loaded.contains(h))) + }) + } + pub fn seen(self, hash: &str) -> Option { + if !self.same() { + return None; + } + self.0.with(|s| { + s.items.get(hash).filter(|c| c.known || c.read.is_some()).map(|c| c.seen(s.metadata.effective().generation)) + }) + } + pub fn bookmarked(self, hash: &str) -> Option { + if !self.same() { + return None; + } + self.0.with(|s| s.items.get(hash).and_then(CachedItem::bookmarked)) + } + pub fn watch(self, hash: &str) { + let key = format!("card-{}", next_id()); + let cleanup = key.clone(); + // The constructing card owns this registration. Register retirement + // before any signal writes; subscription plumbing must not become a + // dependency of the parent render/memo or of its cleanup callback. + on_cleanup(move || untrack(|| self.unwatch_batch(&cleanup))); + untrack(|| self.watch_batch(&key, &[hash.into()])); + } + pub fn watch_batch(self, owner: &str, hashes: &[String]) { + let values = hashes.iter().filter_map(|h| canonical_hash(h)).collect::>(); + WATCHES.with(|w| { + w.borrow_mut().insert(owner.into(), values); + }); + if let Some(b) = current() { + b.load_watched(); + b.watches(); + } + crate::notification_sync::wake_urgent(); + } + pub fn unwatch_batch(self, owner: &str) { + WATCHES.with(|w| { + w.borrow_mut().remove(owner); + }); + if let Some(b) = current() { + b.watches(); + } + } + pub fn retry(self) { + crate::notification_sync::wake_urgent(); + if let Some(b) = current() { + b.inner.disk_failed.set(false); + b.bootstrap(); + b.persist(); + b.load_watched(); + b.watches(); + b.drain(); + } + } + pub fn toggle_bookmark(self, hash: &str) -> Result<(), String> { + let hash = canonical(hash)?; + let b = require()?; + self.bookmarked(&hash).ok_or("Wait for this post's saved state to load")?; + b.action(hash, Action::Toggle) + } + pub fn record_seen(self, hash: &str, at: i64) -> Result<(), String> { + let hash = canonical(hash)?; + let b = require()?; + if self.seen(&hash) == Some(true) { + return Ok(()); + } + b.action(hash, Action::Seen(at.min(now()).max(1))) + } + pub fn forget_seen(self, hash: &str) -> Result<(), String> { + let hash = canonical(hash)?; + require()?.action(hash, Action::Unread) + } + pub fn clear_seen(self) -> Result<(), String> { + let b = require()?; + if !self.0.with_untracked(|s| s.metadata.known) { + return Err("Wait for Library state to load before clearing history".into()); + } + if self.0.with_untracked(|s| s.metadata.pending_clear.is_some()) { + return Ok(()); + } + b.clear_action()?; + Ok(()) + } + pub async fn page(self, kind: PrivateKind, cursor: Option) -> Result { + let b = require()?; + b.page(kind, cursor).await + } +} +fn empty(fid: u64, hash: &str, generation: u64) -> CachedItem { + CachedItem { server: Item { fid, hash: hash.into(), generation, ..Item::default() }, ..CachedItem::default() } +} +fn current() -> Option { + CURRENT.with(|c| c.borrow().clone()) +} +fn require() -> Result { + let b = current().filter(|b| b.active()).ok_or("Sign in to use your Library")?; + if b.library.0.with_untracked(|s| s.capability == Some(false)) { + return Err("Private Library sync is not configured on this server".into()); + } + Ok(b) +} + +#[derive(Clone)] +struct Broker { + fid: u64, + library: Library, + inner: Rc, +} +struct Inner { + stopped: Cell, + attempt: Cell, + bridge: RefCell>, + base: Cell, + disk: Cell, + disk_failed: Cell, + bootstrapping: Cell, + clear_busy: Cell, + loading: RefCell>, + groups: RefCell>, + writing: Cell, + drain_again: Cell, + persisting: Cell, + writes: RefCell>, + in_flight: RefCell>, + buffered: RefCell>, + busy: RefCell>, + touches: RefCell>, + clock: Cell, +} +#[derive(Clone)] +enum Write { + Item(CachedItem), + Metadata(Metadata), +} +#[derive(Clone)] +enum Action { + Toggle, + Seen(i64), + Unread, +} +impl Action { + fn bookmark(&self) -> bool { + matches!(self, Self::Toggle) + } + fn pending(&self, item: &CachedItem) -> bool { + if self.bookmark() { item.bookmark.is_some() } else { item.read.is_some() } + } +} +#[derive(Clone)] +enum Flight { + Read(PendingRead), + Bookmark(PendingBookmark), +} +struct Lease { + bridge: JsValue, + tag: String, + hashes: BTreeSet, +} +impl Drop for Lease { + fn drop(&mut self) { + if let Ok(f) = method(&self.bridge, "unwatch") { + let _ = f.call1(&self.bridge, &self.tag.clone().into()); + } + } +} +const MAX_MEMORY: usize = 2000; +fn evictions( + items: &BTreeMap, + protected: &BTreeSet, + touches: &BTreeMap, +) -> Vec { + let mut eligible: Vec<_> = items + .iter() + .filter(|(hash, item)| !protected.contains(*hash) && !item.pending()) + .map(|(hash, _)| (touches.get(hash).copied().unwrap_or(0), hash.clone())) + .collect(); + eligible.sort(); + eligible.into_iter().take(items.len().saturating_sub(MAX_MEMORY)).map(|(_, hash)| hash).collect() +} +impl Broker { + fn active(&self) -> bool { + !self.inner.stopped.get() + && self.library.1.user.with_untracked(|u| u.as_ref().map(|u| u.fid)) == Some(self.fid) + && current().is_some_and(|b| Rc::ptr_eq(&b.inner, &self.inner)) + } + fn touch(&self, hash: &str) { + let next = self.inner.clock.get().wrapping_add(1); + self.inner.clock.set(next); + self.inner.touches.borrow_mut().insert(hash.into(), next); + } + fn prune(&self) { + if !self.active() { + return; + } + let mut protected = all_watches(); + protected.extend(self.inner.busy.borrow().iter().map(|(hash, _)| hash.clone())); + protected.extend(self.inner.in_flight.borrow().keys().cloned()); + protected.extend(self.inner.loading.borrow().iter().cloned()); + self.library.0.update(|s| { + for hash in evictions(&s.items, &protected, &self.inner.touches.borrow()) { + s.items.remove(&hash); + s.loaded.remove(&hash); + s.checked.remove(&hash); + } + s.loaded.retain(|hash| protected.contains(hash) || s.items.contains_key(hash)); + self.inner.touches.borrow_mut().retain(|hash, _| s.items.contains_key(hash)); + }); + } + fn check(&self, attempt: u64) -> Result<(), String> { + if self.active() && self.inner.attempt.get() == attempt && self.inner.base.get() { + Ok(()) + } else { + Err("Library session changed or disconnected".into()) + } + } + fn failed(&self, error: String) { + if self.active() { + self.inner.groups.borrow_mut().clear(); + self.library.0.update(|s| { + s.status = SyncStatus::Failed(error); + s.checked.clear(); + }); + } + } + fn sequence(&self) -> u64 { + let random = crate::keys::random::<4>().map(u32::from_le_bytes).unwrap_or(0) as u64 % 1_000_000; + let base = (js_sys::Date::now().max(0.0) as u64).saturating_mul(1_000_000).saturating_add(random); + self.library.0.update(|s| s.metadata.sequence = base.max(s.metadata.sequence.saturating_add(1))); + self.library.0.with_untracked(|s| s.metadata.sequence) + } + fn stop(&self) { + self.inner.stopped.set(true); + self.inner.base.set(false); + self.inner.groups.borrow_mut().clear(); + self.inner.bridge.borrow_mut().take(); + } + fn persist_item(&self, hash: &str) { + if let Some(item) = self.library.0.with_untracked(|s| s.items.get(hash).cloned()) { + self.inner.writes.borrow_mut().push_back(Write::Item(item)); + self.persist(); + } + } + fn persist_meta(&self) { + let value = self.library.0.with_untracked(|s| s.metadata.clone()); + self.inner.writes.borrow_mut().push_back(Write::Metadata(value)); + self.persist(); + } + fn persist(&self) { + if !self.active() || self.inner.persisting.replace(true) { + return; + } + let b = self.clone(); + leptos::task::spawn_local(async move { + loop { + let Some(write) = b.inner.writes.borrow_mut().pop_front() else { break }; + let answer = match &write { + Write::Item(item) => library_disk::keep_item(b.fid, item).await, + Write::Metadata(meta) => match serde_json::to_string(meta) { + Ok(json) => library_disk::keep_account(b.fid, &json).await, + Err(e) => Err(e.to_string()), + }, + }; + // Finish every already accepted disk write under its captured + // FID even after sign-out; never send it over an old socket. + if let Err(e) = answer { + b.inner.writes.borrow_mut().push_front(write); + b.inner.disk_failed.set(true); + b.failed(format!("Your Library change could not be saved on this device: {e}")); + break; + } + if matches!(&write, Write::Metadata(_)) && b.active() { + match library_disk::account(b.fid).await { + Ok(Some(json)) => match serde_json::from_str(&json) { + Ok(meta) => b.merge_metadata(meta), + Err(_) => { + b.inner.disk_failed.set(true); + b.failed("Unreadable Library metadata".into()); + break; + } + }, + Ok(None) => { + b.inner.disk_failed.set(true); + b.failed("Library metadata was not committed".into()); + break; + } + Err(error) => { + b.inner.disk_failed.set(true); + b.failed(error); + break; + } + } + } + } + b.inner.persisting.set(false); + if b.active() { + b.drain(); + } + }); + } + fn load_watched(&self) { + let hashes: Vec<_> = all_watches() + .into_iter() + .filter(|h| { + !self.library.0.with_untracked(|s| s.loaded.contains(h)) + && self.inner.loading.borrow_mut().insert(h.clone()) + }) + .collect(); + if hashes.is_empty() { + return; + } + let b = self.clone(); + leptos::task::spawn_local(async move { + let rows = library_disk::items(b.fid, &hashes).await; + if !b.active() { + return; + } + for hash in &hashes { + b.inner.loading.borrow_mut().remove(hash); + } + match rows { + Ok(rows) => { + b.merge_cached(rows); + b.library.0.update(|s| s.loaded.extend(hashes)); + } + Err(e) => { + b.inner.disk_failed.set(true); + b.failed(e); + } + } + }); + } + fn merge_cached(&self, rows: Vec) { + self.library.0.update(|s| { + for row in rows { + let hash = row.server.hash.clone(); + self.touch(&hash); + let flight = self.inner.in_flight.borrow().get(&hash).cloned(); + let (read, bookmark) = match flight { + Some(Flight::Read(sent)) => (Some(sent.sequence), None), + Some(Flight::Bookmark(sent)) => (None, Some(sent.sequence)), + None => (None, None), + }; + let c = s + .items + .entry(hash) + .or_insert_with(|| empty(self.fid, &row.server.hash, s.metadata.effective().generation)); + let old = c.clone(); + c.merge_disk_preserving_inflight(&row, read, bookmark); + if old != *c { + s.revision += 1; + } + } + }); + self.prune(); + } + fn bridge(&self) -> Result { + self.inner.bridge.borrow().clone().ok_or_else(|| "Library is disconnected".into()) + } + fn raw(&self, name: &str, args: &[JsValue]) -> Result { + self.check(self.inner.attempt.get())?; + let bridge = self.bridge()?; + method(&bridge, name)?.apply(&bridge, &args.iter().collect::()).map_err(crate::stdb_bridge::js_error) + } + async fn call(&self, name: &str, args: &[JsValue]) -> Result<(), String> { + let attempt = self.inner.attempt.get(); + let result: Called = parse(crate::notification_sync::bounded(self.raw(name, args)?).await?)?; + self.check(attempt)?; + result.result() + } + async fn snapshot(&self, accessor: &str, query: String) -> Result, String> { + let attempt = self.inner.attempt.get(); + let bridge = self.bridge()?; + let tag = format!("library-once-{}", next_id()); + let _lease = Lease { bridge, tag: tag.clone(), hashes: BTreeSet::new() }; + let json = serde_json::to_string(&[query]).map_err(|e| e.to_string())?; + let value = self.raw("snapshot", &[tag.into(), accessor.into(), json.into()])?; + let rows = parse(crate::notification_sync::bounded(value).await?)?; + self.check(attempt)?; + Ok(rows) + } + async fn membership(&self, hashes: &[String]) -> Result, String> { + if hashes.is_empty() || hashes.len() > 100 { + return Err("Invalid Library membership batch".into()); + } + let rows = self + .snapshot("myLibraryMembership", format!("SELECT * FROM my_library_membership('{}')", hashes.join(","))) + .await?; + self.validate_membership(hashes, rows) + } + fn validate_membership(&self, hashes: &[String], rows: Vec) -> Result, String> { + let generation = self.library.0.with_untracked(|s| s.metadata.server.generation); + let wanted: BTreeSet<_> = hashes.iter().cloned().collect(); + let rows = fold_rows(rows, self.fid, generation) + .into_iter() + .filter(|(hash, _)| wanted.contains(hash)) + .collect::>(); + if rows.len() != wanted.len() { + return Err("Library membership is incomplete; try again".into()); + } + Ok(hashes.iter().filter_map(|h| rows.get(h).cloned()).collect()) + } + fn observe(&self, row: Item) { + if self.inner.in_flight.borrow().contains_key(&row.hash) { + let account = self.library.0.with_untracked(|s| s.metadata.effective()); + let mut buffered = self.inner.buffered.borrow_mut(); + if let Some(old) = buffered.get_mut(&row.hash) { + merge_row(old, row, &account); + } else { + buffered.insert(row.hash.clone(), row); + } + return; + } + let hash = row.hash.clone(); + self.touch(&hash); + let mut changed = false; + self.library.0.update(|s| { + let account = s.metadata.effective(); + let c = s.items.entry(hash.clone()).or_insert_with(|| empty(self.fid, &hash, account.generation)); + let old = c.clone(); + c.observe(row, &account); + changed = old != *c; + if changed { + s.revision += 1; + } + }); + if changed { + self.persist_item(&hash); + } + self.prune(); + } + fn watches(&self) { + self.prune(); + if !self.active() + || !self.inner.base.get() + || !self.inner.disk.get() + || !self.library.0.with_untracked(|s| s.metadata.known && s.metadata.pending_clear.is_none()) + { + return; + } + let hashes: Vec<_> = all_watches().into_iter().collect(); + let groups: BTreeMap<_, _> = + hashes.chunks(100).map(|hs| (hs.join(","), hs.iter().cloned().collect::>())).collect(); + self.inner.groups.borrow_mut().retain(|signature, lease| { + if groups.contains_key(signature) { + true + } else { + self.library.0.update(|s| { + for hash in &lease.hashes { + s.checked.remove(hash); + } + }); + false + } + }); + for (signature, hashes) in groups { + if self.inner.groups.borrow().contains_key(&signature) { + continue; + } + let Ok(bridge) = self.bridge() else { return }; + let tag = format!("library-live-{}", next_id()); + let query = serde_json::to_string(&[format!("SELECT * FROM my_library_membership('{signature}')")]) + .unwrap_or_default(); + let Ok(value) = self.raw("watch", &[tag.clone().into(), query.into()]) else { continue }; + self.inner + .groups + .borrow_mut() + .insert(signature.clone(), Lease { bridge, tag: tag.clone(), hashes: hashes.clone() }); + let b = self.clone(); + let attempt = self.inner.attempt.get(); + leptos::task::spawn_local(async move { + let result = crate::notification_sync::bounded(value).await; + if b.check(attempt).is_err() + || !b.inner.groups.borrow().get(&signature).is_some_and(|lease| lease.tag == tag) + { + return; + } + let answer = result.and_then(|_| { + let bridge = b.bridge()?; + let value = method(&bridge, "rows")? + .call1(&bridge, &"myLibraryMembership".into()) + .map_err(crate::stdb_bridge::js_error)?; + b.validate_membership(&hashes.iter().cloned().collect::>(), parse(value)?) + }); + match answer { + Ok(rows) => { + for row in rows { + b.observe(row); + } + b.library.0.update(|s| { + s.checked.extend(hashes); + }); + b.drain(); + } + Err(e) => { + b.inner.groups.borrow_mut().remove(&signature); + b.failed(e); + } + } + }); + } + } + async fn page(&self, kind: PrivateKind, cursor: Option) -> Result { + if self.library.status() != SyncStatus::Ready { + return Err("Connect your Library to load this page; saved offline changes are kept".into()); + } + let generation = self.library.0.with_untracked(|s| s.metadata.server.generation); + if self.library.0.with_untracked(|s| s.metadata.pending_clear.is_some()) { + return Err("Your history clear is still syncing".into()); + } + let after = match cursor { + Some(c) if c.generation != generation => return Err("History changed; refresh this list".into()), + Some(c) => c.after, + None => String::new(), + }; + if !valid_cursor(&after) { + return Err("Invalid Library cursor".into()); + } + let rows = self + .snapshot( + "myLibraryPage", + format!( + "SELECT * FROM my_library_page({},{generation},'{after}',25)", + if kind == PrivateKind::Seen { 0 } else { 1 } + ), + ) + .await?; + if self.library.generation() != generation { + return Err("History changed; refresh this list".into()); + } + let account = self.library.0.with_untracked(|s| s.metadata.effective()); + let mut folded = fold_rows(rows, self.fid, generation); + self.library.0.with_untracked(|s| { + for (hash, row) in &mut folded { + if let Some(cache) = s.items.get(hash).filter(|c| c.known) { + merge_row(row, cache.server.clone(), &account); + } + } + }); + let mut rows: Vec<_> = folded + .into_values() + .filter(|r| match kind { + PrivateKind::Seen => r.seen, + PrivateKind::Bookmarks => r.bookmarked, + }) + .filter(|r| after.is_empty() || before(r, &after, kind)) + .collect(); + rows.sort_by(|a, b| stamp(b, kind).cmp(&stamp(a, kind)).then_with(|| b.hash.cmp(&a.hash))); + rows.dedup_by(|a, b| a.hash == b.hash); + let more = rows.len() > 25; + rows.truncate(25); + let entries: Vec<_> = rows.iter().map(|r| PrivateEntry { hash: r.hash.clone(), at: stamp(r, kind) }).collect(); + for row in rows { + self.observe(row); + } + let next = if more { + entries.last().map(|e| PrivateCursor { generation, after: format!("{}:{}", e.at, e.hash) }) + } else { + None + }; + Ok(PrivatePage { entries, next }) + } + fn merge_metadata(&self, meta: Metadata) { + if meta.fid != self.fid { + return; + } + if meta.known { + observe_account(self, meta.server.clone()); + } + self.library.0.update(|s| { + let old = s.metadata.clone(); + s.metadata.merge_disk(&meta); + let account = s.metadata.effective(); + let mut changed = old != s.metadata; + for (hash, c) in &mut s.items { + if !self.inner.in_flight.borrow().contains_key(hash) { + let before = c.clone(); + c.reconcile(&account); + changed |= *c != before; + } + } + if changed { + s.revision += 1; + } + }); + } + fn clear_action(&self) -> Result<(), String> { + if self.inner.clear_busy.replace(true) { + return Err("A history clear is already being saved".into()); + } + let b = self.clone(); + leptos::task::spawn_local(async move { + let result = async { + let kept = library_disk::account(b.fid).await?; + if !b.active() { + return Err("Library session changed".into()); + } + if let Some(json) = kept { + let meta = serde_json::from_str(&json).map_err(|_| "Unreadable Library metadata")?; + b.merge_metadata(meta); + } + if b.library.0.with_untracked(|s| s.metadata.pending_clear.is_some()) { + return Err("A history clear from this account is still syncing".into()); + } + let sequence = b.sequence(); + let random = crate::keys::random::<16>().ok_or("Couldn't create a clear operation")?; + let operation = random.iter().map(|b| format!("{b:02x}")).collect(); + let mut changed = Vec::new(); + b.library.0.update(|s| { + s.metadata.pending_clear = + Some(PendingClear { operation, expected_generation: s.metadata.server.generation, sequence }); + let account = s.metadata.effective(); + for (hash, c) in &mut s.items { + let old = c.clone(); + c.reconcile(&account); + if old != *c { + changed.push(hash.clone()); + } + } + s.checked.clear(); + s.revision += 1; + }); + b.inner.groups.borrow_mut().clear(); + b.persist_meta(); + for hash in changed { + b.persist_item(&hash); + } + Ok::<_, String>(()) + } + .await; + b.inner.clear_busy.set(false); + if b.active() + && let Err(error) = result + { + b.failed(error); + } + }); + Ok(()) + } + fn bootstrap(&self) { + if self.inner.disk.get() || self.inner.bootstrapping.replace(true) { + return; + } + let fid = self.fid; + let b = self.clone(); + leptos::task::spawn_local(async move { + let result = async { + let metadata = library_disk::account(fid).await?; + let pending = library_disk::pending(fid, 100).await?; + Ok::<_, String>((metadata, pending)) + } + .await; + b.inner.bootstrapping.set(false); + if !b.active() { + return; + } + match result { + Ok((metadata, pending)) => { + if let Some(json) = metadata { + match serde_json::from_str::(&json) { + Ok(meta) => b.merge_metadata(meta), + Err(_) => { + b.inner.disk_failed.set(true); + b.failed("Unreadable Library account metadata".into()); + return; + } + } + } + b.merge_cached(pending); + b.inner.disk.set(true); + b.ready(); + b.watches(); + b.drain(); + } + Err(e) => { + b.inner.disk_failed.set(true); + b.failed(e); + } + } + }); + } + fn action(&self, hash: String, action: Action) -> Result<(), String> { + let pending = self.library.0.with_untracked(|s| s.items.get(&hash).is_some_and(|item| action.pending(item))); + let inflight = self.inner.in_flight.borrow().get(&hash).is_some_and(|flight| { + matches!((action.bookmark(), flight), (true, Flight::Bookmark(_)) | (false, Flight::Read(_))) + }); + let busy = (hash.clone(), action.bookmark()); + if pending || inflight || !self.inner.busy.borrow_mut().insert(busy.clone()) { + if matches!(action, Action::Seen(_)) { + return Ok(()); + } + return Err("Syncing the previous change to this post; try again when it finishes".into()); + } + let b = self.clone(); + leptos::task::spawn_local(async move { + let answer = async { + let old = library_disk::items(b.fid, std::slice::from_ref(&hash)).await?; + if !b.active() { + return Err("Library session changed".into()); + } + b.merge_cached(old); + if b.library.0.with_untracked(|s| s.items.get(&hash).is_some_and(|item| action.pending(item))) { + if matches!(action, Action::Seen(_)) { + return Ok(()); + } + return Err("Another tab is syncing this post's previous change; try again when it finishes".into()); + } + let sequence = b.sequence(); + let mut result: Result<(), String> = Ok(()); + b.library.0.update(|s| { + let generation = s.metadata.effective().generation; + let c = s.items.entry(hash.clone()).or_insert_with(|| empty(b.fid, &hash, generation)); + match action { + Action::Toggle => match c.bookmarked() { + Some(value) => { + c.bookmark = Some(PendingBookmark { + desired: !value, + expected_revision: c.server.bookmark_revision, + sequence, + }) + } + None => result = Err("Wait for this post's saved state to load".into()), + }, + Action::Seen(at) => { + if !c.seen(generation) { + c.read = Some(PendingRead { + generation, + expected_revision: c.server.seen_revision, + intent: ReadIntent::Seen { at }, + sequence, + }); + } + } + Action::Unread => { + c.read = Some(PendingRead { + generation, + expected_revision: c.server.seen_revision, + intent: ReadIntent::Unread, + sequence, + }) + } + } + s.revision += 1; + }); + result?; + b.persist_item(&hash); + Ok::<_, String>(()) + } + .await; + b.inner.busy.borrow_mut().remove(&busy); + if b.active() + && let Err(error) = answer + { + b.failed(error); + } + }); + Ok(()) + } + fn ready(&self) { + if self.inner.base.get() + && self.inner.disk.get() + && !self.inner.disk_failed.get() + && self.library.0.with_untracked(|s| s.metadata.legacy_imported) + { + let restore = self.library.0.with_untracked(|s| s.status != SyncStatus::Ready); + self.library.0.update(|s| { + s.status = SyncStatus::Ready; + }); + // A failed operation retires all managed subscriptions. A later + // successful drain must restore them before presenting Ready. + // Rebuild only after releasing the State/RefCell borrows above. + if restore { + self.watches(); + } + } + } + fn yield_for_clear(&self) -> bool { + let waiting = self.library.0.with_untracked(|s| s.metadata.pending_clear.is_some()); + if waiting { + self.inner.drain_again.set(true); + } + waiting + } + fn drain(&self) { + if !self.active() + || !self.inner.base.get() + || !self.inner.disk.get() + || self.inner.disk_failed.get() + || self.inner.persisting.get() + || !self.inner.writes.borrow().is_empty() + { + return; + } + if self.inner.writing.replace(true) { + self.inner.drain_again.set(true); + return; + } + let b = self.clone(); + let attempt = self.inner.attempt.get(); + leptos::task::spawn_local(async move { + let result = b.drain_all(attempt).await; + b.inner.writing.set(false); + if b.active() && b.inner.attempt.get() == attempt { + match result { + Err(error) => b.failed(error), + Ok(()) => { + if b.inner.drain_again.replace(false) { + b.drain(); + } + } + } + } else if b.active() { + b.drain(); + } + }); + } + async fn drain_all(&self, attempt: u64) -> Result<(), String> { + self.check(attempt)?; + // A clear is a durable, idempotent generation change and precedes reads + // created after it. The module's operation ID makes a lost ACK safe. + let clear = self.library.0.with_untracked(|s| s.metadata.pending_clear.clone()); + if let Some(clear) = clear { + let sent = self + .call("clearSeen", &[(clear.expected_generation as f64).into(), clear.operation.clone().into()]) + .await; + self.check(attempt)?; + self.refresh_account().await?; + self.check(attempt)?; + if self.library.0.with_untracked(|s| s.metadata.pending_clear.is_some()) { + sent?; + } + if self.library.0.with_untracked(|s| s.metadata.pending_clear.is_some()) { + return Err("History clear has not been acknowledged; retry to reconnect".into()); + } + } + self.migrate_legacy(attempt).await?; + self.ready(); + loop { + self.check(attempt)?; + if self.yield_for_clear() { + break; + } + if self.inner.persisting.get() || !self.inner.writes.borrow().is_empty() { + break; + } + let pending = library_disk::pending(self.fid, 100).await?; + self.check(attempt)?; + if pending.is_empty() { + break; + } + // Point membership is authoritative before replay. A higher remote + // tombstone cancels offline intent; never rebase an old conflict. + let hashes: Vec<_> = pending.iter().map(|p| p.server.hash.clone()).collect(); + self.merge_cached(pending); + let rows = self.membership(&hashes).await?; + for row in rows { + let hash = row.hash.clone(); + self.library.0.update(|s| { + let account = s.metadata.effective(); + let c = s.items.entry(hash.clone()).or_insert_with(|| empty(self.fid, &hash, account.generation)); + c.observe(row, &account); + clamp_replay(c, now(), account.cleared_at); + s.revision += 1; + }); + } + let mut candidates = Vec::new(); + for hash in &hashes { + let item = self + .library + .0 + .with_untracked(|s| s.items.get(hash).cloned()) + .ok_or("Missing Library outbox item")?; + library_disk::keep_item(self.fid, &item).await?; + self.check(attempt)?; + // Read back the atomically merged row: another tab can have + // committed newer intent while this snapshot was being saved. + let durable = library_disk::items(self.fid, std::slice::from_ref(hash)).await?; + self.check(attempt)?; + self.merge_cached(durable); + if let Some(item) = + self.library.0.with_untracked(|s| s.items.get(hash).cloned()).filter(CachedItem::pending) + { + candidates.push(item); + } + } + if candidates.is_empty() { + continue; + } + // A clear may have been requested while the point reads/commits + // above were awaiting. Restart with that durable generation change + // before dispatching any intent created after it. + if self.yield_for_clear() { + break; + } + // One reducer flight at a time. Automatic Seen observations share + // a bounded batch; user removals/bookmarks are serialized separately. + let seen: Vec<_> = candidates + .iter() + .filter_map(|c| { + c.read + .as_ref() + .filter(|r| matches!(r.intent, ReadIntent::Seen { .. })) + .map(|r| (c.server.hash.clone(), r.clone())) + }) + .collect(); + if !seen.is_empty() { + let generation = seen[0].1.generation; + let seen: Vec<_> = seen.into_iter().filter(|(_, r)| r.generation == generation).collect(); + for (hash, read) in &seen { + self.inner.in_flight.borrow_mut().insert(hash.clone(), Flight::Read(read.clone())); + } + let entries:Vec<_>=seen.iter().map(|(hash,r)|serde_json::json!({"hash":hash,"at":match r.intent {ReadIntent::Seen{at}=>at.min(now()),ReadIntent::Unread=>0},"expected_revision":r.expected_revision})).collect(); + let result = self + .call( + "markSeenBatch", + &[ + (generation as f64).into(), + serde_json::to_string(&entries).map_err(|e| e.to_string())?.into(), + ], + ) + .await; + self.finish_flights(&seen.iter().map(|(h, _)| h.clone()).collect::>(), result.is_ok(), attempt) + .await?; + result?; + } else if let Some(item) = candidates.first() { + let hash = item.server.hash.clone(); + let result = if let Some(read) = &item.read { + self.inner.in_flight.borrow_mut().insert(hash.clone(), Flight::Read(read.clone())); + self.call( + "forgetSeen", + &[(read.generation as f64).into(), hash.clone().into(), (read.expected_revision as f64).into()], + ) + .await + } else if let Some(bookmark) = &item.bookmark { + self.inner.in_flight.borrow_mut().insert(hash.clone(), Flight::Bookmark(bookmark.clone())); + self.call( + "setBookmark", + &[hash.clone().into(), bookmark.desired.into(), (bookmark.expected_revision as f64).into()], + ) + .await + } else { + continue; + }; + self.finish_flights(std::slice::from_ref(&hash), result.is_ok(), attempt).await?; + result?; + } + } + Ok(()) + } + async fn finish_flights(&self, hashes: &[String], accepted: bool, attempt: u64) -> Result<(), String> { + self.check(attempt)?; + let fresh = self.membership(hashes).await; + self.check(attempt)?; + let rows = match fresh { + Ok(rows) => rows, + Err(error) => { + for hash in hashes { + self.inner.in_flight.borrow_mut().remove(hash); + let buffered = self.inner.buffered.borrow_mut().remove(hash); + if let Some(row) = buffered { + self.observe(row); + } + } + return Err(error); + } + }; + for row in rows { + let hash = row.hash.clone(); + let flight = self.inner.in_flight.borrow_mut().remove(&hash); + let buffered = self.inner.buffered.borrow_mut().remove(&hash); + self.library.0.update(|s| { + let account = s.metadata.effective(); + let c = s.items.entry(hash.clone()).or_insert_with(|| empty(self.fid, &hash, account.generation)); + apply_flight(c, flight, accepted, row, buffered, &account); + s.revision += 1; + }); + if let Some(item) = self.library.0.with_untracked(|s| s.items.get(&hash).cloned()) { + library_disk::keep_item(self.fid, &item).await?; + } + } + Ok(()) + } + async fn refresh_account(&self) -> Result<(), String> { + let attempt = self.inner.attempt.get(); + let bridge = self.bridge()?; + let tag = format!("library-meta-{}", next_id()); + let _lease = Lease { bridge, tag: tag.clone(), hashes: BTreeSet::new() }; + let value = self + .raw("snapshot", &[tag.into(), "myLibraryState".into(), "[\"SELECT * FROM my_library_state\"]".into()])?; + let rows: Vec = parse(crate::notification_sync::bounded(value).await?)?; + self.check(attempt)?; + let row = rows.into_iter().find(|r| r.fid == self.fid).ok_or("Private Library metadata is missing")?; + observe_account(self, row); + Ok(()) + } + async fn migrate_legacy(&self, attempt: u64) -> Result<(), String> { + if self.library.0.with_untracked(|s| s.metadata.legacy_imported) { + return Ok(()); + } + let entries = crate::seen::legacy_entries(self.fid); + for batch in entries.chunks(100) { + self.check(attempt)?; + let hashes: Vec<_> = batch.iter().filter_map(|(hash, _)| canonical_hash(hash)).collect(); + if hashes.is_empty() { + continue; + } + let rows = self.membership(&hashes).await?; + for row in rows { + let hash = row.hash.clone(); + let at = batch.iter().find(|(h, _)| h == &hash).map_or(0, |(_, at)| *at); + let sequence = self.sequence(); + self.library.0.update(|s| { + let account = s.metadata.effective(); + let c = s.items.entry(hash.clone()).or_insert_with(|| empty(self.fid, &hash, account.generation)); + c.observe(row, &account); + if !c.pending() + && !c.server.seen + && c.server.seen_revision == 0 + && at > account.cleared_at + && at <= now() + { + c.read = Some(PendingRead { + generation: account.generation, + expected_revision: 0, + intent: ReadIntent::Seen { at }, + sequence, + }); + } + s.revision += 1; + }); + if let Some(item) = self.library.0.with_untracked(|s| s.items.get(&hash).cloned()) { + library_disk::keep_item(self.fid, &item).await?; + } + } + } + let mut meta = self.library.0.with_untracked(|s| s.metadata.clone()); + meta.legacy_imported = true; + library_disk::keep_account(self.fid, &serde_json::to_string(&meta).map_err(|e| e.to_string())?).await?; + self.check(attempt)?; + self.library.0.update(|s| s.metadata.legacy_imported = true); + Ok(()) + } +} +fn merge_row(current: &mut Item, incoming: Item, account: &Account) { + let mut cache = CachedItem { server: current.clone(), known: true, ..CachedItem::default() }; + cache.observe(incoming, account); + *current = cache.server; +} +fn fold_rows(rows: Vec, fid: u64, generation: u64) -> BTreeMap { + let account = Account { fid, generation, ..Account::default() }; + let mut folded = BTreeMap::::new(); + for mut row in rows + .into_iter() + .filter(|r| r.fid == fid && r.generation == generation && canonical_hash(&r.hash).as_ref() == Some(&r.hash)) + { + row.cursor.clear(); + if let Some(old) = folded.get_mut(&row.hash) { + merge_row(old, row, &account); + } else { + folded.insert(row.hash.clone(), row); + } + } + folded +} +fn apply_flight( + cache: &mut CachedItem, + flight: Option, + accepted: bool, + row: Item, + buffered: Option, + account: &Account, +) { + match (accepted, flight) { + (true, Some(Flight::Read(sent))) => cache.accepted_read(&sent, row, account), + (true, Some(Flight::Bookmark(sent))) => cache.accepted_bookmark(&sent, row, account), + _ => cache.observe(row, account), + } + // A newer remote operation can arrive after the one-shot response was + // captured but before its Promise resumes Rust. It remains authoritative. + if let Some(buffered) = buffered { + cache.observe(buffered, account); + } +} +fn clamp_replay(item: &mut CachedItem, now: i64, cleared_at: i64) { + if let Some(read) = item.read.as_mut() + && let ReadIntent::Seen { at } = &mut read.intent + { + *at = (*at).min(now); + if *at <= cleared_at { + let seq = read.sequence; + item.acknowledged_read(seq); + } + } +} +fn stamp(row: &Item, kind: PrivateKind) -> i64 { + if kind == PrivateKind::Seen { row.seen_at } else { row.bookmark_at } +} +fn valid_cursor(value: &str) -> bool { + value.is_empty() + || value + .split_once(':') + .is_some_and(|(at, h)| at.parse::().is_ok_and(|v| v > 0) && canonical_hash(h).as_deref() == Some(h)) +} +fn before(row: &Item, after: &str, kind: PrivateKind) -> bool { + after + .split_once(':') + .and_then(|(at, h)| at.parse::().ok().map(|at| (stamp(row, kind), row.hash.as_str()) < (at, h))) + .unwrap_or(false) +} + +pub(crate) fn account(fid: Option) { + let Some(library) = handle() else { return }; + if current().map(|b| b.fid) == fid && library.0.with_untracked(|s| s.fid) == fid { + return; + } + if let Some(b) = CURRENT.with(|c| c.borrow_mut().take()) { + b.stop(); + } + crate::pages::library::clear_cached(); + library.0.set(State { + fid, + status: if fid.is_some() { SyncStatus::Connecting } else { SyncStatus::SignedOut }, + metadata: Metadata { fid: fid.unwrap_or(0), ..Metadata::default() }, + ..State::default() + }); + let Some(fid) = fid else { return }; + let b = Broker { + fid, + library, + inner: Rc::new(Inner { + stopped: Cell::new(false), + attempt: Cell::new(0), + bridge: RefCell::new(None), + base: Cell::new(false), + disk: Cell::new(false), + disk_failed: Cell::new(false), + bootstrapping: Cell::new(false), + clear_busy: Cell::new(false), + loading: RefCell::new(BTreeSet::new()), + groups: RefCell::new(BTreeMap::new()), + writing: Cell::new(false), + drain_again: Cell::new(false), + persisting: Cell::new(false), + writes: RefCell::new(VecDeque::new()), + in_flight: RefCell::new(BTreeMap::new()), + buffered: RefCell::new(BTreeMap::new()), + busy: RefCell::new(BTreeSet::new()), + touches: RefCell::new(BTreeMap::new()), + clock: Cell::new(0), + }), + }; + CURRENT.with(|c| *c.borrow_mut() = Some(b.clone())); + b.load_watched(); + b.bootstrap(); +} + +pub(crate) fn capability(fid: u64, enabled: Option) { + if let Some(b) = current().filter(|b| b.fid == fid) { + b.library.0.update(|s| { + s.capability = enabled; + if enabled == Some(false) { + s.status = SyncStatus::Failed("Private Library sync is not configured on this server".into()); + } + }); + } +} +pub(crate) fn bind(fid: u64, attempt: u64, bridge: JsValue) { + if let Some(b) = current().filter(|b| b.fid == fid) { + b.inner.groups.borrow_mut().clear(); + b.inner.in_flight.borrow_mut().clear(); + b.inner.buffered.borrow_mut().clear(); + b.inner.attempt.set(attempt); + b.inner.base.set(false); + *b.inner.bridge.borrow_mut() = Some(bridge); + b.library.0.update(|s| { + s.checked.clear(); + s.status = SyncStatus::Connecting; + }); + } +} +pub(crate) fn base_applied(fid: u64) -> Result<(), String> { + let b = current().filter(|b| b.fid == fid).ok_or("Library session changed")?; + let bridge = b.bridge()?; + let rows: Vec = parse( + method(&bridge, "rows")?.call1(&bridge, &"myLibraryState".into()).map_err(crate::stdb_bridge::js_error)?, + )?; + let row = rows.into_iter().find(|r| r.fid == fid).ok_or("Private Library metadata is missing")?; + b.inner.base.set(true); + observe_account(&b, row); + b.ready(); + b.watches(); + b.drain(); + Ok(()) +} +fn observe_account(b: &Broker, row: Account) { + if row.fid != b.fid { + return; + } + let mut changed = false; + let mut reset = false; + let mut settled = Vec::new(); + b.library.0.update(|s| { + if s.metadata.known + && (row.generation, row.revision) < (s.metadata.server.generation, s.metadata.server.revision) + { + return; + } + let old = s.metadata.clone(); + reset = row.generation != s.metadata.server.generation; + let foreign = s + .metadata + .pending_clear + .as_ref() + .filter(|p| row.generation > p.expected_generation && row.last_clear_id != p.operation) + .map(|p| p.expected_generation + 1); + s.metadata.server = row; + s.metadata.known = true; + if let Some(p) = s.metadata.pending_clear.clone() + && s.metadata.server.generation > p.expected_generation + { + s.metadata.clear_settled = s.metadata.clear_settled.max(p.sequence); + s.metadata.pending_clear = None; + } + let account = s.metadata.effective(); + for (hash, c) in &mut s.items { + let previous = c.clone(); + if foreign.is_some_and(|g| c.read.as_ref().is_some_and(|p| p.generation == g)) + && let Some(read) = c.read.take() + { + c.acknowledged_read(read.sequence); + } + if !b.inner.in_flight.borrow().contains_key(hash) { + c.reconcile(&account); + } + if previous != *c { + settled.push(hash.clone()); + changed = true; + } + } + changed |= old != s.metadata; + if changed { + s.revision += 1; + } + if reset { + s.checked.clear(); + } + }); + if reset { + b.inner.groups.borrow_mut().clear(); + } + if changed { + b.persist_meta(); + } + for hash in settled { + b.persist_item(&hash); + } + b.watches(); +} +pub(crate) fn event(fid: u64, json: &str) { + #[derive(Deserialize)] + struct Event { + t: String, + table: String, + row: Option, + } + let Some(b) = current().filter(|b| b.fid == fid) else { return }; + let Ok(e) = serde_json::from_str::(json) else { return }; + if !matches!(e.t.as_str(), "insert" | "update") { + return; + } + let Some(row) = e.row else { return }; + match e.table.as_str() { + "my_library_state" => { + if let Ok(row) = serde_json::from_value(row) { + observe_account(&b, row); + } + } + "my_library_membership" => { + if let Ok(row) = serde_json::from_value::(row) + && row.fid == fid + { + b.observe(row); + } + } + _ => {} + } +} +pub(crate) fn connection_failed(fid: u64, error: String) { + if let Some(b) = current().filter(|b| b.fid == fid) { + b.inner.base.set(false); + b.inner.groups.borrow_mut().clear(); + b.inner.bridge.borrow_mut().take(); + b.library.0.update(|s| { + s.checked.clear(); + s.status = if web_sys::window().is_some_and(|w| !w.navigator().on_line()) { + SyncStatus::Offline + } else { + SyncStatus::Failed(error) + }; + s.revision += 1; + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn meta(generation: u64, revision: u64) -> Metadata { + Metadata { + fid: 3, + server: Account { fid: 3, generation, revision, ..Account::default() }, + known: true, + ..Metadata::default() + } + } + #[test] + fn disk_metadata_keeps_the_latest_clear_and_never_resurrects_a_settled_operation() { + let mut current = meta(2, 4); + current.pending_clear = Some(PendingClear { operation: "a".repeat(32), expected_generation: 2, sequence: 20 }); + let stale = current.clone(); + let mut remote = meta(3, 5); + remote.clear_settled = 20; + remote.legacy_imported = true; + current.merge_disk(&remote); + current.merge_disk(&stale); + assert_eq!(current.server.generation, 3); + assert_eq!(current.server.revision, 5); + assert!(current.pending_clear.is_none()); + assert!(current.legacy_imported); + assert_eq!(current.clear_settled, 20); + } + #[test] + fn concurrent_clear_metadata_uses_the_newest_local_sequence_and_preserves_server_revision() { + let mut first = meta(1, 9); + first.sequence = 10; + first.pending_clear = Some(PendingClear { operation: "a".repeat(32), expected_generation: 1, sequence: 10 }); + let mut second = meta(1, 7); + second.sequence = 11; + second.pending_clear = Some(PendingClear { operation: "b".repeat(32), expected_generation: 1, sequence: 11 }); + first.merge_disk(&second); + assert_eq!(first.server.revision, 9); + assert_eq!(first.pending_clear.as_ref().unwrap().operation, "b".repeat(32)); + assert_eq!(first.effective().generation, 2); + let foreign = Metadata { fid: 4, ..meta(99, 99) }; + let snapshot = first.clone(); + first.merge_disk(&foreign); + assert_eq!(first, snapshot); + } + #[test] + fn private_page_cursor_is_a_strict_descending_time_hash_boundary() { + let hash = format!("0x{:040x}", 9); + let cursor = format!("100:{hash}"); + assert!(valid_cursor("")); + assert!(valid_cursor(&cursor)); + for invalid in ["0:x", "-1:x", "100:0xnope", "100:x' OR 1=1", "100:0x000000000000000000000000000000000000000A"] + { + assert!(!valid_cursor(invalid)); + } + let row = Item { hash: hash.clone(), seen_at: 100, bookmark_at: 99, ..Item::default() }; + assert!(!before(&row, &cursor, PrivateKind::Seen)); + assert!(before(&row, &cursor, PrivateKind::Bookmarks)); + assert!(before(&Item { hash: format!("0x{:040x}", 8), ..row }, &cursor, PrivateKind::Seen)); + } + #[test] + fn memory_eviction_keeps_active_and_pending_items_and_preserves_recently_touched_rows() { + let mut items = BTreeMap::new(); + let mut touches = BTreeMap::new(); + for i in 0..2003 { + let hash = format!("0x{i:040x}"); + items.insert(hash.clone(), empty(3, &hash, 0)); + touches.insert(hash, i); + } + let active = format!("0x{:040x}", 0); + let pending = format!("0x{:040x}", 1); + items.get_mut(&pending).unwrap().read = Some(PendingRead { + generation: 0, + expected_revision: 0, + intent: ReadIntent::Seen { at: 100 }, + sequence: 1, + }); + let evicted = evictions(&items, &BTreeSet::from([active.clone()]), &touches); + assert_eq!(evicted.len(), 3); + assert!(!evicted.contains(&active)); + assert!(!evicted.contains(&pending)); + assert_eq!(evicted, vec![format!("0x{:040x}", 2), format!("0x{:040x}", 3), format!("0x{:040x}", 4)]); + assert_eq!(items.get(&pending).unwrap().read.as_ref().unwrap().intent, ReadIntent::Seen { at: 100 }); + } + #[test] + fn corrected_clock_replay_clamps_future_observations_but_never_crosses_a_clear() { + let hash = format!("0x{:040x}", 1); + let mut item = empty(3, &hash, 2); + item.read = Some(PendingRead { + generation: 2, + expected_revision: 0, + intent: ReadIntent::Seen { at: 500 }, + sequence: 17, + }); + clamp_replay(&mut item, 200, 100); + assert_eq!(item.read.as_ref().unwrap().intent, ReadIntent::Seen { at: 200 }); + clamp_replay(&mut item, 100, 100); + assert!(item.read.is_none()); + assert_eq!(item.read_settled, 17); + } + #[test] + fn automatic_seen_and_explicit_bookmark_have_independent_pending_domains() { + let mut item = empty(3, &format!("0x{:040x}", 1), 0); + item.read = Some(PendingRead { + generation: 0, + expected_revision: 0, + intent: ReadIntent::Seen { at: 100 }, + sequence: 1, + }); + assert!(Action::Unread.pending(&item)); + assert!(!Action::Toggle.pending(&item)); + item.bookmark = Some(PendingBookmark { desired: true, expected_revision: 0, sequence: 2 }); + assert!(Action::Toggle.pending(&item)); + item.read = None; + assert!(!Action::Seen(101).pending(&item)); + } + #[test] + fn sdk_union_fold_is_order_independent_for_read_and_bookmark_revisions() { + let hash = format!("0x{:040x}", 1); + let base = Item { fid: 3, hash: hash.clone(), ..Item::default() }; + let read = Item { seen: true, seen_at: 100, seen_revision: 4, bookmark_revision: 1, ..base.clone() }; + let bookmark = + Item { bookmarked: true, bookmark_at: 101, bookmark_revision: 5, seen_revision: 2, ..base.clone() }; + let forward = fold_rows(vec![read.clone(), bookmark.clone()], 3, 0); + let reverse = fold_rows(vec![bookmark.clone(), read.clone(), base.clone()], 3, 0); + assert_eq!(forward, reverse); + let row = &forward[&hash]; + assert!(row.seen && row.bookmarked); + assert_eq!(row.seen_revision, 4); + assert_eq!(row.bookmark_revision, 5); + assert!(fold_rows(vec![Item { fid: 4, ..base.clone() }, Item { generation: 1, ..base }], 3, 0).is_empty()); + } + #[test] + fn buffered_remote_remove_after_own_ack_cannot_be_lost_or_restore_old_pending() { + let hash = format!("0x{:040x}", 1); + let base = Item { fid: 3, hash, ..Item::default() }; + let account = Account { fid: 3, ..Account::default() }; + let sent = PendingBookmark { desired: true, expected_revision: 0, sequence: 1 }; + let mut cache = + CachedItem { server: base.clone(), known: true, bookmark: Some(sent.clone()), ..CachedItem::default() }; + let own = Item { bookmarked: true, bookmark_revision: 1, bookmark_at: 100, ..base.clone() }; + let remote = Item { bookmark_revision: 2, ..base.clone() }; + apply_flight(&mut cache, Some(Flight::Bookmark(sent)), true, own, Some(remote), &account); + assert_eq!(cache.bookmarked(), Some(false)); + assert_eq!(cache.server.bookmark_revision, 2); + assert!(cache.bookmark.is_none()); + assert_eq!(cache.bookmark_settled, 1); + } +} diff --git a/crates/allthing-web/src/library_disk.rs b/crates/allthing-web/src/library_disk.rs new file mode 100644 index 00000000..0e4aceef --- /dev/null +++ b/crates/allthing-web/src/library_disk.rs @@ -0,0 +1,617 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Durable private Library cache and outbox. Point reads never enumerate the +//! history; an outbox page scans only one FID's pending-key range. A write is +//! acknowledged only by transaction completion, after both copies commit. + +use std::cell::RefCell; +use std::future::Future; +use std::pin::Pin; +use std::rc::Rc; +use std::task::{Context, Poll, Waker}; + +use allthing_core::library::{CachedItem, canonical_hash}; +use js_sys::Array; +use wasm_bindgen::{JsCast, JsValue, closure::Closure}; +use web_sys::{Event, IdbDatabase, IdbKeyRange, IdbOpenDbRequest, IdbRequest, IdbTransaction, IdbTransactionMode}; + +const DATABASE: &str = "allthing-library-v1"; +const STORE: &str = "records"; +const MAX_RECORD: usize = 65_536; +const MAX_PENDING: u32 = 100; + +fn error(value: JsValue) -> String { + crate::stdb_bridge::js_error(value) +} + +struct Completion { + settled: bool, + result: Option>, + waker: Option, +} + +impl Default for Completion { + fn default() -> Self { + Self { settled: false, result: None, waker: None } + } +} + +impl Completion { + fn finish(state: &RefCell, result: Result) { + let waker = { + let mut state = state.borrow_mut(); + if state.settled { + return; + } + state.settled = true; + state.result = Some(result); + state.waker.take() + }; + if let Some(waker) = waker { + waker.wake(); + } + } + + fn poll(state: &RefCell, cx: &Context<'_>) -> Poll> { + let mut state = state.borrow_mut(); + if let Some(result) = state.result.take() { + Poll::Ready(result) + } else { + state.waker = Some(cx.waker().clone()); + Poll::Pending + } + } +} + +/// A connection belongs to this operation, including a version-change closer. +/// Closing it does not cancel committed data or another operation's connection. +struct Database { + db: IdbDatabase, + changed: Closure, +} + +impl Database { + fn new(db: IdbDatabase) -> Result { + let close = db.clone(); + let changed = Closure::new(move || close.close()); + if let Err(failed) = db.add_event_listener_with_callback("versionchange", changed.as_ref().unchecked_ref()) { + db.close(); + return Err(error(failed)); + } + Ok(Self { db, changed }) + } +} + +impl Drop for Database { + fn drop(&mut self) { + let _ = self.db.remove_event_listener_with_callback("versionchange", self.changed.as_ref().unchecked_ref()); + self.db.close(); + } +} + +/// IndexedDB cannot cancel a pending open request. Normal listeners are owned; +/// cancellation replaces them with GC-owned one-shot upgrade-abort/success-close +/// listeners with no captured request/self cycle, so a late open leaks no DB. +struct Opening { + request: IdbOpenDbRequest, + state: Rc>>, + finished: Rc>, + success: Closure, + failed: Closure, + blocked: Closure, + upgrade: Closure, + timeout: Closure, + timer: Option, +} + +impl Opening { + fn new() -> Result { + let factory = web_sys::window() + .ok_or("No browser window")? + .indexed_db() + .map_err(error)? + .ok_or("Library storage is unavailable")?; + let request = factory.open_with_u32(DATABASE, 1).map_err(error)?; + let state = Rc::new(RefCell::new(Completion::default())); + let finished = Rc::new(std::cell::Cell::new(false)); + let success_state = Rc::downgrade(&state); + let success_request = request.clone(); + let success_finished = finished.clone(); + let failed_state = Rc::downgrade(&state); + let failed_finished = finished.clone(); + let failed_request = request.clone(); + let blocked_state = Rc::downgrade(&state); + let upgrade_state = Rc::downgrade(&state); + let upgrade_request = request.clone(); + let timeout_state = Rc::downgrade(&state); + let mut opening = Self { + request, + state, + finished, + success: Closure::new(move || { + success_finished.set(true); + let result = success_request + .result() + .map_err(error) + .and_then(|value| value.dyn_into::().map_err(error)); + if let Some(state) = success_state.upgrade() { + Completion::finish(&state, result); + } else if let Ok(db) = result { + db.close(); + } + }), + failed: Closure::new(move || { + failed_finished.set(true); + if let Some(state) = failed_state.upgrade() { + let message = failed_request + .error() + .ok() + .flatten() + .map(|error| error.message()) + .unwrap_or_else(|| "Couldn't open Library storage".into()); + Completion::finish(&state, Err(message)); + } + }), + blocked: Closure::new(move || { + if let Some(state) = blocked_state.upgrade() { + Completion::finish(&state, Err("Library storage is blocked by another tab".into())); + } + }), + upgrade: Closure::new(move || { + let result = upgrade_request + .result() + .map_err(error) + .and_then(|value| value.dyn_into::().map_err(error)) + .and_then(|db| db.create_object_store(STORE).map(|_| ()).map_err(error)); + if let Err(error) = result { + if let Some(tx) = upgrade_request.transaction() { + let _ = tx.abort(); + } + if let Some(state) = upgrade_state.upgrade() { + Completion::finish(&state, Err(error)); + } + } + }), + timeout: Closure::new(move || { + if let Some(state) = timeout_state.upgrade() { + Completion::finish(&state, Err("Library storage took too long to open".into())); + } + }), + timer: None, + }; + for (name, callback) in opening.listeners() { + opening.request.add_event_listener_with_callback(name, callback).map_err(error)?; + } + opening.timer = Some(crate::stdb_bridge::set_timeout(opening.timeout.as_ref(), 30_000)?); + Ok(opening) + } + + fn listeners(&self) -> [(&'static str, &js_sys::Function); 4] { + [ + ("success", self.success.as_ref().unchecked_ref()), + ("error", self.failed.as_ref().unchecked_ref()), + ("blocked", self.blocked.as_ref().unchecked_ref()), + ("upgradeneeded", self.upgrade.as_ref().unchecked_ref()), + ] + } +} + +impl Future for Opening { + type Output = Result; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Completion::poll(&self.state, cx) + } +} + +impl Drop for Opening { + fn drop(&mut self) { + if let Some(timer) = self.timer.take() { + crate::stdb_bridge::clear_timeout(&timer); + } + for (name, callback) in self.listeners() { + let _ = self.request.remove_event_listener_with_callback(name, callback); + } + if let Some(Ok(db)) = self.state.borrow_mut().result.take() { + db.close(); + } + if self.finished.get() { + return; + } + if let Some(tx) = self.request.transaction() { + let _ = tx.abort(); + } + let once = web_sys::AddEventListenerOptions::new(); + once.set_once(true); + let close = Closure::::new(|event: Event| { + if let Some(request) = event.target().and_then(|target| target.dyn_into::().ok()) + && let Ok(db) = request.result().and_then(|value| value.dyn_into::()) + { + db.close(); + } + }) + .into_js_value(); + let abort = Closure::::new(|event: Event| { + if let Some(request) = event.target().and_then(|target| target.dyn_into::().ok()) + && let Some(tx) = request.transaction() + { + let _ = tx.abort(); + } + }) + .into_js_value(); + let _ = self.request.add_event_listener_with_callback_and_add_event_listener_options( + "success", + close.unchecked_ref(), + &once, + ); + let _ = self.request.add_event_listener_with_callback_and_add_event_listener_options( + "upgradeneeded", + abort.unchecked_ref(), + &once, + ); + } +} + +struct Transaction { + database: Database, + tx: IdbTransaction, + state: Rc>>>, + requests: Rc>>, + callbacks: Vec<(IdbRequest, Closure)>, + complete: Closure, + failed: Closure, +} + +impl Transaction { + fn new(database: Database, mode: IdbTransactionMode) -> Result { + let tx = database.db.transaction_with_str_and_mode(STORE, mode).map_err(error)?; + let state = Rc::new(RefCell::new(Completion::default())); + let requests = Rc::new(RefCell::new(Vec::::new())); + let completed_state = Rc::downgrade(&state); + let completed_requests = requests.clone(); + let failed_state = Rc::downgrade(&state); + let failed_tx = tx.clone(); + let transaction = Self { + database, + tx, + state, + requests, + callbacks: Vec::new(), + complete: Closure::new(move || { + if let Some(state) = completed_state.upgrade() { + Completion::finish( + &state, + completed_requests.borrow().iter().map(|request| request.result().map_err(error)).collect(), + ); + } + }), + failed: Closure::new(move || { + if let Some(state) = failed_state.upgrade() { + let message = failed_tx + .error() + .map(|error| error.message()) + .unwrap_or_else(|| "Library storage transaction aborted".into()); + Completion::finish(&state, Err(message)); + } + }), + }; + for (name, callback) in transaction.listeners() { + transaction.tx.add_event_listener_with_callback(name, callback).map_err(error)?; + } + Ok(transaction) + } + fn listeners(&self) -> [(&'static str, &js_sys::Function); 3] { + [ + ("complete", self.complete.as_ref().unchecked_ref()), + ("error", self.failed.as_ref().unchecked_ref()), + ("abort", self.failed.as_ref().unchecked_ref()), + ] + } + fn store(&self) -> Result { + self.tx.object_store(STORE).map_err(error) + } + fn read(&self, request: IdbRequest) { + self.requests.borrow_mut().push(request); + } + fn on_success(&mut self, request: IdbRequest, callback: Closure) -> Result<(), String> { + request.add_event_listener_with_callback("success", callback.as_ref().unchecked_ref()).map_err(error)?; + self.callbacks.push((request, callback)); + Ok(()) + } +} + +impl Future for Transaction { + type Output = Result, String>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Completion::poll(&self.state, cx) + } +} + +impl Drop for Transaction { + fn drop(&mut self) { + for (name, callback) in self.listeners() { + let _ = self.tx.remove_event_listener_with_callback(name, callback); + } + for (request, callback) in &self.callbacks { + let _ = request.remove_event_listener_with_callback("success", callback.as_ref().unchecked_ref()); + } + if !self.state.borrow().settled { + let _ = self.tx.abort(); + } + // Keep the connection owned through commit/error/cancellation. + self.database.db.close(); + } +} + +async fn transaction(mode: IdbTransactionMode) -> Result { + Transaction::new(Database::new(Opening::new()?.await?)?, mode) +} + +fn prefix(kind: &str, fid: u64) -> Result { + if fid == 0 { + return Err("Library storage requires a signed-in account".into()); + } + Ok(format!("{kind}:{fid:020}:")) +} + +fn item_key(kind: &str, fid: u64, hash: &str) -> Result { + if canonical_hash(hash).as_deref() != Some(hash) { + return Err("Invalid Library cast hash".into()); + } + Ok(format!("{}{hash}", prefix(kind, fid)?)) +} + +fn encoded_item(fid: u64, item: &CachedItem) -> Result { + item_key("cache", fid, &item.server.hash)?; + if item.server.fid != fid { + return Err("Library cache belongs to another account".into()); + } + let json = serde_json::to_string(item).map_err(|_| "Couldn't encode Library storage".to_owned())?; + if json.len() > MAX_RECORD { + return Err("Library record is too large".into()); + } + Ok(json) +} + +fn decoded_item(fid: u64, value: JsValue) -> Result { + let json = value.as_string().ok_or("Unreadable Library cache record")?; + if json.len() > MAX_RECORD { + return Err("Library record is too large".into()); + } + let item: CachedItem = serde_json::from_str(&json).map_err(|_| "Unreadable Library cache record")?; + encoded_item(fid, &item)?; + Ok(item) +} + +/// Read only the requested points; a cold feed never loads the entire history. +pub async fn items(fid: u64, hashes: &[String]) -> Result, String> { + prefix("cache", fid)?; + let mut unique = std::collections::HashSet::new(); + let keys: Vec<_> = hashes + .iter() + .filter(|hash| unique.insert(hash.as_str())) + .map(|hash| item_key("cache", fid, hash).map(|key| (hash, key))) + .collect::>()?; + if keys.is_empty() { + return Ok(Vec::new()); + } + let tx = transaction(IdbTransactionMode::Readonly).await?; + let store = tx.store()?; + for (_, key) in &keys { + tx.read(store.get(&JsValue::from_str(key)).map_err(error)?); + } + let found = tx.await?; + let mut items = Vec::new(); + for ((hash, _), value) in keys.into_iter().zip(found) { + if value.is_undefined() { + continue; + } + let item = decoded_item(fid, value)?; + if item.server.hash != *hash { + return Err("Library cache hash does not match its key".into()); + } + items.push(item); + } + Ok(items) +} + +/// Cache and outbox updates share one commit; an aborted/quota-limited write +/// cannot leave one new while the other is old. No pending item is evicted. +pub async fn keep_item(fid: u64, item: &CachedItem) -> Result<(), String> { + encoded_item(fid, item)?; + let incoming = item.clone(); + let (cache, pending) = (item_key("cache", fid, &item.server.hash)?, item_key("pending", fid, &item.server.hash)?); + let mut tx = transaction(IdbTransactionMode::Readwrite).await?; + let store = tx.store()?; + let request = store.get(&JsValue::from_str(&cache)).map_err(error)?; + let read = request.clone(); + let state = Rc::downgrade(&tx.state); + let abort = tx.tx.clone(); + let callback = Closure::::new(move || { + // This callback runs while IDB's read/write lock is still active. + // Queue both writes synchronously; awaiting between them could commit + // the transaction before the second tab's merge is durable. + let merged = (|| -> Result<(), String> { + let previous = read.result().map_err(error)?; + let item = if previous.is_undefined() { + incoming.clone() + } else { + let mut previous = decoded_item(fid, previous)?; + if previous.server.hash != incoming.server.hash { + return Err("Library cache hash does not match its key".into()); + } + previous.merge_disk(&incoming); + previous + }; + // Keep validation and serialization after the merge as well. + let json = encoded_item(fid, &item)?; + store.put_with_key(&JsValue::from_str(&json), &JsValue::from_str(&cache)).map_err(error)?; + if item.pending() { + store.put_with_key(&JsValue::from_str(&json), &JsValue::from_str(&pending)).map_err(error)?; + } else { + store.delete(&JsValue::from_str(&pending)).map_err(error)?; + } + Ok(()) + })(); + if let Err(error) = merged { + if let Some(state) = state.upgrade() { + Completion::finish(&state, Err(error)); + } + let _ = abort.abort(); + } + }); + tx.on_success(request, callback)?; + tx.await.map(|_| ()) +} + +/// Scan just this account's outbox, at most100 records; the cache can be much +/// larger. A caller drains later pages by acknowledging/removing pending intent. +pub async fn pending(fid: u64, limit: u32) -> Result, String> { + let lower = prefix("pending", fid)?; + let limit = limit.min(MAX_PENDING); + if limit == 0 { + return Ok(Vec::new()); + } + let range = IdbKeyRange::bound(&JsValue::from_str(&lower), &JsValue::from_str(&format!("{lower}\u{ffff}"))) + .map_err(error)?; + let tx = transaction(IdbTransactionMode::Readonly).await?; + tx.read(tx.store()?.get_all_with_key_and_limit(range.as_ref(), limit).map_err(error)?); + let rows = tx.await?; + let values = Array::from(rows.first().ok_or("Library outbox did not answer")?); + let mut found = Vec::new(); + for value in values.iter() { + let item = decoded_item(fid, value)?; + if !item.pending() { + return Err("Library outbox contains a settled record".into()); + } + found.push(item); + } + Ok(found) +} + +fn account_json(fid: u64, json: &str) -> Result<(), String> { + prefix("account", fid)?; + if json.len() > MAX_RECORD { + return Err("Library account metadata is too large".into()); + } + let value: serde_json::Value = serde_json::from_str(json).map_err(|_| "Unreadable Library account metadata")?; + if !value.is_object() || value.get("fid").and_then(serde_json::Value::as_u64) != Some(fid) { + return Err("Library account metadata belongs to another account".into()); + } + Ok(()) +} + +pub async fn account(fid: u64) -> Result, String> { + let key = prefix("account", fid)?; + let tx = transaction(IdbTransactionMode::Readonly).await?; + tx.read(tx.store()?.get(&JsValue::from_str(&key)).map_err(error)?); + let rows = tx.await?; + let value = rows.first().ok_or("Library account metadata did not answer")?; + if value.is_undefined() { + return Ok(None); + } + let json = value.as_string().ok_or("Unreadable Library account metadata")?; + account_json(fid, &json)?; + Ok(Some(json)) +} + +pub async fn keep_account(fid: u64, value: &str) -> Result<(), String> { + account_json(fid, value)?; + let incoming: crate::library::Metadata = + serde_json::from_str(value).map_err(|_| "Unreadable Library account metadata")?; + let key = prefix("account", fid)?; + let mut tx = transaction(IdbTransactionMode::Readwrite).await?; + let store = tx.store()?; + let request = store.get(&JsValue::from_str(&key)).map_err(error)?; + let read = request.clone(); + let state = Rc::downgrade(&tx.state); + let abort = tx.tx.clone(); + let callback = Closure::::new(move || { + let result = (|| -> Result<(), String> { + let previous = read.result().map_err(error)?; + let mut merged = if previous.is_undefined() { + incoming.clone() + } else { + let json = previous.as_string().ok_or("Unreadable Library account metadata")?; + account_json(fid, &json)?; + let mut previous: crate::library::Metadata = + serde_json::from_str(&json).map_err(|_| "Unreadable Library account metadata")?; + previous.merge_disk(&incoming); + previous + }; + merged.fid = fid; + let json = serde_json::to_string(&merged).map_err(|e| e.to_string())?; + account_json(fid, &json)?; + store.put_with_key(&JsValue::from_str(&json), &JsValue::from_str(&key)).map_err(error)?; + Ok(()) + })(); + if let Err(error) = result { + if let Some(state) = state.upgrade() { + Completion::finish(&state, Err(error)); + } + let _ = abort.abort(); + } + }); + tx.on_success(request, callback)?; + tx.await.map(|_| ()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use std::task::Wake; + + struct Woken(AtomicUsize); + impl Wake for Woken { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + #[test] + fn aborted_commit_cannot_be_replaced_by_a_late_success_callback() { + let woke = Arc::new(Woken(AtomicUsize::new(0))); + let state = + RefCell::new(Completion:: { waker: Some(Waker::from(woke.clone())), ..Completion::default() }); + Completion::finish(&state, Err("QuotaExceededError".into())); + Completion::finish(&state, Ok(7)); + assert_eq!(state.borrow_mut().result.take(), Some(Err("QuotaExceededError".into()))); + assert!(state.borrow().settled); + assert_eq!(woke.0.load(Ordering::Relaxed), 1); + } + + #[test] + fn completion_before_the_first_poll_is_retained() { + let state = RefCell::new(Completion::default()); + Completion::finish(&state, Ok(9usize)); + let woke = Arc::new(Woken(AtomicUsize::new(0))); + let waker = Waker::from(woke.clone()); + assert_eq!(Completion::poll(&state, &Context::from_waker(&waker)), Poll::Ready(Ok(9))); + assert!(state.borrow().settled); + assert_eq!(woke.0.load(Ordering::Relaxed), 0); + } + + #[test] + fn outbox_prefixes_are_account_exact_and_records_are_typed() { + let hash = format!("0x{:040x}", 1); + let key = item_key("pending", 7, &hash).unwrap(); + assert!(key.starts_with(&prefix("pending", 7).unwrap())); + for other in [1, 70, 71, u64::MAX] { + assert!(!key.starts_with(&prefix("pending", other).unwrap())); + } + assert!(item_key("cache", 0, &hash).is_err()); + assert!(item_key("cache", 7, "malformed").is_err()); + let mut item = CachedItem::default(); + item.server.fid = 7; + item.server.hash = hash; + assert!(encoded_item(7, &item).is_ok()); + assert!(encoded_item(8, &item).is_err()); + assert!(account_json(7, r#"{"fid":7,"server":{"generation":1},"operation_sequence":20}"#).is_ok()); + for json in [r#"{"fid":8}"#, r#"{"fid":"7"}"#, "[]", "{"] { + assert!(account_json(7, json).is_err()); + } + assert!(account_json(7, &format!(r#"{{"fid":7,"padding":"{}"}}"#, "x".repeat(MAX_RECORD))).is_err()); + } +} diff --git a/crates/allthing-web/src/notification_sync.rs b/crates/allthing-web/src/notification_sync.rs index ccfae413..6f32626d 100644 --- a/crates/allthing-web/src/notification_sync.rs +++ b/crates/allthing-web/src/notification_sync.rs @@ -34,7 +34,8 @@ struct Sync { } struct Inner { - not_before: f64, + not_before: Cell, + urgent: Cell, stopped: Cell, connecting: Cell, disconnected: Cell, @@ -60,6 +61,7 @@ fn visible_online() -> bool { /// Called by the app's account effect, including sign-out. A stopped /// instance fences every asynchronous completion, even after A → B → A. pub fn account(fid: Option, notifications: Notifications) { + crate::library::account(fid); if CURRENT.with(|s| s.borrow().as_ref().map(|s| s.fid)) == fid { return; } @@ -71,7 +73,8 @@ pub fn account(fid: Option, notifications: Notifications) { fid, notifications, inner: Arc::new(SendWrapper::new(Inner { - not_before: js_sys::Date::now() + 2000.0, + not_before: Cell::new(js_sys::Date::now() + 2000.0), + urgent: Cell::new(false), stopped: Cell::new(false), connecting: Cell::new(false), disconnected: Cell::new(false), @@ -111,6 +114,17 @@ pub fn wake() { } } +/// Foreground Library/feed readiness must not wait for an unrelated signer +/// request. It still yields to browser idle and requires a visible online page; +/// background-only notification sync retains its API-quiet startup delay. +pub(crate) fn wake_urgent() { + if let Some(sync) = CURRENT.with(|s| s.borrow().clone()) { + sync.inner.urgent.set(true); + sync.inner.not_before.set(0.0); + wake(); + } +} + impl Sync { fn active(&self) -> bool { !self.inner.stopped.get() && self.notifications.are_for(self.fid) @@ -136,13 +150,15 @@ impl Sync { } if sync.inner.ready.get() { sync.flush(); - } else if !api::quiet() { + } else if !sync.inner.urgent.get() && !api::quiet() { sync.schedule(250); } else { crate::components::cast::idle(move || sync.connect()); } }, - Duration::from_millis(millis.max((self.inner.not_before - js_sys::Date::now()).max(0.0).ceil() as u64)), + Duration::from_millis( + millis.max((self.inner.not_before.get() - js_sys::Date::now()).max(0.0).ceil() as u64), + ), ); *self.inner.timer.borrow_mut() = timer.ok(); } @@ -190,11 +206,13 @@ impl Sync { sync.detach(); return; } - if result.is_err() { + if let Err(error) = result { + crate::library::connection_failed(sync.fid, error); sync.retry(); return; } sync.inner.failures.set(0); + sync.inner.urgent.set(false); sync.inner.ready.set(true); sync.flush(); }); @@ -213,14 +231,31 @@ impl Sync { // service. Unconfigured servers/fixtures never load the SDK or dial. #[derive(serde::Deserialize)] struct Status { + #[serde(default)] spacetime: Capability, } - #[derive(serde::Deserialize)] + #[derive(Default, serde::Deserialize)] struct Capability { + #[serde(default)] attest: bool, + #[serde(default)] + library: bool, } - let status = api::get::("/api/v1/status").await.map_err(|e| e.message())?; + let status = match api::get::("/api/v1/status").await { + Ok(status) => status, + Err(api::Failure::NotFound(error)) => { + crate::library::capability(self.fid, Some(false)); + self.inner.urgent.set(false); + return Err(error); + } + Err(error) => return Err(error.message()), + }; self.check()?; + let library = status.data.spacetime.library; + crate::library::capability(self.fid, Some(library)); + if !library { + self.inner.urgent.set(false); + } if !status.data.spacetime.attest { return Err("Notification sync is not configured".into()); } @@ -278,8 +313,23 @@ impl Sync { ) .await?; self.check()?; - bounded(self.raw("subscribe", &[QUERY.into()])?).await?; + if library { + crate::library::bind( + self.fid, + self.inner.attempt.get(), + self.inner.bridge.borrow().clone().ok_or("Not connected")?, + ); + } + let query = if library { + "[\"SELECT * FROM my_notification_state\",\"SELECT * FROM my_library_state\"]" + } else { + QUERY + }; + bounded(self.raw("subscribe", &[query.into()])?).await?; self.check()?; + if library { + crate::library::base_applied(self.fid)?; + } self.inner.state.borrow_mut().hydrated(); let state = self.inner.state.borrow(); self.notifications.synced(self.fid, state.seen(), state.baseline()); @@ -297,6 +347,7 @@ impl Sync { } fn event(&self, json: &str) { + crate::library::event(self.fid, json); #[derive(serde::Deserialize)] struct Event { t: String, @@ -312,6 +363,10 @@ impl Sync { let Ok(event) = serde_json::from_str::(json) else { return }; if event.t == "disconnected" { self.inner.disconnected.set(true); + crate::library::connection_failed( + self.fid, + "Library connection interrupted; your offline changes are kept".into(), + ); self.retry(); } else if event.table == "my_notification_state" && matches!(event.t.as_str(), "insert" | "update") @@ -350,7 +405,7 @@ impl Sync { } /// A stalled connection/subscription/reducer cannot hold a retry forever. -async fn bounded(value: JsValue) -> Result { +pub(crate) async fn bounded(value: JsValue) -> Result { let timer = RefCell::new(None); let timeout = Promise::new(&mut |_, reject| { let reject = reject.bind1(&JsValue::NULL, &"Notification sync timed out".into()); diff --git a/crates/allthing-web/src/pages/home.rs b/crates/allthing-web/src/pages/home.rs index 23db731e..0fc05278 100644 --- a/crates/allthing-web/src/pages/home.rs +++ b/crates/allthing-web/src/pages/home.rs @@ -53,7 +53,8 @@ thread_local! { #[component] pub fn Home() -> impl IntoView { let query = use_query_map(); - let sort = Memo::new(move |_| Sort::parse(query.read().get("sort").as_deref(), Sort::Hot)); + let feed = feed_memo(); + let sort = Memo::new(move |_| feed.get().sort(query.read().get("sort").as_deref())); let range = Memo::new(move |_| TimeRange::parse(query.read().get("t").as_deref())); // Its title whenever it shows; kept while another page shows (kept.rs), // it leaves that page's alone. @@ -77,7 +78,6 @@ pub fn Home() -> impl IntoView { // between them re-sorts in place). Following lists the casts of whom a // signed-in reader follows, in all three. let newest = Memo::new(move |_| listed.get() == Sort::New); - let feed = feed_memo(); let who = reader(); let previous = StoredValue::new(who.get_untracked()); let identity = Memo::new(move |_| { @@ -615,13 +615,13 @@ fn Heading(sort: Memo, range: Memo) -> impl IntoView { let client = client_param(&query.read()).map(|fid| fid.to_string()); // So does the tab. let tab = feed.get().param(); - let sort_kept = (sort.get() != Sort::Hot).then(|| sort.get().as_str().to_owned()); + let default_sort = feed.get().default_sort(); view! { - {signed_in.get().then(|| view! { })} + {signed_in.get().then(|| view! { })}
- + { - view! { } + view! { } }
} @@ -632,10 +632,10 @@ fn Heading(sort: Memo, range: Memo) -> impl IntoView { /// A signed-in reader's two feeds, as X's: **For you** (the trending pool, /// with the casts of whom they follow lifted) and **Following** (only -/// theirs, `?feed=following`). Links, keeping the sort, the time range and -/// the client filter; signed out there is only For you, and no tabs. +/// theirs, `?feed=following`). Entering a tab selects its default sort while +/// keeping the time range and client filter. Signed out there are no tabs. #[component] -fn FeedTabs(current: Feed, sort: Option, t: Option, client: Option) -> impl IntoView { +fn FeedTabs(current: Feed, t: Option, client: Option) -> impl IntoView { view! {