From bd52fb04e497c37e36252cad712f48e1c2d15b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentino=20Me=C4=91imorec?= Date: Wed, 21 Oct 2026 16:45:15 +0200 Subject: [PATCH 1/2] Item history: versioned items and signed deletions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every write carries an item_version inside its signed envelope, and a deletion is a signed envelope too. On pull, a device refuses a signed copy older than the last one the server confirmed. Changes from 0.2.0 apps (no version, unsigned deletions) still apply. Strict mode refuses them; it's built but off. The reference server keeps a deletion's envelope (at most 1 KiB, features.signed_tombstones). vault-conform checks it. Spec and docs updated. Signed-off-by: Valentino Međimorec --- ROADMAP.md | 5 + crates/mg-store/src/error.rs | 16 + crates/mg-store/src/history.rs | 159 +++++++ crates/mg-store/src/lib.rs | 1 + crates/mg-store/src/moving.rs | 6 +- crates/mg-store/src/schema.rs | 8 + crates/mg-store/src/sync.rs | 559 ++++++++++++++++++++--- crates/mg-store/src/vault.rs | 118 ++++- crates/mg-sync/src/types.rs | 7 + docs/features/reference-server.md | 1 + docs/features/sync.md | 17 +- docs/features/vault-and-unlock.md | 2 +- docs/features/vault-crypto.md | 4 +- docs/sync-crypto-spec.md | 38 +- server/src/api.rs | 26 +- server/tests/protocol.rs | 39 +- spec/openapi.yaml | 16 +- tools/vault-conform/src/checks/vaults.rs | 73 ++- tools/vault-conform/tests/reference.rs | 4 + 19 files changed, 991 insertions(+), 108 deletions(-) create mode 100644 crates/mg-store/src/history.rs diff --git a/ROADMAP.md b/ROADMAP.md index a39e1a5..3c17081 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -45,6 +45,11 @@ xterm.js in the webview avoids writing a terminal renderer. with progress, cancel and replace, mkdir/rename/chmod/delete - [x] `mg-sync`: vault-sync/v1 client (prelogin/register/login, token rotation, pull/push) - [x] Sync engine in `mg-store`: paginated pull, rollback guard, conflict copies, sealed refresh token +- [x] Item history: every write carries a version inside its signed envelope, deletions are signed, and + a device refuses a copy of an item older than one it has seen (servers keep the signed deletion: + `signed_tombstones`) +- [ ] Strict item history on by default (refusing unversioned changes, which only apps before item history + send), and signed vault checkpoints for devices with no copy to compare - [x] Reference sync server (`server/`, axum + SQLite) with protocol + two-device e2e tests - [x] Desktop sync: enable from an existing vault, sign in on a fresh device, background sync, disconnect - [x] Password change / recovery / new recovery key while synced (server first, other devices sign in again diff --git a/crates/mg-store/src/error.rs b/crates/mg-store/src/error.rs index d305bf5..4f03cea 100644 --- a/crates/mg-store/src/error.rs +++ b/crates/mg-store/src/error.rs @@ -30,6 +30,22 @@ pub enum Error { Untrusted(String), #[error("vault data is corrupt: {0}")] Corrupt(String), + /// A signed copy of an item older than one this device has seen: a server serving an old + /// envelope again. + #[error( + "the sync server sent an older copy of item {item} (version {remote}) than this device already has (version {seen}); refusing to roll it back" + )] + OlderItem { + item: String, + seen: i64, + remote: i64, + }, + /// With strict item history: a change without a version, or a deletion without a + /// signature, to an item that has one. + #[error( + "the sync server sent a change to item {item} that isn't signed with a version; refusing it" + )] + UnversionedItem { item: String }, #[error(transparent)] Crypto(#[from] mg_vault::Error), #[error("database: {0}")] diff --git a/crates/mg-store/src/history.rs b/crates/mg-store/src/history.rs new file mode 100644 index 0000000..7f710f4 --- /dev/null +++ b/crates/mg-store/src/history.rs @@ -0,0 +1,159 @@ +//! Item history: the version every write carries inside its signed envelope, and signed +//! deletions, so a server can't serve a device an older copy of an item than one it has seen. +//! +//! An item's plaintext is its JSON with an `item_version`: the time of the write in milliseconds, +//! or one more than the highest version the writer knows of the item when that is larger. A +//! deletion is an envelope too, `{"type":"deleted","item_version":n}`, sealed for the same item. +//! Apps before item history (0.2.0) ignore both: they write items without a version and +//! deletions without an envelope, which a device accepts while such apps may still be in use +//! (see `LocalVault::set_strict_history`). + +use serde::Deserialize; +use zeroize::Zeroizing; + +use crate::vault::now; +use crate::{Error, ItemData, Result}; + +/// The plaintext field that holds an item's version. +const FIELD: &str = "item_version"; +/// The `type` of a deletion's plaintext. +const DELETED: &str = "deleted"; + +/// What an envelope says: an item (`data`) or a deletion (`None`), and its version (`None`: +/// written by an app before item history). +#[derive(Debug)] +pub(crate) struct Copy { + pub data: Option, + pub version: Option, +} + +/// The version of a write after `known`, the highest version the writer has of the item. +pub(crate) fn next(known: i64) -> i64 { + known.saturating_add(1).max(now()) +} + +/// `data` as an item's plaintext with `version`. +pub(crate) fn plaintext(data: &ItemData, version: i64) -> Result>> { + let json = Zeroizing::new(serde_json::to_vec(data).map_err(|e| Error::Invalid(e.to_string()))?); + // An item is always a JSON object with a `type`: the version goes in before its end. + let (end, body) = json + .split_last() + .filter(|(end, body)| **end == b'}' && body.len() > 1) + .ok_or_else(|| Error::Invalid("an item must be a JSON object".into()))?; + let field = format!(",\"{FIELD}\":{version}"); + let mut out = Zeroizing::new(Vec::with_capacity(json.len() + field.len())); + out.extend_from_slice(body); + out.extend_from_slice(field.as_bytes()); + out.push(*end); + Ok(out) +} + +/// A deletion's plaintext. +pub(crate) fn deletion(version: i64) -> Vec { + format!("{{\"type\":\"{DELETED}\",\"{FIELD}\":{version}}}").into_bytes() +} + +/// `plain` (an item or a deletion, of any kind, from any app) with its version set to `version`. +pub(crate) fn with_version(plain: &[u8], version: i64) -> Result>> { + let mut value: serde_json::Value = + serde_json::from_slice(plain).map_err(|e| Error::Corrupt(e.to_string()))?; + let object = value + .as_object_mut() + .ok_or_else(|| Error::Corrupt("an item must be a JSON object".into()))?; + object.insert(FIELD.into(), version.into()); + Ok(Zeroizing::new( + serde_json::to_vec(&value).map_err(|e| Error::Invalid(e.to_string()))?, + )) +} + +/// Reads an opened envelope of item `id`. +pub(crate) fn read(id: &str, plain: &[u8]) -> Result { + #[derive(Deserialize)] + struct Head { + #[serde(rename = "type")] + kind: String, + #[serde(default)] + item_version: Option, + } + let corrupt = |e: serde_json::Error| Error::Corrupt(format!("item {id}: {e}")); + let head: Head = serde_json::from_slice(plain).map_err(corrupt)?; + // Versions start at 1; anything else counts as none. + let version = head.item_version.filter(|v| *v > 0); + let data = if head.kind == DELETED { + None + } else { + Some(serde_json::from_slice(plain).map_err(corrupt)?) + }; + Ok(Copy { data, version }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Snippet; + + fn snippet() -> ItemData { + ItemData::Snippet(Snippet { + name: "uptime".into(), + body: "uptime".into(), + tags: vec![], + }) + } + + #[test] + fn items_and_deletions_carry_their_version() { + let plain = plaintext(&snippet(), 1_760_000_000_123).unwrap(); + assert_eq!( + std::str::from_utf8(&plain).unwrap(), + r#"{"type":"snippet","name":"uptime","body":"uptime","item_version":1760000000123}"# + ); + let copy = read("I", &plain).unwrap(); + assert_eq!( + (copy.data, copy.version), + (Some(snippet()), Some(1_760_000_000_123)) + ); + + let gone = deletion(7); + assert_eq!(gone, br#"{"type":"deleted","item_version":7}"#); + let copy = read("I", &gone).unwrap(); + assert_eq!((copy.data, copy.version), (None, Some(7))); + } + + #[test] + fn items_from_older_apps_have_no_version() { + let plain = serde_json::to_vec(&snippet()).unwrap(); + let copy = read("I", &plain).unwrap(); + assert_eq!((copy.data, copy.version), (Some(snippet()), None)); + // Not a version: zero or below. + let copy = read( + "I", + br#"{"type":"snippet","name":"a","body":"b","item_version":0}"#, + ) + .unwrap(); + assert_eq!(copy.version, None); + } + + #[test] + fn a_new_version_keeps_what_this_app_doesnt_know() { + let plain = br#"{"type":"future_kind","shape":{"x":1},"item_version":3}"#; + let copy = read("I", plain).unwrap(); + assert_eq!( + (copy.data, copy.version), + (Some(ItemData::Unknown), Some(3)) + ); + let again: serde_json::Value = + serde_json::from_slice(&with_version(plain, 9).unwrap()).unwrap(); + assert_eq!( + again, + serde_json::json!({"type": "future_kind", "shape": {"x": 1}, "item_version": 9}) + ); + } + + #[test] + fn versions_only_grow() { + let far_ahead = now() + 86_400_000; + assert_eq!(next(far_ahead), far_ahead + 1); + assert!(next(0) >= now() - 1000); + assert_eq!(next(i64::MAX), i64::MAX); + } +} diff --git a/crates/mg-store/src/lib.rs b/crates/mg-store/src/lib.rs index 126ae57..ff99a3f 100644 --- a/crates/mg-store/src/lib.rs +++ b/crates/mg-store/src/lib.rs @@ -10,6 +10,7 @@ mod blobs; mod device; mod error; pub mod export; +mod history; pub mod model; pub mod moving; mod schema; diff --git a/crates/mg-store/src/moving.rs b/crates/mg-store/src/moving.rs index 3a5d92a..6c065e7 100644 --- a/crates/mg-store/src/moving.rs +++ b/crates/mg-store/src/moving.rs @@ -311,11 +311,7 @@ impl LocalVault { let done = (|| -> Result { let stamp = now(); for p in going.iter().filter(|p| p.carry == Carry::Move) { - self.conn.execute( - "UPDATE items SET envelope = NULL, deleted = 1, dirty = 1, updated_at = ?3 - WHERE vault_id = ?1 AND id = ?2", - rusqlite::params![from, p.id, stamp], - )?; + self.tombstone(&from, &p.id, stamp)?; } let mut forget = Vec::new(); for (id, data, old_blob) in &mut writes { diff --git a/crates/mg-store/src/schema.rs b/crates/mg-store/src/schema.rs index 3015600..0815d03 100644 --- a/crates/mg-store/src/schema.rs +++ b/crates/mg-store/src/schema.rs @@ -145,6 +145,14 @@ const MIGRATIONS: &[&str] = &[ ", // v7: where to get the plan a refused blob needs. "ALTER TABLE blobs ADD COLUMN upgrade_url TEXT;", + // v8: item history. `version`: the one in the item's envelope (0: none, from an app before + // item history). `seen_version`: the one of the last copy the server confirmed, pulled or + // pushed (0: it had none). NULL in both: not read yet, from before this migration. A + // deletion made on this device keeps its signed envelope in `envelope`. + " + ALTER TABLE items ADD COLUMN version INTEGER; + ALTER TABLE items ADD COLUMN seen_version INTEGER; + ", ]; pub(crate) fn migrate(conn: &mut Connection) -> Result<()> { diff --git a/crates/mg-store/src/sync.rs b/crates/mg-store/src/sync.rs index 933ead1..ef2db27 100644 --- a/crates/mg-store/src/sync.rs +++ b/crates/mg-store/src/sync.rs @@ -17,8 +17,10 @@ use rusqlite::{OptionalExtension, params}; use ulid::Ulid; use zeroize::Zeroizing; -use crate::vault::{Access, Unlocked, context, decrypt, envelope_key_id, now}; -use crate::{BlobKind, Error, ItemData, LocalVault, Result}; +use crate::vault::{ + Access, Unlocked, context, decrypt, envelope_key_id, now, open_envelope, version_in, +}; +use crate::{BlobKind, Error, ItemData, LocalVault, Result, history}; /// Pseudo item id under which the refresh token is sealed (never a valid ULID, so no clash). const TOKEN_ID: &str = "sync:refresh-token"; @@ -90,9 +92,12 @@ pub struct Outgoing { pub id: String, pub base_rev: i64, pub deleted: bool, + /// The item, or a deletion's signed envelope (`None`: a deletion from before item history). pub envelope: Option>, /// Local `updated_at` when read; an edit during the push keeps the row dirty. pub updated_at: i64, + /// The version in `envelope` (`None`: not known). + pub version: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -609,7 +614,7 @@ impl LocalVault { [vault], )?; let mut stmt = self.conn.prepare( - "SELECT id, rev, deleted, envelope, updated_at FROM items + "SELECT id, rev, deleted, envelope, updated_at, version FROM items WHERE vault_id = ?1 AND dirty = 1 ORDER BY updated_at, id LIMIT ?2", )?; let rows = stmt.query_map(params![vault, (limit + skip.len()) as i64], |r| { @@ -619,6 +624,7 @@ impl LocalVault { deleted: r.get(2)?, envelope: r.get(3)?, updated_at: r.get(4)?, + version: r.get(5)?, }) })?; let mut out = Vec::new(); @@ -635,26 +641,47 @@ impl LocalVault { } /// Applies server items in one transaction; every envelope is verified (author and vault - /// key) before anything is written. Returns (items applied, conflicts). `cursor` advances - /// `head_rev` when given. + /// key) before anything is written, and no item goes back to an older version than this + /// device has seen of it. Returns (items applied, conflicts). `cursor` advances `head_rev` + /// when given. fn apply_remote( &mut self, vault: &str, items: &[RemoteItem], cursor: Option, ) -> Result<(usize, usize)> { + let strict = self.strict_history; let keys = self.keys.as_ref().ok_or(Error::Locked)?; - for r in items.iter().filter(|r| !r.deleted) { - let env = r - .envelope - .as_deref() - .ok_or_else(|| Error::Corrupt(format!("item {} has no envelope", r.id)))?; - decrypt(keys, vault, &r.id, env, r.author_id.as_deref())?; + let mut copies = Vec::with_capacity(items.len()); + for r in items { + let copy = match (&r.envelope, r.deleted) { + (Some(env), false) => { + let plain = open_envelope(keys, vault, &r.id, env, r.author_id.as_deref())?; + history::read(&r.id, &plain)? + } + (None, false) => { + return Err(Error::Corrupt(format!("item {} has no envelope", r.id))); + } + // A deletion is signed when it carries its author's deletion envelope; anything + // else counts as unsigned (as from an app before item history). + (Some(env), true) => open_envelope(keys, vault, &r.id, env, r.author_id.as_deref()) + .and_then(|plain| history::read(&r.id, &plain)) + .ok() + .filter(|copy| copy.data.is_none()) + .unwrap_or(history::Copy { + data: None, + version: None, + }), + (None, true) => history::Copy { + data: None, + version: None, + }, + }; // Items are looked up by id: a shared vault must not bring in a new one that would // stand in for a live item of the personal vault. Moving between vaults keeps ids (the // old copy becomes a tombstone, and the personal vault syncs first in every round), // and an id this vault already had here is no newcomer (an item moved back and forth). - if vault != keys.vault_id { + if copy.data.is_some() && vault != keys.vault_id { let clash = self .conn .query_row( @@ -671,68 +698,101 @@ impl LocalVault { ))); } } + copies.push(copy); } let (mut applied, mut conflicts) = (0, 0); let stamp = now(); let tx = self.conn.transaction()?; - for r in items { + for (r, copy) in items.iter().zip(copies) { + let deleted = copy.data.is_none(); let local: Option = tx .query_row( - "SELECT rev, dirty, deleted, envelope, author_id FROM items WHERE vault_id = ?1 AND id = ?2", + "SELECT rev, dirty, deleted, envelope, author_id, seen_version FROM items WHERE vault_id = ?1 AND id = ?2", params![vault, r.id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, ) .optional()?; - match local { - None if r.deleted => continue, - Some((rev, ..)) if rev >= r.rev => continue, - Some((_, true, local_deleted, local_env, local_author)) => { - let local_data = match (local_deleted, local_env) { - (false, Some(env)) => { - Some(decrypt(keys, vault, &r.id, &env, local_author.as_deref())?) - } - _ => None, - }; - let remote_data = match (&r.deleted, &r.envelope) { - (false, Some(env)) => { - Some(decrypt(keys, vault, &r.id, env, r.author_id.as_deref())?) - } - _ => None, - }; - // The same content on both sides (say, both re-sealed it): nothing lost. - let same = local_data == remote_data; - if !same { - conflicts += 1; + match &local { + None if deleted => continue, + Some((rev, ..)) if *rev >= r.rev => continue, + _ => {} + } + if let Some((_, dirty, _, local_env, local_author, seen)) = &local { + // The version of the last copy the server confirmed; for a copy from before + // item history, the one its envelope holds (unless it has a change on top). + let seen = match (seen, dirty, local_env) { + (Some(seen), ..) => *seen, + (None, false, Some(env)) => { + version_in(keys, vault, &r.id, env, local_author.as_deref()) } - // Local edit lost against the server: keep it as a separate item. - if let (false, Some(mut data)) = (same, local_data) { - mark_conflict(&mut data); - let copy_id = Ulid::generate().to_string(); - let plain = Zeroizing::new( - serde_json::to_vec(&data).map_err(|e| Error::Invalid(e.to_string()))?, - ); - let copy = envelope::seal( - keys.current(vault)?, - &keys.account, - &context(vault, ©_id), - &plain, - )?; - tx.execute( - "INSERT INTO items (id, vault_id, envelope, deleted, rev, dirty, updated_at) - VALUES (?1, ?2, ?3, 0, 0, 1, ?4)", - params![copy_id, vault, copy, stamp], - )?; + _ => 0, + }; + match copy.version { + Some(v) if v < seen => { + return Err(Error::OlderItem { + item: r.id.clone(), + seen, + remote: v, + }); + } + None if strict && seen > 0 => { + return Err(Error::UnversionedItem { item: r.id.clone() }); } + _ => {} } - _ => {} } + if let Some((_, true, local_deleted, local_env, local_author, _)) = local { + let local_data = match (local_deleted, local_env) { + (false, Some(env)) => { + Some(decrypt(keys, vault, &r.id, &env, local_author.as_deref())?) + } + _ => None, + }; + // The same content on both sides (say, both re-sealed it): nothing lost. + let same = local_data == copy.data; + if !same { + conflicts += 1; + } + // Local edit lost against the server: keep it as a separate item. + if let (false, Some(mut data)) = (same, local_data) { + mark_conflict(&mut data); + let copy_id = Ulid::generate().to_string(); + let version = history::next(0); + let plain = history::plaintext(&data, version)?; + let sealed = envelope::seal( + keys.current(vault)?, + &keys.account, + &context(vault, ©_id), + &plain, + )?; + tx.execute( + "INSERT INTO items (id, vault_id, envelope, deleted, rev, dirty, updated_at, version, seen_version) + VALUES (?1, ?2, ?3, 0, 0, 1, ?4, ?5, 0)", + params![copy_id, vault, sealed, stamp, version], + )?; + } + } + // Without a version (an app before item history), the next versioned copy may be + // from a device that never saw the earlier ones: it starts over. + let version = copy.version.unwrap_or(0); tx.execute( - "INSERT INTO items (id, vault_id, envelope, deleted, rev, dirty, updated_at, author_id) - VALUES (?1, ?2, ?3, ?4, ?5, 0, ?6, ?7) + "INSERT INTO items (id, vault_id, envelope, deleted, rev, dirty, updated_at, author_id, version, seen_version) + VALUES (?1, ?2, ?3, ?4, ?5, 0, ?6, ?7, ?8, ?8) ON CONFLICT (vault_id, id) DO UPDATE SET envelope = excluded.envelope, deleted = excluded.deleted, - rev = excluded.rev, dirty = 0, updated_at = excluded.updated_at, author_id = excluded.author_id", - params![r.id, vault, if r.deleted { None } else { r.envelope.as_deref() }, r.deleted, r.rev, stamp, - r.author_id.as_deref().filter(|a| vault != keys.vault_id && Some(*a) != keys.account_id.as_deref())], + rev = excluded.rev, dirty = 0, updated_at = excluded.updated_at, author_id = excluded.author_id, + version = excluded.version, seen_version = excluded.seen_version", + params![r.id, vault, if deleted { None } else { r.envelope.as_deref() }, deleted, r.rev, stamp, + r.author_id.as_deref().filter(|a| vault != keys.vault_id && Some(*a) != keys.account_id.as_deref()), + version], )?; applied += 1; } @@ -748,10 +808,10 @@ impl LocalVault { fn mark_pushed(&mut self, vault: &str, sent: &Outgoing, rev: i64) -> Result<()> { self.conn.execute( - "UPDATE items SET rev = ?3, author_id = NULL, + "UPDATE items SET rev = ?3, author_id = NULL, seen_version = ?5, dirty = CASE WHEN updated_at = ?4 THEN 0 ELSE 1 END WHERE vault_id = ?1 AND id = ?2", - params![vault, sent.id, rev, sent.updated_at], + params![vault, sent.id, rev, sent.updated_at, sent.version], )?; Ok(()) } @@ -776,10 +836,13 @@ impl LocalVault { return Ok(false); } let plain = crate::vault::open_envelope(keys, vault, id, &env, author.as_deref())?; + // A new version, whatever app wrote it and whatever kind it is. + let version = self.next_version(vault, id)?; + let plain = history::with_version(&plain, version)?; let env = envelope::seal(current, &keys.account, &context(vault, id), &plain)?; self.conn.execute( - "UPDATE items SET envelope = ?3, author_id = NULL, dirty = 1 WHERE vault_id = ?1 AND id = ?2", - params![vault, id, env], + "UPDATE items SET envelope = ?3, author_id = NULL, dirty = 1, version = ?4 WHERE vault_id = ?1 AND id = ?2", + params![vault, id, env, version], )?; Ok(true) } @@ -881,8 +944,15 @@ fn unlocked_from((account, vault_key): (AccountKeys, VaultKey), import: AccountI } } -/// `(rev, dirty, deleted, envelope, author_id)` of a local item. -type LocalRow = (i64, bool, bool, Option>, Option); +/// `(rev, dirty, deleted, envelope, author_id, seen_version)` of a local item. +type LocalRow = ( + i64, + bool, + bool, + Option>, + Option, + Option, +); fn mark_conflict(data: &mut ItemData) { const SUFFIX: &str = " (conflict)"; @@ -1122,6 +1192,8 @@ mod tests { refuse_blobs: Mutex>, /// Pull requests answered. pulls: AtomicI64, + /// Drop the envelope a deletion carries, as servers from before signed deletions do. + drops_deletion_envelopes: std::sync::atomic::AtomicBool, } impl Fake { @@ -1160,7 +1232,12 @@ mod tests { }, _ => { let rev = self.head.fetch_add(1, Ordering::SeqCst) + 1; - let envelope = if o.deleted { None } else { o.envelope }; + let envelope = + if o.deleted && self.drops_deletion_envelopes.load(Ordering::SeqCst) { + None + } else { + o.envelope + }; store.insert( o.id.clone(), RemoteItem { @@ -1176,6 +1253,18 @@ mod tests { }) .collect() } + + /// The server's copy of item `id`. + fn copy(&self, id: &str) -> RemoteItem { + self.items.lock().unwrap()[id].clone() + } + + /// Makes `copy` the newest revision of its item: a server serving what it kept, or a + /// change another app pushed. + fn serve(&self, mut copy: RemoteItem) { + copy.rev = self.head.fetch_add(1, Ordering::SeqCst) + 1; + self.items.lock().unwrap().insert(copy.id.clone(), copy); + } } impl Remote for Fake { @@ -1961,4 +2050,342 @@ mod tests { b.lock(); b.unlock(PW).unwrap(); } + + /// What a device pushes for item `id` of its personal vault: `data` with `version`, or + /// without one (`None`: an app before item history), or a deletion (`data: None`), unsigned + /// without a version the way those apps send it. + fn pushed_by( + v: &Mutex, + id: &str, + data: Option<&ItemData>, + version: Option, + ) -> RemoteItem { + let v = v.lock().unwrap(); + let keys = v.unlocked().unwrap(); + let vault = keys.vault_id.clone(); + let plain = match (data, version) { + (Some(data), Some(n)) => history::plaintext(data, n).unwrap().to_vec(), + (Some(data), None) => serde_json::to_vec(data).unwrap(), + (None, Some(n)) => history::deletion(n), + (None, None) => Vec::new(), + }; + let envelope = (!plain.is_empty()).then(|| { + envelope::seal( + keys.current(&vault).unwrap(), + &keys.account, + &context(&vault, id), + &plain, + ) + .unwrap() + }); + RemoteItem { + id: id.to_owned(), + rev: 0, + deleted: data.is_none(), + envelope, + author_id: None, + } + } + + /// The version in this device's copy of item `id` and the last one the server confirmed. + fn versions(v: &Mutex, id: &str) -> (Option, Option) { + v.lock() + .unwrap() + .conn + .query_row( + "SELECT version, seen_version FROM items WHERE id = ?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap() + } + + fn refused_as_older(r: core::result::Result) -> bool { + matches!(r, Err(SyncError::Store(Error::OlderItem { .. }))) + } + + /// Every write carries a version; a server serving a device an older copy than it has seen + /// gets refused, and the device keeps what it has. + #[tokio::test] + async fn an_older_copy_of_an_item_is_refused() { + let (a, b) = two_devices(); + let server = Fake::default(); + let id = a.lock().unwrap().upsert(None, host("db")).unwrap().id; + sync(&a, &server).await; + let first = server.copy(&id); + a.lock().unwrap().upsert(Some(&id), host("db2")).unwrap(); + sync(&a, &server).await; + sync(&b, &server).await; + let (version, seen) = versions(&b, &id); + assert!(version.unwrap() > 1_700_000_000_000); + assert_eq!(version, seen); + assert_eq!(versions(&a, &id), (version, version), "pushed is confirmed"); + + let second = server.copy(&id); + server.serve(first); + assert!(refused_as_older(sync_once(&b, &server).await)); + assert!( + refused_as_older(sync_once(&b, &server).await), + "until it stops" + ); + assert_eq!(labels(&b), ["db2"]); + // The server serves the right copy again: sync goes on. + server.serve(second); + sync(&b, &server).await; + sync(&a, &server).await; + a.lock().unwrap().upsert(Some(&id), host("db3")).unwrap(); + sync(&a, &server).await; + sync(&b, &server).await; + assert_eq!(labels(&b), ["db3"]); + } + + /// A deletion is signed, with a version after the item's: replaying the item's last copy + /// doesn't bring it back, and a deletion the server makes up for a newer copy is refused too. + #[tokio::test] + async fn a_deleted_item_cant_be_brought_back() { + let (a, b) = two_devices(); + let server = Fake::default(); + let id = a.lock().unwrap().upsert(None, host("db")).unwrap().id; + sync(&a, &server).await; + sync(&b, &server).await; + let live = server.copy(&id); + a.lock().unwrap().delete(&id).unwrap(); + sync(&a, &server).await; + let gone = server.copy(&id); + assert!( + gone.deleted && gone.envelope.is_some(), + "the deletion is signed" + ); + sync(&b, &server).await; + assert!(labels(&b).is_empty()); + assert!(versions(&b, &id).1.unwrap() > versions(&a, &id).1.unwrap() - 1); + + server.serve(live); + assert!(refused_as_older(sync_once(&b, &server).await)); + assert!(labels(&b).is_empty()); + + // The same item back (moved back, say), then the old deletion again. + let back = pushed_by( + &a, + &id, + Some(&host("db again")), + Some(history::next(0) + 10), + ); + server.serve(back); + sync(&b, &server).await; + assert_eq!(labels(&b), ["db again"]); + server.serve(gone); + assert!(refused_as_older(sync_once(&b, &server).await)); + assert_eq!(labels(&b), ["db again"]); + } + + /// Apps before item history write items without a version and delete without signing: + /// their changes still apply, and versions carry on after them. + #[tokio::test] + async fn changes_from_apps_before_item_history_still_sync() { + let (a, b) = two_devices(); + let server = Fake::default(); + let id = a.lock().unwrap().upsert(None, host("db")).unwrap().id; + sync(&a, &server).await; + sync(&b, &server).await; + + server.serve(pushed_by(&a, &id, Some(&host("db (old app)")), None)); + sync(&b, &server).await; + assert_eq!(labels(&b), ["db (old app)"]); + assert_eq!(versions(&b, &id), (Some(0), Some(0))); + + // After it, a device that never saw the earlier versions may write a lower one (its + // clock behind, say): it counts from there. + server.serve(pushed_by(&a, &id, Some(&host("db (slow clock)")), Some(5))); + sync(&b, &server).await; + assert_eq!(labels(&b), ["db (slow clock)"]); + server.serve(pushed_by(&a, &id, Some(&host("db (older)")), Some(4))); + assert!(refused_as_older(sync_once(&b, &server).await)); + + // B's next edit comes after all of it; A takes it. + server.serve(pushed_by(&a, &id, Some(&host("db (slow clock)")), Some(5))); + sync(&b, &server).await; + b.lock() + .unwrap() + .upsert(Some(&id), host("db (new app)")) + .unwrap(); + sync(&b, &server).await; + sync(&a, &server).await; + assert_eq!(labels(&a), ["db (new app)"]); + + server.serve(pushed_by(&a, &id, None, None)); + sync(&a, &server).await; + assert!(labels(&a).is_empty(), "an unsigned deletion applies"); + } + + /// With strict item history, an item that has a version takes no unversioned change and no + /// unsigned deletion; items that never had a version still do. + #[tokio::test] + async fn strict_history_refuses_unversioned_changes() { + let (a, b) = two_devices(); + let server = Fake::default(); + b.lock().unwrap().set_strict_history(true); + let id = a.lock().unwrap().upsert(None, host("db")).unwrap().id; + sync(&a, &server).await; + sync(&b, &server).await; + let current = server.copy(&id); + let unversioned = |r: core::result::Result| { + matches!(r, Err(SyncError::Store(Error::UnversionedItem { .. }))) + }; + + server.serve(pushed_by(&a, &id, Some(&host("db (old app)")), None)); + assert!(unversioned(sync_once(&b, &server).await)); + server.serve(pushed_by(&a, &id, None, None)); + assert!(unversioned(sync_once(&b, &server).await)); + // A deletion envelope that isn't one: a live copy, or another item's. + let mut fake = current.clone(); + fake.deleted = true; + server.serve(fake); + assert!(unversioned(sync_once(&b, &server).await)); + assert_eq!(labels(&b), ["db"]); + server.serve(current); + sync(&b, &server).await; + + let old = Ulid::generate().to_string(); + server.serve(pushed_by(&a, &old, Some(&host("from an old app")), None)); + sync(&b, &server).await; + assert_eq!(labels(&b), ["db", "from an old app"]); + server.serve(pushed_by(&a, &old, None, None)); + sync(&b, &server).await; + assert_eq!(labels(&b), ["db"]); + } + + /// A copy stored before item history (no versions in its row) still counts with the version + /// its envelope holds: the next write comes after it, and nothing older is taken. + #[tokio::test] + async fn copies_from_before_item_history_keep_their_version() { + let (a, b) = two_devices(); + let server = Fake::default(); + let id = a.lock().unwrap().upsert(None, host("db")).unwrap().id; + sync(&a, &server).await; + let ahead = history::next(0) + 86_400_000; + server.serve(pushed_by( + &a, + &id, + Some(&host("db (clock ahead)")), + Some(ahead), + )); + sync(&b, &server).await; + b.lock() + .unwrap() + .conn + .execute("UPDATE items SET version = NULL, seen_version = NULL", []) + .unwrap(); + + server.serve(pushed_by( + &a, + &id, + Some(&host("db (older)")), + Some(ahead - 1), + )); + assert!(refused_as_older(sync_once(&b, &server).await)); + + server.serve(pushed_by( + &a, + &id, + Some(&host("db (clock ahead)")), + Some(ahead), + )); + sync(&b, &server).await; + b.lock() + .unwrap() + .conn + .execute("UPDATE items SET version = NULL, seen_version = NULL", []) + .unwrap(); + b.lock() + .unwrap() + .upsert(Some(&id), host("db (edited)")) + .unwrap(); + assert_eq!(versions(&b, &id).0, Some(ahead + 1)); + } + + /// Re-sealing under a vault's new key (finishing a rotation) is a write: a new version, the + /// same content. + #[test] + fn resealing_gives_a_new_version() { + let mut v = fast(LocalVault::open_in_memory().unwrap()); + v.create(PW).unwrap(); + let old = VaultKey::generate().unwrap(); + v.conn + .execute( + "INSERT INTO vaults (id, key_id, wrapped_vault_key, created_at, personal, org_id, permission, name) + VALUES ('TEAM', ?1, x'02', 1, 0, 'ORG', 'write', 'Team')", + [old.id().as_slice()], + ) + .unwrap(); + let ring = |keys: Vec| crate::vault::Ring { + current: *keys.last().unwrap().id(), + keys, + access: Access::Write, + }; + let new = VaultKey::generate().unwrap(); + let (old_id, new_id) = (*old.id(), *new.id()); + v.keys + .as_mut() + .unwrap() + .shared + .insert("TEAM".into(), ring(vec![old])); + let id = v.upsert_in("TEAM", None, host("db")).unwrap().id; + let before: i64 = v + .conn + .query_row("SELECT version FROM items WHERE id = ?1", [&id], |r| { + r.get(0) + }) + .unwrap(); + v.keys + .as_mut() + .unwrap() + .shared + .get_mut("TEAM") + .unwrap() + .keys + .push(new); + v.keys + .as_mut() + .unwrap() + .shared + .get_mut("TEAM") + .unwrap() + .current = new_id; + assert_ne!(old_id, new_id); + + assert_eq!(v.reseal_old_items("TEAM").unwrap(), 1); + let (env, version): (Vec, i64) = v + .conn + .query_row( + "SELECT envelope, version FROM items WHERE id = ?1", + [&id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(envelope_key_id(&env).unwrap(), new_id); + assert!(version > before); + let keys = v.unlocked().unwrap(); + let copy = + history::read(&id, &open_envelope(keys, "TEAM", &id, &env, None).unwrap()).unwrap(); + assert_eq!((copy.data, copy.version), (Some(host("db")), Some(version))); + } + + /// A server from before signed deletions drops their envelope: deletions still sync. + #[tokio::test] + async fn deletions_sync_through_servers_that_drop_their_envelope() { + let (a, b) = two_devices(); + let server = Fake::default(); + server + .drops_deletion_envelopes + .store(true, Ordering::SeqCst); + let id = a.lock().unwrap().upsert(None, host("db")).unwrap().id; + sync(&a, &server).await; + sync(&b, &server).await; + a.lock().unwrap().delete(&id).unwrap(); + sync(&a, &server).await; + assert!(server.copy(&id).envelope.is_none()); + sync(&b, &server).await; + assert!(labels(&b).is_empty()); + } } diff --git a/crates/mg-store/src/vault.rs b/crates/mg-store/src/vault.rs index 4bda5dd..7693f9c 100644 --- a/crates/mg-store/src/vault.rs +++ b/crates/mg-store/src/vault.rs @@ -11,7 +11,7 @@ use ulid::Ulid; use zeroize::Zeroizing; use crate::model::SCHEMA_VERSION; -use crate::{Error, ForwardRule, Host, HostAuth, Item, ItemData, Result, schema}; +use crate::{Error, ForwardRule, Host, HostAuth, Item, ItemData, Result, history, schema}; pub const MIN_PASSWORD_CHARS: usize = 10; /// Longest allowed chain of jump hosts in front of a target. @@ -175,6 +175,9 @@ pub struct LocalVault { pub(crate) conn: Connection, pub(crate) keys: Option, kdf_cost: Option<(u32, u32, u32)>, + /// Refuse unversioned changes to items that have a version (see + /// [`set_strict_history`](Self::set_strict_history)). + pub(crate) strict_history: bool, } impl LocalVault { @@ -207,9 +210,17 @@ impl LocalVault { conn, keys: None, kdf_cost: None, + strict_history: false, }) } + /// Strict item history: once an item has a version here, a change to it without one or a + /// deletion without a signature (what apps before item history send, and what a server + /// could make up) is refused instead of applied. Off while such apps may still write. + pub fn set_strict_history(&mut self, strict: bool) { + self.strict_history = strict; + } + /// Argon2id cost (`m_kib`, `t`, `p`) for passwords set from now on. The `mg-vault` floors still /// apply. Defaults to [`KdfParams::recommended`]. pub fn with_kdf_cost(mut self, m_kib: u32, t: u32, p: u32) -> Self { @@ -641,9 +652,9 @@ impl LocalVault { /// [`put`](Self::put) without the checks, for writes that are checked together afterwards /// (a move, whose items may refer to each other in a circle). pub(crate) fn write(&mut self, vault_id: &str, id: &str, data: &ItemData) -> Result { + let version = self.next_version(vault_id, id)?; let keys = self.unlocked()?; - let plain = - Zeroizing::new(serde_json::to_vec(data).map_err(|e| Error::Invalid(e.to_string()))?); + let plain = history::plaintext(data, version)?; let env = envelope::seal( keys.current(vault_id)?, &keys.account, @@ -652,14 +663,62 @@ impl LocalVault { )?; let now = now(); self.conn.execute( - "INSERT INTO items (id, vault_id, envelope, deleted, dirty, updated_at, author_id) VALUES (?1, ?2, ?3, 0, 1, ?4, NULL) + "INSERT INTO items (id, vault_id, envelope, deleted, dirty, updated_at, author_id, version, seen_version) + VALUES (?1, ?2, ?3, 0, 1, ?4, NULL, ?5, 0) ON CONFLICT (vault_id, id) DO UPDATE SET envelope = excluded.envelope, deleted = 0, dirty = 1, - updated_at = excluded.updated_at, author_id = NULL", - params![id, vault_id, env, now], + updated_at = excluded.updated_at, author_id = NULL, + version = excluded.version", + params![id, vault_id, env, now, version], )?; Ok(now) } + /// The version for the next write of item `id` of `vault`: after the highest this device + /// knows of it (its own copy's, or the last one the server confirmed). + pub(crate) fn next_version(&self, vault: &str, id: &str) -> Result { + let keys = self.unlocked()?; + let row: Option = self + .conn + .query_row( + "SELECT version, seen_version, envelope, author_id FROM items WHERE vault_id = ?1 AND id = ?2", + params![vault, id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .optional()?; + let known = match row { + None => 0, + Some((version, seen, env, author)) => { + let own = match (version, env) { + (Some(v), _) => v, + // A copy from before item history: its envelope may still hold a version + // (another app wrote it). + (None, Some(env)) => version_in(keys, vault, id, &env, author.as_deref()), + (None, None) => 0, + }; + own.max(seen.unwrap_or(0)) + } + }; + Ok(history::next(known)) + } + + /// Replaces item `id` of `vault` with a signed deletion for the next sync. + pub(crate) fn tombstone(&mut self, vault: &str, id: &str, stamp: i64) -> Result<()> { + let version = self.next_version(vault, id)?; + let keys = self.unlocked()?; + let env = envelope::seal( + keys.current(vault)?, + &keys.account, + &context(vault, id), + &history::deletion(version), + )?; + self.conn.execute( + "UPDATE items SET envelope = ?3, deleted = 1, dirty = 1, updated_at = ?4, version = ?5 + WHERE vault_id = ?1 AND id = ?2", + params![vault, id, env, stamp, version], + )?; + Ok(()) + } + /// Replaces the item with a tombstone so the deletion can sync. pub fn delete(&mut self, id: &str) -> Result<()> { let item = self.item(id)?; @@ -726,11 +785,7 @@ impl LocalVault { users.join(", ") ))); } - self.conn.execute( - "UPDATE items SET envelope = NULL, deleted = 1, dirty = 1, updated_at = ?3 - WHERE vault_id = ?1 AND id = ?2", - params![item.vault_id, id, now()], - )?; + self.tombstone(&item.vault_id, id, now())?; // A session log's recording, or an attachment's file, goes with it. let blobs = item.data.blob_refs(); if !blobs.is_empty() { @@ -1465,6 +1520,24 @@ pub(crate) fn open_envelope( Ok(envelope::open(vk, &signer, &context(vault, id), env)?) } +/// `(version, seen_version, envelope, author_id)` of a local item. +type VersionRow = (Option, Option, Option>, Option); + +/// The version in a stored envelope of item `id`; 0 when it has none or can't be read. +pub(crate) fn version_in( + keys: &Unlocked, + vault: &str, + id: &str, + env: &[u8], + author: Option<&str>, +) -> i64 { + open_envelope(keys, vault, id, env, author) + .and_then(|plain| history::read(id, &plain)) + .ok() + .and_then(|copy| copy.version) + .unwrap_or(0) +} + pub(crate) fn decrypt( keys: &Unlocked, vault: &str, @@ -1746,6 +1819,12 @@ mod tests { v.upsert(Some(&h.id), edited.clone()).unwrap(); assert_eq!(v.item(&h.id).unwrap().data, edited); + let before: i64 = v + .conn + .query_row("SELECT version FROM items WHERE id = ?1", [&s.id], |r| { + r.get(0) + }) + .unwrap(); v.delete(&s.id).unwrap(); assert!(matches!(v.item(&s.id), Err(Error::NotFound(_)))); assert_eq!(v.items().unwrap().len(), 2); @@ -1757,10 +1836,19 @@ mod tests { |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), ) .unwrap(); - assert!( - deleted && envelope.is_none() && dirty, - "deletes become syncable tombstones" - ); + assert!(deleted && dirty, "deletes become syncable tombstones"); + // Signed by this account, for this item, with a version after the item's. + let plain = open_envelope( + v.unlocked().unwrap(), + &s.vault_id, + &s.id, + &envelope.expect("a signed deletion"), + None, + ) + .unwrap(); + let gone = history::read(&s.id, &plain).unwrap(); + assert!(gone.data.is_none()); + assert!(gone.version.unwrap() > before); } #[test] diff --git a/crates/mg-sync/src/types.rs b/crates/mg-sync/src/types.rs index a567e7e..f77863a 100644 --- a/crates/mg-sync/src/types.rs +++ b/crates/mg-sync/src/types.rs @@ -127,8 +127,15 @@ pub struct Features { /// Organizations can put their vaults behind their identity provider. #[serde(default, skip_serializing_if = "Option::is_none")] pub sso: Option, + /// The server keeps the envelope a deletion carries (the item's signed deletion, at most + /// [`MAX_DELETION_ENVELOPE`] bytes) and returns it with the tombstone. + #[serde(default)] + pub signed_tombstones: bool, } +/// The most a deletion's envelope may take (`PushItem::envelope` with `deleted`). +pub const MAX_DELETION_ENVELOPE: usize = 1024; + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct BlobFeatures { pub max_blob_bytes: u64, diff --git a/docs/features/reference-server.md b/docs/features/reference-server.md index 105be56..8acfbc1 100644 --- a/docs/features/reference-server.md +++ b/docs/features/reference-server.md @@ -73,6 +73,7 @@ Tokens are 32 random bytes, stored as SHA-256 hashes. The `AuthKey` and the reco | `sharing` | No member or vault limits; `mails_invites: false` (creating an invite returns its token) | | `blobs` | `max_blob_bytes`, `quota_bytes`, kinds `attachment` and `session_log` | | `sso` | `protocols: ["oidc"]` | +| `signed_tombstones` | `true`: a deletion's envelope (at most 1 KiB, any key generation) is kept and returned, for an item the server has | | `limits.tombstone_retention_days` | 90 | | Area | Endpoints (under `/v1` unless noted) | diff --git a/docs/features/sync.md b/docs/features/sync.md index 222cf28..51ac8b0 100644 --- a/docs/features/sync.md +++ b/docs/features/sync.md @@ -171,7 +171,7 @@ The personal vault failing fails the round. A shared vault failing is listed and | `rejected`, `stale_key` | The item is re-sealed under the vault's current key and goes in the next batch | | `rejected`, any other code | Skipped for the rest of the round, still dirty, tried again next round. Not shown anywhere (`SyncReport.rejected`) | -A deletion goes up as a tombstone (`deleted: true`, no envelope). An item deleted before it ever synced (`rev` 0) is removed locally and never sent. A read-only vault only pulls. +A deletion goes up as a tombstone (`deleted: true`) with its signed deletion envelope ([Item history](#item-history)); servers with `signed_tombstones` keep it (for items they have), others drop it. An item deleted before it ever synced (`rev` 0) is removed locally and never sent. A read-only vault only pulls. ## Revisions and anti-rollback @@ -183,8 +183,21 @@ Each vault row keeps `head_rev`, the highest server revision this device has app | A shared vault's item may not take the id of a live personal item (lookups go by id) | "item {id} of a shared vault has the id of one in your personal vault" | | A page whose `head_rev` is below the local `head_rev` | The vault fails: "the sync server sent older data (revision {remote}) than this device already has ({local}); refusing to roll back" (`SyncError::Rollback`) | | A pulled item no newer than the local copy | Ignored | +| A pulled item or deletion with a version below the last one the server confirmed for it | The vault fails: "the sync server sent an older copy of item {id} (version {remote}) than this device already has (version {seen}); refusing to roll it back" (`Error::OlderItem`). Nothing from the page is applied | | `410` on a pull (code `gone` or none; `410 invite_expired` is something else) | The cursor goes back to 0 and the vault is pulled in full; afterwards, clean synced items the server didn't send are deleted here (`drop_unseen`). The `head_rev` check doesn't apply to this full pull; the per-item check does | +## Item history + +Every write carries a version inside its signed envelope (`crates/mg-store/src/history.rs`, the spec's [Item history](../sync-crypto-spec.md#sync-protocol)): the time of the write in milliseconds, or one more than the highest version this device knows of the item when that is larger (`LocalVault::next_version`). Edits, conflict copies, re-seals under a rotated key (`reseal`) and deletions (`LocalVault::tombstone`, from `delete` and from moves) all take a new one. Each item row keeps `version`, the one in its envelope, and `seen_version`, the one of the last copy the server confirmed: set by `apply_remote` and by `mark_pushed`. + +| Pulled | Effect | +| --- | --- | +| A versioned copy at or above `seen_version` | Applied; `seen_version` follows it | +| A versioned copy below it | Refused (`Error::OlderItem`, above) | +| A copy without a version, or a deletion without a valid signed envelope (apps before item history, 0.2.0) | Applied; `seen_version` goes to 0, so the next versioned copy is taken whatever its version (its writer may not have seen the earlier ones) | +| The same, with strict item history on (`LocalVault::set_strict_history`, off for now) | Refused when the item has a version (`Error::UnversionedItem`) | +| An item whose row is from before item history (`seen_version` NULL) | Compared with the version in its stored envelope | + ## Conflicts From the code's own statement: the server copy always wins for an item id, and a local edit that lost is kept as a new "(conflict)" item, so nothing is silently discarded. A conflict is found either when a pull brings a newer revision of an item that is dirty here, or when a push answers `conflict` with the server's current copy. @@ -298,7 +311,7 @@ Organization endpoints are listed in [Teams](teams.md); account and device endpo | Where | What | | --- | --- | -| `crates/mg-store/src/sync.rs` (`cargo test -p mg-store`) | The engine against an in-memory server (`Fake`, two items per page): `items_flow_between_devices`, `concurrent_edits_keep_both_versions`, `edit_beats_delete_as_a_copy_and_delete_loses_to_edit`, `never_synced_deletes_are_not_uploaded`, `tampered_server_data_is_rejected_whole`, `rollback_is_refused`, `up_to_date_vaults_arent_pulled`, `a_device_with_its_own_vault_joins_an_account`, `a_shared_vault_cant_shadow_a_personal_item`, `kinds_from_newer_apps_are_kept_but_not_listed`, `token_and_config_roundtrip_and_disable`, `vaults_without_a_stored_recovery_proof_rotate_on_registration`, `adopt_requires_the_right_password_and_a_fresh_device`, and the blob tests (`session_log_blobs_upload_fetch_and_delete`, `refused_blobs_are_reported_and_retried`, `attachments_sync_and_go_with_their_host`) | +| `crates/mg-store/src/sync.rs` (`cargo test -p mg-store`) | The engine against an in-memory server (`Fake`, two items per page): `items_flow_between_devices`, `concurrent_edits_keep_both_versions`, `edit_beats_delete_as_a_copy_and_delete_loses_to_edit`, `never_synced_deletes_are_not_uploaded`, `tampered_server_data_is_rejected_whole`, `rollback_is_refused`, `an_older_copy_of_an_item_is_refused`, `a_deleted_item_cant_be_brought_back`, `changes_from_apps_before_item_history_still_sync`, `strict_history_refuses_unversioned_changes`, `copies_from_before_item_history_keep_their_version`, `resealing_gives_a_new_version`, `deletions_sync_through_servers_that_drop_their_envelope`, `up_to_date_vaults_arent_pulled`, `a_device_with_its_own_vault_joins_an_account`, `a_shared_vault_cant_shadow_a_personal_item`, `kinds_from_newer_apps_are_kept_but_not_listed`, `token_and_config_roundtrip_and_disable`, `vaults_without_a_stored_recovery_proof_rotate_on_registration`, `adopt_requires_the_right_password_and_a_fresh_device`, and the blob tests (`session_log_blobs_upload_fetch_and_delete`, `refused_blobs_are_reported_and_retried`, `attachments_sync_and_go_with_their_host`) | | `crates/mg-sync/src/client.rs` | `payment_errors_say_what_to_do` | | `server/tests/e2e.rs`, `protocol.rs`, `account.rs` | Real vaults through the reference server over HTTP: a second device signs in and syncs, push, pull, conflicts and tombstones, refresh rotation and reuse detection, email verification and two-step verification where offered. `VAULT_SYNC_URL=https://… cargo test -p vault-sync-server --test protocol --test e2e --test account` runs them against another server | | `tools/vault-conform` | The `auth`, `account` and `vaults` groups check a server's side of all of this. See [Development](../development.md#conformance-suite) | diff --git a/docs/features/vault-and-unlock.md b/docs/features/vault-and-unlock.md index 342b80b..05dd0b6 100644 --- a/docs/features/vault-and-unlock.md +++ b/docs/features/vault-and-unlock.md @@ -62,7 +62,7 @@ Store-level rules that apply to every type: | Saving an existing item into another vault | `to put an item in another vault, copy or move it` | | Deleting an item others refer to | ` is still used by: ` | -Deleting leaves a tombstone (`envelope` NULL, `deleted` 1) so the deletion syncs. Deleting a host deletes its attachments; deleting a group makes what its hosts took from it their own. +Deleting leaves a tombstone (`deleted` 1) holding the signed deletion (`{"type":"deleted","item_version":n}` in an envelope of the item) so the deletion syncs, and other devices can tell it from one a server made up ([Item history](sync.md#item-history)). Deleting a host deletes its attachments; deleting a group makes what its hosts took from it their own. ## First run diff --git a/docs/features/vault-crypto.md b/docs/features/vault-crypto.md index d0694e5..9f1cffe 100644 --- a/docs/features/vault-crypto.md +++ b/docs/features/vault-crypto.md @@ -149,6 +149,7 @@ The revision memory lives in `mg-store`: `vaults.head_rev` is the highest server | --- | --- | | A pull answers with `head_rev` below the device's | `SyncError::Rollback`: "the sync server sent older data (revision R) than this device already has (L); refusing to roll back". Nothing from that answer is applied | | A pulled item whose `rev` is not newer than the local copy's | Skipped (`apply_remote`) | +| A pulled item or deletion whose version (inside its signed envelope) is below the last one the server confirmed for it | `Error::OlderItem`: "the sync server sent an older copy of item … than this device already has …; refusing to roll it back". Nothing from that page is applied ([Item history](sync.md#item-history)) | | `410 Gone` (the history was purged) | The cursor resets to 0 and the vault is pulled in full; the `head_rev` check is off for that pass, the per-item check stays. Synced, unchanged items the server no longer lists are dropped (`drop_unseen`) | | Sync turned off | `head_rev` back to 0 for every vault: another server is another history | @@ -172,7 +173,8 @@ How a round pages and resolves conflicts: [Sync](sync.md). | --- | --- | | `mg-vault` unit tests, one module each | Derivation determinism and domain separation, weak parameters refused (`kdf.rs`); wrap, wrong password, recovery and device domains, truncated blobs (`account.rs`); code shape, forgiving input, typos (`recovery_code.rs`); seal, other account, tampering, low-order keys (`vault.rs`); bucket sizes, trailing zeros, moved items, bit flips, wrong author, wrong key, oversize (`envelope.rs`); binding to vault and kind, SHA-256 vector, size limit (`blob.rs`); fixed vectors computed independently (`sharing.rs`) | | `mg-store` `vault.rs` tests | `persists_and_stores_only_ciphertext` (no hostname, password or key bytes in the file), `tampered_or_swapped_envelopes_are_rejected`, `another_accounts_keys_are_refused` | -| `mg-store` `sync.rs` tests | `rollback_is_refused` | +| `mg-store` `sync.rs` tests | `rollback_is_refused`, `an_older_copy_of_an_item_is_refused`, `a_deleted_item_cant_be_brought_back` | +| `mg-store` `history.rs` tests | The plaintext with its version, deletions, items from older apps and newer kinds | `cargo test -p mg-vault -p mg-store`. Tests derive at the floor parameters (`fast_params()`, `LocalVault::with_kdf_cost(MIN_M_KIB, MIN_T, 1)`), 64 MiB each. CI runs `cargo test --workspace --all-features` on Linux and these crates on macOS and Windows. diff --git a/docs/sync-crypto-spec.md b/docs/sync-crypto-spec.md index 24f8734..e42eb8e 100644 --- a/docs/sync-crypto-spec.md +++ b/docs/sync-crypto-spec.md @@ -24,13 +24,13 @@ A fully compromised server, ours or a self-hoster's, learns metadata only: accou | Adversary | Can | Cannot | | --- | --- | --- | -| Malicious / breached server | Read ciphertext and metadata, drop or replay old blobs, deny service | Decrypt items, forge items (AEAD + signatures), make a client accept a vault revision below one it has seen | +| Malicious / breached server | Read ciphertext and metadata, drop or replay old blobs, deny service | Decrypt items, forge items (AEAD + signatures), make a client accept a vault revision below one it has seen, or a signed copy of an item older than one it has seen ([item history](#sync-protocol)) | | Network attacker | Nothing beyond TLS failure | Anything: TLS required (the client refuses plain `http://` except to this computer), plus E2E | | Stolen unlocked device | Everything in that vault | Survive revocation for future changes | | Stolen locked device | Offline brute-force of master password | Succeed quickly (Argon2id, high cost) | | Malicious self-host operator | Same as breached server | Same as breached server | -**Rollback defense.** Each client remembers the highest vault revision it has seen. A server answering with a lower one is flagged as an error, not silently accepted. The rollback guard covers the vault's revision, not each item's history. +**Rollback defense.** Each client remembers the highest vault revision it has seen. A server answering with a lower one is flagged as an error, not silently accepted. Items have a guard of their own: every write carries a version inside its signed envelope, deletions are signed, and a client refuses a copy of an item older than one it has seen (Item history, under Sync protocol). **KDF bounds.** KDF parameters come from the server at prelogin, so the client enforces floors (≥ 64 MiB, ≥ 2 passes, ≥ 16-byte salt) and refuses to derive below them, and ceilings (≤ 1 GiB, ≤ 10 passes, ≤ 64-byte salt), so a server can't make a device allocate or compute without bound. @@ -96,6 +96,8 @@ All HKDF is HKDF-SHA256; all info strings are prefixed `morphgate/v1/`. | Blob | `0x01 ‖ key_id(16) ‖ nonce(24) ‖ XChaCha20-Poly1305(VK, aad, pad₄ₖ(plaintext))`, aad = `blob ‖ u16 len ‖ vault_id ‖ u16 len ‖ kind ‖ 0x01 ‖ key_id`; named by lowercase hex SHA-256 of the whole blob; no signature (the signed item that refers to it holds the hash) | | Envelope AAD | `u16 len ‖ vault_id ‖ u16 len ‖ item_id ‖ u16 schema_version ‖ 0x01 ‖ key_id` | | Envelope signature | Ed25519 over `envelope-sig ‖ u32 len(aad) ‖ aad ‖ version ‖ key_id ‖ nonce ‖ ct`; verified before decrypting | +| Item plaintext | The item's JSON (Data model) with `item_version`: the time of the write in Unix milliseconds, or one more than the highest version the writer knows of the item when that is larger | +| Deletion | An envelope of the deleted item (same AAD) whose plaintext is `{"type":"deleted","item_version":n}` | | Padding | ISO/IEC 7816-4 (`0x80` then zeros) to a multiple of 256 bytes | | Vault `meta` | An envelope with item id `vault-meta` (JSON `{name, directory?}`), under the vault key, signed by the vault's creator (`created_by`) | @@ -132,7 +134,7 @@ The server knows three things: accounts, vaults and items. Items are opaque enve **Plaintext item types (client-only schema, `schema_version` 1)** -Serialized as JSON with a `type` tag, then sealed into an envelope. +Serialized as JSON with a `type` tag and the write's `item_version` (Item history), then sealed into an envelope. | Type | Fields | Status | | --- | --- | --- | @@ -162,7 +164,7 @@ The client is offline-first: the vault exists on the device before any server is - One SQLite file in the app data dir (`vault.sqlite3`, mode 0600, `secure_delete` on). - `account`: KDF params, public keys, `wrapped_private_keys`, `wrapped_private_keys_rk` (same values the server gets at register). - `vaults`: id, `key_id`, `wrapped_vault_key`, `head_rev` (highest server revision seen, for the rollback guard). -- `items`: id, vault_id, envelope (NULL for tombstones), `deleted`, `rev` (server rev the copy is based on; 0 = never synced), `dirty` (unpushed local change), `updated_at`. +- `items`: id, vault_id, envelope (NULL for tombstones, except a deletion made on this device, which keeps its signed envelope), `deleted`, `rev` (server rev the copy is based on; 0 = never synced), `dirty` (unpushed local change), `updated_at`, `version` (the one in the envelope) and `seen_version` (the one of the last copy the server confirmed). - Nothing typed is stored in plaintext: hostnames, users, passwords and keys exist only inside envelopes. - Unlock = Argon2id + unwrap account keys + open vault key, all in memory. Lock drops every key and closes open sessions. - Optional per device: a random device key in the OS keychain (`device_unlock` holds the account keys wrapped under HKDF(device key, `device-kek`)). It unlocks without the master password, survives password changes and recovery, never syncs, and is tried automatically only at app launch. The device can also require Touch ID, Windows Hello or polkit (Linux) before the key is read. That is a check made by the app: it stops someone at an unlocked computer, not malware running as the user. @@ -175,7 +177,7 @@ The client is offline-first: the vault exists on the device before any server is Sync is a per-vault revision log. Pull is "give me everything after rev N". Push is a batch with optimistic concurrency per item. Conflicts resolve on the client, because only the client can read the data. 1. **Pull.** `GET /vaults/{id}/changes?since={rev}&limit=500` returns items with `rev > since` (tombstones included) plus `head_rev`. Page until `has_more = false`. -2. **Decrypt & apply.** Verify signature, decrypt, merge into local SQLite. Store `last_rev`. +2. **Decrypt & apply.** Verify signature, decrypt, check the item's version (Item history), merge into local SQLite. Store `last_rev`. 3. **Push.** `POST /vaults/{id}/changes` with a batch of `{id, base_rev, deleted, envelope}`. - Server accepts an item if its current rev equals `base_rev`, or the item is new. - Otherwise it rejects that item with `conflict` in the per-item result and returns the current server copy. @@ -184,7 +186,29 @@ Sync is a per-vault revision log. Pull is "give me everything after rev N". Push 5. **Realtime nudge (optional).** `GET /vaults/{id}/events` via SSE sends only `{head_rev}`. Clients then pull. Self-hosters can skip this; clients fall back to polling: every 60 s while someone uses the app, every 5 minutes while nobody does (window hidden, or no input for 5 minutes), and at once on coming back. Local edits push within seconds either way. 6. **Quiet rounds are cheap.** A round lists the account's organizations (`GET /orgs`) and vaults (`GET /vaults`). A vault whose `head_rev` there equals the device's `last_rev` isn't pulled. An organization whose `rev` (optional) equals the one the device saw last isn't fetched again: its details, members, device approvals and vault members are reused, and verified again locally (the device's own pins may have changed). So a quiet round is two requests, however many organizations and vaults there are. -**Tombstones.** Deletes are items with `deleted: true` and an empty envelope. The server keeps them for at least 90 days. A client whose `last_rev` is older than the oldest retained tombstone gets `410 Gone` and does a full resync. +**Tombstones.** Deletes are items with `deleted: true`. A client with item history sends the deletion's signed envelope along (at most 1 KiB, under any key generation of the vault); a server with `features.signed_tombstones` keeps it for an item it has and returns it with the tombstone, and others ignore it. The server keeps tombstones for at least 90 days. A client whose `last_rev` is older than the oldest retained tombstone gets `410 Gone` and does a full resync. + +**Item history.** The vault's revision only says how far a device has read. Each item also has a +version inside its signed envelope, so a server can't serve a device a copy of an item older +than one it has seen: + +- Every write carries `item_version` (Wire formats): an edit, a conflict copy, a re-seal under a + rotated key, a deletion. Versions grow along an item's history because a server takes a write + only on the item's latest revision (`base_rev`); the clock keeps them growing for an item made + again after its tombstone was purged. +- A client remembers the version of the last copy of each item the server confirmed (pulled or + pushed). A signed copy with a lower version, item or deletion, is refused: the vault's sync + stops with an error, as for a vault revision below the one seen. An equal version is taken (the + same write again, or an older app's re-seal of it, which keeps the plaintext). +- Apps from before item history (0.2.0) write items without `item_version` and delete without an + envelope, and ignore both in what they read. A client takes such a change, and the next + versioned copy starts the count again (its writer may not have seen the earlier ones). So the + guard holds once every app that writes to a vault has item history. Strict item history, a + client setting, refuses both kinds of change to an item that has a version; it is off while + such apps may still write. +- Not covered: a client without a copy of an item has nothing to compare it with (a new device, + or one starting over after `410`), so a server can still serve it an old copy or leave an item + out. That needs signed checkpoints of a vault's state. **Blobs and session logs.** What is too big for an item goes in a blob (`/vaults/{id}/blobs/{hash}`, behind `features.blobs`): sealed under the vault key (Wire formats), @@ -303,7 +327,7 @@ We publish `vault-sync/v1` as an OpenAPI 3.1 document plus a conformance test su | [`tools/vault-conform`](../tools/vault-conform) (runs against a URL) | Apache-2.0 OR MIT | Pass/fail per endpoint, with throwaway accounts: auth, devices, sync (conflicts, 410, pagination), sharing, blobs | | Reference server ([`server/`](../server), Rust + SQLite, single binary, Dockerfile) | AGPL-3.0 | So self-hosting is `docker run`, not "implement the spec" | -**Capability discovery.** `GET /.well-known/vault-sync`, which needs no session (like the sign-up and sign-in steps), returns the spec version, limits, and optional features (`Capabilities` in the spec: `registration_open`, `terms`, `email_verification`, `sse`, `totp`, `blobs`, `sso`, `sharing`, `billing`). The client adapts its UI from that response; for example, it shows no plans when `billing` is absent. +**Capability discovery.** `GET /.well-known/vault-sync`, which needs no session (like the sign-up and sign-in steps), returns the spec version, limits, and optional features (`Capabilities` in the spec: `registration_open`, `terms`, `email_verification`, `sse`, `totp`, `signed_tombstones`, `blobs`, `sso`, `sharing`, `billing`). The client adapts its UI from that response; for example, it shows no plans when `billing` is absent. **Versioning.** The path prefix is `/v1`. Additive changes are minor and must be ignored by older clients. Breaking changes go to `/v2`, and the client supports N and N-1. diff --git a/server/src/api.rs b/server/src/api.rs index dcc1d74..de5a22a 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -221,6 +221,7 @@ async fn capabilities(State(s): State) -> Json { sso: Some(SsoFeatures { protocols: vec!["oidc".into()], }), + signed_tombstones: true, }, }) } @@ -915,16 +916,20 @@ async fn push( results.push(reject("bad_request", "item id must be a ULID")); continue; } - let envelope = if item.deleted { - None - } else { - item.envelope.map(|e| e.0) - }; + // A deletion may carry its author's signed deletion: kept and returned with it. + let envelope = item.envelope.map(|e| e.0); match &envelope { None if !item.deleted => { results.push(reject("bad_request", "envelope required")); continue; } + Some(e) if item.deleted && e.len() > MAX_DELETION_ENVELOPE => { + results.push(reject( + "too_large", + "a deletion's envelope is at most 1024 bytes", + )); + continue; + } Some(e) if e.len() > MAX_ENVELOPE => { results.push(reject("too_large", "envelope exceeds max_envelope_bytes")); continue; @@ -934,7 +939,8 @@ async fn push( results.push(reject("bad_request", "envelope too short")); continue; } - Some(e) if e[1..17] != current_key[..] => { + // A deletion carries no secret: any key generation the vault had will do. + Some(e) if !item.deleted && e[1..17] != current_key[..] => { results.push(reject( "stale_key", "the vault has a newer key: re-encrypt and push again", @@ -960,6 +966,9 @@ async fn push( }, ) .optional()?; + // A deletion's envelope only for an item the server has: for one it never had, it would + // take room that no quota counts. + let envelope = envelope.filter(|_| !(item.deleted && current.is_none())); if let Some(cur) = current.filter(|cur| cur.rev != item.base_rev) { results.push(PushResult { id: item.id, @@ -971,7 +980,10 @@ async fn push( continue; } head += 1; - let key_id = envelope.as_ref().map(|e| e[1..17].to_vec()); + let key_id = envelope + .as_ref() + .filter(|_| !item.deleted) + .map(|e| e[1..17].to_vec()); tx.execute( "INSERT INTO items (vault_id, id, rev, deleted, device_id, envelope, updated_at, author_id, key_id) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) diff --git a/server/tests/protocol.rs b/server/tests/protocol.rs index d4e1098..62c648f 100644 --- a/server/tests/protocol.rs +++ b/server/tests/protocol.rs @@ -186,6 +186,43 @@ async fn push_pull_conflicts_and_tombstones() { let tomb = page.items.iter().find(|i| i.id == b).unwrap(); assert!(tomb.deleted && tomb.envelope.is_none()); + // A signed deletion: its envelope is kept and comes back (any key generation, at most 1 KiB). + let signed = ulid(); + c.push(&vault, vec![put(&signed, 0, 5)]).await.unwrap(); + let deletion = |base_rev, len| PushItem { + id: signed.clone(), + base_rev, + deleted: true, + envelope: Some(bytes(len, 7)), + }; + let r = c.push(&vault, vec![deletion(5, 1025)]).await.unwrap(); + assert_eq!(r.results[0].status, PushStatus::Rejected); + let r = c.push(&vault, vec![deletion(5, 400)]).await.unwrap(); + assert_eq!(r.results[0].status, PushStatus::Accepted); + let page = c.pull(&vault, 5, 500).await.unwrap(); + let tomb = page.items.iter().find(|i| i.id == signed).unwrap(); + assert!(tomb.deleted); + assert_eq!(tomb.envelope, Some(bytes(400, 7))); + assert!(c.capabilities().await.unwrap().features.signed_tombstones); + // An item the server never had: its deletion keeps no envelope. + let unknown = ulid(); + let r = c + .push( + &vault, + vec![PushItem { + id: unknown.clone(), + base_rev: 0, + deleted: true, + envelope: Some(bytes(400, 7)), + }], + ) + .await + .unwrap(); + assert_eq!(r.results[0].status, PushStatus::Accepted); + let page = c.pull(&vault, 6, 500).await.unwrap(); + assert_eq!(page.items.len(), 1); + assert!(page.items[0].deleted && page.items[0].envelope.is_none()); + // Pagination. let page = c.pull(&vault, 0, 1).await.unwrap(); assert!(page.has_more && page.items.len() == 1); @@ -207,7 +244,7 @@ async fn push_pull_conflicts_and_tombstones() { assert!(r.results.iter().all(|x| x.status == PushStatus::Rejected)); let v = c.list_vaults().await.unwrap(); - assert_eq!((v.len(), v[0].head_rev, v[0].item_count), (1, 4, Some(1))); + assert_eq!((v.len(), v[0].head_rev, v[0].item_count), (1, 7, Some(1))); } #[tokio::test] diff --git a/spec/openapi.yaml b/spec/openapi.yaml index 6d6c44a..2a67b7c 100644 --- a/spec/openapi.yaml +++ b/spec/openapi.yaml @@ -1481,6 +1481,12 @@ components: description: Sign-up needs the code from POST /auth/register/start; email changes are available. sse: { type: boolean } totp: { type: boolean } + signed_tombstones: + type: boolean + description: >- + The server keeps the envelope a deletion carries (its author's signed deletion, + at most 1024 bytes) for an item it has, and returns it with the tombstone. Servers + without it ignore that envelope; clients send it either way. blobs: type: object description: Encrypted session logs and attachments (the paths under /vaults/{vault_id}/blobs). @@ -1736,7 +1742,8 @@ components: $ref: '#/components/schemas/B64' description: | version(1) || key_id(16) || nonce(24) || ciphertext || sig(64). - Empty when deleted = true. + When deleted = true: the deletion's envelope its author pushed (`signed_tombstones`), + or absent. updated_at: { type: string, format: date-time } author_id: type: string @@ -1764,7 +1771,12 @@ components: minimum: 0 description: Omit or 0 for new items. deleted: { type: boolean } - envelope: { $ref: '#/components/schemas/B64' } + envelope: + $ref: '#/components/schemas/B64' + description: >- + Required unless deleted = true. A deletion may carry its signed deletion (at most 1024 + bytes, any key generation of the vault; `too_large` beyond): a server with + `signed_tombstones` keeps it for an item it has, others ignore it. PushRequest: type: object diff --git a/tools/vault-conform/src/checks/vaults.rs b/tools/vault-conform/src/checks/vaults.rs index 3b24ce6..89804c2 100644 --- a/tools/vault-conform/src/checks/vaults.rs +++ b/tools/vault-conform/src/checks/vaults.rs @@ -1,6 +1,6 @@ //! Vaults and their items: create, list, push, pull, conflicts, tombstones, privacy. -use mg_sync::{Error, PushItem, PushStatus}; +use mg_sync::{B64, Error, MAX_DELETION_ENVELOPE, PushItem, PushStatus}; use reqwest::Method; use super::auth::outcome; @@ -292,6 +292,7 @@ pub async fn run(ctx: &Ctx, r: &mut Report) { .await; gone_after_purge(ctx, r, &c).await; + signed_deletions(ctx, r, &c).await; events(ctx, r, &c, &vault).await; r.check( @@ -309,6 +310,76 @@ pub async fn run(ctx: &Ctx, r: &mut Report) { .await; } +/// A deletion's envelope (its author's signed deletion) kept and returned: where the server +/// offers it. +async fn signed_deletions(ctx: &Ctx, r: &mut Report, c: &mg_sync::SyncClient) { + const E: (&str, &str) = ( + "POST /vaults/{vault_id}/changes", + "keeps a deletion's envelope (at most 1 KiB) and returns it with the tombstone", + ); + if !ctx + .caps + .as_ref() + .is_some_and(|c| c.features.signed_tombstones) + { + r.skip( + G, + E.0, + E.1, + "the server doesn't offer features.signed_tombstones", + ); + return; + } + r.check(G, E.0, E.1, async { + let vault = ulid(); + c.create_vault(&vault_req(&vault)).await.map_err(shown)?; + let id = ulid(); + c.push(&vault, vec![put(&id, 0, 1)]).await.map_err(shown)?; + let deletion = |envelope| PushItem { + id: id.clone(), + base_rev: 1, + deleted: true, + envelope: Some(envelope), + }; + let mut big = envelope(2).0; + big.resize(MAX_DELETION_ENVELOPE + 1, 2); + match c.push(&vault, vec![deletion(B64(big))]).await { + Ok(resp) => ensure( + resp.results + .iter() + .all(|x| x.status == PushStatus::Rejected), + "took a deletion's envelope over 1 KiB", + )?, + Err(Error::Api { + status: 400 | 413, .. + }) => {} + Err(e) => return Err(shown(e)), + } + let resp = c + .push(&vault, vec![deletion(envelope(3))]) + .await + .map_err(shown)?; + ensure( + resp.results + .iter() + .all(|x| x.status == PushStatus::Accepted), + format!("the deletion wasn't accepted: {:?}", resp.results), + )?; + let page = c.pull(&vault, 1, 500).await.map_err(shown)?; + let tomb = page + .items + .iter() + .find(|i| i.id == id) + .ok_or("the tombstone is missing")?; + ensure( + tomb.deleted && tomb.envelope == Some(envelope(3)), + "the tombstone came back without the deletion's envelope", + )?; + c.delete_vault(&vault).await.map_err(shown) + }) + .await; +} + /// `410` after tombstones are purged: only with a command that makes the server purge now. async fn gone_after_purge(ctx: &Ctx, r: &mut Report, c: &mg_sync::SyncClient) { const E: (&str, &str) = ( diff --git a/tools/vault-conform/tests/reference.rs b/tools/vault-conform/tests/reference.rs index f1015fb..1b0ea81 100644 --- a/tools/vault-conform/tests/reference.rs +++ b/tools/vault-conform/tests/reference.rs @@ -23,6 +23,10 @@ async fn the_reference_server_conforms() { .collect(); assert!(failed.is_empty(), "{}", failed.join("\n")); assert!(report.passed() > 60, "{}", report.text(false)); + // It keeps signed deletions, so that check runs. + assert!(report.checks.iter().any(|c| { + c.what.starts_with("keeps a deletion's envelope") && matches!(c.outcome, Outcome::Pass) + })); // What the reference server doesn't offer, or this run can't do, is skipped and says why. for c in &report.checks { if let Outcome::Skip(why) = &c.outcome { From ac0e0e4ef4b908ca2ffca69794a6e35b8e9c7662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentino=20Me=C4=91imorec?= Date: Wed, 21 Oct 2026 17:28:39 +0200 Subject: [PATCH 2/2] Updates: Install waits for a running check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Valentino Međimorec --- apps/desktop/e2e/appimage.mjs | 18 +++++++++++++++--- apps/desktop/src/AboutPanel.tsx | 3 ++- docs/features/updates.md | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/desktop/e2e/appimage.mjs b/apps/desktop/e2e/appimage.mjs index 69ade5a..aff9357 100644 --- a/apps/desktop/e2e/appimage.mjs +++ b/apps/desktop/e2e/appimage.mjs @@ -60,6 +60,18 @@ async function quickConnect(s) { await s.exec(`document.querySelector("button[type=submit].primary").click(); return true`); } +/** Clicks "Check now" in About and waits for that check to finish: the update shown before it + * may be from the check at launch, and Install takes the one the last finished check found. */ +async function checkNow(s) { + await s.exec(`const b = [...document.querySelectorAll("button")].find((b) => b.innerText.includes("Check now") && !b.disabled); + if (!b) throw new Error("no enabled button Check now"); + window.__checked = false; + new MutationObserver((_, o) => { if (b.innerText.includes("Check now")) { window.__checked = true; o.disconnect(); } }) + .observe(b, { subtree: true, childList: true, characterData: true }); + b.click(); return true`); + await s.wait(() => s.exec(`return window.__checked || null`), "the update check", 60000); +} + let s = await session(app); try { // A guessable master password is refused, and the form says why. @@ -861,7 +873,7 @@ try { // A tampered download is refused and the AppImage stays as it was. const before = sha(app); copyFileSync(`${feed}/latest-tampered.json`, `${feed}/latest.json`); - await s.clickButton("Check now"); + await checkNow(s); await s.waitText(`MorphGate ${next} is available`); await s.clickButton("Install and restart"); assert.match(await s.wait(s.panelError, "the refusal", 120000), /signature/); @@ -879,7 +891,7 @@ try { // The signed update installs, and the app restarts by itself. copyFileSync(`${feed}/latest-good.json`, `${feed}/latest.json`); - await s.clickButton("Check now"); + await checkNow(s); await s.waitText(`MorphGate ${next} is available`); await s.screenshot(`${shots}/update-available.png`); await s.clickButton("Install and restart"); @@ -922,7 +934,7 @@ try { await s.exec(`document.querySelector("form button[type=submit]").click(); return true`); await s.wait(() => s.exec(`return document.querySelector("button.brand") ? true : null`), "the workspace", 60000); assert.match(await openAbout(s), new RegExp(`Version ${next} · AppImage`)); - await s.clickButton("Check now"); + await checkNow(s); await s.waitText("up to date"); await s.screenshot(`${shots}/updated.png`); ok(`unlocked the same vault in ${next}, which is up to date`); diff --git a/apps/desktop/src/AboutPanel.tsx b/apps/desktop/src/AboutPanel.tsx index 1e91a36..4cae962 100644 --- a/apps/desktop/src/AboutPanel.tsx +++ b/apps/desktop/src/AboutPanel.tsx @@ -131,7 +131,8 @@ export function AboutPanel({ update, onUpdate, onAutoCheckChanged, onClose }: Pr

-
diff --git a/docs/features/updates.md b/docs/features/updates.md index b8823aa..badf138 100644 --- a/docs/features/updates.md +++ b/docs/features/updates.md @@ -54,7 +54,7 @@ After installing, `app.restart()` quits. On the way out the shell stops tunnels, | **Check for updates automatically** | `set_auto_update_check`. Hint: "At launch and twice a day MorphGate asks GitHub for the latest release. Nothing from your vault is sent. Updates are signed, and MorphGate installs only ones signed with its key." | | Update notice | "MorphGate X is available", "Released …", and the release notes | | **Install and restart** | `update_install`. Before it: "Open terminals, file browsers, tunnels and VPNs close when MorphGate restarts." (plus "Installing a package asks for an administrator password." for deb and rpm). During: a progress bar, "Downloading… 12.3 MiB of 40.1 MiB", then "Installing… MorphGate restarts when it's done." | -| **Check now** | `update_check`; "Checking…" while it runs; "MorphGate is up to date." when nothing is newer | +| **Check now** | `update_check`; "Checking…" while it runs, and **Install and restart** waits for it (Install takes the update the last finished check found); "MorphGate is up to date." when nothing is newer | | **Close** | Back to the workspace | On phones the Updates section is one line instead: "MorphGate updates through the App Store." on iOS, "MorphGate updates through Google Play, or by installing the next APK." on Android.