Item history: versioned items and signed deletions

ROADMAP.md

@@ -45,6 +45,11 @@ xterm.js in the webview avoids writing a terminal renderer.
       with progress, cancel and replace, mkdir/rename/chmod/delete
 - [x] `mg-sync`: vault-sync/v1 client (prelogin/register/login, token rotation, pull/push)
 - [x] Sync engine in `mg-store`: paginated pull, rollback guard, conflict copies, sealed refresh token
+- [x] Item history: every write carries a version inside its signed envelope, deletions are signed, and
+      a device refuses a copy of an item older than one it has seen (servers keep the signed deletion:
+      `signed_tombstones`)
+- [ ] Strict item history on by default (refusing unversioned changes, which only apps before item history
+      send), and signed vault checkpoints for devices with no copy to compare
 - [x] Reference sync server (`server/`, axum + SQLite) with protocol + two-device e2e tests
 - [x] Desktop sync: enable from an existing vault, sign in on a fresh device, background sync, disconnect
 - [x] Password change / recovery / new recovery key while synced (server first, other devices sign in again

apps/desktop/e2e/appimage.mjs

@@ -60,6 +60,18 @@ async function quickConnect(s) {
   await s.exec(`document.querySelector("button[type=submit].primary").click(); return true`);
 }
 
+/** Clicks "Check now" in About and waits for that check to finish: the update shown before it
+ *  may be from the check at launch, and Install takes the one the last finished check found. */
+async function checkNow(s) {
+  await s.exec(`const b = [...document.querySelectorAll("button")].find((b) => b.innerText.includes("Check now") && !b.disabled);
+    if (!b) throw new Error("no enabled button Check now");
+    window.__checked = false;
+    new MutationObserver((_, o) => { if (b.innerText.includes("Check now")) { window.__checked = true; o.disconnect(); } })
+      .observe(b, { subtree: true, childList: true, characterData: true });
+    b.click(); return true`);
+  await s.wait(() => s.exec(`return window.__checked || null`), "the update check", 60000);
+}
+
 let s = await session(app);
 try {
   // A guessable master password is refused, and the form says why.
@@ -861,7 +873,7 @@ try {
   // A tampered download is refused and the AppImage stays as it was.
   const before = sha(app);
   copyFileSync(`${feed}/latest-tampered.json`, `${feed}/latest.json`);
-  await s.clickButton("Check now");
+  await checkNow(s);
   await s.waitText(`MorphGate ${next} is available`);
   await s.clickButton("Install and restart");
   assert.match(await s.wait(s.panelError, "the refusal", 120000), /signature/);
@@ -879,7 +891,7 @@ try {
 
   // The signed update installs, and the app restarts by itself.
   copyFileSync(`${feed}/latest-good.json`, `${feed}/latest.json`);
-  await s.clickButton("Check now");
+  await checkNow(s);
   await s.waitText(`MorphGate ${next} is available`);
   await s.screenshot(`${shots}/update-available.png`);
   await s.clickButton("Install and restart");
@@ -922,7 +934,7 @@ try {
   await s.exec(`document.querySelector("form button[type=submit]").click(); return true`);
   await s.wait(() => s.exec(`return document.querySelector("button.brand") ? true : null`), "the workspace", 60000);
   assert.match(await openAbout(s), new RegExp(`Version ${next} · AppImage`));
-  await s.clickButton("Check now");
+  await checkNow(s);
   await s.waitText("up to date");
   await s.screenshot(`${shots}/updated.png`);
   ok(`unlocked the same vault in ${next}, which is up to date`);

apps/desktop/src/AboutPanel.tsx

@@ -131,7 +131,8 @@ export function AboutPanel({ update, onUpdate, onAutoCheckChanged, onClose }: Pr
                     </p>
                     <div className="actions">
                       <span className="spacer" />
-                      <button type="button" className="primary" onClick={install}>
+                      {/* Install takes the update the last finished check found. */}
+                      <button type="button" className="primary" disabled={checking} onClick={install}>
                         Install and restart
                       </button>
                     </div>

crates/mg-store/src/error.rs

@@ -30,6 +30,22 @@ pub enum Error {
     Untrusted(String),
     #[error("vault data is corrupt: {0}")]
     Corrupt(String),
+    /// A signed copy of an item older than one this device has seen: a server serving an old
+    /// envelope again.
+    #[error(
+        "the sync server sent an older copy of item {item} (version {remote}) than this device already has (version {seen}); refusing to roll it back"
+    )]
+    OlderItem {
+        item: String,
+        seen: i64,
+        remote: i64,
+    },
+    /// With strict item history: a change without a version, or a deletion without a
+    /// signature, to an item that has one.
+    #[error(
+        "the sync server sent a change to item {item} that isn't signed with a version; refusing it"
+    )]
+    UnversionedItem { item: String },
     #[error(transparent)]
     Crypto(#[from] mg_vault::Error),
     #[error("database: {0}")]

crates/mg-store/src/history.rs

@@ -0,0 +1,159 @@
+//! Item history: the version every write carries inside its signed envelope, and signed
+//! deletions, so a server can't serve a device an older copy of an item than one it has seen.
+//!
+//! An item's plaintext is its JSON with an `item_version`: the time of the write in milliseconds,
+//! or one more than the highest version the writer knows of the item when that is larger. A
+//! deletion is an envelope too, `{"type":"deleted","item_version":n}`, sealed for the same item.
+//! Apps before item history (0.2.0) ignore both: they write items without a version and
+//! deletions without an envelope, which a device accepts while such apps may still be in use
+//! (see `LocalVault::set_strict_history`).
+
+use serde::Deserialize;
+use zeroize::Zeroizing;
+
+use crate::vault::now;
+use crate::{Error, ItemData, Result};
+
+/// The plaintext field that holds an item's version.
+const FIELD: &str = "item_version";
+/// The `type` of a deletion's plaintext.
+const DELETED: &str = "deleted";
+
+/// What an envelope says: an item (`data`) or a deletion (`None`), and its version (`None`:
+/// written by an app before item history).
+#[derive(Debug)]
+pub(crate) struct Copy {
+    pub data: Option<ItemData>,
+    pub version: Option<i64>,
+}
+
+/// The version of a write after `known`, the highest version the writer has of the item.
+pub(crate) fn next(known: i64) -> i64 {
+    known.saturating_add(1).max(now())
+}
+
+/// `data` as an item's plaintext with `version`.
+pub(crate) fn plaintext(data: &ItemData, version: i64) -> Result<Zeroizing<Vec<u8>>> {
+    let json = Zeroizing::new(serde_json::to_vec(data).map_err(|e| Error::Invalid(e.to_string()))?);
+    // An item is always a JSON object with a `type`: the version goes in before its end.
+    let (end, body) = json
+        .split_last()
+        .filter(|(end, body)| **end == b'}' && body.len() > 1)
+        .ok_or_else(|| Error::Invalid("an item must be a JSON object".into()))?;
+    let field = format!(",\"{FIELD}\":{version}");
+    let mut out = Zeroizing::new(Vec::with_capacity(json.len() + field.len()));
+    out.extend_from_slice(body);
+    out.extend_from_slice(field.as_bytes());
+    out.push(*end);
+    Ok(out)
+}
+
+/// A deletion's plaintext.
+pub(crate) fn deletion(version: i64) -> Vec<u8> {
+    format!("{{\"type\":\"{DELETED}\",\"{FIELD}\":{version}}}").into_bytes()
+}
+
+/// `plain` (an item or a deletion, of any kind, from any app) with its version set to `version`.
+pub(crate) fn with_version(plain: &[u8], version: i64) -> Result<Zeroizing<Vec<u8>>> {
+    let mut value: serde_json::Value =
+        serde_json::from_slice(plain).map_err(|e| Error::Corrupt(e.to_string()))?;
+    let object = value
+        .as_object_mut()
+        .ok_or_else(|| Error::Corrupt("an item must be a JSON object".into()))?;
+    object.insert(FIELD.into(), version.into());
+    Ok(Zeroizing::new(
+        serde_json::to_vec(&value).map_err(|e| Error::Invalid(e.to_string()))?,
+    ))
+}
+
+/// Reads an opened envelope of item `id`.
+pub(crate) fn read(id: &str, plain: &[u8]) -> Result<Copy> {
+    #[derive(Deserialize)]
+    struct Head {
+        #[serde(rename = "type")]
+        kind: String,
+        #[serde(default)]
+        item_version: Option<i64>,
+    }
+    let corrupt = |e: serde_json::Error| Error::Corrupt(format!("item {id}: {e}"));
+    let head: Head = serde_json::from_slice(plain).map_err(corrupt)?;
+    // Versions start at 1; anything else counts as none.
+    let version = head.item_version.filter(|v| *v > 0);
+    let data = if head.kind == DELETED {
+        None
+    } else {
+        Some(serde_json::from_slice(plain).map_err(corrupt)?)
+    };
+    Ok(Copy { data, version })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::Snippet;
+
+    fn snippet() -> ItemData {
+        ItemData::Snippet(Snippet {
+            name: "uptime".into(),
+            body: "uptime".into(),
+            tags: vec![],
+        })
+    }
+
+    #[test]
+    fn items_and_deletions_carry_their_version() {
+        let plain = plaintext(&snippet(), 1_760_000_000_123).unwrap();
+        assert_eq!(
+            std::str::from_utf8(&plain).unwrap(),
+            r#"{"type":"snippet","name":"uptime","body":"uptime","item_version":1760000000123}"#
+        );
+        let copy = read("I", &plain).unwrap();
+        assert_eq!(
+            (copy.data, copy.version),
+            (Some(snippet()), Some(1_760_000_000_123))
+        );
+
+        let gone = deletion(7);
+        assert_eq!(gone, br#"{"type":"deleted","item_version":7}"#);
+        let copy = read("I", &gone).unwrap();
+        assert_eq!((copy.data, copy.version), (None, Some(7)));
+    }
+
+    #[test]
+    fn items_from_older_apps_have_no_version() {
+        let plain = serde_json::to_vec(&snippet()).unwrap();
+        let copy = read("I", &plain).unwrap();
+        assert_eq!((copy.data, copy.version), (Some(snippet()), None));
+        // Not a version: zero or below.
+        let copy = read(
+            "I",
+            br#"{"type":"snippet","name":"a","body":"b","item_version":0}"#,
+        )
+        .unwrap();
+        assert_eq!(copy.version, None);
+    }
+
+    #[test]
+    fn a_new_version_keeps_what_this_app_doesnt_know() {
+        let plain = br#"{"type":"future_kind","shape":{"x":1},"item_version":3}"#;
+        let copy = read("I", plain).unwrap();
+        assert_eq!(
+            (copy.data, copy.version),
+            (Some(ItemData::Unknown), Some(3))
+        );
+        let again: serde_json::Value =
+            serde_json::from_slice(&with_version(plain, 9).unwrap()).unwrap();
+        assert_eq!(
+            again,
+            serde_json::json!({"type": "future_kind", "shape": {"x": 1}, "item_version": 9})
+        );
+    }
+
+    #[test]
+    fn versions_only_grow() {
+        let far_ahead = now() + 86_400_000;
+        assert_eq!(next(far_ahead), far_ahead + 1);
+        assert!(next(0) >= now() - 1000);
+        assert_eq!(next(i64::MAX), i64::MAX);
+    }
+}

crates/mg-store/src/lib.rs

@@ -10,6 +10,7 @@ mod blobs;
 mod device;
 mod error;
 pub mod export;
+mod history;
 pub mod model;
 pub mod moving;
 mod schema;

crates/mg-store/src/moving.rs

@@ -311,11 +311,7 @@ impl LocalVault {
         let done = (|| -> Result<Moved> {
             let stamp = now();
             for p in going.iter().filter(|p| p.carry == Carry::Move) {
-                self.conn.execute(
-                    "UPDATE items SET envelope = NULL, deleted = 1, dirty = 1, updated_at = ?3
-                     WHERE vault_id = ?1 AND id = ?2",
-                    rusqlite::params![from, p.id, stamp],
-                )?;
+                self.tombstone(&from, &p.id, stamp)?;
             }
             let mut forget = Vec::new();
             for (id, data, old_blob) in &mut writes {

crates/mg-store/src/schema.rs

@@ -145,6 +145,14 @@ const MIGRATIONS: &[&str] = &[
     ",
     // v7: where to get the plan a refused blob needs.
     "ALTER TABLE blobs ADD COLUMN upgrade_url TEXT;",
+    // v8: item history. `version`: the one in the item's envelope (0: none, from an app before
+    // item history). `seen_version`: the one of the last copy the server confirmed, pulled or
+    // pushed (0: it had none). NULL in both: not read yet, from before this migration. A
+    // deletion made on this device keeps its signed envelope in `envelope`.
+    "
+    ALTER TABLE items ADD COLUMN version INTEGER;
+    ALTER TABLE items ADD COLUMN seen_version INTEGER;
+    ",
 ];
 
 pub(crate) fn migrate(conn: &mut Connection) -> Result<()> {