feat(tables): --layout on tables add, ivf_rebalance_sweep job type, SQL UPDATE/DELETE in the skill

Cargo.toml

@@ -16,7 +16,7 @@ path = "src/main.rs"
 # behind a shared multi-thread tokio runtime. The `arrow` feature backs result
 # decode; `upload_file` runs the presigned direct-to-storage upload flow
 # (see src/sdk.rs::Api::upload).
-hotdata = { version = "0.20.0", features = ["arrow"] }
+hotdata = { version = "0.21.0", features = ["arrow"] }
 # Shared multi-thread runtime for the sync wrapper; block_on is called
 # concurrently from rayon worker threads (see src/indexes.rs). The presigned
 # upload (src/sdk.rs::Api::upload) drives the SDK's async `upload_file` on this

plugin/skills/hotdata/SKILL.md

@@ -124,7 +124,7 @@ hotdata databases load (--catalog <alias> | --database <id>) --table <table> [--
 
 # Also available via tables subcommand
 hotdata databases tables list [--database <id>] [--schema <name>] [--workspace-id <workspace_id>] [--output table|json|yaml]
-hotdata databases tables add <table|schema.table> [--database <id>] [--schema public] [--key <col>]... [--key-determines <col>]... [--sorted-by <col>[=asc|desc]]... [--partition-by <col>[=identity|year|month|day|hour]]... [--output table|json|yaml]
+hotdata databases tables add <table|schema.table> [--database <id>] [--schema public] [--key <col>]... [--key-determines <col>]... [--sorted-by <col>[=asc|desc]]... [--partition-by <col>[=identity|year|month|day|hour]]... [--layout analytical|point_lookup] [--output table|json|yaml]
 hotdata databases tables load <table> [--database <id>] [--schema public] (--file <path> | --url <url> | --upload-id <id> | --result-id <id>) [--mode replace|append|delete|update|upsert] [--append] [--format csv|json|parquet] [--key <col>]... [--retry-timeout <secs>] [--workspace-id <workspace_id>]
 hotdata databases tables remove <table> [--database <id>] [--schema public] [--workspace-id <workspace_id>]
 ```
@@ -144,7 +144,7 @@ hotdata databases tables remove <table> [--database <id>] [--schema public] [--w
 - **Concurrent loads** — loads into one table run one at a time. While another load holds the table the server answers 409 `RESOURCE_LOCKED` with a `Retry-After`; the CLI waits and retries for up to `--retry-timeout` seconds (default 60). Pass `--retry-timeout 0` to fail immediately instead. Loads into different tables never block each other.
 - **Load modes** (`--mode`, default `replace`) — `replace` supersedes the table's contents; `append` adds rows; `delete`, `update`, and `upsert` match existing rows **by key**. `--append` is the old shorthand for `--mode append` and still works, but the two cannot be combined. The keyed modes need a key: declare one with `databases tables add --key`, or name it per-load with `--key` (repeat for a composite key). `delete` uploads only the key columns; `update` replaces matching rows and ignores unmatched ones; `upsert` inserts the unmatched instead. Keyed modes are not available with `--result-id`.
 - `tables list` — lists tables with `TABLE` (`<catalog>.<schema>.<table>`), `SYNCED`, `LAST_SYNC`. Uses current database when `--database` is omitted.
-- `tables add` — declares a table **with its key and storage layout**, which a load cannot infer. `--key` (repeatable) is what enables the `delete`/`update`/`upsert` load modes on that table. `--sorted-by <col>` or `<col>=desc` sets sort order; `--partition-by <col>` partitions on the value, `<col>=month` (or `year`/`day`/`hour`) on a calendar part — one partition per calendar month needs **both** `<col>=year` and `<col>=month`, or every March shares a partition. Sort and partition are fixed once the table exists. `--key-determines` (repeatable) asserts a column's value is fixed by the key: it prunes keyed loads harder, and is **correctness-affecting** — declare it only where the invariant really holds, or a keyed load can leave a duplicate key behind. Re-adding an existing table is a conflict (409), and `tables remove` does not clear the declaration — the table leaves the listing but the name stays declared and still conflicts. So **a key cannot be retrofitted onto a table declared without one**: declare it with `--key` up front, or use a new table name.
+- `tables add` — declares a table **with its key and storage layout**, which a load cannot infer. `--key` (repeatable) is what enables the `delete`/`update`/`upsert` load modes on that table. `--sorted-by <col>` or `<col>=desc` sets sort order; `--partition-by <col>` partitions on the value, `<col>=month` (or `year`/`day`/`hour`) on a calendar part — one partition per calendar month needs **both** `<col>=year` and `<col>=month`, or every March shares a partition. `--layout point_lookup` writes small data pages so rows are cheap to fetch by id — pick it for a table whose main reader is a vector index; scans of it read more pages (default `analytical`). Sort, partition and layout are fixed once the table exists. `--key-determines` (repeatable) asserts a column's value is fixed by the key: it prunes keyed loads harder, and is **correctness-affecting** — declare it only where the invariant really holds, or a keyed load can leave a duplicate key behind. Re-adding an existing table is a conflict (409), and `tables remove` does not clear the declaration — the table leaves the listing but the name stays declared and still conflicts. So **a key cannot be retrofitted onto a table declared without one**: declare it with `--key` up front, or use a new table name.
 - `tables load` — publishes to an instant-database table from a local file (`--file`), a remote URL (`--url`), a pre-staged upload (`--upload-id`), or a saved query result (`--result-id`, must belong to the target database). Same `--mode`, `--format`, and `--key` flags as the top-level `load` above.
 - `tables remove` — drops a table from the instant database.
 - `attach` — attaches **another instant database** to this one, so its **live** tables become visible inside this database's query scope. Name the other database by name, catalog alias, or id. Defaults to the current database; target another with `--database`. `--alias` sets the SQL name it answers to (defaults to the attached database's own catalog alias). **Required when that alias is `default`** — the stock name for a database created without `--catalog` — since `default` cannot be attached under its own name. This is how you **join across databases** — see [Querying across databases](#querying-across-databases-attach). Read-only: loads still target your own database, and attaching is not transitive — you see what you attached, not what it attached.
@@ -270,7 +270,7 @@ hotdata databases query status <query_run_id>
 - **A query runs inside one instant database** (current database or `--database`); with none set it fails *"a database is required."* The scope sees the database's own catalog **plus whatever is attached to it only**. To read another database's tables or join across databases, attach it first — see [Querying across databases (attach)](#querying-across-databases-attach).
 - Use `hotdata databases tables list` and `hotdata databases tables show` for discovery — not `information_schema` via `query`. (Discovery lists every workspace table; queryability still requires the table's catalog to be in the current database's scope.)
 - **PostgreSQL dialect.** Quote non-lowercase columns with double quotes. To write DuckDB/Postgres/Snowflake SQL instead, pass `--dialect` (server-side transpile; writes follow the same rules as `hotsql`) — details in **`hotdata-analytics`**.
-- **SQL writes.** `query` runs `INSERT INTO t [(cols)] VALUES …|SELECT …` (append only), `CREATE TABLE t (col TYPE, …)`, `CREATE TABLE t AS SELECT …`, and `DROP TABLE [IF EXISTS] t` against the **current database's own tables**; attached databases stay read-only. The API key needs the `write` permission. The result is one row, `rows_affected`. Each `INSERT` is a full publish (write lock, new generation, index maintenance), so **batch rows into one statement** and use `hotdata databases load` for anything of size. Refused: `UPDATE`/`DELETE` (*"not supported yet"* — use `hotdata databases load --mode update|delete|upsert` by key), `INSERT OVERWRITE`/`OR REPLACE`/`ON CONFLICT`/`RETURNING`, column constraints on `CREATE TABLE` (`NOT NULL`, `DEFAULT`, `PRIMARY KEY`, `UNIQUE` — every column is nullable), `ALTER`, views, `COPY`, transactions. A table with a vector index must be inserted into with a column list that omits the generated embedding column. Table layout (`--key`, partition, sort) is declared with `databases tables add`, not in SQL.
+- **SQL writes.** `query` runs `INSERT INTO t [(cols)] VALUES …|SELECT …`, `UPDATE t SET col = expr, … [WHERE …]`, `DELETE FROM t [WHERE …]`, `CREATE TABLE t (col TYPE, …)`, `CREATE TABLE t AS SELECT …`, and `DROP TABLE [IF EXISTS] t` against the **current database's own tables**; attached databases stay read-only. The API key needs the `write` permission. The result is one row, `rows_affected` (for `UPDATE`, the rows its `WHERE` matched). `UPDATE`/`DELETE` match rows by their internal rowid, so the table **needs no key**, and subqueries in `WHERE` work. Each write is a full publish (write lock, new generation, index maintenance), so **batch rows into one statement** and use `hotdata databases load` for anything of size. For bulk changes by key, `hotdata databases load --mode update|delete|upsert` is still the better tool. Refused: a target alias, `DELETE … LIMIT`/`USING`/`ORDER BY`, `UPDATE … FROM`/joins, `SET t.col` or the same column twice, `SET` of a key or `--key-determines` column on a table that declares `--key-determines`, `INSERT OVERWRITE`/`OR REPLACE`/`ON CONFLICT`/`RETURNING`, column constraints on `CREATE TABLE` (`NOT NULL`, `DEFAULT`, `PRIMARY KEY`, `UNIQUE` — every column is nullable), `ALTER`, views, `COPY`, transactions. A table with a vector index must be inserted into with a column list that omits the generated embedding column. Table layout (`--key`, partition, sort, `--layout`) is declared with `databases tables add`, not in SQL.
 - Async runs return `query_run_id` → poll with `query status <id>` (do not re-run the same heavy SQL). `query status` exit codes: `0` succeeded, `1` failed, `2` still running (poll again), `3` succeeded but the result is a truncated/incomplete preview.
 - **Large results: `-o csv` / `-o json` are complete, `-o table` is capped.** The server returns inline rows only up to a bounded cap and persists the full set under a `result_id`. For `csv` and `json` the CLI **streams** that full result batch by batch, so output size is unbounded and memory stays flat — pipe a big result to a file with `-o csv`/`-o json`, never `table`. For `table` the CLI fetches at most **10,000 rows** and the footer says `N of TOTAL rows — INCOMPLETE PREVIEW` (with `?` when the server sent no total); the process exits **3** so a pipeline cannot mistake it for the whole set. Same rules for `hotdata databases results get`. A result the server is still writing is waited for (up to 5 minutes, honoring `Retry-After`) rather than returned partial. If the full result can't be retrieved, the CLI prints the inline preview, a `warning:` to stderr, and exits 3. `-o json` carries `row_count`, `total_row_count`, and `truncated` — branch on `truncated`, not on the row count.
 - **Numbers print exactly.** A `DECIMAL(38,2)` or other wide number is printed with every digit the service sent, in every output format; `-o json` emits it as an unquoted JSON number. One consequence: a list or struct cell prints on a single line in `-o json` (re-indenting would mean re-parsing and rounding it); scalars are unaffected.
@@ -284,7 +284,7 @@ hotdata jobs list [--workspace-id <workspace_id>] [--job-type <type>] [--status
 hotdata jobs <job_id> [--workspace-id <workspace_id>] [--output table|json|yaml]
 ```
 - `list` shows only active jobs (`pending`, `running`) by default. Use `--all` to see all jobs.
-- `--job-type`: `create_index`, `managed_load`, `database_fork`. (`database_fork` is accepted for forward compatibility; today a fork completes synchronously and records no job.) (`data_refresh_table` / `data_refresh_connection` were retired server-side — the engine is push-only; nothing refreshes.)
+- `--job-type`: `create_index`, `managed_load`, `database_fork`, `ivf_rebalance_sweep` (background IVF index rebalancing). (`database_fork` is accepted for forward compatibility; today a fork completes synchronously and records no job.) (`data_refresh_table` / `data_refresh_connection` were retired server-side — the engine is push-only; nothing refreshes.)
 - `--status`: `pending`, `running`, `succeeded`, `partially_succeeded`, `failed`.
 - Use `hotdata jobs <job_id>` to inspect a specific job's status, error, and result.
 

src/commands/databases.rs

@@ -428,6 +428,12 @@ pub enum DatabaseTablesCommands {
         #[arg(long = "partition-by")]
         partition_by: Vec<String>,
 
+        /// Storage layout. `point_lookup` writes small data pages so rows are
+        /// cheap to fetch by id (what vector-index answers do), at the cost of
+        /// slower scans. Default: `analytical`. Fixed once the table is created.
+        #[arg(long, value_parser = ["analytical", "point_lookup"])]
+        layout: Option<String>,
+
         /// Output format
         #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])]
         output: String,
@@ -1085,6 +1091,7 @@ fn table_declaration_body(
     key_determines: &[String],
     sorted_by: &[serde_json::Value],
     partition_by: &[serde_json::Value],
+    layout: Option<&str>,
 ) -> serde_json::Value {
     let mut body = serde_json::json!({ "name": table });
     if !key.is_empty() {
@@ -1101,6 +1108,9 @@ fn table_declaration_body(
     if !partition_by.is_empty() {
         body["partition_by"] = serde_json::json!(partition_by);
     }
+    if let Some(layout) = layout {
+        body["layout"] = serde_json::json!(layout);
+    }
     body
 }
 
@@ -1115,6 +1125,7 @@ pub fn add_table(
     key_determines: &[String],
     sorted_by: &[String],
     partition_by: &[String],
+    layout: Option<&str>,
     output: &str,
 ) {
     use crossterm::style::Stylize;
@@ -1148,7 +1159,14 @@ pub fn add_table(
     let api = Api::new(Some(workspace_id));
     let db = resolve_database(&api, &database);
 
-    let body = table_declaration_body(table, key, key_determines, &sorted_by, &partition_by);
+    let body = table_declaration_body(
+        table,
+        key,
+        key_determines,
+        &sorted_by,
+        &partition_by,
+        layout,
+    );
 
     let (status, resp) = declare_table(&api, &db.id, schema, &body);
 
@@ -1170,6 +1188,7 @@ pub fn add_table(
         "key_determines": key_determines,
         "sorted_by": sorted_by,
         "partition_by": partition_by,
+        "layout": layout.unwrap_or("analytical"),
     });
     match output {
         "json" => println!("{}", serde_json::to_string_pretty(&declared).unwrap()),
@@ -1198,6 +1217,9 @@ pub fn add_table(
             if !partition_by.is_empty() {
                 println!("parts:   {}", describe_partition_keys(&partition_by));
             }
+            if let Some(layout) = layout {
+                println!("layout:  {layout}");
+            }
         }
     }
 }
@@ -4392,7 +4414,7 @@ mod tests {
             .with_status(200)
             .with_header("content-type", "application/json")
             .with_body(
-                r#"{"count":1,"limit":1000,"tables":[{"connection":"default","schema":"public","table":"b","synced":true,"partition_by":[],"sorted_by":[],"constant_per_key":[],"last_sync":null}],"has_more":false,"next_cursor":null}"#,
+                r#"{"count":1,"limit":1000,"tables":[{"connection":"default","schema":"public","table":"b","synced":true,"partition_by":[],"sorted_by":[],"constant_per_key":[],"layout":"analytical","last_sync":null}],"has_more":false,"next_cursor":null}"#,
             )
             .create();
         let page0 = server
@@ -4404,7 +4426,7 @@ mod tests {
             .with_status(200)
             .with_header("content-type", "application/json")
             .with_body(
-                r#"{"count":1,"limit":1000,"tables":[{"connection":"default","schema":"public","table":"a","synced":false,"partition_by":[],"sorted_by":[],"constant_per_key":[],"last_sync":null}],"has_more":true,"next_cursor":"cur2"}"#,
+                r#"{"count":1,"limit":1000,"tables":[{"connection":"default","schema":"public","table":"a","synced":false,"partition_by":[],"sorted_by":[],"constant_per_key":[],"layout":"analytical","last_sync":null}],"has_more":true,"next_cursor":"cur2"}"#,
             )
             .create();
 
@@ -4518,7 +4540,7 @@ mod tests {
             .with_status(200)
             .with_header("content-type", "application/json")
             .with_body(
-                r#"{"count":1,"limit":5,"tables":[{"connection":"default","schema":"public","table":"a","synced":true,"partition_by":[],"sorted_by":[],"constant_per_key":[],"last_sync":null}],"has_more":true,"next_cursor":"page2"}"#,
+                r#"{"count":1,"limit":5,"tables":[{"connection":"default","schema":"public","table":"a","synced":true,"partition_by":[],"sorted_by":[],"constant_per_key":[],"layout":"analytical","last_sync":null}],"has_more":true,"next_cursor":"page2"}"#,
             )
             .expect(1)
             .create();
@@ -4544,7 +4566,7 @@ mod tests {
             .with_status(200)
             .with_header("content-type", "application/json")
             .with_body(
-                r#"{"count":1,"limit":1000,"tables":[{"connection":"default","schema":"public","table":"orders","synced":true,"partition_by":[],"sorted_by":[],"constant_per_key":[],"last_sync":null}],"has_more":false,"next_cursor":null}"#,
+                r#"{"count":1,"limit":1000,"tables":[{"connection":"default","schema":"public","table":"orders","synced":true,"partition_by":[],"sorted_by":[],"constant_per_key":[],"layout":"analytical","last_sync":null}],"has_more":false,"next_cursor":null}"#,
             )
             .create();
 
@@ -4602,7 +4624,8 @@ mod tests {
                 &cols(&["order_id"]),
                 &cols(&["event_date"]),
                 &[],
-                &[]
+                &[],
+                None
             ),
             serde_json::json!({
                 "name": "orders",
@@ -4611,9 +4634,13 @@ mod tests {
             })
         );
         assert_eq!(
-            table_declaration_body("orders", &[], &[], &[], &[]),
+            table_declaration_body("orders", &[], &[], &[], &[], None),
             serde_json::json!({"name": "orders"})
         );
+        assert_eq!(
+            table_declaration_body("orders", &[], &[], &[], &[], Some("point_lookup")),
+            serde_json::json!({"name": "orders", "layout": "point_lookup"})
+        );
     }
 
     #[test]