diff --git a/.cargo/config.toml b/.cargo/config.toml index 08ed59af..3cb0dc48 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-or-later [alias] -xtask = "run --quiet --release --package xtask --" +xtask = "run --quiet --release --locked --package xtask --" # The browser app's Leptos views are type-erased (`--cfg erase_components`): # each reactive closure is boxed instead of monomorphizing the view code for diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a7c765c..6957dc4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: name: performance budget (allthing.social) if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-24.04 - # Up to 30 minutes waiting for the deploy (scripts/wait-for-deploy.mjs), + # Up to 30 minutes waiting for the deploy (cargo xtask wait-for-deploy), # then the measurements. timeout-minutes: 45 steps: @@ -79,6 +79,9 @@ jobs: # The history the deploy wait needs to tell whether a later live # commit includes this one. fetch-depth: 0 + - name: Install the pinned Rust toolchain + run: rustup toolchain install + - uses: Swatinem/rust-cache@v2 - uses: actions/setup-node@v6 with: node-version: 22 @@ -86,7 +89,7 @@ jobs: - run: npm ci - run: npx playwright install --with-deps chromium - name: Wait for the intended deployment - run: node scripts/wait-for-deploy.mjs https://allthing.social "$(git rev-parse HEAD)" + run: cargo xtask wait-for-deploy https://allthing.social "$(git rev-parse HEAD)" - run: npx playwright test e2e/perf.spec.ts env: E2E_BASE_URL: https://allthing.social @@ -147,12 +150,12 @@ jobs: ~/.cache/worker-build key: browser-tools-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} - run: npm ci - - run: node scripts/npm-audit.mjs + - run: cargo xtask npm-audit - run: npm run test:tools - run: npx playwright install --with-deps chromium - run: bash scripts/build.sh - name: Test the local build without live data - run: sh scripts/test-browser-fixtures.sh + run: cargo xtask browser-fixtures - name: Test the static copy run: | bash scripts/build-pages.sh diff --git a/.gitignore b/.gitignore index ac7e70af..ac4b8608 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Rust and build output /target/ +/spacetime/spacetimedb/target/ /dist/ /dist-pages/ /crates/allthing-worker/build/ diff --git a/AGENTS.md b/AGENTS.md index b5fa1aab..054e9f14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,6 +5,7 @@ Allthing is a Farcaster client for reading, written in Rust: a Leptos WebAssembl - **One QR flow: sign in, and write, with a key the reader approves once.** By the owner's decision, where the Worker endorses keys there is one approval and one dialog: the reader approves Allthing's key request in the Farcaster app (a QR code on a computer, a link to the app on a phone), which signs them in (`POST /api/v1/auth/key`) and lets them cast, reply, quote, like, recast, follow and delete from here, with no second scan or separate grant step. A reader signed in without a key in this browser (signed in before keys, after Revoke permissions, or with its storage cleared) meets the same dialog, in its approve form, on their first write (`Session::grant`), never by itself on a page load. Never add a second approval dialog or QR flow. Sign In With Farcaster by QR code is only the fallback where the Worker endorses no keys, and makes no relay request where it does (`allthing-worker/src/auth.rs`, `allthing-core/src/siwf.rs`, `allthing-web/src/components/signin.rs`; see `docs/ARCHITECTURE.md#sign-in`). Once a reader has approved Allthing in the Farcaster app, casts, replies, quotes, likes, recasts, follows and deletes go from Allthing: the browser makes and keeps the key and signs (`allthing-core/src/app_key.rs` and `message.rs`, `allthing-web/src/keys.rs` and `components/writer.rs`; see `docs/ARCHITECTURE.md#liking-and-recasting`), and the Worker keeps scheduled casts the browser signed until their time (`allthing-worker/src/scheduled_casts.rs`, `allthing-data/src/scheduled.rs`). Allthing asks for those keys as its own Farcaster account, @allthingapp (FID 3356318, `ALLTHING_FID` in `wrangler.toml`), which its owner registered for it, and shows the casts "via Allthing" (`allthing-core/src/clients.rs`). By the owner's decision that is Allthing's only account: never add another, a seed phrase import or a wallet connection. The Worker endorses readers' keys with that account's custody key, the `ALLTHING_CUSTODY_KEY` secret, set in Cloudflare only (`allthing-worker/src/app_key.rs`; see `docs/DEPLOY.md#secrets`), and uses it for nothing else; it never holds a reader's key or signs as a reader. Until the secret is set (`writes.ready` in `/api/v1/status`), and for a reader signed out, a write opens the cast, the profile or a draft ready to post in the Farcaster app instead (`allthing-web/src/components/handoff.rs`, `allthing-core/src/compose.rs`; see `docs/ARCHITECTURE.md#handing-writes-to-the-farcaster-app`). Never add other login methods, wallet connections or write paths unless asked to. Messages (`/messages`) is Allthing's own, on SpacetimeDB (`allthing-web/src/pages/messages`, `spacetime/`; see `docs/ARCHITECTURE.md#messages-on-spacetimedb`): never add Farcaster Direct Cast API calls or an API-key paste step to it unless asked. - **Never hide content** because of who posted it. Allthing shows no account labels; spam is judged by a cast's content (see Mini apps and spam below). +- **Rust first.** The owner prefers Rust for application behavior and native tooling. Put pure shared rules in `allthing-core`, interactive UI in `allthing-web`, and native build/check policy in `xtask`. Keep `web/paint.js`'s inert first-card renderer: the owner chose it after a paired cold-load comparison beat the Rust prototype (see `docs/PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer`). Retain the existing Node static test server (`e2e/pages-server.mjs`) after an unresolved pre-mount stall in migration CI; the server's role in that stall was not established (see `docs/PERFORMANCE.md#retained-node-static-test-server`). Keep the small JavaScript loader and inline preload/data hints, the SpacetimeDB TypeScript SDK adapter/generated bindings, and official Playwright browser tests at their platform boundaries. CSS, SQL, HTML and config remain platform data. Preserve measured performance, license rules and independent test oracles; do not mask handwritten files with Linguist attributes. - **Where code goes.** - Pure logic and data types, with unit tests, go in `crates/allthing-core`. - Provider quirks go in `crates/allthing-data/src/hypersnap.rs` and `crates/allthing-core/src/normalize.rs`. @@ -19,7 +20,8 @@ Allthing is a Farcaster client for reading, written in Rust: a Leptos WebAssembl - **Licensing.** Licensing is AGPL-3.0-or-later. Start new files with an SPDX header. New dependencies must pass `cargo deny check licenses`. - **Secrets.** Never commit secrets, tokens, `.dev.vars` or `.env` files, or session URLs. The Worker's secrets (`SESSION_SECRET`, `SOLANA_RPC_URL`) are set in Cloudflare; see `docs/DEPLOY.md#secrets`. - **Checks.** - - Run `bash scripts/check.sh` (fmt, Clippy native and wasm32, tests, license audit). + - Run `cargo xtask check` (fmt, Clippy native and wasm32, tests, license audit). + - Run `cargo xtask module-check` for the isolated SpacetimeDB crate (fmt, native/Wasm Clippy, tests, release Wasm build and license audit); `spacetime/check.sh` is a thin compatibility wrapper. - End to end: `bash scripts/build.sh && npx playwright test`. Playwright starts `wrangler dev` on port 8787. - Against production: `E2E_BASE_URL=https://allthing.social npx playwright test`. - The static copy: `bash scripts/build-pages.sh && E2E_PAGES=1 npx playwright test`. @@ -32,3 +34,12 @@ Allthing is a Farcaster client for reading, written in Rust: a Leptos WebAssembl - **Speed.** Keep within `docs/PERFORMANCE.md`. Prefetches go through `api::prefetch`, which caps and dedupes them. - **Mini apps and spam.** Spam by the rules in `allthing-core/src/spam.rs` is filtered out server-side (`thread::without_spam`, `thread::prune_spam`). Casts with mini app or frame links are marked (`Cast.mini_app`) and kept in feed, profile, channel and thread payloads; the browser hides them unless the reader turned on Show mini app casts (`allthing-web/src/miniapps.rs`, `Cast::hidden_for`), the default being hidden, and never runs them. Search, reply previews and notifications still drop both (`Cast::hidden`). Keep new list endpoints doing one or the other. - **Deploying.** Cloudflare Workers Builds deploys `main` with `bash scripts/build.sh` and `npx wrangler deploy`; see `docs/DEPLOY.md`. + +## Conservative GitHub Actions use + +- Follow the current user approval policy for GitHub Actions budgets and exceptions. Preserve stricter project pauses and required release evidence. +- Run proportionate validation locally first, inspect the diff, and batch coherent changes before remote CI. Avoid repeated dispatch or rerun loops. +- Before any dispatch, rerun, push, pull-request creation or update, merge, or trigger change that can start Actions, inspect repository run history for the current UTC day using read-only tools. Check every workflow, branch and actor, including queued, in-progress and completed runs, all rerun attempts, and attempts of older runs. Account for duplicate push/PR events, downstream `workflow_run` chains, scheduled services and GitHub-generated workflows. Estimate the resulting runs and runner minutes; a skipped job or cancelled run is not a daily cap. Coordinate parallel agents because a read-only check is only a snapshot. +- Escalate exception requests, unknown monthly usage, and CI or release conflicts to the coordinating assistant under the current user approval policy before triggering remote work. Report unavailable usage as unknown; do not expand billing permissions to obtain it. +- Checkpoint and source-publication instructions remain subject to this preflight and the current user approval policy. If publication is blocked, retain durable work locally and report the precise blocker. Preserve required checks and service/deployment behavior; do not bypass checks or alter triggers to fit a budget. +- Include this operational guidance in delegated tasks. Apply the current user approval policy to each proposed action; do not copy permission grants or numeric approval thresholds into repository instructions or memory. diff --git a/CHANGELOG.md b/CHANGELOG.md index fd4e3e91..232a649d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ All notable changes to Allthing are recorded here. The format follows [Keep a Ch ### Development +- Move build orchestration, Brotli compression, repository validation, asset-credit checks, deployment waiting, isolated database-module checks, audit handling and browser fixture orchestration into native Rust `xtask` commands. Retain official Playwright tests and independent cryptography, SQL and SDK codec checks. + - Live layout and interaction checks wait for loaded app content instead of loading placeholders or inert early cards; first-paint and navigation timing limits remain unchanged. - Browser checks wait for app handover after measuring the first painted card, compare Back with the position at the actual click, and await exact page-load counts. Local media checks run database commands asynchronously so CI's Node 22 client can retire closed connections, and retain bounded Worker diagnostics on failure. Existing timing limits, request counts and upload assertions remain. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9cece43c..1621bcea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ Everyone taking part is expected to follow the [code of conduct](CODE_OF_CONDUCT - `npx playwright test`: the whole suite, including `e2e/allthing.spec.ts` against live Farcaster data. Playwright starts `wrangler dev` on port 8787. Tests run at 1280×900; the phone-width tests and some fixture tests run at 390×844. - `npx playwright test e2e/perf.spec.ts`, for anything on the reading path. - `bash scripts/build-pages.sh && E2E_PAGES=1 npx playwright test`, if you touched the static copy. -- `npm run test:tools`, if you changed the static test server (`e2e/pages-server.mjs`). +- `cargo test --locked -p xtask`, if you changed native build/check tooling. Run `npm run test:tools` for the static test server and the independent cryptography, SQL and SDK codec checks. - For UI changes, attach before-and-after screenshots at both widths to the pull request. The live-data suite saves screenshots in `docs/screenshots/` (set `E2E_SCREENSHOT_DIR` to save them elsewhere). Only the WebP images the README shows (`home.webp`, `profile.webp`, `dau.webp`) are tracked; replace one only when the README should change. diff --git a/Cargo.lock b/Cargo.lock index 76269df2..073faded 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allthing-core" version = "0.5.1" @@ -228,6 +243,27 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -2304,6 +2340,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tokio" version = "1.53.3" @@ -2403,6 +2445,15 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -2756,7 +2807,18 @@ name = "xtask" version = "0.5.1" dependencies = [ "allthing-core", + "base64", + "brotli", + "ed25519-dalek", + "getrandom 0.4.3", + "libc", + "serde_json", + "sha2 0.10.9", + "toml", + "unicode-normalization", + "url", "wasm_split_cli_support", + "zeroize", ] [[package]] diff --git a/README.md b/README.md index 4d88b37e..e7312fdf 100644 --- a/README.md +++ b/README.md @@ -57,14 +57,16 @@ Allthing is an independent Farcaster client, written in Rust, that shows trendin ### Fast and frugal - **One API call per page**, cached at the edge, so the whole site runs on the Cloudflare Workers Free plan. -- **Early paint.** The front page draws its first cards before the app has loaded (about 1.2 s on a slow phone). +- **Early paint.** A small JavaScript renderer draws the front page's first cards from its existing response while the Rust app downloads. It stays in JavaScript because it beat the Rust prototype in the [paired cold-load comparison](docs/PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer). - **Split WebAssembly.** Pages, the composer, writing, sign-in and the DAU dashboard are code of their own, fetched when used, so every visit downloads less. - **Instant navigation.** Posts open from the feed's own data, cards are prefetched within a capped budget, and Back returns to the same place. CI holds smoothness to budgets ([docs/PERFORMANCE.md](docs/PERFORMANCE.md)). - **Accessible.** Keyboard throughout, alt text, focus kept in dialogs, reduced motion respected, and it fits a phone. ## Architecture -Allthing is one Cloudflare Worker that serves a static WebAssembly app and a small JSON API. All of it is Rust, apart from generated JavaScript glue and a few lines of boot script. +Allthing is one Cloudflare Worker that serves a WebAssembly app and a small JSON API. Application logic, the Leptos interface, the API Worker and the SpacetimeDB module are written in Rust. Native Rust tooling also owns builds, compression and repository checks. + +JavaScript loads WebAssembly, starts the page's data request before it arrives and renders its first inert cards. That early renderer is a measured performance exception to the Rust-first policy. Messages keeps a TypeScript adapter for the official SpacetimeDB SDK and its generated bindings; browser tests keep the official Playwright TypeScript runner and the small Node static-site test server. CSS, SQL, HTML and configuration stay in their platform formats. ```text ┌── static files ──▶ Workers Static Assets: the app, styles, fonts (free) @@ -82,7 +84,7 @@ The Worker builds each payload from [Hypersnap](https://github.com/farcasterorg/ | [`allthing-data`](crates/allthing-data) | Page payload builders and guarded upstream clients, behind a small `Host` trait so they run in the Worker and in the browser. | | [`allthing-worker`](crates/allthing-worker) | The `/api/v1` Worker (workers-rs): routing, the per-IP rate limit, the edge cache and request coalescing. | | [`allthing-web`](crates/allthing-web) | The browser app: Leptos, rendered client-side, compiled to WebAssembly. | -| [`xtask`](xtask) | Build tooling: fingerprinted static assets and the static copy. | +| [`xtask`](xtask) | Native build orchestration, fingerprinted assets, Brotli compression, repository and credit checks, attestation key generation and deployment waiting. | More in [docs/](docs): @@ -110,18 +112,23 @@ The first build also installs `wasm-bindgen` and `worker-build` into `target/too Checks and tests: ```sh -bash scripts/check.sh # version, CSP hashes, changelog links, rustfmt, Clippy, unit tests, cargo-deny +cargo xtask check # version, CSP hashes, changelog links, rustfmt, Clippy, unit tests, cargo-deny +cargo xtask module-check # isolated SpacetimeDB crate: fmt, native/Wasm Clippy, tests, release build, licenses npx playwright install chromium # once -npx playwright test '\.fixture\.spec\.ts$' # deterministic browser regressions, as CI runs them +cargo xtask browser-fixtures # functional fixtures, then isolated smoothness, as CI runs them npx playwright test # everything, including live Farcaster data E2E_BASE_URL=https://allthing.social E2E_EXPECT_WRITES_READY=true npx playwright test # production bash scripts/build-pages.sh && E2E_PAGES=1 npx playwright test # the static copy ``` -Playwright starts `wrangler dev` on port 8787 by itself. CI runs `check.sh` and the fixture regressions on pull requests and pushes to `main`, and checks the [performance budget](docs/PERFORMANCE.md) against production after each push to `main`. Cloudflare Workers Builds deploys `main` ([docs/DEPLOY.md](docs/DEPLOY.md)); the repository holds no secrets. +Playwright starts `wrangler dev` on port 8787 by itself, or `node e2e/pages-server.mjs` for the static copy. CI runs `cargo xtask check` and the fixture regressions on pull requests and pushes to `main`, and checks the [performance budget](docs/PERFORMANCE.md) against production after each push to `main`. Cloudflare Workers Builds deploys `main` ([docs/DEPLOY.md](docs/DEPLOY.md)); the repository holds no secrets. + +Other native commands include `cargo xtask check-repo` (metadata only), `cargo xtask check-credits` (read-only online asset-credit audit), `cargo xtask npm-audit`, and `cargo xtask wait-for-deploy https://allthing.social `. Attestation key generation is documented with its secret-handling precautions in [DEPLOY.md](docs/DEPLOY.md#secrets). ## Contributing +**Rust first.** Add application logic to the Rust crates and native build or validation policy to `xtask`. Retain the measured faster JavaScript early-card renderer, the existing Node static test server and necessary browser, SDK and official test-tool boundaries; document exceptions and compare performance before migrating them. Preserve performance, licensing and independent test oracles; language statistics are not a reason to hide or reclassify handwritten files. + Bug reports, fixes and focused improvements are welcome, within Allthing's scope: it reads, its one sign-in is Farcaster's, and it writes only what a reader asks for, with a key they approved. Read [CONTRIBUTING.md](CONTRIBUTING.md) for the ground rules, the workflow and the checks, and follow the [code of conduct](CODE_OF_CONDUCT.md). Commits and pull request titles are [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/), each change a reader or operator would notice gets a line in the [changelog](CHANGELOG.md), and versions follow [Semantic Versioning](https://semver.org) ([docs/RELEASING.md](docs/RELEASING.md)). ## Security diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ef2b555d..d774ec9b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,7 +1,7 @@ # Architecture -Allthing is a Farcaster client for reading: it shows Farcaster, and the Worker posts nothing of its own. It is one Cloudflare Worker that serves two things: a static WebAssembly app, and a small JSON API that the app calls once per page. Readers can [sign in with Farcaster](#sign-in), by scanning a QR code with the Farcaster app, by one approval that also lets them [like, recast, cast, reply, quote, follow and delete](#liking-and-recasting) with a key their browser keeps; the Worker holds [scheduled casts](#scheduled-casts) the browser signed until their time. [Messages](#messages-on-spacetimedb) are SpacetimeDB chat and end-to-end encrypted DMs. The Worker also keeps the images and videos readers [attach](#attachments) to their casts, in R2, and serves them at `/m/`. All of it is Rust, except generated glue, a short boot script, a one-line inline script that starts the page's data request, and a script that draws the front page's first cards from it before the app has come. +Allthing is a Farcaster client for reading: it shows Farcaster, and the Worker posts nothing of its own. It is one Cloudflare Worker that serves two things: a static WebAssembly app, and a small JSON API that the app calls once per page. Readers can [sign in with Farcaster](#sign-in), by scanning a QR code with the Farcaster app, by one approval that also lets them [like, recast, cast, reply, quote, follow and delete](#liking-and-recasting) with a key their browser keeps; the Worker holds [scheduled casts](#scheduled-casts) the browser signed until their time. [Messages](#messages-on-spacetimedb) are SpacetimeDB chat and end-to-end encrypted DMs. The Worker also keeps the images and videos readers [attach](#attachments) to their casts, in R2, and serves them at `/m/`. Application behavior and most build/check tooling are Rust. The early-card renderer stays in JavaScript after a measured cold-load comparison; the loader and inline hints start data and WebAssembly downloads; Messages uses a narrow TypeScript adapter to its official SDK. Generated bindings, the official Playwright TypeScript test runner and the Node static-site test server stay at those platform boundaries. ```text Browser Cloudflare (one Worker, Free plan) Upstreams @@ -41,9 +41,15 @@ Allthing is a Farcaster client for reading: it shows Farcaster, and the Worker p | `allthing-data` | native + wasm32 | The page builders that assemble each payload, and the Hypersnap and RPC clients behind guards. The runtime is abstracted as a `Host` (clock, HTTP with a timeout, background work), so the same code runs in the Worker and in the browser. | | `allthing-worker` | wasm32 (Worker) | The `/api/v1` handler: routing, the per-IP rate limit, the edge cache with coalescing, and a `Host` over Workers `fetch`, `AbortController`, `waitUntil` and D1. Its cron handler runs the indexer. `media.rs` keeps readers' [attachments](#attachments) in R2 and serves them. `auth.rs` is [sign-in](#sign-in) and `app_key.rs` [endorses readers' keys](#liking-and-recasting); between them they hold the Worker's cryptography (secp256k1 signer recovery and signing, Keccak-256, HMAC-SHA256) and the QR encoder, none of which ships to browsers. A reader's own key is their browser's (WebCrypto's Ed25519). | | `allthing-web` | wasm32 (browser) | The Leptos app (client-side rendering): routes, pages, components, the media viewer, the HLS player, prefetching and scroll restoration, and liking and recasting with the reader's key (`keys.rs`, `components/writer.rs`). With the `static-host` feature it builds payloads in-process through a browser `Host`. | -| `xtask` | native | `cargo xtask dist` builds `dist/`; `cargo xtask pages` builds the static copy in `dist-pages/`. | +| `xtask` | native | Build orchestration, Brotli-11 compression, esbuild CLI/SDK-version checks, metadata and credit audits, attestation key generation, optimizer discovery, fixture phases, npm audit, isolated SpacetimeDB validation and deployment waiting. `dist` builds `dist/`; `pages` builds `dist-pages/`. | -`scripts/build.sh` runs `worker-build` for the Worker and `cargo xtask dist` for the app. The xtask runs `cargo build`, wasm-split (which cuts out the parts of the app loaded when used), `wasm-bindgen --target web` and `wasm-opt -Oz`, then writes fingerprinted files (`allthing-.wasm`, and the folder `allthing-/` with its JavaScript, the parts and their loader; `allthing-.css`, `boot-.js`) and fills them into `web/index.html`. Both WebAssembly builds use `opt-level = "z"`, LTO, one codegen unit and `panic = "abort"`. +`scripts/build.sh` only bootstraps rustup and the pinned toolchain on images without Rust, then runs `cargo xtask build`. The native build selects the locked tools, builds the Worker, strips its relocations and builds the browser assets. `cargo xtask dist` runs `cargo build`, wasm-split for the interactive app, `wasm-bindgen --target web` and `wasm-opt -Oz`. It writes fingerprinted Wasm, glue, loaders, CSS and HTML, and compresses the optional `.wasm.br` copies in Rust at quality 11, window 24. The Messages adapter is bundled through the installed esbuild CLI after checking its SDK version. WebAssembly release builds use `opt-level = "z"`, LTO, one codegen unit and `panic = "abort"`. + +## Language boundaries + +Rust is the default for new application logic and native tooling. Pure shared behavior belongs in `allthing-core`; the interactive app, data layer, Worker and SpacetimeDB module keep their separate Rust targets. Build, filesystem, validation and deployment policy belongs in `xtask`. + +The JavaScript early-card renderer (`web/paint.js`) is an explicit measured exception: the owner kept it after it beat the Rust prototype in a [paired cold-load comparison](PERFORMANCE.md#measured-exception-the-javascript-first-card-renderer). The small `boot.js` loader and inline data/preload hints also run before WebAssembly can start. Messages retains the official TypeScript SDK adapter and generated bindings; browser tests retain Playwright's official TypeScript runner, browser-side probes, independent protocol oracles and the existing Node static test server (`e2e/pages-server.mjs`). The test server stays after an unresolved pre-mount stall during migration CI, with causality unproven; [PERFORMANCE.md](PERFORMANCE.md#retained-node-static-test-server) records the conservative rollback. Wrangler and esbuild remain platform tools. CSS, SQL, HTML, fixtures and configuration stay in their appropriate formats. Keep exceptions narrow and documented; preserve licensing and measured behavior instead of reclassifying handwritten files to change language statistics. ## Browser-local seen posts @@ -240,7 +246,7 @@ Allthing counts Farcaster's daily active accounts as [SnapMeter](https://github. - An emoji is 1.6 em tall in the text, 1.4 em in a one-line excerpt (a search suggestion, the composer's), which clips what leaves its line, and 48 px when a cast is one to six of them and nothing else. - Each is a `` whose `prefers-reduced-motion: reduce` source is the still. Its alt text is the shortcode, so copied text keeps it, and what Pepe does is read out as its description (`aria-description`), not shown in a native tooltip. Each loads lazily in cast text, and at once in an excerpt, which is in view as it shows. - `/assets/` files are cached for a year as immutable, so a changed image takes a new folder (`v2/`). - - `web/emoji/CREDITS.md` names each one's source and maker. A unit test keeps the files, the table and the credits in step, reading each size from the WebP header; another keeps `emoji::MAKERS` the same as the credits, for the picker, which names the maker of the Pepe pointed at or focused beside its shortcode. `node scripts/check-emoji-sources.mjs` reads every source, emoji and stickers, and checks that it is still there under the terms its credit gives, by the maker it names (7TV's and FrankerFaceZ's APIs, emoji.gg's page); it only reads, and exits 1 naming any that isn't. + - `web/emoji/CREDITS.md` names each one's source and maker. A unit test keeps the files, the table and the credits in step, reading each size from the WebP header; another keeps `emoji::MAKERS` the same as the credits, for the picker, which names the maker of the Pepe pointed at or focused beside its shortcode. `cargo xtask check-credits` reads every source, emoji and stickers, and checks that it is still there under the terms its credit gives, by the maker it names (7TV's and FrankerFaceZ's APIs, emoji.gg's page); it only reads, and exits 1 naming any that isn't. - **Stickers.** 32 of the animated emoji are stickers too (`allthing_core::emoji::STICKERS`): 7TV's 128 px GIF of each, unchanged, under `/assets/stickers/v1/`, with a still of its fullest frame, and a moving tile for the picker (`.tile.webp`, an animated WebP of the same size made from the GIF, lossy at quality 40 or lossless, whichever is smaller: about a third of its bytes). - A cast carries a sticker as an image embed of its GIF at `https://allthing.social/assets/stickers/v1/.gif`, so every Farcaster app shows it. GIF is what they all play. - Allthing knows its own stickers by that address (`emoji::sticker_of`). `MediaGrid` shows each one apart from the cast's photos: 128 px tall with no frame, loaded from the site itself, and as its still with reduced motion. Its alt text is "Sticker: Pepe …". @@ -458,15 +464,15 @@ Notifications and counts come from the same node (Hypersnap), and a like or repl ## Messages on SpacetimeDB The public `#general` room and end-to-end encrypted conversations (Messages) live in a SpacetimeDB database, `allthing` on maincloud, off Farcaster: nothing is written to Snapchain. The module is in `spacetime/spacetimedb` (its handoff, schema and reducers in `spacetime/HANDOFF.md`), checked by CI's `SpacetimeDB module` job. - **Who is who.** A browser connects with its own SpacetimeDB identity. After Sign In With Farcaster, it asks the Worker for an attestation (`POST /api/v1/auth/attest`): the Ed25519 signature, by the `ATTEST_KEY` secret, of its identity, the signed-in account and an expiry 10 minutes on (`allthing_core::stdb`, `crates/allthing-worker/src/attest.rs`). The module's `verify_fid` checks it against the public half pinned in `attest_key.rs` (ed25519-dalek, strict), against the caller's own identity, so an attestation is no use to anyone else, and marks the identity's FID verified. Once a key is pinned, only verified identities can publish a key, chat, create rooms or send messages. The key is the Worker's own; it is never the custody key, and the Worker never holds a reader's key. -- **The client.** `/messages`, from the account menu, is one inbox (`pages/messages`): a list of conversations, Allthing's public `#general` room first, then the reader's encrypted conversations, newest first; the one open beside it (`?room=general`, `?to=`), or New message (`?new=1`), which finds a person by name with the site's search. On a phone the list and the open conversation take turns, with a back button. Enter sends, Shift+Enter starts a new line. It asks for no API key and calls no Farcaster Direct Cast API: Farcaster's own Direct Casts stay in the Farcaster app. The page is a split part of its own (`split_realtime.wasm`, `pages/messages/realtime`), and it loads the SpacetimeDB TypeScript SDK as one bundled module script, `assets/stdb-.js` (`web/stdb/bridge.ts`, `scripts/build-stdb.mjs`), for a signed-in reader only. The Rust SDK builds for browsers but is BSL-1.1, which the license audit refuses. No other page fetches either. Names and pictures come from Hypersnap's `user/bulk`. The browser keeps its SpacetimeDB token and an X25519 key pair (WebCrypto, unexportable) in IndexedDB `allthing-stdb`, one record per account. After connecting it calls `register`, then `verify_fid` with the Worker's attestation, then `set_public_key`. Signed out, the page asks the reader to sign in; the static copy points to allthing.social. Tested by `e2e/realtime-messages.fixture.spec.ts` at 1280 and 390, with a stand-in database (axe included). -- **Content Security Policy.** SpacetimeDB 2.13.0 builds product and sum codecs with `Function`, which the site’s existing policy rejects. `web/stdb/csp-codec.ts`, imported before the generated bindings, replaces only those public codec factories with cached closures; SDK primitive and array codecs and BSATN wire encoding stay in use. `scripts/build-stdb.mjs` requires a deliberate adapter review when the SDK version changes. `tests/stdb-codecs.test.mjs` compares wire bytes and decoded values with the unmodified SDK, including recursive types and inherited field names. `e2e/stdb-csp.fixture.spec.ts` runs the actual bundled SDK under the production policy against a BSATN WebSocket peer and mock Worker attestation: connection, rows, online count, verification and sending must work without a CSP violation. The mock attestation does not replace the release check with a real signed-in account and the live module. No `unsafe-eval` permission is added. +- **The client.** `/messages`, from the account menu, is one inbox (`pages/messages`): a list of conversations, Allthing's public `#general` room first, then the reader's encrypted conversations, newest first; the one open beside it (`?room=general`, `?to=`), or New message (`?new=1`), which finds a person by name with the site's search. On a phone the list and the open conversation take turns, with a back button. Enter sends, Shift+Enter starts a new line. It asks for no API key and calls no Farcaster Direct Cast API: Farcaster's own Direct Casts stay in the Farcaster app. The page is a split part of its own (`split_realtime.wasm`, `pages/messages/realtime`), and it loads the SpacetimeDB TypeScript SDK as one bundled module script, `assets/stdb-.js` (`web/stdb/bridge.ts`, `xtask/src/browser_assets.rs`), for a signed-in reader only. The Rust SDK builds for browsers but is BSL-1.1, which the license audit refuses. No other page fetches either. Names and pictures come from Hypersnap's `user/bulk`. The browser keeps its SpacetimeDB token and an X25519 key pair (WebCrypto, unexportable) in IndexedDB `allthing-stdb`, one record per account. After connecting it calls `register`, then `verify_fid` with the Worker's attestation, then `set_public_key`. Signed out, the page asks the reader to sign in; the static copy points to allthing.social. Tested by `e2e/realtime-messages.fixture.spec.ts` at 1280 and 390, with a stand-in database (axe included). +- **Content Security Policy.** SpacetimeDB 2.13.0 builds product and sum codecs with `Function`, which the site’s existing policy rejects. `web/stdb/csp-codec.ts`, imported before the generated bindings, replaces only those public codec factories with cached closures; SDK primitive and array codecs and BSATN wire encoding stay in use. `xtask/src/browser_assets.rs` requires a deliberate adapter review when the SDK version changes. `tests/stdb-codecs.test.mjs` compares wire bytes and decoded values with the unmodified SDK, including recursive types and inherited field names. `e2e/stdb-csp.fixture.spec.ts` runs the actual bundled SDK under the production policy against a BSATN WebSocket peer and mock Worker attestation: connection, rows, online count, verification and sending must work without a CSP violation. The mock attestation does not replace the release check with a real signed-in account and the live module. No `unsafe-eval` permission is added. - **Encryption** is in `allthing_core::e2e`: X25519 (WebCrypto), then HKDF-SHA256, then XChaCha20-Poly1305, with associated data that binds both parties, their key versions and their keys. The explainer (`components/encryption.rs`, text in `e2e::messages_explainer`) is reusable by later encrypted features. ## Allthing's poll The front page asks its readers one question of Allthing's own: "Would you like an Allthing mobile client?", Yes or No (`allthing_core::poll`, the poll `mobile-client-v1`). Anyone can vote, signed in or not. -- **Where it shows.** On the front page only (hot, new and top, and every time range), as one card in two places (`components/poll.rs`), labelled "Allthing poll". From 1280 px wide, it sits in the gutter right of the feed, placed beside the feed's column, which doesn't move, and stays in view under the header as the feed scrolls (`position: sticky`). Narrower, it is a card after the feed's fourth (`CastList`'s inset), drawn with the cards around it and kept after the fourth as they are sorted again: it comes once the four before it are drawn, it is keyed by the card it follows, so a list drawn anew (another sort, time range or tab) draws it anew too, as its cards are, rather than moving it, and it is laid out as they are (`content-visibility: auto`, its height kept while the front page is kept, `kept.rs`), so a new list never shifts it into view. web/paint.js's first cards don't have it, so on a narrow screen they stop at the fourth, and nothing moves when the app takes over from them. Its body keeps one height (112 px), asking or showing the results, so a vote moves nothing. What fills the body (Yes and No, the results, voting, and reading and keeping the vote) is code of its own (`split_poll.wasm`), fetched once the front page is idle: the card's frame, with its question and its body's box, is drawn with the cards, and the body fills in place. +- **Where it shows.** On the front page only (hot, new and top, and every time range), as one card in two places (`components/poll.rs`), labelled "Allthing poll". From 1280 px wide, it sits in the gutter right of the feed, placed beside the feed's column, which doesn't move, and stays in view under the header as the feed scrolls (`position: sticky`). Narrower, it is a card after the feed's fourth (`CastList`'s inset), drawn with the cards around it and kept after the fourth as they are sorted again: it comes once the four before it are drawn, it is keyed by the card it follows, so a list drawn anew (another sort, time range or tab) draws it anew too, as its cards are, rather than moving it, and it is laid out as they are (`content-visibility: auto`, its height kept while the front page is kept, `kept.rs`), so a new list never shifts it into view. The early renderer's first cards don't have it, so on a narrow screen they stop at the fourth, and nothing moves when the app takes over from them. Its body keeps one height (112 px), asking or showing the results, so a vote moves nothing. What fills the body (Yes and No, the results, voting, and reading and keeping the vote) is code of its own (`split_poll.wasm`), fetched once the front page is idle: the card's frame, with its question and its body's box, is drawn with the cards, and the body fills in place. - **Voting.** Before a vote it shows no numbers. Yes or No shows pressed at once, and both wait while `POST /api/v1/poll` (`{"poll":"mobile-client-v1","choice":"yes"|"no"}`) is answered. Counted, the card shows each answer's share and count ("Yes 63% (5)", in whole percents that add up to 100, by the largest remainder), how many voted, and "You voted Yes.", and a polite live region says it all. A vote that doesn't go through (no answer, a 5xx) gives the buttons back, with why. - **One vote per network.** The Worker counts a vote for the address Cloudflare saw it come from (`CF-Connecting-IP`): an IPv4 address, or an IPv6 address's /64, which one device's addresses rotate within (an IPv4 address written as IPv6 is that IPv4 address). A network that has voted is told so, with the counts, and never which way it voted: "A vote from your network was already counted." People who share an address, behind an office's or a carrier's NAT (common on phones), share one vote, and a reader on two networks can vote twice: it is a show of hands, not a count of people. - **What is kept.** For each network that voted, D1 keeps an HMAC-SHA256 of the poll and the network, keyed with the `SESSION_SECRET` secret (of `allthing poll v1`, a newline and `mobile-client-v1|`), in hex, and its choice (`poll_votes`, from `migrations/0008_poll.sql`), and each answer's count (`poll_counts`). Never the address: it isn't logged, kept or answered with. Without the secret the keys can't be matched to addresses, and with the poll in them, a network's votes in two polls can't be matched to each other. One D1 batch, one transaction, counts the vote and keeps it unless the network has voted, and reads the counts, so a vote counts once and nothing counts the votes again. Changing the secret lets every network vote again. @@ -508,11 +514,12 @@ The header's small "● 12" says how many people are reading Allthing now. Hover - **Unit tests**: `cargo test` covers the core against recorded Hypersnap responses, link shortening in the app, and the xtask's CSS minifier and fingerprinting. - **Protocol test vectors**: `crates/allthing-core/tests/message_vectors.rs` checks the messages a reader signs (`allthing_core::message`) against known Farcaster messages, byte for byte by their BLAKE3 hash, with their Ed25519 signatures verified against their signers: Snapchain's own test vectors (two casts in `farcasterxyz/snapchain`'s `engine_tests.rs`, signed by its test key), the `reactionById` example in Snapchain's HTTP API reference (a like, by its hash), and five messages on mainnet as Hypersnap served them (`tests/fixtures/snapchain-messages.json`, each with its URL): a like, a recast, an unlike, a quote embedding another cast's id, and a deleted cast. These are what Allthing writes for a like, a recast, a quote and their undoing (`ReactionAdd`, `ReactionRemove`, `CastAdd` with a `CastId` embed, `CastRemove`). Apps differ in how they encode a cast (`@farcaster/core` writes empty mention lists that Snapchain's own encoder leaves out), and the network keeps either as signed; the quote is one encoded as Allthing and Snapchain encode it. -- **Lints**: `scripts/check.sh` runs rustfmt and Clippy with `-D warnings` for the native and wasm32 targets, and the `cargo deny` license audit. -- **Tooling**: `npm run test:tools` checks the static test server with temporary fixture directories (malformed paths, root boundaries, redirects and deep-link fallback), and the production deployment wait against a local HTTP server (revision matching, transient failures and bounded timeouts). CI runs it before the browser fixtures. -- **Browser regressions**: `npx playwright test '\.fixture\.spec\.ts$'` runs deterministic browser tests against the local build, with API and media responses supplied by fixtures. CI runs these on pull requests and pushes to `main`; new deterministic suites should use the `.fixture.spec.ts` suffix. They exercise the code under review without depending on live data. Failure reports are uploaded for 7 days. +- **Lints**: `cargo xtask check` runs rustfmt and Clippy with `-D warnings` for the native and wasm32 targets, and the `cargo deny` license audit. +- **Tooling**: `cargo test --locked -p xtask` covers native build/check policy, metadata/CSP/release links, credit terms, attestation handling and the deployment wait (revision matching, local Git ancestry/fetching, transient HTTP errors, bounded stalled bodies and child cleanup). `npm run test:tools` retains the Node static-server oracles, actual SDK/browser interfaces and browser-specific fixture contracts. CI runs native checks in its check job and the remaining Node tests before browser fixtures. +- **SpacetimeDB module**: `cargo xtask module-check` runs in the isolated `spacetime/spacetimedb` crate with workspace `RUSTFLAGS` cleared: rustfmt, native and Wasm Clippy, unit tests, a release Wasm build and the license audit when `cargo-deny` is installed. CI installs it, so the license check remains required there. `spacetime/check.sh` only forwards to that native command. +- **Browser regressions**: `cargo xtask browser-fixtures` runs functional fixtures with two workers, then calibrated smoothness alone with one; both phases run and either failure fails the job. The underlying `npx playwright test '\.fixture\.spec\.ts$'` runner exercises deterministic browser tests against the local build, with API and media responses supplied by fixtures. CI runs these on pull requests and pushes to `main`; new deterministic suites should use the `.fixture.spec.ts` suffix. They exercise the code under review without depending on live data. Failure reports are uploaded for 7 days. - **Static payloads**: CI also builds the `/Allthing` copy and runs `E2E_PAGES=1 npx playwright test e2e/static-body-limit.fixture.spec.ts e2e/static-metadata.fixture.spec.ts e2e/static-navigation.fixture.spec.ts e2e/static-follows.fixture.spec.ts e2e/preloaded-replies.fixture.spec.ts e2e/poll.fixture.spec.ts`. These fixtures exercise the browser Host's capped HTML streams, cleanup after read failures, complete uncapped JSON responses, and linked-page metadata filtering with inert fallback text and actual frame declarations, check that what the app opens itself (a cast's card, a search, a replies count, the time range) stays under `/Allthing`, that follows navigation works under that prefix, that a replies preview built in the browser, by the code a popover batch builds replies with, leaves out spam and mini apps, and that the copy has no poll. -- **Performance**: `e2e/perf.spec.ts` checks the budget in [PERFORMANCE.md](PERFORMANCE.md); CI runs it against production after each push to `main`. +- **Performance**: `e2e/perf.spec.ts` checks the budget in [PERFORMANCE.md](PERFORMANCE.md); CI runs it against production after each push to `main`, after `cargo xtask wait-for-deploy` has checked the intended version without warming Home. - **Writes land, or are taken back**: `e2e/verified-writes.fixture.spec.ts`, at 1280 and 390 px, likes, recasts, quotes and their undoing with the reader's real WebCrypto key against a stand-in node (`e2e/helpers/node.ts`, which every spec that writes uses). The stand-in checks each message as Snapchain would (BLAKE3 of its data is its hash, its Ed25519 signature is by the key the reader approved, mainnet, the reader's fid, the time) and answers the read-backs from what it merged. It covers a write merged and shown; taken but never shown, which is undone after 15 s, with Retry; refused, with the node's reason; an unknown key, which offers the approval; a busy node, then success; a lost answer, the same message again and a duplicate, counted once; each undoing; and a quote shown at once, posted once shown, or gone with its draft back. - **A reply in its thread**: `e2e/reply-in-thread.fixture.spec.ts`, at 1280 and 390 px: a reply at the top of its post's replies the moment Reply is pressed, before the node answers, as posting (faded, its actions inert), then posted, with its time linking to it, and once when the page lists it; a reply to a reply under it; the count on the post and in the feed; refused, or not shown in 15 s, gone with the count back and the composer open again with the draft and why, and Retry sending the same message. - **Smoothness**: `e2e/smoothness.fixture.spec.ts`, one of the browser regressions, times a signed-in reader's taps, navigations, Back and presses at 390 px (touch, the CPU slowed 2×) and at 1280 px against fixture data: INP, long tasks, layout shift, blank frames, where Back puts the reader, and whether a press shows in its first frame ([PERFORMANCE.md](PERFORMANCE.md#smoothness)). diff --git a/docs/BUDGET.md b/docs/BUDGET.md index 7990eb6c..dc1462ad 100644 --- a/docs/BUDGET.md +++ b/docs/BUDGET.md @@ -7,7 +7,7 @@ The relevant Free plan limits, as of October 2026: | Limit | Value | Allthing's design | |---|---|---| -| Worker requests | 100,000/day (resets 00:00 UTC) | `/api/*`, `/version`, `/wasm/*` and `/m/*` run the Worker; everything else is a static asset. `/wasm/*` is the app's WebAssembly, fetched once per browser (it is cached for a year), so it adds one request to a first visit only. `/m/*` is readers' attachments, a request for each a browser doesn't have yet ([§12](#12-attachments-r2)) | +| Worker requests | 100,000/day (resets 00:00 UTC) | `/api/*`, `/version`, `/wasm/*` and `/m/*` run the Worker; everything else is a static asset. `/wasm/*` serves the main app WebAssembly, cached for a year: its cold code load adds one request. `/m/*` is readers' attachments, a request for each a browser doesn't have yet ([§12](#12-attachments-r2)) | | Static asset requests | free and unlimited | The app, styles, fonts, emoji and stickers are all static, and so are the parts of the app fetched when used (the DAU page's dashboard, the composer, React's picker, signing in, the list on /scheduled, writing, the Messages page, and the pages of posts, profiles, channels and search). Other Farcaster apps load a cast's sticker from allthing.social too, as a static asset | | CPU time | 10 ms per invocation | Payloads are parsed once per refresh and cached; cache hits copy bytes | | Subrequests | 50 per invocation (`fetch` **and** Cache API calls) | Hard budget of 45 per invocation, enforced in code | @@ -15,11 +15,11 @@ The relevant Free plan limits, as of October 2026: ## 1. What a page view costs -The browser app is static. A page view runs the Worker for its one API call and nothing else: +The page-payload API counts below are unchanged. Code assets are cached separately; a cold `/wasm/*` load also runs the Worker, as accounted for below. | Page | Worker requests | Free static requests | |---|---|---| -| Front page `/` | 1 (`/api/v1/home`) | index.html, boot.js, app JS + WASM, CSS, font (first visit only: hashed, immutable) | +| Front page `/` | 1 (`/api/v1/home`) | index.html, boot.js, paint.js, app JS glue, CSS, font (first visit only: hashed, immutable); Wasm transport is counted below | | Front page `/`, signed in, **For you** | 1 (`/api/v1/home`, or `/api/v1/latest` in new, as signed out); and, once the page is painted, at most one `/api/v1/following` every 10 minutes per account through `api::prefetch`, to renew whom the reader follows (`followed.rs`), which also makes the Following tab instant. See the note below | as above | | Front page `/`, signed in, **Following** (`?feed=following`) | 1 (`/api/v1/following`, the reader's own, in hot, new and top), plus up to nine older payloads as its end approaches; automatic reads stop at the selected time limit or after ten payloads total. Load more posts explicitly grants ten more. All filtering, including seen rows, shares this cap | as above | | Channel `/c/` | 1 (`/api/v1/channel`) | none after the first visit | @@ -85,7 +85,9 @@ A reader who opens posts they hovered spends nothing extra; idle viewport prefet **Capacity:** about **40,000 page views a day** with a typical mix, including prefetches, popovers and preloaded previews (about 2.5 Worker requests per view). That assumes about half of all views are feeds read with a mouse, each adding one to three batches. Since 21 October 2026 the next run loads a screen ahead of the reader, a batch more for one who stops just short of it, and a page view may load up to eight runs, where it was four; before, one or two batches a feed view gave about 43,000 views a day. Phones preload too (since 20 October 2026): a phone's feed view adds the same one to three batches, and the facts of the authors it stops at, about 4 to 6 more requests in all. If a quarter of all views are phone feeds, capacity falls to about 28,000 page views a day (about 3.6 requests per view). `TOUCH_CARD_ACCOUNTS_PER_WINDOW` in `crates/allthing-web/src/api.rs` and `MAX_BATCHES` trade complete cards on phones for capacity. That is before counting views that cost nothing (sorting, back/forward, revisits). `MAX_BATCHES` in `crates/allthing-web/src/preload.rs` trades preloaded cards for capacity: without preloads it is about 60,000 (about 1.6 requests per view). Each profile card a reader opens adds up to 2 more, and nothing for that account again within 5 minutes. A preview the reader opens adds up to 2 per person it names, the cards then opened from it nothing: 20 for a list of ten, and at most 60 per reader per 5 minutes. If one view in ten opens one, capacity falls to about 23,000 page views a day (about 4.3 requests per view); one in twenty, about 30,000 (about 3.3). `ROW_ACCOUNTS_PER_WINDOW` in `crates/allthing-web/src/api.rs` trades complete cards for capacity: past it, a card reads its own facts when it opens and shows when they come. Since 22 October 2026 the counts in view are kept current (`/api/v1/counts`), about **0.8 more requests per page view** at a typical 40-second view (one tick, sometimes two runs, nothing for views shorter than 30 s), so the typical capacity is about **30,000 page views a day** (about 3.3 requests per view), and about 22,000 with a quarter of views on phone feeds. A reader who keeps reading costs about 2 to 4 a minute (2 runs every 30 s); a tab left alone stops after 2 minutes. `EVERY_MS`, `PER_TICK` and `IDLE_MS` in `crates/allthing-web/src/counts.rs` trade current counts for capacity. Signed-in readers' bells come on top of page views: about 20 requests an hour for each tab left in view, one every 3 minutes (`EVERY` in `crates/allthing-web/src/components/notifications.rs`), so 100 readers with a tab in view for two hours a day cost about 4,000 requests, 4% of the day's allowance. The header's presence beats come on top of this, per reader at once rather than per view: at most 8,640 a day up to about 90 readers at once, which is about 2,600 views at these figures ([§14](#14-presence)). -The production performance CI job checks `/version` before measuring a deployment: one Worker request every 10 seconds until the commit is live, for at most 30 minutes (at most 180 requests per run), with no upstream calls. Workers Builds usually makes a commit live 6 to 16 minutes after the push, so a run usually checks 30 to 90 times. +`cargo xtask wait-for-deploy`, used by the production performance CI job, checks `/version` before measuring a deployment: one Worker request every 10 seconds until the commit is live, for at most 30 minutes (at most 180 requests per run), with no upstream calls. Workers Builds usually makes a commit live 6 to 16 minutes after the push, so a run usually checks 30 to 90 times. + +**Early rendering:** `web/paint.js` reuses the already-started Home response and performs no additional API, D1, cron, upstream or Worker call. Its fingerprinted JavaScript is a free immutable static asset; no independent early-renderer Wasm is shipped. The static copy omits early rendering. The API/prefetch capacity estimates above exclude the main app's cold `/wasm/*` transport request; count that against the same 100,000-request allowance when estimating cold traffic. Native tooling changes no application request budget. **Messages CSP compatibility:** the closure-based SDK codec adapter adds 449 Brotli bytes (2,111 raw bytes) to the Messages-only JavaScript bundle in the measured build. It adds no requests, subscriptions, upstream subrequests or server cache keys; connection and attestation budgets above are unchanged. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 9b82ad5f..a8b6aee8 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -27,18 +27,18 @@ Pushing to `main` then builds and deploys. The first deploy also: ### What the build does -`scripts/build.sh` runs on the Workers Builds image (Ubuntu, 2 vCPUs and a 20-minute limit on the Free plan). The image has no Rust, so the script installs it: +`scripts/build.sh` runs on the Workers Builds image (Ubuntu, 2 vCPUs and a 20-minute limit on the Free plan). The image has no Rust, so the shell wrapper installs it and hands build policy to native Rust: 1. rustup and the toolchain pinned in `rust-toolchain.toml` (Rust 1.99.1 with the `wasm32-unknown-unknown` target); -2. `wasm-bindgen` CLI at the version in `Cargo.lock` (a prebuilt binary from its GitHub release), and `worker-build` (compiled with `cargo install`); +2. `cargo xtask build` selects and installs `wasm-bindgen` CLI at the version in `Cargo.lock` (a prebuilt binary from its GitHub release), and `worker-build` (compiled with `cargo install`); 3. the Worker: `worker-build --release` in `crates/allthing-worker` → `crates/allthing-worker/build/`, then `cargo xtask strip-worker`, which drops from its WebAssembly the relocations the workspace's `--emit-relocs` leaves for the app's wasm-split (`linking` and `reloc.*`, more than half its bytes); -4. the browser app: `cargo xtask dist` → `dist/` (fingerprinted WebAssembly and its brotli-11 copy, written with Node's zlib, JS glue, the parts loaded when used and their loader, CSS, fonts, `index.html`, `_headers`). +4. the browser app: `cargo xtask dist` → `dist/` (fingerprinted app WebAssembly and its Brotli-11 copy written in Rust, generated glue, the app's split parts, the JavaScript loader and early-card renderer, CSS, fonts, `index.html`, `_headers`). Native Rust checks the pinned SpacetimeDB SDK and invokes the installed esbuild CLI to bundle its TypeScript adapter. -A clean build on 2 CPUs takes about 5 minutes, most of it compiling (see "Build timing" below). `npx wrangler deploy` then uploads the Worker and the `dist/` assets. Only changed assets are uploaded. +The earlier clean-build timing below predates the native tooling migration; it is historical evidence, not a timing guarantee for this build. `npx wrangler deploy` then uploads the Worker and the `dist/` assets. Only changed assets are uploaded. ### Build timing -A rehearsal from a fresh clone, on a 2-CPU Linux machine with no Rust and empty caches (as on Workers Builds), took **4 min 51 s**: +An earlier rehearsal, before the native tooling migration, from a fresh clone on a 2-CPU Linux machine with no Rust and empty caches took **4 min 51 s**: | Step | Time | |---|---| @@ -54,6 +54,8 @@ A rehearsal from a fresh clone, on a 2-CPU Linux machine with no Rust and empty 3. **Recommended:** make `www` redirect with a real HTTP 301 at no Worker cost. Go to the zone's **Rules → Redirect Rules**, choose the template **Redirect from WWW to root**, and deploy it. Without it, `www.allthing.social` still redirects: API calls get a 301 from the Worker, and pages are sent to the apex by the first script that runs (`web/boot.js`). 4. Optional protections that stop abusive traffic before it reaches the Worker, so it doesn't count against the 100,000 requests a day: a **WAF rate-limiting rule** on `/api/` and **Bot Fight Mode** (both under the zone's Security settings). +Before production performance measurements, `cargo xtask wait-for-deploy https://allthing.social ` polls only `/version`: every 10 seconds for at most 30 minutes, with a 5-second request deadline. A later live revision is accepted only when local Git ancestry proves it contains the intended commit; unknown commits trigger a bounded read-only fetch of `origin/main`. This does not warm Home or change application request budgets. CI retains its existing 45-minute performance-job cap. + ## Rolling back In the dashboard, **Workers & Pages → allthing → Deployments** lists every version; pick one and choose **Rollback**. A rollback restores the Worker and its static assets together. @@ -73,6 +75,8 @@ E2E_PAGES=1 npx playwright test # check it locally With the redirect in place, `…/Allthing/u/dwr` lands on `https://allthing.social/u/dwr`. +The local static-site test server remains `node e2e/pages-server.mjs [port] [base] [root]` (defaults: `4173`, `/Allthing`, `dist-pages/`). Playwright selects it automatically with `E2E_PAGES=1`. It binds loopback and reproduces the static host's base-path redirect, directory index and `404.html` behavior. Its existing Node oracles remain; the migration CI startup stall and conservative rollback are documented in [PERFORMANCE.md](PERFORMANCE.md#retained-node-static-test-server). + ## Secrets - **`SESSION_SECRET`** (needed for sign-in and the poll): the key that signs sign-in cookies ([ARCHITECTURE.md](ARCHITECTURE.md#sign-in)), and that keys the HMACs [Allthing's poll](ARCHITECTURE.md#allthings-poll) keeps votes under, at least 32 characters, random. Make one with `openssl rand -base64 48` and set it once with `npx wrangler secret put SESSION_SECRET`, or in the Cloudflare dashboard (Workers & Pages → `allthing` → Settings → Variables and Secrets, type *Secret*). It is never committed, logged or sent anywhere. Without it, "Sign in" says sign-in isn't set up and the poll hides itself, and everything else works. Changing it signs everyone out, and lets every network vote in the poll again. For local development, put a different random value in a `.dev.vars` file next to `wrangler.toml` (git-ignored) as `SESSION_SECRET=…`. `wrangler dev` presents requests to the Worker at the route's host, so a sign-in started locally names `allthing.social` (over http). @@ -85,9 +89,9 @@ With the redirect in place, `…/Allthing/u/dwr` lands on `https://allthing.soci - **Taking it back.** Delete the secret: writes go to the Farcaster app again at once, and keys readers already approved keep working until they remove Allthing in the Farcaster app. If it may have leaked, move the account to a new custody address straight away. - **`ATTEST_KEY`** (for Messages on SpacetimeDB): the Ed25519 private key, 64 hex digits, with which the Worker attests that a signed-in reader's SpacetimeDB identity is their Farcaster account (`POST /api/v1/auth/attest`; [ARCHITECTURE.md](ARCHITECTURE.md#messages-on-spacetimedb)). The module's `verify_fid` checks the attestation against the key's public half, pinned in `spacetime/spacetimedb/src/attest_key.rs`. It is a key of its own, used for nothing else: never `ALLTHING_CUSTODY_KEY` or a reader's key. - **What it can do.** Make the module believe any SpacetimeDB identity is any Farcaster account, for chat and encrypted messages. It can't read anyone's messages, which only readers' browsers can decrypt, or touch a Farcaster account. - - **Making and setting it.** From the repository's root: `node scripts/attest-key.mjs | npx wrangler secret put ATTEST_KEY`. The script makes a new key pair, hands the private half straight to Wrangler (it refuses to print it to a terminal), and writes the public half into `attest_key.rs` and on the screen. Commit that file, then republish the module (`cd spacetime && spacetime publish -s maincloud allthing`; see `spacetime/HANDOFF.md`). Open `/api/v1/status`: `spacetime.attest` is `true` and `spacetime.public_key` is what the script printed. Any other key means the module and the Worker disagree, and no one can verify. The key itself never shows. Never put it in the repository, a chat, an issue or a log. + - **Making and setting it.** From the repository's root: `cargo xtask attest-key | npx wrangler secret put ATTEST_KEY`. The native command makes a new key pair, hands the private half straight to Wrangler (it refuses to print it to a terminal), and writes the public half into `attest_key.rs` and on the screen. Commit that file, then republish the module (`cd spacetime && spacetime publish -s maincloud allthing`; see `spacetime/HANDOFF.md`). Open `/api/v1/status`: `spacetime.attest` is `true` and `spacetime.public_key` is the public half printed on stderr. Any other key means the module and the Worker disagree, and no one can verify. The key itself never shows. Never put it in the repository, a chat, an issue or a log. - **Until it is set**, `spacetime.attest` is `false`, the attestation endpoint answers 404, and the module, with its all-zero placeholder key, refuses `verify_fid` and lets unverified FIDs speak as before, which Messages says. - - **Changing it.** Run the script again and republish: attestations signed with the old key stop verifying, and identities already verified stay verified. + - **Changing it.** Run the native command again and republish: attestations signed with the old key stop verifying, and identities already verified stay verified. - **`ALLTHING_FID`** (in `wrangler.toml`, public): the account Allthing asks for readers' keys as, 3356318, @allthingapp. A fork sets its own. For the account's picture, Allthing serves its logo as a 512×512 PNG at `https://allthing.social/pfp.png` (`web/pfp.png`), since Farcaster apps show an SVG picture as blank. The URL is cached for a year (`web/_headers`), so a new logo gets a new file name. - **`HYPERSNAP_ORIGIN`**: never set it. Only the end-to-end tests do, to put a stand-in on their own machine in the place of Hypersnap's node; the Worker ignores anything but a local port (`http://127.0.0.1:`). - **`ALLTHING_SPONSOR`** (optional): `1` to have the account pay the onchain fee for readers' approvals, which the Farcaster app takes from it in warps (it needs at least 100). Without it, any fee is the reader's. Set it as a *Secret* too: a deploy drops plain variables set only in the dashboard (Wrangler's `keep_vars` is off). diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 6f7b6282..a52788ba 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -21,7 +21,7 @@ The post-opening check also retains `post-open-diagnostics.json` for each attemp ## Smoothness -CI runs functional fixture files with two workers, then runs `smoothness.fixture.spec.ts` alone with one worker (`scripts/test-browser-fixtures.sh`). Both phases run even if the first fails, and either failure fails the existing browser job. Their HTML reports and test results are kept separately, including from the later static checks; the production performance job, test retries, timing assertions and 45-minute job cap are unchanged. +CI runs functional fixture files with two workers, then runs `smoothness.fixture.spec.ts` alone with one worker (`cargo xtask browser-fixtures`). Both phases run even if the first fails, and either failure fails the existing browser job. Their HTML reports and test results are kept separately, including from the later static checks; the production performance job, test retries, timing assertions and 45-minute job cap are unchanged. How the app answers a reader's taps, clicks and scrolls: `e2e/smoothness.fixture.spec.ts`, which CI's browser job runs on every pull request and push to `main`, on the local build with the Worker's API, the node and the images as stand-ins (the API answers after 60 to 180 ms). A signed-in reader with an approved key, on a phone (390 × 844, touch, the CPU slowed 2×) and on a computer (1280 × 900, a mouse): @@ -136,15 +136,31 @@ At 2× what the app answers in time stays under the budget even on a busy machin - **A page's data is requested by a one-line inline script** at the top of `index.html`, so it starts while the HTML is still being parsed, alongside the WebAssembly download. `api.rs` (`early_response`) takes that pending response instead of fetching again. - It covers the pages people arrive on: the front page (its sort and time range), `/p/`, `/u/` and `/c/`. It normalizes the hash, name or id as the app does (lowercase; an `0x` or `@` prefix), so the paths match; anything it doesn't recognize, such as a name with other characters, is left to the app. - On production the post, profile and channel requests used to start only once the app had booted, about 450 ms in (the front page's always started at once). The response is still the page's one API call. - - The script is allowed by its hash in the CSP (`web/_headers`), and `scripts/check.sh` fails if the two disagree. + - The script is allowed by its hash in the CSP (`web/_headers`), and `cargo xtask check-repo` fails if the two disagree. - Starting the request from `boot.js` was tried first, but a module script has to download before it runs, so the request still began about when the app's own JS arrived (700–990 ms on production). - The static copy, served under `/Allthing/`, skips it. -- **The front page's first cards are drawn from that data before the app has come** (`web/paint.js`). On a slow line the data comes seconds before the WebAssembly (on the speed report's phone, at about 1.0 s against 3.1 s), so a small script draws the cards the app draws first: in the app's order (hot or top, with the reader's favorites and those they see less of), with the app's markup and the usage bar above them, under the boot screen's header. They only show: they're inert and hidden from screen readers. Once mounted, the app is drawn hidden under them, and when its own cards are drawn, in the same boxes, they take their place (`boot.rs`), without their entrance animation again. The [speed report](#speed-report) times both: the first cast on screen, and the app's own first. - - Only the front page, for hot and top over the past hour, day or week, and only from a fresh answer (not one the Worker marks stale). At most the first 6 cards, and never past one the app draws otherwise: a quote, a video, a sticker, or a single image it can't size (one not on Cloudflare Images, such as a Tenor or KLIPY GIF). +- **The front page's first cards are drawn from that data before the app has come** (`web/paint.js`). On a slow line the data comes seconds before the WebAssembly (on the historical speed report's phone, at about 1.0 s against 3.1 s), so a small script draws the cards the app draws first: in the app's order (hot or top, with the reader's favorites and those they see less of), with the app's markup and the usage bar above them, under the boot screen's header. They only show: they're inert and hidden from screen readers. Once mounted, the app is drawn hidden under them, and when its own cards are drawn, in the same boxes, they take their place (`boot.rs`), without their entrance animation again. The [speed report](#speed-report) times both: the first cast on screen, and the app's own first. + - Only the front page, for hot and top over the past hour, 6 hours, 12 hours, day or week, and only from a fresh answer (not one the Worker marks stale). At most six cards at 1280 px and above, or four below it, where the app inserts its poll. Quotes use the app's depth limit; a video, sticker or single image it cannot size stops the prefix. Following, new, client filters, long ranges and browsers with local seen history remain with the app. - A single image's tile has its final box before the image comes. Cloudflare Images describes the variant a feed shows (`…/fit=scale-down,w=720,format=json`, about 110 bytes, sent with `access-control-allow-origin: *`), the tile takes that size, and so does the app's tile for the same image (`media::known_size`). A card with one is drawn once its description has come, a round trip after the data. Nothing on the first screen moves as its images come, there or after the app takes over; without this, a single image grew its card when it loaded, pushing the cards below it down. - Its images (avatars, emoji, link thumbnails and media) wait until the WebAssembly is in, and the app's own cards then find them on their way. Asked for at once, in a trial on the speed report's phone, they shared the slow line with it and the app's first card came 1.4 s later. - If the app can't load, `boot.js`'s panel replaces the cards. Leaving the front page, or its data failing in the app, hands over at once, and the app's cards replace them at most 4 s after the app has mounted, whatever else. - - It is 38 KB (11 KB at brotli), and adds no request to the Worker. The static copy doesn't load it. + - It adds no request to the Worker. The static copy doesn't load it. Historical JavaScript size and timing measurements below describe their recorded builds. + +#### Measured exception: the JavaScript first-card renderer + +The Rust-first tooling work also tried an independent Rust Wasm first-card renderer. The owner chose to keep `web/paint.js` after the JavaScript version was faster in a paired cold local comparison. Three alternating rounds per profile used the same synthetic 40-cast Home response and Brotli delivery; medians, rounded to milliseconds: + +| Measurement | Retained JavaScript renderer | Rust Wasm prototype | +|---|---:|---:| +| Desktop first card (1280 px, 10 Mbps, 40 ms RTT, native CPU) | 215 ms | 376 ms | +| Phone first card (390 px, 1.6 Mbps, 150 ms RTT, 4× CPU slowdown) | 944 ms | 2,027 ms | +| Phone app's own first cards visible after handover | 4,009 ms | 4,540 ms | + +The JavaScript baseline used preserved pre-migration local assets; it was not independently rebuilt from an exact source commit. These are local prototype results, not production release evidence or a general Rust-versus-JavaScript benchmark. The Rust prototype needed an additional Wasm/glue download; the retained renderer does not. This supports keeping this measured exception, not a claim that native Rust tooling improves browser startup. Preserve the existing early-renderer regression tests and require matched performance evidence before revisiting it. + +#### Retained Node static test server + +The small `e2e/pages-server.mjs` and its three existing Node oracles remain. In [migration CI 40232102342](https://github.com/ael-dev3/Allthing/actions/runs/40232102342), the first static poll fixture at 1280 px stalled on the boot screen before the app mounted, then passed on retry. That first attempt remains rejected. The pending resource was not recorded, so the cause was not established. Keeping the existing test server is a conservative rollback while startup diagnostics improve, with no claim that Rust caused the stall or that the rollback improves performance. The existing assertions and their 25-second expectation deadline remain. ### 2. Prefetch, under a strict budget @@ -224,8 +240,8 @@ Who liked and who recasted a feed card, and its replies, show the moment their p - **Server time**: every API answer says how long the Worker spent on it in a `Server-Timing` header, named after how it was served (`fresh;dur=3`, or `miss;dur=1840` for a build the request waited for). Browsers show it in their network panel and in `PerformanceResourceTiming.serverTiming`. Clocks in a Worker only advance across I/O, so it measures waiting on the cache and upstream, not CPU time. - **Compact JSON**: payloads skip empty and default fields (`skip_serializing_if`). Cloudflare compresses responses with Brotli for clients that accept it. - **Immutable assets**: the app's `.wasm`, `.js` and `.css` are fingerprinted and served with `Cache-Control: public, max-age=31536000, immutable` (`web/_headers`). -- **The WebAssembly at brotli's highest level**: the edge compresses static assets on the fly at a fast level, which left the 1.5 MB module at about 550 KB. The build also writes a copy compressed at quality 11 (`.wasm.br`, about 434 KB), and the page loads the module from `/wasm/`, which the Worker answers with that copy (`Content-Encoding: br`, passed through as it is) to browsers that accept brotli, and with the module itself to the rest. That is a fifth less to download before anything renders. If the Worker can't answer, `boot.js` loads `/assets/`, the static copy. -- **The Worker's WebAssembly without the relocations**: the workspace links all its WebAssembly with `--emit-relocs` (`.cargo/config.toml`), which the browser app's wasm-split reads. The Worker's module carried those sections too, `linking` (2.5 MB) and `reloc.CODE` (0.4 MB), more than half its bytes: nothing reads them there, and once worker-build's wasm-opt has rewritten the code they no longer describe it. `cargo xtask strip-worker`, which `scripts/build.sh` runs after worker-build, drops them and keeps every other byte. Measured on main at 74bc5ca (21 October 2026): +- **The WebAssembly at brotli's highest level**: the native Rust build writes optional `.wasm.br` copies at quality 11, window 24, the same settings previously used by Node's zlib. `/wasm/` serves those bytes with `Content-Encoding: br` to browsers that accept Brotli, and the original module otherwise. If that route fails, the loaders fall back to `/assets/`. In an earlier build, fast edge compression left a 1.5 MB app module at about 550 KB, compared with about 434 KB at quality 11. Those sizes describe that historical build, not a measured improvement from changing compression implementations. +- **The Worker's WebAssembly without the relocations**: the workspace links all its WebAssembly with `--emit-relocs` (`.cargo/config.toml`), which the browser app's wasm-split reads. The Worker's module carried those sections too, `linking` (2.5 MB) and `reloc.CODE` (0.4 MB), more than half its bytes: nothing reads them there, and once worker-build's wasm-opt has rewritten the code they no longer describe it. `cargo xtask strip-worker`, which the native `cargo xtask build` runs after worker-build, drops them and keeps every other byte. Measured on main at 74bc5ca (21 October 2026): - `wrangler deploy --dry-run`: 2,568.39 KiB and 794.12 KiB gzip, where it was 5,442.53 KiB and 1,403.01 KiB, against the Free plan's 3 MB compressed limit; - compiling the module in Node 22's V8: 8.3 ms of CPU where it took 11.3 (medians of 15); - the first request to `/api/v1/status` under `wrangler dev`, which loads the Worker: 822 ms where it took 1,123 ms (medians of 5, alternating, on a busy machine). diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 88a1ae3c..3d184dfd 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -9,8 +9,8 @@ How Allthing is versioned, how changes reach `main`, and how a release is cut. ` - **MINOR** (`0.3.0` → `0.4.0`) for new features or anything a reader would notice as a change; - **PATCH** (`0.3.0` → `0.3.1`) for fixes only. - `1.0.0` will mean the API paths and URLs are stable. -- **One source of truth**: `version` under `[workspace.package]` in the root `Cargo.toml`. Every crate inherits it (`version.workspace = true`). `package.json` repeats it for npm tooling, and `scripts/check.sh` fails if the two differ. -- **Where it shows**: the site footer (`v0.3.0 (abc1234)`), the Worker's `/version` endpoint (`{"name":"allthing","version":"0.3.0","commit":"abc1234"}`), and the Worker's upstream `User-Agent`. The commit is stamped at build time from `WORKERS_CI_COMMIT_SHA` (Workers Builds) or `git rev-parse` (`scripts/build.sh`). +- **One source of truth**: `version` under `[workspace.package]` in the root `Cargo.toml`. Every crate inherits it (`version.workspace = true`). `package.json` repeats it for npm tooling, and `cargo xtask check-repo` (also run by `cargo xtask check`) fails if the two differ. +- **Where it shows**: the site footer (`v0.3.0 (abc1234)`), the Worker's `/version` endpoint (`{"name":"allthing","version":"0.3.0","commit":"abc1234"}`), and the Worker's upstream `User-Agent`. The commit is stamped at build time from `WORKERS_CI_COMMIT_SHA` (Workers Builds) or `git rev-parse` (the native `cargo xtask build` invoked by `scripts/build.sh`). - Releases are not named v1, v2, v3. A release is its version number, and its notes say what changed. ## Commits @@ -42,7 +42,7 @@ Scopes are optional, for example `feat(profile): …`, `fix(worker): …` or `pe 2. On a branch `chore/release-X.Y.Z`: - set `version` in `Cargo.toml` (workspace) and `package.json`, and run `cargo check` so `Cargo.lock` follows; - in `CHANGELOG.md`, move `[Unreleased]` items under `## [X.Y.Z] - YYYY-MM-DD` and update the compare links at the bottom; - - run `bash scripts/check.sh` and the end-to-end suite (`npx playwright test`). + - run `cargo xtask check`, `cargo xtask module-check` for the isolated SpacetimeDB module, and the end-to-end suite (`npx playwright test`). 3. Open the PR `chore(release): X.Y.Z`, wait for green CI, and squash-merge it. 4. Tag the merge commit on `main` and push the tag: ```sh @@ -54,7 +54,7 @@ Scopes are optional, for example `feat(profile): …`, `fix(worker): …` or `pe ```sh gh release create vX.Y.Z --title "Allthing X.Y.Z" --notes-file <(awk '/^## \[X.Y.Z\]/{f=1;next} /^## \[/{f=0} f' CHANGELOG.md) ``` -6. Workers Builds deploys the merge. Check that `https://allthing.social/version` reports `X.Y.Z` and the merge commit. Then run `E2E_BASE_URL=https://allthing.social E2E_EXPECT_WRITES_READY=true npx playwright test`. The readiness contract defaults to `false` for local development without a custody secret; deployments that endorse keys set it explicitly to `true` (`true` and `false` are the only accepted values). +6. Workers Builds deploys the merge. `cargo xtask wait-for-deploy https://allthing.social ` waits for its version stamp, accepting a later deployment only when local Git ancestry proves it includes that commit. It requests only `/version`, with no Home/API warming. Check that `https://allthing.social/version` reports `X.Y.Z` and the merge commit. Then run `E2E_BASE_URL=https://allthing.social E2E_EXPECT_WRITES_READY=true npx playwright test`. The readiness contract defaults to `false` for local development without a custody secret; deployments that endorse keys set it explicitly to `true` (`true` and `false` are the only accepted values). ## Hotfixes diff --git a/e2e/poll.fixture.spec.ts b/e2e/poll.fixture.spec.ts index a4b11ec8..37a8a7ae 100644 --- a/e2e/poll.fixture.spec.ts +++ b/e2e/poll.fixture.spec.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import axe from "axe-core"; -import { expect, test, type Page } from "@playwright/test"; +import { expect, test, type Page, type Request } from "@playwright/test"; /* * Allthing's poll on the front page: "Would you like an Allthing mobile @@ -391,24 +391,98 @@ test.describe("the poll", () => { }); for (const width of [1280, 390]) { - test(`the static copy, which has no Worker to vote with, has no poll, at ${width} px`, async ({ page }) => { + test(`the static copy, which has no Worker to vote with, has no poll, at ${width} px`, async ({ page }, info) => { test.skip(!process.env.E2E_PAGES, "the static copy, under /Allthing"); - await page.setViewportSize({ width, height: 844 }); - // Its trending casts, from Hypersnap in the browser: more than four. - await page.route("https://haatz.quilibrium.com/**", (route) => { - const url = new URL(route.request().url()); - const casts = url.pathname.endsWith("/feed/trending") && !url.searchParams.has("cursor") - ? pool.slice(0, 8).map((cast) => ({ - hash: cast.hash, author: cast.author, text: cast.text, timestamp: new Date(cast.ts * 1000).toISOString(), - embeds: [], replies: { count: 0 }, reactions: { likes_count: cast.likes, recasts_count: 0 }, - })) - : null; - return casts - ? route.fulfill({ json: { casts }, headers: { "access-control-allow-origin": "*" } }) - : route.fulfill({ status: 404, json: {}, headers: { "access-control-allow-origin": "*" } }); + // Keep transport/startup evidence if this fails before the app mounts. + // Never retain bodies, console arguments, cookies or URL queries. + const started = Date.now(); + const safeURL = (raw: string) => { + try { + const url = new URL(raw); + return /^https?:$/.test(url.protocol) ? `${url.origin}${url.pathname}`.slice(0, 240) : url.protocol; + } catch { return "unparseable URL"; } + }; + const categories = (message: string) => ["wasm", "compile", "instantiate", "import", "fetch", "network", "cors", "csp", "mime", "panic", "unreachable", "memory", "timeout", "abort"].filter(word => message.toLowerCase().includes(word)); + const events: object[] = []; + let dropped = 0; + const record = (event: object) => { + if (events.length === 64) { events.shift(); dropped++; } + events.push({ at: Date.now() - started, ...event }); + }; + const ids = new WeakMap(); + let nextId = 0; + const id = (request: Request) => { + if (!ids.has(request)) ids.set(request, ++nextId); + return ids.get(request)!; + }; + const pending = new Map(); + let pendingTotal = 0; + page.on("request", request => { + const entry = { id: id(request), method: request.method(), url: safeURL(request.url()) }; + pendingTotal++; + if (pending.size < 32) pending.set(request, entry); + record({ kind: "request", ...entry }); }); - await page.goto("/Allthing/"); - await expect(page.locator("#app .cast-card:not(.skeleton)").nth(5)).toBeVisible(); - await expect(page.locator(".poll-card, .poll-gutter")).toHaveCount(0); + page.on("response", response => { + const headers = response.headers(); + record({ kind: "response", id: id(response.request()), status: response.status(), contentType: headers["content-type"]?.slice(0, 100), contentLength: /^\d+$/.test(headers["content-length"] ?? "") ? Number(headers["content-length"]) : undefined }); + }); + page.on("requestfinished", request => { + pendingTotal--; + pending.delete(request); + const timing = request.timing(); + record({ kind: "finished", id: id(request), requestStart: timing.requestStart, responseStart: timing.responseStart, responseEnd: timing.responseEnd }); + }); + page.on("requestfailed", request => { + pendingTotal--; + pending.delete(request); + record({ kind: "failed", id: id(request), code: request.failure()?.errorText.match(/net::ERR_[A-Z_]+/)?.[0] ?? "unclassified network failure" }); + }); + page.on("pageerror", error => record({ kind: "pageerror", name: ["Error", "TypeError", "ReferenceError", "SyntaxError", "RangeError", "CompileError", "LinkError", "RuntimeError"].includes(error.name) ? error.name : "Error", categories: categories(error.message), scripts: [...(error.stack ?? "").split("\n").filter(line => /^\s*at /.test(line)).join("\n").matchAll(/https?:\/\/[^\s)]+/g)].slice(0, 4).map(match => safeURL(match[0])) })); + page.on("console", message => { + const location = message.location(); + record({ kind: "console", type: message.type(), categories: categories(message.text()), url: safeURL(location.url), line: location.lineNumber, column: location.columnNumber }); + }); + try { + await page.setViewportSize({ width, height: 844 }); + // Its trending casts, from Hypersnap in the browser: more than four. + await page.route("https://haatz.quilibrium.com/**", (route) => { + const url = new URL(route.request().url()); + const casts = url.pathname.endsWith("/feed/trending") && !url.searchParams.has("cursor") + ? pool.slice(0, 8).map((cast) => ({ + hash: cast.hash, author: cast.author, text: cast.text, timestamp: new Date(cast.ts * 1000).toISOString(), + embeds: [], replies: { count: 0 }, reactions: { likes_count: cast.likes, recasts_count: 0 }, + })) + : null; + return casts + ? route.fulfill({ json: { casts }, headers: { "access-control-allow-origin": "*" } }) + : route.fulfill({ status: 404, json: {}, headers: { "access-control-allow-origin": "*" } }); + }); + await page.goto("/Allthing/"); + await expect(page.locator("#app .cast-card:not(.skeleton)").nth(5)).toBeVisible(); + await expect(page.locator(".poll-card, .poll-gutter")).toHaveCount(0); + } catch (error) { + // Diagnostic failure must not replace the original assertion failure. + try { + const state = await Promise.race([page.evaluate(() => { + const safe = (raw: string) => { + try { const url = new URL(raw); return /^https?:$/.test(url.protocol) ? `${url.origin}${url.pathname}`.slice(0, 240) : url.protocol; } + catch { return "unparseable URL"; } + }; + const boot = document.getElementById("boot"), app = document.getElementById("app"); + return { + url: safe(location.href), readyState: document.readyState, visibility: document.visibilityState, online: navigator.onLine, + boot: { present: Boolean(boot), painted: boot?.hasAttribute("data-painted"), hidden: boot?.getAttribute("aria-hidden") }, + app: { present: Boolean(app), children: app?.childElementCount, cards: app?.querySelectorAll(".cast-card:not(.skeleton)").length }, + resources: (performance.getEntriesByType("resource") as PerformanceResourceTiming[]).slice(-48).map(entry => ({ + url: safe(entry.name), initiator: entry.initiatorType, start: entry.startTime, duration: entry.duration, + responseStart: entry.responseStart, responseEnd: entry.responseEnd, transferSize: entry.transferSize, encodedBodySize: entry.encodedBodySize, + })), + }; + }).catch(() => ({ unavailable: true })), new Promise(resolve => setTimeout(() => resolve({ diagnosticTimeout: true }), 1000))]); + await info.attach("static-startup-diagnostics", { contentType: "application/json", body: Buffer.from(JSON.stringify({ elapsed: Date.now() - started, dropped, events, pendingTotal, pendingEntriesTruncated: pendingTotal > pending.size, pending: [...pending.values()], state })) }); + } catch { /* The original failure remains authoritative. */ } + throw error; + } }); } diff --git a/scripts/attest-key.mjs b/scripts/attest-key.mjs deleted file mode 100644 index 5e6f07aa..00000000 --- a/scripts/attest-key.mjs +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env node -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// Makes Allthing's FID attestation key (Ed25519) for SpacetimeDB messages -// (docs/DEPLOY.md#secrets, ATTEST_KEY). Run it from the repository's root, -// piping it straight into Wrangler, so the private key goes only to -// Cloudflare and is never shown, saved or pasted: -// -// node scripts/attest-key.mjs | npx wrangler secret put ATTEST_KEY -// -// It writes the public half into the module -// (spacetime/spacetimedb/src/attest_key.rs) and says it on stderr; commit -// that file and republish the module. Running it again makes a new key: -// the old one stops verifying once the module is republished with the new. -import { generateKeyPairSync } from "node:crypto"; -import { readFileSync, writeFileSync } from "node:fs"; - -// Refuse before making anything if the key would land on a screen. -if (process.stdout.isTTY) { - process.stderr.write("Not printing the private key to a terminal: pipe this into `npx wrangler secret put ATTEST_KEY`.\n"); - process.exit(1); -} - -const FILE = "spacetime/spacetimedb/src/attest_key.rs"; -const { publicKey, privateKey } = generateKeyPairSync("ed25519"); -const seed = Buffer.from(privateKey.export({ format: "jwk" }).d, "base64url"); -const pub = Buffer.from(publicKey.export({ format: "jwk" }).x, "base64url"); -if (seed.length !== 32 || pub.length !== 32) throw new Error("unexpected Ed25519 key length"); - -const source = readFileSync(FILE, "utf8"); -const bytes = [...pub].map((b) => `0x${b.toString(16).padStart(2, "0")}`).join(", "); -const next = source.replace(/pub const ATTEST_PUBLIC_KEY: \[u8; 32\] = \[[^\]]*\];/, `pub const ATTEST_PUBLIC_KEY: [u8; 32] = [${bytes}];`); -if (next === source && !source.includes(bytes)) throw new Error(`no ATTEST_PUBLIC_KEY in ${FILE}`); -writeFileSync(FILE, next); - -process.stderr.write(`ATTEST_KEY public half: ${pub.toString("hex")}\nwritten to ${FILE}; run cargo fmt in spacetime/spacetimedb, commit it and republish the module.\n`); -process.stdout.write(seed.toString("hex")); diff --git a/scripts/build-pages.sh b/scripts/build-pages.sh index e9130bf4..466835e4 100644 --- a/scripts/build-pages.sh +++ b/scripts/build-pages.sh @@ -5,14 +5,11 @@ # same Rust app, compiled with the `static-host` feature so the browser builds # each page's payload itself (there is no Worker behind a static host). # Run scripts/build.sh first: it installs the toolchain and wasm-bindgen. +# Rust owns optimizer discovery and all build logic. # # bash scripts/build-pages.sh [base] # base path, default /Allthing set -euo pipefail cd "$(dirname "$0")/.." -export PATH="$HOME/.cargo/bin:$PWD/target/tools/bin:$PATH" -if [ -z "${WASM_OPT_BIN:-}" ]; then - WASM_OPT_BIN="$(bash scripts/find-wasm-opt.sh)" -fi -export WASM_OPT_BIN -cargo xtask pages "${1:-/Allthing}" +export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:$PWD/target/tools/bin:$PATH" +exec cargo xtask pages "${1:-/Allthing}" diff --git a/scripts/build-stdb.mjs b/scripts/build-stdb.mjs deleted file mode 100644 index 61a0bda4..00000000 --- a/scripts/build-stdb.mjs +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env node -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// Bundles web/stdb/bridge.ts, the SpacetimeDB TypeScript SDK and the -// module's generated bindings into one minified ES module for the browser, -// which `cargo xtask dist` fingerprints into dist/assets/stdb-.js. -// The app adds it only when a signed-in reader opens /messages. -// -// node scripts/build-stdb.mjs -import { build } from "esbuild"; -import { readFile } from "node:fs/promises"; - -// The CSP codec adapter is verified against this SDK's BSATN factories. -// Fail clearly on upgrades, rather than silently restoring runtime codegen. -const sdk = JSON.parse(await readFile(new URL("../package.json", import.meta.resolve("spacetimedb")), "utf8")); -if (sdk.version !== "2.13.0") throw new Error(`Review web/stdb/csp-codec.ts before upgrading SpacetimeDB ${sdk.version}`); - -const out = process.argv[2]; -if (!out) { - console.error("usage: node scripts/build-stdb.mjs "); - process.exit(2); -} -await build({ - entryPoints: [new URL("../web/stdb/bridge.ts", import.meta.url).pathname], - outfile: out, - bundle: true, - format: "esm", - platform: "browser", - target: ["es2022"], - minify: true, - legalComments: "eof", - logLevel: "warning", -}); diff --git a/scripts/build.sh b/scripts/build.sh index 1b058f09..61ebbf72 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,77 +1,14 @@ #!/usr/bin/env bash # SPDX-License-Identifier: AGPL-3.0-or-later -# -# Builds everything Cloudflare serves for Allthing: -# - crates/allthing-worker/build/ the Worker (Rust, via worker-build) -# - dist/ static assets: the WebAssembly app, styles -# and fonts -# -# Cloudflare Workers Builds runs this as the build command (its image has no -# Rust, so this installs it); `npx wrangler deploy` then uploads both. It -# works the same on a laptop. -# -# bash scripts/build.sh +# Cloudflare Workers Builds starts without Rust. Bootstrap the pinned +# toolchain, then let the native xtask install tools and build both targets. set -euo pipefail cd "$(dirname "$0")/.." -log() { printf '\n==> %s\n' "$*" >&2; } -version_of() { awk -v name="$1" '$0 == "name = \"" name "\"" { getline; gsub(/version = |"/, ""); print; exit }' Cargo.lock; } - -# 1. Rust, at the version and with the wasm32 target pinned in rust-toolchain.toml. -export PATH="$HOME/.cargo/bin:$PATH" +export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:$PATH" if ! command -v rustup >/dev/null 2>&1; then - log "Installing rustup" - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none + curl --proto '=https' --proto-redir '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none fi rustup toolchain install >&2 cargo --version >&2 - -TOOLS="$PWD/target/tools" -mkdir -p "$TOOLS/bin" -export PATH="$TOOLS/bin:$PATH" -platform="$(uname -s)-$(uname -m)" - -# 2. wasm-bindgen CLI matching the wasm-bindgen crate in Cargo.lock. -WBG_VERSION="$(version_of wasm-bindgen)" -if ! wasm-bindgen --version 2>/dev/null | grep -q " $WBG_VERSION\$"; then - log "Installing wasm-bindgen-cli $WBG_VERSION" - release="wasm-bindgen-$WBG_VERSION-x86_64-unknown-linux-musl" - if [ "$platform" = "Linux-x86_64" ] && - curl -fsSL "https://github.com/wasm-bindgen/wasm-bindgen/releases/download/$WBG_VERSION/$release.tar.gz" | tar -xz -C "$TOOLS"; then - cp "$TOOLS/$release/wasm-bindgen" "$TOOLS/bin/" - else - cargo install wasm-bindgen-cli --version "$WBG_VERSION" --locked --root "$TOOLS" - fi -fi -WASM_BINDGEN_BIN="$(command -v wasm-bindgen)" -export WASM_BINDGEN_BIN - -# 3. worker-build, matching the worker crate. -WB_VERSION="$(version_of worker)" -if ! worker-build --version 2>/dev/null | grep -q "^$WB_VERSION\$"; then - log "Installing worker-build $WB_VERSION" - cargo install worker-build --version "$WB_VERSION" --locked --root "$TOOLS" -fi - -# Version stamp: the workspace version is in Cargo.toml; the commit comes -# from Workers Builds or the local checkout. -ALLTHING_COMMIT="$(printf '%s' "${WORKERS_CI_COMMIT_SHA:-$(git rev-parse HEAD 2>/dev/null || echo dev)}" | cut -c1-7)" -export ALLTHING_COMMIT - -# 4. The Worker. worker-build also fetches esbuild and wasm-opt. The -# workspace links all its WebAssembly with --emit-relocs, for the app's -# wasm-split (.cargo/config.toml); the Worker's module needs none of the -# sections that leaves, which would be more than half the upload. -log "Building the Worker" -(cd crates/allthing-worker && worker-build --release) -cargo xtask strip-worker - -# 5. The browser app, optimized with the wasm-opt worker-build fetched. -if [ -z "${WASM_OPT_BIN:-}" ]; then - WASM_OPT_BIN="$(bash scripts/find-wasm-opt.sh)" -fi -export WASM_OPT_BIN -log "Building the browser app" -cargo xtask dist - -log "Built dist/ and crates/allthing-worker/build/" +exec cargo xtask build "$@" diff --git a/scripts/check-emoji-sources.mjs b/scripts/check-emoji-sources.mjs deleted file mode 100644 index 85688a25..00000000 --- a/scripts/check-emoji-sources.mjs +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -// Checks that every Pepe credited in web/emoji/CREDITS.md and -// web/stickers/CREDITS.md is still where its credit says, still shared as -// the credit's terms say, and by whom it says: -// -// node scripts/check-emoji-sources.mjs -// -// - 7TV: the emote is listed, with no flags, and its owner is the maker. -// - FrankerFaceZ: the emote is public, not hidden, approved, and its owner -// is the maker. -// - emoji.gg: the emoji's page is there and still under "Do whatever you -// want with this emoji, have fun!". -// -// It only reads, one source at a time, and exits 1 if any isn't so, naming -// it, so it can be taken out (see the credits' last paragraph). - -import { readFileSync } from "node:fs"; - -const FILES = ["web/emoji/CREDITS.md", "web/stickers/CREDITS.md"]; -const FREE = "Do whatever you want with this emoji, have fun!"; -const AGENT = "allthing-credits-check (+https://github.com/ael-dev3/Allthing)"; - -/** Each credit's maker and source, by source: one emote can be both an emoji and a sticker. */ -function credits() { - const found = new Map(); - for (const file of FILES) { - for (const line of readFileSync(file, "utf8").split("\n")) { - const cells = line.split("|").map((c) => c.trim()); - const source = cells.find((c) => c.startsWith("https://")); - if (!line.startsWith("| `") || !source) continue; - const code = cells[1].replaceAll("`", "").replaceAll(":", ""); - const by = cells[cells.indexOf(source) - 1]; - const seen = found.get(source); - if (seen && seen.by !== by) throw new Error(`${source} is credited to ${seen.by} and to ${by}`); - found.set(source, { by, codes: [...(seen?.codes ?? []), code] }); - } - } - return found; -} - -const same = (a, b) => a.normalize("NFKC").toLowerCase() === b.normalize("NFKC").toLowerCase(); - -async function get(url, json = true) { - const res = await fetch(url, { headers: { "user-agent": AGENT, accept: json ? "application/json" : "text/html" } }); - if (!res.ok) return { status: res.status }; - return { status: res.status, body: json ? await res.json() : await res.text() }; -} - -/** Why `source` isn't as its credit says, or nothing. */ -async function check(source, by) { - const seventv = source.match(/^https:\/\/7tv\.app\/emotes\/([0-9A-Z]+)$/); - const ffz = source.match(/^https:\/\/www\.frankerfacez\.com\/emoticon\/(\d+)-/); - if (seventv) { - const { status, body } = await get(`https://7tv.io/v3/emotes/${seventv[1]}`); - if (!body) return `7TV answers ${status}`; - if (!body.listed) return "no longer listed on 7TV"; - if (body.flags) return `flagged on 7TV (${body.flags})`; - const owner = body.owner ?? {}; - // A maker who has deleted their account: 7TV keeps the emote, with no - // owner, and its credit says so, as 7TV did. - const gone = !body.owner && by === "*DeletedUser"; - if (!gone && ![owner.username, owner.display_name].some((name) => name && same(name, by))) return `7TV names ${owner.username ?? "no one"} as its owner`; - } else if (ffz) { - const { status, body } = await get(`https://api.frankerfacez.com/v1/emote/${ffz[1]}`); - const emote = body?.emote; - if (!emote) return `FrankerFaceZ answers ${status}`; - if (!emote.public || emote.hidden || emote.status !== 1) return "no longer public and approved on FrankerFaceZ"; - const owner = emote.owner ?? {}; - if (![owner.name, owner.display_name].some((name) => name && same(name, by))) return `FrankerFaceZ names ${owner.name ?? "no one"} as its owner`; - } else if (source.startsWith("https://emoji.gg/emoji/")) { - const { status, body } = await get(source, false); - if (!body) return `emoji.gg answers ${status}`; - if (!body.includes(FREE)) return "no longer under emoji.gg's free license"; - } else { - return "a source this doesn't know"; - } - return null; -} - -const all = credits(); -const wrong = []; -for (const [source, { by, codes }] of all) { - const why = await check(source, by).catch((error) => `couldn't be read (${error.message})`); - if (why) wrong.push(`${codes.join(", ")}: ${why} (${source})`); -} -console.log(`${all.size - wrong.length} of ${all.size} sources are as credited.`); -for (const line of wrong) console.log(` ${line}`); -process.exit(wrong.length ? 1 : 0); diff --git a/scripts/check.sh b/scripts/check.sh index 62fff280..c1346c0f 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -1,46 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: AGPL-3.0-or-later -# Checks run locally and in CI: formatting, lints for the native and -# WebAssembly targets, unit tests, and the dependency license audit. +# Compatibility entry point; repository checks and validation run in Rust. set -eu cd "$(dirname "$0")/.." - -# One version: the workspace Cargo.toml; package.json must agree. -cargo_version="$(awk -F'"' '/^\[workspace.package\]/ {w=1} w && /^version/ {print $2; exit}' Cargo.toml)" -npm_version="$(awk -F'"' '/"version"/ {print $4; exit}' package.json)" -if [ "$cargo_version" != "$npm_version" ]; then - echo "version mismatch: Cargo.toml $cargo_version, package.json $npm_version" >&2 - exit 1 -fi - -# Every inline ' web/index.html | grep -v ' src=' | while IFS= read -r tag; do - body="${tag#*>}" - body="${body%}" - hash="sha256-$(printf '%s' "$body" | openssl dgst -sha256 -binary | openssl base64 -A)" - if ! grep -qF "'$hash'" web/_headers; then - echo "web/_headers lacks the CSP hash of an inline script in web/index.html ($hash)" >&2 - exit 1 - fi -done - -# Every released version in the changelog links to its tag (a compare link, -# or the first release's page), not to a commit. -for version in $(sed -nE 's/^## \[([0-9]+\.[0-9]+\.[0-9]+)\].*/\1/p' CHANGELOG.md); do - v=$(printf '%s' "$version" | sed 's/\./\\./g') - if ! grep -qE "^\[$v\]: https://github\.com/ael-dev3/Allthing/(compare/v[0-9.]+\.\.\.v$v|releases/tag/v$v)$" CHANGELOG.md; then - echo "CHANGELOG.md: [$version] should link to the v$version tag" >&2 - exit 1 - fi -done - -cargo fmt --all -- --check -cargo clippy --locked -p allthing-core -p xtask --all-targets -- -D warnings -cargo clippy --locked -p allthing-data -p allthing-web -p allthing-worker --target wasm32-unknown-unknown -- -D warnings -cargo clippy --locked -p allthing-web --features static-host --target wasm32-unknown-unknown -- -D warnings -cargo test --locked -p allthing-core -p allthing-data -p allthing-web -p allthing-worker -p xtask -if command -v cargo-deny >/dev/null 2>&1; then - cargo deny --log-level error check licenses -else - echo "cargo-deny is not installed; skipping the license audit (cargo install cargo-deny)" >&2 -fi +exec cargo xtask check "$@" diff --git a/scripts/find-wasm-opt.sh b/scripts/find-wasm-opt.sh deleted file mode 100644 index 790f6847..00000000 --- a/scripts/find-wasm-opt.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-or-later -# Find the optimizer worker-build downloaded. Its dirs_next cache is -# ~/Library/Caches on macOS, even when XDG_CACHE_HOME is set. Optional -# arguments are cache roots to search (also useful for isolated tests). -set -euo pipefail - -if [ -n "${WASM_OPT_BIN:-}" ]; then - printf '%s\n' "$WASM_OPT_BIN" - exit 0 -fi -if command -v wasm-opt >/dev/null 2>&1; then - command -v wasm-opt - exit 0 -fi - -# Match worker-build's Binaryen archive names; don't pick an optimizer -# cached for a different architecture on a shared or migrated machine. -case "$(uname -s)-$(uname -m)" in - Darwin-arm64|Darwin-aarch64) target=arm64-macos ;; - Darwin-x86_64) target=x86_64-macos ;; - Linux-aarch64|Linux-arm64) target=aarch64-linux ;; - Linux-x86_64) target=x86_64-linux ;; - *) exit 0 ;; -esac -if [ "$#" -eq 0 ]; then - set -- "${XDG_CACHE_HOME:-$HOME/.cache}" "$HOME/Library/Caches" "${TMPDIR:-/tmp}" -fi -for cache_root in "$@"; do - for candidate in "$cache_root"/worker-build/wasm-opt-"$target"-*/bin/wasm-opt; do - if [ -x "$candidate" ]; then - printf '%s\n' "$candidate" - exit 0 - fi - done -done diff --git a/scripts/npm-audit.mjs b/scripts/npm-audit.mjs deleted file mode 100644 index d6bfdd04..00000000 --- a/scripts/npm-audit.mjs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -// CI's npm audit: fails on a high or critical advisory, as -// `npm audit --audit-level=high` does, but only warns when the registry's -// audit service can't answer (it has returned 400 and 503 for whole days), -// so an outage there doesn't stop every pull request's browser tests. -import { spawnSync } from "node:child_process"; - -const run = spawnSync("npm", ["audit", "--json", "--audit-level=high"], { encoding: "utf8" }); -let report; -try { - report = JSON.parse(run.stdout); -} catch { - report = null; -} -if (!report || report.error) { - const why = report?.error?.summary ?? report?.error ?? run.stderr.trim().split("\n").pop() ?? "no answer"; - console.log(`::warning title=npm audit unavailable::${typeof why === "string" ? why : JSON.stringify(why)}`); - process.exit(0); -} -const found = report.metadata?.vulnerabilities ?? {}; -const serious = (found.high ?? 0) + (found.critical ?? 0); -console.log(`npm audit: ${JSON.stringify(found)}`); -if (serious > 0) { - spawnSync("npm", ["audit", "--audit-level=high"], { stdio: "inherit" }); - process.exit(1); -} diff --git a/scripts/test-browser-fixtures.sh b/scripts/test-browser-fixtures.sh deleted file mode 100644 index 5d1adce8..00000000 --- a/scripts/test-browser-fixtures.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: AGPL-3.0-or-later -# Keep calibrated smoothness measurements alone on the runner. The other -# fixtures run first with two workers, under their existing assertions. -set -u -cd "$(dirname "$0")/.." || exit 1 - -if [ "$#" -ne 0 ]; then - echo "Usage: sh scripts/test-browser-fixtures.sh" >&2 - exit 2 -fi - -functional=0 -echo "Functional fixtures (2 workers)" -PLAYWRIGHT_HTML_OUTPUT_DIR=playwright-report/functional \ - npx playwright test '^(?!.*[/\\]smoothness\.fixture\.spec\.ts$).*\.fixture\.spec\.ts$' \ - --workers=2 --output=test-results/functional || functional=$? - -smoothness=0 -echo "Smoothness fixtures (1 worker, after functional fixtures finish)" -PLAYWRIGHT_HTML_OUTPUT_DIR=playwright-report/smoothness \ - npx playwright test '[/\\]smoothness\.fixture\.spec\.ts$' \ - --workers=1 --output=test-results/smoothness || smoothness=$? - -# Run both phases, including after an assertion failure or runner error. -if [ "$functional" -ne 0 ] || [ "$smoothness" -ne 0 ]; then - exit 1 -fi diff --git a/scripts/wait-for-deploy.mjs b/scripts/wait-for-deploy.mjs deleted file mode 100644 index 80ce995e..00000000 --- a/scripts/wait-for-deploy.mjs +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -import { execFile } from "node:child_process"; -import { setTimeout as delay } from "node:timers/promises"; -import { pathToFileURL } from "node:url"; -import { promisify } from "node:util"; - -const sha = /^[a-f0-9]{40}$/i; -const deployedSha = /^[a-f0-9]{7,40}$/i; - -// The Worker stamps its commit with seven hex digits; accept a longer stamp too. -export function revisionMatches(actual, expected) { - return typeof actual === "string" && deployedSha.test(actual) && typeof expected === "string" && sha.test(expected) - && expected.toLowerCase().startsWith(actual.toLowerCase()); -} - -/** - * Whether a deployed commit includes `expected`, from the local clone's - * history: Workers Builds skips a commit that a newer push supersedes, so a - * commit may only ever be live as part of a later one. A deployed commit - * the clone doesn't know yet is fetched with `remote`'s `branch` first. - * Settled answers are kept, since a commit's history never changes. - */ -export function gitIncludes({ cwd = process.cwd(), remote = "origin", branch = "main", timeoutMs = 60_000 } = {}) { - const run = promisify(execFile); - const git = (...args) => run("git", args, { cwd, timeout: timeoutMs }).then(() => true, () => false); - const known = sha => git("cat-file", "-e", `${sha}^{commit}`); - const settled = new Map(); - return async (deployed, expected) => { - const key = `${deployed} ${expected}`; - if (settled.has(key)) return settled.get(key); - if (!(await known(deployed)) && !((await git("fetch", "--quiet", remote, branch)) && (await known(deployed)))) { - return false; - } - const answer = await git("merge-base", "--is-ancestor", expected, deployed); - settled.set(key, answer); - return answer; - }; -} - -// Workers Builds takes 6 to 16 minutes from a push to main until the commit -// is live, and about 25 when a second merge queues behind the first. -// `includes(deployed, expected)`, when given, also accepts a later deployed -// commit that contains the expected one (see `gitIncludes`). -export async function waitForDeploy(baseURL, expected, { - timeoutMs = 1_800_000, - pollIntervalMs = 10_000, - requestTimeoutMs = 5000, - includes = null, - log = () => {}, -} = {}) { - if (typeof expected !== "string" || !sha.test(expected)) throw new TypeError("Expected a full 40-digit Git commit"); - for (const value of [timeoutMs, pollIntervalMs, requestTimeoutMs]) { - if (!Number.isFinite(value) || value <= 0) throw new TypeError("Wait durations must be positive and finite"); - } - const url = new URL("/version", baseURL); - if (!["http:", "https:"].includes(url.protocol)) throw new TypeError("Expected an HTTP(S) site URL"); - const deadline = performance.now() + timeoutMs; - let last = "no response"; - while (performance.now() < deadline) { - try { - const response = await fetch(url, { - cache: "no-store", - headers: { accept: "application/json", "cache-control": "no-cache" }, - signal: AbortSignal.timeout(Math.max(1, Math.ceil(Math.min(requestTimeoutMs, deadline - performance.now())))), - }); - if (!response.ok) { - last = `HTTP ${response.status}`; - await response.body?.cancel(); - } else { - const body = await response.json(); - if (revisionMatches(body?.commit, expected)) { - log(`Deployment ${body.commit} is live`); - return body.commit; - } - if (includes && typeof body?.commit === "string" && deployedSha.test(body.commit) - && await includes(body.commit, expected)) { - log(`Deployment ${body.commit} is live, and includes ${expected.slice(0, 7)}`); - return body.commit; - } - last = typeof body?.commit === "string" && deployedSha.test(body.commit) - ? `commit ${body.commit}` : "unrecognized version response"; - } - } catch (error) { - last = error.name === "TimeoutError" || error.name === "AbortError" ? "version request timed out" : "version request failed"; - } - log(`Waiting for deployment ${expected.slice(0, 7)}: ${last}`); - const remaining = deadline - performance.now(); - if (remaining > 0) await delay(Math.min(pollIntervalMs, remaining)); - } - throw new Error(`Deployment ${expected.slice(0, 7)} was not live within ${timeoutMs / 1000}s (${last})`); -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const [, , baseURL, expected] = process.argv; - if (!baseURL || !expected) { - console.error("Usage: node scripts/wait-for-deploy.mjs "); - process.exitCode = 1; - } else { - try { - await waitForDeploy(baseURL, expected, { log: console.log, includes: gitIncludes() }); - } catch (error) { - console.error(error.message); - process.exitCode = 1; - } - } -} diff --git a/spacetime/HANDOFF.md b/spacetime/HANDOFF.md index a1aaf20c..bdacd98d 100644 --- a/spacetime/HANDOFF.md +++ b/spacetime/HANDOFF.md @@ -1,7 +1,7 @@ # Allthing SpacetimeDB module — handoff -Module: `spacetime/spacetimedb` in this repository (Rust, spacetimedb 2.13). Builds with `spacetime build`; `sh spacetime/check.sh` runs its checks (CI's `SpacetimeDB module` job). +Module: `spacetime/spacetimedb` in this repository (Rust, spacetimedb 2.13). Builds with `spacetime build`; from the repository root, `cargo xtask module-check` runs its isolated checks (CI's `SpacetimeDB module` job). `sh spacetime/check.sh` is a thin compatibility wrapper. The native command clears workspace `RUSTFLAGS` and retains rustfmt, native/Wasm Clippy, unit tests, the release Wasm build and the license audit when `cargo-deny` is installed (required by CI). Target DB name: **`allthing`** (fall back to `allthing-chat` if the name is taken) on maincloud. URL once published: `https://maincloud.spacetimedb.com` / ws, database `allthing`. Dashboard: https://spacetimedb.com/allthing STATUS: published to maincloud as `allthing` (checked 21 October 2026: `GET /v1/database/allthing` answers, and its schema has @@ -12,7 +12,7 @@ these tables and reducers, with `general` seeded as room 1). export PATH=$HOME/.local/bin:$HOME/.cargo/bin:$PATH spacetime login # interactive, as ael cd spacetime && spacetime publish -s maincloud allthing -y -spacetime generate --lang typescript --out-dir /src/module_bindings --module-path spacetimedb +spacetime generate --lang typescript --out-dir ../web/stdb/module_bindings --module-path spacetimedb ``` ## Schema @@ -31,23 +31,23 @@ Per-user views: `my_dm_inbox` and `my_dm_outbox` show the caller's DMs only. Tha ## E2E design (the server sees only ciphertext) - The client generates an X25519 keypair. The private key stays in IndexedDB (non-extractable where possible), and only the public key goes to `set_public_key`. - Per message: shared = X25519(my_priv, their_pub), then key = HKDF-SHA256(shared, info="allthing-dm-v1"|sorted identities|key versions). - Encrypt with XChaCha20-Poly1305 (libsodium / @noble/ciphers), a random 24-byte nonce, and AAD = sender|recipient|key versions. + Encrypt with XChaCha20-Poly1305 (`allthing-core::e2e` in Rust), a random 24-byte nonce, and AAD = sender|recipient|key versions. - Both parties can decrypt (static-static DH), so the outbox can be read back. Key versions let clients pick the right key after a rotation. - Future: X3DH/Double Ratchet for forward secrecy, plus multi-device keys (one key row per device). ## TODOs 1. ~~FID verification~~: done. `verify_fid(fid, expiry, signature)` checks the Worker's Ed25519 attestation of - `allthing-spacetimedb-fid-v1|identity_hex|fid|expiry` against the key pinned in `src/attest_key.rs` (all zeros until - `node scripts/attest-key.mjs` sets it; see docs/DEPLOY.md#secrets). Once pinned, only verified identities may speak. + `allthing-spacetimedb-fid-v1|identity_hex|fid|expiry` against the key pinned in `src/attest_key.rs`. `cargo xtask attest-key` generates a replacement only when explicitly rotating it; pipe its private output directly to Wrangler as described in docs/DEPLOY.md#secrets. Only verified identities may speak once the key is pinned. 2. Pruning of expired typing rows (add a scheduled reducer; clients can filter by `expires_at` for now). Message retention/pagination. 3. Rooms are public, with no membership or moderation. Add `room_member` plus visibility views for private rooms. 4. RLS: views are used instead of `client_visibility_filter`. Users can still query the public `user_key`/`user` tables, which is intended. 5. Run wasm-opt for a smaller module. ## Client integration (Allthing web) -1. `npm i spacetimedb`, then generate TS bindings (see above). -2. `DbConnection.builder().withUri('wss://maincloud.spacetimedb.com').withModuleName('allthing').withToken(localStorage token)`. - Persist the token that comes back in onConnect. -3. After Farcaster sign-in, call `register(fid)` (later `verify_fid`), then `set_public_key`. +The UI and encryption logic are Rust. `web/stdb/bridge.ts` is the narrow adapter to the official TypeScript SDK; generated bindings live in `web/stdb/module_bindings`. The Rust SDK is not substituted because its BSL-1.1 license fails the repository's current license policy. `cargo xtask dist` bundles the adapter through the installed esbuild CLI and checks the pinned SDK version; the CSP codec adapter remains covered by wire-format and browser tests. + +1. Use `npm ci` for the pinned SDK and regenerate bindings from the Rust module when its schema changes (see above). +2. The Messages-only adapter connects to `wss://maincloud.spacetimedb.com`, database `allthing`, keeping its per-account token in IndexedDB. +3. After Farcaster sign-in, call `register`, obtain the Worker's identity-bound attestation, call `verify_fid`, then `set_public_key`. 4. Subscribe to `room`, `chat_message WHERE room_id=…`, `typing`, `user`, `user_key`, `my_dm_inbox`, `my_dm_outbox`. 5. Throttle `set_typing` (~every 3 s while typing). DMs are encrypted/decrypted entirely on the client. diff --git a/spacetime/check.sh b/spacetime/check.sh index 7fd519d4..512c31e8 100755 --- a/spacetime/check.sh +++ b/spacetime/check.sh @@ -1,21 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: AGPL-3.0-or-later -# Checks for the SpacetimeDB module (spacetime/spacetimedb), which is its own -# crate outside the workspace: formatting, Clippy, unit tests, a release -# build for wasm32 (what `spacetime publish` uploads) and the license audit. +# Compatibility entry point; Rust validates the isolated SpacetimeDB module. set -eu -cd "$(dirname "$0")/spacetimedb" -# The workspace's wasm32 rustflags (.cargo/config.toml) are for the browser -# app; the module builds without them, as `spacetime build` would elsewhere. -export RUSTFLAGS="" -cargo fmt -- --check -cargo clippy --locked --all-targets -- -D warnings -cargo clippy --locked --target wasm32-unknown-unknown -- -D warnings -cargo test --locked -cargo build --locked --release --target wasm32-unknown-unknown -if command -v cargo-deny >/dev/null 2>&1; then - # cargo-deny finds the repository's deny.toml in a parent directory. - cargo deny --log-level error check licenses -else - echo "cargo-deny is not installed; skipping the license audit" >&2 -fi +cd "$(dirname "$0")/.." +exec cargo xtask module-check "$@" diff --git a/spacetime/spacetimedb/src/attest_key.rs b/spacetime/spacetimedb/src/attest_key.rs index c0794b79..1f43684d 100644 --- a/spacetime/spacetimedb/src/attest_key.rs +++ b/spacetime/spacetimedb/src/attest_key.rs @@ -2,7 +2,7 @@ //! The public half of Allthing's FID attestation key, whose private half is //! the Worker's `ATTEST_KEY` secret (docs/DEPLOY.md#secrets). All zeros //! means not set up: `verify_fid` refuses, and unverified FIDs may still -//! speak, as before. `node scripts/attest-key.mjs` makes a key pair, gives +//! speak, as before. `cargo xtask attest-key` makes a key pair, gives //! the private half to `wrangler secret put ATTEST_KEY` and writes the //! public half here; the module must then be republished. diff --git a/tests/browser-fixture-phases.test.mjs b/tests/browser-fixture-phases.test.mjs deleted file mode 100644 index 37b1d19e..00000000 --- a/tests/browser-fixture-phases.test.mjs +++ /dev/null @@ -1,76 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { test } from "node:test"; -import { fileURLToPath } from "node:url"; - -const script = fileURLToPath(new URL("../scripts/test-browser-fixtures.sh", import.meta.url)); - -async function run(functional, smoothness, args = []) { - const scratch = await mkdtemp(path.join(tmpdir(), "allthing-browser-phases-")); - try { - const log = path.join(scratch, "calls"); - const npx = path.join(scratch, "npx"); - await writeFile(npx, `#!/bin/sh -printf 'phase\nhtml=%s\nci=%s\n' "$PLAYWRIGHT_HTML_OUTPUT_DIR" "$CI" >> "$PHASE_LOG" -printf 'arg=%s\n' "$@" >> "$PHASE_LOG" -case "$*" in - *--workers=2*) exit "$FUNCTIONAL_EXIT" ;; - *--workers=1*) exit "$SMOOTHNESS_EXIT" ;; - *) exit 90 ;; -esac -`); - await chmod(npx, 0o755); - const result = spawnSync("sh", [script, ...args], { - encoding: "utf8", - env: { ...process.env, PATH: `${scratch}:${process.env.PATH}`, CI: "1", PHASE_LOG: log, FUNCTIONAL_EXIT: String(functional), SMOOTHNESS_EXIT: String(smoothness) }, - }); - const calls = await readFile(log, "utf8").catch((error) => { - if (error.code === "ENOENT") return ""; - throw error; - }); - return { ...result, calls: calls.split("phase\n").filter(Boolean).map((call) => call.trim().split("\n")) }; - } finally { - await rm(scratch, { recursive: true, force: true }); - } -} - -test("runs functional then quiet smoothness with separate reports and unchanged CI policy", async () => { - const result = await run(0, 0); - assert.equal(result.status, 0, result.stderr); - assert.equal(result.calls.length, 2); - assert.deepEqual(result.calls.map((call) => call.slice(0, 2)), [ - ["html=playwright-report/functional", "ci=1"], - ["html=playwright-report/smoothness", "ci=1"], - ]); - assert.deepEqual(result.calls.map((call) => call.filter((arg) => arg.startsWith("arg=--"))), [ - ["arg=--workers=2", "arg=--output=test-results/functional"], - ["arg=--workers=1", "arg=--output=test-results/smoothness"], - ]); -}); - -test("still measures smoothness when functional assertions or the runner fail", async () => { - for (const status of [1, 127]) { - const result = await run(status, 0); - assert.equal(result.status, 1); - assert.equal(result.calls.length, 2); - assert.ok(result.calls[1].includes("arg=--workers=1")); - } -}); - -test("either phase failing makes the aggregate fail", async () => { - for (const statuses of [[0, 1], [1, 1]]) { - const result = await run(...statuses); - assert.equal(result.status, 1); - assert.equal(result.calls.length, 2); - } -}); - -test("rejects extra CLI arguments instead of overriding the phase worker counts", async () => { - const result = await run(0, 0, ["--workers=8"]); - assert.equal(result.status, 2); - assert.equal(result.calls.length, 0); -}); diff --git a/tests/find-wasm-opt.test.mjs b/tests/find-wasm-opt.test.mjs deleted file mode 100644 index e6812204..00000000 --- a/tests/find-wasm-opt.test.mjs +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { after, before, test } from "node:test"; -import { fileURLToPath } from "node:url"; - -const script = fileURLToPath(new URL("../scripts/find-wasm-opt.sh", import.meta.url)); -const target = process.platform === "darwin" - ? `${process.arch === "arm64" ? "arm64" : "x86_64"}-macos` - : `${process.arch === "arm64" ? "aarch64" : "x86_64"}-linux`; -let scratch; -before(async () => { scratch = await mkdtemp(path.join(tmpdir(), "allthing-wasm-opt-")); }); -after(async () => { await rm(scratch, { recursive: true, force: true }); }); - -async function binary(file, executable = true) { - await mkdir(path.dirname(file), { recursive: true }); - await writeFile(file, "#!/bin/sh\nexit 0\n"); - if (executable) await chmod(file, 0o755); - return file; -} -function find(roots, env = {}) { - return execFileSync("/bin/bash", [script, ...roots], { - encoding: "utf8", env: { ...process.env, PATH: "/usr/bin:/bin", WASM_OPT_BIN: "", ...env }, - }).trim(); -} - -test("honors an explicit optimizer path, including spaces", () => { - const explicit = path.join(scratch, "chosen optimizer", "wasm-opt"); - assert.equal(find([], { WASM_OPT_BIN: explicit }), explicit); -}); - -test("uses the installed PATH optimizer before a cached one", async () => { - const installed = await binary(path.join(scratch, "tools", "wasm-opt")); - assert.equal(find([scratch], { PATH: `${path.dirname(installed)}:/usr/bin:/bin` }), installed); -}); - -test("finds a platform cache after an empty XDG root and skips other architectures or non-executables", async () => { - const cache = path.join(scratch, "Library", "Caches with spaces"); - await binary(path.join(cache, "worker-build", "wasm-opt-wrong-platform-132", "bin", "wasm-opt")); - await binary(path.join(cache, "worker-build", `wasm-opt-${target}-131`, "bin", "wasm-opt"), false); - const wanted = await binary(path.join(cache, "worker-build", `wasm-opt-${target}-132`, "bin", "wasm-opt")); - assert.equal(find([path.join(scratch, "empty-xdg"), cache]), wanted); -}); - -test("an absent optimizer remains optional", () => { - assert.equal(find([path.join(scratch, "no-cache")]), ""); -}); diff --git a/tests/wait-for-deploy.test.mjs b/tests/wait-for-deploy.test.mjs deleted file mode 100644 index a521c8ff..00000000 --- a/tests/wait-for-deploy.test.mjs +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { createServer } from "node:http"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { test } from "node:test"; -import { gitIncludes, revisionMatches, waitForDeploy } from "../scripts/wait-for-deploy.mjs"; - -const expected = "a1234567890abcdef1234567890abcdef1234567"; -const fast = { timeoutMs: 200, requestTimeoutMs: 40, pollIntervalMs: 10 }; - -async function server(t, handler) { - const app = createServer(handler); - await new Promise(resolve => app.listen(0, "127.0.0.1", resolve)); - t.after(async () => { - app.closeAllConnections(); - await new Promise(resolve => app.close(resolve)); - }); - return `http://127.0.0.1:${app.address().port}`; -} - -test("only a valid deployed revision matching the expected commit is accepted", () => { - assert.equal(expected.length, 40); - for (const actual of [expected.slice(0, 7), expected, expected.toUpperCase()]) { - assert.equal(revisionMatches(actual, expected), true); - } - for (const actual of [undefined, null, 42, "dev", "", "a12345", "g123456", "b123456", `${expected}a`]) { - assert.equal(revisionMatches(actual, expected), false); - } - assert.equal(revisionMatches("a123456", "dev"), false); -}); - -test("waits through the previous release and bypasses cached version responses", async t => { - let reads = 0; - const url = await server(t, (req, res) => { - assert.equal(req.url, "/version"); - assert.equal(req.headers["cache-control"], "no-cache"); - res.setHeader("content-type", "application/json"); - res.end(JSON.stringify({ commit: ++reads === 1 ? "b123456" : expected.slice(0, 7) })); - }); - assert.equal(await waitForDeploy(url, expected, { ...fast, timeoutMs: 1000 }), expected.slice(0, 7)); - assert.equal(reads, 2); -}); - -test("recovers from HTTP errors, malformed JSON, and unstamped builds", async t => { - let reads = 0; - const url = await server(t, (_, res) => { - reads++; - if (reads === 1) { res.writeHead(503); res.end("not ready"); } - else if (reads === 2) res.end("not JSON"); - else res.end(JSON.stringify({ commit: reads === 3 ? "dev" : expected })); - }); - assert.equal(await waitForDeploy(url, expected, { ...fast, timeoutMs: 1000 }), expected); - assert.equal(reads, 4); -}); - -test("fails within the overall deadline if the intended release never arrives", async t => { - let reads = 0; - const url = await server(t, (_, res) => { reads++; res.end(JSON.stringify({ commit: "b123456" })); }); - const started = performance.now(); - await assert.rejects(waitForDeploy(url, expected, fast), /was not live within 0\.2s/); - assert.ok(reads > 0); - assert.ok(performance.now() - started < 1500); -}); - -for (const bodyStarted of [false, true]) { - test(`aborts a stalled ${bodyStarted ? "response body" : "connection"} and respects the overall deadline`, async t => { - let reads = 0; - const url = await server(t, (_, res) => { - reads++; - if (bodyStarted) { res.writeHead(200); res.write('{"commit":'); } - }); - const started = performance.now(); - // Room for several 40 ms requests even when a busy runner is slow to - // make the first: within 0.2s, it sometimes made only one. - await assert.rejects(waitForDeploy(url, expected, { ...fast, timeoutMs: 600 }), /was not live within 0\.6s/); - assert.ok(reads >= 2, "a stalled request must leave room to retry"); - assert.ok(performance.now() - started < 1500); - }); -} - -test("accepts a later deployment that includes the expected commit", async t => { - const asked = []; - const includes = async (deployed, of) => { asked.push([deployed, of]); return deployed === "c123456"; }; - let reads = 0; - const url = await server(t, (_, res) => { - reads++; - // The previous release, an unstamped build, then a later commit that includes the pushed one. - res.end(JSON.stringify({ commit: ["b123456", "dev", "c123456"][Math.min(reads, 3) - 1] })); - }); - assert.equal(await waitForDeploy(url, expected, { ...fast, timeoutMs: 1000, includes }), "c123456"); - assert.deepEqual(asked, [["b123456", expected], ["c123456", expected]], "only valid stamps are looked up"); -}); - -test("a deployment that doesn't include the expected commit is still waited past", async t => { - const url = await server(t, (_, res) => res.end(JSON.stringify({ commit: "b123456" }))); - await assert.rejects(waitForDeploy(url, expected, { ...fast, includes: async () => false }), /was not live within/); -}); - -test("gitIncludes follows the clone's history, fetching commits it doesn't know yet", async t => { - const dir = mkdtempSync(join(tmpdir(), "wait-for-deploy-")); - t.after(() => rmSync(dir, { recursive: true, force: true })); - const git = (cwd, ...args) => - execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "-c", "init.defaultBranch=main", ...args], - { cwd, encoding: "utf8" }).trim(); - const commit = (cwd, name) => { - writeFileSync(join(cwd, name), name); - git(cwd, "add", name); - git(cwd, "commit", "-q", "-m", name); - return git(cwd, "rev-parse", "HEAD"); - }; - const origin = join(dir, "origin.git"); - const work = join(dir, "work"); - const clone = join(dir, "clone"); - git(dir, "init", "-q", "--bare", origin); - git(dir, "init", "-q", work); - git(work, "remote", "add", "origin", origin); - const old = commit(work, "a"); - const pushed = commit(work, "b"); - git(work, "push", "-q", "origin", "HEAD:main"); - git(dir, "clone", "-q", origin, clone); - // Merged after the clone was made: a later commit on main, and one elsewhere. - const later = commit(work, "c"); - git(work, "push", "-q", "origin", "HEAD:main"); - git(work, "checkout", "-q", "-b", "side", old); - const side = commit(work, "d"); - git(work, "push", "-q", "origin", "side"); - - const includes = gitIncludes({ cwd: clone }); - const short = sha => sha.slice(0, 7); - assert.equal(await includes(short(later), pushed), true, "a later commit on main, fetched"); - assert.equal(await includes(short(pushed), pushed), true); - assert.equal(await includes(short(old), pushed), false, "the previous release"); - assert.equal(await includes(short(side), pushed), false, "not on main"); - assert.equal(await includes("0000000", pushed), false, "unknown anywhere"); -}); - -test("rejects an invalid target or duration before requesting the site", async () => { - await assert.rejects(waitForDeploy("http://127.0.0.1:1", "dev"), /full 40-digit Git commit/); - await assert.rejects(waitForDeploy("http://127.0.0.1:1", expected, { timeoutMs: 0 }), /positive and finite/); - await assert.rejects(waitForDeploy("file:///tmp/site", expected), /HTTP\(S\) site URL/); -}); diff --git a/web/emoji/CREDITS.md b/web/emoji/CREDITS.md index 5fdd0ddc..de9b5e36 100644 --- a/web/emoji/CREDITS.md +++ b/web/emoji/CREDITS.md @@ -14,7 +14,7 @@ The images in `v1/` are not covered by Allthing's AGPL license: each belongs to Anything with a stated restriction was left out, among them emoji.gg's default license, which allows only platforms that don't charge for access. Some animations were re-encoded smaller, at most 150 KB, and some at half their frame rate. -Each source was checked again on 21 October 2026 with `node scripts/check-emoji-sources.mjs`. It reads each one and checks that it is still there under the same terms (listed with no flags on 7TV; public, not hidden and approved on FrankerFaceZ; under the free license on emoji.gg), and that the maker credited is still its owner. All 219 are. `:pepe_eat:`'s maker has deleted their 7TV account: 7TV still lists the emote, with no owner, as `*DeletedUser`, and so does its credit. Run the script again before adding to the pack, and take out any Pepe it names. +Each source was checked again on 21 October 2026 with `cargo xtask check-credits`. It reads each one and checks that it is still there under the same terms (listed with no flags on 7TV; public, not hidden and approved on FrankerFaceZ; under the free license on emoji.gg), and that the maker credited is still its owner. All 219 are. `:pepe_eat:`'s maker has deleted their 7TV account: 7TV still lists the emote, with no owner, as `*DeletedUser`, and so does its credit. Run the script again before adding to the pack, and take out any Pepe it names. In Allthing, the picker names each Pepe's maker beside its shortcode when it is pointed at or focused, and its **Credits** link opens this page. `allthing_core::emoji::MAKERS` holds the makers, and a unit test keeps it the same as the table below. diff --git a/web/stdb/bridge.ts b/web/stdb/bridge.ts index 72f4505e..03d1c842 100644 --- a/web/stdb/bridge.ts +++ b/web/stdb/bridge.ts @@ -7,7 +7,7 @@ // browser too, but under the Business Source License, which Allthing's // AGPL and its license audit don't take; the TypeScript SDK is ISC. // -// `scripts/build-stdb.mjs` bundles this into one file, `assets/stdb-.js`, +// The native xtask bundles this through esbuild into one file, `assets/stdb-.js`, // which the app adds as a module script only when a signed-in reader opens // /messages: no other page fetches it. It sets // `window.allthingStdb`, and the app talks to it in JSON strings, so the diff --git a/web/stickers/CREDITS.md b/web/stickers/CREDITS.md index a93e2616..252349de 100644 --- a/web/stickers/CREDITS.md +++ b/web/stickers/CREDITS.md @@ -6,7 +6,7 @@ Allthing's Pepe stickers: 32 of the animated emotes in its emoji pack ([../emoji A sticker goes out as an image embed of its GIF at `https://allthing.social/assets/stickers/v1/`, so every Farcaster app shows it, and Allthing shows it as a sticker. GIF is what every Farcaster app plays. -The images in `v1/` are not covered by Allthing's AGPL license: each belongs to whoever made it, credited below with where it came from. They were chosen as the emoji were: public emotes, listed on 7TV with no flags, with no stated restriction. `node scripts/check-emoji-sources.mjs` checks them with the emoji, and found all 32 as credited on 21 October 2026. The picker names each one's maker when it is pointed at or focused. +The images in `v1/` are not covered by Allthing's AGPL license: each belongs to whoever made it, credited below with where it came from. They were chosen as the emoji were: public emotes, listed on 7TV with no flags, with no stated restriction. `cargo xtask check-credits` checks them with the emoji, and found all 32 as credited on 21 October 2026. The picker names each one's maker when it is pointed at or focused. If you made one of these and would rather it were credited differently, or not here, please open an issue. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index bc30eb96..be9558d6 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later [package] name = "xtask" -description = "Build tasks for Allthing: bundle the browser app and its static copy" +description = "Native build, validation and deployment tooling for Allthing" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -16,3 +16,15 @@ publish = false allthing-core = { path = "../crates/allthing-core" } # Cuts the browser app's #[wasm_split] parts into modules of their own. wasm_split_cli_support = "0.2.4" +# Native build and validation tools; none of these ship to the browser or Worker. +base64 = "0.22.1" +brotli = "8" +ed25519-dalek = "2.2.0" +getrandom = "0.4.3" +libc = "0.2" +serde_json.workspace = true +sha2 = "0.10.9" +toml = "1.1.8" +unicode-normalization = "0.1.25" +url = "2.5.8" +zeroize = "1" diff --git a/xtask/src/browser_assets.rs b/xtask/src/browser_assets.rs new file mode 100644 index 00000000..1686dcf6 --- /dev/null +++ b/xtask/src/browser_assets.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Native compression and browser SDK build orchestration. + +use std::fs; +use std::io::Write; +use std::path::Path; +use std::process::Command; + +use super::{Result, read, run}; + +pub fn build_bridge(root: &Path, out: &Path) -> Result> { + let manifest = root.join("node_modules/spacetimedb/package.json"); + let package: serde_json::Value = + serde_json::from_slice(&read(&manifest)?).map_err(|e| format!("{}: {e}", manifest.display()))?; + // The CSP adapter is verified against this SDK's actual BSATN factories. + // An upgrade must not silently reintroduce its runtime code generation. + if package["version"].as_str() != Some("2.13.0") { + return Err(format!("review web/stdb/csp-codec.ts before upgrading SpacetimeDB {}", package["version"])); + } + // npm may replace esbuild's launcher with its native executable during + // postinstall. Execute it directly; passing that binary to Node disables + // Messages by trying to parse machine code as JavaScript. + run(Command::new(root.join("node_modules/esbuild/bin/esbuild")) + .current_dir(root) + .args([ + "web/stdb/bridge.ts", + "--bundle", + "--format=esm", + "--platform=browser", + "--target=es2022", + "--minify", + "--legal-comments=eof", + "--log-level=warning", + ]) + .arg(format!("--outfile={}", out.display())))?; + read(out) +} + +fn compress(bytes: &[u8]) -> Result> { + let params = + brotli::enc::BrotliEncoderParams { quality: 11, lgwin: 24, size_hint: bytes.len(), ..Default::default() }; + let mut compressed = Vec::new(); + { + let mut writer = brotli::CompressorWriter::with_params(&mut compressed, 64 * 1024, ¶ms); + writer.write_all(bytes).map_err(|e| e.to_string())?; + } + Ok(compressed) +} + +/// Keep the Worker's uncompressed fallback if an optional compressed copy +/// cannot be written. Compression itself no longer needs Node or its zlib. +pub fn brotli(input: &Path, output: &Path) { + let _ = fs::remove_file(output); + let result = read(input).and_then(|bytes| compress(&bytes)).and_then(|bytes| super::write(output, bytes)); + if let Err(why) = result { + let _ = fs::remove_file(output); + eprintln!("xtask: could not compress {}: {why}; the Worker will serve it as is", input.display()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + + #[test] + fn native_brotli_roundtrips_binary_wasm_and_empty_files() { + for bytes in [Vec::new(), [0, 97, 115, 109, 1, 0, 0, 0].to_vec(), (0..=255).cycle().take(128 * 1024).collect()] + { + let compressed = compress(&bytes).unwrap(); + let mut decoded = Vec::new(); + brotli::Decompressor::new(compressed.as_slice(), 64 * 1024).read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, bytes); + } + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 25f3207e..83793efe 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -16,6 +16,9 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode}; +mod browser_assets; +mod tooling; + type Result = std::result::Result; fn main() -> ExitCode { @@ -24,13 +27,23 @@ fn main() -> ExitCode { Some("dist") => site(&Site::workers()), Some("pages") => site(&Site::pages(args.get(1).map_or("/Allthing", String::as_str))), Some("strip-worker") => strip_worker(&args.get(1).map_or_else(|| root().join(WORKER_WASM), PathBuf::from)), - _ => Err("usage: cargo xtask dist | cargo xtask pages [base] | cargo xtask strip-worker [path]".into()), + Some("build") => tooling::build::run(&root(), &args[1..]), + Some("check") => tooling::metadata::check_repo(&root()).and_then(|()| tooling::commands::check(&root(), &args[1..])), + Some("check-repo") if args.len() == 1 => tooling::metadata::check_repo(&root()), + Some("check-credits") if args.len() == 1 => tooling::metadata::check_emoji_sources(&root()), + Some("find-wasm-opt") => tooling::find_wasm_opt::run(&args[1..]), + Some("browser-fixtures") => tooling::commands::browser_fixtures(&root(), &args[1..]), + Some("module-check") => tooling::commands::module_check(&root(), &args[1..]), + Some("npm-audit") => tooling::commands::npm_audit(&root(), &args[1..]), + Some("attest-key") => tooling::attest::run(&root(), &args[1..]), + Some("wait-for-deploy") => tooling::deploy::run(&root(), &args[1..]), + _ => Err("usage: cargo xtask dist | pages [base] | build | check | check-repo | check-credits | find-wasm-opt [cache...] | browser-fixtures | module-check | npm-audit | attest-key | wait-for-deploy | strip-worker [path]".into()), }; match result { Ok(()) => ExitCode::SUCCESS, Err(error) => { eprintln!("xtask: {error}"); - ExitCode::FAILURE + if error.starts_with("usage:") { ExitCode::from(2) } else { ExitCode::FAILURE } } } } @@ -55,20 +68,9 @@ fn available(program: &str) -> bool { Command::new(program).arg("--version").output().is_ok_and(|o| o.status.success()) } -/// Write `input` compressed at brotli's highest level to `output`, with -/// Node's zlib (wrangler needs Node anyway). Without Node no copy is written, -/// and the Worker serves the module as it is. +/// Write the optional native Brotli-11 copy the Worker serves. fn brotli(input: &Path, output: &Path) { - const SCRIPT: &str = "const z = require('zlib'), fs = require('fs'); \ - const bytes = fs.readFileSync(process.argv[1]); \ - fs.writeFileSync(process.argv[2], z.brotliCompressSync(bytes, { params: { \ - [z.constants.BROTLI_PARAM_QUALITY]: 11, [z.constants.BROTLI_PARAM_LGWIN]: 24, \ - [z.constants.BROTLI_PARAM_SIZE_HINT]: bytes.length } }));"; - let _ = fs::remove_file(output); - match Command::new(tool("NODE_BIN", "node")).args(["-e", SCRIPT]).arg(input).arg(output).status() { - Ok(status) if status.success() => {} - _ => eprintln!("xtask: could not compress {} with Node; the Worker will serve it as is", input.display()), - } + browser_assets::brotli(input, output); } /// The Pepe pack as web/paint.js reads it: `{"code": [width, height, @@ -178,9 +180,7 @@ fn site(site: &Site) -> Result<()> { // without them: Messages says so, and the rest of the site is whole. let out = root.join("target/stdb-bridge/bridge.js"); let _ = fs::remove_file(&out); - let built = - run(Command::new(tool("NODE_BIN", "node")).current_dir(&root).arg("scripts/build-stdb.mjs").arg(&out)) - .and_then(|()| read(&out)); + let built = browser_assets::build_bridge(&root, &out); match built { Ok(bytes) => { let name = format!("stdb-{}.js", fingerprint(&bytes)); @@ -211,7 +211,8 @@ fn site(site: &Site) -> Result<()> { "target/web-split/main.wasm", ]))?; let wasm_path = bindgen_out.join("allthing_bg.wasm"); - let wasm_opt = tool("WASM_OPT_BIN", "wasm-opt"); + let wasm_opt = tooling::find_wasm_opt::current(&tooling::find_wasm_opt::default_roots())? + .map_or_else(|| "wasm-opt".to_owned(), |path| path.to_string_lossy().into_owned()); if available(&wasm_opt) { for module in std::iter::once(&wasm_path).chain(&parts) { optimize(&wasm_opt, module)?; diff --git a/xtask/src/tooling/attest.rs b/xtask/src/tooling/attest.rs new file mode 100644 index 00000000..5bbcad31 --- /dev/null +++ b/xtask/src/tooling/attest.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Generate the existing Ed25519 attestation seed for a pipe to Wrangler. +//! Only its public key is written to the repository or diagnostics. + +use ed25519_dalek::SigningKey; +use std::fs::{self, OpenOptions}; +use std::io::{IsTerminal, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use zeroize::{Zeroize, Zeroizing}; + +pub type Result = std::result::Result; +const FILE: &str = "spacetime/spacetimedb/src/attest_key.rs"; +const DECLARATION: &str = "pub const ATTEST_PUBLIC_KEY: [u8; 32] = ["; +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn allowed(args: &[String], terminal: bool) -> Result<()> { + if !args.is_empty() { + return Err("usage: cargo xtask attest-key | npx wrangler secret put ATTEST_KEY".into()); + } + if terminal { + return Err( + "Not printing the private key to a terminal: pipe this into `npx wrangler secret put ATTEST_KEY`.".into() + ); + } + Ok(()) +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(DIGITS[usize::from(byte >> 4)] as char); + out.push(DIGITS[usize::from(byte & 15)] as char); + } + out +} + +fn replace_public(source: &str, public: &[u8; 32]) -> Result { + let mut positions = source.match_indices(DECLARATION); + let start = + positions.next().map(|(start, _)| start).ok_or("no ATTEST_PUBLIC_KEY declaration in attestation module")?; + if positions.next().is_some() { + return Err("more than one ATTEST_PUBLIC_KEY declaration in attestation module".into()); + } + let body = start + DECLARATION.len(); + let end = body + source[body..].find("];").ok_or("unterminated ATTEST_PUBLIC_KEY declaration")? + 2; + let existing = &source[body..end - 2]; + // Refuse to replace expressions or comments; the pinned key is a plain + // array, so a changed template cannot accidentally overwrite other code. + let literals: Vec<&str> = existing.split(',').map(str::trim).filter(|value| !value.is_empty()).collect(); + if literals.len() != 32 + || !literals.iter().all(|value| { + value + .strip_prefix("0x") + .is_some_and(|digits| digits.len() == 2 && digits.bytes().all(|byte| byte.is_ascii_hexdigit())) + }) + { + return Err("ATTEST_PUBLIC_KEY must contain exactly 32 hexadecimal byte literals".into()); + } + let mut next = String::with_capacity(source.len()); + next.push_str(&source[..body]); + next.push('\n'); + for bytes in public.chunks(8) { + next.push_str(" "); + for byte in bytes { + next.push_str(&format!("0x{byte:02x}, ")); + } + next.pop(); + next.push('\n'); + } + next.push_str("];"); + next.push_str(&source[end..]); + Ok(next) +} + +struct Lock(PathBuf); +impl Lock { + fn acquire(path: &Path) -> Result { + let path = path.with_extension("rs.lock"); + OpenOptions::new().create_new(true).write(true).open(&path).map_err(|error| { + format!("could not lock attestation key source (another generator may be running): {error}") + })?; + Ok(Self(path)) + } +} +impl Drop for Lock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +struct Pending(PathBuf); +impl Drop for Pending { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +fn write_public(path: &Path, original: &str, next: &str) -> Result<()> { + let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?; + if !metadata.is_file() { + return Err("attestation key source must be a regular file".into()); + } + let pending_path = + path.with_extension(format!("rs.{}-{}.tmp", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); + let mut file = + OpenOptions::new().create_new(true).write(true).open(&pending_path).map_err(|error| error.to_string())?; + let pending = Pending(pending_path); + file.write_all(next.as_bytes()).map_err(|error| error.to_string())?; + file.set_permissions(metadata.permissions()).map_err(|error| error.to_string())?; + file.sync_all().map_err(|error| error.to_string())?; + if fs::read_to_string(path).map_err(|error| error.to_string())? != original { + return Err("attestation key source changed during generation; no private key was printed".into()); + } + fs::rename(&pending.0, path).map_err(|error| error.to_string()) +} + +pub fn run(root: &Path, args: &[String]) -> Result<()> { + // This guard precedes reading, locking, generating or changing anything. + allowed(args, std::io::stdout().is_terminal())?; + let path = root.join(FILE); + let _lock = Lock::acquire(&path)?; + let original = fs::read_to_string(&path).map_err(|error| format!("{}: {error}", path.display()))?; + // Validate the template before requesting any entropy. + replace_public(&original, &[0; 32])?; + let mut seed = Zeroizing::new([0; 32]); + getrandom::fill(seed.as_mut()).map_err(|_| "could not read operating-system randomness")?; + let key = SigningKey::from_bytes(&seed); + let public = key.verifying_key().to_bytes(); + let next = replace_public(&original, &public)?; + write_public(&path, &original, &next)?; + eprintln!( + "ATTEST_KEY public half: {}\nwritten to {FILE}; run cargo fmt in spacetime/spacetimedb, commit it and republish the module.", + hex(&public) + ); + let mut secret = Zeroizing::new(hex(seed.as_ref())); + let output = std::io::stdout().lock().write_all(secret.as_bytes()); + seed.zeroize(); + secret.zeroize(); + output.map_err(|_| { + "public key was updated but the private key could not be delivered to the pipe; generate again".into() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn template() -> String { + format!("// before\n{DECLARATION}{}];\n// after\n", "0x00, ".repeat(32)) + } + + #[test] + fn terminal_and_extra_arguments_are_refused_before_any_work() { + assert!(allowed(&[], true).unwrap_err().contains("Not printing")); + assert!(allowed(&["--show-secret".into()], false).is_err()); + assert!(allowed(&[], false).is_ok()); + } + + #[test] + fn ed25519_seed_to_public_key_matches_rfc8032_test_vector_one() { + let seed = [ + 0x9d, 0x61, 0xb1, 0x9d, 0xef, 0xfd, 0x5a, 0x60, 0xba, 0x84, 0x4a, 0xf4, 0x92, 0xec, 0x2c, 0xc4, 0x44, 0x49, + 0xc5, 0x69, 0x7b, 0x32, 0x69, 0x19, 0x70, 0x3b, 0xac, 0x03, 0x1c, 0xae, 0x7f, 0x60, + ]; + let key = SigningKey::from_bytes(&seed); + assert_eq!( + hex(&key.verifying_key().to_bytes()), + "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a" + ); + } + + #[test] + fn only_public_array_changes_and_repeated_public_pin_is_idempotent() { + let original = template(); + let next = replace_public(&original, &[0x7f; 32]).unwrap(); + assert!(next.starts_with("// before\n")); + assert!(next.ends_with("// after\n")); + assert_eq!(next.matches("0x7f").count(), 32); + assert_eq!(replace_public(&next, &[0x7f; 32]).unwrap(), next); + assert!(!next.contains("PRIVATE")); + } + + #[test] + fn unexpected_templates_are_not_overwritten() { + for source in [ + "nothing".into(), + format!("{DECLARATION}0x00];"), + format!("{}{}", template(), template()), + format!("{DECLARATION}make_key()];"), + ] { + assert!(replace_public(&source, &[0; 32]).is_err()); + } + } + + #[test] + fn public_source_update_is_atomic_and_detects_concurrent_changes() { + let root = std::env::temp_dir().join(format!( + "allthing-attest-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&root).unwrap(); + let path = root.join("attest_key.rs"); + let original = template(); + fs::write(&path, &original).unwrap(); + let lock = Lock::acquire(&path).unwrap(); + assert!(Lock::acquire(&path).is_err()); + let next = replace_public(&original, &[7; 32]).unwrap(); + write_public(&path, &original, &next).unwrap(); + assert_eq!(fs::read_to_string(&path).unwrap(), next); + assert!(write_public(&path, &original, "must not overwrite").is_err()); + assert_eq!(fs::read_to_string(&path).unwrap(), next); + drop(lock); + assert!(!path.with_extension("rs.lock").exists()); + assert_eq!(fs::read_dir(&root).unwrap().count(), 1); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/xtask/src/tooling/build.rs b/xtask/src/tooling/build.rs new file mode 100644 index 00000000..7bb5c992 --- /dev/null +++ b/xtask/src/tooling/build.rs @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! The native part of the Cloudflare build, after the tiny rustup bootstrap. +//! Version selection, optimizer discovery and command ordering have one owner. + +use super::{commands, find_wasm_opt}; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub type Result = std::result::Result; + +/// Read Cargo.lock as TOML rather than relying on adjacent lines. A build +/// must not silently choose an arbitrary version if a CLI's crate occurs +/// at more than one version. +fn locked_version(lock: &str, name: &str) -> Result { + let lock: toml::Value = toml::from_str(lock).map_err(|error| format!("Cargo.lock: {error}"))?; + let packages = lock.get("package").and_then(toml::Value::as_array).ok_or("Cargo.lock has no packages")?; + let mut versions: Vec<&str> = packages + .iter() + .filter(|package| package.get("name").and_then(toml::Value::as_str) == Some(name)) + .filter_map(|package| package.get("version").and_then(toml::Value::as_str)) + .collect(); + versions.sort_unstable(); + versions.dedup(); + match versions.as_slice() { + [version] + if !version.is_empty() + && version.bytes().all(|byte| byte.is_ascii_alphanumeric() || b".-+".contains(&byte)) => + { + Ok((*version).to_owned()) + } + [] => Err(format!("Cargo.lock has no version for {name}")), + _ => Err(format!("Cargo.lock needs one unambiguous version for {name}")), + } +} + +fn version_matches(output: &[u8], version: &str, named: bool) -> bool { + let output = String::from_utf8_lossy(output); + let text = output.trim(); + if named { text == format!("wasm-bindgen {version}") } else { text == version } +} + +fn matching_tool(name: &str, version: &str, path: &OsStr, named: bool) -> Option { + let binary = commands::tool_on_path(name, path).ok()?; + let output = Command::new(&binary).arg("--version").env("PATH", path).output().ok()?; + (output.status.success() && version_matches(&output.stdout, version, named)).then_some(binary) +} + +fn commit_stamp(ci: Option<&str>, git: Option<&str>) -> Result { + let value = ci.filter(|value| !value.is_empty()).or_else(|| git.filter(|value| !value.is_empty())).unwrap_or("dev"); + if value == "dev" { + return Ok(value.to_owned()); + } + if !(7..=40).contains(&value.len()) || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("build commit must be a 7–40 digit Git revision".into()); + } + Ok(value[..7].to_owned()) +} + +fn stamp(root: &Path) -> Result { + let ci = std::env::var("WORKERS_CI_COMMIT_SHA").ok(); + let git = Command::new("git") + .current_dir(root) + .args(["rev-parse", "HEAD"]) + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned()); + commit_stamp(ci.as_deref(), git.as_deref()) +} + +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Scratch(PathBuf); +impl Scratch { + fn new(tools: &Path) -> Result { + for _ in 0..100 { + let path = + tools.join(format!(".wasm-bindgen-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); + match fs::create_dir(&path) { + Ok(()) => return Ok(Self(path)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("{}: {error}", path.display())), + } + } + Err("could not allocate a build-tool download directory".into()) + } +} +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// Use the same upstream Linux-musl release as before. Only the named +/// binary is streamed out of the tar, never arbitrary archive paths. Both +/// download and redirection stay HTTPS; failure falls back to cargo install. +fn prebuilt_bindgen(tools: &Path, version: &str, path: &OsStr) -> Result<()> { + let scratch = Scratch::new(tools)?; + let release = format!("wasm-bindgen-{version}-x86_64-unknown-linux-musl"); + let archive = scratch.0.join("release.tar.gz"); + let binary = scratch.0.join("wasm-bindgen"); + let url = format!("https://github.com/wasm-bindgen/wasm-bindgen/releases/download/{version}/{release}.tar.gz"); + commands::run( + Command::new("curl") + .env("PATH", path) + .args([ + "--fail", + "--location", + "--silent", + "--show-error", + "--proto", + "=https", + "--proto-redir", + "=https", + "--tlsv1.2", + "--max-time", + "120", + "--max-filesize", + "33554432", + "--output", + ]) + .arg(&archive) + .arg(url), + )?; + let output = fs::File::create(&binary).map_err(|error| error.to_string())?; + commands::run( + Command::new("tar") + .env("PATH", path) + .args(["-xzOf"]) + .arg(&archive) + .arg(format!("{release}/wasm-bindgen")) + .stdout(Stdio::from(output)), + )?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&binary, fs::Permissions::from_mode(0o755)).map_err(|error| error.to_string())?; + } + let checked = Command::new(&binary).arg("--version").output().map_err(|error| error.to_string())?; + if !checked.status.success() || !version_matches(&checked.stdout, version, true) { + return Err("downloaded wasm-bindgen does not report the locked version".into()); + } + fs::rename(&binary, tools.join("bin/wasm-bindgen")).map_err(|error| error.to_string()) +} + +fn install(root: &Path, path: &OsStr, tools: &Path, name: &str, version: &str) -> Result<()> { + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + commands::run( + Command::new(cargo) + .current_dir(root) + .env("PATH", path) + .args(["install", name, "--version", version, "--locked", "--root"]) + .arg(tools), + ) +} + +pub fn run(root: &Path, args: &[String]) -> Result<()> { + if !args.is_empty() { + return Err("usage: cargo xtask build".into()); + } + let lock = fs::read_to_string(root.join("Cargo.lock")).map_err(|error| error.to_string())?; + let bindgen_version = locked_version(&lock, "wasm-bindgen")?; + let worker_version = locked_version(&lock, "worker")?; + let tools = root.join("target/tools"); + fs::create_dir_all(tools.join("bin")).map_err(|error| error.to_string())?; + let path = commands::build_path(root, std::env::var_os("PATH").as_deref())?; + + if matching_tool("wasm-bindgen", &bindgen_version, &path, true).is_none() { + eprintln!("Installing wasm-bindgen-cli {bindgen_version}"); + let prebuilt = if std::env::consts::OS == "linux" && std::env::consts::ARCH == "x86_64" { + prebuilt_bindgen(&tools, &bindgen_version, &path) + } else { + Err("no prebuilt binary for this platform".into()) + }; + if let Err(why) = prebuilt { + eprintln!("xtask: {why}; compiling wasm-bindgen-cli"); + install(root, &path, &tools, "wasm-bindgen-cli", &bindgen_version)?; + } + } + let bindgen = matching_tool("wasm-bindgen", &bindgen_version, &path, true) + .ok_or("wasm-bindgen is not at the locked version after installation")?; + if matching_tool("worker-build", &worker_version, &path, false).is_none() { + eprintln!("Installing worker-build {worker_version}"); + install(root, &path, &tools, "worker-build", &worker_version)?; + } + let worker = matching_tool("worker-build", &worker_version, &path, false) + .ok_or("worker-build is not at the locked version after installation")?; + let commit = stamp(root)?; + let executable = std::env::current_exe().map_err(|error| error.to_string())?; + let configured = |command: &mut Command| { + command.env("PATH", &path).env("ALLTHING_COMMIT", &commit).env("WASM_BINDGEN_BIN", &bindgen); + }; + eprintln!("Building the Worker"); + let mut build_worker = Command::new(worker); + build_worker.current_dir(root.join("crates/allthing-worker")).arg("--release"); + configured(&mut build_worker); + commands::run(&mut build_worker)?; + // Invoke this already-built executable, not a recursive cargo run. + let mut strip = Command::new(&executable); + strip.current_dir(root).arg("strip-worker"); + configured(&mut strip); + commands::run(&mut strip)?; + let optimizer = find_wasm_opt::find( + std::env::var_os("WASM_OPT_BIN"), + Some(path.clone()), + &find_wasm_opt::default_roots(), + find_wasm_opt::platform(std::env::consts::OS, std::env::consts::ARCH), + )?; + eprintln!("Building the browser app"); + let mut browser = Command::new(executable); + browser + .current_dir(root) + .arg("dist") + .env("WASM_OPT_BIN", optimizer.map_or_else(OsString::new, PathBuf::into_os_string)); + configured(&mut browser); + commands::run(&mut browser)?; + eprintln!("Built dist/ and crates/allthing-worker/build/"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lock_version_is_structural_and_missing_or_ambiguous_versions_fail() { + let lock = "version = 4\n[[package]]\nversion = '0.2.108'\nname = 'wasm-bindgen'\n[[package]]\nname = 'worker'\nversion = '0.8.5'\n"; + assert_eq!(locked_version(lock, "wasm-bindgen").unwrap(), "0.2.108"); + assert_eq!(locked_version(lock, "worker").unwrap(), "0.8.5"); + assert!(locked_version(lock, "missing").is_err()); + assert!(locked_version(&format!("{lock}[[package]]\nname = 'worker'\nversion = '0.9.0'\n"), "worker").is_err()); + assert!(locked_version("not toml = ", "worker").is_err()); + } + + #[test] + fn version_checks_reject_stale_or_prefixed_tool_output() { + assert!(version_matches(b"wasm-bindgen 0.2.108\n", "0.2.108", true)); + assert!(!version_matches(b"wasm-bindgen 0.2.1089\n", "0.2.108", true)); + assert!(version_matches(b"0.8.5\n", "0.8.5", false)); + assert!(!version_matches(b"worker-build 0.8.5\n", "0.8.5", false)); + } + + #[test] + fn commit_stamp_uses_ci_before_git_and_rejects_untrusted_non_revisions() { + assert_eq!(commit_stamp(Some("a1234567890abcdef1234567890abcdef1234567"), Some("b123456")).unwrap(), "a123456"); + assert_eq!(commit_stamp(Some(""), Some("b123456")).unwrap(), "b123456"); + assert_eq!(commit_stamp(None, None).unwrap(), "dev"); + assert!(commit_stamp(Some("$(secret)"), None).is_err()); + assert!(commit_stamp(Some("abc"), None).is_err()); + } +} diff --git a/xtask/src/tooling/commands.rs b/xtask/src/tooling/commands.rs new file mode 100644 index 00000000..a1e37744 --- /dev/null +++ b/xtask/src/tooling/commands.rs @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Native orchestration for existing checks and browser fixture phases. +//! These commands do not alter Playwright's retries, assertions or budgets. + +use super::find_wasm_opt; +use serde_json::Value; +use std::ffi::{OsStr, OsString}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +pub type Result = std::result::Result; + +fn no_args(args: &[String], usage: &str) -> Result<()> { + if args.is_empty() { Ok(()) } else { Err(format!("usage: cargo xtask {usage}")) } +} + +pub fn run(command: &mut Command) -> Result<()> { + let shown = format!("{command:?}"); + eprintln!("$ {shown}"); + let status = command.status().map_err(|error| format!("could not run {shown}: {error}"))?; + if status.success() { Ok(()) } else { Err(format!("{shown} failed with {status}")) } +} + +/// Functional fixtures first, then calibrated measurements by themselves, +/// even when the first runner fails to start or an assertion fails. +pub fn browser_fixtures(root: &Path, args: &[String]) -> Result<()> { + no_args(args, "browser-fixtures")?; + let npx = std::env::var_os("NPX_BIN").unwrap_or_else(|| "npx".into()); + let phases = fixture_phases(); + let failures = run_phases(&phases, |phase| { + eprintln!("{}", phase.label); + run(Command::new(&npx).current_dir(root).env("PLAYWRIGHT_HTML_OUTPUT_DIR", phase.report).args(phase.args)) + }); + if failures.is_empty() { Ok(()) } else { Err(failures.join("\n")) } +} + +struct Phase { + label: &'static str, + report: &'static str, + args: &'static [&'static str], +} + +fn fixture_phases() -> [Phase; 2] { + [ + Phase { + label: "Functional fixtures (2 workers)", + report: "playwright-report/functional", + args: &[ + "playwright", + "test", + r"^(?!.*[/\\]smoothness\.fixture\.spec\.ts$).*\.fixture\.spec\.ts$", + "--workers=2", + "--output=test-results/functional", + ], + }, + Phase { + label: "Smoothness fixtures (1 worker, after functional fixtures finish)", + report: "playwright-report/smoothness", + args: &[ + "playwright", + "test", + r"[/\\]smoothness\.fixture\.spec\.ts$", + "--workers=1", + "--output=test-results/smoothness", + ], + }, + ] +} + +fn run_phases(phases: &[Phase], mut run: impl FnMut(&Phase) -> Result<()>) -> Vec { + phases.iter().filter_map(|phase| run(phase).err()).collect() +} + +/// The exact native/wasm checks formerly listed by scripts/check.sh. +/// Repository version/CSP/changelog checks run before this in the dispatcher. +pub fn check(root: &Path, args: &[String]) -> Result<()> { + no_args(args, "check")?; + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + for args in check_steps() { + run(Command::new(&cargo).current_dir(root).args(args))?; + } + if find_wasm_opt::on_path(OsStr::new("cargo-deny"), std::env::var_os("PATH").as_deref()).is_some() { + run(Command::new(&cargo).current_dir(root).args(["deny", "--log-level", "error", "check", "licenses"]))?; + } else { + eprintln!("cargo-deny is not installed; skipping the license audit (cargo install cargo-deny)"); + } + Ok(()) +} + +fn check_steps() -> [&'static [&'static str]; 5] { + [ + &["fmt", "--all", "--", "--check"], + &["clippy", "--locked", "-p", "allthing-core", "-p", "xtask", "--all-targets", "--", "-D", "warnings"], + &[ + "clippy", + "--locked", + "-p", + "allthing-data", + "-p", + "allthing-web", + "-p", + "allthing-worker", + "--target", + "wasm32-unknown-unknown", + "--", + "-D", + "warnings", + ], + &[ + "clippy", + "--locked", + "-p", + "allthing-web", + "--features", + "static-host", + "--target", + "wasm32-unknown-unknown", + "--", + "-D", + "warnings", + ], + &[ + "test", + "--locked", + "-p", + "allthing-core", + "-p", + "allthing-data", + "-p", + "allthing-web", + "-p", + "allthing-worker", + "-p", + "xtask", + ], + ] +} + +/// The SpacetimeDB module remains outside the application workspace. Its +/// own manifest and lockfile are selected by cwd; the browser's relocation +/// flags must never leak into its Wasm publish build. +pub fn module_check(root: &Path, args: &[String]) -> Result<()> { + no_args(args, "module-check")?; + let module = root.join("spacetime/spacetimedb"); + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + for args in module_steps() { + run(Command::new(&cargo).current_dir(&module).env("RUSTFLAGS", "").args(args))?; + } + if find_wasm_opt::on_path(OsStr::new("cargo-deny"), std::env::var_os("PATH").as_deref()).is_some() { + run(Command::new(&cargo).current_dir(&module).env("RUSTFLAGS", "").args([ + "deny", + "--log-level", + "error", + "check", + "licenses", + ]))?; + } else { + eprintln!("cargo-deny is not installed; skipping the license audit"); + } + Ok(()) +} + +fn module_steps() -> [&'static [&'static str]; 5] { + [ + &["fmt", "--", "--check"], + &["clippy", "--locked", "--all-targets", "--", "-D", "warnings"], + &["clippy", "--locked", "--target", "wasm32-unknown-unknown", "--", "-D", "warnings"], + &["test", "--locked"], + &["build", "--locked", "--release", "--target", "wasm32-unknown-unknown"], + ] +} + +#[derive(Debug, PartialEq)] +enum Audit { + Available { vulnerabilities: Value, serious: bool }, + Unavailable(String), +} + +fn audit_report(bytes: &[u8], stderr: &[u8]) -> Audit { + let report: Value = match serde_json::from_slice(bytes) { + Ok(report) => report, + Err(_) => return Audit::Unavailable(last_line(stderr)), + }; + if let Some(error) = report.get("error").filter(|error| !error.is_null()) { + return Audit::Unavailable( + error.get("summary").and_then(Value::as_str).map_or_else(|| error.to_string(), str::to_owned), + ); + } + let Some(found) = report.pointer("/metadata/vulnerabilities").filter(|found| found.is_object()) else { + return Audit::Unavailable("unrecognized npm audit response".into()); + }; + let (Some(high), Some(critical)) = + (found.get("high").and_then(Value::as_u64), found.get("critical").and_then(Value::as_u64)) + else { + return Audit::Unavailable("unrecognized npm audit severity counts".into()); + }; + Audit::Available { vulnerabilities: found.clone(), serious: high > 0 || critical > 0 } +} + +fn last_line(bytes: &[u8]) -> String { + String::from_utf8_lossy(bytes).lines().rev().find(|line| !line.trim().is_empty()).unwrap_or("no answer").to_owned() +} + +fn warning(message: &str) { + // npm's registry supplies this text: it must remain one workflow command. + let escaped = message.replace('%', "%25").replace('\r', "%0D").replace('\n', "%0A"); + println!("::warning title=npm audit unavailable::{escaped}"); +} + +struct Captured { + stdout: Vec, + stderr: Vec, +} + +fn bounded_read(mut reader: impl Read, limit: usize) -> std::io::Result<(Vec, bool)> { + let mut retained = Vec::new(); + let mut buffer = [0; 8192]; + let mut truncated = false; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + return Ok((retained, truncated)); + } + let keep = count.min(limit.saturating_sub(retained.len())); + retained.extend_from_slice(&buffer[..keep]); + truncated |= keep < count; + } +} + +fn stop(child: &mut Child) { + #[cfg(unix)] + { + // SAFETY: capture creates a new child process group. Terminating + // that group also stops helpers retaining its inherited pipes. + unsafe { + libc::kill(-(child.id() as libc::pid_t), libc::SIGKILL); + } + } + let _ = child.kill(); + let _ = child.wait(); +} + +enum Pipe { + Out(std::io::Result<(Vec, bool)>), + Err(std::io::Result<(Vec, bool)>), +} + +/// Drain both pipes concurrently, but never wait for a pipe outside the +/// deadline. A child may have exited while a helper still owns stdout. +fn capture(command: &mut Command, timeout: Duration) -> Result { + let deadline = Instant::now().checked_add(timeout).ok_or("npm audit timeout is too large")?; + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + let stdout = child.stdout.take().ok_or("missing child stdout")?; + let stderr = child.stderr.take().ok_or("missing child stderr")?; + let (sender, receiver) = mpsc::sync_channel(2); + let errors = sender.clone(); + thread::spawn(move || { + let _ = sender.send(Pipe::Out(bounded_read(stdout, 4 * 1024 * 1024))); + }); + thread::spawn(move || { + let _ = errors.send(Pipe::Err(bounded_read(stderr, 64 * 1024))); + }); + let (mut out, mut err) = (None, None); + let mut exited = false; + loop { + if !exited { + match child.try_wait() { + Ok(status) => exited = status.is_some(), + Err(error) => { + stop(&mut child); + return Err(error.to_string()); + } + } + } + if exited && out.is_some() && err.is_some() { + return Ok(Captured { + stdout: out.take().expect("stdout present"), + stderr: err.take().expect("stderr present"), + }); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + stop(&mut child); + return Err("npm audit service timed out".into()); + } + let step = remaining.min(Duration::from_millis(10)); + if out.is_some() && err.is_some() { + thread::sleep(step); + continue; + } + let message = match receiver.recv_timeout(step) { + Ok(message) => message, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => { + stop(&mut child); + return Err("npm audit output reader stopped".into()); + } + }; + let (is_out, result) = match message { + Pipe::Out(result) => (true, result), + Pipe::Err(result) => (false, result), + }; + match result { + Ok((bytes, false)) => { + if is_out { + out = Some(bytes) + } else { + err = Some(bytes) + } + } + Ok((_, true)) => { + stop(&mut child); + return Err("npm audit response exceeded the diagnostic limit".into()); + } + Err(error) => { + stop(&mut child); + return Err(format!("npm audit output failed: {error}")); + } + } + } +} + +pub fn npm_audit(root: &Path, args: &[String]) -> Result<()> { + no_args(args, "npm-audit")?; + let npm = std::env::var_os("NPM_BIN").unwrap_or_else(|| "npm".into()); + let captured = match capture( + Command::new(&npm).current_dir(root).args(["audit", "--json", "--audit-level=high"]), + Duration::from_secs(180), + ) { + Ok(captured) => captured, + Err(error) => { + warning(&error); + return Ok(()); + } + }; + match audit_report(&captured.stdout, &captured.stderr) { + Audit::Unavailable(why) => { + warning(&why); + Ok(()) + } + Audit::Available { vulnerabilities, serious } => { + println!("npm audit: {vulnerabilities}"); + if serious { + // A second readable report helps people fix the advisories; + // its status cannot turn the already-known failure green. + match capture( + Command::new(&npm).current_dir(root).args(["audit", "--audit-level=high"]), + Duration::from_secs(180), + ) { + Ok(report) => { + let readable = + [String::from_utf8_lossy(&report.stdout), String::from_utf8_lossy(&report.stderr)]; + for line in readable.iter().flat_map(|text| text.lines()) { + // Registry-supplied text must not become an Actions command. + println!("npm audit: {line}"); + } + } + Err(why) => eprintln!("npm audit: readable advisory report unavailable ({why})"), + } + Err("npm audit found high or critical advisories".into()) + } else { + Ok(()) + } + } + } +} + +/// PATH for the post-bootstrap build; no global environment mutation is +/// necessary (or safe in a multi-threaded Rust process). +pub fn build_path(root: &Path, inherited: Option<&OsStr>) -> Result { + let mut paths = vec![root.join("target/tools/bin")]; + paths.extend(inherited.into_iter().flat_map(std::env::split_paths)); + std::env::join_paths(paths).map_err(|error| error.to_string()) +} + +pub fn tool_on_path(program: &str, path: &OsStr) -> Result { + find_wasm_opt::on_path(OsStr::new(program), Some(path)) + .ok_or_else(|| format!("{program} is not executable on PATH")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn both_fixture_phases_always_run_and_either_failure_fails_the_aggregate() { + for failed in [vec![true, false], vec![false, true], vec![true, true], vec![false, false]] { + let mut order = Vec::new(); + let errors = run_phases(&fixture_phases(), |phase| { + let index = order.len(); + order.push(phase.report); + if failed[index] { Err("runner failed or could not start".into()) } else { Ok(()) } + }); + assert_eq!(order, ["playwright-report/functional", "playwright-report/smoothness"]); + assert_eq!(errors.len(), failed.iter().filter(|value| **value).count()); + } + } + + #[test] + fn fixture_arguments_preserve_worker_counts_filters_outputs_and_default_retry_policy() { + let phases = fixture_phases(); + assert_eq!( + phases[0].args, + [ + "playwright", + "test", + r"^(?!.*[/\\]smoothness\.fixture\.spec\.ts$).*\.fixture\.spec\.ts$", + "--workers=2", + "--output=test-results/functional" + ] + ); + assert_eq!( + phases[1].args, + [ + "playwright", + "test", + r"[/\\]smoothness\.fixture\.spec\.ts$", + "--workers=1", + "--output=test-results/smoothness" + ] + ); + assert!(browser_fixtures(Path::new("."), &["--workers=8".into()]).is_err()); + } + + #[test] + fn native_and_both_wasm_lint_targets_and_tests_are_retained_locked() { + let steps = check_steps(); + assert_eq!(steps.len(), 5); + assert_eq!(steps[0], ["fmt", "--all", "--", "--check"]); + assert!(steps[1].contains(&"--all-targets")); + assert!(steps[2].contains(&"allthing-worker")); + assert!(steps[2].contains(&"wasm32-unknown-unknown")); + assert!(steps[3].windows(2).any(|args| args == ["--features", "static-host"])); + for step in &steps[1..] { + assert!(step.contains(&"--locked")); + } + for name in ["allthing-core", "allthing-data", "allthing-web", "allthing-worker", "xtask"] { + assert!(steps[4].contains(&name)); + } + } + + #[test] + fn registry_outages_and_malformed_schema_are_unavailable_not_advisory_passes() { + assert_eq!( + audit_report(b"not json", b"first\n503 registry unavailable\n"), + Audit::Unavailable("503 registry unavailable".into()) + ); + assert_eq!( + audit_report(br#"{"error":{"summary":"registry 400"}}"#, b""), + Audit::Unavailable("registry 400".into()) + ); + assert!(matches!(audit_report(br#"{"advisories":[]}"#, b""), Audit::Unavailable(_))); + assert!(matches!( + audit_report(br#"{"metadata":{"vulnerabilities":{"high":"0","critical":0}}}"#, b""), + Audit::Unavailable(_) + )); + } + + #[test] + fn only_high_or_critical_advisories_fail_regardless_of_other_severities() { + for (high, critical, serious) in [(0, 0, false), (1, 0, true), (0, 1, true), (u64::MAX, u64::MAX, true)] { + let report = serde_json::json!({"metadata":{"vulnerabilities":{"high":high,"critical":critical,"moderate":99,"low":99}}}); + assert!( + matches!(audit_report(&serde_json::to_vec(&report).unwrap(), b""), Audit::Available { serious: found, .. } if found == serious) + ); + } + } + + #[test] + fn bounded_diagnostics_are_drained_without_retaining_the_excess() { + let (bytes, truncated) = bounded_read(std::io::Cursor::new(vec![7; 1024]), 33).unwrap(); + assert_eq!(bytes, vec![7; 33]); + assert!(truncated); + assert_eq!(bounded_read(std::io::Cursor::new(b"fine"), 33).unwrap(), (b"fine".to_vec(), false)); + } + + #[cfg(unix)] + #[test] + fn npm_capture_deadline_stops_helpers_that_keep_pipes_after_parent_exit() { + let scratch = std::env::temp_dir().join(format!("allthing-npm-descendant-{}", std::process::id())); + std::fs::create_dir_all(&scratch).unwrap(); + let marker = scratch.join("must-not-be-written"); + let started = Instant::now(); + let result = capture( + Command::new("/bin/sh") + .args(["-c", "(sleep 0.4; printf leaked > \"$1\") & exit 0", "fixture"]) + .arg(&marker), + Duration::from_millis(80), + ); + assert!(result.err().unwrap().contains("timed out")); + assert!(started.elapsed() < Duration::from_secs(2)); + thread::sleep(Duration::from_millis(500)); + assert!(!marker.exists(), "the owned descendant process group must be terminated"); + std::fs::remove_dir_all(scratch).unwrap(); + } + + #[test] + fn module_validation_retains_isolated_native_wasm_release_and_license_steps() { + let steps = module_steps(); + assert_eq!(steps[0], ["fmt", "--", "--check"]); + assert_eq!(steps[1], ["clippy", "--locked", "--all-targets", "--", "-D", "warnings"]); + assert_eq!(steps[2], ["clippy", "--locked", "--target", "wasm32-unknown-unknown", "--", "-D", "warnings"]); + assert_eq!(steps[3], ["test", "--locked"]); + assert_eq!(steps[4], ["build", "--locked", "--release", "--target", "wasm32-unknown-unknown"]); + } +} diff --git a/xtask/src/tooling/deploy.rs b/xtask/src/tooling/deploy.rs new file mode 100644 index 00000000..acc5bec9 --- /dev/null +++ b/xtask/src/tooling/deploy.rs @@ -0,0 +1,675 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Wait for the intended Cloudflare deployment without warming Home or its API. + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::io::Read; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +use serde_json::Value; +use url::{Host, Url}; + +type Result = std::result::Result; + +#[derive(Clone, Copy)] +struct Options { + timeout: Duration, + poll_interval: Duration, + request_timeout: Duration, + git_timeout: Duration, +} + +impl Default for Options { + fn default() -> Self { + Self { + timeout: Duration::from_secs(1_800), + poll_interval: Duration::from_secs(10), + request_timeout: Duration::from_secs(5), + git_timeout: Duration::from_secs(60), + } + } +} + +impl Options { + fn validate(self) -> Result<()> { + if [self.timeout, self.poll_interval, self.request_timeout, self.git_timeout].contains(&Duration::ZERO) { + return Err("Wait durations must be positive and finite".into()); + } + Ok(()) + } +} + +struct Tools { + curl: OsString, + git: OsString, +} + +impl Default for Tools { + fn default() -> Self { + Self { + curl: std::env::var_os("CURL_BIN").unwrap_or_else(|| "curl".into()), + git: std::env::var_os("GIT_BIN").unwrap_or_else(|| "git".into()), + } + } +} + +fn valid_sha(value: &str, full: bool) -> bool { + (if full { value.len() == 40 } else { (7..=40).contains(&value.len()) }) + && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn revision_matches(actual: &str, expected: &str) -> bool { + valid_sha(actual, false) + && valid_sha(expected, true) + && expected.to_ascii_lowercase().starts_with(&actual.to_ascii_lowercase()) +} + +/// Plain HTTP is useful only for local fixtures. Production checks never +/// downgrade to an insecure origin or accept credentials in a URL. +fn allowed_url(url: &Url) -> bool { + let local = match url.host() { + Some(Host::Ipv4(ip)) => ip.is_loopback(), + Some(Host::Ipv6(ip)) => ip.is_loopback(), + Some(Host::Domain(name)) => name.eq_ignore_ascii_case("localhost"), + None => false, + }; + url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && (url.scheme() == "https" || (url.scheme() == "http" && local)) +} + +fn version_url(base: &str) -> Result { + let base = Url::parse(base).map_err(|_| "Expected an HTTP(S) site URL")?; + if !allowed_url(&base) { + return Err("Expected an HTTPS site URL (HTTP is allowed only for loopback tests)".into()); + } + base.join("/version").map_err(|_| "Expected an HTTP(S) site URL".into()) +} + +#[derive(Debug, PartialEq, Eq)] +enum ChildError { + TimedOut, + TooLarge, + Failed(String), +} + +/// Every child has a group of its own. In particular, killing `git fetch` +/// must also stop its transport helper and close inherited output pipes. +fn stop(child: &mut Child) { + #[cfg(unix)] + { + // SAFETY: process_group(0) below makes this child's PID its group + // ID. The negative PID targets only the group this command owns. + unsafe { + libc::kill(-(child.id() as libc::pid_t), libc::SIGKILL); + } + } + let _ = child.kill(); + let _ = child.wait(); +} + +fn capture( + command: &mut Command, + deadline: Instant, + limit: usize, +) -> std::result::Result<(ExitStatus, Vec), ChildError> { + if Instant::now() >= deadline { + return Err(ChildError::TimedOut); + } + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| ChildError::Failed(e.to_string()))?; + let stdout = child.stdout.take().ok_or_else(|| ChildError::Failed("missing child stdout".into()))?; + let (sender, receiver) = mpsc::sync_channel(1); + thread::spawn(move || { + let mut bytes = Vec::new(); + let result = stdout + .take(limit as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|e| ChildError::Failed(e.to_string())) + .and(if bytes.len() > limit { Err(ChildError::TooLarge) } else { Ok(bytes) }); + let _ = sender.send(result); + }); + let mut output = None; + loop { + if output.is_none() { + match receiver.try_recv() { + Ok(Ok(bytes)) => output = Some(bytes), + Ok(Err(error)) => { + stop(&mut child); + return Err(error); + } + Err(mpsc::TryRecvError::Disconnected) => { + stop(&mut child); + return Err(ChildError::Failed("child output reader stopped".into())); + } + Err(mpsc::TryRecvError::Empty) => {} + } + } + let status = match child.try_wait() { + Ok(status) => status, + Err(error) => { + stop(&mut child); + return Err(ChildError::Failed(error.to_string())); + } + }; + if let Some(status) = status { + if let Some(bytes) = output { + return Ok((status, bytes)); + } + match receiver.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(Ok(bytes)) => return Ok((status, bytes)), + Ok(Err(error)) => { + stop(&mut child); + return Err(error); + } + Err(_) => { + stop(&mut child); + return Err(ChildError::TimedOut); + } + } + } + if Instant::now() >= deadline { + stop(&mut child); + // Also prove inherited pipes closed, without an unbounded join + // if a broken external tool escaped its process group. + if output.is_none() && receiver.recv_timeout(Duration::from_millis(200)).is_err() { + return Err(ChildError::Failed("child output did not close after cancellation".into())); + } + return Err(ChildError::TimedOut); + } + thread::sleep(Duration::from_millis(5).min(deadline.saturating_duration_since(Instant::now()))); + } +} + +#[derive(Debug)] +enum FetchError { + TimedOut, + Failed, +} + +fn fetch_version(url: &Url, curl: &OsString, deadline: Instant) -> std::result::Result<(u16, String), FetchError> { + let mut url = url.clone(); + for _ in 0..=20 { + if !allowed_url(&url) { + return Err(FetchError::Failed); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(FetchError::TimedOut); + } + let seconds = remaining.as_secs_f64().to_string(); + // Redirects are followed explicitly, so each target is checked + // before a request. All hops share the same request deadline. + let result = capture( + Command::new(curl).args([ + "--silent", + "--show-error", + "--proto", + "=http,https", + "--max-time", + &seconds, + "--connect-timeout", + &seconds, + "--max-filesize", + "65536", + "--noproxy", + "localhost,127.0.0.1,::1", + "--header", + "accept: application/json", + "--header", + "cache-control: no-cache", + "--write-out", + "\n%{http_code}\n%{redirect_url}", + "--url", + url.as_str(), + ]), + deadline, + 72 * 1024, + ); + let (status, bytes) = match result { + Ok(value) => value, + Err(ChildError::TimedOut) => return Err(FetchError::TimedOut), + Err(_) => return Err(FetchError::Failed), + }; + if !status.success() { + return Err(if status.code() == Some(28) { FetchError::TimedOut } else { FetchError::Failed }); + } + let output = String::from_utf8(bytes).map_err(|_| FetchError::Failed)?; + let mut pieces = output.rsplitn(3, '\n'); + let redirect = pieces.next().ok_or(FetchError::Failed)?; + let status = pieces.next().ok_or(FetchError::Failed)?.parse::().map_err(|_| FetchError::Failed)?; + let body = pieces.next().ok_or(FetchError::Failed)?; + if body.len() > 64 * 1024 { + return Err(FetchError::Failed); + } + if matches!(status, 301 | 302 | 303 | 307 | 308) && !redirect.is_empty() { + url = Url::parse(redirect).map_err(|_| FetchError::Failed)?; + continue; + } + return Ok((status, body.into())); + } + Err(FetchError::Failed) +} + +struct History<'a> { + root: &'a Path, + git: OsString, + timeout: Duration, + settled: BTreeMap<(String, String), bool>, +} + +impl<'a> History<'a> { + fn new(root: &'a Path, git: &OsString, timeout: Duration) -> Self { + Self { root, git: git.clone(), timeout, settled: BTreeMap::new() } + } + + fn git(&self, args: &[&str], deadline: Instant) -> Option { + capture( + Command::new(&self.git).current_dir(self.root).args(args), + deadline.min(Instant::now() + self.timeout), + 64 * 1024, + ) + .ok() + .and_then(|(status, _)| status.code()) + } + + fn includes(&mut self, deployed: &str, expected: &str, deadline: Instant) -> bool { + if !valid_sha(deployed, false) || !valid_sha(expected, true) { + return false; + } + let key = (deployed.to_ascii_lowercase(), expected.to_ascii_lowercase()); + if let Some(answer) = self.settled.get(&key) { + return *answer; + } + let object = format!("{}^{{commit}}", key.0); + if self.git(&["cat-file", "-e", &object], deadline) != Some(0) + && !(self.git(&["fetch", "--quiet", "origin", "main"], deadline) == Some(0) + && self.git(&["cat-file", "-e", &object], deadline) == Some(0)) + { + return false; + } + match self.git(&["merge-base", "--is-ancestor", &key.1, &key.0], deadline) { + Some(code @ (0 | 1)) => { + let answer = code == 0; + self.settled.insert(key, answer); + answer + } + _ => false, + } + } +} + +fn wait_for_deploy( + url: &Url, + expected: &str, + options: Options, + tools: &Tools, + mut includes: impl FnMut(&str, &str, Instant) -> bool, + mut log: impl FnMut(String), +) -> Result { + if !valid_sha(expected, true) { + return Err("Expected a full 40-digit Git commit".into()); + } + options.validate()?; + let deadline = Instant::now().checked_add(options.timeout).ok_or("Wait duration is too large")?; + let mut last = "no response".to_owned(); + while Instant::now() < deadline { + let request_deadline = deadline.min(Instant::now() + options.request_timeout); + match fetch_version(url, &tools.curl, request_deadline) { + Ok((status, _)) if !(200..300).contains(&status) => last = format!("HTTP {status}"), + Ok((_, body)) => match serde_json::from_str::(&body) { + Ok(body) => { + let actual = body.get("commit").and_then(Value::as_str); + if let Some(actual) = actual.filter(|actual| valid_sha(actual, false)) { + if Instant::now() < deadline && revision_matches(actual, expected) { + log(format!("Deployment {actual} is live")); + return Ok(actual.into()); + } + if includes(actual, expected, deadline) && Instant::now() < deadline { + log(format!("Deployment {actual} is live, and includes {}", &expected[..7])); + return Ok(actual.into()); + } + last = format!("commit {actual}"); + } else { + last = "unrecognized version response".into(); + } + } + Err(_) => last = "version request failed".into(), + }, + Err(FetchError::TimedOut) => last = "version request timed out".into(), + Err(FetchError::Failed) => last = "version request failed".into(), + } + log(format!("Waiting for deployment {}: {last}", &expected[..7])); + thread::sleep(options.poll_interval.min(deadline.saturating_duration_since(Instant::now()))); + } + Err(format!("Deployment {} was not live within {}s ({last})", &expected[..7], options.timeout.as_secs_f64())) +} + +pub fn run(root: &Path, args: &[String]) -> Result<()> { + if args.len() != 2 { + return Err("usage: cargo xtask wait-for-deploy ".into()); + } + if !valid_sha(&args[1], true) { + return Err("Expected a full 40-digit Git commit".into()); + } + let url = version_url(&args[0])?; + let options = Options::default(); + let tools = Tools::default(); + let mut history = History::new(root, &tools.git, options.git_timeout); + wait_for_deploy( + &url, + &args[1], + options, + &tools, + |deployed, expected, deadline| history.includes(deployed, expected, deadline), + |message| println!("{message}"), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::io::Write; + use std::net::TcpListener; + use std::path::PathBuf; + use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, + }; + + const EXPECTED: &str = "a1234567890abcdef1234567890abcdef1234567"; + fn fast() -> Options { + Options { + timeout: Duration::from_secs(2), + request_timeout: Duration::from_millis(120), + poll_interval: Duration::from_millis(10), + git_timeout: Duration::from_millis(120), + } + } + fn response(status: u16, body: &str) -> Vec { + format!("HTTP/1.1 {status} test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()) + .into_bytes() + } + + struct Http { + url: Url, + requests: Arc>>, + stop: Arc, + worker: Option>, + } + impl Http { + fn new(handler: impl Fn(usize) -> Vec + Send + Sync + 'static, stall: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = version_url(&format!("http://{}/ignored?q=ignored", listener.local_addr().unwrap())).unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let stop = Arc::new(AtomicBool::new(false)); + let handler = Arc::new(handler); + let (asked, stopped) = (Arc::clone(&requests), Arc::clone(&stop)); + let worker = thread::spawn(move || { + let mut children = Vec::new(); + while !stopped.load(Ordering::Relaxed) { + match listener.accept() { + Ok((mut stream, _)) => { + let (asked, stopped, handler) = + (Arc::clone(&asked), Arc::clone(&stopped), Arc::clone(&handler)); + children.push(thread::spawn(move || { + stream.set_read_timeout(Some(Duration::from_millis(200))).unwrap(); + stream.set_write_timeout(Some(Duration::from_millis(200))).unwrap(); + let mut request = Vec::new(); + let mut byte = [0]; + while request.len() < 8192 && stream.read(&mut byte).unwrap_or(0) == 1 { + request.push(byte[0]); + if request.ends_with(b"\r\n\r\n") { + break; + } + } + let number = { + let mut asked = asked.lock().unwrap(); + asked.push(String::from_utf8(request).unwrap()); + asked.len() + }; + let _ = stream.write_all(&handler(number)); + while stall && !stopped.load(Ordering::Relaxed) { + thread::sleep(Duration::from_millis(5)); + } + })); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(2)) + } + Err(error) => panic!("local server: {error}"), + } + } + for child in children { + child.join().unwrap(); + } + }); + Self { url, requests, stop, worker: Some(worker) } + } + } + impl Drop for Http { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + worker.join().unwrap(); + } + } + } + + #[test] + fn only_full_expected_and_valid_matching_stamps_are_accepted() { + for actual in [&EXPECTED[..7], EXPECTED, &EXPECTED.to_ascii_uppercase()] { + assert!(revision_matches(actual, EXPECTED)); + } + for actual in ["dev", "", "a12345", "g123456", "b123456", &format!("{EXPECTED}a")] { + assert!(!revision_matches(actual, EXPECTED)); + } + assert!(!revision_matches("a123456", "dev")); + assert!(version_url("file:///tmp/site").is_err()); + assert!(version_url("http://example.com").is_err()); + assert!(version_url("https://secret:password@example.com").is_err()); + assert!(Options { timeout: Duration::ZERO, ..fast() }.validate().is_err()); + assert_eq!(Options::default().timeout, Duration::from_secs(1800)); + assert_eq!(Options::default().poll_interval, Duration::from_secs(10)); + assert_eq!(Options::default().request_timeout, Duration::from_secs(5)); + } + + #[test] + fn polls_only_uncached_version_through_old_errors_malformed_and_unstamped_releases() { + let server = Http::new( + |n| match n { + 1 => response(200, r#"{"commit":"b123456"}"#), + 2 => response(503, "not ready"), + 3 => response(200, "not JSON"), + 4 => response(200, r#"{"commit":"dev"}"#), + 5 => response(200, r#"{"commit":42}"#), + _ => response(200, &format!(r#"{{"commit":"{EXPECTED}"}}"#)), + }, + false, + ); + let actual = + wait_for_deploy(&server.url, EXPECTED, fast(), &Tools::default(), |_, _, _| false, |_| {}).unwrap(); + assert_eq!(actual, EXPECTED); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 6); + for request in requests.iter() { + assert!(request.starts_with("GET /version HTTP/1.1\r\n"), "{request}"); + assert!(request.to_ascii_lowercase().contains("cache-control: no-cache\r\n")); + assert!(request.to_ascii_lowercase().contains("accept: application/json\r\n")); + } + } + + #[test] + fn later_including_commits_are_accepted_but_invalid_stamps_never_reach_git() { + let server = Http::new( + |n| { + response( + 200, + match n { + 1 => r#"{"commit":"b123456"}"#, + 2 => r#"{"commit":"dev"}"#, + _ => r#"{"commit":"c123456"}"#, + }, + ) + }, + false, + ); + let mut asked = Vec::new(); + let actual = wait_for_deploy( + &server.url, + EXPECTED, + fast(), + &Tools::default(), + |actual, expected, _| { + asked.push((actual.to_owned(), expected.to_owned())); + actual == "c123456" + }, + |_| {}, + ) + .unwrap(); + assert_eq!(actual, "c123456"); + assert_eq!(asked, [("b123456".into(), EXPECTED.into()), ("c123456".into(), EXPECTED.into())]); + } + + #[test] + fn wrong_release_and_stalled_headers_or_body_respect_monotonic_deadlines() { + for (stall, bytes) in [ + (false, response(200, r#"{"commit":"b123456"}"#)), + (true, Vec::new()), + (true, b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{\"commit\":".to_vec()), + ] { + let server = Http::new(move |_| bytes.clone(), stall); + let start = Instant::now(); + let options = Options { timeout: Duration::from_millis(600), ..fast() }; + let result = wait_for_deploy(&server.url, EXPECTED, options, &Tools::default(), |_, _, _| false, |_| {}); + assert!(result.unwrap_err().contains("was not live within 0.6s")); + assert!(start.elapsed() < Duration::from_millis(1500)); + assert!(server.requests.lock().unwrap().len() >= 2); + } + } + + #[test] + fn redirects_cannot_read_files_or_downgrade_to_public_http_and_bodies_are_bounded() { + let destination = Http::new(|_| response(200, &format!(r#"{{"commit":"{EXPECTED}"}}"#)), false); + let location = destination.url.to_string(); + let redirect = Http::new( + move |_| format!("HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n").into_bytes(), + false, + ); + let (status, body) = + fetch_version(&redirect.url, &Tools::default().curl, Instant::now() + Duration::from_secs(1)).unwrap(); + assert_eq!(status, 200); + assert_eq!(serde_json::from_str::(&body).unwrap()["commit"], EXPECTED); + assert_eq!(destination.requests.lock().unwrap().len(), 1); + for location in ["file:///etc/passwd", "http://example.com/version", "https://user:secret@example.com/version"] + { + let response = format!("HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n"); + let server = Http::new(move |_| response.as_bytes().to_vec(), false); + assert!( + fetch_version(&server.url, &Tools::default().curl, Instant::now() + Duration::from_secs(1)).is_err() + ); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } + let server = Http::new(|_| response(200, &"x".repeat(70 * 1024)), false); + assert!(fetch_version(&server.url, &Tools::default().curl, Instant::now() + Duration::from_secs(1)).is_err()); + } + + struct Scratch(PathBuf); + impl Scratch { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + for _ in 0..100 { + let path = std::env::temp_dir().join(format!( + "allthing-deploy-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + if fs::create_dir(&path).is_ok() { + return Self(path); + } + } + panic!("could not create fixture directory"); + } + } + impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + fn git(root: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .current_dir(root) + .args(["-c", "user.name=t", "-c", "user.email=t@example.com", "-c", "init.defaultBranch=main"]) + .args(args) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + fn commit(root: &Path, name: &str) -> String { + fs::write(root.join(name), name).unwrap(); + git(root, &["add", name]); + git(root, &["commit", "-q", "-m", name]); + git(root, &["rev-parse", "HEAD"]) + } + + #[test] + fn ancestry_fetches_unknown_main_commits_and_caches_settled_answers() { + let scratch = Scratch::new(); + let work = scratch.0.join("work"); + let clone = scratch.0.join("clone"); + git(&scratch.0, &["init", "-q", work.to_str().unwrap()]); + let old = commit(&work, "a"); + let expected = commit(&work, "b"); + git(&scratch.0, &["clone", "-q", "--no-local", work.to_str().unwrap(), clone.to_str().unwrap()]); + let later = commit(&work, "c"); + git(&work, &["checkout", "-q", "-b", "side", &old]); + let side = commit(&work, "d"); + git(&work, &["checkout", "-q", "main"]); + let tools = Tools::default(); + let mut history = History::new(&clone, &tools.git, Duration::from_secs(2)); + let deadline = Instant::now() + Duration::from_secs(10); + assert!(history.includes(&later[..7], &expected, deadline)); + assert!(history.includes(&expected[..7], &expected, deadline)); + assert!(!history.includes(&old[..7], &expected, deadline)); + assert!(!history.includes(&side[..7], &expected, deadline)); + assert!(!history.includes("0000000", &expected, deadline)); + history.git = "missing-git-fixture".into(); + assert!(history.includes(&later[..7], &expected, deadline)); + assert!(!history.includes(&old[..7], &expected, deadline)); + assert_eq!(history.settled.len(), 3); + } + + #[cfg(unix)] + #[test] + fn deadlines_kill_helpers_that_inherit_the_output_pipe() { + // A shell stands in for git with a transport helper. Killing only + // the parent leaves sleep holding stdout open for thirty seconds. + let start = Instant::now(); + let error = capture( + Command::new("sh").args(["-c", "sleep 30 & wait"]), + Instant::now() + Duration::from_millis(100), + 1024, + ) + .unwrap_err(); + assert_eq!(error, ChildError::TimedOut); + assert!(start.elapsed() < Duration::from_secs(1)); + } +} diff --git a/xtask/src/tooling/find_wasm_opt.rs b/xtask/src/tooling/find_wasm_opt.rs new file mode 100644 index 00000000..20ee93c7 --- /dev/null +++ b/xtask/src/tooling/find_wasm_opt.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! One optimizer lookup for Worker builds, static builds and the command line. + +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::{Path, PathBuf}; + +pub type Result = std::result::Result; + +pub fn executable(path: &Path) -> bool { + let Ok(metadata) = fs::metadata(path) else { return false }; + if !metadata.is_file() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + metadata.permissions().mode() & 0o111 != 0 + } + #[cfg(not(unix))] + { + true + } +} + +pub fn on_path(program: &OsStr, path: Option<&OsStr>) -> Option { + let name = Path::new(program); + if name.components().count() > 1 { + return executable(name).then(|| name.to_owned()); + } + std::env::split_paths(path?).map(|dir| dir.join(name)).find(|candidate| executable(candidate)) +} + +/// Binaryen's platform names as worker-build caches them. An unsupported +/// machine may still supply an explicit binary or have one on PATH. +pub fn platform(os: &str, arch: &str) -> Option<&'static str> { + match (os, arch) { + ("macos" | "Darwin", "arm64" | "aarch64") => Some("arm64-macos"), + ("macos" | "Darwin", "x86_64") => Some("x86_64-macos"), + ("linux" | "Linux", "arm64" | "aarch64") => Some("aarch64-linux"), + ("linux" | "Linux", "x86_64") => Some("x86_64-linux"), + _ => None, + } +} + +pub fn default_roots() -> Vec { + let home = std::env::var_os("HOME").map(PathBuf::from); + let mut roots = Vec::new(); + if let Some(cache) = std::env::var_os("XDG_CACHE_HOME").filter(|value| !value.is_empty()) { + roots.push(cache.into()); + } else if let Some(home) = &home { + roots.push(home.join(".cache")); + } + if let Some(home) = home { + roots.push(home.join("Library/Caches")); + } + roots.push( + std::env::var_os("TMPDIR").filter(|value| !value.is_empty()).map_or_else(std::env::temp_dir, PathBuf::from), + ); + roots +} + +/// Override, installed tool, then each supplied cache root in order. Cache +/// names are sorted like the old shell glob; a different architecture or a +/// non-executable file can never be picked from a shared cache. +pub fn find( + override_path: Option, + path: Option, + roots: &[PathBuf], + target: Option<&str>, +) -> Result> { + if let Some(path) = override_path.filter(|value| !value.is_empty()) { + return Ok(Some(path.into())); + } + if let Some(installed) = on_path(OsStr::new("wasm-opt"), path.as_deref()) { + return Ok(Some(installed)); + } + let Some(target) = target else { return Ok(None) }; + let prefix = format!("wasm-opt-{target}-"); + for root in roots { + let mut candidates: Vec = match fs::read_dir(root.join("worker-build")) { + Ok(entries) => entries.filter_map(|entry| entry.ok().map(|entry| entry.path())).collect(), + Err(error) + if matches!(error.kind(), std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied) => + { + continue; + } + Err(error) => return Err(format!("{}: {error}", root.display())), + }; + candidates.sort(); + for candidate in candidates { + if candidate.file_name().and_then(OsStr::to_str).is_some_and(|name| name.starts_with(&prefix)) { + let binary = candidate.join("bin/wasm-opt"); + if executable(&binary) { + return Ok(Some(binary)); + } + } + } + } + Ok(None) +} + +pub fn current(roots: &[PathBuf]) -> Result> { + find( + std::env::var_os("WASM_OPT_BIN"), + std::env::var_os("PATH"), + roots, + platform(std::env::consts::OS, std::env::consts::ARCH), + ) +} + +pub fn run(args: &[String]) -> Result<()> { + let roots = if args.is_empty() { default_roots() } else { args.iter().map(PathBuf::from).collect() }; + if let Some(path) = current(&roots)? { + println!("{}", path.display()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT: AtomicU64 = AtomicU64::new(0); + struct Scratch(PathBuf); + impl Scratch { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "allthing-optimizer-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + fn binary(&self, relative: &str, executable: bool) -> PathBuf { + let path = self.0.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"#!/bin/sh\nexit 0\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(if executable { 0o755 } else { 0o644 })).unwrap(); + } + path + } + } + impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + #[test] + fn explicit_path_with_spaces_is_not_reinterpreted_or_required_to_exist() { + let path = OsString::from("/chosen optimizer/wasm-opt"); + assert_eq!(find(Some(path.clone()), None, &[], None).unwrap(), Some(PathBuf::from(path))); + } + + #[test] + fn installed_path_precedes_cache_and_directories_are_not_executables() { + let scratch = Scratch::new(); + let installed = scratch.binary("tools with spaces/wasm-opt", true); + scratch.binary("worker-build/wasm-opt-arm64-macos-132/bin/wasm-opt", true); + let search = std::env::join_paths([installed.parent().unwrap()]).unwrap(); + assert_eq!( + find(None, Some(search), std::slice::from_ref(&scratch.0), Some("arm64-macos")).unwrap(), + Some(installed) + ); + assert!(!executable(&scratch.0)); + } + + #[test] + fn platform_cache_skips_wrong_architecture_and_non_executable_after_empty_root() { + let scratch = Scratch::new(); + scratch.binary("cache with spaces/worker-build/wasm-opt-x86_64-linux-132/bin/wasm-opt", true); + scratch.binary("cache with spaces/worker-build/wasm-opt-arm64-macos-131/bin/wasm-opt", false); + let wanted = scratch.binary("cache with spaces/worker-build/wasm-opt-arm64-macos-132/bin/wasm-opt", true); + let roots = [scratch.0.join("empty-xdg"), scratch.0.join("cache with spaces")]; + assert_eq!(find(None, Some(OsString::new()), &roots, Some("arm64-macos")).unwrap(), Some(wanted)); + } + + #[test] + fn absent_optimizer_and_unknown_architecture_remain_optional() { + let scratch = Scratch::new(); + assert_eq!(find(None, None, std::slice::from_ref(&scratch.0), Some("arm64-macos")).unwrap(), None); + assert_eq!(platform("windows", "x86_64"), None); + assert_eq!(platform("Darwin", "aarch64"), Some("arm64-macos")); + assert_eq!(find(None, None, &[scratch.0.join("absent")], None).unwrap(), None); + } +} diff --git a/xtask/src/tooling/metadata.rs b/xtask/src/tooling/metadata.rs new file mode 100644 index 00000000..9710da10 --- /dev/null +++ b/xtask/src/tooling/metadata.rs @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Repository metadata checks and the opt-in, read-only asset credit audit. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::Read; +use std::path::Path; +use std::process::{Command, Stdio}; + +use base64::Engine; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use unicode_normalization::UnicodeNormalization; + +type Result = std::result::Result; + +pub fn check_repo(root: &Path) -> Result<()> { + let read = |name: &str| fs::read_to_string(root.join(name)).map_err(|e| format!("{name}: {e}")); + check_versions(&read("Cargo.toml")?, &read("package.json")?)?; + check_inline_scripts(&read("web/index.html")?, &read("web/_headers")?)?; + check_changelog(&read("CHANGELOG.md")?)?; + println!("Repository versions, inline-script CSP hashes and release links agree."); + Ok(()) +} + +fn check_versions(cargo: &str, package: &str) -> Result<()> { + // Value::from_str parses a standalone TOML value in toml 1.x; this is + // a complete document, including its workspace tables and comments. + let cargo: toml::Value = toml::from_str(cargo).map_err(|e| format!("Cargo.toml: {e}"))?; + let cargo = cargo + .get("workspace") + .and_then(|v| v.get("package")) + .and_then(|v| v.get("version")) + .and_then(toml::Value::as_str) + .filter(|v| !v.is_empty()) + .ok_or("Cargo.toml lacks workspace.package.version")?; + let package: Value = serde_json::from_str(package).map_err(|e| format!("package.json: {e}"))?; + let package = package + .get("version") + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + .ok_or("package.json lacks a string version")?; + if cargo != package { + return Err(format!("version mismatch: Cargo.toml {cargo}, package.json {package}")); + } + Ok(()) +} + +/// Only the effective global script directive can authorize inline scripts; +/// a hash in a comment, another header or a narrower path is no evidence. +fn script_hashes(headers: &str) -> Result> { + let mut route = ""; + let mut policies = Vec::new(); + for line in headers.lines() { + if line.trim().is_empty() || line.trim_start().starts_with('#') { + continue; + } + if !line.starts_with(char::is_whitespace) { + route = line.trim(); + continue; + } + if route == "/*" + && let Some((name, value)) = line.trim().split_once(':') + && name.eq_ignore_ascii_case("Content-Security-Policy") + { + policies.push(value.trim()); + } + } + if policies.len() != 1 { + return Err("web/_headers must have one global Content-Security-Policy".into()); + } + let mut directives = BTreeMap::new(); + for directive in policies[0].split(';') { + let mut words = directive.split_whitespace(); + if let Some(name) = words.next() + && directives.insert(name.to_ascii_lowercase(), words.collect::>()).is_some() + { + return Err(format!("web/_headers repeats CSP directive {name}")); + } + } + let words = ["script-src-elem", "script-src", "default-src"] + .into_iter() + .find_map(|name| directives.get(name)) + .ok_or("web/_headers has no script policy")?; + Ok(words + .iter() + .filter_map(|word| word.strip_prefix('\'').and_then(|s| s.strip_suffix('\''))) + .filter(|word| word.starts_with("sha256-")) + .map(str::to_owned) + .collect()) +} + +fn tag_end(html: &str, start: usize) -> Result { + let mut quote = None; + for (i, byte) in html.as_bytes().iter().enumerate().skip(start) { + match (quote, byte) { + (Some(q), b) if q == *b => quote = None, + (None, b'\'' | b'"') => quote = Some(*byte), + (None, b'>') => return Ok(i), + _ => {} + } + } + Err("web/index.html contains an unterminated tag".into()) +} + +fn attributes(tag: &str) -> Result> { + let bytes = tag.as_bytes(); + let mut i = 0; + let mut names = BTreeSet::new(); + while i < bytes.len() { + while i < bytes.len() && (bytes[i].is_ascii_whitespace() || bytes[i] == b'/') { + i += 1; + } + if i == bytes.len() { + break; + } + let first = i; + while i < bytes.len() && !bytes[i].is_ascii_whitespace() && !b"=/".contains(&bytes[i]) { + i += 1; + } + if i == first { + return Err("web/index.html contains a malformed script attribute".into()); + } + let name = tag[first..i].to_ascii_lowercase(); + if !names.insert(name.clone()) { + return Err(format!("web/index.html repeats script attribute {name}")); + } + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if bytes.get(i) != Some(&b'=') { + continue; + } + i += 1; + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if let Some(quote @ (b'\'' | b'"')) = bytes.get(i).copied() { + i += 1; + while i < bytes.len() && bytes[i] != quote { + i += 1; + } + if i == bytes.len() { + return Err("web/index.html contains an unterminated script attribute".into()); + } + i += 1; + } else { + while i < bytes.len() && !bytes[i].is_ascii_whitespace() { + i += 1; + } + } + } + Ok(names) +} + +fn check_inline_scripts(html: &str, headers: &str) -> Result<()> { + let hashes = script_hashes(headers)?; + let lower = html.to_ascii_lowercase(); + let mut cursor = 0; + while let Some(relative) = lower[cursor..].find('<') { + let start = cursor + relative; + if lower[start..].starts_with("").ok_or("web/index.html contains an unclosed comment")? + 3; + continue; + } + let name_start = start + 1; + let name_end = lower[name_start..] + .find(|c: char| !c.is_ascii_alphanumeric() && c != '-') + .map_or(lower.len(), |n| name_start + n); + let name = &lower[name_start..name_end]; + let open_end = tag_end(html, name_end)?; + cursor = open_end + 1; + // These are raw-text/RCDATA elements in the source shell: their + // literal '", + &headers(&hash(body)) + ) + .is_ok() + ); + assert!(check_inline_scripts("", &headers(&hash(body))).is_err()); + assert_eq!(hash("alert(1)"), "sha256-bhHHL3z2vDgxUt0W3dWQOrprscmda2Y5pLsLg4GF+pI="); + } + + #[test] + fn csp_only_accepts_the_effective_script_policy() { + let good = hash("run()"); + let html = ""; + for policy in [ + format!("# '{good}'\n/*\n Content-Security-Policy: script-src 'self'"), + format!("/*\n Content-Security-Policy: script-src 'self'; img-src '{good}'"), + format!("/assets/*\n Content-Security-Policy: script-src '{good}'"), + format!("/*\n Content-Security-Policy: script-src '{good}'; script-src-elem 'none'"), + ] { + assert!(check_inline_scripts(html, &policy).is_err()); + } + } + + #[test] + fn csp_distinguishes_external_scripts_attributes_comments_and_raw_text() { + let policy = headers(&hash("real()")); + assert!(check_inline_scripts("", &policy).is_ok()); + assert!(check_inline_scripts("", &policy).is_err()); + assert!(check_inline_scripts("", &policy).is_err()); + assert!(check_inline_scripts("