feat(lore-0023): rate limits per route and client, client IP from a configured header

apps/facilitator/README.md

@@ -12,7 +12,49 @@ discovery routes. Design: [the M1 spine](../../docs/architecture/m1-spine.md) (A
 - `SETTLE_TIMEOUT_MS` (20 s, or the budget when that is shorter; at least 1 s, at most the
   budget): how long a repeat of a sent payment waits for it to turn final.
 - `NETWORK=stellar:pubnet` refuses to start, whatever the store: `/settle` would spend there
-  with no spend budgets (0024), no rate limits (0023) and no channel lease (0025) yet.
+  with no spend budgets (0024) and no channel lease (0025) yet, and the store's tables would
+  grow without bound until pruning (0050) lands.
+
+## Callers and rate limits
+
+Every route is open: no caller authenticates, and there are no API keys (task 0060). `/verify`
+and `/settle` send no CORS headers, so a browser page can't call them; a seller's server calls
+them for its buyers.
+
+Each client has a budget per route and window. Past it the route answers 429
+`{"error":"too_many_requests"}` with `Retry-After` in seconds; the other routes still answer.
+Every request counts, a malformed or oversized one too. `/supported` has no limit.
+
+| Setting                | Default  | Meaning                                                                             |
+| ---------------------- | -------- | ----------------------------------------------------------------------------------- |
+| `RATE_LIMIT_WINDOW_MS` | 60 000   | the window, aligned to the clock: every count starts again at its end               |
+| `RATE_LIMIT_VERIFY`    | 600      | `/verify` requests per window and client                                            |
+| `RATE_LIMIT_SETTLE`    | 600      | `/settle` requests per window and client; at least `/verify`'s                      |
+| `RATE_LIMIT_DISCOVERY` | 120      | `/discovery/*` requests per window and client                                       |
+| `BODY_LIMIT_BYTES`     | 65 536   | a larger request body is a 413                                                      |
+| `CLIENT_IP`            | required | `cloudflare`: the `CF-Connecting-IP` header; `socket`: the connection's own address |
+
+- **A client is its IP**, an IPv6 one its /56. The seller's server calls `/verify` and `/settle`
+  for all of its buyers, so a budget is a budget per seller: a seller above about 10 paid requests
+  a second needs higher values. `/settle`'s budget is never below `/verify`'s, since the seller
+  settles after serving: a payment refused there was work given away. The spend budgets (AD-18),
+  not these, bound the fees the facilitator pays.
+- **Sellers on Cloudflare Workers share one budget.** A Worker's request to another Cloudflare
+  zone arrives with `CF-Connecting-IP: 2a06:98c0:3600::103` for every Worker, and sellers behind
+  one cloud NAT share its address the same way. A per-seller budget needs API keys (0060).
+- **`CLIENT_IP` is required**, so it can't be forgotten behind the tunnel, where every client
+  would share the tunnel's address and one budget. `cloudflare` reads `CF-Connecting-IP`, which
+  Cloudflare sets on every request and refuses from a client; `socket` takes the connection's own
+  address, for runs with no proxy in front (local runs and tests). Any other value refuses to
+  start, and so does a leftover `TRUSTED_PROXY_HOPS`. `X-Forwarded-For` is never read: a client
+  can put any address in it.
+- **With `cloudflare`, a request without the header, or with anything but one address in it,** is a
+  500 and an error log line: only `cloudflared` should reach the service.
+- **Windows are aligned to the clock**, so a client can send up to twice its budget across a
+  window's end, and every blocked client is released at the same moment.
+- **The counters live in the store** (`rate_limits` with `STORE=postgres`). A store error is a 500,
+  logged. A client over its budget is answered from the process until its window ends, without a
+  store write.
 
 ## The store
 

apps/facilitator/package.json

@@ -25,6 +25,7 @@
     "@x402/extensions": "~2.28.0",
     "@x402/stellar": "~2.28.0",
     "express": "^5.2.1",
+    "express-rate-limit": "^8.7.0",
     "pg": "^8.23.1",
     "undici": "~7.30.0",
     "zod": "^3.25.76"

apps/facilitator/scripts/settle-smoke.ts

@@ -85,6 +85,7 @@ if (created.status !== "SUCCESS") throw new Error(`channel setup ${created.statu
 const facilitatorServer = await main(
   {
     FACILITATOR_SECRET: facilitator.secret(),
+    CLIENT_IP: "socket",
     CHANNELS: channelKeys.map((keypair) => keypair.publicKey()).join(","),
     PORT: "0",
     LOG_LEVEL: "warn",

apps/facilitator/scripts/verify-smoke.ts

@@ -81,6 +81,7 @@ async function verifyWith(
   // A fresh facilitator key: verify never spends from it.
   const config = parseConfig({
     FACILITATOR_SECRET: Keypair.random().secret(),
+    CLIENT_IP: "socket",
     CHANNELS: channels,
     LOG_LEVEL: "info",
     MIN_VALIDITY_LEDGERS: process.env.MIN_VALIDITY_LEDGERS,

apps/facilitator/src/config.ts

@@ -178,11 +178,23 @@ const envSchema = z
     LEASE_TTL_MS: ms(1_000).default("30000"),
 
     BODY_LIMIT_BYTES: int(1_024).default("65536"),
-    /** Proxy hops in front of the service whose `X-Forwarded-For` entries are trusted (AD-11). */
-    TRUSTED_PROXY_HOPS: int(0, 10).default("0"),
+    /**
+     * Where the client's IP comes from (AD-11). `cloudflare`: the `CF-Connecting-IP` header, which
+     * Cloudflare sets on every request and refuses from a client; only `cloudflared` reaches the
+     * service (ADR 0011). `socket`: the connection's own address, with no proxy in front, as in
+     * local runs and tests. Required: forgotten behind the tunnel, every client would share the
+     * tunnel's address and one budget. `X-Forwarded-For` is never read.
+     */
+    CLIENT_IP: oneOf(["cloudflare", "socket"]),
+    /** Replaced by `CLIENT_IP`; refused, so a leftover can't look like it still applies. */
+    TRUSTED_PROXY_HOPS: z.string().optional(),
     RATE_LIMIT_WINDOW_MS: ms(1_000).default("60000"),
-    RATE_LIMIT_VERIFY: int(1).default("120"),
-    RATE_LIMIT_SETTLE: int(1).default("60"),
+    /**
+     * Requests per window and client. The seller's server calls `/verify` and `/settle` for all of
+     * its buyers, so these are budgets per seller; fees are bounded by the spend budgets (AD-18).
+     */
+    RATE_LIMIT_VERIFY: int(1).default("600"),
+    RATE_LIMIT_SETTLE: int(1).default("600"),
     RATE_LIMIT_DISCOVERY: int(1).default("120"),
 
     /** Rolling window of every spend budget (AD-18); a store window, not a timer. */
@@ -229,15 +241,25 @@ const envSchema = z
     message: "must be at most SETTLE_BUDGET_MS",
     path: ["SETTLE_TIMEOUT_MS"],
   })
+  // The seller settles after serving the resource, so a payment that passed `/verify` and is
+  // refused at `/settle` was work given away.
+  .refine((env) => env.RATE_LIMIT_SETTLE >= env.RATE_LIMIT_VERIFY, {
+    message: "must be at least RATE_LIMIT_VERIFY",
+    path: ["RATE_LIMIT_SETTLE"],
+  })
+  .refine((env) => env.TRUSTED_PROXY_HOPS === undefined, {
+    message: "is replaced by CLIENT_IP",
+    path: ["TRUSTED_PROXY_HOPS"],
+  })
   .refine((env) => env.STORE !== "memory" || env.NETWORK === STELLAR_NETWORKS.testnet, {
     message: "must be postgres outside testnet",
     path: ["STORE"],
   })
-  // `/settle` spends on pubnet, so not before spend budgets, rate limits and the channel lease.
-  // The task that lands the last of them lifts this.
+  // `/settle` spends on pubnet, so not before spend budgets and the channel lease, nor while the
+  // store's tables grow without bound. The task that lands the last of them lifts this.
   .refine((env) => env.NETWORK !== STELLAR_NETWORKS.pubnet, {
     message:
-      "stellar:pubnet is refused until spend budgets (0024), rate limits (0023) and the channel lease (0025) land",
+      "stellar:pubnet is refused until spend budgets (0024), the channel lease (0025) and store pruning (0050) land",
     path: ["NETWORK"],
   })
   .refine((env) => env.DATABASE_URL !== undefined || env.STORE !== "postgres", {
@@ -282,7 +304,7 @@ export interface Config {
   };
   http: {
     bodyLimitBytes: number;
-    trustedProxyHops: number;
+    clientIp: "cloudflare" | "socket";
     rateLimit: { windowMs: number; verify: number; settle: number; discovery: number };
   };
   spend: {
@@ -360,7 +382,7 @@ export function parseConfig(env: Record<string, string | undefined>): Config {
     },
     http: {
       bodyLimitBytes: e.BODY_LIMIT_BYTES,
-      trustedProxyHops: e.TRUSTED_PROXY_HOPS,
+      clientIp: e.CLIENT_IP,
       rateLimit: {
         windowMs: e.RATE_LIMIT_WINDOW_MS,
         verify: e.RATE_LIMIT_VERIFY,

apps/facilitator/src/http/app.ts

@@ -9,6 +9,7 @@ import type { SettlementHooks } from "../settlement/hooks.js";
 import type { Settle } from "../settlement/settle.js";
 import type { Verify, VerifyTrace } from "../settlement/verify.js";
 import { EXTENSION_RESPONSES_HEADER, encodeExtensionResponses } from "./extension-responses.js";
+import { rateLimit } from "./rate-limit.js";
 
 /** What the composition root hands to the HTTP adapter. Lanes add their services here. */
 export interface AppDeps {
@@ -39,14 +40,28 @@ const notImplemented: RequestHandler = (_req, res) => {
  */
 export const MAX_BODY_DEPTH = 256;
 
-/** Whether `value` nests deeper than `max` arrays and objects, counted without recursion. */
-function nestsDeeper(value: unknown, max: number): boolean {
+/**
+ * A NUL, or a lone UTF-16 surrogate (`\p{Cs}` matches only an unpaired one under the `u` flag).
+ * PostgreSQL refuses a NUL in `text` and `jsonb`, and `jsonb` a lone surrogate, so either would
+ * fail the settlement's store write with a 500.
+ */
+const UNSTORABLE_TEXT = /[\0\p{Cs}]/u;
+
+/**
+ * Whether `value` nests deeper than `max` arrays and objects, or holds a string or key matching
+ * {@link UNSTORABLE_TEXT}; walked without recursion.
+ */
+function refusedBody(value: unknown, max: number): boolean {
   const stack: [unknown, number][] = [[value, 0]];
   while (stack.length > 0) {
     const [item, depth] = stack.pop()!;
+    if (typeof item === "string" && UNSTORABLE_TEXT.test(item)) return true;
     if (item === null || typeof item !== "object") continue;
     if (depth >= max) return true;
-    for (const child of Object.values(item)) stack.push([child, depth + 1]);
+    for (const [key, child] of Object.entries(item)) {
+      if (UNSTORABLE_TEXT.test(key)) return true;
+      stack.push([child, depth + 1]);
+    }
   }
   return false;
 }
@@ -57,11 +72,12 @@ function nestsDeeper(value: unknown, max: number): boolean {
  *
  * The schema takes any non-empty `amount`, and upstream's `BigInt(amount)` throws on anything but
  * an integer, which it reports as `unexpected_verify_error`: our 500. So `amount` must be digits.
- * A body nested deeper than {@link MAX_BODY_DEPTH} is a 400 too, before anything recurses on it.
+ * A body nested deeper than {@link MAX_BODY_DEPTH} is a 400 too, before anything recurses on it,
+ * and so is a NUL or a lone surrogate anywhere in it.
  */
 export function parsePaymentRequest(body: unknown): VerifyRequest | undefined {
   if (typeof body !== "object" || body === null) return undefined;
-  if (nestsDeeper(body, MAX_BODY_DEPTH)) return undefined;
+  if (refusedBody(body, MAX_BODY_DEPTH)) return undefined;
   const { x402Version, paymentPayload, paymentRequirements } = body as Record<string, unknown>;
   const requirements = PaymentRequirementsV2Schema.safeParse(paymentRequirements);
   const valid =
@@ -74,23 +90,34 @@ export function parsePaymentRequest(body: unknown): VerifyRequest | undefined {
 
 /**
  * The inbound HTTP adapter (AD-13). `/discovery/resources` answers 501 until its task lands
- * (0032). There is a body size limit and
- * no CORS anywhere (spine "HTTP hardening"); rate limits come in 0023.
+ * (0032). There is a body size limit and no CORS anywhere (spine "HTTP hardening"). `/verify`,
+ * `/settle` and `/discovery/*` each have a rate limit (AD-11), applied before the body is read, so
+ * a malformed or oversized request counts too; `/supported` has none.
  */
 export function createApp({
   config,
   logger,
+  stores,
   verify,
   settle,
   extensionResponses,
   supported,
 }: AppDeps): Express {
   const app = express();
   app.disable("x-powered-by");
-  app.use(express.json({ limit: config.http.bodyLimitBytes }));
+  const limit = (route: string, budget: number) =>
+    rateLimit({
+      route,
+      limit: budget,
+      windowMs: config.http.rateLimit.windowMs,
+      clientIp: config.http.clientIp,
+      store: stores.rateLimits,
+      logger,
+    });
+  const json = express.json({ limit: config.http.bodyLimitBytes });
 
   // A verify outcome, valid or not, is a 200 with an x402 body; a thrown error is a 500.
-  app.post("/verify", async (req, res) => {
+  app.post("/verify", limit("verify", config.http.rateLimit.verify), json, async (req, res) => {
     const request = parsePaymentRequest(req.body);
     if (!request) {
       res.status(400).json({ error: code(400) });
@@ -112,7 +139,7 @@ export function createApp({
   });
   // Like `/verify`: any settle outcome is a 200 with an x402 body (AD-6). A thrown error, such as
   // upstream verify failing or the store down before the claim, is a 500.
-  app.post("/settle", async (req, res) => {
+  app.post("/settle", limit("settle", config.http.rateLimit.settle), json, async (req, res) => {
     const request = parsePaymentRequest(req.body);
     if (!request) {
       res.status(400).json({ error: code(400) });
@@ -157,6 +184,7 @@ export function createApp({
   app.get("/supported", (_req, res) => {
     res.json(supported);
   });
+  app.use("/discovery", limit("discovery", config.http.rateLimit.discovery));
   app.get("/discovery/resources", notImplemented);
 
   app.use((_req, res) => {

apps/facilitator/src/http/rate-limit.ts

@@ -0,0 +1,73 @@
+import { isIP } from "node:net";
+import type { RequestHandler } from "express";
+import { ipKeyGenerator } from "express-rate-limit";
+import type { Logger } from "../logger.js";
+import type { RateLimitStore } from "../ports/index.js";
+
+export interface RateLimit {
+  /** The route's name in the counter's key and the logs: `verify`, `settle`, `discovery`. */
+  route: string;
+  /** Requests per window and client. */
+  limit: number;
+  windowMs: number;
+  /** `cloudflare`: the `CF-Connecting-IP` header; `socket`: the connection's own address. */
+  clientIp: "cloudflare" | "socket";
+  store: RateLimitStore;
+  logger: Logger;
+}
+
+/**
+ * One route's budget (AD-11): a fixed window per client, counted in `RateLimitStore`, and a 429
+ * past it. Every request counts, a malformed one too. An IPv6 client is counted per /56, so one
+ * /64 can't rotate past the budget. A store error fails the request with a 500.
+ *
+ * A client over its budget is remembered in the process until its window ends and answered
+ * without a store hit, so its flood can't hold the store's few connections from everyone else.
+ */
+export function rateLimit({
+  route,
+  limit,
+  windowMs,
+  clientIp,
+  store,
+  logger,
+}: RateLimit): RequestHandler {
+  /** Clients over budget, with the end of their window in milliseconds since the epoch. */
+  const blocked = new Map<string, number>();
+
+  return async (req, res, next) => {
+    const ip = clientIp === "cloudflare" ? req.get("cf-connecting-ip") : req.socket.remoteAddress;
+    if (!ip || isIP(ip) === 0) {
+      // Only `cloudflared` reaches the service, and Cloudflare sets the header to one address on
+      // every request. Anything else never reaches the store; the value is the caller's, so it isn't
+      // logged.
+      logger.error(ip ? "client_ip_invalid" : "client_ip_missing", {
+        route,
+        source: clientIp,
+      });
+      res.status(500).json({ error: "internal_error" });
+      return;
+    }
+    const client = ipKeyGenerator(ip);
+    const now = Date.now();
+
+    let resetAt = blocked.get(client);
+    if (resetAt === undefined || resetAt <= now) {
+      // A store error rejects, and Express answers it with a 500.
+      const hit = await store.hit(`${route}:ip:${client}`, windowMs, now);
+      if (hit.count <= limit) {
+        next();
+        return;
+      }
+      resetAt = hit.resetAt;
+      // Requests already in flight find the budget spent too; only the first one logs.
+      const logged = (blocked.get(client) ?? 0) > now;
+      for (const [key, end] of blocked) if (end <= now) blocked.delete(key);
+      blocked.set(client, resetAt);
+      // `ip` is the address to block at the edge; `client` is what was counted (an IPv6 /56).
+      if (!logged) logger.warn("rate_limited", { route, ip, client, limit, windowMs });
+    }
+    res.setHeader("Retry-After", String(Math.max(1, Math.ceil((resetAt - now) / 1000))));
+    res.status(429).json({ error: "too_many_requests" });
+  };
+}

apps/facilitator/test/build-verifier.test.ts

@@ -46,6 +46,7 @@ const CURRENT = 1_000;
 
 const config = parseConfig({
   FACILITATOR_SECRET: secret,
+  CLIENT_IP: "socket",
   CHANNELS: channel,
   RPC_URLS: "https://rpc-a.example,https://rpc-b.example",
   MAX_FEE_STROOPS: "300000",