diff --git a/apps/core/src/constants/event-bus.constant.ts b/apps/core/src/constants/event-bus.constant.ts index ad9c595da83..de642c1f1ce 100644 --- a/apps/core/src/constants/event-bus.constant.ts +++ b/apps/core/src/constants/event-bus.constant.ts @@ -1,6 +1,7 @@ export enum EventBusEvents { EmailInit = 'email.init', TokenExpired = 'token.expired', + OwnerRolesChanged = 'auth.owner-roles-changed', CleanAggregateCache = 'cache.aggregate', SystemException = 'system.exception', ConfigChanged = 'config.changed', diff --git a/apps/core/src/modules/auth/auth.service.ts b/apps/core/src/modules/auth/auth.service.ts index 15328a98ed7..4d070f79216 100644 --- a/apps/core/src/modules/auth/auth.service.ts +++ b/apps/core/src/modules/auth/auth.service.ts @@ -6,6 +6,7 @@ import { Injectable, InternalServerErrorException, } from '@nestjs/common' +import { EventEmitter2 } from '@nestjs/event-emitter' import { hashPassword } from 'better-auth/crypto' import { omit } from 'es-toolkit/compat' import { customAlphabet } from 'nanoid' @@ -13,6 +14,7 @@ import { customAlphabet } from 'nanoid' import { SECRET_STORAGE_V2 } from '~/app.config' import { RequestContext } from '~/common/contexts/request.context' import { AppErrorCode, createAppException } from '~/common/errors' +import { EventBusEvents } from '~/constants/event-bus.constant' import { alphabet } from '~/constants/other.constant' import { SnowflakeService } from '~/shared/id/snowflake.service' import { getAvatar } from '~/utils/tool.util' @@ -49,6 +51,7 @@ export class AuthService { @Inject(AuthInstanceInjectKey) private readonly authInstance: InjectAuthInstance, private readonly snowflakeService: SnowflakeService, + private readonly eventEmitter: EventEmitter2, ) {} private normalizeOptional(value?: string | null) { @@ -375,7 +378,7 @@ export class AuthService { if (reader?.bannedAt) { return null } - if (reader?.role && !sessionUser.role) { + if (reader?.role) { sessionUser = { ...sessionUser, role: reader.role } } } @@ -418,6 +421,7 @@ export class AuthService { await this.readerRepository.setOwnersExceptToReader(target.id) await this.readerRepository.setRole(target.id, 'owner') + this.eventEmitter.emit(EventBusEvents.OwnerRolesChanged) const ownerCount = await this.readerRepository.countOwners() if (ownerCount !== 1) { @@ -445,6 +449,7 @@ export class AuthService { } await this.readerRepository.setRole(target.id, 'reader') + this.eventEmitter.emit(EventBusEvents.OwnerRolesChanged) return 'OK' } diff --git a/apps/core/src/modules/backup/backup.service.ts b/apps/core/src/modules/backup/backup.service.ts index 774850f569c..6f60af4e9ef 100644 --- a/apps/core/src/modules/backup/backup.service.ts +++ b/apps/core/src/modules/backup/backup.service.ts @@ -9,6 +9,7 @@ import { } from 'node:fs/promises' import { tmpdir } from 'node:os' import path, { join, resolve } from 'node:path' +import { Readable } from 'node:stream' import { Injectable, @@ -114,6 +115,9 @@ export class BackupService { const zip = await new JSZip().loadAsync(buffer) const entries = Object.values(zip.files) + if (entries.length > 10_000) + throw new Error('Backup exceeds entry count limit') + let expandedBytes = 0 for (const entry of entries) { const name = entry.name if (path.isAbsolute(name) || /^[a-z]:[/\\]/i.test(name)) { @@ -139,8 +143,23 @@ export class BackupService { continue } await mkdir(path.dirname(target), { recursive: true }) - const content = await entry.async('nodebuffer') - await writeFile(target, Uint8Array.from(content)) + async function* boundedContent() { + let entryBytes = 0 + for await (const chunk of new Readable().wrap( + entry.nodeStream('nodebuffer'), + )) { + entryBytes += chunk.length + expandedBytes += chunk.length + if ( + entryBytes > 256 * 1024 * 1024 || + expandedBytes > 1024 * 1024 * 1024 + ) { + throw new Error('Backup exceeds expanded size limit') + } + yield chunk + } + } + await writeFile(target, boundedContent()) } } @@ -437,20 +456,18 @@ export class BackupService { const backupDataDirFilenames = await readdir(backupDataDir) - await Promise.all( - backupDataDirFilenames.map(async (filename) => { - // Containment: source must stay within the extracted backup_data dir, - // destination within DATA_DIR. Rejects `..`/absolute filenames. - const fullpath = this.assertContained(backupDataDir, filename) - const targetPath = this.assertContained(DATA_DIR, filename) + for (const filename of backupDataDirFilenames) { + // Containment: source must stay within the extracted backup_data dir, + // destination within DATA_DIR. Rejects `..`/absolute filenames. + const fullpath = this.assertContained(backupDataDir, filename) + const targetPath = this.assertContained(DATA_DIR, filename) - await rm(targetPath, { recursive: true, force: true }) + await rm(targetPath, { recursive: true, force: true }) - await $throw( - `cp -r ${this.shellQuote(fullpath)} ${this.shellQuote(targetPath)}`, - ) - }), - ) + await $throw( + `cp -r ${this.shellQuote(fullpath)} ${this.shellQuote(targetPath)}`, + ) + } // SECURITY: never auto-install dependencies from a restored archive. // An uploaded archive is attacker-controlled; running `npm install` on its diff --git a/apps/core/src/modules/comment/comment.interceptor.ts b/apps/core/src/modules/comment/comment.interceptor.ts index 0913550b72a..94e17cd6d0e 100644 --- a/apps/core/src/modules/comment/comment.interceptor.ts +++ b/apps/core/src/modules/comment/comment.interceptor.ts @@ -4,53 +4,39 @@ import type { NestInterceptor, } from '@nestjs/common' import { Injectable } from '@nestjs/common' -import { isNotNil } from 'es-toolkit' -import { cloneDeep, isArrayLike, isObjectLike } from 'es-toolkit/compat' import { map } from 'rxjs' import { getNestExecutionContextRequest } from '~/transformers/get-req.transformer' import { getAvatar } from '~/utils/tool.util' +import { CommentViews } from './comment.views' + @Injectable() export class CommentFilterEmailInterceptor implements NestInterceptor { intercept(context: ExecutionContext, next: CallHandler) { const request = this.getRequest(context) - // Already authenticated — skip filtering - const isAuthenticated = request.user - if (isAuthenticated) { - return next.handle() - } - return next.handle().pipe( - map(function handle(data: any) { - if (!data) { - return data - } - try { - if (isArrayLike(data?.data)) { - data?.data?.forEach((item: any, i: number) => { - if (isNotNil(item.mail)) { - data.data[i].avatar = getAvatar(item.mail) - delete data.data[i].mail - } - if (item.replies) { - handle({ data: data.data[i].replies }) - } - }) - } - - if (isObjectLike(data)) { - Reflect.deleteProperty(data, 'mail') - } + if (request.hasAdminAccess ?? request.isAuthenticated) return next.handle() + return next.handle().pipe(map((data) => this.projectPublic(data))) + } - return cloneDeep(data) - } catch (error) { - if (isDev) { - console.error(error) - } - return cloneDeep(data) - } - }), - ) + private projectPublic(value: any): any { + if (!value || typeof value !== 'object' || value instanceof Date) + return value + if (Array.isArray(value)) + return value.map((item) => this.projectPublic(item)) + if ('id' in value && 'text' in value && 'refType' in value) { + return CommentViews.public.parse({ + ...value, + avatar: value.avatar || (value.mail ? getAvatar(value.mail) : null), + replies: value.replies?.map((item: any) => this.projectPublic(item)), + children: value.children?.map((item: any) => this.projectPublic(item)), + parent: value.parent, + }) + } + // Keep the explicit-envelope symbol and pagination metadata intact. + for (const key of Object.keys(value)) + value[key] = this.projectPublic(value[key]) + return value } get getRequest() { diff --git a/apps/core/src/modules/comment/comment.views.ts b/apps/core/src/modules/comment/comment.views.ts index 0bab0193611..0e3f64d4135 100644 --- a/apps/core/src/modules/comment/comment.views.ts +++ b/apps/core/src/modules/comment/comment.views.ts @@ -87,6 +87,57 @@ const AuthorActivityItemSchema = z.object({ }) export const CommentViews = { + public: CommentDetailSchema.omit({ + mail: true, + ip: true, + agent: true, + location: true, + state: true, + }) + .extend({ + ref: CommentRefSummarySchema.strip().nullable().optional(), + moderation: z + .object({ + status: z.enum(['published', 'pending', 'rejected']), + reviewer: z.enum(['ai', 'owner']).optional(), + receipt: z.string().optional(), + }) + .optional(), + replyWindow: z + .object({ + total: z.number(), + returned: z.number(), + threshold: z.number(), + hasHidden: z.boolean(), + hiddenCount: z.number(), + nextCursor: z.string().optional(), + }) + .optional(), + anchor: z.unknown().optional(), + reader: z + .object({ + id: z.string(), + isMember: z.boolean().optional(), + name: z.string().nullable().optional(), + handle: z.string().nullable().optional(), + username: z.string().nullable().optional(), + displayUsername: z.string().nullable().optional(), + image: z.string().nullable().optional(), + membership: z + .object({ + status: z.string(), + plan: z.string(), + }) + .nullable() + .optional(), + }) + .nullable() + .optional(), + replies: z.array(z.unknown()).optional(), + children: z.array(z.unknown()).optional(), + parent: CommentParentPreviewSchema.nullable().optional(), + }) + .strip(), moderation: z.object({ status: z.enum(['published', 'pending', 'rejected']), reviewer: z.enum(['ai', 'owner']).optional(), diff --git a/apps/core/src/modules/enrichment/providers/open-graph/browser-fetch.service.ts b/apps/core/src/modules/enrichment/providers/open-graph/browser-fetch.service.ts index e6f5498cd81..d162981ec87 100644 --- a/apps/core/src/modules/enrichment/providers/open-graph/browser-fetch.service.ts +++ b/apps/core/src/modules/enrichment/providers/open-graph/browser-fetch.service.ts @@ -252,16 +252,20 @@ export class BrowserFetchService { } } - private buildBaseArgs(slot: PoolSlot): string[] { + private async buildBaseArgs(slot: PoolSlot): Promise { return [ '--session', slot.name, + '--proxy', + await this.pool.getProxyUrl(), + '--proxy-bypass', + '<-loopback>', '--user-agent', OG_USER_AGENT, '--headers', JSON.stringify({ 'Accept-Language': OG_ACCEPT_LANGUAGE }), '--args', - OG_LAUNCH_ARGS, + `${OG_LAUNCH_ARGS},--disable-quic,--force-webrtc-ip-handling-policy=disable_non_proxied_udp`, ] } @@ -275,7 +279,7 @@ export class BrowserFetchService { const { stdout } = await execFileAsync( executable, [ - ...this.buildBaseArgs(slot), + ...(await this.buildBaseArgs(slot)), 'batch', '--bail', '--json', @@ -303,7 +307,7 @@ export class BrowserFetchService { const { stdout } = await execFileAsync( executable, [ - ...this.buildBaseArgs(slot), + ...(await this.buildBaseArgs(slot)), 'batch', '--bail', '--json', @@ -334,7 +338,7 @@ export class BrowserFetchService { const res = await execFileAsync( executable, [ - ...this.buildBaseArgs(slot), + ...(await this.buildBaseArgs(slot)), 'network', 'requests', '--filter', @@ -416,7 +420,7 @@ export class BrowserFetchService { await execFileAsync( executable, [ - ...this.buildBaseArgs(slot), + ...(await this.buildBaseArgs(slot)), 'set', 'viewport', String(SCREENSHOT_VIEWPORT_WIDTH), @@ -433,7 +437,7 @@ export class BrowserFetchService { await execFileAsync( executable, [ - ...this.buildBaseArgs(slot), + ...(await this.buildBaseArgs(slot)), 'screenshot', '--screenshot-format', SCREENSHOT_CLI_FORMAT, diff --git a/apps/core/src/modules/enrichment/providers/open-graph/safe-fetch.ts b/apps/core/src/modules/enrichment/providers/open-graph/safe-fetch.ts index 7c7f190b08a..cbdd635b855 100644 --- a/apps/core/src/modules/enrichment/providers/open-graph/safe-fetch.ts +++ b/apps/core/src/modules/enrichment/providers/open-graph/safe-fetch.ts @@ -1,8 +1,7 @@ -import { isDev } from '~/global/env.global' -import { - assertHostnameSafe, - parseAndValidateUrl, -} from '~/processors/agent-browser/url-guard' +import { fetch, type Response } from 'undici' + +import { parseAndValidateUrl } from '~/processors/agent-browser/url-guard' +import { publicHttpDispatcher } from '~/utils/ssrf.util' export { isPrivateIp, @@ -44,7 +43,7 @@ export class FetchSizeExceededError extends Error { * Hardened HTTP fetcher for arbitrary user-supplied URLs. Defends against: * * - non-http(s) schemes - * - private/loopback/link-local DNS resolution (best-effort; TOCTOU window) + * - private/loopback/link-local DNS resolution at socket connection * - oversized response bodies (streamed truncation) * - slow / hung peers (AbortSignal timeout) * - redirect loops + cross-host redirects to internal targets @@ -61,7 +60,6 @@ export async function safeFetch( let currentUrl = rawUrl for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { const parsed = parseAndValidateUrl(currentUrl) - if (!isDev) await assertHostnameSafe(parsed.hostname) const ac = new AbortController() const timer = setTimeout(() => ac.abort(), opts.timeoutMs) @@ -69,53 +67,55 @@ export async function safeFetch( let res: Response try { res = await fetch(parsed, { + dispatcher: publicHttpDispatcher, method: 'GET', redirect: 'manual', signal: ac.signal, headers: buildRequestHeaders(accept, opts.userAgent ?? DEFAULT_UA, hop), }) - } finally { - clearTimeout(timer) - } - if (isRedirect(res.status)) { - const loc = res.headers.get('location') - if (!loc) - throw new Error(`Redirect without location header (${res.status})`) - try { - await res.body?.cancel() - } catch { - // ignore — the body will be GC'd + if (isRedirect(res.status)) { + const loc = res.headers.get('location') + if (!loc) + throw new Error(`Redirect without location header (${res.status})`) + try { + await res.body?.cancel() + } catch { + // ignore — the body will be GC'd + } + currentUrl = new URL(loc, parsed).toString() + continue } - currentUrl = new URL(loc, parsed).toString() - continue - } - if (!res.ok) { - try { - await res.body?.cancel() - } catch { - // ignore + if (!res.ok) { + try { + await res.body?.cancel() + } catch { + // ignore + } + throw new Error(`HTTP ${res.status} for ${parsed.toString()}`) } - throw new Error(`HTTP ${res.status} for ${parsed.toString()}`) - } - const contentType = (res.headers.get('content-type') || '').toLowerCase() - if (!accept.some((prefix) => contentType.startsWith(prefix))) { - try { - await res.body?.cancel() - } catch { - // ignore + const contentType = (res.headers.get('content-type') || '').toLowerCase() + if (!accept.some((prefix) => contentType.startsWith(prefix))) { + try { + await res.body?.cancel() + } catch { + // ignore + } + throw new Error(`Unsupported content-type: ${contentType || '∅'}`) } - throw new Error(`Unsupported content-type: ${contentType || '∅'}`) - } - const { body, truncated } = await readWithLimit(res, opts.maxBodyBytes) - return { - finalUrl: parsed.toString(), - contentType, - body, - truncated, + const { body, truncated } = await readWithLimit(res, opts.maxBodyBytes) + return { + finalUrl: parsed.toString(), + contentType, + body, + truncated, + } + } finally { + ac.abort() + clearTimeout(timer) } } diff --git a/apps/core/src/modules/file/file-reference.repository.ts b/apps/core/src/modules/file/file-reference.repository.ts index d65c34460a3..715725f78b6 100644 --- a/apps/core/src/modules/file/file-reference.repository.ts +++ b/apps/core/src/modules/file/file-reference.repository.ts @@ -12,6 +12,7 @@ import { sql, } from 'drizzle-orm' +import { AppErrorCode, createAppException } from '~/common/errors' import { PG_DB_TOKEN } from '~/constants/system.constant' import { aiTtsBlocks, fileReferences, fileUsages } from '~/database/schema' import { @@ -48,6 +49,21 @@ const mapRow = (row: typeof fileReferences.$inferSelect): FileReferenceRow => ({ @Injectable() export class FileReferenceRepository extends BaseRepository { + async withReaderUploadLock( + readerId: string, + upload: () => Promise, + ): Promise { + return this.db.transaction(async (tx) => { + const result = await tx.execute( + sql`SELECT pg_try_advisory_xact_lock(hashtextextended(${`reader-upload:${readerId}`}, 0)) AS acquired`, + ) + if (!result.rows[0]?.acquired) { + throw createAppException(AppErrorCode.COMMENT_UPLOAD_RATE_LIMITED) + } + return upload() + }) + } + constructor( @Inject(PG_DB_TOKEN) db: AppDatabase, private readonly snowflake: SnowflakeService, diff --git a/apps/core/src/modules/file/file-reference.service.ts b/apps/core/src/modules/file/file-reference.service.ts index 418e7d4f752..c1d386f0ace 100644 --- a/apps/core/src/modules/file/file-reference.service.ts +++ b/apps/core/src/modules/file/file-reference.service.ts @@ -43,6 +43,10 @@ export interface ReaderImageDiff { @Injectable() export class FileReferenceService { + withReaderUploadLock(readerId: string, upload: () => Promise) { + return this.fileReferenceRepository.withReaderUploadLock(readerId, upload) + } + private readonly logger = new Logger(FileReferenceService.name) private readonly storageRoot: string diff --git a/apps/core/src/modules/file/reader-upload-quota.interceptor.ts b/apps/core/src/modules/file/reader-upload-quota.interceptor.ts index f2a384f936e..7a2abc5167c 100644 --- a/apps/core/src/modules/file/reader-upload-quota.interceptor.ts +++ b/apps/core/src/modules/file/reader-upload-quota.interceptor.ts @@ -4,7 +4,7 @@ import type { NestInterceptor, } from '@nestjs/common' import { Injectable, Logger, PayloadTooLargeException } from '@nestjs/common' -import { catchError } from 'rxjs' +import { catchError, defer, from, lastValueFrom } from 'rxjs' import { AppErrorCode, createAppException } from '~/common/errors' import { CommentRepository } from '~/modules/comment/comment.repository' @@ -77,27 +77,40 @@ export class ReaderUploadQuotaInterceptor implements NestInterceptor { } } - const hourlyLimit = config.readerHourlyUploadCount ?? 10 - if (hourlyLimit > 0) { - const since = new Date(Date.now() - 60 * 60 * 1000) - const count = await this.fileReferenceService.countReaderUploadsSince( - readerId, - since, - ) - if (count >= hourlyLimit) { - throw createAppException(AppErrorCode.COMMENT_UPLOAD_RATE_LIMITED) - } - } + return defer(() => + from( + this.fileReferenceService.withReaderUploadLock(readerId, async () => { + const hourlyLimit = config.readerHourlyUploadCount ?? 10 + if (hourlyLimit > 0) { + const since = new Date(Date.now() - 60 * 60 * 1000) + const count = + await this.fileReferenceService.countReaderUploadsSince( + readerId, + since, + ) + if (count >= hourlyLimit) { + throw createAppException(AppErrorCode.COMMENT_UPLOAD_RATE_LIMITED) + } + } - const totalBytesLimitMB = config.readerTotalActiveBytesMB ?? 50 - if (totalBytesLimitMB > 0) { - const totalBytes = - await this.fileReferenceService.sumReaderActiveBytes(readerId) - if (totalBytes >= totalBytesLimitMB * 1024 * 1024) { - throw createAppException(AppErrorCode.COMMENT_UPLOAD_QUOTA_EXCEEDED) - } - } + const totalBytesLimitMB = config.readerTotalActiveBytesMB ?? 50 + if (totalBytesLimitMB > 0) { + const totalBytes = + await this.fileReferenceService.sumReaderActiveBytes(readerId) + request.commentUploadMaxFileSize = Math.min( + request.commentUploadMaxFileSize!, + Math.max(0, totalBytesLimitMB * 1024 * 1024 - totalBytes), + ) + if (totalBytes >= totalBytesLimitMB * 1024 * 1024) { + throw createAppException( + AppErrorCode.COMMENT_UPLOAD_QUOTA_EXCEEDED, + ) + } + } - return handleUpload() + return lastValueFrom(handleUpload()) + }), + ), + ) } } diff --git a/apps/core/src/modules/link/link-avatar.service.ts b/apps/core/src/modules/link/link-avatar.service.ts index 96044e6c104..cdcc745661e 100644 --- a/apps/core/src/modules/link/link-avatar.service.ts +++ b/apps/core/src/modules/link/link-avatar.service.ts @@ -8,9 +8,10 @@ import type { FetchResponse } from 'ofetch' import { AppErrorCode, createAppException } from '~/common/errors' import { alphabet } from '~/constants/other.constant' import { HttpService } from '~/processors/helper/helper.http.service' +import { readResponseBuffer } from '~/utils/http-response.util' import { validateImageBuffer } from '~/utils/image.util' import { AsyncQueue } from '~/utils/queue.util' -import { assertPublicHttpUrl } from '~/utils/ssrf.util' +import { assertPublicHttpUrl, publicHttpDispatcher } from '~/utils/ssrf.util' import { ConfigsService } from '../configs/configs.service' import { FileService } from '../file/file.service' @@ -102,10 +103,16 @@ export class LinkAvatarService { return false } - let response: FetchResponse + let response: FetchResponse> + let buffer: Buffer try { - response = await this.http.fetch.raw(avatar, { - responseType: 'arrayBuffer', + response = await this.http.fetch.raw< + ReadableStream, + 'stream' + >(avatar, { + responseType: 'stream', + dispatcher: publicHttpDispatcher, + signal: AbortSignal.timeout(10_000), timeout: 10_000, redirect: 'error', retry: 0, @@ -115,6 +122,10 @@ export class LinkAvatarService { referer: refererHeader, }, }) + buffer = await readResponseBuffer( + new Response(response._data, { headers: response.headers }), + 8 * 1024 * 1024, + ) } catch (error: any) { this.logger.warn( `Failed to fetch friend link ${doc.id} avatar; skipping internalization: ${error?.message || String(error)}`, @@ -122,7 +133,6 @@ export class LinkAvatarService { return false } - const buffer = Buffer.from(response._data as ArrayBuffer) const contentType = response.headers.get('content-type') ?? undefined const normalizedContentType = this.normalizeMimeType(contentType) diff --git a/apps/core/src/modules/note/note.controller.ts b/apps/core/src/modules/note/note.controller.ts index f914a328fc1..1b64682defb 100644 --- a/apps/core/src/modules/note/note.controller.ts +++ b/apps/core/src/modules/note/note.controller.ts @@ -199,16 +199,15 @@ export class NoteController { const { password, single: isSingle } = query const visibleOnly = !isAuthenticated + if (!isAuthenticated && this.noteService.checkNoteIsSecret(current)) { + throw createAppException(AppErrorCode.NOTE_NOT_FOUND) + } + if (!isAuthenticated) { current.location = null current.coordinates = null } - current.text = - !isAuthenticated && this.noteService.checkNoteIsSecret(current) - ? '' - : current.text - if ( !(await this.noteService.checkPasswordToAccess(current.id, password)) && !isAuthenticated @@ -456,11 +455,17 @@ export class NoteController { throw createAppException(AppErrorCode.NOTE_NOT_FOUND, { id }) } - if (!isAuthenticated && !current.isPublished) { + if ( + !isAuthenticated && + (!current.isPublished || this.noteService.checkNoteIsSecret(current)) + ) { throw createAppException(AppErrorCode.NOTE_NOT_FOUND, { id }) } if (!isAuthenticated) { + if (!(await this.noteService.checkPasswordToAccess(id))) { + throw createAppException(AppErrorCode.NOTE_FORBIDDEN) + } current.location = null current.coordinates = null } @@ -515,7 +520,13 @@ export class NoteController { const { id } = params const currentDocument = await this.noteService.findById(id) - if (!currentDocument) { + if ( + !currentDocument || + (!isAuthenticated && + (!currentDocument.isPublished || + currentDocument.hasPassword || + this.noteService.checkNoteIsSecret(currentDocument))) + ) { return withMeta([], new MetaObjectBuilder().view('card').build()) } diff --git a/apps/core/src/modules/push/push.controller.ts b/apps/core/src/modules/push/push.controller.ts index fa0ed3b2bc7..f6e174cf951 100644 --- a/apps/core/src/modules/push/push.controller.ts +++ b/apps/core/src/modules/push/push.controller.ts @@ -2,6 +2,7 @@ import { Body, Post } from '@nestjs/common' import { ApiController } from '~/common/decorators/api-controller.decorator' import { CurrentReaderId } from '~/common/decorators/current-user.decorator' +import { HasAdminAccess } from '~/common/decorators/role.decorator' import { type PushActivationRequestDto, @@ -18,7 +19,8 @@ export class PushController { @Body({ schema: PushActivationRequestSchema }) body: PushActivationRequestDto, @CurrentReaderId() readerId?: string, + @HasAdminAccess() ownerAuthorized = false, ) { - return this.service.activate(readerId, body) + return this.service.activate(readerId, body, ownerAuthorized) } } diff --git a/apps/core/src/modules/push/push.service.ts b/apps/core/src/modules/push/push.service.ts index 4c3a0fe08a9..7b90b974f75 100644 --- a/apps/core/src/modules/push/push.service.ts +++ b/apps/core/src/modules/push/push.service.ts @@ -111,6 +111,7 @@ export class PushService implements OnModuleInit, OnModuleDestroy { async activate( readerId: string | undefined, input: PushActivationRequestDto, + ownerAuthorized = false, ) { PushSecretVault.assertConfigured() const relayUrl = resolveAllowedPushRelayOrigin(input.relayUrl) @@ -118,6 +119,10 @@ export class PushService implements OnModuleInit, OnModuleDestroy { const sourceOrigin = await this.sourceOrigin() const claimBody: Record = { ticket: input.activationTicket, + owner_authorized: ownerAuthorized, + ...(!existing && process.env.MX_PUSH_RELAY_ENROLLMENT_TOKEN + ? { enrollment_token: process.env.MX_PUSH_RELAY_ENROLLMENT_TOKEN } + : {}), source_origin: sourceOrigin, source_label: new URL(sourceOrigin).hostname, } diff --git a/apps/core/src/modules/subscribe/subscribe.controller.ts b/apps/core/src/modules/subscribe/subscribe.controller.ts index 0c805d6af7a..de8520d293e 100644 --- a/apps/core/src/modules/subscribe/subscribe.controller.ts +++ b/apps/core/src/modules/subscribe/subscribe.controller.ts @@ -1,4 +1,6 @@ import { Body, Delete, Get, Post, Query } from '@nestjs/common' +import { Throttle } from '@nestjs/throttler' +import { z } from 'zod' import { ApiController } from '~/common/decorators/api-controller.decorator' import { Auth } from '~/common/decorators/auth.decorator' @@ -46,7 +48,23 @@ export class SubscribeController { ) } + @Get('/confirm') + @HTTPDecorators.RawResponse + async confirm( + @Query({ + schema: z.object({ email: z.email(), token: z.string().max(256) }), + }) + query: { + email: string + token: string + }, + ) { + await this.service.confirm(query.email, query.token) + return 'Subscription confirmed' + } + @Post('/') + @Throttle({ default: { limit: 3, ttl: 60_000 } }) async subscribe(@Body({ schema: SubscribeSchema }) body: SubscribeDto) { if (!(await this.service.checkEnable())) { throw createAppException(AppErrorCode.SUBSCRIBE_NOT_ENABLED) diff --git a/apps/core/src/modules/subscribe/subscribe.service.ts b/apps/core/src/modules/subscribe/subscribe.service.ts index 3ea4d9e38c2..2c4c15e75ea 100644 --- a/apps/core/src/modules/subscribe/subscribe.service.ts +++ b/apps/core/src/modules/subscribe/subscribe.service.ts @@ -1,5 +1,10 @@ import cluster from 'node:cluster' -import { randomBytes } from 'node:crypto' +import { + createHash, + createHmac, + randomBytes, + timingSafeEqual, +} from 'node:crypto' import type { CoAction } from '@innei/next-async' import { Co } from '@innei/next-async' @@ -9,6 +14,7 @@ import ejs from 'ejs' import { LRUCache } from 'lru-cache' import type Mail from 'nodemailer/lib/mailer' +import { apiRoutePrefix } from '~/common/decorators/api-controller.decorator' import { AppErrorCode, createAppException } from '~/common/errors' import { BusinessEvents, EventScope } from '~/constants/business-event.constant' import { isMainProcess } from '~/global/env.global' @@ -17,6 +23,8 @@ import { EmailService } from '~/processors/helper/helper.email.service' import type { IEventManagerHandlerDisposer } from '~/processors/helper/helper.event.service' import { EventManagerService } from '~/processors/helper/helper.event.service' import { UrlBuilderService } from '~/processors/helper/helper.url-builder.service' +import { RedisService } from '~/processors/redis/redis.service' +import { getRedisKey } from '~/utils/redis.util' import { truncateAtBoundary } from '~/utils/text-summary.util' import { ConfigsService } from '../configs/configs.service' @@ -33,9 +41,6 @@ import { defaultSubscribeForRenderProps } from './subscribe.email.default' import { SubscribeRepository } from './subscribe.repository' import { SubscribeMailType } from './subscribe-mail.enum' -type Email = string -type SubscriberEntry = { subscribe: number; cancelToken: string } - @Injectable() export class SubscribeService implements OnModuleInit, OnModuleDestroy { constructor( @@ -46,10 +51,9 @@ export class SubscribeService implements OnModuleInit, OnModuleDestroy { private readonly urlBuilderService: UrlBuilderService, private readonly emailService: EmailService, private readonly ownerService: OwnerService, + private readonly redis: RedisService, ) {} - private subscribeMap = new Map() - public get repository() { return this.subscribeRepository } @@ -92,15 +96,6 @@ export class SubscribeService implements OnModuleInit, OnModuleDestroy { private async observeEvents() { if (!isMainProcess && cluster.isWorker && cluster.worker?.id !== 1) return - const docs = await this.subscribeRepository.findAll() - - for (const doc of docs) { - this.subscribeMap.set(doc.email, { - subscribe: doc.subscribe, - cancelToken: doc.cancelToken, - }) - } - // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this @@ -121,12 +116,11 @@ export class SubscribeService implements OnModuleInit, OnModuleDestroy { const { serverUrl } = await self.configService.get('url') const detailLink = await self.urlBuilderService.buildWithBaseUrl(noteOrPost) - for (const [ - email, - { subscribe, cancelToken }, - ] of self.subscribeMap.entries()) { + const subscribers = await self.subscribeRepository.findAll() + for (const { email, subscribe, cancelToken, verified } of subscribers) { + if (!verified) continue if (!(subscribe & subscribeBit)) continue - const unsubscribeLink = `${serverUrl}/subscribe/unsubscribe?email=${encodeURIComponent(email)}&cancelToken=${encodeURIComponent(cancelToken)}` + const unsubscribeLink = `${serverUrl.replace(/\/$/, '')}${apiRoutePrefix}/subscribe/unsubscribe?email=${encodeURIComponent(email)}&cancelToken=${encodeURIComponent(cancelToken)}` self.sendEmail( email, { @@ -195,20 +189,72 @@ export class SubscribeService implements OnModuleInit, OnModuleDestroy { } async subscribe(email: string, subscribe: number) { - const isExist = await this.subscribeRepository.findByEmail(email) - let cancelToken: string - if (isExist) { - await this.subscribeRepository.updateByEmail(email, { subscribe }) - cancelToken = isExist.cancelToken - } else { - cancelToken = String(this.createCancelToken(email)) - await this.subscribeRepository.create({ + email = email.trim().toLowerCase() + const emailHash = createHash('sha256').update(email).digest('hex') + const admitted = await this.redis + .getClient() + .set(getRedisKey(`subscribe:confirm:${emailHash}`), '1', 'EX', 60, 'NX') + if (!admitted) return + let subscriber = await this.subscribeRepository.findByEmail(email) + if (!subscriber) { + subscriber = await this.subscribeRepository.create({ email, - cancelToken, - subscribe, + cancelToken: this.createCancelToken(email), + subscribe: 0, + verified: false, }) } - this.subscribeMap.set(email, { subscribe, cancelToken }) + const expires = Date.now() + 15 * 60_000 + const payload = `${expires}.${subscribe}` + const signature = createHmac('sha256', subscriber.cancelToken) + .update(`${email}.${payload}`) + .digest('hex') + const { serverUrl } = await this.configService.get('url') + const { mailOptions } = await this.configService.waitForConfigReady() + const url = new URL( + `${serverUrl.replace(/\/$/, '')}${apiRoutePrefix}/subscribe/confirm`, + ) + url.searchParams.set('email', email) + url.searchParams.set('token', `${payload}.${signature}`) + await this.emailService.send({ + from: mailOptions.from || mailOptions.smtp?.user, + to: email, + subject: 'Confirm your email subscription', + text: `Confirm your subscription or preference change within 15 minutes:\n${url.toString()}\nIgnore this email if you did not request it.`, + }) + } + + async confirm(email: string, token: string) { + email = email.trim().toLowerCase() + const subscriber = await this.subscribeRepository.findByEmail(email) + const [expires, bit, signature] = token.split('.') + const subscribe = Number(bit) + const allowedBits = Object.values(SubscribeTypeToBitMap).reduce( + (a, b) => a | b, + 0, + ) + if ( + !subscriber || + !/^\d+$/.test(expires) || + Number(expires) <= Date.now() || + !Number.isSafeInteger(subscribe) || + subscribe <= 0 || + subscribe > allowedBits || + (subscribe & ~allowedBits) !== 0 || + !/^[\da-f]{64}$/.test(signature ?? '') + ) { + throw createAppException(AppErrorCode.INVALID_PARAMETER) + } + const expected = createHmac('sha256', subscriber.cancelToken) + .update(`${email}.${expires}.${bit}`) + .digest() + if (!timingSafeEqual(expected, Buffer.from(signature, 'hex'))) + throw createAppException(AppErrorCode.INVALID_PARAMETER) + await this.subscribeRepository.updateByEmail(email, { + subscribe, + verified: true, + }) + return true } async unsubscribe(email: string, token: string) { @@ -216,7 +262,6 @@ export class SubscribeService implements OnModuleInit, OnModuleDestroy { if (!model) return false if (model.cancelToken === token) { await this.subscribeRepository.deleteByEmail(email) - this.subscribeMap.delete(email) return true } return false @@ -225,15 +270,11 @@ export class SubscribeService implements OnModuleInit, OnModuleDestroy { async unsubscribeBatch(emails?: string[], all?: boolean) { if (all) { const count = await this.subscribeRepository.deleteAll() - this.subscribeMap.clear() return count } if (!emails?.length) return 0 const count = await this.subscribeRepository.deleteByEmails(emails) - for (const email of emails) { - this.subscribeMap.delete(email) - } return count } diff --git a/apps/core/src/modules/webhook/webhook.service.ts b/apps/core/src/modules/webhook/webhook.service.ts index d5acf773a84..564eac78453 100644 --- a/apps/core/src/modules/webhook/webhook.service.ts +++ b/apps/core/src/modules/webhook/webhook.service.ts @@ -12,7 +12,7 @@ import { EventManagerService } from '~/processors/helper/helper.event.service' import { EventPayloadEnricherService } from '~/processors/helper/helper.event-payload.service' import { HttpService } from '~/processors/helper/helper.http.service' import type { BasicPagerDto } from '~/shared/dto/pager.dto' -import { isPrivateIp } from '~/utils/ssrf.util' +import { isPrivateIp, publicHttpDispatcher } from '~/utils/ssrf.util' import { WebhookRepository } from './webhook.repository' import type { WebhookModel, WebhookRow } from './webhook.types' @@ -185,6 +185,7 @@ export class WebhookService implements OnModuleInit, OnModuleDestroy { try { const response = await this.httpService.fetch.raw(webhook.payloadUrl, { method: 'POST', + dispatcher: publicHttpDispatcher, body: clonedPayload, headers, redirect: 'error', @@ -266,11 +267,7 @@ function isBlockedIP(rawIp: string): boolean { * webhook target that points at loopback / link-local / private / metadata * addresses. Returns the resolved IP that the request will actually hit. * - * NOTE: this is checked at delivery time (the real gate). A DNS-rebinding - * attacker can still pass this check and then re-point the hostname before the - * HTTP socket connects; `maxRedirects: 0` removes the redirect-based bypass but - * not rebinding. Pinning the connection to `resolvedIp` would be needed to - * fully close that window. + * Delivery also validates DNS in publicHttpDispatcher at socket connection time. */ async function assertWebhookTargetIsPublic(rawUrl: string): Promise { let url: URL diff --git a/apps/core/src/processors/agent-browser/agent-browser-pool.service.ts b/apps/core/src/processors/agent-browser/agent-browser-pool.service.ts index 47e662d6985..15da700e106 100644 --- a/apps/core/src/processors/agent-browser/agent-browser-pool.service.ts +++ b/apps/core/src/processors/agent-browser/agent-browser-pool.service.ts @@ -16,6 +16,10 @@ import { AGENT_BROWSER_DEFAULT_MAX_SIZE, describeAgentBrowserError, } from './agent-browser.constants' +import { + type BrowserEgressProxy, + createBrowserEgressProxy, +} from './browser-egress-proxy' const execFileAsync = promisify(execFile) @@ -77,6 +81,13 @@ export class AgentBrowserSessionPool implements OnModuleDestroy { private shuttingDown = false private readonly sessionPrefix = `agent-browser-${process.pid}-${randomUUID()}` private nextSession = 0 + private proxy?: Promise + + async getProxyUrl(): Promise { + if (this.shuttingDown) throw new Error('Browser pool shut down') + this.proxy ??= createBrowserEgressProxy() + return (await this.proxy).url + } constructor(@Optional() options?: AgentBrowserSessionPoolOptions) { const maxSize = options?.maxSize ?? AGENT_BROWSER_DEFAULT_MAX_SIZE @@ -190,6 +201,7 @@ export class AgentBrowserSessionPool implements OnModuleDestroy { if (this.inFlightCloses.size > 0) { await Promise.all(this.inFlightCloses) } + if (this.proxy) await (await this.proxy).close() } /** Mark before launching: a failed CLI may already have created its daemon. */ diff --git a/apps/core/src/processors/agent-browser/agent-browser.service.ts b/apps/core/src/processors/agent-browser/agent-browser.service.ts index 690346f52a9..fedb5ac829e 100644 --- a/apps/core/src/processors/agent-browser/agent-browser.service.ts +++ b/apps/core/src/processors/agent-browser/agent-browser.service.ts @@ -101,7 +101,7 @@ export class AgentBrowserService { try { slot = await this.pool.acquire({ signal: ac.signal }) this.pool.markLive(slot) - const baseArgs = this.buildBaseArgs(slot) + const baseArgs = await this.buildBaseArgs(slot) const finalUrl = await this.runNavigation( executable, baseArgs, @@ -176,10 +176,14 @@ export class AgentBrowserService { } } - private buildBaseArgs(slot: PoolSlot): string[] { + private async buildBaseArgs(slot: PoolSlot): Promise { return [ '--session', slot.name, + '--proxy', + await this.pool.getProxyUrl(), + '--proxy-bypass', + '<-loopback>', '--user-agent', AGENT_BROWSER_DEFAULT_UA, '--headers', @@ -187,7 +191,7 @@ export class AgentBrowserService { 'Accept-Language': AGENT_BROWSER_DEFAULT_ACCEPT_LANGUAGE, }), '--args', - AGENT_BROWSER_DEFAULT_LAUNCH_ARGS, + `${AGENT_BROWSER_DEFAULT_LAUNCH_ARGS},--disable-quic,--force-webrtc-ip-handling-policy=disable_non_proxied_udp`, ] } diff --git a/apps/core/src/processors/agent-browser/browser-egress-proxy.ts b/apps/core/src/processors/agent-browser/browser-egress-proxy.ts new file mode 100644 index 00000000000..cad54219fd8 --- /dev/null +++ b/apps/core/src/processors/agent-browser/browser-egress-proxy.ts @@ -0,0 +1,118 @@ +import { lookup } from 'node:dns/promises' +import { createServer, request as httpRequest } from 'node:http' +import { connect, isIP, type Socket } from 'node:net' + +import { isDev } from '~/global/env.global' + +import { + isPrivateIp, + parseAndValidateUrl, + stripIpv6Brackets, + UnsafeUrlError, +} from './url-guard' + +export interface BrowserEgressProxy { + url: string + close: () => Promise +} + +/** Chromium sends every navigation, redirect, subresource and WebSocket through this proxy. */ +export async function createBrowserEgressProxy(): Promise { + const sockets = new Set() + let closing = false + const track = (socket: Socket) => { + sockets.add(socket) + socket.on('error', () => socket.destroy()) + socket.setTimeout(30_000, () => socket.destroy()) + socket.once('close', () => sockets.delete(socket)) + return socket + } + const resolve = async (raw: string) => { + const url = parseAndValidateUrl(raw) + const host = stripIpv6Brackets(url.hostname) + const addresses = isIP(host) + ? [{ address: host, family: isIP(host) }] + : await lookup(host, { all: true }) + if ( + !addresses.length || + (!isDev && addresses.some((a) => isPrivateIp(a.address, a.family))) + ) + throw new UnsafeUrlError('Private browser destination') + return { url, address: addresses[0].address } + } + const server = createServer(async (incoming, outgoing) => { + try { + const { url, address } = await resolve(incoming.url || '') + if (closing || incoming.destroyed || outgoing.destroyed) return + if (url.protocol !== 'http:') + throw new UnsafeUrlError('Expected proxy HTTP URL') + const upstream = httpRequest( + { + hostname: address, + port: url.port || 80, + method: incoming.method, + path: `${url.pathname}${url.search}`, + headers: { ...incoming.headers, host: url.host }, + }, + (response) => { + outgoing.writeHead(response.statusCode || 502, response.headers) + response.pipe(outgoing) + }, + ) + upstream.on('socket', track) + upstream.on('error', () => { + if (!outgoing.headersSent) outgoing.writeHead(502) + outgoing.end() + }) + outgoing.on('close', () => upstream.destroy()) + incoming.pipe(upstream) + } catch { + outgoing.writeHead(403).end() + } + }) + server.maxConnections = 64 + server.on('connection', track) + server.on('connect', (request, client, head) => { + void (async () => { + try { + const { url, address } = await resolve(`https://${request.url}`) + if (closing || client.destroyed) return + const upstream = track( + connect({ host: address, port: Number(url.port || 443) }), + ) + upstream.on('error', () => client.destroy()) + client.on('error', () => upstream.destroy()) + client.on('close', () => upstream.destroy()) + upstream.on('close', () => client.destroy()) + upstream.once('connect', () => { + if (client.destroyed) { + upstream.destroy() + return + } + client.write('HTTP/1.1 200 Connection Established\r\n\r\n') + if (head.length) upstream.write(head) + client.pipe(upstream).pipe(client) + }) + } catch { + client.end('HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n') + } + })() + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + server.off('error', reject) + resolve() + }) + }) + const port = (server.address() as { port: number }).port + return { + url: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve, reject) => { + closing = true + for (const socket of sockets) socket.destroy() + server.close((error) => (error ? reject(error) : resolve())) + }), + } +} diff --git a/apps/core/src/processors/agent-browser/url-guard.ts b/apps/core/src/processors/agent-browser/url-guard.ts index 2d4b4866ff5..5ee2835971b 100644 --- a/apps/core/src/processors/agent-browser/url-guard.ts +++ b/apps/core/src/processors/agent-browser/url-guard.ts @@ -104,14 +104,32 @@ function isPrivateIpv4(addr: string): boolean { } function isPrivateIpv6(addr: string): boolean { - const lower = addr.toLowerCase() - if (lower === '::' || lower === '::1') return true - // IPv4-mapped (::ffff:a.b.c.d) and IPv4-compatible (::a.b.c.d) forms - const v4mapped = /^::(?:ffff:)?((?:\d+\.){3}\d+)$/i.exec(addr) - if (v4mapped) return isPrivateIpv4(v4mapped[1]) - if (lower.startsWith('fe8') || lower.startsWith('fe9')) return true - if (lower.startsWith('fea') || lower.startsWith('feb')) return true - if (lower.startsWith('fc') || lower.startsWith('fd')) return true - if (lower.startsWith('ff')) return true - return false + if (isIP(addr) !== 6 || addr.includes('%')) return true + const normalized = new URL(`http://[${addr}]/`).hostname.slice(1, -1) + const [left, right] = normalized.split('::') + const head = left ? left.split(':') : [] + const tail = right ? right.split(':') : [] + const words = + right === undefined + ? head + : [ + ...head, + ...Array.from({ length: 8 - head.length - tail.length }, () => '0'), + ...tail, + ] + const parts = words.map((word) => Number.parseInt(word, 16)) + if ( + parts.slice(0, 5).every((word) => word === 0) && + (parts[5] === 0 || parts[5] === 0xffff) + ) { + return isPrivateIpv4( + `${parts[6] >> 8}.${parts[6] & 255}.${parts[7] >> 8}.${parts[7] & 255}`, + ) + } + // Only globally routable unicast; reject tunneling ranges as well. + return ( + (parts[0] & 0xe000) !== 0x2000 || + parts[0] === 0x2002 || + (parts[0] === 0x2001 && parts[1] === 0) + ) } diff --git a/apps/core/src/processors/gateway/shared/auth.gateway.ts b/apps/core/src/processors/gateway/shared/auth.gateway.ts index 85a23cdf65e..6864860917d 100644 --- a/apps/core/src/processors/gateway/shared/auth.gateway.ts +++ b/apps/core/src/processors/gateway/shared/auth.gateway.ts @@ -30,6 +30,7 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { super(namespace, bus, presence) } + private roleGeneration = 0 tokenSocketIdMap = new Map>() private readonly socketIdTokenMap = new Map() @@ -55,6 +56,7 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { } async handleConnection(ws: WebSocket, request: IncomingMessage) { + const generation = this.roleGeneration const conn = this.trackConnection(ws) await this.presence.addConnection(namespace, conn.id) @@ -68,7 +70,11 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { const session = await this.authService.getSessionUserFromHeaders(headers) if (session?.user?.role === 'owner') { - return this.completeConnection(conn, session.session?.token) + return this.completeConnection( + conn, + session.session?.token, + generation, + ) } } @@ -76,8 +82,7 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { const headerAuthorization = request.headers.authorization const apiKey = (Array.isArray(headerApiKey) ? headerApiKey[0] : headerApiKey) || - headerAuthorization || - queryToken(request) + headerAuthorization if (!apiKey) { return this.authFailed(conn) } @@ -91,13 +96,18 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { return this.authFailed(conn) } - this.completeConnection(conn, result.id) + this.completeConnection(conn, result.id, generation) } // A close during the auth lookups already unbound this connection; binding // now would strand the token entry, since nothing unbinds it again. - private completeConnection(conn: WsConnection, token?: string) { + private completeConnection( + conn: WsConnection, + token: string | undefined, + generation: number, + ) { if (!this.resolveConnection(conn.ws)) return + if (generation !== this.roleGeneration) return this.authFailed(conn) this.sendConnectGreeting(conn) if (token) this.bindToken(token, conn) } @@ -122,7 +132,28 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { return closed } + @OnEvent(EventBusEvents.OwnerRolesChanged) + handleOwnerRolesChanged() { + this.closeForRoleChange() + this.bus.publish({ + ns: this.namespace, + event: EventBusEvents.OwnerRolesChanged, + }) + } + + private closeForRoleChange() { + this.roleGeneration++ + for (const conn of this.registry.all()) { + this.authFailed(conn) + void this.handleDisconnect(conn.ws) + } + } + protected override deliverLocal(frame: WsBusFrame) { + if (frame.event === EventBusEvents.OwnerRolesChanged) { + this.closeForRoleChange() + return + } if ( frame.event === EventBusEvents.TokenExpired && typeof frame.payload === 'string' @@ -149,14 +180,3 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { return AuthGateway } - -function queryToken(request: IncomingMessage): string | undefined { - try { - return ( - new URL(request.url ?? '', 'ws://localhost').searchParams.get('token') ?? - undefined - ) - } catch { - return undefined - } -} diff --git a/apps/core/src/processors/helper/helper.http.service.ts b/apps/core/src/processors/helper/helper.http.service.ts index 4634804e051..17d0c3865ab 100644 --- a/apps/core/src/processors/helper/helper.http.service.ts +++ b/apps/core/src/processors/helper/helper.http.service.ts @@ -3,7 +3,7 @@ import { inspect } from 'node:util' import { Injectable, Logger } from '@nestjs/common' import { $Fetch, createFetch, FetchOptions } from 'ofetch' import pc from 'picocolors' -import { Agent } from 'undici' +import { Agent, fetch } from 'undici' import { DEBUG_MODE } from '~/app.config' import { isDev } from '~/global/env.global' @@ -85,7 +85,10 @@ body: ${this.prettyStringify(ctx.options.body)}`, }, } - return createFetch({ defaults }) + return createFetch({ + defaults, + fetch: fetch as unknown as typeof globalThis.fetch, + }) } private prettyStringify(data: unknown) { diff --git a/apps/core/src/processors/helper/helper.image.service.ts b/apps/core/src/processors/helper/helper.image.service.ts index 84ea3a804ae..f2b9b2951d5 100644 --- a/apps/core/src/processors/helper/helper.image.service.ts +++ b/apps/core/src/processors/helper/helper.image.service.ts @@ -5,9 +5,10 @@ import { rgbaToThumbHash } from 'thumbhash' import { ConfigsService } from '~/modules/configs/configs.service' import type { ImageModel } from '~/shared/types/legacy-model.type' +import { readResponseBuffer } from '~/utils/http-response.util' import { pickImagesFromMarkdown } from '~/utils/pic.util' import { AsyncQueue } from '~/utils/queue.util' -import { assertPublicHttpUrl } from '~/utils/ssrf.util' +import { assertPublicHttpUrl, publicHttpDispatcher } from '~/utils/ssrf.util' import { requireDepsWithInstall } from '~/utils/tool.util' import { HttpService } from './helper.http.service' @@ -117,10 +118,12 @@ export class ImageService implements OnModuleInit { // disable redirect following so a public host cannot 30x us into one. await assertPublicHttpUrl(image, { allowHttp: true }) const response = await this.httpService.fetch.raw< - ArrayBuffer, - 'arrayBuffer' + ReadableStream, + 'stream' >(image, { - responseType: 'arrayBuffer', + responseType: 'stream', + dispatcher: publicHttpDispatcher, + signal: AbortSignal.timeout(10_000), timeout: 10_000, redirect: 'error', retry: 0, @@ -133,9 +136,12 @@ export class ImageService implements OnModuleInit { const imageType = response.headers.get('content-type') ?? '' - const buffer = Buffer.from(response._data as ArrayBuffer) + const buffer = await readResponseBuffer( + new Response(response._data, { headers: response.headers }), + 8 * 1024 * 1024, + ) const sharp = await requireDepsWithInstall('sharp') - const sharped = sharp(buffer) as Sharp + const sharped = sharp(buffer, { limitInputPixels: 40_000_000 }) as Sharp const metadata = await sharped.metadata() const size = { height: metadata.height, diff --git a/apps/core/src/utils/http-response.util.ts b/apps/core/src/utils/http-response.util.ts new file mode 100644 index 00000000000..82f6e91dfdb --- /dev/null +++ b/apps/core/src/utils/http-response.util.ts @@ -0,0 +1,24 @@ +/** Bound decoded bytes as well as Content-Length (which may describe compressed data). */ +export async function readResponseBuffer( + response: Response, + limit: number, +): Promise { + const reader = response.body?.getReader() + if (!reader) return Buffer.alloc(0) + try { + if (Number(response.headers.get('content-length')) > limit) + throw new Error(`Response exceeds ${limit} bytes`) + const chunks: Uint8Array[] = [] + let size = 0 + for (;;) { + const { value, done } = await reader.read() + if (done) return Buffer.concat(chunks, size) + size += value.byteLength + if (size > limit) throw new Error(`Response exceeds ${limit} bytes`) + chunks.push(value) + } + } finally { + await reader.cancel().catch(() => {}) + reader.releaseLock() + } +} diff --git a/apps/core/src/utils/s3.util.ts b/apps/core/src/utils/s3.util.ts index 37a094f0812..f606f16ca41 100644 --- a/apps/core/src/utils/s3.util.ts +++ b/apps/core/src/utils/s3.util.ts @@ -3,6 +3,8 @@ import type { Readable } from 'node:stream' import { isDev } from '~/global/env.global' +import { readResponseBuffer } from './http-response.util' + export interface S3UploaderOptions { bucket: string region: string @@ -593,6 +595,29 @@ export class S3Uploader { } } + private async fetchResponse( + url: string, + options: RequestInit, + ): Promise { + const response = await fetch(url, { + ...options, + redirect: 'error', + signal: AbortSignal.timeout(60_000), + }) + // S3 control/error responses are small; never buffer an arbitrary endpoint's body. + const bytes = await readResponseBuffer(response, 1024 * 1024) + return new Response( + options.method === 'HEAD' || [204, 205, 304].includes(response.status) + ? null + : new Uint8Array(bytes), + { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }, + ) + } + private async signedRequest(options: { method: string objectKey: string @@ -704,7 +729,7 @@ export class S3Uploader { } try { - return await fetch(requestUrl, fetchOptions) + return await this.fetchResponse(requestUrl, fetchOptions) } finally { if (isDev) { if (originalTlsReject === undefined) { @@ -813,7 +838,10 @@ export class S3Uploader { } try { - const response = await fetch(requestUrl, fetchOptions as RequestInit) + const response = await this.fetchResponse( + requestUrl, + fetchOptions as RequestInit, + ) if (response.status === 404) return if (!response.ok) { @@ -933,7 +961,10 @@ export class S3Uploader { try { let response: Response try { - response = await fetch(requestUrl, fetchOptions as RequestInit) + response = await this.fetchResponse( + requestUrl, + fetchOptions as RequestInit, + ) } catch (error) { throw createS3TransportError(error) } diff --git a/apps/core/src/utils/sandbox/sandbox-worker.runtime.js b/apps/core/src/utils/sandbox/sandbox-worker.runtime.js index ce0bc8e8af6..1a24850b386 100644 --- a/apps/core/src/utils/sandbox/sandbox-worker.runtime.js +++ b/apps/core/src/utils/sandbox/sandbox-worker.runtime.js @@ -51,39 +51,46 @@ function ipv4ToParts(ip) { function isPrivateIpv4(ip) { const p = ipv4ToParts(ip) - if (!p) return false + if (!p) return true const [a, b] = p if (a === 0) return true // 0.0.0.0/8 if (a === 10) return true // 10.0.0.0/8 RFC1918 if (a === 127) return true // 127.0.0.0/8 loopback if (a === 169 && b === 254) return true // 169.254.0.0/16 link-local (+ metadata 169.254.169.254) if (a === 172 && b >= 16 && b <= 31) return true // 172.16.0.0/12 RFC1918 + if (a === 192 && b === 0) return true + if (a === 198 && (b === 18 || b === 19)) return true if (a === 192 && b === 168) return true // 192.168.0.0/16 RFC1918 if (a === 100 && b >= 64 && b <= 127) return true // 100.64.0.0/10 CGNAT if (a >= 224) return true // 224.0.0.0/4 multicast + 240.0.0.0/4 reserved return false } -function isPrivateIpv6(ip) { - let addr = ip.toLowerCase() - if (addr.startsWith('[') && addr.endsWith(']')) addr = addr.slice(1, -1) - // strip zone id - const pct = addr.indexOf('%') - if (pct !== -1) addr = addr.slice(0, pct) - if (addr === '::1' || addr === '::') return true // loopback / unspecified +function isPrivateIpv6(addr) { + if (net.isIP(addr) !== 6 || addr.includes('%')) return true + const normalized = new URL(`http://[${addr}]/`).hostname.slice(1, -1) + const [left, right] = normalized.split('::') + const head = left ? left.split(':') : [] + const tail = right ? right.split(':') : [] + const words = + right === undefined + ? head + : [...head, ...Array(8 - head.length - tail.length).fill('0'), ...tail] + const parts = words.map((word) => Number.parseInt(word, 16)) if ( - addr.startsWith('fe8') || - addr.startsWith('fe9') || - addr.startsWith('fea') || - addr.startsWith('feb') + parts.slice(0, 5).every((word) => word === 0) && + (parts[5] === 0 || parts[5] === 0xffff) + ) { + return isPrivateIpv4( + `${parts[6] >> 8}.${parts[6] & 255}.${parts[7] >> 8}.${parts[7] & 255}`, + ) + } + // Only globally routable unicast; reject tunneling ranges as well. + return ( + (parts[0] & 0xe000) !== 0x2000 || + parts[0] === 0x2002 || + (parts[0] === 0x2001 && parts[1] === 0) ) - return true // fe80::/10 link-local - if (addr.startsWith('fc') || addr.startsWith('fd')) return true // fc00::/7 ULA - if (addr.startsWith('ff')) return true // ff00::/8 multicast - // IPv4-mapped / -compatible: ::ffff:a.b.c.d or ::a.b.c.d - const mapped = addr.match(/(?:::ffff:|::)(\d+\.\d+\.\d+\.\d+)$/) - if (mapped) return isPrivateIpv4(mapped[1]) - return false } function isPrivateIp(ip) { @@ -91,9 +98,35 @@ function isPrivateIp(ip) { if (kind === 4) return isPrivateIpv4(ip) if (kind === 6) return isPrivateIpv6(ip) // Not a literal IP — be conservative if it somehow reaches here. - return false + return true } +const { Agent, fetch: realFetch, WebSocket: RealWebSocket } = require( + workerData.undiciPath, +) +const egressDispatcher = new Agent({ + connect: { + lookup(hostname, options, callback) { + dnsPromises.lookup(hostname, { all: true, family: options.family }).then( + (addresses) => { + if ( + !addresses.length || + addresses.some((entry) => isPrivateIp(entry.address)) + ) { + return callback(new Error('SSRF guard: blocked DNS address'), '', 0) + } + callback( + null, + options.all ? addresses : addresses[0].address, + addresses[0].family, + ) + }, + (error) => callback(error, '', 0), + ) + }, + }, +}) + async function assertEgressUrlSafe(input) { let url try { @@ -164,7 +197,6 @@ const MAX_REDIRECTS = 20 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]) function createGuardedFetch() { - const realFetch = globalThis.fetch if (typeof realFetch !== 'function') return realFetch return async function guardedFetch(input, init) { const userRedirect = init && init.redirect @@ -174,7 +206,7 @@ function createGuardedFetch() { // error: undici throws on redirect). if (userRedirect === 'manual' || userRedirect === 'error') { await assertEgressUrlSafe(input) - return realFetch(input, init) + return realFetch(input, { ...init, dispatcher: egressDispatcher }) } // Auto-follow path (redirect unspecified or 'follow'). @@ -190,7 +222,10 @@ function createGuardedFetch() { for (let hop = 0; ; hop++) { await assertEgressUrlSafe(currentUrl) - const res = await realFetch(currentInput, currentInit) + const res = await realFetch(currentInput, { + ...currentInit, + dispatcher: egressDispatcher, + }) const location = res.headers.get('location') if (!REDIRECT_STATUSES.has(res.status) || !location) { @@ -237,7 +272,6 @@ function createGuardedFetch() { // Wrap WebSocket so the connection target is validated before the handshake. function createGuardedWebSocket() { - const RealWebSocket = globalThis.WebSocket if (typeof RealWebSocket !== 'function') return RealWebSocket return new Proxy(RealWebSocket, { construct(target, args) { @@ -256,10 +290,17 @@ function createGuardedWebSocket() { 'SSRF guard: blocked private WebSocket target ' + hostname, ) } - // Async DNS resolution can't be awaited in a synchronous constructor; - // literal-IP targets are blocked above, hostnames pass through. The - // fetch path remains the fully-guarded egress channel. - return Reflect.construct(target, args) + if ( + !['ws:', 'wss:'].includes(url.protocol) || + hostname.toLowerCase().replace(/\.$/, '') === 'localhost' || + /\.(localhost|local|internal)\.?$/i.test(hostname) + ) { + throw new Error('SSRF guard: blocked WebSocket target') + } + return Reflect.construct(target, [ + rawUrl, + { protocols: args[1], dispatcher: egressDispatcher }, + ]) }, }) } @@ -387,11 +428,17 @@ function createSandboxRequire(basePath) { // '@mx-space/extra'). Requiring user-installed packages from the data // dir's node_modules is a documented snippet feature, so resolve it // through the normal resolver jailed to requireBasePath. + if ( + moduleId.includes('\\') || + moduleId.includes(':') || + moduleId.split('/').some((part) => part === '.' || part === '..') + ) { + throw new Error('require: invalid package specifier') + } return baseRequire(moduleId) } - // Relative / absolute paths resolve against requireBasePath as before. - return baseRequire(moduleId) + throw new Error('require: relative and absolute paths are not allowed') } } diff --git a/apps/core/src/utils/sandbox/sandbox.service.ts b/apps/core/src/utils/sandbox/sandbox.service.ts index 4c9aa3bb8b9..1b7c81c0560 100644 --- a/apps/core/src/utils/sandbox/sandbox.service.ts +++ b/apps/core/src/utils/sandbox/sandbox.service.ts @@ -1,3 +1,4 @@ +import { createRequire } from 'node:module' import { Worker } from 'node:worker_threads' import type { @@ -76,6 +77,9 @@ export class SandboxService { private options: Required private workerCode = sandboxWorkerCode private initialized = false + private initialization?: Promise + private creatingWorkers = 0 + private waitingExecutions = 0 private idleCleanupInterval: ReturnType | null = null constructor(options: SandboxServiceOptions) { @@ -92,6 +96,12 @@ export class SandboxService { async initialize(): Promise { if (this.initialized) return + if (this.initialization) return this.initialization + this.initialization = this.initializePool() + return this.initialization + } + + private async initializePool(): Promise { // Create the minimum number of Workers const initPromises: Promise[] = [] for (let i = 0; i < this.options.minWorkers; i++) { @@ -153,6 +163,7 @@ export class SandboxService { eval: true, workerData: { requireBasePath: this.options.requireBasePath, + undiciPath: createRequire(import.meta.url).resolve('undici'), }, // Cap each Worker's resource usage so it does not impact the main process resourceLimits: { @@ -281,28 +292,53 @@ export class SandboxService { } } - private async getAvailableWorker(): Promise { - let worker = this.workers.find((w) => !w.busy) - - if (!worker && this.workers.length < this.options.maxWorkers) { - worker = await this.createWorker() - this.workers.push(worker) + private async getAvailableWorker(timeout: number): Promise { + const reserve = () => { + const worker = this.workers.find((w) => !w.busy && !w.terminating) + if (worker) worker.busy = true + return worker } - - if (!worker) { - await new Promise((resolve) => { - const checkInterval = setInterval(() => { - const available = this.workers.find((w) => !w.busy) - if (available) { - clearInterval(checkInterval) - resolve() + const available = reserve() + if (available) return available + if (this.workers.length + this.creatingWorkers < this.options.maxWorkers) { + this.creatingWorkers++ + try { + const worker = await this.createWorker() + worker.busy = true + this.workers.push(worker) + return worker + } finally { + this.creatingWorkers-- + } + } + if (this.waitingExecutions >= this.options.maxWorkers * 4) { + throw new Error('Sandbox capacity exhausted') + } + this.waitingExecutions++ + try { + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + clearInterval(poll) + reject(new Error('Sandbox capacity wait timeout')) + }, timeout) + const poll = setInterval(() => { + if (!this.initialized) { + clearInterval(poll) + clearTimeout(timer) + reject(new Error('Sandbox shut down')) + return } - }, 100) + const worker = reserve() + if (worker) { + clearInterval(poll) + clearTimeout(timer) + resolve(worker) + } + }, 10) }) - worker = this.workers.find((w) => !w.busy)! + } finally { + this.waitingExecutions-- } - - return worker } async execute( @@ -317,11 +353,12 @@ export class SandboxService { await this.initialize() } - const pooledWorker = await this.getAvailableWorker() - pooledWorker.busy = true + const started = Date.now() + const requestedTimeout = options?.timeout ?? this.options.defaultTimeout + const pooledWorker = await this.getAvailableWorker(requestedTimeout) const id = generateId() - const timeout = options?.timeout ?? this.options.defaultTimeout + const timeout = Math.max(1, requestedTimeout - (Date.now() - started)) const namespace = options?.namespace ?? `${context.model.reference}/${context.model.name}` @@ -376,6 +413,7 @@ export class SandboxService { this.workers = [] this.pendingExecutions.clear() this.initialized = false + this.initialization = undefined } getStats(): { diff --git a/apps/core/src/utils/ssrf.util.ts b/apps/core/src/utils/ssrf.util.ts index 4c5206536c0..b6a8f437512 100644 --- a/apps/core/src/utils/ssrf.util.ts +++ b/apps/core/src/utils/ssrf.util.ts @@ -1,5 +1,11 @@ +import { lookup } from 'node:dns/promises' + +import { Agent } from 'undici' + +import { isDev } from '~/global/env.global' import { assertHostnameSafe, + isPrivateIp, parseAndValidateUrl, UnsafeUrlError, } from '~/processors/agent-browser/url-guard' @@ -29,11 +35,7 @@ export interface AssertPublicHttpUrlOptions { * is a single source of truth shared with the agent-browser and Open Graph * fetchers. * - * NOTE (DNS rebinding): this is a TOCTOU check. A rebinding attacker can pass - * the lookup here and then re-point the hostname before the HTTP socket - * connects. Callers MUST additionally disable redirect following - * (`maxRedirects: 0`) to remove the redirect-based bypass. Fully closing the - * rebinding window would require pinning the connection to the resolved IP. + * Requests must use publicHttpDispatcher so DNS is validated at connection time. */ export async function assertPublicHttpUrl( rawUrl: string, @@ -46,3 +48,31 @@ export async function assertPublicHttpUrl( await assertHostnameSafe(url.hostname) return url } + +/** The checked DNS answers are the exact addresses handed to the socket. */ +export const publicHttpDispatcher = new Agent({ + connect: { + lookup(hostname, options, callback) { + lookup(hostname, { all: true, family: options.family }).then( + (addresses) => { + if ( + !addresses.length || + (!isDev && + addresses.some((entry) => + isPrivateIp(entry.address, entry.family), + )) + ) { + callback(new UnsafeUrlError('Blocked outbound DNS address'), '', 0) + return + } + callback( + null, + options.all ? addresses : addresses[0].address, + addresses[0].family, + ) + }, + (error) => callback(error, '', 0), + ) + }, + }, +}) diff --git a/apps/core/test/helper/setup-e2e.ts b/apps/core/test/helper/setup-e2e.ts index 81883009ef3..e2e2b01f3c9 100644 --- a/apps/core/test/helper/setup-e2e.ts +++ b/apps/core/test/helper/setup-e2e.ts @@ -21,6 +21,14 @@ export const setupE2EApp = async (module: TestingModule | ModuleMetadata) => { const app = nextModule.createNestApplication(fastifyApp) + app + .getHttpAdapter() + .getInstance() + .addHook('onRequest', async (request) => { + // Mirror the global role annotation on public routes as well as @Auth routes. + request.hasAdminAccess = Boolean(request.headers['test-token']) + request.isAuthenticated = request.hasAdminAccess + }) app.useGlobalPipes( requestCaseNormalizationPipeInstance, standardSchemaValidationPipeInstance, diff --git a/apps/core/test/src/contracts/comment.contract.spec.ts b/apps/core/test/src/contracts/comment.contract.spec.ts index 396c29598e9..092ffd48e1c 100644 --- a/apps/core/test/src/contracts/comment.contract.spec.ts +++ b/apps/core/test/src/contracts/comment.contract.spec.ts @@ -30,15 +30,12 @@ const EXPECTED_COMMENT_MODEL_KEYS = [ 'created_at', 'ref_type', 'ref_id', - 'state', 'author', 'text', // `mail` is admin-only; CommentFilterEmailInterceptor strips it on // unauthenticated endpoints, so SDK marks it optional. Asserted separately // on the admin list test below. 'url', - 'ip', - 'agent', 'pin', 'avatar', 'parent_comment_id', @@ -46,9 +43,7 @@ const EXPECTED_COMMENT_MODEL_KEYS = [ 'reply_count', 'latest_reply_at', 'is_deleted', - 'deleted_at', 'is_whispers', - 'location', 'auth_provider', 'reader_id', 'edited_at', diff --git a/apps/core/test/src/contracts/yohaku/comment-thread.contract.spec.ts b/apps/core/test/src/contracts/yohaku/comment-thread.contract.spec.ts index dc1083a2a32..9a13494bc1f 100644 --- a/apps/core/test/src/contracts/yohaku/comment-thread.contract.spec.ts +++ b/apps/core/test/src/contracts/yohaku/comment-thread.contract.spec.ts @@ -162,7 +162,6 @@ describe('Yohaku contract — comment thread (e2e)', () => { 'author', 'text', 'avatar', - 'state', 'pin', 'parent_comment_id', 'reply_count', @@ -182,6 +181,6 @@ describe('Yohaku contract — comment thread (e2e)', () => { const body = res.json() assertNoLegacyKeys(body, { allowed: allowedCommentKeys }) - assertHasKeys(body.data, ['id', 'author', 'text', 'created_at', 'state']) + assertHasKeys(body.data, ['id', 'author', 'text', 'created_at']) }) }) diff --git a/apps/core/test/src/database/protect-stored-secrets.spec.ts b/apps/core/test/src/database/protect-stored-secrets.spec.ts index 4c4fa5b535a..63a6c3ba73b 100644 --- a/apps/core/test/src/database/protect-stored-secrets.spec.ts +++ b/apps/core/test/src/database/protect-stored-secrets.spec.ts @@ -86,6 +86,7 @@ it('upgrades real credential rows without changing their usable values, and can new OwnerRepository(context.db, snowflake), { get: () => auth } as any, snowflake, + { emit: () => false } as any, ) expect(await service.verifyApiKey('legacy-raw-api-key')).toMatchObject({ id: 'legacy-key', diff --git a/apps/core/test/src/modules/auth/auth-bearer-narrowing.e2e-spec.ts b/apps/core/test/src/modules/auth/auth-bearer-narrowing.e2e-spec.ts index c036c056e68..4f8df0ade3f 100644 --- a/apps/core/test/src/modules/auth/auth-bearer-narrowing.e2e-spec.ts +++ b/apps/core/test/src/modules/auth/auth-bearer-narrowing.e2e-spec.ts @@ -65,6 +65,7 @@ const buildAuthService = () => { ownerRepository as any, authInstance as any, { nextId: () => '740375270589665280' } as any, + { emit: vi.fn() } as any, ) vi.spyOn(service, 'getSessionUser').mockResolvedValue(null) vi.spyOn(service, 'isOwnerReaderId').mockImplementation( diff --git a/apps/core/test/src/modules/auth/auth.service.spec.ts b/apps/core/test/src/modules/auth/auth.service.spec.ts index cb98d368569..73a86377adf 100644 --- a/apps/core/test/src/modules/auth/auth.service.spec.ts +++ b/apps/core/test/src/modules/auth/auth.service.spec.ts @@ -52,6 +52,7 @@ const createService = () => { ownerRepository as any, authInstance as any, snowflakeService as any, + { emit: vi.fn() } as any, ) return { authInstance, authRepository, readerRepository, service } } diff --git a/apps/core/test/src/modules/backup/backup.service.spec.ts b/apps/core/test/src/modules/backup/backup.service.spec.ts index e66b31119cf..1676416daf4 100644 --- a/apps/core/test/src/modules/backup/backup.service.spec.ts +++ b/apps/core/test/src/modules/backup/backup.service.spec.ts @@ -43,3 +43,16 @@ describe('BackupService path validation', () => { expect(rm).not.toHaveBeenCalled() }) }) + +it('rejects a ZIP with too many expanded entries before extraction', async () => { + const JSZip = (await import('jszip')).default + const zip = new JSZip() + for (let i = 0; i < 10001; i++) zip.file(`entry-${i}`, '') + const service = new BackupService({} as any, {} as any, {} as any) + await expect( + (service as any).safeUnzip( + await zip.generateAsync({ type: 'nodebuffer' }), + '/tmp/mx-security-archive-limit', + ), + ).rejects.toThrow(/entries|limit/i) +}) diff --git a/apps/core/test/src/modules/comment/comment-privacy.spec.ts b/apps/core/test/src/modules/comment/comment-privacy.spec.ts new file mode 100644 index 00000000000..e15a763f34f --- /dev/null +++ b/apps/core/test/src/modules/comment/comment-privacy.spec.ts @@ -0,0 +1,77 @@ +import { firstValueFrom, of } from 'rxjs' +import { describe, expect, it } from 'vitest' + +import { + isExplicitSuccessEnvelope, + withMeta, +} from '~/common/response/envelope.types' +import { CommentFilterEmailInterceptor } from '~/modules/comment/comment.interceptor' + +const comment = () => ({ + id: '1', + text: 'hello', + author: 'reader', + refId: '2', + refType: 'post', + state: 0, + createdAt: new Date(), + mail: 'private@example.com', + ip: '127.0.0.1', + agent: 'private agent', + location: 'private location', + meta: 'private metadata', + reader: { + id: '3', + name: 'Reader', + image: 'avatar', + email: 'secret@example.com', + role: 'owner', + banReason: 'private', + }, +}) + +const filter = (value: unknown, request = {}) => + firstValueFrom( + new CommentFilterEmailInterceptor().intercept( + { switchToHttp: () => ({ getRequest: () => request }) } as any, + { handle: () => of(value) }, + ), + ) + +describe('public comment privacy', () => { + it.each(['detail', 'list', 'thread'])( + 'projects nested %s responses for signed-in readers too', + async (shape) => { + const value = + shape === 'detail' + ? comment() + : shape === 'list' + ? withMeta([{ ...comment(), replies: [comment()] }], {}) + : { replies: [comment()], remaining: 0, done: true } + const result = await filter(value, { + user: { role: 'reader' }, + hasAdminAccess: false, + }) + const serialized = JSON.stringify(result) + for (const secret of [ + 'private@example.com', + '127.0.0.1', + 'private agent', + 'private location', + 'private metadata', + 'secret@example.com', + 'banReason', + ]) { + expect(serialized).not.toContain(secret) + } + expect(serialized).toContain('hello') + if (shape === 'list') expect(isExplicitSuccessEnvelope(result)).toBe(true) + }, + ) + it('preserves owner moderation fields', async () => { + expect(await filter(comment(), { hasAdminAccess: true })).toMatchObject({ + ip: '127.0.0.1', + mail: 'private@example.com', + }) + }) +}) diff --git a/apps/core/test/src/modules/enrichment/browser-fetch.service.spec.ts b/apps/core/test/src/modules/enrichment/browser-fetch.service.spec.ts index dba3df08b40..c718545ff2f 100644 --- a/apps/core/test/src/modules/enrichment/browser-fetch.service.spec.ts +++ b/apps/core/test/src/modules/enrichment/browser-fetch.service.spec.ts @@ -269,7 +269,7 @@ describe('BrowserFetchService', () => { ) expect(headers['Accept-Language']).toBe(OG_ACCEPT_LANGUAGE) expect(navCall.args).toContain('--args') - expect(navCall.args[navCall.args.indexOf('--args') + 1]).toBe( + expect(navCall.args[navCall.args.indexOf('--args') + 1]).toContain( OG_LAUNCH_ARGS, ) const navBatchCommands = parseBatchArgs(navCall.args).subCommands diff --git a/apps/core/test/src/modules/file/reader-upload-lock.spec.ts b/apps/core/test/src/modules/file/reader-upload-lock.spec.ts new file mode 100644 index 00000000000..467dcf98566 --- /dev/null +++ b/apps/core/test/src/modules/file/reader-upload-lock.spec.ts @@ -0,0 +1,55 @@ +import * as schema from '@mx-space/db-schema/schema' +import { drizzle } from 'drizzle-orm/node-postgres' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { expect, it } from 'vitest' + +import { FileReferenceRepository } from '~/modules/file/file-reference.repository' + +it('serializes the full upload accounting interval across replicas', async () => { + const database = await createIsolatedPgDatabase() + const pool = new Pool({ + connectionString: database.getConnectionUri(), + max: 4, + }) + const repository = new FileReferenceRepository( + drizzle(pool, { schema }) as any, + {} as any, + ) + let release!: () => void + let entered!: () => void + const started = new Promise((resolve) => { + entered = resolve + }) + const holding = new Promise((resolve) => { + release = resolve + }) + try { + const first = (repository as any).withReaderUploadLock( + 'reader-1', + async () => { + entered() + await holding + }, + ) + await started + await expect( + (repository as any).withReaderUploadLock( + 'reader-1', + async () => 'overlap', + ), + ).rejects.toThrow() + release() + await first + expect( + await (repository as any).withReaderUploadLock( + 'reader-1', + async () => 'released', + ), + ).toBe('released') + } finally { + release?.() + await pool.end() + await database.drop() + } +}, 30_000) diff --git a/apps/core/test/src/modules/note/note.controller.e2e-spec.ts b/apps/core/test/src/modules/note/note.controller.e2e-spec.ts index 03c9344e5db..2bbf6a43449 100644 --- a/apps/core/test/src/modules/note/note.controller.e2e-spec.ts +++ b/apps/core/test/src/modules/note/note.controller.e2e-spec.ts @@ -399,14 +399,9 @@ describe('NoteController', () => { }, }) - const response = await controller.getNoteByNid( - { nid: 1 } as any, - false, - {} as any, - 'fake-ip', - ) - - expect(response.meta.tts).toEqual({ available: false }) + await expect( + controller.getNoteByNid({ nid: 1 } as any, false, {} as any, 'fake-ip'), + ).rejects.toThrow() }) it('keeps meta.tts for the owner of a future-dated secret note', async () => { @@ -529,3 +524,43 @@ describe('NoteController', () => { }) }) }) + +describe('note confidentiality', () => { + it('blocks password-protected direct ID reads', async () => { + const { controller } = createController({ + noteService: { + findById: vi.fn().mockResolvedValue(makeNote({ hasPassword: true })), + checkPasswordToAccess: vi.fn().mockResolvedValue(false), + }, + }) + await expect( + controller.getOneNote({ id: 'note-1' } as any, false), + ).rejects.toThrow() + }) + it('blocks scheduled bodies before translation or enrichment', async () => { + const { controller } = createController({ + noteService: { + findByNid: vi + .fn() + .mockResolvedValue(makeNote({ content: 'secret lexical body' })), + checkNoteIsSecret: vi.fn().mockReturnValue(true), + }, + }) + await expect( + controller.getNoteByNid({ nid: 1 } as any, false, {} as any, 'ip'), + ).rejects.toThrow() + }) + it('does not include a protected anchor note in public navigation lists', async () => { + const { controller } = createController({ + noteService: { + findById: vi.fn().mockResolvedValue(makeNote({ hasPassword: true })), + }, + }) + const result = await controller.getNoteList( + { size: 10 } as any, + { id: 'note-1' } as any, + false, + ) + expect(result.data).toEqual([]) + }) +}) diff --git a/apps/core/test/src/modules/push/push.controller.spec.ts b/apps/core/test/src/modules/push/push.controller.spec.ts index 82424cc70ff..3a48e62af53 100644 --- a/apps/core/test/src/modules/push/push.controller.spec.ts +++ b/apps/core/test/src/modules/push/push.controller.spec.ts @@ -12,7 +12,7 @@ const activationBody = { const contextWith = (request: Record) => ({ - switchToHttp: () => ({ getRequest: () => request }), + switchToHttp: () => ({ getRequest: () => ({ headers: {}, ...request }) }), }) as unknown as ExecutionContext const customParamFactories = (methodName: string) => { @@ -49,9 +49,13 @@ describe('PushController', () => { it('resolves the reader id from the request without demanding a session', () => { const factories = customParamFactories('activate') - expect(factories).toHaveLength(1) + expect(factories).toHaveLength(2) - const factory = factories[0]!.factory! + const factory = factories.find( + (entry) => + entry.factory!(undefined, contextWith({ readerId: 'reader-1' })) === + 'reader-1', + )!.factory! expect(factory(undefined, contextWith({ readerId: 'reader-1' }))).toBe( 'reader-1', ) @@ -68,7 +72,11 @@ describe('PushController', () => { relayUrl: 'https://push.example.com', bindingId: 'bnd_remote', }) - expect(service.activate).toHaveBeenCalledWith('reader-1', activationBody) + expect(service.activate).toHaveBeenCalledWith( + 'reader-1', + activationBody, + false, + ) }) it('activates anonymously when no reader session is present', async () => { @@ -76,7 +84,11 @@ describe('PushController', () => { await controller.activate(activationBody) - expect(service.activate).toHaveBeenCalledWith(undefined, activationBody) + expect(service.activate).toHaveBeenCalledWith( + undefined, + activationBody, + false, + ) }) it('no longer exposes the reader-scoped endpoints Relay owns', () => { diff --git a/apps/core/test/src/modules/push/push.service.spec.ts b/apps/core/test/src/modules/push/push.service.spec.ts index 4834a6d3919..9f3a1b06c37 100644 --- a/apps/core/test/src/modules/push/push.service.spec.ts +++ b/apps/core/test/src/modules/push/push.service.spec.ts @@ -660,6 +660,7 @@ describe('PushService', () => { const [, init] = fetchMock.mock.calls[0]! expect(JSON.parse(String(init.body))).toEqual({ ticket: 't'.repeat(32), + owner_authorized: false, source_origin: 'https://core.example.com', source_label: 'core.example.com', }) @@ -681,6 +682,7 @@ describe('PushService', () => { expect(JSON.parse(String(fetchMock.mock.calls[0]![1].body))).toEqual({ ticket: 't'.repeat(32), + owner_authorized: false, source_origin: 'https://core.example.com', source_label: 'core.example.com', reader_id: 'reader-1', diff --git a/apps/core/test/src/modules/subscribe/subscribe.service.spec.ts b/apps/core/test/src/modules/subscribe/subscribe.service.spec.ts index 984c7288aba..5a5120e9dc5 100644 --- a/apps/core/test/src/modules/subscribe/subscribe.service.spec.ts +++ b/apps/core/test/src/modules/subscribe/subscribe.service.spec.ts @@ -65,6 +65,7 @@ describe('SubscribeService public content mail', () => { urlBuilderService as never, emailService as never, ownerService as never, + { getClient: () => ({ set: async () => 'OK' }) } as never, ) beforeEach(() => { @@ -73,6 +74,7 @@ describe('SubscribeService public content mail', () => { subscribeRepository.findAll.mockResolvedValue([ { email: 'reader@example.com', + verified: true, subscribe: SubscribePostCreateBit, cancelToken: 'token-1', }, @@ -135,3 +137,54 @@ describe('SubscribeService public content mail', () => { await service.onModuleDestroy() }) }) + +describe('subscription email ownership', () => { + it('leaves existing preferences unchanged until the mailbox link is confirmed', async () => { + const row = { + email: 'reader@example.com', + subscribe: 1, + verified: true, + cancelToken: 'secret-token', + } + const repository = { + findByEmail: vi.fn(async () => row), + updateByEmail: vi.fn(), + } + const email = { send: vi.fn() } + const admission = vi + .fn() + .mockResolvedValueOnce('OK') + .mockResolvedValue(null) + const service = new SubscribeService( + repository as any, + {} as any, + {} as any, + { + get: async () => ({ serverUrl: 'https://example.com' }), + waitForConfigReady: async () => ({ + mailOptions: { from: 'blog@example.com' }, + seo: { title: 'Blog' }, + }), + } as any, + {} as any, + email as any, + {} as any, + { getClient: () => ({ set: admission }) } as any, + ) + await service.subscribe(row.email, 2) + await service.subscribe(row.email, 1) + expect(repository.updateByEmail).not.toHaveBeenCalled() + expect(email.send).toHaveBeenCalledOnce() + const link = new URL( + email.send.mock.calls[0][0].text.match(/https:\/\/\S+/)[0], + ) + await (service as any).confirm(row.email, link.searchParams.get('token')) + expect(repository.updateByEmail).toHaveBeenCalledWith(row.email, { + subscribe: 2, + verified: true, + }) + await expect( + (service as any).confirm(row.email, 'forged'), + ).rejects.toThrow() + }) +}) diff --git a/apps/core/test/src/processors/browser-egress.spec.ts b/apps/core/test/src/processors/browser-egress.spec.ts new file mode 100644 index 00000000000..c069a79e4dc --- /dev/null +++ b/apps/core/test/src/processors/browser-egress.spec.ts @@ -0,0 +1,50 @@ +import { request } from 'node:http' + +import { describe, expect, it, vi } from 'vitest' + +import { createBrowserEgressProxy } from '~/processors/agent-browser/browser-egress-proxy' + +vi.mock('~/global/env.global', () => ({ isDev: false })) + +describe('browser egress proxy', () => { + it.each([ + 'http://127.0.0.1/private', + 'http://[::ffff:7f00:1]/private', + 'http://localhost/private', + ])('blocks browser requests to %s before connecting', async (target) => { + const proxy = await createBrowserEgressProxy() + try { + const status = await new Promise((resolve, reject) => { + const req = request(proxy.url, { path: target }, (res) => { + res.resume() + resolve(res.statusCode) + }) + req.on('error', reject) + req.end() + }) + expect(status).toBe(403) + } finally { + await proxy.close() + } + }) + it('blocks CONNECT tunnels to metadata', async () => { + const proxy = await createBrowserEgressProxy() + try { + const status = await new Promise((resolve, reject) => { + const req = request(proxy.url, { + method: 'CONNECT', + path: '169.254.169.254:80', + }) + req.on('connect', (res, socket) => { + socket.destroy() + resolve(res.statusCode) + }) + req.on('error', reject) + req.end() + }) + expect(status).toBe(403) + } finally { + await proxy.close() + } + }) +}) diff --git a/apps/core/test/src/processors/gateway/e2e/admin-gateway.e2e-spec.ts b/apps/core/test/src/processors/gateway/e2e/admin-gateway.e2e-spec.ts index e8cbe4802d8..bb402c61dd3 100644 --- a/apps/core/test/src/processors/gateway/e2e/admin-gateway.e2e-spec.ts +++ b/apps/core/test/src/processors/gateway/e2e/admin-gateway.e2e-spec.ts @@ -71,18 +71,36 @@ describe('AdminEventsGateway e2e', () => { } }) - it('connects via a valid query token', async () => { + it('rejects reusable credentials in URL queries', async () => { testApp.authService.verifyApiKey.mockResolvedValueOnce({ - referenceId: 'reader-2', + id: 'key-id', + referenceId: 'owner', }) testApp.authService.isOwnerReaderId.mockResolvedValueOnce(true) - const client = await WsTestClient.connect( testApp.adminUrl('token=valid-query-token'), ) + try { + expect((await client.waitForClose()).code).toBe(4401) + } finally { + await client.closeAndWait() + testApp.authService.verifyApiKey.mockReset() + testApp.authService.isOwnerReaderId.mockReset() + } + }) + + it('closes established admin sockets when owner roles change', async () => { + testApp.authService.getSessionUserFromHeaders.mockResolvedValueOnce({ + user: { id: 'owner', role: 'owner' }, + session: { token: 'role-token' }, + }) + const client = await WsTestClient.connect(testApp.adminUrl(), { + headers: { cookie: 'session=owner' }, + }) try { await client.waitFor((e) => e.event === BusinessEvents.GATEWAY_CONNECT) - await assertLiveConnection(client) + testApp.eventEmitter.emit('auth.owner-roles-changed') + expect((await client.waitForClose()).code).toBe(4401) } finally { await client.closeAndWait() } diff --git a/apps/core/test/src/utils/outbound-security.spec.ts b/apps/core/test/src/utils/outbound-security.spec.ts new file mode 100644 index 00000000000..b45e60c67d3 --- /dev/null +++ b/apps/core/test/src/utils/outbound-security.spec.ts @@ -0,0 +1,115 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { safeFetch } from '~/modules/enrichment/providers/open-graph/safe-fetch' +import { + isPrivateIp, + parseAndValidateUrl, +} from '~/processors/agent-browser/url-guard' +import { readResponseBuffer } from '~/utils/http-response.util' +import { assertPublicHttpUrl, publicHttpDispatcher } from '~/utils/ssrf.util' + +const { lookupMock } = vi.hoisted(() => ({ lookupMock: vi.fn() })) +vi.mock('undici', async (original) => ({ + ...(await original()), + fetch: (...args: Parameters) => + globalThis.fetch(...args), +})) +vi.mock('node:dns/promises', () => ({ lookup: lookupMock })) +vi.mock('~/global/env.global', () => ({ isDev: false })) + +const options = { timeoutMs: 30, maxBodyBytes: 8 } +afterEach(() => vi.unstubAllGlobals()) + +describe('outbound request security', () => { + it.each([ + '::ffff:7f00:1', + '0:0:0:0:0:ffff:a00:1', + '::7f00:1', + 'fe80::1', + 'fc00::1', + ])('blocks normalized private IPv6 %s', (ip) => { + expect(isPrivateIp(ip, 6)).toBe(true) + expect(() => parseAndValidateUrl(`http://[${ip}]/`)).toThrow() + }) + it('retains the deadline after headers arrive', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async (_url, init) => + new Response( + new ReadableStream({ + start(controller) { + init.signal.addEventListener( + 'abort', + () => controller.error(new Error('deadline')), + { once: true }, + ) + }, + }), + { headers: { 'content-type': 'text/html' } }, + ), + ), + ) + const result = safeFetch('https://8.8.8.8/', options) + await expect( + Promise.race([ + result, + new Promise((resolve) => setTimeout(() => resolve('hung'), 100)), + ]), + ).rejects.toThrow('deadline') + }) + it('uses a connection dispatcher and caps chunked bodies', async () => { + const fetch = vi.fn( + async () => + new Response('0123456789', { + headers: { 'content-type': 'text/html' }, + }), + ) + vi.stubGlobal('fetch', fetch) + expect(await safeFetch('https://8.8.8.8/', options)).toMatchObject({ + body: '01234567', + truncated: true, + }) + expect(fetch.mock.calls[0]?.[1]).toHaveProperty('dispatcher') + }) +}) + +describe('bounded binary responses', () => { + it.each([undefined, '20'])( + 'rejects oversized bodies with content-length %s', + async (length) => { + let cancelled = false + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(20)) + }, + cancel() { + cancelled = true + }, + }), + { headers: length ? { 'content-length': length } : {} }, + ) + await expect(readResponseBuffer(response, 8)).rejects.toThrow('exceeds') + expect(cancelled).toBe(true) + }, + ) + it('returns exact bytes for an allowed image', async () => { + expect( + await readResponseBuffer(new Response(new Uint8Array([1, 2, 3])), 3), + ).toEqual(Buffer.from([1, 2, 3])) + }) +}) +it('rejects a rebinding answer at socket connection time', async () => { + lookupMock + .mockResolvedValueOnce([{ address: '8.8.8.8', family: 4 }]) + .mockResolvedValueOnce([{ address: '127.0.0.1', family: 4 }]) + const url = await assertPublicHttpUrl('https://rebinding-security.test/') + const { fetch } = await vi.importActual('undici') + await expect( + fetch(url, { dispatcher: publicHttpDispatcher }).catch((error) => { + throw error.cause ?? error + }), + ).rejects.toThrow('Blocked outbound DNS address') + expect(lookupMock).toHaveBeenCalledTimes(2) +}) diff --git a/apps/core/test/src/utils/s3.util.spec.ts b/apps/core/test/src/utils/s3.util.spec.ts index 44d500f19a2..0528cb905c6 100644 --- a/apps/core/test/src/utils/s3.util.spec.ts +++ b/apps/core/test/src/utils/s3.util.spec.ts @@ -671,3 +671,42 @@ describe('S3Uploader.objectExists', () => { } }) }) + +describe('S3 response admission', () => { + afterEach(() => vi.unstubAllGlobals()) + it('disables redirects and bounds the complete response lifetime', async () => { + const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 200 })) + vi.stubGlobal('fetch', fetch) + const uploader = new S3Uploader({ + endpoint: 'https://storage.example.com', + accessKey: 'test', + secretKey: 'test', + bucket: 'test', + region: 'auto', + }) + await uploader.uploadBuffer(Buffer.from('test'), 'test.txt', 'text/plain') + expect(fetch.mock.calls[0][1].redirect).toBe('error') + expect(fetch.mock.calls[0][1].signal).toBeInstanceOf(AbortSignal) + }) + it('rejects oversized storage responses before buffering', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response('oops', { + status: 500, + headers: { 'content-length': '999999999' }, + }), + ), + ) + const uploader = new S3Uploader({ + endpoint: 'https://storage.example.com', + accessKey: 'test', + secretKey: 'test', + bucket: 'test', + region: 'auto', + }) + await expect( + uploader.uploadBuffer(Buffer.from('test'), 'test.txt', 'text/plain'), + ).rejects.toThrow(/exceed|transport/i) + }) +}) diff --git a/apps/core/test/src/utils/sandbox/sandbox.service.spec.ts b/apps/core/test/src/utils/sandbox/sandbox.service.spec.ts index 1a3e3e20b44..30aa160a2df 100644 --- a/apps/core/test/src/utils/sandbox/sandbox.service.spec.ts +++ b/apps/core/test/src/utils/sandbox/sandbox.service.spec.ts @@ -514,3 +514,75 @@ describe('SandboxService', () => { }) }) }) + +describe('sandbox authority boundaries', () => { + const context = { + req: {}, + res: {}, + isAuthenticated: false, + model: { id: '1', name: 'test', reference: 'test' }, + } + test('rejects execution of writable files outside the VM', async () => { + const { mkdtemp, writeFile, rm } = await import('node:fs/promises') + const { tmpdir } = await import('node:os') + const dir = await mkdtemp(`${tmpdir()}/mx-sandbox-security-`) + const sandbox = new SandboxService({ + maxWorkers: 1, + bridgeHandlers: {}, + requireBasePath: `${dir}/`, + }) + try { + await writeFile(`${dir}/escape.cjs`, 'module.exports = typeof process') + const result = await sandbox.execute( + `async function handler(ctx, require) { return await require(${JSON.stringify(`${dir}/escape.cjs`)}) }`, + context, + ) + expect(result.success).toBe(false) + } finally { + await sandbox.shutdown() + await rm(dir, { recursive: true }) + } + }) + test('rejects overload without accumulating waiting requests', async () => { + const sandbox = new SandboxService({ maxWorkers: 1, bridgeHandlers: {} }) + await sandbox.initialize() + const first = sandbox.execute( + 'async function handler() { await new Promise(r => setTimeout(r, 200)); return 1 }', + context, + ) + try { + const queued = Promise.allSettled( + Array.from({ length: 4 }, () => + sandbox.execute('function handler() { return 2 }', context), + ), + ) + const result = sandbox.execute('function handler() { return 2 }', context) + await expect( + Promise.race([ + result, + new Promise((resolve) => setTimeout(() => resolve('queued'), 50)), + ]), + ).rejects.toThrow(/capacity|busy/i) + expect((await first).success).toBe(true) + await queued + } finally { + await first + await sandbox.shutdown() + } + }) + test.each(['ws://localhost:9', 'ws://127.0.0.1:9', 'ws://[::ffff:7f00:1]:9'])( + 'blocks private WebSocket target %s', + async (url) => { + const sandbox = new SandboxService({ maxWorkers: 1, bridgeHandlers: {} }) + try { + const result = await sandbox.execute( + `function handler() { new WebSocket(${JSON.stringify(url)}); return 'connected' }`, + context, + ) + expect(result.success).toBe(false) + } finally { + await sandbox.shutdown() + } + }, + ) +}) diff --git a/apps/core/test/src/utils/telemetry-security.spec.ts b/apps/core/test/src/utils/telemetry-security.spec.ts new file mode 100644 index 00000000000..3f9f42f6a88 --- /dev/null +++ b/apps/core/test/src/utils/telemetry-security.spec.ts @@ -0,0 +1,101 @@ +import { describe, expect, it, vi } from 'vitest' + +import worker from '../../../../telemetry/src/index' + +describe('telemetry admission', () => { + const payload = { + instanceId: '00000000-0000-4000-8000-000000000001', + version: '1.0.0', + event: 'startup', + } + it('rejects oversized bodies before touching the database', async () => { + const prepare = vi.fn() + const response = await worker.fetch( + new Request('https://telemetry.test/collect', { + method: 'POST', + body: JSON.stringify({ ...payload, padding: 'x'.repeat(4096) }), + }), + { + DB: { prepare }, + COLLECT_LIMITER: { limit: async () => ({ success: true }) }, + } as any, + {} as any, + ) + expect(response.status).toBe(413) + expect(prepare).not.toHaveBeenCalled() + }) + it('rejects rate-limited callers before touching D1', async () => { + const prepare = vi.fn() + const response = await worker.fetch( + new Request('https://telemetry.test/collect', { + method: 'POST', + body: JSON.stringify(payload), + }), + { + DB: { prepare }, + COLLECT_LIMITER: { limit: async () => ({ success: false }) }, + } as any, + {} as any, + ) + expect(response.status).toBe(429) + expect(prepare).not.toHaveBeenCalled() + }) +}) + +it('deduplicates collection and refuses new rows after the daily budget (real SQLite)', async () => { + const { DatabaseSync } = await import('node:sqlite') + const { readFileSync } = await import('node:fs') + const database = new DatabaseSync(':memory:') + database.exec( + readFileSync( + new URL('../../../../telemetry/schema.sql', import.meta.url), + 'utf8', + ), + ) + const env = { + COLLECT_LIMITER: { limit: async () => ({ success: true }) }, + DB: { + prepare: (sql: string) => ({ + bind: (...values: string[]) => ({ + run: async () => ({ + meta: { + changes: Number(database.prepare(sql).run(...values).changes), + }, + }), + }), + }), + }, + } + const collect = (id: string) => + worker.fetch( + new Request('https://telemetry.test/collect', { + method: 'POST', + body: JSON.stringify({ + instanceId: id, + version: '1.0.0', + event: 'startup', + }), + }), + env as any, + {} as any, + ) + try { + expect((await collect('00000000-0000-4000-8000-000000000001')).status).toBe( + 200, + ) + expect((await collect('00000000-0000-4000-8000-000000000001')).status).toBe( + 429, + ) + database.exec( + `WITH RECURSIVE rows(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM rows WHERE n < 9999) INSERT INTO telemetry(instance_id,version,event) SELECT 'test-' || n,'1.0.0','startup' FROM rows`, + ) + expect((await collect('00000000-0000-4000-8000-000000000002')).status).toBe( + 429, + ) + expect( + database.prepare('SELECT count(*) AS count FROM telemetry').get()?.count, + ).toBe(10000) + } finally { + database.close() + } +}) diff --git a/apps/ios/Packages/SpaceCore/Sources/SpaceCore/AuthenticationMiddleware.swift b/apps/ios/Packages/SpaceCore/Sources/SpaceCore/AuthenticationMiddleware.swift index 8385a87004f..38c009f658a 100644 --- a/apps/ios/Packages/SpaceCore/Sources/SpaceCore/AuthenticationMiddleware.swift +++ b/apps/ios/Packages/SpaceCore/Sources/SpaceCore/AuthenticationMiddleware.swift @@ -17,7 +17,10 @@ public struct AuthenticationMiddleware: ClientMiddleware { next: (HTTPRequest, HTTPBody?, URL) async throws -> (HTTPResponse, HTTPBody?) ) async throws -> (HTTPResponse, HTTPBody?) { var request = request - if let token = try? tokenStore.read(), !token.isEmpty { + let pairingOperation = operationID == "requestDeviceCode" || operationID == "pollDeviceToken" + if pairingOperation || baseURL.scheme?.lowercased() != "https" { + request.headerFields[.authorization] = nil + } else if let token = try? tokenStore.read(), !token.isEmpty { request.headerFields[.authorization] = "Bearer \(token)" } return try await next(request, body, baseURL) diff --git a/apps/ios/Packages/SpaceCore/Sources/SpaceCore/ServerEndpoint.swift b/apps/ios/Packages/SpaceCore/Sources/SpaceCore/ServerEndpoint.swift index c0f6bfc3076..f61f6f3bbf8 100644 --- a/apps/ios/Packages/SpaceCore/Sources/SpaceCore/ServerEndpoint.swift +++ b/apps/ios/Packages/SpaceCore/Sources/SpaceCore/ServerEndpoint.swift @@ -13,14 +13,12 @@ public struct ServerEndpoint: Sendable, Equatable { baseURL.appending(path: ServerEndpoint.pathPrefix.dropFirst()) } - /// Plaintext HTTP is accepted only for private and `.local` hosts, matching - /// the `NSAllowsLocalNetworking` exception the app ships with. Public - /// plaintext is refused outright rather than degraded. + /// Owner credentials require TLS on public and local networks. public init(baseURL: URL) throws { guard let scheme = baseURL.scheme?.lowercased(), let host = baseURL.host() else { throw SpaceTransportError.invalidServerURL(baseURL.absoluteString) } - if scheme == "http", !ServerEndpoint.isLocalHost(host) { + if scheme == "http" { throw SpaceTransportError.insecureScheme(host) } guard scheme == "http" || scheme == "https" else { @@ -29,18 +27,16 @@ public struct ServerEndpoint: Sendable, Equatable { self.baseURL = baseURL } - static func isLocalHost(_ host: String) -> Bool { - if host == "localhost" || host.hasSuffix(".local") { return true } - let parts = host.split(separator: ".").compactMap { Int($0) } - guard parts.count == 4, parts.allSatisfy({ (0...255).contains($0) }) else { - return false - } - switch (parts[0], parts[1]) { - case (10, _): return true - case (127, _): return true - case (192, 168): return true - case (172, 16...31): return true - default: return false - } + public var credentialOrigin: String { + var origin = URLComponents(url: baseURL, resolvingAgainstBaseURL: false)! + origin.scheme = origin.scheme?.lowercased() + origin.host = origin.host?.lowercased() + origin.user = nil + origin.password = nil + origin.path = "" + origin.query = nil + origin.fragment = nil + origin.port = baseURL.port == 443 ? nil : baseURL.port + return origin.string! } } diff --git a/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/AuthenticationMiddlewareTests.swift b/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/AuthenticationMiddlewareTests.swift index 800cc19a471..120a39c2d9a 100644 --- a/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/AuthenticationMiddlewareTests.swift +++ b/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/AuthenticationMiddlewareTests.swift @@ -43,3 +43,30 @@ import Testing private final class Captured: @unchecked Sendable { var request: HTTPRequest? } + +@Suite struct PairingCredentialPrivacyTests { + @Test(arguments: ["requestDeviceCode", "pollDeviceToken"]) + func pairingNeverSendsAPreviousToken(_ operation: String) async throws { + let middleware = AuthenticationMiddleware(tokenStore: InMemoryTokenStore(token: "previous-owner-secret")) + _ = try await middleware.intercept( + HTTPRequest(method: .post, scheme: nil, authority: nil, path: "/device"), + body: nil, baseURL: URL(string: "https://different.example/api/v3")!, operationID: operation + ) { request, _, _ in + #expect(request.headerFields[.authorization] == nil) + return (HTTPResponse(status: .ok), nil) + } + } +} + +@Suite struct CredentialTransportBoundaryTests { + @Test(arguments: ["requestDeviceCode", "pollDeviceToken"]) + func removesPreviouslyAttachedHeadersFromPairing(_ operation: String) async throws { + let middleware = AuthenticationMiddleware(tokenStore: InMemoryTokenStore(token: "old-token")) + var request = HTTPRequest(method: .post, scheme: nil, authority: nil, path: "/device") + request.headerFields[.authorization] = "Bearer previous-header" + _ = try await middleware.intercept(request, body: nil, baseURL: URL(string: "https://different.example")!, operationID: operation) { request, _, _ in + #expect(request.headerFields[.authorization] == nil) + return (HTTPResponse(status: .ok), nil) + } + } +} diff --git a/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/ServerEndpointTests.swift b/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/ServerEndpointTests.swift index 97150a966f2..f0e814ec601 100644 --- a/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/ServerEndpointTests.swift +++ b/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/ServerEndpointTests.swift @@ -17,8 +17,10 @@ import Testing "http://172.16.0.1", "http://127.0.0.1:2333", ]) - func allowsPlaintextOnLocalNetworks(_ raw: String) throws { - _ = try ServerEndpoint(baseURL: URL(string: raw)!) + func refusesPlaintextOnLocalNetworks(_ raw: String) { + #expect(throws: SpaceTransportError.self) { + _ = try ServerEndpoint(baseURL: URL(string: raw)!) + } } @Test(arguments: ["http://example.com", "http://172.32.0.1", "http://8.8.8.8"]) @@ -52,3 +54,16 @@ import Testing #expect(url == ServerEndpoint.pathPrefix) } } + +@Suite struct CredentialOriginTests { + @Test func canonicalizesTheOriginWithoutPathOrDefaultPort() throws { + let first = try ServerEndpoint(baseURL: URL(string: "https://EXAMPLE.com:443/old/path")!) + let second = try ServerEndpoint(baseURL: URL(string: "https://example.com")!) + #expect(first.credentialOrigin == second.credentialOrigin) + #expect(first.credentialOrigin != (try ServerEndpoint(baseURL: URL(string: "https://example.com:8443")!)).credentialOrigin) + } + @Test func acceptsAnHTTPSIPv6Origin() throws { + let endpoint = try ServerEndpoint(baseURL: URL(string: "https://[::1]:8443")!) + #expect(URL(string: endpoint.credentialOrigin)?.port == 8443) + } +} diff --git a/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/WebHandoffServiceTests.swift b/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/WebHandoffServiceTests.swift index 34b52986899..8280f55b3c8 100644 --- a/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/WebHandoffServiceTests.swift +++ b/apps/ios/Packages/SpaceCore/Tests/SpaceCoreTests/WebHandoffServiceTests.swift @@ -13,11 +13,11 @@ import Testing ), ]) let client = Client( - serverURL: URL(string: "http://127.0.0.1:2444/api/v3")!, + serverURL: URL(string: "https://127.0.0.1:2444/api/v3")!, configuration: SpaceClient.configuration, transport: transport ) - let endpoint = try ServerEndpoint(baseURL: URL(string: "http://127.0.0.1:2444")!) + let endpoint = try ServerEndpoint(baseURL: URL(string: "https://127.0.0.1:2444")!) let service = WebHandoffService(client: client, endpoint: endpoint) let url = try await service.makeURL(for: .comments) diff --git a/apps/ios/README.md b/apps/ios/README.md index 67c1f403efb..ece52302533 100644 --- a/apps/ios/README.md +++ b/apps/ios/README.md @@ -44,11 +44,13 @@ PG_URL="postgres://mx:mx@127.0.0.1:5433/mx_verify" NODE_ENV=production \ REDIS_HOST=127.0.0.1 REDIS_PORT=6380 SNOWFLAKE_WORKER_ID=7 \ JWT_SECRET=<32+ chars> PORT=2444 node ../core/out/main.mjs -make verify SPACE_SERVER=http://127.0.0.1:2444 \ +make verify SPACE_SERVER=https://mx-verify.example.com \ SPACE_USERNAME= SPACE_PASSWORD= \ PG_DATABASE=mx_verify ``` +Expose the local verification server through a trusted HTTPS reverse proxy at the address passed as `SPACE_SERVER`. The app requires HTTPS on local networks too. + The suite skips itself when `SPACE_TEST_SERVER` is unset, so `make test` stays green without a server. @@ -105,7 +107,9 @@ To add an endpoint: register it in ## Transport policy -`ServerEndpoint` refuses plaintext HTTP for public hosts and allows it only for -`localhost`, `*.local`, and RFC1918 addresses — matching the -`NSAllowsLocalNetworking` exception in `Info.plist`. `NSAllowsArbitraryLoads` -is deliberately absent; enabling it invites an App Review question. +`ServerEndpoint` requires HTTPS on every host, including loopback, `.local`, +and private addresses. Use a trusted certificate for LAN development. +Session tokens are stored in Keychain accounts keyed by the canonical HTTPS +origin. Upgrading from the global token record requires pairing again; legacy +tokens are never automatically attached to a newly selected server. Pairing +requests never carry an existing bearer token. diff --git a/apps/ios/Space/AppContainer.swift b/apps/ios/Space/AppContainer.swift index 15b9aa60928..e9087f8fd59 100644 --- a/apps/ios/Space/AppContainer.swift +++ b/apps/ios/Space/AppContainer.swift @@ -7,7 +7,9 @@ import UIKit final class AppContainer { static let shared = AppContainer() - let tokenStore: any TokenStore = KeychainTokenStore() + private func tokenStore(for endpoint: ServerEndpoint) -> any TokenStore { + KeychainTokenStore(account: endpoint.credentialOrigin) + } let serverStore = ServerStore() private var pushManager: PushNotificationManager? @@ -16,8 +18,9 @@ final class AppContainer { private init() {} private var pairedEndpoint: ServerEndpoint? { - let hasToken = ((try? tokenStore.read()) ?? nil)?.isEmpty == false - return hasToken ? serverStore.read() : nil + guard let endpoint = serverStore.read() else { return nil } + let hasToken = ((try? tokenStore(for: endpoint).read()) ?? nil)?.isEmpty == false + return hasToken ? endpoint : nil } func makeRootViewController() -> UIViewController { @@ -46,7 +49,7 @@ final class AppContainer { } func makeClient(_ endpoint: ServerEndpoint) -> SpaceClient { - SpaceClient(endpoint: endpoint, tokenStore: tokenStore) + SpaceClient(endpoint: endpoint, tokenStore: tokenStore(for: endpoint)) } func makePairingFlow() -> UIViewController { @@ -61,7 +64,7 @@ final class AppContainer { let client = makeClient(endpoint) let controller = PairingCodeViewController( endpoint: endpoint, - pairing: PairingService(spaceClient: client, tokenStore: tokenStore) + pairing: PairingService(spaceClient: client, tokenStore: tokenStore(for: endpoint)) ) { [weak self] in self?.serverStore.write(endpoint) self?.swapRoot() @@ -89,7 +92,7 @@ final class AppContainer { } func clearPairing() { - try? tokenStore.clear() + if let endpoint = serverStore.read() { try? tokenStore(for: endpoint).clear() } serverStore.clear() pushManager = nil } diff --git a/apps/ios/Space/Info.plist b/apps/ios/Space/Info.plist index 76b8d5679a8..b411368cc1d 100644 --- a/apps/ios/Space/Info.plist +++ b/apps/ios/Space/Info.plist @@ -22,11 +22,6 @@ $(CURRENT_PROJECT_VERSION) ITSAppUsesNonExemptEncryption - NSAppTransportSecurity - - NSAllowsLocalNetworking - - SpaceAPNsEnvironment $(SPACE_APNS_ENVIRONMENT) SpacePushAppID diff --git a/apps/ios/project.yml b/apps/ios/project.yml index 101eb0ad2cf..6351ec96129 100644 --- a/apps/ios/project.yml +++ b/apps/ios/project.yml @@ -43,8 +43,6 @@ targets: UIWindowSceneSessionRoleApplication: - UISceneConfigurationName: Default Configuration UISceneDelegateClassName: $(PRODUCT_MODULE_NAME).SceneDelegate - NSAppTransportSecurity: - NSAllowsLocalNetworking: true SpacePushRelayURL: $(SPACE_PUSH_RELAY_URL) SpacePushAppID: $(SPACE_PUSH_APP_ID) SpaceAPNsEnvironment: $(SPACE_APNS_ENVIRONMENT) diff --git a/apps/ios/scripts/approve-device.sh b/apps/ios/scripts/approve-device.sh index e883750bae1..ee7200adc6c 100755 --- a/apps/ios/scripts/approve-device.sh +++ b/apps/ios/scripts/approve-device.sh @@ -2,13 +2,17 @@ # Stands in for the operator approving a pairing code in a browser: signs in as # the owner, waits for a pending device code, and approves it. # -# SPACE_SERVER=http://127.0.0.1:2444 \ +# SPACE_SERVER=https://mx-verify.example.com \ # SPACE_USERNAME=innei SPACE_PASSWORD=... \ # PG_CONTAINER=mx-pg-dev PG_DATABASE=mx_ios_verify \ # ./scripts/approve-device.sh set -euo pipefail SERVER=${SPACE_SERVER:?SPACE_SERVER is required} +if [[ "$SERVER" != https://* ]]; then + printf 'SPACE_SERVER must use HTTPS\n' >&2 + exit 1 +fi USERNAME=${SPACE_USERNAME:?SPACE_USERNAME is required} PASSWORD=${SPACE_PASSWORD:?SPACE_PASSWORD is required} PG_CONTAINER=${PG_CONTAINER:-mx-pg-dev} diff --git a/apps/ios/scripts/seed-verify-data.sh b/apps/ios/scripts/seed-verify-data.sh index 3bfd0b3048a..b3ed4a38938 100755 --- a/apps/ios/scripts/seed-verify-data.sh +++ b/apps/ios/scripts/seed-verify-data.sh @@ -5,6 +5,10 @@ set -euo pipefail SERVER=${SPACE_SERVER:?SPACE_SERVER is required} +if [[ "$SERVER" != https://* ]]; then + printf 'SPACE_SERVER must use HTTPS\n' >&2 + exit 1 +fi USERNAME=${SPACE_USERNAME:?SPACE_USERNAME is required} PASSWORD=${SPACE_PASSWORD:?SPACE_PASSWORD is required} LINK=${SEED_LINK:-https://bgm.tv/subject/265} diff --git a/apps/push-relay-rs/README.md b/apps/push-relay-rs/README.md index d8c1925b037..9551412da5b 100644 --- a/apps/push-relay-rs/README.md +++ b/apps/push-relay-rs/README.md @@ -10,6 +10,7 @@ on a dedicated thread, with foreign keys enabled and WAL journaling. export PUSH_RELAY_DATABASE_PATH=./push-relay.sqlite3 export PUSH_RELAY_PUBLIC_URL=http://localhost:8787 export PUSH_RELAY_DATA_KEY="$(openssl rand -hex 32)" +export PUSH_RELAY_ENROLLMENT_TOKEN="$(openssl rand -hex 32)" export PUSH_RELAY_APPS_JSON='[{"id":"space","bundleId":"your.bundle.id","teamId":"YOUR_TEAM","keyId":"YOUR_KEY","privateKeyPath":"/path/to/AuthKey.p8"}]' cargo run --release -- migrate cargo run --release @@ -46,6 +47,33 @@ survive restarts; interrupted `processing` deliveries are requeued on startup. Delivery remains at least once: a crash after APNs accepts a message but before the result is saved may cause that notification to be sent again. +## Enrollment and admission limits + +Set the same secret (32–256 characters) as `PUSH_RELAY_ENROLLMENT_TOKEN` on +Relay and `MX_PUSH_RELAY_ENROLLMENT_TOKEN` on each approved Core. It authorizes +the first source claim. When it is unset, new sources are refused; existing +sources continue using their source credentials. Keep this operator secret +out of client applications. Deploy Relay and Core together. + +Space administrative bindings now require Core to attest owner authority. +Migration `0003_binding_owner_authorization.sql` leaves old bindings without +that grant. Reactivate Space push from a newly paired owner session after +upgrading. Yohaku reader/public subscription bindings remain available. + +Registration rejects active duplicate app/environment/token combinations and +caps token encodings at 256 bytes. Storage is bounded at 100,000 installations, +1,000 new installations per hour, 10,000 sources, 100,000 bindings, 1,000 +bindings per source, and 32 per installation. Unbound installations expire +after a day; expired activation tickets are removed and each installation +can retain at most ten tickets. + +Events are admitted atomically: 60 per source per minute, 1,000 per source +per day, and 10,000 globally per day. Pending deliveries are capped at 10,000 +and daily fanout at 100,000. Budget failures return HTTP 429; retries of an +already accepted event remain idempotent. Events and their deliveries expire +after 30 days when new events are accepted. These are fixed safety ceilings; +review capacity before changing them. + ## Upgrading from PostgreSQL SQLite migrations initialize a new SQLite database. To preserve devices, diff --git a/apps/push-relay-rs/migrations/0003_binding_owner_authorization.sql b/apps/push-relay-rs/migrations/0003_binding_owner_authorization.sql new file mode 100644 index 00000000000..d7ae9ff2b93 --- /dev/null +++ b/apps/push-relay-rs/migrations/0003_binding_owner_authorization.sql @@ -0,0 +1,6 @@ +-- Existing bindings must be reactivated by a core that verifies owner authority. +ALTER TABLE push_bindings ADD COLUMN owner_authorized INTEGER NOT NULL DEFAULT 0 CHECK (owner_authorized IN (0, 1)); +CREATE INDEX IF NOT EXISTS push_events_source_created_idx ON push_events (source_id, created_at); +CREATE INDEX IF NOT EXISTS push_events_created_idx ON push_events (created_at); +CREATE INDEX IF NOT EXISTS push_deliveries_created_idx ON push_deliveries (created_at); +CREATE INDEX IF NOT EXISTS push_installations_created_idx ON push_installations (created_at); diff --git a/apps/push-relay-rs/src/config.rs b/apps/push-relay-rs/src/config.rs index ce6f78d1fa1..f3da1498255 100644 --- a/apps/push-relay-rs/src/config.rs +++ b/apps/push-relay-rs/src/config.rs @@ -60,6 +60,7 @@ pub struct Config { pub public_url: String, pub database_path: String, pub data_key: String, + pub enrollment_token: Option, pub apps: HashMap, } @@ -183,6 +184,9 @@ pub fn load() -> Result { .to_string(), database_path: database_path()?, data_key: required("PUSH_RELAY_DATA_KEY")?, + enrollment_token: env::var("PUSH_RELAY_ENROLLMENT_TOKEN") + .ok() + .filter(|token| token.len() >= 32 && token.len() <= 256), apps, }) } diff --git a/apps/push-relay-rs/src/fanout.rs b/apps/push-relay-rs/src/fanout.rs index 343445d2203..19d8eb05a1b 100644 --- a/apps/push-relay-rs/src/fanout.rs +++ b/apps/push-relay-rs/src/fanout.rs @@ -10,6 +10,7 @@ pub const FANOUT_DELIVERY_INSERT_SQL: &str = "INSERT INTO push_deliveries JOIN push_installations i ON i.id = b.installation_id WHERE b.source_id = ?2 AND b.revoked_at IS NULL AND i.revoked_at IS NULL AND i.app_id = ?4 + AND (i.app_id <> 'space' OR b.owner_authorized = 1) AND (?5 IS NULL OR b.reader_id = ?5) AND (?6 IS NULL OR COALESCE(json_extract(b.preferences, '$.' || ?6), 0) = 1)"; diff --git a/apps/push-relay-rs/src/main.rs b/apps/push-relay-rs/src/main.rs index 617677eaa18..c6f09e7446c 100644 --- a/apps/push-relay-rs/src/main.rs +++ b/apps/push-relay-rs/src/main.rs @@ -52,6 +52,7 @@ async fn serve() -> Result<(), String> { DataVault::new(&config.data_key)?, config.public_url.clone(), app_ids, + config.enrollment_token.clone(), )); let worker = DeliveryWorker::new( store, diff --git a/apps/push-relay-rs/src/migrate.rs b/apps/push-relay-rs/src/migrate.rs index ac7541a10d7..763f17831a6 100644 --- a/apps/push-relay-rs/src/migrate.rs +++ b/apps/push-relay-rs/src/migrate.rs @@ -9,6 +9,10 @@ const MIGRATIONS: &[(&str, &str)] = &[ "0002_binding_reader_preferences.sql", include_str!("../migrations/0002_binding_reader_preferences.sql"), ), + ( + "0003_binding_owner_authorization.sql", + include_str!("../migrations/0003_binding_owner_authorization.sql"), + ), ]; pub fn run(connection: &mut Connection) -> rusqlite::Result<()> { diff --git a/apps/push-relay-rs/src/requests.rs b/apps/push-relay-rs/src/requests.rs index f351523676d..bffeb4b1de5 100644 --- a/apps/push-relay-rs/src/requests.rs +++ b/apps/push-relay-rs/src/requests.rs @@ -5,7 +5,9 @@ use crate::protocol::{ApnsEnvironment, Issue, PushPreferences}; use crate::validate::{Checker, decode, is_url}; fn is_apns_token(value: &str) -> bool { - value.len() >= 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) + (64..=512).contains(&value.len()) + && value.len() % 2 == 0 + && value.bytes().all(|b| b.is_ascii_hexdigit()) } #[derive(Deserialize, Debug)] @@ -58,6 +60,9 @@ pub struct ClaimSourceActivation { pub source_label: Option, pub reader_id: Option, pub preferences: Option, + #[serde(default)] + pub owner_authorized: bool, + pub enrollment_token: Option, } impl ClaimSourceActivation { @@ -84,6 +89,11 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn rejects_oversized_apns_tokens() { + assert!(RegisterInstallation::parse(json!({ "app_id": "space", "apns_environment": "development", "apns_token": "ab".repeat(4096) })).is_err()); + } + #[test] fn register_installation_validation() { let ok = json!({ "app_id": "space", "apns_environment": "development", "apns_token": "ab".repeat(32) }); diff --git a/apps/push-relay-rs/src/service.rs b/apps/push-relay-rs/src/service.rs index 65365c3754a..cd347e02ee9 100644 --- a/apps/push-relay-rs/src/service.rs +++ b/apps/push-relay-rs/src/service.rs @@ -24,7 +24,15 @@ pub enum RelayError { impl From for RelayError { fn from(error: DbError) -> Self { - Self::Internal(error.0) + if error.0.contains("relay_admission_limited") { + http( + 429, + "relay_admission_limited", + "Relay registration or delivery budget exceeded", + ) + } else { + Self::Internal(error.0) + } } } @@ -67,6 +75,7 @@ pub struct RelayService { vault: DataVault, public_url: String, app_ids: HashSet, + enrollment_token_hash: Option, } impl RelayService { @@ -75,12 +84,14 @@ impl RelayService { vault: DataVault, public_url: String, app_ids: HashSet, + enrollment_token: Option, ) -> Self { Self { store, vault, public_url, app_ids, + enrollment_token_hash: enrollment_token.map(|token| credential_hash(&token)), } } @@ -184,6 +195,20 @@ impl RelayService { None => None, }; + if existing_source.is_none() + && !self + .enrollment_token_hash + .as_ref() + .zip(parsed.enrollment_token.as_ref()) + .is_some_and(|(expected, token)| credentials_match(token, expected)) + { + return Err(http( + 403, + "source_enrollment_required", + "Source creation requires operator enrollment approval", + )); + } + let installation_id = self .store .claim_activation_ticket(&credential_hash(&parsed.ticket), now) @@ -196,6 +221,21 @@ impl RelayService { ) })?; + let installation = self + .store + .find_installation(&installation_id) + .await? + .ok_or_else(|| http(410, "installation_invalid", "Installation is unavailable"))?; + if installation.app_id == crate::fanout::SPACE_APP_ID + && (!parsed.owner_authorized || parsed.reader_id.is_none()) + { + return Err(http( + 403, + "owner_required", + "Space notifications require an authenticated owner", + )); + } + let (source_id, source_secret) = match existing_source { Some(source) if source.origin != parsed.source_origin => { return Err(http( @@ -228,6 +268,7 @@ impl RelayService { installation_id: &installation_id, reader_id: parsed.reader_id.as_deref(), preferences: &parsed.preferences.unwrap_or_default(), + owner_authorized: parsed.owner_authorized, }) .await?; @@ -411,6 +452,62 @@ impl RelayService { mod tests { use super::*; + #[tokio::test] + async fn self_enrolled_installation_cannot_mint_a_source() { + let store = Store::open(":memory:").await.unwrap(); + store.migrate().await.unwrap(); + let relay = RelayService::new( + store, + DataVault::new(&"11".repeat(32)).unwrap(), + "https://relay.test".into(), + HashSet::from(["space".into()]), + None, + ); + let result = relay + .claim_source_activation( + None, + json!({ "ticket": "t".repeat(32), "source_origin": "https://victim.test" }), + ) + .await; + assert!(matches!(result, Err(RelayError::Http { status: 403, .. }))); + } + + #[tokio::test] + async fn approved_sources_still_need_owner_authority_for_space() { + let store = Store::open(":memory:").await.unwrap(); + store.migrate().await.unwrap(); + let relay = RelayService::new( + store.clone(), + DataVault::new(&"11".repeat(32)).unwrap(), + "https://relay.test".into(), + HashSet::from(["space".into()]), + Some("operator-approval-token-32-characters".into()), + ); + let installation = relay.register_installation(json!({ "app_id": "space", "apns_environment": "production", "apns_token": "ab".repeat(32) })).await.ok().unwrap(); + let authorization = format!( + "Installation {}.{}", + installation["installation_id"].as_str().unwrap(), + installation["installation_secret"].as_str().unwrap() + ); + let ticket = relay + .create_activation_ticket(Some(&authorization)) + .await + .ok() + .unwrap(); + let rejected = relay.claim_source_activation(None, json!({ "ticket": ticket["ticket"], "source_origin": "https://core.test", "enrollment_token": "operator-approval-token-32-characters" })).await; + assert!(matches!( + rejected, + Err(RelayError::Http { status: 403, .. }) + )); + let ticket = relay + .create_activation_ticket(Some(&authorization)) + .await + .ok() + .unwrap(); + let accepted = relay.claim_source_activation(None, json!({ "ticket": ticket["ticket"], "source_origin": "https://core.test", "enrollment_token": "operator-approval-token-32-characters", "owner_authorized": true, "reader_id": "owner" })).await; + assert!(accepted.is_ok()); + } + #[test] fn parses_authorization() { assert_eq!( diff --git a/apps/push-relay-rs/src/store.rs b/apps/push-relay-rs/src/store.rs index fbc4c6e9043..9cdc6348366 100644 --- a/apps/push-relay-rs/src/store.rs +++ b/apps/push-relay-rs/src/store.rs @@ -35,6 +35,7 @@ type Result = std::result::Result; pub struct InstallationRecord { pub id: String, pub secret_hash: String, + pub app_id: String, pub revoked_at: Option, } @@ -78,6 +79,7 @@ pub struct NewBinding<'a> { pub installation_id: &'a str, pub reader_id: Option<&'a str>, pub preferences: &'a PushPreferences, + pub owner_authorized: bool, } fn delivery(row: &Row<'_>) -> rusqlite::Result { @@ -106,6 +108,10 @@ pub struct Store { connection: Connection, } +fn admission_error() -> rusqlite::Error { + rusqlite::Error::InvalidParameterName("relay_admission_limited".into()) +} + impl Store { pub async fn open(path: &str) -> Result { let connection = Connection::open(path).await?; @@ -151,6 +157,14 @@ impl Store { ); self.connection .call(move |conn| { + conn.execute("DELETE FROM push_installations WHERE updated_at < (unixepoch() - 86400) * 1000 AND NOT EXISTS (SELECT 1 FROM push_bindings WHERE installation_id = push_installations.id AND revoked_at IS NULL)", [])?; + let admitted: bool = conn.query_row( + "SELECT (SELECT count(*) FROM push_installations) < 100000 + AND (SELECT count(*) FROM push_installations WHERE created_at >= (unixepoch() - 3600) * 1000) < 1000 + AND NOT EXISTS (SELECT 1 FROM push_installations WHERE app_id = ?1 AND apns_environment = ?2 AND token_hash = ?3 AND revoked_at IS NULL)", + params![values.1, values.2, values.3], |row| row.get(0), + )?; + if !admitted { return Err(admission_error()); } conn.execute( "INSERT INTO push_installations (id, app_id, apns_environment, token_hash, token_ciphertext, secret_hash) @@ -168,12 +182,13 @@ impl Store { self.connection .call(move |conn| { conn.query_row( - "SELECT id, secret_hash, revoked_at FROM push_installations WHERE id = ?1", + "SELECT id, secret_hash, app_id, revoked_at FROM push_installations WHERE id = ?1", [id], |row| { Ok(InstallationRecord { id: row.get("id")?, secret_hash: row.get("secret_hash")?, + app_id: row.get("app_id")?, revoked_at: row.get("revoked_at")?, }) }, @@ -226,6 +241,9 @@ impl Store { ); self.connection .call(move |conn| { + conn.execute("DELETE FROM push_activation_tickets WHERE expires_at < unixepoch() * 1000", [])?; + let count: i64 = conn.query_row("SELECT count(*) FROM push_activation_tickets WHERE installation_id = ?1", [&values.2], |row| row.get(0))?; + if count >= 10 { return Err(admission_error()); } conn.execute( "INSERT INTO push_activation_tickets (id, ticket_hash, installation_id, expires_at) VALUES (?1, ?2, ?3, ?4)", @@ -273,6 +291,8 @@ impl Store { ); self.connection .call(move |conn| { + let count: i64 = conn.query_row("SELECT count(*) FROM push_sources", [], |row| row.get(0))?; + if count >= 10000 { return Err(admission_error()); } conn.execute( "INSERT INTO push_sources (id, origin, label, secret_ciphertext) VALUES (?1, ?2, ?3, ?4)", values, @@ -309,14 +329,20 @@ impl Store { input.installation_id.to_owned(), input.reader_id.map(str::to_owned), serde_json::to_value(input.preferences).map_err(|e| DbError(e.to_string()))?, + input.owner_authorized, ); self.connection .call(move |conn| { + let existing: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM push_bindings WHERE source_id = ?1 AND installation_id = ?2)", params![values.1, values.2], |row| row.get(0))?; + if !existing { + let allowed: bool = conn.query_row("SELECT (SELECT count(*) FROM push_bindings WHERE source_id = ?1) < 1000 AND (SELECT count(*) FROM push_bindings WHERE installation_id = ?2) < 32 AND (SELECT count(*) FROM push_bindings) < 100000", params![values.1, values.2], |row| row.get(0))?; + if !allowed { return Err(admission_error()); } + } conn.query_row( - "INSERT INTO push_bindings (id, source_id, installation_id, reader_id, preferences) - VALUES (?1, ?2, ?3, ?4, ?5) + "INSERT INTO push_bindings (id, source_id, installation_id, reader_id, preferences, owner_authorized) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT (source_id, installation_id) - DO UPDATE SET revoked_at = NULL, reader_id = excluded.reader_id, + DO UPDATE SET revoked_at = NULL, reader_id = excluded.reader_id, owner_authorized = excluded.owner_authorized, updated_at = unixepoch() * 1000 RETURNING id", values, @@ -431,6 +457,19 @@ impl Store { .call(move |conn| { let tx = conn.transaction()?; let (event_id, source_id, delivery_id, event_type, subject) = values; + tx.execute("DELETE FROM push_events WHERE created_at < ?1", [now - 30 * 24 * 60 * 60 * 1000])?; + let existing: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM push_events WHERE source_id = ?1 AND id = ?2)", params![source_id, event_id], |row| row.get(0))?; + if !existing { + let admitted: bool = tx.query_row( + "SELECT (SELECT count(*) FROM push_events WHERE source_id = ?1 AND created_at >= ?2) < 60 + AND (SELECT count(*) FROM push_events WHERE source_id = ?1 AND created_at >= ?3) < 1000 + AND (SELECT count(*) FROM push_events WHERE created_at >= ?3) < 10000 + AND (SELECT count(*) FROM push_deliveries WHERE status IN ('pending', 'processing', 'retrying')) < 10000 + AND (SELECT count(*) FROM push_deliveries WHERE created_at >= ?3) < 100000", + params![source_id, now - 60000, now - 86400000], |row| row.get(0), + )?; + if !admitted { return Err(admission_error()); } + } let inserted = tx.execute( "INSERT INTO push_events (id, source_id, delivery_id, event_type, subject, payload, event_time, created_at) @@ -447,6 +486,8 @@ impl Store { } else { tx.execute(FANOUT_DELIVERY_INSERT_SQL, params![event_id, source_id, now, app_id, reader_id, preference_key])? as u64 }; + let within_budget: bool = tx.query_row("SELECT (SELECT count(*) FROM push_deliveries WHERE status IN ('pending', 'processing', 'retrying')) <= 10000 AND (SELECT count(*) FROM push_deliveries WHERE created_at >= ?1) <= 100000", [now - 86400000], |row| row.get(0))?; + if !within_budget { return Err(admission_error()); } tx.commit()?; Ok(deliveries) }) @@ -591,7 +632,7 @@ mod tests { id, app_id: app, apns_environment: ApnsEnvironment::Production, - token_hash: &"a".repeat(64), + token_hash: &crate::crypto::credential_hash(id), token_ciphertext: "encrypted-token", secret_hash: &"b".repeat(64), }) @@ -604,6 +645,7 @@ mod tests { installation_id: id, reader_id: reader, preferences, + owner_authorized: app == "space" && reader.is_some(), }) .await .unwrap(); @@ -629,7 +671,14 @@ mod tests { #[tokio::test] async fn activation_tickets_are_single_use_and_expire_at_the_boundary() { let store = store().await; - bind(&store, "ins_1", "space", None, &PushPreferences::default()).await; + bind( + &store, + "ins_1", + "space", + Some("owner"), + &PushPreferences::default(), + ) + .await; let now = Utc::now(); let expires = now + TimeDelta::minutes(10); let hash = "c".repeat(64); @@ -676,7 +725,7 @@ mod tests { async fn events_route_by_app_reader_preferences_and_revocation_without_duplicates() { let store = store().await; let preferences = PushPreferences::default(); - bind(&store, "space", "space", None, &preferences).await; + bind(&store, "space", "space", Some("owner"), &preferences).await; bind(&store, "r1", "yohaku", Some("reader1"), &preferences).await; bind(&store, "r2", "yohaku", Some("reader2"), &preferences).await; let disabled = PushPreferences { @@ -784,6 +833,7 @@ mod tests { installation_id: "r1", reader_id: Some("reader3"), preferences: &preferences, + owner_authorized: false, }) .await .unwrap(); @@ -800,7 +850,14 @@ mod tests { #[tokio::test] async fn event_acceptance_rolls_back_when_fanout_fails() { let store = store().await; - bind(&store, "ins_1", "space", None, &PushPreferences::default()).await; + bind( + &store, + "ins_1", + "space", + Some("owner"), + &PushPreferences::default(), + ) + .await; store.connection.call(|conn| conn.execute_batch( "CREATE TRIGGER reject_delivery BEFORE INSERT ON push_deliveries BEGIN SELECT RAISE(ABORT, 'disk failure fixture'); END;" )).await.unwrap(); @@ -828,7 +885,14 @@ mod tests { #[tokio::test] async fn claims_do_not_overlap_and_retries_wait_until_due() { let store = store().await; - bind(&store, "ins_1", "space", None, &PushPreferences::default()).await; + bind( + &store, + "ins_1", + "space", + Some("owner"), + &PushPreferences::default(), + ) + .await; let now = Utc::now(); store .accept_event("src_1", "d1", &comment("c1"), now) @@ -940,4 +1004,89 @@ mod tests { store.connection.close().await.unwrap(); std::fs::remove_file(path).unwrap(); } + #[tokio::test] + async fn unauthorized_space_binding_never_receives_owner_notifications() { + let store = store().await; + bind( + &store, + "anonymous", + "space", + None, + &PushPreferences::default(), + ) + .await; + assert_eq!( + store + .accept_event("src_1", "d1", &comment("private"), Utc::now()) + .await + .unwrap(), + 0 + ); + } + + #[tokio::test] + async fn event_budget_is_atomic_and_retries_remain_idempotent() { + let store = store().await; + let now = Utc::now(); + for i in 0..60 { + store + .accept_event("src_1", &format!("d{i}"), &comment(&format!("e{i}")), now) + .await + .unwrap(); + } + assert!( + store + .accept_event("src_1", "overflow", &comment("overflow"), now) + .await + .is_err() + ); + assert!( + store + .accept_event("src_1", "retry", &comment("e0"), now) + .await + .is_ok() + ); + assert!( + store + .accept_event( + "src_1", + "later", + &comment("later"), + now + TimeDelta::seconds(61) + ) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn repeated_device_tokens_cannot_allocate_more_rows() { + let store = store().await; + bind( + &store, + "device", + "space", + Some("owner"), + &PushPreferences::default(), + ) + .await; + let result = store + .create_installation(NewInstallation { + id: "duplicate", + app_id: "space", + apns_environment: ApnsEnvironment::Production, + token_hash: &crate::crypto::credential_hash("device"), + token_ciphertext: "ciphertext", + secret_hash: &"c".repeat(64), + }) + .await; + assert!(result.is_err()); + assert!( + store + .find_installation("duplicate") + .await + .unwrap() + .is_none() + ); + } } diff --git a/apps/telemetry/src/index.ts b/apps/telemetry/src/index.ts index 4e077a43b52..e378ec27e8e 100644 --- a/apps/telemetry/src/index.ts +++ b/apps/telemetry/src/index.ts @@ -1,5 +1,8 @@ interface Env { DB: D1Database + COLLECT_LIMITER: { + limit: (input: { key: string }) => Promise<{ success: boolean }> + } } interface TelemetryPayload { @@ -35,7 +38,42 @@ function safeJsonStringify(value: unknown): string { async function handleCollect(request: Request, env: Env): Promise { try { - const payload = (await request.json()) as TelemetryPayload + const admission = await env.COLLECT_LIMITER.limit({ + key: request.headers.get('CF-Connecting-IP') || 'unknown', + }) + if (!admission.success) return jsonResponse({ error: 'Rate limited' }, 429) + const reader = request.body?.getReader() + if (!reader) return jsonResponse({ error: 'Missing body' }, 400) + const chunks: Uint8Array[] = [] + let size = 0 + try { + for (;;) { + const { value, done } = await reader.read() + if (done) break + size += value.byteLength + if (size > 1024) return jsonResponse({ error: 'Body too large' }, 413) + chunks.push(value) + } + } finally { + await reader.cancel() + reader.releaseLock() + } + let payload: TelemetryPayload + try { + payload = JSON.parse(await new Blob(chunks).text()) + } catch { + return jsonResponse({ error: 'Invalid JSON' }, 400) + } + if ( + !payload || + typeof payload.instanceId !== 'string' || + typeof payload.version !== 'string' || + typeof payload.event !== 'string' || + (payload.nodeVersion !== undefined && + typeof payload.nodeVersion !== 'string') + ) { + return jsonResponse({ error: 'Invalid fields' }, 400) + } if (!payload.instanceId || !payload.version || !payload.event) { return jsonResponse({ error: 'Missing required fields' }, 400) @@ -56,17 +94,24 @@ async function handleCollect(request: Request, env: Env): Promise { return jsonResponse({ error: 'Field too long' }, 400) } - await env.DB.prepare( - 'INSERT INTO telemetry (instance_id, version, node_version, event) VALUES (?, ?, ?, ?)', + const result = await env.DB.prepare( + `INSERT INTO telemetry (instance_id, version, node_version, event) SELECT ?, ?, ?, ? + WHERE (SELECT count(*) FROM telemetry) < 100000 + AND (SELECT count(*) FROM telemetry WHERE created_at >= datetime('now', '-1 day')) < 10000 + AND NOT EXISTS (SELECT 1 FROM telemetry WHERE instance_id = ? AND event = ? AND created_at >= datetime('now', '-1 minute'))`, ) .bind( payload.instanceId, payload.version, payload.nodeVersion || null, payload.event, + payload.instanceId, + payload.event, ) .run() + if (result.meta.changes === 0) + return jsonResponse({ error: 'Collection budget exceeded' }, 429) return jsonResponse({ success: true }) } catch (error) { console.error('Collect error:', error) diff --git a/apps/telemetry/wrangler.toml b/apps/telemetry/wrangler.toml index 4508cf9c84a..0aa6d4359fb 100644 --- a/apps/telemetry/wrangler.toml +++ b/apps/telemetry/wrangler.toml @@ -10,3 +10,8 @@ crons = ["0 3 * * *"] binding = "DB" database_name = "mx-telemetry" database_id = "26369d03-d91b-48e5-97c6-0523c0b51417" + +[[ratelimits]] +name = "COLLECT_LIMITER" +namespace_id = "1002" +simple = { limit = 10, period = 60 } diff --git a/dockerfile b/dockerfile index 3f20748a0c0..b05cb6a5b38 100644 --- a/dockerfile +++ b/dockerfile @@ -47,9 +47,9 @@ COPY --from=builder /app/out . RUN npm i sharp -g RUN npm i sharp -# Nest 12 WsAdapter loads `ws` at runtime via createRequire(import.meta.url). +# WsAdapter and sandbox workers load these packages outside the Vite bundle. # The production image only copies the Vite bundle, so install it next to sharp. -RUN npm i ws@8.21.3 +RUN npm i ws@8.21.3 undici@8.11.2 COPY --chmod=755 docker-entrypoint.sh . COPY docker-browser-watchdog.mjs . diff --git a/docs/security/2026-10-23-daybreak-blue-remediation.md b/docs/security/2026-10-23-daybreak-blue-remediation.md new file mode 100644 index 00000000000..e47786b2adc --- /dev/null +++ b/docs/security/2026-10-23-daybreak-blue-remediation.md @@ -0,0 +1,59 @@ +# Daybreak Blue 扫描审查与修复 + +扫描 ID：`cdb5ad99-2e3f-4c0a-b59e-79eb7cfdf134`。输入为用户提供的 `scan-bundle (1).tar.gz`，冻结版本为 `038657b8c995cff0cd769ef7ed16dd209a4aea2f`。 + +报告列出的 23 个问题均在当前代码中确认了相应缺口。以下修复保存在本地工作区，尚未发布。扫描本身为静态审查；本次补充了行为测试、真实 PostgreSQL/SQLite 验证、Chromium 请求证据和构建验证。 + +| # | 扫描 ruleId | 审查结果及处理 | +| --- | ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `public-comment-sensitive-data` | 公开评论改为明确字段投影，覆盖详情、分页、回复和嵌套 reader；去掉邮箱、IP、UA、位置、内部元数据及管理状态。所有者继续收到管理字段，普通已登录读者也走公开投影。SDK 同步可选字段。 | +| 2 | `note-direct-id-visibility-bypass` | ID 读取补齐定时发布与口令检查；公开导航列表不包含受保护的当前条目。 | +| 3 | `scheduled-note-alternate-representation` | 定时笔记在公开详情入口提前拒绝，发生在翻译、摘要、TTS、富化和 Lexical 内容处理之前。 | +| 4 | `websocket-stale-authorization` | 所有者转移/撤销触发管理员 socket 失效，并经 Redis bus 通知其他实例；处理中握手也会失效。HTTP 会话使用数据库中的当前角色。 | +| 5 | `websocket-query-credential` | 管理 WebSocket 删除 query token 回退，保留 cookie 与认证头。 | +| 6 | `serverless-relative-module-sandbox-escape` | require 禁止相对/绝对路径及包路径穿越，阻断可写资产中的 CommonJS 文件在 VM 外执行。保留已有的受控内置模块与安装包功能。 | +| 7 | `serverless-waiter-queue-exhaustion` | 限制等候请求数量，等候时间计入执行预算；创建和分配 worker 时预留容量，避免并发超配。 | +| 8 | `sandbox-websocket-private-egress` | WebSocket 与 fetch 都在实际 socket DNS 查询处检查地址；拦截 localhost、私网、映射 IPv6 与私网重绑定。 | +| 9 | `outbound-url-address-validation-bypass` | 统一规范化 IPv6 判断；HTTP 使用配套的 Undici fetch/Agent，在连接时检查并使用同一组 DNS 地址，覆盖 OG、oEmbed、图片、头像及 webhook。 | +| 10 | `browser-post-navigation-destination-check` | 浏览器所有 HTTP/CONNECT 流量经过本地出站代理，逐连接验证并固定地址，涵盖重定向、脚本、子资源和 WebSocket。关闭 QUIC 与未代理的 WebRTC UDP。 | +| 11 | `unbounded-image-response-buffering` | 图片和头像流式读取，累计上限 8 MiB；图片解码限制 4000 万像素。 | +| 12 | `safe-fetch-body-timeout-gap` | 超时覆盖响应正文，所有退出路径中止连接并清理计时器。 | +| 13 | `reader-upload-quota-race` | 每个 reader 使用 PostgreSQL advisory transaction lock，覆盖配额查询、文件接收、存储和记账；剩余字节数也限制单次文件大小。 | +| 14 | `backup-decompression-resource-exhaustion` | ZIP 限 10000 条目、单条目 256 MiB、展开总量 1 GiB；流式写入，恢复文件逐项复制。保留路径与符号链接检查。 | +| 15 | `telemetry-unbounded-ingestion` | 请求体限 1 KiB；Cloudflare 限流、同实例/事件一分钟去重、每日 10000 条和总计 100000 条上限；条件插入由 SQLite 原子执行。 | +| 16 | `subscription-missing-owner-verification` | 邮箱持有人经 15 分钟有效的签名链接确认后才激活或修改偏好；未验证记录不投递，确认邮件有跨实例冷却和入口限流。 | +| 17 | `s3-unvalidated-redirect` | 所有 S3 请求拒绝重定向。 | +| 18 | `s3-unbounded-transport-response` | S3 请求及正文共用 60 秒期限，控制/错误响应流式限 1 MiB。 | +| 19 | `push-owner-binding-authz` | Core 从可信请求权限生成所有者声明；Relay 验证 Space 绑定必须带所有者授权及 reader 身份，扇出只选择获授权的 Space 绑定。 | +| 20 | `relay-installation-storage-abuse` | 设备 token 编码长度受限，拒绝活跃重复 token；设备注册速率、持久化行数、票据数及无绑定设备保留时间均受限。 | +| 21 | `relay-self-issued-source-amplification` | 新来源必须携带运营方注册凭证；绑定、事件速率、每日发送量、队列长度及保留时间有硬上限，事务回滚超额扇出。 | +| 22 | `ios-cross-origin-stale-token` | Keychain account 按规范化来源隔离；配对请求不发送旧 token。旧的全局 token 不自动迁移。 | +| 23 | `ios-plaintext-owner-credentials` | 所有服务器端点必须 HTTPS，认证中间件也不向 HTTP 发送 bearer。更新本地验证说明。 | + +## 升级要求 + +- Relay 设置 `PUSH_RELAY_ENROLLMENT_TOKEN`，获准接入的 Core 设置相同的 `MX_PUSH_RELAY_ENROLLMENT_TOKEN`（32–256 字符）。未配置时禁止创建新来源，已有来源仍使用自己的凭证。 +- Relay 执行新增 SQLite migration 0003。旧 Space 绑定没有所有者标记，需要所有者重新配对并激活推送。请同时更新 Relay 与 Core。 +- iOS 旧全局 token 被隔离，升级后重新配对 HTTPS 地址；局域网开发也需要受信任的 TLS 证书或 HTTPS 代理。 +- 旧邮箱订阅若没有 verified 标记，将停止收到内容通知，收件人需重新确认订阅。公开订阅调用成功表示确认邮件已请求发送。 +- 遥测需随 `wrangler.toml` 一起部署，以提供 `COLLECT_LIMITER` 绑定；本次没有修改线上 Worker、D1 或账户策略。 +- 备份、上传、图片、S3 和 Relay 的新资源上限可能拒绝超大输入，具体阈值见上表及 Relay README。 + +APNs token 使用有上限的可变长度编码，而不是扫描建议的固定长度；Apple 明确要求避免假定 token 大小，见 [Apple 通知请求文档](https://developer.apple.com/documentation/usernotifications/sending-notification-requests-to-apns)。 + +沙箱继续只面向可信所有者脚本；本次封堵报告指出的可写模块入口和网络/排队缺口，未更换 Node VM 运行机制。 + +## 验证记录 + +- Core：全量 `pnpm -C apps/core exec vitest run --maxWorkers=4` 有 3090 个测试通过、25 个跳过；唯一未执行文件为 `tts-runtime.adapter.spec.ts`，原因是测试 Redis 的随机端口 52506 冲突。该文件使用单 worker 独立复跑，14 个测试全部通过。合并结果为 3104 通过、11 个按条件跳过。后续订阅冷却、浏览器代理生命周期和网络改动另有针对性复跑。 +- `pnpm typecheck`：响应 envelope 检查、AI 校验检查和 Core 类型检查通过。 +- 管理端 `pnpm -C apps/admin typecheck` 通过；公共 SDK 的 200 个测试、推送协议的 26 个测试和协议类型检查通过；WebSocket SDK 测试通过。 +- Rust Relay：在 `rust:1` 容器运行 `cargo test --locked`，27 个测试通过，包括真实 SQLite 的配额、重复 token、来源注册、Space 所有者授权、幂等重试和事务回滚。 +- iOS Core：`swift test --package-path apps/ios/Packages/SpaceCore`，49 个测试通过。覆盖 HTTP 拒绝、配对凭证清理、IPv6 和默认端口的来源规范化。 +- iOS App：`xcodebuild -project apps/ios/Space.xcodeproj -scheme Space -destination 'generic/platform=iOS Simulator' -skipPackagePluginValidation CODE_SIGNING_ALLOWED=NO build` 通过。宿主工具使用其默认 SDK；不能全局强制 `-sdk iphonesimulator`，否则 OpenAPI 生成器被错误地编译为 iOS 库。 +- Core 的 `pnpm -C apps/core run bundle` 通过；Node 26 容器直接加载沙箱 worker 与运行时 Undici 并成功执行。 +- Chromium 实测：私网目标返回代理的 HTTP 403，私网测试服务收到 0 次请求。DNS 重绑定另经真实 Undici 连接器验证。 +- 遥测使用真实 SQLite 验证去重及每日预算；Worker 独立类型检查通过。 +- 修改范围 ESLint 无错误（raw sandbox JS 按仓库规则忽略），`git diff --check` 通过。 + +未进行生产部署、线上账号策略检查、APNs 真机发送或 Daybreak Blue 远端重扫；未更改全仓扫描覆盖率结论。 diff --git a/packages/api-client/controllers/subscribe.ts b/packages/api-client/controllers/subscribe.ts index fed1f8ca846..8b2bdf4def0 100644 --- a/packages/api-client/controllers/subscribe.ts +++ b/packages/api-client/controllers/subscribe.ts @@ -3,6 +3,7 @@ import type { IController } from '~/interfaces/controller' import type { IRequestHandler } from '~/interfaces/request' import type { SubscribeType } from '~/models/subscribe' import { autoBind } from '~/utils/auto-bind' + import type { HTTPClient } from '../core' declare module '@mx-space/api-client' { @@ -38,6 +39,7 @@ export class SubscribeController implements IController { }>() } + /** Sends a confirmation email. Preferences change only after the recipient confirms. */ subscribe(email: string, types: SubscribeType[]) { return this.proxy.post({ data: { diff --git a/packages/api-client/models/comment.ts b/packages/api-client/models/comment.ts index cdb99cfaf40..9a824fa3b84 100644 --- a/packages/api-client/models/comment.ts +++ b/packages/api-client/models/comment.ts @@ -71,14 +71,14 @@ export interface CommentModel { createdAt: string refType: CollectionRefTypes refId: string - state: number + state?: number author: string | null text: string /** 仅鉴权 admin 端点附带；公开端点由 `CommentFilterEmailInterceptor` 剥离。 */ mail?: string | null url: string | null - ip: string | null - agent: string | null + ip?: string | null + agent?: string | null pin: boolean avatar: string | null @@ -88,10 +88,10 @@ export interface CommentModel { replyCount: number latestReplyAt: string | null isDeleted: boolean - deletedAt: string | null + deletedAt?: string | null isWhispers: boolean - location: string | null + location?: string | null authProvider: string | null readerId: string | null diff --git a/packages/push-protocol/src/protocol.ts b/packages/push-protocol/src/protocol.ts index 6624a6de4fc..31588f9a4e3 100644 --- a/packages/push-protocol/src/protocol.ts +++ b/packages/push-protocol/src/protocol.ts @@ -204,7 +204,7 @@ export const RegisterInstallationSchema = z .object({ app_id: z.string().min(1).max(64), apns_environment: z.enum(['development', 'production']), - apns_token: z.string().regex(/^[\da-f]{64,}$/i), + apns_token: z.string().regex(/^(?:[\da-f]{2}){32,256}$/i), }) .strict() @@ -220,6 +220,8 @@ export const ClaimSourceActivationSchema = z source_label: z.string().trim().min(1).max(128).optional(), reader_id: resourceId.optional(), preferences: PushPreferencesSchema.optional(), + owner_authorized: z.boolean().optional(), + enrollment_token: z.string().min(32).max(256).optional(), }) .strict() diff --git a/packages/ws-client/test/client.spec.ts b/packages/ws-client/test/client.spec.ts index d2d2dbf4e0b..0f3fc5a7645 100644 --- a/packages/ws-client/test/client.spec.ts +++ b/packages/ws-client/test/client.spec.ts @@ -118,7 +118,7 @@ describe('createWsClient', () => { const client = createWsClient({ url: `ws://127.0.0.1:${port}`, - query: { token: 'abc', room: '1' }, + query: { trace: 'abc', room: '1' }, webSocketImpl: NodeWebSocket as unknown as typeof WebSocket, pingIntervalMs: 60_000, }) @@ -127,7 +127,7 @@ describe('createWsClient', () => { }) const url = await requestUrl - expect(url).toContain('token=abc') + expect(url).toContain('trace=abc') expect(url).toContain('room=1') })