From 49b6605c0f0bc9d807cecc79b54288bfd6878d68 Mon Sep 17 00:00:00 2001 From: harveysang Date: Thu, 17 Sep 2026 23:48:04 +0800 Subject: [PATCH] Unify phone execution with host-specific visual policies and typed outcomes --- packages/device-runtime/src/actions.ts | 288 ++++++++++++++++++ packages/device-runtime/src/concurrency.ts | 50 +++ packages/device-runtime/src/device-fleet.ts | 197 ++++++++++++ packages/device-runtime/src/errors.ts | 46 +++ .../device-runtime/src/frame-comparison.ts | 19 ++ packages/device-runtime/src/image.ts | 21 ++ .../device-runtime/src/phone-controller.ts | 252 +++++++++++++++ .../device-runtime/src/phone-execution.ts | 188 ++++++++++++ .../tests/controller-contract.ts | 77 +++++ .../device-runtime/tests/image-fixture.ts | 2 + plugins/opengui/src/adb.ts | 288 +----------------- plugins/opengui/src/cli.ts | 13 +- plugins/opengui/src/concurrency.ts | 52 +--- plugins/opengui/src/daemon.ts | 9 +- plugins/opengui/src/device-fleet.ts | 192 +----------- plugins/opengui/src/errors.ts | 1 + plugins/opengui/src/image.ts | 20 +- plugins/opengui/src/phone-controller.ts | 189 +----------- plugins/opengui/src/phone-execution.ts | 181 +---------- plugins/opengui/tests/daemon.spec.ts | 10 + .../opengui/tests/runtime-contract.spec.ts | 4 + workbuddy-plugin/src/adb.ts | 288 +----------------- workbuddy-plugin/src/concurrency.ts | 51 +--- workbuddy-plugin/src/device-fleet.ts | 198 +----------- workbuddy-plugin/src/errors.ts | 47 +-- workbuddy-plugin/src/phone-controller.ts | 249 +-------------- workbuddy-plugin/src/phone-execution.ts | 189 +----------- workbuddy-plugin/src/screenshot.ts | 9 +- workbuddy-plugin/src/vision.ts | 22 +- .../tests/runtime-contract.spec.ts | 4 + 30 files changed, 1199 insertions(+), 1957 deletions(-) create mode 100644 packages/device-runtime/src/actions.ts create mode 100644 packages/device-runtime/src/concurrency.ts create mode 100644 packages/device-runtime/src/device-fleet.ts create mode 100644 packages/device-runtime/src/errors.ts create mode 100644 packages/device-runtime/src/frame-comparison.ts create mode 100644 packages/device-runtime/src/image.ts create mode 100644 packages/device-runtime/src/phone-controller.ts create mode 100644 packages/device-runtime/src/phone-execution.ts create mode 100644 packages/device-runtime/tests/controller-contract.ts create mode 100644 packages/device-runtime/tests/image-fixture.ts create mode 100644 plugins/opengui/src/errors.ts create mode 100644 plugins/opengui/tests/runtime-contract.spec.ts create mode 100644 workbuddy-plugin/tests/runtime-contract.spec.ts diff --git a/packages/device-runtime/src/actions.ts b/packages/device-runtime/src/actions.ts new file mode 100644 index 0000000..95abc3e --- /dev/null +++ b/packages/device-runtime/src/actions.ts @@ -0,0 +1,288 @@ +/** Opaque identity of one completed phone observation. */ +export type ObservationId = string & { readonly __observationId: unique symbol } + +/** + * Brand a validated or generated observation identifier. + * @param value Raw observation identifier. + * @returns The same string with its observation-id brand. + */ +export function ObservationId(value: string): ObservationId { + return value as ObservationId +} + +/** One row returned by `adb devices -l`. */ +export interface AdbDevice { + serial: string + state: string + model?: string + product?: string + device?: string +} + +/** The logical Android display coordinate space. */ +export interface ScreenSize { + width: number + height: number +} + +/** Device input dimensions plus the screenshot pixel space shown to the model. */ +export interface PhoneCoordinateSpace extends ScreenSize { + screenshotWidth: number + screenshotHeight: number +} + +/** Tight visible bounds of one target in the current screenshot. */ +export interface TargetBoundingBox { + left: number + top: number + right: number + bottom: number +} + +/** The closed set of operations exposed to the phone subagent. */ +export type PhoneAction = + | { action: 'observe' } + | { action: 'tap'; observationId: ObservationId; targetBBox: TargetBoundingBox } + | { action: 'swipe'; observationId: ObservationId; x1: number; y1: number; x2: number; y2: number; durationMs?: number } + | { action: 'text'; observationId: ObservationId; text: string } + | { action: 'key'; observationId: ObservationId; key: 'Back' | 'Home' | 'Enter' | 'AppSwitch' } + | { action: 'launch'; observationId: ObservationId; packageName: string } + | { action: 'wait'; observationId: ObservationId; waitMs: number } + +const KEY_CODES = { + Back: 'KEYCODE_BACK', + Home: 'KEYCODE_HOME', + Enter: 'KEYCODE_ENTER', + AppSwitch: 'KEYCODE_APP_SWITCH', +} as const + +const ADB_INPUT_TEXT = /^[A-Za-z0-9 .,_@:/+=!?-]*$/u + +/** Whether Android's shell input-text command can preserve this value exactly. */ +export function canUseAdbInputText(text: string): boolean { + return ADB_INPUT_TEXT.test(text) +} + +function requiredNumber(value: unknown, action: string, fields: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`opengui: ${action} requires ${fields}`) + } + return value +} + +function requiredObservationId(value: unknown, action: string): ObservationId { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`opengui: ${action} requires the current observationId`) + } + return ObservationId(value) +} + +function assertNever(value: never): never { + throw new Error(`opengui: unsupported validated action ${JSON.stringify(value)}`) +} + +function requiredTargetBBox(value: unknown): TargetBoundingBox { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('opengui: tap requires targetBBox from the current screenshot') + } + const input = value as Record + return { + left: requiredNumber(input.left, 'tap', 'targetBBox.left, top, right, and bottom'), + top: requiredNumber(input.top, 'tap', 'targetBBox.left, top, right, and bottom'), + right: requiredNumber(input.right, 'tap', 'targetBBox.left, top, right, and bottom'), + bottom: requiredNumber(input.bottom, 'tap', 'targetBBox.left, top, right, and bottom'), + } +} + +/** + * Convert untrusted tool arguments into the closed phone-action union. + * @param input Raw arguments supplied to the phone tool. + * @returns A validated action containing every required field. + */ +export function normalizePhoneAction(input: Record): PhoneAction { + switch (input.action) { + case 'observe': return { action: 'observe' } + case 'tap': + return { + action: 'tap', + observationId: requiredObservationId(input.observationId, 'tap'), + targetBBox: requiredTargetBBox(input.targetBBox), + } + case 'swipe': { + const action: Extract = { + action: 'swipe', + observationId: requiredObservationId(input.observationId, 'swipe'), + x1: requiredNumber(input.x1, 'swipe', 'x1, y1, x2, and y2'), + y1: requiredNumber(input.y1, 'swipe', 'x1, y1, x2, and y2'), + x2: requiredNumber(input.x2, 'swipe', 'x1, y1, x2, and y2'), + y2: requiredNumber(input.y2, 'swipe', 'x1, y1, x2, and y2'), + } + if (input.durationMs !== undefined) { + action.durationMs = requiredNumber(input.durationMs, 'swipe', 'an integer durationMs') + } + return action + } + case 'text': + if (typeof input.text !== 'string') throw new Error('opengui: text requires text as a string') + return { action: 'text', observationId: requiredObservationId(input.observationId, 'text'), text: input.text } + case 'key': + if (input.key !== 'Back' && input.key !== 'Home' && input.key !== 'Enter' && input.key !== 'AppSwitch') { + throw new Error('opengui: key requires one of Back, Home, Enter, or AppSwitch') + } + return { action: 'key', observationId: requiredObservationId(input.observationId, 'key'), key: input.key } + case 'launch': + if (typeof input.packageName !== 'string') throw new Error('opengui: launch requires packageName as a string') + return { action: 'launch', observationId: requiredObservationId(input.observationId, 'launch'), packageName: input.packageName } + case 'wait': + return { + action: 'wait', + observationId: requiredObservationId(input.observationId, 'wait'), + waitMs: requiredNumber(input.waitMs, 'wait', 'an integer waitMs'), + } + default: + throw new Error('opengui: unsupported action') + } +} + +/** + * Parse `adb devices -l` without treating offline/unauthorized rows as usable. + * @param output Standard output from `adb devices -l`. + * @returns Parsed device rows in their original order. + */ +export function parseDevices(output: string): AdbDevice[] { + return output.split(/\r?\n/u).slice(1).map(line => line.trim()).filter(Boolean).map((line) => { + const [serial = '', state = 'unknown', ...fields] = line.split(/\s+/u) + const attributes = new Map() + for (const field of fields) { + const separator = field.indexOf(':') + if (separator > 0) attributes.set(field.slice(0, separator), field.slice(separator + 1)) + } + const model = attributes.get('model') + const product = attributes.get('product') + const device = attributes.get('device') + return { + serial, + state, + ...(model === undefined ? {} : { model }), + ...(product === undefined ? {} : { product }), + ...(device === undefined ? {} : { device }), + } + }).filter(device => device.serial.length > 0) +} + +/** + * Pick one deterministic target for compatibility with single-device callers. + * @param devices Device rows returned by {@link parseDevices}. + * @returns The lexicographically first authorized serial. + */ +export function selectAuthorizedSerial(devices: readonly AdbDevice[]): string { + const serial = devices.filter(device => device.state === 'device') + .map(device => device.serial).sort((a, b) => a.localeCompare(b))[0] + if (serial === undefined) { + throw new Error('opengui: no authorized Android device is connected; connect at least one phone and accept its USB debugging prompt') + } + return serial +} + +/** + * Parse the logical display size used by screenshots and input. + * @param output Standard output from `adb shell wm size`. + * @returns The effective logical width and height. + */ +export function parseScreenSize(output: string): ScreenSize { + const match = output.match(/Override size:\s*(\d+)x(\d+)/iu) + ?? output.match(/Physical size:\s*(\d+)x(\d+)/iu) + ?? output.match(/(\d+)x(\d+)/u) + const width = Number(match?.[1] ?? 0) + const height = Number(match?.[2] ?? 0) + if (!(width > 0 && height > 0)) throw new Error('opengui: ADB did not report a valid display size') + return { width, height } +} + +function screenshotCoordinate(value: number, modelTotal: number, inputTotal: number, name: string): string { + if (!Number.isFinite(value) || value < 0 || value > modelTotal) { + throw new Error(`opengui: ${name} must be inside the current screenshot's ${modelTotal}px axis`) + } + return String(Math.round((value * inputTotal) / modelTotal)) +} + +function targetCenter(box: TargetBoundingBox, screen: PhoneCoordinateSpace): { x: string; y: string } { + if (!(box.right > box.left) || !(box.bottom > box.top)) { + throw new Error('opengui: targetBBox must have positive width and height') + } + if (box.left < 0 || box.top < 0 || box.right > screen.screenshotWidth || box.bottom > screen.screenshotHeight) { + throw new Error(`opengui: targetBBox must fit the current ${screen.screenshotWidth}x${screen.screenshotHeight} screenshot`) + } + return { + x: screenshotCoordinate((box.left + box.right) / 2, screen.screenshotWidth, screen.width, 'targetBBox center x'), + y: screenshotCoordinate((box.top + box.bottom) / 2, screen.screenshotHeight, screen.height, 'targetBBox center y'), + } +} + +/** + * Build the allowlisted device-side command for one validated operation. + * @param action A validated phone action. + * @param screen Device input dimensions and the screenshot pixel space shown to the model. + * @returns ADB arguments, or no arguments for a pure observation. + */ +export function actionCommand(action: PhoneAction, screen: PhoneCoordinateSpace): string[] | undefined { + switch (action.action) { + case 'observe': return undefined + case 'tap': { + const center = targetCenter(action.targetBBox, screen) + return ['shell', 'input', 'tap', center.x, center.y] + } + case 'swipe': { + const duration = action.durationMs ?? 300 + if (!Number.isInteger(duration) || duration < 50 || duration > 2_000) { + throw new Error('opengui: durationMs must be an integer from 50 through 2000') + } + return [ + 'shell', 'input', 'swipe', + screenshotCoordinate(action.x1, screen.screenshotWidth, screen.width, 'x1'), + screenshotCoordinate(action.y1, screen.screenshotHeight, screen.height, 'y1'), + screenshotCoordinate(action.x2, screen.screenshotWidth, screen.width, 'x2'), + screenshotCoordinate(action.y2, screen.screenshotHeight, screen.height, 'y2'), + String(duration), + ] + } + case 'text': { + if (action.text.length < 1 || action.text.length > 500 || !/^[A-Za-z0-9 .,!?_@+:'"()-]+$/u.test(action.text)) { + throw new Error('opengui: text must be 1-500 characters supported by Android adb input text') + } + return ['shell', 'input', 'text', action.text.replaceAll(' ', '%s')] + } + case 'key': return ['shell', 'input', 'keyevent', KEY_CODES[action.key]] + case 'launch': + if (!/^[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)+$/u.test(action.packageName)) { + throw new Error('opengui: packageName must be a valid Android application id') + } + return ['shell', 'monkey', '-p', action.packageName, '-c', 'android.intent.category.LAUNCHER', '1'] + case 'wait': { + if (!Number.isInteger(action.waitMs) || action.waitMs < 100 || action.waitMs > 10_000) { + throw new Error('opengui: waitMs must be an integer from 100 through 10000') + } + return undefined + } + default: return assertNever(action) + } +} + +/** + * Build the bounded ADB text-input sequence for Android's safe ASCII set. + * Unicode is intentionally rejected here so callers cannot mistake an + * unacknowledged shell clipboard attempt for successful input; it is handled + * by the acknowledged scrcpy control channel instead. + * @param text Text to enter in the focused Android field. + * @returns One allowlisted ADB argument array. + */ +export function textInputCommands(text: string): string[][] { + if (text.length < 1 || [...text].length > 500 || text.includes('\0')) { + throw new Error('opengui: text must contain 1-500 Unicode characters without NUL') + } + if (canUseAdbInputText(text)) { + return [['shell', 'input', 'text', text.replaceAll(' ', '%s')]] + } + throw new Error('opengui: Unicode text requires acknowledged scrcpy clipboard input') +} + diff --git a/packages/device-runtime/src/concurrency.ts b/packages/device-runtime/src/concurrency.ts new file mode 100644 index 0000000..8bd687a --- /dev/null +++ b/packages/device-runtime/src/concurrency.ts @@ -0,0 +1,50 @@ +/** Fair abort-aware permit pool used to bound expensive host/device media work. */ +export class AsyncSemaphore { + private active = 0 + private readonly queued: Array<{ + readonly signal: AbortSignal + readonly resolve: (release: () => void) => void + readonly reject: (reason: unknown) => void + }> = [] + + constructor(readonly limit: number) { + if (!Number.isSafeInteger(limit) || limit < 1) throw new Error('semaphore limit must be a positive integer') + } + + async acquire(signal: AbortSignal): Promise<() => void> { + signal.throwIfAborted() + if (this.active < this.limit) return this.grant() + return await new Promise<() => void>((resolvePermit, rejectPermit) => { + const request = { signal, resolve: (release: () => void): void => { + signal.removeEventListener('abort', onAbort) + resolvePermit(release) + }, reject: rejectPermit } + const onAbort = (): void => { + const index = this.queued.indexOf(request) + if (index >= 0) this.queued.splice(index, 1) + rejectPermit(signal.reason) + } + signal.addEventListener('abort', onAbort, { once: true }) + this.queued.push(request) + }) + } + + private grant(): () => void { + this.active += 1 + let released = false + return () => { + if (released) return + released = true + this.active -= 1 + while (this.queued.length > 0) { + const next = this.queued.shift()! + if (next.signal.aborted) { + next.reject(next.signal.reason) + continue + } + next.resolve(this.grant()) + break + } + } + } +} diff --git a/packages/device-runtime/src/device-fleet.ts b/packages/device-runtime/src/device-fleet.ts new file mode 100644 index 0000000..92c49fa --- /dev/null +++ b/packages/device-runtime/src/device-fleet.ts @@ -0,0 +1,197 @@ +import { randomUUID } from 'node:crypto' +import type { AdbDevice } from './actions.ts' + +/** Host-private device identity paired with its browser-safe presentation. */ +export interface FleetDevice { + readonly id: string + readonly serial: string + readonly label: string + readonly model?: string +} + +/** Browser-visible device choice; serial deliberately never crosses this seam. */ +export interface FleetDeviceView { + readonly id: string + readonly label: string + readonly model?: string + readonly selected: boolean +} + +/** Read-only connection state exposed by host device discovery. */ +export interface FleetDeviceStatusView { + readonly id: string + readonly label: string + readonly model?: string + readonly state: string + readonly connected: boolean + readonly authorized: boolean +} + +export interface DeviceFleetSnapshot { + readonly devices: readonly FleetDeviceView[] + readonly selectedDeviceIds: readonly string[] +} + +export type DiscoverDevices = (signal: AbortSignal) => Promise + +interface DeviceRecord { + id: string + serial: string +} + +function displayModel(device: AdbDevice): string | undefined { + const raw = device.model?.trim() + return raw ? raw.replaceAll('_', ' ') : undefined +} + +/** + * Owns authorized-device discovery, opaque browser identities, and the selection + * applied atomically to the next OpenGUI task. + */ +export class DeviceFleet { + private readonly records = new Map() + private readonly selected = new Set() + + constructor( + private readonly discover: DiscoverDevices, + private readonly createId: () => string = randomUUID, + ) {} + + async snapshot(signal: AbortSignal): Promise { + const devices = await this.discover(signal) + this.syncRecords(devices, false) + const authorized = devices.filter(device => device.state === 'device') + .toSorted((a, b) => a.serial.localeCompare(b.serial)) + + // Preserve the single-phone experience. With multiple phones, an explicit + // choice is required unless a prior still-connected choice already exists. + if (authorized.length === 1 && this.selected.size === 0) { + const only = this.records.get(authorized[0]!.serial) + if (only !== undefined) this.selected.add(only.id) + } + + const modelCounts = new Map() + for (const device of authorized) { + const model = displayModel(device) ?? 'Android 手机' + modelCounts.set(model, (modelCounts.get(model) ?? 0) + 1) + } + const modelIndexes = new Map() + const views = authorized.map((device): FleetDeviceView => { + const record = this.records.get(device.serial)! + const model = displayModel(device) + const base = model ?? 'Android 手机' + const index = (modelIndexes.get(base) ?? 0) + 1 + modelIndexes.set(base, index) + const label = (modelCounts.get(base) ?? 0) > 1 ? `${base} ${index}` : base + return { + id: record.id, + label, + ...(model === undefined ? {} : { model }), + selected: this.selected.has(record.id), + } + }) + return { + devices: views, + selectedDeviceIds: views.filter(device => device.selected).map(device => device.id), + } + } + + /** List every ADB row, including unauthorized and offline devices, without exposing serials. */ + async inspect(signal: AbortSignal): Promise { + const devices = (await this.discover(signal)).toSorted((a, b) => a.serial.localeCompare(b.serial)) + this.syncRecords(devices, true) + const modelCounts = new Map() + for (const device of devices) { + const model = displayModel(device) ?? 'Android phone' + modelCounts.set(model, (modelCounts.get(model) ?? 0) + 1) + } + const modelIndexes = new Map() + return devices.map((device): FleetDeviceStatusView => { + const record = this.records.get(device.serial)! + const model = displayModel(device) + const base = model ?? 'Android phone' + const index = (modelIndexes.get(base) ?? 0) + 1 + modelIndexes.set(base, index) + return { + id: record.id, + label: (modelCounts.get(base) ?? 0) > 1 ? `${base} ${index}` : base, + ...(model === undefined ? {} : { model }), + state: device.state, + connected: true, + authorized: device.state === 'device', + } + }) + } + + /** Materialize only rows in the caller's discovery snapshot; do not rediscover per phone. */ + resolveInspected(deviceId: string): string | undefined { + return [...this.records.values()].find(record => record.id === deviceId)?.serial + } + + async select(deviceIds: readonly string[], signal: AbortSignal): Promise { + const snapshot = await this.snapshot(signal) + const available = new Set(snapshot.devices.map(device => device.id)) + const unique = [...new Set(deviceIds)] + if (unique.some(id => !available.has(id))) { + throw new Error('opengui: device selection contains a disconnected or unknown phone') + } + this.selected.clear() + for (const id of unique) this.selected.add(id) + return this.snapshot(signal) + } + + /** Resolve browser-safe ids to current Host-private devices for an immediate operation. */ + async resolveConnected(deviceIds: readonly string[], signal: AbortSignal): Promise { + const snapshot = await this.snapshot(signal) + return this.materialize(snapshot, deviceIds) + } + + async selectedDevices(signal: AbortSignal): Promise { + const snapshot = await this.snapshot(signal) + if (snapshot.devices.length === 0) { + throw new Error('opengui: no authorized Android device is connected; connect a phone and accept its USB debugging prompt') + } + if (snapshot.selectedDeviceIds.length === 0) { + throw new Error('opengui: multiple phones are connected; select at least one device with opengui_open_session') + } + return this.materialize(snapshot, snapshot.selectedDeviceIds) + } + + private materialize(snapshot: DeviceFleetSnapshot, deviceIds: readonly string[]): readonly FleetDevice[] { + const views = new Map(snapshot.devices.map(device => [device.id, device])) + const byId = new Map([...this.records.values()].map(record => [record.id, record])) + return [...new Set(deviceIds)].map((id) => { + const view = views.get(id) + const record = byId.get(id) + if (view === undefined || record === undefined) { + throw new Error('opengui: device is disconnected or unknown') + } + return { + id, + serial: record.serial, + label: view.label, + ...(view.model === undefined ? {} : { model: view.model }), + } + }) + } + + + private syncRecords(devices: readonly AdbDevice[], includeUnavailable: boolean): void { + const connectedSerials = new Set(devices.map(device => device.serial)) + for (const [serial, record] of this.records) { + if (connectedSerials.has(serial)) continue + // Retain opaque identity for frozen sessions across physical reconnects. + // Current discovery still gates materialization; a remembered id grants no access. + this.selected.delete(record.id) + } + const candidates = includeUnavailable ? devices : devices.filter(device => device.state === 'device') + for (const device of candidates.toSorted((a, b) => { + const authorization = Number(b.state === 'device') - Number(a.state === 'device') + return authorization === 0 ? a.serial.localeCompare(b.serial) : authorization + })) { + if (!this.records.has(device.serial)) { + this.records.set(device.serial, { id: this.createId(), serial: device.serial }) + } + } + } +} diff --git a/packages/device-runtime/src/errors.ts b/packages/device-runtime/src/errors.ts new file mode 100644 index 0000000..b894c97 --- /dev/null +++ b/packages/device-runtime/src/errors.ts @@ -0,0 +1,46 @@ +export type ExecutionState = 'not_executed' | 'executed' | 'outcome_unknown' +export type Recovery = 'observe' | 'reconnect' | 'wait' | 'replan' | 'stop' + +/** Structured execution evidence, never an instruction to replay a mutation. */ +export class OpenGuiError extends Error { + constructor( + readonly code: string, + message: string, + readonly executionState: ExecutionState = 'not_executed', + readonly recovery: Recovery = 'stop', + ) { super(message); this.name = 'OpenGuiError' } +} + +export function errorInfo(error: unknown): { code: string; message: string; executionState: ExecutionState; recovery: Recovery } { + if (error instanceof OpenGuiError) return { code: error.code, message: error.message, executionState: error.executionState, recovery: error.recovery } + const message = error instanceof Error ? error.message : String(error) + const code = /stale|observe.*before|observation.*unavailable|current frame/u.test(message) ? 'observation_required' + : /invalid arguments|unknown tool|must be|is required/u.test(message) ? 'invalid_arguments' + : /waiting_for_display/u.test(message) ? 'waiting_for_display' + : /no screen progress|repeated action/u.test(message) ? 'no_progress' + : /operation.*limit|budget/u.test(message) ? 'budget_exhausted' + : /device offline|device not found|not connected/u.test(message) ? 'device_offline' + : /disconnected|ECONNRESET|EPIPE|ECONNREFUSED/u.test(message) ? 'connection_lost' + : /locked by another/u.test(message) ? 'device_busy' + : /cancelled|aborted|session is closed/u.test(message) ? 'cancelled' : 'operation_failed' + const recovery: Recovery = code === 'observation_required' ? 'observe' + : code === 'connection_lost' ? 'reconnect' + : code === 'waiting_for_display' || code === 'device_busy' || code === 'device_offline' ? 'wait' + : code === 'invalid_arguments' || code === 'no_progress' ? 'replan' : 'stop' + return { code, message, executionState: 'not_executed', recovery } +} + +/** Retry only explicitly transient, non-mutating work. */ +export async function retryRead(operation: () => Promise, signal: AbortSignal): Promise { + for (let attempt = 0; ; attempt++) { + signal.throwIfAborted() + try { return await operation() } catch (error) { + if (signal.aborted || attempt >= 2 || !/ECONNRESET|EPIPE|ETIMEDOUT|ECONNREFUSED|EAI_AGAIN|fetch failed|device offline|device .*not found|transport error|temporarily unavailable/iu.test(String(error))) throw error + await new Promise((resolve, reject) => { + const abort = (): void => { clearTimeout(timer); reject(signal.reason) } + const timer = setTimeout(() => { signal.removeEventListener('abort', abort); resolve() }, attempt === 0 ? 250 : 1000) + signal.addEventListener('abort', abort, { once: true }) + }) + } + } +} diff --git a/packages/device-runtime/src/frame-comparison.ts b/packages/device-runtime/src/frame-comparison.ts new file mode 100644 index 0000000..92974b8 --- /dev/null +++ b/packages/device-runtime/src/frame-comparison.ts @@ -0,0 +1,19 @@ +export interface VisualFrame { readonly pixels: Buffer; readonly width: number; readonly height: number } +export interface ImageRegion { readonly left: number; readonly top: number; readonly right: number; readonly bottom: number } + +/** Ignore isolated clock/cursor/compression noise, but never mask a target region. */ +export function frameChanged(before: VisualFrame, after: VisualFrame, region?: ImageRegion): boolean { + if (before.width !== after.width || before.height !== after.height) return true + const left = Math.max(0, Math.floor((region?.left ?? 0) * before.width)) + const right = Math.min(before.width, Math.ceil((region?.right ?? 1) * before.width)) + const top = Math.max(0, Math.floor((region?.top ?? 0) * before.height)) + const bottom = Math.min(before.height, Math.ceil((region?.bottom ?? 1) * before.height)) + const total = (right - left) * (bottom - top) + if (total <= 0) return true + let changed = 0 + for (let y = top; y < bottom; y++) for (let x = left; x < right; x++) { + const offset = (y * before.width + x) * 3 + if ([0, 1, 2].some(channel => Math.abs(before.pixels[offset + channel]! - after.pixels[offset + channel]!) > 20)) changed++ + } + return changed / total > (region ? 0.01 : 0.02) +} diff --git a/packages/device-runtime/src/image.ts b/packages/device-runtime/src/image.ts new file mode 100644 index 0000000..bbd98f3 --- /dev/null +++ b/packages/device-runtime/src/image.ts @@ -0,0 +1,21 @@ +/** Encoded screenshot shape; the macOS encoder has no native npm dependency. */ +export interface EncodedPhoneScreenshot { + readonly data: Buffer + readonly width: number + readonly height: number + readonly sourceWidth?: number + readonly sourceHeight?: number +} +/** Read screencap's current orientation and pixel size, not wm's natural size. */ +export function pngDimensions(data: Buffer): { width: number; height: number } { + if (data.length < 24 || data.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a' + || data.readUInt32BE(8) !== 13 || data.toString('ascii', 12, 16) !== 'IHDR') { + throw new Error('opengui: screenshot is not a PNG with an IHDR header') + } + const width = data.readUInt32BE(16) + const height = data.readUInt32BE(20) + if (width < 1 || height < 1 || width > 65_535 || height > 65_535) { + throw new Error('opengui: invalid screenshot dimensions') + } + return { width, height } +} diff --git a/packages/device-runtime/src/phone-controller.ts b/packages/device-runtime/src/phone-controller.ts new file mode 100644 index 0000000..5c2a6db --- /dev/null +++ b/packages/device-runtime/src/phone-controller.ts @@ -0,0 +1,252 @@ +import { createHash } from 'node:crypto' +import { + actionCommand, + canUseAdbInputText, + normalizePhoneAction, + parseScreenSize, + textInputCommands, +} from './actions.ts' +import type { ObservationId, PhoneCoordinateSpace } from './actions.ts' +import { AsyncSemaphore } from './concurrency.ts' +import type { EncodedPhoneScreenshot } from './image.ts' +import { PhoneExecutionState, PhoneOperationQueue, waitForPhoneUi } from './phone-execution.ts' +import type { PhoneExecutionSnapshot } from './phone-execution.ts' +import { errorInfo, OpenGuiError, retryRead } from './errors.ts' +import { frameChanged, type VisualFrame } from './frame-comparison.ts' +import { pngDimensions } from './image.ts' + +/** Host-neutral phone observation used by host adapters. */ +export interface RawPhoneObservation { + readonly observationId: ObservationId + readonly unchangedFromObservationId?: ObservationId + readonly serial: string + readonly width: number + readonly height: number + readonly foregroundPackage: string + readonly capturedAt?: string + readonly settled?: boolean + readonly image: { + readonly data: Buffer + readonly mediaType: 'image/jpeg' + readonly bytes: number + readonly width: number + readonly height: number + readonly name: string + } +} + +interface StoredObservation { + readonly value: RawPhoneObservation + readonly fingerprint: string +} + +export interface PhoneControllerOptions { + readonly runAdb: ( + args: readonly string[], + signal: AbortSignal, + buffer?: boolean, + ) => Promise + readonly discoverTarget: (signal: AbortSignal) => Promise + readonly validateTarget?: (serial: string, signal: AbortSignal) => Promise + readonly pasteUnicode: (serial: string, text: string, signal: AbortSignal) => Promise + readonly encodeScreenshot: (source: Buffer) => Promise + readonly maxOperations: () => number + readonly mediaPermits?: AsyncSemaphore + readonly now?: () => number + readonly sampleFrame?: (image: Buffer) => Promise + readonly readScreenSize?: boolean + readonly retryReads?: boolean + readonly settleIntervalMs?: number + readonly settleTimeoutMs?: number +} + +function currentPackage(output: string): string { + return output.match(/(?:mCurrentFocus|mFocusedApp)=[^\n]*?\bu\d+\s+([A-Za-z0-9._]+)\//u)?.[1] + ?? output.match(/(?:topResumedActivity|mResumedActivity)[^\n]*?\bu\d+\s+([A-Za-z0-9._]+)\//u)?.[1] + ?? '' +} + +/** + * Execution semantics shared by independent host runtimes. + * Host adapters provide only target discovery, process execution, and Unicode + * clipboard transport; safety and observation semantics live here. + */ +export class PhoneController { + private readonly observations = new WeakMap() + private readonly execution = new PhoneExecutionState() + private readonly queue = new PhoneOperationQueue() + private readonly mediaPermits: AsyncSemaphore + private readonly now: () => number + + constructor(private readonly options: PhoneControllerOptions) { + this.mediaPermits = options.mediaPermits ?? new AsyncSemaphore(2) + this.now = options.now ?? Date.now + } + + /** Freeze an actor to one Host-private device serial. */ + assignTarget(actor: object, serial: string): void { + this.execution.assignTarget(actor, serial) + } + + /** Return counters without exposing or mutating the current observation. */ + status(actor: object): PhoneExecutionSnapshot { + return this.execution.snapshot(actor) + } + + invalidate(actor: object): void { this.execution.consumeObservation(actor) } + + /** Observe without accepting arbitrary device commands. */ + observe(actor: object, signal: AbortSignal): Promise { + return this.execute(actor, { action: 'observe' }, signal) + } + + /** Execute exactly one validated operation and always return the resulting frame. */ + async execute( + actor: object, + input: Record, + signal: AbortSignal, + ): Promise { + return this.queue.run(actor, async () => { + let dispatched = false + try { + signal.throwIfAborted() + this.execution.beginOperation(actor, this.options.maxOperations()) + const normalized = { ...input } + delete normalized.verifyCurrentFrame + const action = normalizePhoneAction(normalized) + const serial = await this.targetFor(actor, signal) + await this.options.validateTarget?.(serial, signal) + if (action.action === 'observe') return this.capture(actor, serial, signal) + + const before = this.execution.current(actor, action.observationId) + const stored = this.observations.get(actor) + if (stored === undefined || stored.value.observationId !== action.observationId) { + throw new Error('opengui: current phone observation is unavailable') + } + const screen: PhoneCoordinateSpace = { + width: stored.value.width, + height: stored.value.height, + screenshotWidth: stored.value.image.width, + screenshotHeight: stored.value.image.height, + } + if (action.action === 'wait') { + this.execution.consumeObservation(actor) + await waitForPhoneUi(action.waitMs, signal) + return this.capture(actor, serial, signal) + } + + if (this.options.sampleFrame) { + // Consume the supplied credential while checking the current real frame. + // If it changed, the model must see a new observation, never reuse coordinates. + const current = await this.capture(actor, serial, signal) + const currentState = this.execution.current(actor, current.observationId) + const region = action.action === 'tap' ? { + left: action.targetBBox.left / stored.value.image.width, + top: action.targetBBox.top / stored.value.image.height, + right: action.targetBBox.right / stored.value.image.width, + bottom: action.targetBBox.bottom / stored.value.image.height, + } : undefined + if (current.width !== stored.value.width || current.height !== stored.value.height + || current.foregroundPackage !== stored.value.foregroundPackage + || (before.visual && currentState.visual && (frameChanged(before.visual, currentState.visual) || (region && frameChanged(before.visual, currentState.visual, region))))) { + throw new OpenGuiError('screen_changed', 'opengui: phone changed before dispatch; observe the new screen before acting', 'not_executed', 'observe') + } + + } + + const scrcpyText = action.action === 'text' && !canUseAdbInputText(action.text) + const command = action.action === 'text' ? undefined : actionCommand(action, screen) + const commands = action.action === 'text' + ? scrcpyText ? [] : textInputCommands(action.text) + : command === undefined ? [] : [command] + if (commands.length === 0 && !scrcpyText) { + throw new Error('opengui: action did not resolve to a device command') + } + + const signature = JSON.stringify(scrcpyText ? ['scrcpy-text', action.text] : commands) + this.execution.assertActionAllowed(actor, signature, before) + this.execution.consumeObservation(actor) + signal.throwIfAborted() + dispatched = true + if (scrcpyText) await this.options.pasteUnicode(serial, action.text, signal) + else for (const candidate of commands) await this.options.runAdb(['-s', serial, ...candidate], signal) + + const after = this.options.sampleFrame + ? await this.captureSettled(actor, serial, signal) : await this.capture(actor, serial, signal) + const afterState = this.execution.current(actor, after.observationId) + this.execution.recordActionResult(actor, signature, before, afterState) + return after + } catch (error) { + this.execution.consumeObservation(actor) + const info = errorInfo(error) + throw new OpenGuiError(info.code, info.message, dispatched ? 'outcome_unknown' : info.executionState, dispatched ? 'observe' : info.recovery) + } + }) + } + + private async captureSettled(actor: object, serial: string, signal: AbortSignal): Promise { + const interval = this.options.settleIntervalMs ?? 250 + const timeout = this.options.settleTimeoutMs ?? 2000 + const deadline = Date.now() + timeout + let previous = await this.capture(actor, serial, signal) + while (Date.now() + interval <= deadline) { + const before = this.execution.current(actor, previous.observationId) + await waitForPhoneUi(interval, signal) + const current = await this.capture(actor, serial, signal) + const after = this.execution.current(actor, current.observationId) + if (before.visual && after.visual && !frameChanged(before.visual, after.visual)) return { ...current, settled: true } + previous = current + } + return { ...previous, settled: false } + } + + private async targetFor(actor: object, signal: AbortSignal): Promise { + return this.execution.resolveTarget(actor, () => this.options.discoverTarget(signal)) + } + + private async capture(actor: object, serial: string, signal: AbortSignal): Promise { + this.execution.consumeObservation(actor) + const releaseMedia = await this.mediaPermits.acquire(signal) + try { + const read = () => Promise.all([ + this.options.readScreenSize ? this.options.runAdb(['-s', serial, 'shell', 'wm', 'size'], signal) : Promise.resolve(''), + this.options.runAdb(['-s', serial, 'shell', 'dumpsys', 'window', 'windows'], signal), + this.options.runAdb(['-s', serial, 'exec-out', 'screencap', '-p'], signal, true), + ]) + const [sizeRaw, focusRaw, pngRaw] = await (this.options.retryReads ? retryRead(read, signal) : read()) + const png = Buffer.isBuffer(pngRaw) ? pngRaw : Buffer.from(pngRaw) + const screen = this.options.readScreenSize ? parseScreenSize(String(sizeRaw)) : pngDimensions(png) + const encoded = await this.options.encodeScreenshot(png) + signal.throwIfAborted() + const fingerprint = createHash('sha256').update(encoded.data).digest('hex') + const previous = this.observations.get(actor) + const unchanged = previous?.fingerprint === fingerprint ? previous : undefined + const observationId = this.execution.nextObservationId(actor) + const value: RawPhoneObservation = { + observationId, + ...(unchanged === undefined ? {} : { unchangedFromObservationId: unchanged.value.observationId }), + serial, + width: encoded.sourceWidth ?? screen.width, + height: encoded.sourceHeight ?? screen.height, + foregroundPackage: currentPackage(String(focusRaw)), + capturedAt: new Date(this.now()).toISOString(), + settled: false, + image: unchanged?.value.image ?? { + data: encoded.data, + mediaType: 'image/jpeg', + bytes: encoded.data.byteLength, + width: encoded.width, + height: encoded.height, + name: `opengui-phone-${this.now()}.jpg`, + }, + } + this.observations.set(actor, { value, fingerprint }) + const visual = await this.options.sampleFrame?.(encoded.data) + signal.throwIfAborted() + this.execution.recordObservation(actor, { observationId, screenshotFingerprint: fingerprint, ...(visual ? { visual } : {}) }) + return value + } finally { + releaseMedia() + } + } +} diff --git a/packages/device-runtime/src/phone-execution.ts b/packages/device-runtime/src/phone-execution.ts new file mode 100644 index 0000000..eb5aa92 --- /dev/null +++ b/packages/device-runtime/src/phone-execution.ts @@ -0,0 +1,188 @@ +import { ObservationId } from './actions.ts' +import { randomUUID } from 'node:crypto' +import type { ObservationId as ObservationIdType } from './actions.ts' +import { frameChanged, type VisualFrame } from './frame-comparison.ts' + +/** Minimal observation identity retained outside the durable tool result. */ +export interface PhoneFrameState { + observationId: ObservationIdType + screenshotFingerprint: string + visual?: VisualFrame +} + +interface NoProgressState { + signature: string + screenshotFingerprint: string + count: number + frame: PhoneFrameState +} + +interface AgentPhoneState { + identity: string + operations: number + observationSequence: number + targetSerial?: string + latest?: PhoneFrameState + noProgress?: NoProgressState +} + +/** Read-only execution counters used by Host adapters and status tools. */ +export interface PhoneExecutionSnapshot { + readonly operations: number + readonly observationSequence: number + readonly targetSerial?: string + readonly observationId?: ObservationIdType +} + +/** Per-child freshness, operation-budget, and repeated-no-progress enforcement. */ +export class PhoneExecutionState { + private readonly agents = new WeakMap() + + private state(agent: object): AgentPhoneState { + const existing = this.agents.get(agent) + if (existing !== undefined) return existing + const created: AgentPhoneState = { identity: randomUUID(), operations: 0, observationSequence: 0 } + this.agents.set(agent, created) + return created + } + + /** Bind a newly published actor to one Host-private serial before its first tool call. */ + assignTarget(agent: object, serial: string): void { + const state = this.state(agent) + if (state.targetSerial !== undefined && state.targetSerial !== serial) { + throw new Error('opengui: phone agent is already bound to another device') + } + state.targetSerial = serial + } + + /** Count one tool operation and enforce the actor's bounded action budget. */ + beginOperation(agent: object, maxOperations: number): void { + const state = this.state(agent) + state.operations += 1 + if (state.operations > maxOperations) { + throw new Error(`opengui: phone task exceeded its ${maxOperations}-operation limit`) + } + } + + /** Resolve the task's phone once and reuse that serial for every later call. */ + async resolveTarget(agent: object, discover: () => Promise): Promise { + const state = this.state(agent) + if (state.targetSerial !== undefined) return state.targetSerial + const selected = await discover() + state.targetSerial = selected + return selected + } + + /** Allocate an actor-local observation id that the next mutation must echo. */ + nextObservationId(agent: object): ObservationIdType { + const state = this.state(agent) + state.observationSequence += 1 + return ObservationId(`phone-observation-${state.identity}-${state.observationSequence}`) + } + + /** Return the current frame, when the actor has observed one. */ + latest(agent: object): PhoneFrameState | undefined { + return this.state(agent).latest + } + + /** Consume before dispatch so a failed or cancelled mutation cannot reuse its old frame. */ + consumeObservation(agent: object): void { + delete this.state(agent).latest + } + + /** Publish a completed observation as the only current frame. */ + recordObservation(agent: object, frame: PhoneFrameState): void { + const state = this.state(agent) + if (state.latest !== undefined && this.changed(state.latest, frame)) { + delete state.noProgress + } + state.latest = frame + } + + /** Require an action to name the exact current frame and return that frame. */ + current(agent: object, observationId: ObservationIdType): PhoneFrameState { + const latest = this.state(agent).latest + if (latest === undefined) throw new Error('opengui: observe the phone before performing an action') + if (latest.observationId !== observationId) { + throw new Error(`opengui: stale observationId ${observationId}; use current observationId ${latest.observationId}`) + } + return latest + } + + /** Reject a fourth identical action after three unchanged resulting frames. */ + assertActionAllowed(agent: object, signature: string, before: PhoneFrameState): void { + const state = this.state(agent) + const noProgress = state.noProgress + if (noProgress?.count === 3 + && noProgress.signature === signature + && !this.changed(noProgress.frame, before)) { + throw new Error('opengui: repeated action made no screen progress three times; choose another action or report blocked') + } + } + + /** Update the repeated-action fuse from the frame before and after one mutation. */ + recordActionResult(agent: object, signature: string, before: PhoneFrameState, after: PhoneFrameState): void { + const state = this.state(agent) + if (this.changed(before, after)) { + delete state.noProgress + return + } + const previous = state.noProgress + state.noProgress = previous?.signature === signature && !this.changed(previous.frame, after) + ? { ...previous, count: previous.count + 1, frame: after } + : { signature, screenshotFingerprint: after.screenshotFingerprint, count: 1, frame: after } + } + + private changed(before: PhoneFrameState, after: PhoneFrameState): boolean { + return before.visual && after.visual ? frameChanged(before.visual, after.visual) : before.screenshotFingerprint !== after.screenshotFingerprint + } + + /** Return a copy of one actor's bounded execution state. */ + snapshot(agent: object): PhoneExecutionSnapshot { + const state = this.state(agent) + return { + operations: state.operations, + observationSequence: state.observationSequence, + ...(state.targetSerial === undefined ? {} : { targetSerial: state.targetSerial }), + ...(state.latest === undefined ? {} : { observationId: state.latest.observationId }), + } + } +} + +/** Serialize one actor's tool calls while allowing different actors to proceed in parallel. */ +export class PhoneOperationQueue { + private readonly tails = new WeakMap>() + + async run(agent: object, operation: () => Promise): Promise { + const previous = this.tails.get(agent) ?? Promise.resolve() + let release!: () => void + const gate = new Promise((resolve) => { release = resolve }) + const tail = previous.catch(() => {}).then(() => gate) + this.tails.set(agent, tail) + await previous.catch(() => {}) + try { + return await operation() + } finally { + release() + if (this.tails.get(agent) === tail) this.tails.delete(agent) + } + } +} + +/** Wait for an explicit UI-settle operation and honor cancellation. */ +export function waitForPhoneUi(waitMs: number, signal: AbortSignal): Promise { + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const finish = (): void => { + signal.removeEventListener('abort', abort) + resolve() + } + const timer = setTimeout(finish, waitMs) + const abort = (): void => { + clearTimeout(timer) + signal.removeEventListener('abort', abort) + reject(signal.reason instanceof Error ? signal.reason : new Error('opengui: phone wait cancelled')) + } + signal.addEventListener('abort', abort, { once: true }) + }) +} diff --git a/packages/device-runtime/tests/controller-contract.ts b/packages/device-runtime/tests/controller-contract.ts new file mode 100644 index 0000000..faa343a --- /dev/null +++ b/packages/device-runtime/tests/controller-contract.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import type { PhoneController, PhoneControllerOptions } from '../src/phone-controller.ts' +import { jpeg } from './image-fixture.ts' + +type Test = (name: string, body: () => Promise) => unknown + +/** Run the same fault-injection contract against both production host adapters. */ +export function controllerContract(it: Test, create: (options: PhoneControllerOptions) => PhoneController): void { + function fixture() { + const png = Buffer.alloc(24) + Buffer.from('89504e470d0a1a0a0000000d49484452', 'hex').copy(png) + png.writeUInt32BE(2, 16); png.writeUInt32BE(2, 20) + let failCapture = false + let failAfterDispatch = false + let dispatched = 0 + const controller = create({ + runAdb: async args => { + if (args.includes('screencap')) { + if (failCapture) throw new Error('injected capture failure') + return png + } + if (args.includes('wm')) return 'Physical size: 2x2' + if (args.includes('dumpsys')) return 'mCurrentFocus=Window{ u0 com.example/.Main }' + if (args.includes('keyevent')) { dispatched++; if (failAfterDispatch) failCapture = true } + return '' + }, + discoverTarget: async () => 'test-device', pasteUnicode: async () => {}, + encodeScreenshot: async () => ({ data: jpeg, width: 2, height: 2 }), + maxOperations: () => 100, settleIntervalMs: 1, settleTimeoutMs: 3, + }) + const actor = {} + const signal = AbortSignal.timeout(5000) + return { controller, actor, signal, dispatched: () => dispatched, + failCapture: () => { failCapture = true }, + failAfterDispatch: () => { failAfterDispatch = true }, + } + } + + it('revokes an observation when a later read fails before any action', async () => { + const f = fixture() + const before = await f.controller.observe(f.actor, f.signal) + f.failCapture() + await assert.rejects(f.controller.observe(f.actor, f.signal), /capture failure/) + await assert.rejects(f.controller.execute(f.actor, { action: 'key', key: 'Home', observationId: before.observationId }, f.signal), /observe the phone/) + assert.equal(f.dispatched(), 0) + }) + + it('classifies dispatched capture failure as unknown and refuses replay', async () => { + const f = fixture() + const before = await f.controller.observe(f.actor, f.signal) + f.failAfterDispatch() + const action = { action: 'key', key: 'Home', observationId: before.observationId } + await assert.rejects(f.controller.execute(f.actor, action, f.signal), { executionState: 'outcome_unknown' }) + await assert.rejects(f.controller.execute(f.actor, action, f.signal), { executionState: 'not_executed' }) + assert.equal(f.dispatched(), 1) + }) + + it('serializes racing actions and consumes their shared observation once', async () => { + const f = fixture() + const before = await f.controller.observe(f.actor, f.signal) + const action = { action: 'key', key: 'Home', observationId: before.observationId } + const results = await Promise.allSettled([ + f.controller.execute(f.actor, action, f.signal), + f.controller.execute(f.actor, action, f.signal), + ]) + assert.equal(results.filter(result => result.status === 'fulfilled').length, 1) + assert.equal(f.dispatched(), 1) + }) + + it('never dispatches a request cancelled before it acquires execution', async () => { + const f = fixture() + const before = await f.controller.observe(f.actor, f.signal) + const abort = new AbortController(); abort.abort(new Error('test cancelled')) + await assert.rejects(f.controller.execute(f.actor, { action: 'key', key: 'Home', observationId: before.observationId }, abort.signal), /cancelled/) + assert.equal(f.dispatched(), 0) + }) +} diff --git a/packages/device-runtime/tests/image-fixture.ts b/packages/device-runtime/tests/image-fixture.ts new file mode 100644 index 0000000..070f918 --- /dev/null +++ b/packages/device-runtime/tests/image-fixture.ts @@ -0,0 +1,2 @@ +// Synthetic 2x2 image; contains no device or user data. +export const jpeg = Buffer.from('/9j/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAACAAIDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAABP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AJQAcK//2Q==', 'base64') diff --git a/plugins/opengui/src/adb.ts b/plugins/opengui/src/adb.ts index 7c24970..3a12fcc 100644 --- a/plugins/opengui/src/adb.ts +++ b/plugins/opengui/src/adb.ts @@ -4,293 +4,7 @@ import { constants } from 'node:fs' import { fileURLToPath } from 'node:url' import { dirname, join } from 'node:path' -/** Opaque identity of one completed phone observation. */ -export type ObservationId = string & { readonly __observationId: unique symbol } - -/** - * Brand a validated or generated observation identifier. - * @param value Raw observation identifier. - * @returns The same string with its observation-id brand. - */ -export function ObservationId(value: string): ObservationId { - return value as ObservationId -} - -/** One row returned by `adb devices -l`. */ -export interface AdbDevice { - serial: string - state: string - model?: string - product?: string - device?: string -} - -/** The logical Android display coordinate space. */ -export interface ScreenSize { - width: number - height: number -} - -/** Device input dimensions plus the screenshot pixel space shown to the model. */ -export interface PhoneCoordinateSpace extends ScreenSize { - screenshotWidth: number - screenshotHeight: number -} - -/** Tight visible bounds of one target in the current screenshot. */ -export interface TargetBoundingBox { - left: number - top: number - right: number - bottom: number -} - -/** The closed set of operations exposed to the phone subagent. */ -export type PhoneAction = - | { action: 'observe' } - | { action: 'tap'; observationId: ObservationId; targetBBox: TargetBoundingBox } - | { action: 'swipe'; observationId: ObservationId; x1: number; y1: number; x2: number; y2: number; durationMs?: number } - | { action: 'text'; observationId: ObservationId; text: string } - | { action: 'key'; observationId: ObservationId; key: 'Back' | 'Home' | 'Enter' | 'AppSwitch' } - | { action: 'launch'; observationId: ObservationId; packageName: string } - | { action: 'wait'; observationId: ObservationId; waitMs: number } - -const KEY_CODES = { - Back: 'KEYCODE_BACK', - Home: 'KEYCODE_HOME', - Enter: 'KEYCODE_ENTER', - AppSwitch: 'KEYCODE_APP_SWITCH', -} as const - -const ADB_INPUT_TEXT = /^[A-Za-z0-9 .,_@:/+=!?-]*$/u - -/** Whether Android's shell input-text command can preserve this value exactly. */ -export function canUseAdbInputText(text: string): boolean { - return ADB_INPUT_TEXT.test(text) -} - -function requiredNumber(value: unknown, action: string, fields: string): number { - if (typeof value !== 'number' || !Number.isFinite(value)) { - throw new Error(`opengui: ${action} requires ${fields}`) - } - return value -} - -function requiredObservationId(value: unknown, action: string): ObservationId { - if (typeof value !== 'string' || value.trim().length === 0) { - throw new Error(`opengui: ${action} requires the current observationId`) - } - return ObservationId(value) -} - -function assertNever(value: never): never { - throw new Error(`opengui: unsupported validated action ${JSON.stringify(value)}`) -} - -function requiredTargetBBox(value: unknown): TargetBoundingBox { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - throw new Error('opengui: tap requires targetBBox from the current screenshot') - } - const input = value as Record - return { - left: requiredNumber(input.left, 'tap', 'targetBBox.left, top, right, and bottom'), - top: requiredNumber(input.top, 'tap', 'targetBBox.left, top, right, and bottom'), - right: requiredNumber(input.right, 'tap', 'targetBBox.left, top, right, and bottom'), - bottom: requiredNumber(input.bottom, 'tap', 'targetBBox.left, top, right, and bottom'), - } -} - -/** - * Convert untrusted tool arguments into the closed phone-action union. - * @param input Raw arguments supplied to the phone tool. - * @returns A validated action containing every required field. - */ -export function normalizePhoneAction(input: Record): PhoneAction { - switch (input.action) { - case 'observe': return { action: 'observe' } - case 'tap': - return { - action: 'tap', - observationId: requiredObservationId(input.observationId, 'tap'), - targetBBox: requiredTargetBBox(input.targetBBox), - } - case 'swipe': { - const action: Extract = { - action: 'swipe', - observationId: requiredObservationId(input.observationId, 'swipe'), - x1: requiredNumber(input.x1, 'swipe', 'x1, y1, x2, and y2'), - y1: requiredNumber(input.y1, 'swipe', 'x1, y1, x2, and y2'), - x2: requiredNumber(input.x2, 'swipe', 'x1, y1, x2, and y2'), - y2: requiredNumber(input.y2, 'swipe', 'x1, y1, x2, and y2'), - } - if (input.durationMs !== undefined) { - action.durationMs = requiredNumber(input.durationMs, 'swipe', 'an integer durationMs') - } - return action - } - case 'text': - if (typeof input.text !== 'string') throw new Error('opengui: text requires text as a string') - return { action: 'text', observationId: requiredObservationId(input.observationId, 'text'), text: input.text } - case 'key': - if (input.key !== 'Back' && input.key !== 'Home' && input.key !== 'Enter' && input.key !== 'AppSwitch') { - throw new Error('opengui: key requires one of Back, Home, Enter, or AppSwitch') - } - return { action: 'key', observationId: requiredObservationId(input.observationId, 'key'), key: input.key } - case 'launch': - if (typeof input.packageName !== 'string') throw new Error('opengui: launch requires packageName as a string') - return { action: 'launch', observationId: requiredObservationId(input.observationId, 'launch'), packageName: input.packageName } - case 'wait': - return { - action: 'wait', - observationId: requiredObservationId(input.observationId, 'wait'), - waitMs: requiredNumber(input.waitMs, 'wait', 'an integer waitMs'), - } - default: - throw new Error('opengui: unsupported action') - } -} - -/** - * Parse `adb devices -l` without treating offline/unauthorized rows as usable. - * @param output Standard output from `adb devices -l`. - * @returns Parsed device rows in their original order. - */ -export function parseDevices(output: string): AdbDevice[] { - return output.split(/\r?\n/u).slice(1).map(line => line.trim()).filter(Boolean).map((line) => { - const [serial = '', state = 'unknown', ...fields] = line.split(/\s+/u) - const attributes = new Map() - for (const field of fields) { - const separator = field.indexOf(':') - if (separator > 0) attributes.set(field.slice(0, separator), field.slice(separator + 1)) - } - const model = attributes.get('model') - const product = attributes.get('product') - const device = attributes.get('device') - return { - serial, - state, - ...(model === undefined ? {} : { model }), - ...(product === undefined ? {} : { product }), - ...(device === undefined ? {} : { device }), - } - }).filter(device => device.serial.length > 0) -} - -/** - * Pick one deterministic target for compatibility with single-device callers. - * @param devices Device rows returned by {@link parseDevices}. - * @returns The lexicographically first authorized serial. - */ -export function selectAuthorizedSerial(devices: readonly AdbDevice[]): string { - const serial = devices.filter(device => device.state === 'device') - .map(device => device.serial).sort((a, b) => a.localeCompare(b))[0] - if (serial === undefined) { - throw new Error('opengui: no authorized Android device is connected; connect at least one phone and accept its USB debugging prompt') - } - return serial -} - -/** - * Parse the logical display size used by screenshots and input. - * @param output Standard output from `adb shell wm size`. - * @returns The effective logical width and height. - */ -export function parseScreenSize(output: string): ScreenSize { - const match = output.match(/Override size:\s*(\d+)x(\d+)/iu) - ?? output.match(/Physical size:\s*(\d+)x(\d+)/iu) - ?? output.match(/(\d+)x(\d+)/u) - const width = Number(match?.[1] ?? 0) - const height = Number(match?.[2] ?? 0) - if (!(width > 0 && height > 0)) throw new Error('opengui: ADB did not report a valid display size') - return { width, height } -} - -function screenshotCoordinate(value: number, modelTotal: number, inputTotal: number, name: string): string { - if (!Number.isFinite(value) || value < 0 || value > modelTotal) { - throw new Error(`opengui: ${name} must be inside the current screenshot's ${modelTotal}px axis`) - } - return String(Math.round((value * inputTotal) / modelTotal)) -} - -function targetCenter(box: TargetBoundingBox, screen: PhoneCoordinateSpace): { x: string; y: string } { - if (!(box.right > box.left) || !(box.bottom > box.top)) { - throw new Error('opengui: targetBBox must have positive width and height') - } - if (box.left < 0 || box.top < 0 || box.right > screen.screenshotWidth || box.bottom > screen.screenshotHeight) { - throw new Error(`opengui: targetBBox must fit the current ${screen.screenshotWidth}x${screen.screenshotHeight} screenshot`) - } - return { - x: screenshotCoordinate((box.left + box.right) / 2, screen.screenshotWidth, screen.width, 'targetBBox center x'), - y: screenshotCoordinate((box.top + box.bottom) / 2, screen.screenshotHeight, screen.height, 'targetBBox center y'), - } -} - -/** - * Build the allowlisted device-side command for one validated operation. - * @param action A validated phone action. - * @param screen Device input dimensions and the screenshot pixel space shown to the model. - * @returns ADB arguments, or no arguments for a pure observation. - */ -export function actionCommand(action: PhoneAction, screen: PhoneCoordinateSpace): string[] | undefined { - switch (action.action) { - case 'observe': return undefined - case 'tap': { - const center = targetCenter(action.targetBBox, screen) - return ['shell', 'input', 'tap', center.x, center.y] - } - case 'swipe': { - const duration = action.durationMs ?? 300 - if (!Number.isInteger(duration) || duration < 50 || duration > 2_000) { - throw new Error('opengui: durationMs must be an integer from 50 through 2000') - } - return [ - 'shell', 'input', 'swipe', - screenshotCoordinate(action.x1, screen.screenshotWidth, screen.width, 'x1'), - screenshotCoordinate(action.y1, screen.screenshotHeight, screen.height, 'y1'), - screenshotCoordinate(action.x2, screen.screenshotWidth, screen.width, 'x2'), - screenshotCoordinate(action.y2, screen.screenshotHeight, screen.height, 'y2'), - String(duration), - ] - } - case 'text': { - if (action.text.length < 1 || action.text.length > 500 || !/^[A-Za-z0-9 .,!?_@+:'"()-]+$/u.test(action.text)) { - throw new Error('opengui: text must be 1-500 characters supported by Android adb input text') - } - return ['shell', 'input', 'text', action.text.replaceAll(' ', '%s')] - } - case 'key': return ['shell', 'input', 'keyevent', KEY_CODES[action.key]] - case 'launch': - if (!/^[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)+$/u.test(action.packageName)) { - throw new Error('opengui: packageName must be a valid Android application id') - } - return ['shell', 'monkey', '-p', action.packageName, '-c', 'android.intent.category.LAUNCHER', '1'] - case 'wait': { - if (!Number.isInteger(action.waitMs) || action.waitMs < 100 || action.waitMs > 10_000) { - throw new Error('opengui: waitMs must be an integer from 100 through 10000') - } - return undefined - } - default: return assertNever(action) - } -} - -/** - * Build the bounded ADB text-input sequence for Android's safe ASCII set. - * Unicode is intentionally rejected here so callers cannot mistake an - * unacknowledged shell clipboard attempt for successful input; it is handled - * by the acknowledged scrcpy control channel instead. - * @param text Text to enter in the focused Android field. - * @returns One allowlisted ADB argument array. - */ -export function textInputCommands(text: string): string[][] { - if (text.length < 1 || [...text].length > 500 || text.includes('\0')) { - throw new Error('opengui: text must contain 1-500 Unicode characters without NUL') - } - if (canUseAdbInputText(text)) { - return [['shell', 'input', 'text', text.replaceAll(' ', '%s')]] - } - throw new Error('opengui: Unicode text requires acknowledged scrcpy clipboard input') -} +export * from '../../../packages/device-runtime/src/actions.ts' /** * Resolve the ADB executable installed inside this plugin. diff --git a/plugins/opengui/src/cli.ts b/plugins/opengui/src/cli.ts index 0068ed1..5246d2c 100644 --- a/plugins/opengui/src/cli.ts +++ b/plugins/opengui/src/cli.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { OpenGuiError, errorInfo } from './errors.ts' import { execFile } from 'node:child_process' import { realpathSync } from 'node:fs' import { access } from 'node:fs/promises' @@ -84,7 +85,10 @@ export async function runCli(argv: readonly string[], signal = new AbortControll } if (name === '--shutdown-daemon') { const result = await sendRequest(daemonEndpoint(), request('__shutdown__'), signal) - if (!result.ok) throw new Error(result.error) + if (!result.ok) { + if (result.failure) throw new OpenGuiError(result.failure.code, result.failure.message, result.failure.executionState, result.failure.recovery) + throw new Error(result.error) + } return result.result } const source = raw ?? (process.stdin.isTTY ? '{}' : await readStdin()) @@ -93,7 +97,10 @@ export async function runCli(argv: readonly string[], signal = new AbortControll if (!process.env.CODEX_THREAD_ID?.trim()) throw new Error('opengui: CODEX_THREAD_ID is required; run from a local Codex task') const endpoint = await ensureDaemon(fileURLToPath(import.meta.url), dataDirectory(), AbortSignal.any([signal, AbortSignal.timeout(15_000)])) const result = await sendRequest(endpoint, request(name, args as Record), signal) - if (!result.ok) throw new Error(result.error) + if (!result.ok) { + if (result.failure) throw new OpenGuiError(result.failure.code, result.failure.message, result.failure.executionState, result.failure.recovery) + throw new Error(result.error) + } return result.result } @@ -121,7 +128,7 @@ if (isEntry) { runCli(process.argv.slice(2), controller.signal) .then(result => process.stdout.write(JSON.stringify(result, null, 2) + '\n')) .catch(error => { - process.stderr.write(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }) + '\n') + process.stderr.write(JSON.stringify({ error: error instanceof Error ? error.message : String(error), ...(error instanceof OpenGuiError ? { failure: errorInfo(error) } : {}) }) + '\n') process.exitCode = 1 }) } diff --git a/plugins/opengui/src/concurrency.ts b/plugins/opengui/src/concurrency.ts index 0266eb1..26adabd 100644 --- a/plugins/opengui/src/concurrency.ts +++ b/plugins/opengui/src/concurrency.ts @@ -1,51 +1 @@ -/** Fair abort-aware permit pool used to bound expensive host/device media work. */ -export class AsyncSemaphore { - private active = 0 - private readonly queued: Array<{ - readonly signal: AbortSignal - readonly resolve: (release: () => void) => void - readonly reject: (reason: unknown) => void - }> = [] - - constructor(readonly limit: number) { - if (!Number.isSafeInteger(limit) || limit < 1) throw new Error('semaphore limit must be a positive integer') - } - - async acquire(signal: AbortSignal): Promise<() => void> { - signal.throwIfAborted() - if (this.active < this.limit) return this.grant() - return await new Promise<() => void>((resolvePermit, rejectPermit) => { - const request = { - signal, - resolve: (release: () => void) => { signal.removeEventListener('abort', onAbort); resolvePermit(release) }, - reject: rejectPermit, - } - const onAbort = (): void => { - const index = this.queued.indexOf(request) - if (index >= 0) this.queued.splice(index, 1) - rejectPermit(signal.reason) - } - signal.addEventListener('abort', onAbort, { once: true }) - this.queued.push(request) - }) - } - - private grant(): () => void { - this.active += 1 - let released = false - return () => { - if (released) return - released = true - this.active -= 1 - while (this.queued.length > 0) { - const next = this.queued.shift()! - if (next.signal.aborted) { - next.reject(next.signal.reason) - continue - } - next.resolve(this.grant()) - break - } - } - } -} +export * from '../../../packages/device-runtime/src/concurrency.ts' diff --git a/plugins/opengui/src/daemon.ts b/plugins/opengui/src/daemon.ts index bf07416..07a5d30 100644 --- a/plugins/opengui/src/daemon.ts +++ b/plugins/opengui/src/daemon.ts @@ -1,3 +1,4 @@ +import { errorInfo } from './errors.ts' import { randomUUID } from 'node:crypto' import { spawn } from 'node:child_process' import { chmod, lstat, open, readFile, rm } from 'node:fs/promises' @@ -19,7 +20,7 @@ export interface Request { args: Record owner?: string } -export interface Response { ok: boolean; result?: unknown; error?: string } +export interface Response { ok: boolean; result?: unknown; error?: string; failure?: ReturnType } export interface Hello { version: string; protocol: number; activeSessions: number; activeViewers?: number } export function request(name: string, args: Record = {}, owner = process.env.CODEX_THREAD_ID): Request { return { version: VERSION, protocol: PROTOCOL_VERSION, name, args, ...(owner ? { owner } : {}) } @@ -173,6 +174,7 @@ export async function startDaemon(options: DaemonOptions): Promise<{ endpoint: s if (!input.includes('\n')) return received = true const operation = (async () => { + let actionCompleted = false try { const value = JSON.parse(input.trim()) as Request if (value.name === '__ping__') { @@ -208,6 +210,7 @@ export async function startDaemon(options: DaemonOptions): Promise<{ endpoint: s const result = value.name === 'opengui_list_sessions' ? { sessions: service.listSessions().filter(item => owners.get(item.sessionId) === value.owner) } : await callOpenGuiTool(service, value.name, value.args, signal, confirmed, value.owner) + actionCompleted = value.name === 'opengui_act' if (value.name === 'opengui_open_session') { ownedSession = (result as { sessionId: string }).sessionId owners.set(ownedSession, value.owner) @@ -226,7 +229,9 @@ export async function startDaemon(options: DaemonOptions): Promise<{ endpoint: s await service.cancel(ownedSession).catch(() => {}) await observations.remove(ownedSession).catch(() => {}) } - respond({ ok: false, error: error instanceof Error ? error.message : String(error) }) + const failure = errorInfo(error) + if (actionCompleted) { failure.executionState = 'outcome_unknown'; failure.recovery = 'observe' } + respond({ ok: false, error: failure.message, failure }) } finally { lastRequest = Date.now() const retained = new Set(service.listSessions().map(item => item.sessionId)) diff --git a/plugins/opengui/src/device-fleet.ts b/plugins/opengui/src/device-fleet.ts index b6ad49d..383b40f 100644 --- a/plugins/opengui/src/device-fleet.ts +++ b/plugins/opengui/src/device-fleet.ts @@ -1,191 +1 @@ -import { randomUUID } from 'node:crypto' -import type { AdbDevice } from './adb.ts' - -/** Host-private device identity paired with its browser-safe presentation. */ -export interface FleetDevice { - readonly id: string - readonly serial: string - readonly label: string - readonly model?: string -} - -/** Browser-visible device choice; serial deliberately never crosses this seam. */ -export interface FleetDeviceView { - readonly id: string - readonly label: string - readonly model?: string - readonly selected: boolean -} - -/** Read-only connection state exposed by Codex device discovery. */ -export interface FleetDeviceStatusView { - readonly id: string - readonly label: string - readonly model?: string - readonly state: string - readonly connected: boolean - readonly authorized: boolean -} - -export interface DeviceFleetSnapshot { - readonly devices: readonly FleetDeviceView[] - readonly selectedDeviceIds: readonly string[] -} - -export type DiscoverDevices = (signal: AbortSignal) => Promise - -interface DeviceRecord { - id: string - serial: string -} - -function displayModel(device: AdbDevice): string | undefined { - const raw = device.model?.trim() - return raw ? raw.replaceAll('_', ' ') : undefined -} - -/** - * Owns authorized-device discovery, opaque browser identities, and the selection - * applied atomically to the next OpenGUI task. - */ -export class DeviceFleet { - private readonly records = new Map() - private readonly selected = new Set() - - constructor( - private readonly discover: DiscoverDevices, - private readonly createId: () => string = randomUUID, - ) {} - - async snapshot(signal: AbortSignal): Promise { - const devices = await this.discover(signal) - this.syncRecords(devices, false) - const authorized = devices.filter(device => device.state === 'device') - .toSorted((a, b) => a.serial.localeCompare(b.serial)) - - // Preserve the single-phone experience. With multiple phones, an explicit - // choice is required unless a prior still-connected choice already exists. - if (authorized.length === 1 && this.selected.size === 0) { - const only = this.records.get(authorized[0]!.serial) - if (only !== undefined) this.selected.add(only.id) - } - - const modelCounts = new Map() - for (const device of authorized) { - const model = displayModel(device) ?? 'Android 手机' - modelCounts.set(model, (modelCounts.get(model) ?? 0) + 1) - } - const modelIndexes = new Map() - const views = authorized.map((device): FleetDeviceView => { - const record = this.records.get(device.serial)! - const model = displayModel(device) - const base = model ?? 'Android 手机' - const index = (modelIndexes.get(base) ?? 0) + 1 - modelIndexes.set(base, index) - const label = (modelCounts.get(base) ?? 0) > 1 ? `${base} ${index}` : base - return { - id: record.id, - label, - ...(model === undefined ? {} : { model }), - selected: this.selected.has(record.id), - } - }) - return { - devices: views, - selectedDeviceIds: views.filter(device => device.selected).map(device => device.id), - } - } - - /** List every ADB row, including unauthorized and offline devices, without exposing serials. */ - async inspect(signal: AbortSignal): Promise { - const devices = (await this.discover(signal)).toSorted((a, b) => a.serial.localeCompare(b.serial)) - this.syncRecords(devices, true) - const modelCounts = new Map() - for (const device of devices) { - const model = displayModel(device) ?? 'Android phone' - modelCounts.set(model, (modelCounts.get(model) ?? 0) + 1) - } - const modelIndexes = new Map() - return devices.map((device): FleetDeviceStatusView => { - const record = this.records.get(device.serial)! - const model = displayModel(device) - const base = model ?? 'Android phone' - const index = (modelIndexes.get(base) ?? 0) + 1 - modelIndexes.set(base, index) - return { - id: record.id, - label: (modelCounts.get(base) ?? 0) > 1 ? `${base} ${index}` : base, - ...(model === undefined ? {} : { model }), - state: device.state, - connected: true, - authorized: device.state === 'device', - } - }) - } - - async select(deviceIds: readonly string[], signal: AbortSignal): Promise { - const snapshot = await this.snapshot(signal) - const available = new Set(snapshot.devices.map(device => device.id)) - const unique = [...new Set(deviceIds)] - if (unique.some(id => !available.has(id))) { - throw new Error('opengui: device selection contains a disconnected or unknown phone') - } - this.selected.clear() - for (const id of unique) this.selected.add(id) - return this.snapshot(signal) - } - - /** Resolve browser-safe ids to current Host-private devices for an immediate operation. */ - async resolveConnected(deviceIds: readonly string[], signal: AbortSignal): Promise { - const snapshot = await this.snapshot(signal) - return this.materialize(snapshot, deviceIds) - } - - async selectedDevices(signal: AbortSignal): Promise { - const snapshot = await this.snapshot(signal) - if (snapshot.devices.length === 0) { - throw new Error('opengui: no authorized Android device is connected; connect a phone and accept its USB debugging prompt') - } - if (snapshot.selectedDeviceIds.length === 0) { - throw new Error('opengui: multiple phones are connected; select at least one in the OpenGUI Tab before running /opengui') - } - return this.materialize(snapshot, snapshot.selectedDeviceIds) - } - - private materialize(snapshot: DeviceFleetSnapshot, deviceIds: readonly string[]): readonly FleetDevice[] { - const views = new Map(snapshot.devices.map(device => [device.id, device])) - const byId = new Map([...this.records.values()].map(record => [record.id, record])) - return [...new Set(deviceIds)].map((id) => { - const view = views.get(id) - const record = byId.get(id) - if (view === undefined || record === undefined) { - throw new Error('opengui: device is disconnected or unknown') - } - return { - id, - serial: record.serial, - label: view.label, - ...(view.model === undefined ? {} : { model: view.model }), - } - }) - } - - - private syncRecords(devices: readonly AdbDevice[], includeUnavailable: boolean): void { - const connectedSerials = new Set(devices.map(device => device.serial)) - for (const [serial, record] of this.records) { - if (connectedSerials.has(serial)) continue - // Retain identity for frozen sessions when the same serial reconnects. - this.selected.delete(record.id) - } - const candidates = includeUnavailable ? devices : devices.filter(device => device.state === 'device') - for (const device of candidates.toSorted((a, b) => { - const authorization = Number(b.state === 'device') - Number(a.state === 'device') - return authorization === 0 ? a.serial.localeCompare(b.serial) : authorization - })) { - if (!this.records.has(device.serial)) { - this.records.set(device.serial, { id: this.createId(), serial: device.serial }) - } - } - } -} +export * from '../../../packages/device-runtime/src/device-fleet.ts' diff --git a/plugins/opengui/src/errors.ts b/plugins/opengui/src/errors.ts new file mode 100644 index 0000000..fa68df5 --- /dev/null +++ b/plugins/opengui/src/errors.ts @@ -0,0 +1 @@ +export * from '../../../packages/device-runtime/src/errors.ts' diff --git a/plugins/opengui/src/image.ts b/plugins/opengui/src/image.ts index b88e9c5..704c4e6 100644 --- a/plugins/opengui/src/image.ts +++ b/plugins/opengui/src/image.ts @@ -1,19 +1 @@ -/** Encoded screenshot shape; the macOS encoder has no native npm dependency. */ -export interface EncodedPhoneScreenshot { - readonly data: Buffer - readonly width: number - readonly height: number -} -/** Read screencap's current orientation and pixel size, not wm's natural size. */ -export function pngDimensions(data: Buffer): { width: number; height: number } { - if (data.length < 24 || data.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a' - || data.readUInt32BE(8) !== 13 || data.toString('ascii', 12, 16) !== 'IHDR') { - throw new Error('opengui: screenshot is not a PNG with an IHDR header') - } - const width = data.readUInt32BE(16) - const height = data.readUInt32BE(20) - if (width < 1 || height < 1 || width > 65_535 || height > 65_535) { - throw new Error('opengui: invalid screenshot dimensions') - } - return { width, height } -} +export * from '../../../packages/device-runtime/src/image.ts' diff --git a/plugins/opengui/src/phone-controller.ts b/plugins/opengui/src/phone-controller.ts index 9cd0100..f51d071 100644 --- a/plugins/opengui/src/phone-controller.ts +++ b/plugins/opengui/src/phone-controller.ts @@ -1,188 +1 @@ -import { createHash } from 'node:crypto' -import { - actionCommand, - canUseAdbInputText, - normalizePhoneAction, - textInputCommands, -} from './adb.ts' -import type { ObservationId, PhoneCoordinateSpace } from './adb.ts' -import { AsyncSemaphore } from './concurrency.ts' -import type { EncodedPhoneScreenshot } from './image.ts' -import { pngDimensions } from './image.ts' -import { PhoneExecutionState, PhoneOperationQueue, waitForPhoneUi } from './phone-execution.ts' -import type { PhoneExecutionSnapshot } from './phone-execution.ts' - -/** Host-neutral phone observation used by the standalone Codex CLI. */ -export interface RawPhoneObservation { - readonly observationId: ObservationId - readonly unchangedFromObservationId?: ObservationId - readonly serial: string - readonly width: number - readonly height: number - readonly foregroundPackage: string - readonly image: { - readonly data: Buffer - readonly mediaType: 'image/jpeg' - readonly bytes: number - readonly width: number - readonly height: number - readonly name: string - } -} - -interface StoredObservation { - readonly value: RawPhoneObservation - readonly fingerprint: string -} - -export interface PhoneControllerOptions { - readonly runAdb: ( - args: readonly string[], - signal: AbortSignal, - buffer?: boolean, - ) => Promise - readonly discoverTarget: (signal: AbortSignal) => Promise - readonly validateTarget?: (serial: string, signal: AbortSignal) => Promise - readonly pasteUnicode: (serial: string, text: string, signal: AbortSignal) => Promise - readonly encodeScreenshot: (source: Buffer) => Promise - readonly maxOperations: () => number - readonly mediaPermits?: AsyncSemaphore - readonly now?: () => number -} - -function currentPackage(output: string): string { - return output.match(/(?:mCurrentFocus|mFocusedApp)=[^\n]*?\bu\d+\s+([A-Za-z0-9._]+)\//u)?.[1] - ?? output.match(/(?:topResumedActivity|mResumedActivity)[^\n]*?\bu\d+\s+([A-Za-z0-9._]+)\//u)?.[1] - ?? '' -} - -/** - * The standalone Codex phone execution kernel. - * Host adapters provide only target discovery, process execution, and Unicode - * clipboard transport; safety and observation semantics live here. - */ -export class PhoneController { - private readonly observations = new WeakMap() - private readonly execution = new PhoneExecutionState() - private readonly queue = new PhoneOperationQueue() - private readonly mediaPermits: AsyncSemaphore - private readonly now: () => number - - constructor(private readonly options: PhoneControllerOptions) { - this.mediaPermits = options.mediaPermits ?? new AsyncSemaphore(2) - this.now = options.now ?? Date.now - } - - /** Freeze an actor to one Host-private device serial. */ - assignTarget(actor: object, serial: string): void { - this.execution.assignTarget(actor, serial) - } - - /** Return counters without exposing or mutating the current observation. */ - status(actor: object): PhoneExecutionSnapshot { - return this.execution.snapshot(actor) - } - - /** Observe without accepting arbitrary device commands. */ - observe(actor: object, signal: AbortSignal): Promise { - return this.execute(actor, { action: 'observe' }, signal) - } - - /** Execute exactly one validated operation and always return the resulting frame. */ - async execute( - actor: object, - input: Record, - signal: AbortSignal, - ): Promise { - return this.queue.run(actor, async () => { - signal.throwIfAborted() - this.execution.beginOperation(actor, this.options.maxOperations()) - const action = normalizePhoneAction(input) - const serial = await this.targetFor(actor, signal) - await this.options.validateTarget?.(serial, signal) - if (action.action === 'observe') return this.capture(actor, serial, signal) - - const before = this.execution.current(actor, action.observationId) - const stored = this.observations.get(actor) - if (stored === undefined || stored.value.observationId !== action.observationId) { - throw new Error('opengui: current phone observation is unavailable') - } - const screen: PhoneCoordinateSpace = { - width: stored.value.width, - height: stored.value.height, - screenshotWidth: stored.value.image.width, - screenshotHeight: stored.value.image.height, - } - if (action.action === 'wait') { - actionCommand(action, screen) - this.execution.consumeObservation(actor) - await waitForPhoneUi(action.waitMs, signal) - return this.capture(actor, serial, signal) - } - - const scrcpyText = action.action === 'text' && !canUseAdbInputText(action.text) - const command = action.action === 'text' ? undefined : actionCommand(action, screen) - const commands = action.action === 'text' - ? scrcpyText ? [] : textInputCommands(action.text) - : command === undefined ? [] : [command] - if (commands.length === 0 && !scrcpyText) { - throw new Error('opengui: action did not resolve to a device command') - } - - const signature = JSON.stringify(scrcpyText ? ['scrcpy-text', action.text] : commands) - this.execution.assertActionAllowed(actor, signature) - this.execution.consumeObservation(actor) - signal.throwIfAborted() - if (scrcpyText) await this.options.pasteUnicode(serial, action.text, signal) - else for (const candidate of commands) await this.options.runAdb(['-s', serial, ...candidate], signal) - - const after = await this.capture(actor, serial, signal) - const afterState = this.execution.current(actor, after.observationId) - this.execution.recordActionResult(actor, signature, before.screenshotFingerprint, afterState.screenshotFingerprint) - return after - }) - } - - private async targetFor(actor: object, signal: AbortSignal): Promise { - return this.execution.resolveTarget(actor, () => this.options.discoverTarget(signal)) - } - - private async capture(actor: object, serial: string, signal: AbortSignal): Promise { - const releaseMedia = await this.mediaPermits.acquire(signal) - try { - const [focusRaw, pngRaw] = await Promise.all([ - this.options.runAdb(['-s', serial, 'shell', 'dumpsys', 'window', 'windows'], signal), - this.options.runAdb(['-s', serial, 'exec-out', 'screencap', '-p'], signal, true), - ]) - const png = Buffer.isBuffer(pngRaw) ? pngRaw : Buffer.from(pngRaw) - const screen = pngDimensions(png) - const encoded = await this.options.encodeScreenshot(png) - signal.throwIfAborted() - const fingerprint = createHash('sha256').update(encoded.data).digest('hex') - const previous = this.observations.get(actor) - const unchanged = previous?.fingerprint === fingerprint ? previous : undefined - const observationId = this.execution.nextObservationId(actor) - const value: RawPhoneObservation = { - observationId, - ...(unchanged === undefined ? {} : { unchangedFromObservationId: unchanged.value.observationId }), - serial, - width: screen.width, - height: screen.height, - foregroundPackage: currentPackage(String(focusRaw)), - image: unchanged?.value.image ?? { - data: encoded.data, - mediaType: 'image/jpeg', - bytes: encoded.data.byteLength, - width: encoded.width, - height: encoded.height, - name: `opengui-phone-${this.now()}.jpg`, - }, - } - this.observations.set(actor, { value, fingerprint }) - this.execution.recordObservation(actor, { observationId, screenshotFingerprint: fingerprint }) - return value - } finally { - releaseMedia() - } - } -} +export * from '../../../packages/device-runtime/src/phone-controller.ts' diff --git a/plugins/opengui/src/phone-execution.ts b/plugins/opengui/src/phone-execution.ts index b1b3d61..858a141 100644 --- a/plugins/opengui/src/phone-execution.ts +++ b/plugins/opengui/src/phone-execution.ts @@ -1,180 +1 @@ -import { ObservationId } from './adb.ts' -import { randomUUID } from 'node:crypto' -import type { ObservationId as ObservationIdType } from './adb.ts' - -/** Minimal observation identity retained outside the durable tool result. */ -export interface PhoneFrameState { - observationId: ObservationIdType - screenshotFingerprint: string -} - -interface NoProgressState { - signature: string - screenshotFingerprint: string - count: number -} - -interface AgentPhoneState { - readonly observationNonce: string - operations: number - observationSequence: number - targetSerial?: string - latest?: PhoneFrameState - noProgress?: NoProgressState -} - -/** Read-only execution counters used by Host adapters and status tools. */ -export interface PhoneExecutionSnapshot { - readonly operations: number - readonly observationSequence: number - readonly targetSerial?: string - readonly observationId?: ObservationIdType -} - -/** Per-child freshness, operation-budget, and repeated-no-progress enforcement. */ -export class PhoneExecutionState { - private readonly agents = new WeakMap() - - private state(agent: object): AgentPhoneState { - const existing = this.agents.get(agent) - if (existing !== undefined) return existing - const created: AgentPhoneState = { operations: 0, observationSequence: 0, observationNonce: randomUUID() } - this.agents.set(agent, created) - return created - } - - /** Bind a newly published actor to one Host-private serial before its first tool call. */ - assignTarget(agent: object, serial: string): void { - const state = this.state(agent) - if (state.targetSerial !== undefined && state.targetSerial !== serial) { - throw new Error('opengui: phone agent is already bound to another device') - } - state.targetSerial = serial - } - - /** Count one tool operation and enforce the actor's bounded action budget. */ - beginOperation(agent: object, maxOperations: number): void { - const state = this.state(agent) - state.operations += 1 - if (state.operations > maxOperations) { - throw new Error(`opengui: phone task exceeded its ${maxOperations}-operation limit`) - } - } - - /** Resolve the task's phone once and reuse that serial for every later call. */ - async resolveTarget(agent: object, discover: () => Promise): Promise { - const state = this.state(agent) - if (state.targetSerial !== undefined) return state.targetSerial - const selected = await discover() - state.targetSerial = selected - return selected - } - - /** Allocate an actor-local observation id that the next mutation must echo. */ - nextObservationId(agent: object): ObservationIdType { - const state = this.state(agent) - state.observationSequence += 1 - return ObservationId(`phone-observation-${state.observationNonce}-${state.observationSequence}`) - } - - /** Return the current frame, when the actor has observed one. */ - latest(agent: object): PhoneFrameState | undefined { - return this.state(agent).latest - } - - /** Publish a completed observation as the only current frame. */ - consumeObservation(agent: object): void { - delete this.state(agent).latest - } - - recordObservation(agent: object, frame: PhoneFrameState): void { - const state = this.state(agent) - if (state.latest !== undefined && state.latest.screenshotFingerprint !== frame.screenshotFingerprint) { - delete state.noProgress - } - state.latest = frame - } - - /** Require an action to name the exact current frame and return that frame. */ - current(agent: object, observationId: ObservationIdType): PhoneFrameState { - const latest = this.state(agent).latest - if (latest === undefined) throw new Error('opengui: observe the phone before performing an action') - if (latest.observationId !== observationId) { - throw new Error(`opengui: stale observationId ${observationId}; use current observationId ${latest.observationId}`) - } - return latest - } - - /** Reject a fourth identical action after three unchanged resulting frames. */ - assertActionAllowed(agent: object, signature: string): void { - const state = this.state(agent) - const noProgress = state.noProgress - if (noProgress?.count === 3 - && noProgress.signature === signature - && noProgress.screenshotFingerprint === state.latest?.screenshotFingerprint) { - throw new Error('opengui: repeated action made no screen progress three times; choose another action or report blocked') - } - } - - /** Update the repeated-action fuse from the frame before and after one mutation. */ - recordActionResult(agent: object, signature: string, beforeFingerprint: string, afterFingerprint: string): void { - const state = this.state(agent) - if (beforeFingerprint !== afterFingerprint) { - delete state.noProgress - return - } - const previous = state.noProgress - state.noProgress = previous?.signature === signature && previous.screenshotFingerprint === afterFingerprint - ? { ...previous, count: previous.count + 1 } - : { signature, screenshotFingerprint: afterFingerprint, count: 1 } - } - - /** Return a copy of one actor's bounded execution state. */ - snapshot(agent: object): PhoneExecutionSnapshot { - const state = this.state(agent) - return { - operations: state.operations, - observationSequence: state.observationSequence, - ...(state.targetSerial === undefined ? {} : { targetSerial: state.targetSerial }), - ...(state.latest === undefined ? {} : { observationId: state.latest.observationId }), - } - } -} - -/** Serialize one actor's tool calls while allowing different actors to proceed in parallel. */ -export class PhoneOperationQueue { - private readonly tails = new WeakMap>() - - async run(agent: object, operation: () => Promise): Promise { - const previous = this.tails.get(agent) ?? Promise.resolve() - let release!: () => void - const gate = new Promise((resolve) => { release = resolve }) - const tail = previous.catch(() => {}).then(() => gate) - this.tails.set(agent, tail) - await previous.catch(() => {}) - try { - return await operation() - } finally { - release() - if (this.tails.get(agent) === tail) this.tails.delete(agent) - } - } -} - -/** Wait for an explicit UI-settle operation and honor cancellation. */ -export function waitForPhoneUi(waitMs: number, signal: AbortSignal): Promise { - signal.throwIfAborted() - return new Promise((resolve, reject) => { - const finish = (): void => { - signal.removeEventListener('abort', abort) - resolve() - } - const timer = setTimeout(finish, waitMs) - const abort = (): void => { - clearTimeout(timer) - signal.removeEventListener('abort', abort) - reject(signal.reason instanceof Error ? signal.reason : new Error('opengui: phone wait cancelled')) - } - signal.addEventListener('abort', abort, { once: true }) - }) -} +export * from '../../../packages/device-runtime/src/phone-execution.ts' diff --git a/plugins/opengui/tests/daemon.spec.ts b/plugins/opengui/tests/daemon.spec.ts index 828bc7a..5d74e15 100644 --- a/plugins/opengui/tests/daemon.spec.ts +++ b/plugins/opengui/tests/daemon.spec.ts @@ -6,6 +6,7 @@ import { join } from 'node:path' import { createConnection } from 'node:net' import { CodexOpenGuiService } from '../src/codex/service.ts' import { assertVersion, request as makeRequest, sendRequest, startDaemon } from '../src/daemon.ts' +import { OpenGuiError } from '../src/errors.ts' import { FakeHost } from './fixtures.ts' const cleanup: (() => Promise)[] = [] @@ -27,6 +28,15 @@ async function open(endpoint: string): Promise { } describe('standalone daemon transport', () => { + it('preserves an unknown action outcome across the daemon transport', async () => { + const server = await daemon(), sessionId = await open(server.endpoint) + server.host.act = async () => { throw new OpenGuiError('capture_failed', 'capture failed after dispatch', 'outcome_unknown', 'observe') } + const response = await sendRequest(server.endpoint, request('opengui_act', { + sessionId, action: 'key', key: 'Home', observationId: 'frame', externalSideEffect: 'none', + })) + expect(response).toMatchObject({ ok: false, failure: { executionState: 'outcome_unknown', recovery: 'observe' } }) + }) + it('scopes discovery and every session operation to the originating task', async () => { const server = await daemon(), sessionId = await open(server.endpoint) const other = (name: string, args: Record = {}) => sendRequest(server.endpoint, makeRequest(name, args, 'task-b')) diff --git a/plugins/opengui/tests/runtime-contract.spec.ts b/plugins/opengui/tests/runtime-contract.spec.ts new file mode 100644 index 0000000..7c255c6 --- /dev/null +++ b/plugins/opengui/tests/runtime-contract.spec.ts @@ -0,0 +1,4 @@ +import { it } from 'vitest' +import { PhoneController } from '../src/phone-controller.ts' +import { controllerContract } from '../../../packages/device-runtime/tests/controller-contract.ts' +controllerContract(it, options => new PhoneController(options)) diff --git a/workbuddy-plugin/src/adb.ts b/workbuddy-plugin/src/adb.ts index bafadd8..3b754fb 100644 --- a/workbuddy-plugin/src/adb.ts +++ b/workbuddy-plugin/src/adb.ts @@ -4,293 +4,7 @@ import { constants } from 'node:fs' import { fileURLToPath } from 'node:url' import { dirname, join } from 'node:path' -/** Opaque identity of one completed phone observation. */ -export type ObservationId = string & { readonly __observationId: unique symbol } - -/** - * Brand a validated or generated observation identifier. - * @param value Raw observation identifier. - * @returns The same string with its observation-id brand. - */ -export function ObservationId(value: string): ObservationId { - return value as ObservationId -} - -/** One row returned by `adb devices -l`. */ -export interface AdbDevice { - serial: string - state: string - model?: string - product?: string - device?: string -} - -/** The logical Android display coordinate space. */ -export interface ScreenSize { - width: number - height: number -} - -/** Device input dimensions plus the screenshot pixel space shown to the model. */ -export interface PhoneCoordinateSpace extends ScreenSize { - screenshotWidth: number - screenshotHeight: number -} - -/** Tight visible bounds of one target in the current screenshot. */ -export interface TargetBoundingBox { - left: number - top: number - right: number - bottom: number -} - -/** The closed set of operations exposed to the phone subagent. */ -export type PhoneAction = - | { action: 'observe' } - | { action: 'tap'; observationId: ObservationId; targetBBox: TargetBoundingBox } - | { action: 'swipe'; observationId: ObservationId; x1: number; y1: number; x2: number; y2: number; durationMs?: number } - | { action: 'text'; observationId: ObservationId; text: string } - | { action: 'key'; observationId: ObservationId; key: 'Back' | 'Home' | 'Enter' | 'AppSwitch' } - | { action: 'launch'; observationId: ObservationId; packageName: string } - | { action: 'wait'; observationId: ObservationId; waitMs: number } - -const KEY_CODES = { - Back: 'KEYCODE_BACK', - Home: 'KEYCODE_HOME', - Enter: 'KEYCODE_ENTER', - AppSwitch: 'KEYCODE_APP_SWITCH', -} as const - -const ADB_INPUT_TEXT = /^[A-Za-z0-9 .,_@:/+=!?-]*$/u - -/** Whether Android's shell input-text command can preserve this value exactly. */ -export function canUseAdbInputText(text: string): boolean { - return ADB_INPUT_TEXT.test(text) -} - -function requiredNumber(value: unknown, action: string, fields: string): number { - if (typeof value !== 'number' || !Number.isFinite(value)) { - throw new Error(`opengui: ${action} requires ${fields}`) - } - return value -} - -function requiredObservationId(value: unknown, action: string): ObservationId { - if (typeof value !== 'string' || value.trim().length === 0) { - throw new Error(`opengui: ${action} requires the current observationId`) - } - return ObservationId(value) -} - -function assertNever(value: never): never { - throw new Error(`opengui: unsupported validated action ${JSON.stringify(value)}`) -} - -function requiredTargetBBox(value: unknown): TargetBoundingBox { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - throw new Error('opengui: tap requires targetBBox from the current screenshot') - } - const input = value as Record - return { - left: requiredNumber(input.left, 'tap', 'targetBBox.left, top, right, and bottom'), - top: requiredNumber(input.top, 'tap', 'targetBBox.left, top, right, and bottom'), - right: requiredNumber(input.right, 'tap', 'targetBBox.left, top, right, and bottom'), - bottom: requiredNumber(input.bottom, 'tap', 'targetBBox.left, top, right, and bottom'), - } -} - -/** - * Convert untrusted tool arguments into the closed phone-action union. - * @param input Raw arguments supplied to the phone tool. - * @returns A validated action containing every required field. - */ -export function normalizePhoneAction(input: Record): PhoneAction { - switch (input.action) { - case 'observe': return { action: 'observe' } - case 'tap': - return { - action: 'tap', - observationId: requiredObservationId(input.observationId, 'tap'), - targetBBox: requiredTargetBBox(input.targetBBox), - } - case 'swipe': { - const action: Extract = { - action: 'swipe', - observationId: requiredObservationId(input.observationId, 'swipe'), - x1: requiredNumber(input.x1, 'swipe', 'x1, y1, x2, and y2'), - y1: requiredNumber(input.y1, 'swipe', 'x1, y1, x2, and y2'), - x2: requiredNumber(input.x2, 'swipe', 'x1, y1, x2, and y2'), - y2: requiredNumber(input.y2, 'swipe', 'x1, y1, x2, and y2'), - } - if (input.durationMs !== undefined) { - action.durationMs = requiredNumber(input.durationMs, 'swipe', 'an integer durationMs') - } - return action - } - case 'text': - if (typeof input.text !== 'string') throw new Error('opengui: text requires text as a string') - return { action: 'text', observationId: requiredObservationId(input.observationId, 'text'), text: input.text } - case 'key': - if (input.key !== 'Back' && input.key !== 'Home' && input.key !== 'Enter' && input.key !== 'AppSwitch') { - throw new Error('opengui: key requires one of Back, Home, Enter, or AppSwitch') - } - return { action: 'key', observationId: requiredObservationId(input.observationId, 'key'), key: input.key } - case 'launch': - if (typeof input.packageName !== 'string') throw new Error('opengui: launch requires packageName as a string') - return { action: 'launch', observationId: requiredObservationId(input.observationId, 'launch'), packageName: input.packageName } - case 'wait': - return { - action: 'wait', - observationId: requiredObservationId(input.observationId, 'wait'), - waitMs: requiredNumber(input.waitMs, 'wait', 'an integer waitMs'), - } - default: - throw new Error('opengui: unsupported action') - } -} - -/** - * Parse `adb devices -l` without treating offline/unauthorized rows as usable. - * @param output Standard output from `adb devices -l`. - * @returns Parsed device rows in their original order. - */ -export function parseDevices(output: string): AdbDevice[] { - return output.split(/\r?\n/u).slice(1).map(line => line.trim()).filter(Boolean).map((line) => { - const [serial = '', state = 'unknown', ...fields] = line.split(/\s+/u) - const attributes = new Map() - for (const field of fields) { - const separator = field.indexOf(':') - if (separator > 0) attributes.set(field.slice(0, separator), field.slice(separator + 1)) - } - const model = attributes.get('model') - const product = attributes.get('product') - const device = attributes.get('device') - return { - serial, - state, - ...(model === undefined ? {} : { model }), - ...(product === undefined ? {} : { product }), - ...(device === undefined ? {} : { device }), - } - }).filter(device => device.serial.length > 0) -} - -/** - * Pick one deterministic target for compatibility with single-device callers. - * @param devices Device rows returned by {@link parseDevices}. - * @returns The lexicographically first authorized serial. - */ -export function selectAuthorizedSerial(devices: readonly AdbDevice[]): string { - const serial = devices.filter(device => device.state === 'device') - .map(device => device.serial).sort((a, b) => a.localeCompare(b))[0] - if (serial === undefined) { - throw new Error('opengui: no authorized Android device is connected; connect at least one phone and accept its USB debugging prompt') - } - return serial -} - -/** - * Parse the logical display size used by screenshots and input. - * @param output Standard output from `adb shell wm size`. - * @returns The effective logical width and height. - */ -export function parseScreenSize(output: string): ScreenSize { - const match = output.match(/Override size:\s*(\d+)x(\d+)/iu) - ?? output.match(/Physical size:\s*(\d+)x(\d+)/iu) - ?? output.match(/(\d+)x(\d+)/u) - const width = Number(match?.[1] ?? 0) - const height = Number(match?.[2] ?? 0) - if (!(width > 0 && height > 0)) throw new Error('opengui: ADB did not report a valid display size') - return { width, height } -} - -function screenshotCoordinate(value: number, modelTotal: number, inputTotal: number, name: string): string { - if (!Number.isFinite(value) || value < 0 || value > modelTotal) { - throw new Error(`opengui: ${name} must be inside the current screenshot's ${modelTotal}px axis`) - } - return String(Math.round((value * inputTotal) / modelTotal)) -} - -function targetCenter(box: TargetBoundingBox, screen: PhoneCoordinateSpace): { x: string; y: string } { - if (!(box.right > box.left) || !(box.bottom > box.top)) { - throw new Error('opengui: targetBBox must have positive width and height') - } - if (box.left < 0 || box.top < 0 || box.right > screen.screenshotWidth || box.bottom > screen.screenshotHeight) { - throw new Error(`opengui: targetBBox must fit the current ${screen.screenshotWidth}x${screen.screenshotHeight} screenshot`) - } - return { - x: screenshotCoordinate((box.left + box.right) / 2, screen.screenshotWidth, screen.width, 'targetBBox center x'), - y: screenshotCoordinate((box.top + box.bottom) / 2, screen.screenshotHeight, screen.height, 'targetBBox center y'), - } -} - -/** - * Build the allowlisted device-side command for one validated operation. - * @param action A validated phone action. - * @param screen Device input dimensions and the screenshot pixel space shown to the model. - * @returns ADB arguments, or no arguments for a pure observation. - */ -export function actionCommand(action: PhoneAction, screen: PhoneCoordinateSpace): string[] | undefined { - switch (action.action) { - case 'observe': return undefined - case 'tap': { - const center = targetCenter(action.targetBBox, screen) - return ['shell', 'input', 'tap', center.x, center.y] - } - case 'swipe': { - const duration = action.durationMs ?? 300 - if (!Number.isInteger(duration) || duration < 50 || duration > 2_000) { - throw new Error('opengui: durationMs must be an integer from 50 through 2000') - } - return [ - 'shell', 'input', 'swipe', - screenshotCoordinate(action.x1, screen.screenshotWidth, screen.width, 'x1'), - screenshotCoordinate(action.y1, screen.screenshotHeight, screen.height, 'y1'), - screenshotCoordinate(action.x2, screen.screenshotWidth, screen.width, 'x2'), - screenshotCoordinate(action.y2, screen.screenshotHeight, screen.height, 'y2'), - String(duration), - ] - } - case 'text': { - if (action.text.length < 1 || action.text.length > 500 || !/^[A-Za-z0-9 .,!?_@+:'"()-]+$/u.test(action.text)) { - throw new Error('opengui: text must be 1-500 characters supported by Android adb input text') - } - return ['shell', 'input', 'text', action.text.replaceAll(' ', '%s')] - } - case 'key': return ['shell', 'input', 'keyevent', KEY_CODES[action.key]] - case 'launch': - if (!/^[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)+$/u.test(action.packageName)) { - throw new Error('opengui: packageName must be a valid Android application id') - } - return ['shell', 'monkey', '-p', action.packageName, '-c', 'android.intent.category.LAUNCHER', '1'] - case 'wait': { - if (!Number.isInteger(action.waitMs) || action.waitMs < 100 || action.waitMs > 10_000) { - throw new Error('opengui: waitMs must be an integer from 100 through 10000') - } - return undefined - } - default: return assertNever(action) - } -} - -/** - * Build the bounded ADB text-input sequence for Android's safe ASCII set. - * Unicode is intentionally rejected here so callers cannot mistake an - * unacknowledged shell clipboard attempt for successful input; it is handled - * by the acknowledged scrcpy control channel instead. - * @param text Text to enter in the focused Android field. - * @returns One allowlisted ADB argument array. - */ -export function textInputCommands(text: string): string[][] { - if (text.length < 1 || [...text].length > 500 || text.includes('\0')) { - throw new Error('opengui: text must contain 1-500 Unicode characters without NUL') - } - if (canUseAdbInputText(text)) { - return [['shell', 'input', 'text', text.replaceAll(' ', '%s')]] - } - throw new Error('opengui: Unicode text requires acknowledged scrcpy clipboard input') -} +export * from '../../packages/device-runtime/src/actions.ts' /** * Resolve the ADB executable installed inside this plugin. diff --git a/workbuddy-plugin/src/concurrency.ts b/workbuddy-plugin/src/concurrency.ts index 8bd687a..79bee44 100644 --- a/workbuddy-plugin/src/concurrency.ts +++ b/workbuddy-plugin/src/concurrency.ts @@ -1,50 +1 @@ -/** Fair abort-aware permit pool used to bound expensive host/device media work. */ -export class AsyncSemaphore { - private active = 0 - private readonly queued: Array<{ - readonly signal: AbortSignal - readonly resolve: (release: () => void) => void - readonly reject: (reason: unknown) => void - }> = [] - - constructor(readonly limit: number) { - if (!Number.isSafeInteger(limit) || limit < 1) throw new Error('semaphore limit must be a positive integer') - } - - async acquire(signal: AbortSignal): Promise<() => void> { - signal.throwIfAborted() - if (this.active < this.limit) return this.grant() - return await new Promise<() => void>((resolvePermit, rejectPermit) => { - const request = { signal, resolve: (release: () => void): void => { - signal.removeEventListener('abort', onAbort) - resolvePermit(release) - }, reject: rejectPermit } - const onAbort = (): void => { - const index = this.queued.indexOf(request) - if (index >= 0) this.queued.splice(index, 1) - rejectPermit(signal.reason) - } - signal.addEventListener('abort', onAbort, { once: true }) - this.queued.push(request) - }) - } - - private grant(): () => void { - this.active += 1 - let released = false - return () => { - if (released) return - released = true - this.active -= 1 - while (this.queued.length > 0) { - const next = this.queued.shift()! - if (next.signal.aborted) { - next.reject(next.signal.reason) - continue - } - next.resolve(this.grant()) - break - } - } - } -} +export * from '../../packages/device-runtime/src/concurrency.ts' diff --git a/workbuddy-plugin/src/device-fleet.ts b/workbuddy-plugin/src/device-fleet.ts index 40b0411..06d2d29 100644 --- a/workbuddy-plugin/src/device-fleet.ts +++ b/workbuddy-plugin/src/device-fleet.ts @@ -1,197 +1 @@ -import { randomUUID } from 'node:crypto' -import type { AdbDevice } from './adb.ts' - -/** Host-private device identity paired with its browser-safe presentation. */ -export interface FleetDevice { - readonly id: string - readonly serial: string - readonly label: string - readonly model?: string -} - -/** Browser-visible device choice; serial deliberately never crosses this seam. */ -export interface FleetDeviceView { - readonly id: string - readonly label: string - readonly model?: string - readonly selected: boolean -} - -/** Read-only connection state exposed by WorkBuddy device discovery. */ -export interface FleetDeviceStatusView { - readonly id: string - readonly label: string - readonly model?: string - readonly state: string - readonly connected: boolean - readonly authorized: boolean -} - -export interface DeviceFleetSnapshot { - readonly devices: readonly FleetDeviceView[] - readonly selectedDeviceIds: readonly string[] -} - -export type DiscoverDevices = (signal: AbortSignal) => Promise - -interface DeviceRecord { - id: string - serial: string -} - -function displayModel(device: AdbDevice): string | undefined { - const raw = device.model?.trim() - return raw ? raw.replaceAll('_', ' ') : undefined -} - -/** - * Owns authorized-device discovery, opaque browser identities, and the selection - * applied atomically to the next OpenGUI task. - */ -export class DeviceFleet { - private readonly records = new Map() - private readonly selected = new Set() - - constructor( - private readonly discover: DiscoverDevices, - private readonly createId: () => string = randomUUID, - ) {} - - async snapshot(signal: AbortSignal): Promise { - const devices = await this.discover(signal) - this.syncRecords(devices, false) - const authorized = devices.filter(device => device.state === 'device') - .toSorted((a, b) => a.serial.localeCompare(b.serial)) - - // Preserve the single-phone experience. With multiple phones, an explicit - // choice is required unless a prior still-connected choice already exists. - if (authorized.length === 1 && this.selected.size === 0) { - const only = this.records.get(authorized[0]!.serial) - if (only !== undefined) this.selected.add(only.id) - } - - const modelCounts = new Map() - for (const device of authorized) { - const model = displayModel(device) ?? 'Android 手机' - modelCounts.set(model, (modelCounts.get(model) ?? 0) + 1) - } - const modelIndexes = new Map() - const views = authorized.map((device): FleetDeviceView => { - const record = this.records.get(device.serial)! - const model = displayModel(device) - const base = model ?? 'Android 手机' - const index = (modelIndexes.get(base) ?? 0) + 1 - modelIndexes.set(base, index) - const label = (modelCounts.get(base) ?? 0) > 1 ? `${base} ${index}` : base - return { - id: record.id, - label, - ...(model === undefined ? {} : { model }), - selected: this.selected.has(record.id), - } - }) - return { - devices: views, - selectedDeviceIds: views.filter(device => device.selected).map(device => device.id), - } - } - - /** List every ADB row, including unauthorized and offline devices, without exposing serials. */ - async inspect(signal: AbortSignal): Promise { - const devices = (await this.discover(signal)).toSorted((a, b) => a.serial.localeCompare(b.serial)) - this.syncRecords(devices, true) - const modelCounts = new Map() - for (const device of devices) { - const model = displayModel(device) ?? 'Android phone' - modelCounts.set(model, (modelCounts.get(model) ?? 0) + 1) - } - const modelIndexes = new Map() - return devices.map((device): FleetDeviceStatusView => { - const record = this.records.get(device.serial)! - const model = displayModel(device) - const base = model ?? 'Android phone' - const index = (modelIndexes.get(base) ?? 0) + 1 - modelIndexes.set(base, index) - return { - id: record.id, - label: (modelCounts.get(base) ?? 0) > 1 ? `${base} ${index}` : base, - ...(model === undefined ? {} : { model }), - state: device.state, - connected: true, - authorized: device.state === 'device', - } - }) - } - - /** Materialize only rows in the caller's discovery snapshot; do not rediscover per phone. */ - resolveInspected(deviceId: string): string | undefined { - return [...this.records.values()].find(record => record.id === deviceId)?.serial - } - - async select(deviceIds: readonly string[], signal: AbortSignal): Promise { - const snapshot = await this.snapshot(signal) - const available = new Set(snapshot.devices.map(device => device.id)) - const unique = [...new Set(deviceIds)] - if (unique.some(id => !available.has(id))) { - throw new Error('opengui: device selection contains a disconnected or unknown phone') - } - this.selected.clear() - for (const id of unique) this.selected.add(id) - return this.snapshot(signal) - } - - /** Resolve browser-safe ids to current Host-private devices for an immediate operation. */ - async resolveConnected(deviceIds: readonly string[], signal: AbortSignal): Promise { - const snapshot = await this.snapshot(signal) - return this.materialize(snapshot, deviceIds) - } - - async selectedDevices(signal: AbortSignal): Promise { - const snapshot = await this.snapshot(signal) - if (snapshot.devices.length === 0) { - throw new Error('opengui: no authorized Android device is connected; connect a phone and accept its USB debugging prompt') - } - if (snapshot.selectedDeviceIds.length === 0) { - throw new Error('opengui: multiple phones are connected; select at least one in the deviceIds in opengui_open_session') - } - return this.materialize(snapshot, snapshot.selectedDeviceIds) - } - - private materialize(snapshot: DeviceFleetSnapshot, deviceIds: readonly string[]): readonly FleetDevice[] { - const views = new Map(snapshot.devices.map(device => [device.id, device])) - const byId = new Map([...this.records.values()].map(record => [record.id, record])) - return [...new Set(deviceIds)].map((id) => { - const view = views.get(id) - const record = byId.get(id) - if (view === undefined || record === undefined) { - throw new Error('opengui: device is disconnected or unknown') - } - return { - id, - serial: record.serial, - label: view.label, - ...(view.model === undefined ? {} : { model: view.model }), - } - }) - } - - - private syncRecords(devices: readonly AdbDevice[], includeUnavailable: boolean): void { - const connectedSerials = new Set(devices.map(device => device.serial)) - for (const [serial, record] of this.records) { - if (connectedSerials.has(serial)) continue - // Retain opaque identity for frozen sessions across physical reconnects. - // Current discovery still gates materialization; a remembered id grants no access. - this.selected.delete(record.id) - } - const candidates = includeUnavailable ? devices : devices.filter(device => device.state === 'device') - for (const device of candidates.toSorted((a, b) => { - const authorization = Number(b.state === 'device') - Number(a.state === 'device') - return authorization === 0 ? a.serial.localeCompare(b.serial) : authorization - })) { - if (!this.records.has(device.serial)) { - this.records.set(device.serial, { id: this.createId(), serial: device.serial }) - } - } - } -} +export * from '../../packages/device-runtime/src/device-fleet.ts' diff --git a/workbuddy-plugin/src/errors.ts b/workbuddy-plugin/src/errors.ts index b894c97..a856432 100644 --- a/workbuddy-plugin/src/errors.ts +++ b/workbuddy-plugin/src/errors.ts @@ -1,46 +1 @@ -export type ExecutionState = 'not_executed' | 'executed' | 'outcome_unknown' -export type Recovery = 'observe' | 'reconnect' | 'wait' | 'replan' | 'stop' - -/** Structured execution evidence, never an instruction to replay a mutation. */ -export class OpenGuiError extends Error { - constructor( - readonly code: string, - message: string, - readonly executionState: ExecutionState = 'not_executed', - readonly recovery: Recovery = 'stop', - ) { super(message); this.name = 'OpenGuiError' } -} - -export function errorInfo(error: unknown): { code: string; message: string; executionState: ExecutionState; recovery: Recovery } { - if (error instanceof OpenGuiError) return { code: error.code, message: error.message, executionState: error.executionState, recovery: error.recovery } - const message = error instanceof Error ? error.message : String(error) - const code = /stale|observe.*before|observation.*unavailable|current frame/u.test(message) ? 'observation_required' - : /invalid arguments|unknown tool|must be|is required/u.test(message) ? 'invalid_arguments' - : /waiting_for_display/u.test(message) ? 'waiting_for_display' - : /no screen progress|repeated action/u.test(message) ? 'no_progress' - : /operation.*limit|budget/u.test(message) ? 'budget_exhausted' - : /device offline|device not found|not connected/u.test(message) ? 'device_offline' - : /disconnected|ECONNRESET|EPIPE|ECONNREFUSED/u.test(message) ? 'connection_lost' - : /locked by another/u.test(message) ? 'device_busy' - : /cancelled|aborted|session is closed/u.test(message) ? 'cancelled' : 'operation_failed' - const recovery: Recovery = code === 'observation_required' ? 'observe' - : code === 'connection_lost' ? 'reconnect' - : code === 'waiting_for_display' || code === 'device_busy' || code === 'device_offline' ? 'wait' - : code === 'invalid_arguments' || code === 'no_progress' ? 'replan' : 'stop' - return { code, message, executionState: 'not_executed', recovery } -} - -/** Retry only explicitly transient, non-mutating work. */ -export async function retryRead(operation: () => Promise, signal: AbortSignal): Promise { - for (let attempt = 0; ; attempt++) { - signal.throwIfAborted() - try { return await operation() } catch (error) { - if (signal.aborted || attempt >= 2 || !/ECONNRESET|EPIPE|ETIMEDOUT|ECONNREFUSED|EAI_AGAIN|fetch failed|device offline|device .*not found|transport error|temporarily unavailable/iu.test(String(error))) throw error - await new Promise((resolve, reject) => { - const abort = (): void => { clearTimeout(timer); reject(signal.reason) } - const timer = setTimeout(() => { signal.removeEventListener('abort', abort); resolve() }, attempt === 0 ? 250 : 1000) - signal.addEventListener('abort', abort, { once: true }) - }) - } - } -} +export * from '../../packages/device-runtime/src/errors.ts' diff --git a/workbuddy-plugin/src/phone-controller.ts b/workbuddy-plugin/src/phone-controller.ts index 3a0fcb6..c138895 100644 --- a/workbuddy-plugin/src/phone-controller.ts +++ b/workbuddy-plugin/src/phone-controller.ts @@ -1,243 +1,10 @@ -import { createHash } from 'node:crypto' -import { - actionCommand, - canUseAdbInputText, - normalizePhoneAction, - parseScreenSize, - textInputCommands, -} from './adb.ts' -import type { ObservationId, PhoneCoordinateSpace } from './adb.ts' -import { AsyncSemaphore } from './concurrency.ts' -import type { EncodedPhoneScreenshot } from './screenshot.ts' -import { PhoneExecutionState, PhoneOperationQueue, waitForPhoneUi } from './phone-execution.ts' -import type { PhoneExecutionSnapshot } from './phone-execution.ts' -import { errorInfo, OpenGuiError, retryRead } from './errors.ts' -import { frameChanged, sampleFrame } from './vision.ts' - -/** Host-neutral phone observation used by the WorkBuddy runtime. */ -export interface RawPhoneObservation { - readonly observationId: ObservationId - readonly unchangedFromObservationId?: ObservationId - readonly serial: string - readonly width: number - readonly height: number - readonly foregroundPackage: string - readonly capturedAt?: string - readonly settled?: boolean - readonly image: { - readonly data: Buffer - readonly mediaType: 'image/jpeg' - readonly bytes: number - readonly width: number - readonly height: number - readonly name: string - } -} - -interface StoredObservation { - readonly value: RawPhoneObservation - readonly fingerprint: string -} - -export interface PhoneControllerOptions { - readonly runAdb: ( - args: readonly string[], - signal: AbortSignal, - buffer?: boolean, - ) => Promise - readonly discoverTarget: (signal: AbortSignal) => Promise - readonly validateTarget?: (serial: string, signal: AbortSignal) => Promise - readonly pasteUnicode: (serial: string, text: string, signal: AbortSignal) => Promise - readonly encodeScreenshot: (source: Buffer) => Promise - readonly maxOperations: () => number - readonly mediaPermits?: AsyncSemaphore - readonly now?: () => number - readonly settleIntervalMs?: number - readonly settleTimeoutMs?: number -} - -function currentPackage(output: string): string { - return output.match(/(?:mCurrentFocus|mFocusedApp)=[^\n]*?\bu\d+\s+([A-Za-z0-9._]+)\//u)?.[1] - ?? output.match(/(?:topResumedActivity|mResumedActivity)[^\n]*?\bu\d+\s+([A-Za-z0-9._]+)\//u)?.[1] - ?? '' -} - -/** - * One shared execution kernel for the independent WorkBuddy package. - * Host adapters provide only target discovery, process execution, and Unicode - * clipboard transport; safety and observation semantics live here. - */ -export class PhoneController { - private readonly observations = new WeakMap() - private readonly execution = new PhoneExecutionState() - private readonly queue = new PhoneOperationQueue() - private readonly mediaPermits: AsyncSemaphore - private readonly now: () => number - - constructor(private readonly options: PhoneControllerOptions) { - this.mediaPermits = options.mediaPermits ?? new AsyncSemaphore(2) - this.now = options.now ?? Date.now - } - - /** Freeze an actor to one Host-private device serial. */ - assignTarget(actor: object, serial: string): void { - this.execution.assignTarget(actor, serial) - } - - /** Return counters without exposing or mutating the current observation. */ - status(actor: object): PhoneExecutionSnapshot { - return this.execution.snapshot(actor) - } - - invalidate(actor: object): void { this.execution.consumeObservation(actor) } - - /** Observe without accepting arbitrary device commands. */ - observe(actor: object, signal: AbortSignal): Promise { - return this.execute(actor, { action: 'observe' }, signal) - } - - /** Execute exactly one validated operation and always return the resulting frame. */ - async execute( - actor: object, - input: Record, - signal: AbortSignal, - ): Promise { - return this.queue.run(actor, async () => { - let dispatched = false - try { - signal.throwIfAborted() - this.execution.beginOperation(actor, this.options.maxOperations()) - const normalized = { ...input } - delete normalized.verifyCurrentFrame - const action = normalizePhoneAction(normalized) - const serial = await this.targetFor(actor, signal) - await this.options.validateTarget?.(serial, signal) - if (action.action === 'observe') return this.capture(actor, serial, signal) - - const before = this.execution.current(actor, action.observationId) - const stored = this.observations.get(actor) - if (stored === undefined || stored.value.observationId !== action.observationId) { - throw new Error('opengui: current phone observation is unavailable') - } - const screen: PhoneCoordinateSpace = { - width: stored.value.width, - height: stored.value.height, - screenshotWidth: stored.value.image.width, - screenshotHeight: stored.value.image.height, - } - if (action.action === 'wait') { - this.execution.consumeObservation(actor) - await waitForPhoneUi(action.waitMs, signal) - return this.capture(actor, serial, signal) - } - - // Consume the supplied credential while checking the current real frame. - // If it changed, the model must see a new observation, never reuse coordinates. - const current = await this.capture(actor, serial, signal) - const currentState = this.execution.current(actor, current.observationId) - const region = action.action === 'tap' ? { - left: action.targetBBox.left / stored.value.image.width, - top: action.targetBBox.top / stored.value.image.height, - right: action.targetBBox.right / stored.value.image.width, - bottom: action.targetBBox.bottom / stored.value.image.height, - } : undefined - if (current.width !== stored.value.width || current.height !== stored.value.height - || current.foregroundPackage !== stored.value.foregroundPackage - || (before.visual && currentState.visual && (frameChanged(before.visual, currentState.visual) || (region && frameChanged(before.visual, currentState.visual, region))))) { - throw new OpenGuiError('screen_changed', 'opengui: phone changed before dispatch; observe the new screen before acting', 'not_executed', 'observe') - } - - const scrcpyText = action.action === 'text' && !canUseAdbInputText(action.text) - const command = action.action === 'text' ? undefined : actionCommand(action, screen) - const commands = action.action === 'text' - ? scrcpyText ? [] : textInputCommands(action.text) - : command === undefined ? [] : [command] - if (commands.length === 0 && !scrcpyText) { - throw new Error('opengui: action did not resolve to a device command') - } - - const signature = JSON.stringify(scrcpyText ? ['scrcpy-text', action.text] : commands) - this.execution.assertActionAllowed(actor, signature, before) - this.execution.consumeObservation(actor) - signal.throwIfAborted() - dispatched = true - if (scrcpyText) await this.options.pasteUnicode(serial, action.text, signal) - else for (const candidate of commands) await this.options.runAdb(['-s', serial, ...candidate], signal) - - const after = await this.captureSettled(actor, serial, signal) - const afterState = this.execution.current(actor, after.observationId) - this.execution.recordActionResult(actor, signature, before, afterState) - return after - } catch (error) { - this.execution.consumeObservation(actor) - const info = errorInfo(error) - throw new OpenGuiError(info.code, info.message, dispatched ? 'outcome_unknown' : info.executionState, dispatched ? 'observe' : info.recovery) - } - }) - } - - private async captureSettled(actor: object, serial: string, signal: AbortSignal): Promise { - const interval = this.options.settleIntervalMs ?? 250 - const timeout = this.options.settleTimeoutMs ?? 2000 - const deadline = Date.now() + timeout - let previous = await this.capture(actor, serial, signal) - while (Date.now() + interval <= deadline) { - const before = this.execution.current(actor, previous.observationId) - await waitForPhoneUi(interval, signal) - const current = await this.capture(actor, serial, signal) - const after = this.execution.current(actor, current.observationId) - if (before.visual && after.visual && !frameChanged(before.visual, after.visual)) return { ...current, settled: true } - previous = current - } - return { ...previous, settled: false } - } - - private async targetFor(actor: object, signal: AbortSignal): Promise { - return this.execution.resolveTarget(actor, () => this.options.discoverTarget(signal)) - } - - private async capture(actor: object, serial: string, signal: AbortSignal): Promise { - this.execution.consumeObservation(actor) - const releaseMedia = await this.mediaPermits.acquire(signal) - try { - const [sizeRaw, focusRaw, pngRaw] = await retryRead(() => Promise.all([ - this.options.runAdb(['-s', serial, 'shell', 'wm', 'size'], signal), - this.options.runAdb(['-s', serial, 'shell', 'dumpsys', 'window', 'windows'], signal), - this.options.runAdb(['-s', serial, 'exec-out', 'screencap', '-p'], signal, true), - ]), signal) - const screen = parseScreenSize(String(sizeRaw)) - const png = Buffer.isBuffer(pngRaw) ? pngRaw : Buffer.from(pngRaw) - const encoded = await this.options.encodeScreenshot(png) - signal.throwIfAborted() - const fingerprint = createHash('sha256').update(encoded.data).digest('hex') - const previous = this.observations.get(actor) - const unchanged = previous?.fingerprint === fingerprint ? previous : undefined - const observationId = this.execution.nextObservationId(actor) - const value: RawPhoneObservation = { - observationId, - ...(unchanged === undefined ? {} : { unchangedFromObservationId: unchanged.value.observationId }), - serial, - width: encoded.sourceWidth ?? screen.width, - height: encoded.sourceHeight ?? screen.height, - foregroundPackage: currentPackage(String(focusRaw)), - capturedAt: new Date(this.now()).toISOString(), - settled: false, - image: unchanged?.value.image ?? { - data: encoded.data, - mediaType: 'image/jpeg', - bytes: encoded.data.byteLength, - width: encoded.width, - height: encoded.height, - name: `opengui-phone-${this.now()}.jpg`, - }, - } - this.observations.set(actor, { value, fingerprint }) - const visual = await sampleFrame(encoded.data) - signal.throwIfAborted() - this.execution.recordObservation(actor, { observationId, screenshotFingerprint: fingerprint, visual }) - return value - } finally { - releaseMedia() - } +import { PhoneController as RuntimePhoneController, type PhoneControllerOptions } from '../../packages/device-runtime/src/phone-controller.ts' +import { sampleFrame } from './vision.ts' +export type { PhoneControllerOptions, RawPhoneObservation } from '../../packages/device-runtime/src/phone-controller.ts' + +/** WorkBuddy retains its pixel guard, read retries and settled-frame policy. */ +export class PhoneController extends RuntimePhoneController { + constructor(options: PhoneControllerOptions) { + super({ ...options, sampleFrame, readScreenSize: true, retryReads: true }) } } diff --git a/workbuddy-plugin/src/phone-execution.ts b/workbuddy-plugin/src/phone-execution.ts index f1fe76d..cb57a18 100644 --- a/workbuddy-plugin/src/phone-execution.ts +++ b/workbuddy-plugin/src/phone-execution.ts @@ -1,188 +1 @@ -import { ObservationId } from './adb.ts' -import { randomUUID } from 'node:crypto' -import type { ObservationId as ObservationIdType } from './adb.ts' -import { frameChanged, type VisualFrame } from './vision.ts' - -/** Minimal observation identity retained outside the durable tool result. */ -export interface PhoneFrameState { - observationId: ObservationIdType - screenshotFingerprint: string - visual?: VisualFrame -} - -interface NoProgressState { - signature: string - screenshotFingerprint: string - count: number - frame: PhoneFrameState -} - -interface AgentPhoneState { - identity: string - operations: number - observationSequence: number - targetSerial?: string - latest?: PhoneFrameState - noProgress?: NoProgressState -} - -/** Read-only execution counters used by Host adapters and status tools. */ -export interface PhoneExecutionSnapshot { - readonly operations: number - readonly observationSequence: number - readonly targetSerial?: string - readonly observationId?: ObservationIdType -} - -/** Per-child freshness, operation-budget, and repeated-no-progress enforcement. */ -export class PhoneExecutionState { - private readonly agents = new WeakMap() - - private state(agent: object): AgentPhoneState { - const existing = this.agents.get(agent) - if (existing !== undefined) return existing - const created: AgentPhoneState = { identity: randomUUID(), operations: 0, observationSequence: 0 } - this.agents.set(agent, created) - return created - } - - /** Bind a newly published actor to one Host-private serial before its first tool call. */ - assignTarget(agent: object, serial: string): void { - const state = this.state(agent) - if (state.targetSerial !== undefined && state.targetSerial !== serial) { - throw new Error('opengui: phone agent is already bound to another device') - } - state.targetSerial = serial - } - - /** Count one tool operation and enforce the actor's bounded action budget. */ - beginOperation(agent: object, maxOperations: number): void { - const state = this.state(agent) - state.operations += 1 - if (state.operations > maxOperations) { - throw new Error(`opengui: phone task exceeded its ${maxOperations}-operation limit`) - } - } - - /** Resolve the task's phone once and reuse that serial for every later call. */ - async resolveTarget(agent: object, discover: () => Promise): Promise { - const state = this.state(agent) - if (state.targetSerial !== undefined) return state.targetSerial - const selected = await discover() - state.targetSerial = selected - return selected - } - - /** Allocate an actor-local observation id that the next mutation must echo. */ - nextObservationId(agent: object): ObservationIdType { - const state = this.state(agent) - state.observationSequence += 1 - return ObservationId(`phone-observation-${state.identity}-${state.observationSequence}`) - } - - /** Return the current frame, when the actor has observed one. */ - latest(agent: object): PhoneFrameState | undefined { - return this.state(agent).latest - } - - /** Consume before dispatch so a failed or cancelled mutation cannot reuse its old frame. */ - consumeObservation(agent: object): void { - delete this.state(agent).latest - } - - /** Publish a completed observation as the only current frame. */ - recordObservation(agent: object, frame: PhoneFrameState): void { - const state = this.state(agent) - if (state.latest !== undefined && this.changed(state.latest, frame)) { - delete state.noProgress - } - state.latest = frame - } - - /** Require an action to name the exact current frame and return that frame. */ - current(agent: object, observationId: ObservationIdType): PhoneFrameState { - const latest = this.state(agent).latest - if (latest === undefined) throw new Error('opengui: observe the phone before performing an action') - if (latest.observationId !== observationId) { - throw new Error(`opengui: stale observationId ${observationId}; use current observationId ${latest.observationId}`) - } - return latest - } - - /** Reject a fourth identical action after three unchanged resulting frames. */ - assertActionAllowed(agent: object, signature: string, before: PhoneFrameState): void { - const state = this.state(agent) - const noProgress = state.noProgress - if (noProgress?.count === 3 - && noProgress.signature === signature - && !this.changed(noProgress.frame, before)) { - throw new Error('opengui: repeated action made no screen progress three times; choose another action or report blocked') - } - } - - /** Update the repeated-action fuse from the frame before and after one mutation. */ - recordActionResult(agent: object, signature: string, before: PhoneFrameState, after: PhoneFrameState): void { - const state = this.state(agent) - if (this.changed(before, after)) { - delete state.noProgress - return - } - const previous = state.noProgress - state.noProgress = previous?.signature === signature && !this.changed(previous.frame, after) - ? { ...previous, count: previous.count + 1, frame: after } - : { signature, screenshotFingerprint: after.screenshotFingerprint, count: 1, frame: after } - } - - private changed(before: PhoneFrameState, after: PhoneFrameState): boolean { - return before.visual && after.visual ? frameChanged(before.visual, after.visual) : before.screenshotFingerprint !== after.screenshotFingerprint - } - - /** Return a copy of one actor's bounded execution state. */ - snapshot(agent: object): PhoneExecutionSnapshot { - const state = this.state(agent) - return { - operations: state.operations, - observationSequence: state.observationSequence, - ...(state.targetSerial === undefined ? {} : { targetSerial: state.targetSerial }), - ...(state.latest === undefined ? {} : { observationId: state.latest.observationId }), - } - } -} - -/** Serialize one actor's tool calls while allowing different actors to proceed in parallel. */ -export class PhoneOperationQueue { - private readonly tails = new WeakMap>() - - async run(agent: object, operation: () => Promise): Promise { - const previous = this.tails.get(agent) ?? Promise.resolve() - let release!: () => void - const gate = new Promise((resolve) => { release = resolve }) - const tail = previous.catch(() => {}).then(() => gate) - this.tails.set(agent, tail) - await previous.catch(() => {}) - try { - return await operation() - } finally { - release() - if (this.tails.get(agent) === tail) this.tails.delete(agent) - } - } -} - -/** Wait for an explicit UI-settle operation and honor cancellation. */ -export function waitForPhoneUi(waitMs: number, signal: AbortSignal): Promise { - signal.throwIfAborted() - return new Promise((resolve, reject) => { - const finish = (): void => { - signal.removeEventListener('abort', abort) - resolve() - } - const timer = setTimeout(finish, waitMs) - const abort = (): void => { - clearTimeout(timer) - signal.removeEventListener('abort', abort) - reject(signal.reason instanceof Error ? signal.reason : new Error('opengui: phone wait cancelled')) - } - signal.addEventListener('abort', abort, { once: true }) - }) -} +export * from '../../packages/device-runtime/src/phone-execution.ts' diff --git a/workbuddy-plugin/src/screenshot.ts b/workbuddy-plugin/src/screenshot.ts index ab3c8f2..64badf4 100644 --- a/workbuddy-plugin/src/screenshot.ts +++ b/workbuddy-plugin/src/screenshot.ts @@ -1,12 +1,7 @@ import sharp from 'sharp' -export interface EncodedPhoneScreenshot { - readonly data: Buffer - readonly width: number - readonly height: number - readonly sourceWidth?: number - readonly sourceHeight?: number -} +import type { EncodedPhoneScreenshot } from '../../packages/device-runtime/src/image.ts' +export type { EncodedPhoneScreenshot } from '../../packages/device-runtime/src/image.ts' /** Bound model-visible images without requiring a host-specific image helper. */ export async function encodeWorkBuddyPhoneScreenshot(source: Buffer): Promise { diff --git a/workbuddy-plugin/src/vision.ts b/workbuddy-plugin/src/vision.ts index 345e134..febbd47 100644 --- a/workbuddy-plugin/src/vision.ts +++ b/workbuddy-plugin/src/vision.ts @@ -1,27 +1,9 @@ import sharp from 'sharp' - -export interface VisualFrame { readonly pixels: Buffer; readonly width: number; readonly height: number } -export interface ImageRegion { readonly left: number; readonly top: number; readonly right: number; readonly bottom: number } +import type { VisualFrame } from '../../packages/device-runtime/src/frame-comparison.ts' +export * from '../../packages/device-runtime/src/frame-comparison.ts' /** Small RGB samples compare visual change, not JPEG encoding or semantic success. */ export async function sampleFrame(image: Buffer): Promise { const result = await sharp(image).removeAlpha().toColourspace('srgb').resize({ width: 256, withoutEnlargement: true }).raw().toBuffer({ resolveWithObject: true }) return { pixels: result.data, width: result.info.width, height: result.info.height } } - -/** Ignore isolated clock/cursor/compression noise, but never mask a target region. */ -export function frameChanged(before: VisualFrame, after: VisualFrame, region?: ImageRegion): boolean { - if (before.width !== after.width || before.height !== after.height) return true - const left = Math.max(0, Math.floor((region?.left ?? 0) * before.width)) - const right = Math.min(before.width, Math.ceil((region?.right ?? 1) * before.width)) - const top = Math.max(0, Math.floor((region?.top ?? 0) * before.height)) - const bottom = Math.min(before.height, Math.ceil((region?.bottom ?? 1) * before.height)) - const total = (right - left) * (bottom - top) - if (total <= 0) return true - let changed = 0 - for (let y = top; y < bottom; y++) for (let x = left; x < right; x++) { - const offset = (y * before.width + x) * 3 - if ([0, 1, 2].some(channel => Math.abs(before.pixels[offset + channel]! - after.pixels[offset + channel]!) > 20)) changed++ - } - return changed / total > (region ? 0.01 : 0.02) -} diff --git a/workbuddy-plugin/tests/runtime-contract.spec.ts b/workbuddy-plugin/tests/runtime-contract.spec.ts new file mode 100644 index 0000000..04493b7 --- /dev/null +++ b/workbuddy-plugin/tests/runtime-contract.spec.ts @@ -0,0 +1,4 @@ +import { it } from 'vitest' +import { PhoneController } from '../src/phone-controller.ts' +import { controllerContract } from '../../packages/device-runtime/tests/controller-contract.ts' +controllerContract(it, options => new PhoneController(options))