feat(mobile): search by voice with the phone's on-device recogniser

apps/api/Dockerfile

@@ -12,6 +12,8 @@ RUN corepack enable && \
 FROM base AS deps
 WORKDIR /app
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
+# pnpm checks every patched dependency's file hash, installed here or not.
+COPY patches/ patches/
 COPY packages/core/package.json packages/core/
 COPY packages/extension-contracts/package.json packages/extension-contracts/
 COPY packages/db-schema/package.json packages/db-schema/
@@ -44,6 +46,7 @@ RUN pnpm install --frozen-lockfile
 FROM base AS prod-deps
 WORKDIR /app
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
+COPY patches/ patches/
 COPY packages/core/package.json packages/core/
 COPY packages/extension-contracts/package.json packages/extension-contracts/
 COPY packages/db-schema/package.json packages/db-schema/

apps/mobile/app.config.ts

@@ -74,6 +74,7 @@ const config: ExpoConfig = {
       backgroundColor: BRAND_COLOR,
     },
     permissions: [
+      "android.permission.RECORD_AUDIO",
       "android.permission.INTERNET",
       "android.permission.ACCESS_COARSE_LOCATION",
       "android.permission.ACCESS_FINE_LOCATION",
@@ -92,13 +93,13 @@ const config: ExpoConfig = {
       "android.permission.RECEIVE_BOOT_COMPLETED",
     ],
     // Permissions some bundled Expo modules declare by default but this app
-    // must not ship: no remote push, no exact alarms, no media/microphone
+    // must not ship: no remote push, no exact alarms, no media/background-microphone
     // capability.
     blockedPermissions: [
       "com.google.android.c2dm.permission.RECEIVE",
       "android.permission.SCHEDULE_EXACT_ALARM",
       "android.permission.USE_EXACT_ALARM",
-      "android.permission.RECORD_AUDIO",
+      "android.permission.FOREGROUND_SERVICE_MICROPHONE",
       "android.permission.CAMERA",
       "android.permission.READ_EXTERNAL_STORAGE",
       "android.permission.WRITE_EXTERNAL_STORAGE",
@@ -167,6 +168,13 @@ const config: ExpoConfig = {
         enableBackgroundRemoteNotifications: false,
       },
     ],
+    [
+      "expo-speech-recognition",
+      {
+        microphonePermission: IOS_USAGE_DESCRIPTIONS.en.NSMicrophoneUsageDescription,
+        speechRecognitionPermission: IOS_USAGE_DESCRIPTIONS.en.NSSpeechRecognitionUsageDescription,
+      },
+    ],
     "expo-system-ui",
   ],
   // Non-secret build inputs the running app reads through `expo-constants`.

apps/mobile/compliance/apple-privacy-worksheet.md

@@ -36,6 +36,22 @@ below remain pending review; omitted rows do not establish an exemption.
 | User Content | Review text, rating, nickname, photos and public signing identity the user chooses to publish | App Functionality | No | No |
 | Coarse Location | Tile and imagery requests revealing the viewed area, with network metadata such as IP address | App Functionality | No | No |
 
+## On-device voice-search audio
+
+Convert one user-initiated utterance into search text using the OS on-device recogniser on iOS and Android 13+; no network recogniser fallback is permitted.
+
+Audio is processed during the utterance only and is not retained by OpenMapX. Recording persistence is disabled. Transcripts are not logged by voice input; resulting search text follows normal search-request and search-history rules.
+
+User control: Tap the microphone to start and grant microphone permission and, on iOS, speech-recognition authorization; tap again to stop. Backgrounding or leaving the page cancels. Revoke permissions in platform settings.
+
+Proposed collection: Apple No; Google No.
+Proposed Google sharing: No.
+Purpose: App functionality.
+
+Design classification pending candidate device/privacy qualification. Android below API 33 is disabled. iOS requires locale-specific offline support and the fail-closed package patch; no app-triggered iOS model download is used. Model downloads require a network; recognition does not. Only transcript text crosses the bridge, then ordinary search may transmit it.
+
+These are design classifications, pending candidate device/privacy qualification.
+
 ## Tracking
 
 Proposed answer: No. Verify the exact binary's dependencies and privacy manifests,

apps/mobile/compliance/data-practices.json

@@ -500,6 +500,37 @@
       "legalSection": "third-parties",
       "notes": "Pending review of distributor handling, developer access and the exact binary SDK inventory. The retained not-collected values do not establish a real-time-processing exemption or prove that the operator receives no store diagnostics.",
       "encryptedInTransit": true
+    },
+    {
+      "id": "voice-search-audio",
+      "dataType": "Microphone audio for user-initiated on-device voice search",
+      "source": "device-sensor",
+      "leavesDevice": false,
+      "purpose": "Convert one user-initiated utterance into search text using the OS on-device recogniser on iOS and Android 13+; no network recogniser fallback is permitted.",
+      "linkedToAccount": false,
+      "retention": {
+        "kind": "ephemeral",
+        "note": "Audio is processed during the utterance only and is not retained by OpenMapX. Recording persistence is disabled. Transcripts are not logged by voice input; resulting search text follows normal search-request and search-history rules."
+      },
+      "shared": "none",
+      "userControl": "Tap the microphone to start and grant microphone permission and, on iOS, speech-recognition authorization; tap again to stop. Backgrounding or leaving the page cancels. Revoke permissions in platform settings.",
+      "appleLabel": {
+        "collected": false,
+        "category": "Audio Data",
+        "purposes": ["App Functionality"],
+        "linkedToIdentity": false,
+        "usedForTracking": false
+      },
+      "googleDataSafety": {
+        "collected": false,
+        "shared": false,
+        "category": "Audio files",
+        "ephemeral": true,
+        "optional": true,
+        "purposes": ["App functionality"]
+      },
+      "legalSection": "Voice Search (Microphone)",
+      "notes": "Design classification pending candidate device/privacy qualification. Android below API 33 is disabled. iOS requires locale-specific offline support and the fail-closed package patch; no app-triggered iOS model download is used. Model downloads require a network; recognition does not. Only transcript text crosses the bridge, then ordinary search may transmit it."
     }
   ]
 }

apps/mobile/compliance/data-practices.test.ts

@@ -63,6 +63,7 @@ const byId = new Map(practices.map((practice) => [practice.id, practice]));
 
 /** Every behaviour a reviewer would look for. A missing row is a missing answer. */
 const REQUIRED_PRACTICES = [
+  "voice-search-audio",
   "precise-location-active-navigation",
   "background-location-active-navigation",
   "route-request-coordinates",

apps/mobile/compliance/google-data-safety-worksheet.md

@@ -34,6 +34,22 @@ also be reviewed; their absence does not establish that data is not collected.
 | App activity | Review text, rating, nickname, photos and public signing identity the user chooses to publish | Shared | Retained | Optional | App functionality |
 | Location | Tile and imagery requests revealing the viewed area, with network metadata such as IP address | Shared | Processed ephemerally | Required | App functionality |
 
+## On-device voice-search audio
+
+Convert one user-initiated utterance into search text using the OS on-device recogniser on iOS and Android 13+; no network recogniser fallback is permitted.
+
+Audio is processed during the utterance only and is not retained by OpenMapX. Recording persistence is disabled. Transcripts are not logged by voice input; resulting search text follows normal search-request and search-history rules.
+
+User control: Tap the microphone to start and grant microphone permission and, on iOS, speech-recognition authorization; tap again to stop. Backgrounding or leaving the page cancels. Revoke permissions in platform settings.
+
+Proposed collection: Apple No; Google No.
+Proposed Google sharing: No.
+Purpose: App functionality.
+
+Design classification pending candidate device/privacy qualification. Android below API 33 is disabled. iOS requires locale-specific offline support and the fail-closed package patch; no app-triggered iOS model download is used. Model downloads require a network; recognition does not. Only transcript text crosses the bridge, then ordinary search may transmit it.
+
+These are design classifications, pending candidate device/privacy qualification.
+
 ## Security practices to verify
 
 - Verify HTTPS for the binary, served web code and every configured provider path.

apps/mobile/compliance/permissions.test.ts

@@ -37,8 +37,10 @@ describe("every permission is justified", () => {
 });
 
 describe("the permitted surface is minimal", () => {
-  it("asks for exactly two iOS usage descriptions", () => {
+  it("asks for exactly four iOS usage descriptions", () => {
     expect(expectedPermissionSurface().iosUsageDescriptionKeys).toEqual([
+      "NSMicrophoneUsageDescription",
+      "NSSpeechRecognitionUsageDescription",
       "NSLocationWhenInUseUsageDescription",
       "NSLocationAlwaysAndWhenInUseUsageDescription",
     ]);
@@ -50,10 +52,11 @@ describe("the permitted surface is minimal", () => {
     expect(expectedPermissionSurface().iosBackgroundModes).toEqual(["location"]);
   });
 
-  it("asks for twelve Android permissions and no more", () => {
+  it("asks for thirteen Android permissions and no more", () => {
     expect(expectedPermissionSurface().androidPermissions).toEqual([
       "androidx.car.app.NAVIGATION_TEMPLATES",
       "androidx.car.app.ACCESS_SURFACE",
+      "android.permission.RECORD_AUDIO",
       "android.permission.INTERNET",
       "android.permission.ACCESS_COARSE_LOCATION",
       "android.permission.ACCESS_FINE_LOCATION",
@@ -91,7 +94,7 @@ describe("the app config matches the permitted surface", () => {
   it("blocks the capabilities that would otherwise arrive by default", () => {
     for (const id of [
       "com.google.android.c2dm.permission.RECEIVE",
-      "android.permission.RECORD_AUDIO",
+      "android.permission.FOREGROUND_SERVICE_MICROPHONE",
       "android.permission.CAMERA",
       "android.permission.QUERY_ALL_PACKAGES",
     ]) {

apps/mobile/compliance/permissions.ts

@@ -21,6 +21,17 @@ export interface PermissionEntry {
 }
 
 export const IOS_USAGE_DESCRIPTION_KEYS: PermissionEntry[] = [
+  {
+    id: "NSMicrophoneUsageDescription",
+    practiceId: "voice-search-audio",
+    reason: "Linked speech module usage description; voice input remains disabled on iOS.",
+  },
+  {
+    id: "NSSpeechRecognitionUsageDescription",
+    practiceId: "voice-search-audio",
+    reason: "Linked speech module usage description; voice input remains disabled on iOS.",
+  },
+
   {
     id: "NSLocationWhenInUseUsageDescription",
     practiceId: "precise-location-active-navigation",
@@ -53,6 +64,11 @@ export const ANDROID_PERMISSIONS: PermissionEntry[] = [
     practiceId: "background-location-active-navigation",
     reason: "Map-only rendering on the Android Auto surface.",
   },
+  {
+    id: "android.permission.RECORD_AUDIO",
+    practiceId: "voice-search-audio",
+    reason: "On-device voice search initiated by the microphone button.",
+  },
   {
     id: "android.permission.INTERNET",
     practiceId: "route-request-coordinates",
@@ -112,11 +128,11 @@ export const ANDROID_PERMISSIONS: PermissionEntry[] = [
  * Capabilities that must be absent, and why each would be a problem.
  *
  * Redundant with the allowlist by construction, and kept anyway: the message a
- * failing check prints is the useful artefact, and "RECORD_AUDIO is present"
+ * failing check prints is the useful artefact, and "CAMERA is present"
  * says much less than the reason it must not be.
  */
 export const FORBIDDEN_ANDROID_PERMISSIONS: Record<string, string> = {
-  "android.permission.RECORD_AUDIO": "The shell has no microphone feature and declares none.",
+  "android.permission.FOREGROUND_SERVICE_MICROPHONE": "Voice input is foreground only.",
   "android.permission.CAMERA": "The shell has no camera feature.",
   "android.permission.READ_CONTACTS": "Never requested.",
   "android.permission.READ_CALENDAR": "Never requested.",
@@ -142,7 +158,6 @@ export const FORBIDDEN_ANDROID_PERMISSIONS: Record<string, string> = {
 };
 
 export const FORBIDDEN_IOS_KEYS: Record<string, string> = {
-  NSMicrophoneUsageDescription: "The shell has no microphone feature.",
   NSCameraUsageDescription: "The shell has no camera feature.",
   NSContactsUsageDescription: "Never requested.",
   NSCalendarsUsageDescription: "Never requested.",

apps/mobile/compliance/store-answers.mts

@@ -92,6 +92,7 @@ const REQUIRED_COVERAGE: { subject: string; practiceId: string }[] = [
   { subject: "WebView cookies and cache", practiceId: "webview-storage" },
   { subject: "map tile and imagery providers", practiceId: "map-and-media-providers" },
   { subject: "store distribution", practiceId: "store-operational-data" },
+  { subject: "on-device voice-search audio", practiceId: "voice-search-audio" },
 ];
 
 const byId = new Map(registry.practices.map((practice) => [practice.id, practice]));
@@ -115,6 +116,26 @@ const PREAMBLE = `<!-- Generated from data-practices.json by store-answers.mts.
      update registry prose or this template and regenerate. --check verifies
      generated-file parity, not behavior, privacy classifications or approval. -->`;
 
+function voiceAudioSection(): string {
+  const practice = byId.get("voice-search-audio") as Practice;
+  return `## On-device voice-search audio
+
+${practice.purpose}
+
+${practice.retention.note}
+
+User control: ${practice.userControl}
+
+Proposed collection: Apple ${practice.appleLabel.collected ? "Yes" : "No"}; Google ${practice.googleDataSafety.collected ? "Yes" : "No"}.
+Proposed Google sharing: ${practice.googleDataSafety.shared ? "Yes" : "No"}.
+Purpose: ${(practice.googleDataSafety.purposes ?? []).join(", ")}.
+
+${practice.notes}
+
+These are design classifications, pending candidate device/privacy qualification.
+`;
+}
+
 function appleWorksheet(): string {
   const rows = collectedForApple
     .map(
@@ -161,6 +182,7 @@ below remain pending review; omitted rows do not establish an exemption.
 | --- | --- | --- | --- | --- |
 ${rows}
 
+${voiceAudioSection()}
 ## Tracking
 
 Proposed answer: No. Verify the exact binary's dependencies and privacy manifests,
@@ -248,6 +270,7 @@ also be reviewed; their absence does not establish that data is not collected.
 | --- | --- | --- | --- | --- | --- |
 ${rows}
 
+${voiceAudioSection()}
 ## Security practices to verify
 
 - Verify HTTPS for the binary, served web code and every configured provider path.

apps/mobile/config/locales/de.json

@@ -1,4 +1,6 @@
 {
   "NSLocationWhenInUseUsageDescription": "OpenMapX verwendet deinen Standort, um deine Position auf der Karte anzuzeigen und dich entlang einer von dir gestarteten Route zu führen.",
-  "NSLocationAlwaysAndWhenInUseUsageDescription": "OpenMapX verwendet deinen Standort im Hintergrund, damit eine von dir gestartete Navigation dich weiter führt, während der Bildschirm gesperrt ist oder die App nicht verwendet wird. Die Standorterfassung endet, sobald du die Navigation beendest."
+  "NSLocationAlwaysAndWhenInUseUsageDescription": "OpenMapX verwendet deinen Standort im Hintergrund, damit eine von dir gestartete Navigation dich weiter führt, während der Bildschirm gesperrt ist oder die App nicht verwendet wird. Die Standorterfassung endet, sobald du die Navigation beendest.",
+  "NSMicrophoneUsageDescription": "OpenMapX verwendet das Mikrofon für die Sprachsuche, wenn du auf die Mikrofontaste tippst.",
+  "NSSpeechRecognitionUsageDescription": "OpenMapX verwendet die Spracherkennung auf dem Gerät, um deine Stimme beim Tippen auf die Mikrofontaste in Suchtext umzuwandeln."
 }

apps/mobile/config/locales/en.json

@@ -1,4 +1,6 @@
 {
   "NSLocationWhenInUseUsageDescription": "OpenMapX uses your location to show where you are on the map and to guide you along a route you start.",
-  "NSLocationAlwaysAndWhenInUseUsageDescription": "OpenMapX uses your location in the background so navigation you have started keeps guiding you while the screen is locked or the app is not in use. Tracking stops when you end navigation."
+  "NSLocationAlwaysAndWhenInUseUsageDescription": "OpenMapX uses your location in the background so navigation you have started keeps guiding you while the screen is locked or the app is not in use. Tracking stops when you end navigation.",
+  "NSMicrophoneUsageDescription": "OpenMapX uses the microphone for voice search when you tap the microphone button.",
+  "NSSpeechRecognitionUsageDescription": "OpenMapX uses on-device speech recognition to turn your voice into search text when you tap the microphone."
 }

apps/mobile/config/nativeCopy.ts

@@ -19,6 +19,8 @@ export const MOBILE_LOCALES = ["en", "de"] as const;
 export type MobileLocale = (typeof MOBILE_LOCALES)[number];
 
 export interface IosUsageDescriptions {
+  NSMicrophoneUsageDescription: string;
+  NSSpeechRecognitionUsageDescription: string;
   NSLocationWhenInUseUsageDescription: string;
   NSLocationAlwaysAndWhenInUseUsageDescription: string;
 }
@@ -30,12 +32,22 @@ export interface IosUsageDescriptions {
  */
 export const IOS_USAGE_DESCRIPTIONS: Record<MobileLocale, IosUsageDescriptions> = {
   en: {
+    NSMicrophoneUsageDescription:
+      "OpenMapX uses the microphone for voice search when you tap the microphone button.",
+    NSSpeechRecognitionUsageDescription:
+      "OpenMapX uses on-device speech recognition to turn your voice into search text when you tap the microphone.",
+
     NSLocationWhenInUseUsageDescription:
       "OpenMapX uses your location to show where you are on the map and to guide you along a route you start.",
     NSLocationAlwaysAndWhenInUseUsageDescription:
       "OpenMapX uses your location in the background so navigation you have started keeps guiding you while the screen is locked or the app is not in use. Tracking stops when you end navigation.",
   },
   de: {
+    NSMicrophoneUsageDescription:
+      "OpenMapX verwendet das Mikrofon für die Sprachsuche, wenn du auf die Mikrofontaste tippst.",
+    NSSpeechRecognitionUsageDescription:
+      "OpenMapX verwendet die Spracherkennung auf dem Gerät, um deine Stimme beim Tippen auf die Mikrofontaste in Suchtext umzuwandeln.",
+
     NSLocationWhenInUseUsageDescription:
       "OpenMapX verwendet deinen Standort, um deine Position auf der Karte anzuzeigen und dich entlang einer von dir gestarteten Route zu führen.",
     NSLocationAlwaysAndWhenInUseUsageDescription:

apps/mobile/package.json

@@ -33,6 +33,7 @@
     "expo-keep-awake": "~57.0.2",
     "expo-location": "~57.0.20",
     "expo-notifications": "~57.0.21",
+    "expo-speech-recognition": "57.1.1",
     "expo-splash-screen": "~57.0.9",
     "expo-sqlite": "~57.0.3",
     "expo-system-ui": "~57.0.4",

apps/mobile/plugins/nativeConfigTransforms.ts

@@ -27,6 +27,7 @@ export const EXPO_LOCATION_SERVICE_NAMES = ["expo.modules.location.services.Loca
 export const REQUIRED_ANDROID_PERMISSIONS = [
   "androidx.car.app.NAVIGATION_TEMPLATES",
   "androidx.car.app.ACCESS_SURFACE",
+  "android.permission.RECORD_AUDIO",
   "android.permission.INTERNET",
   "android.permission.ACCESS_COARSE_LOCATION",
   "android.permission.ACCESS_FINE_LOCATION",
@@ -214,14 +215,19 @@ export interface AndroidManifestLike {
  * a duplicate `<uses-permission>` node.
  */
 export function applyAndroidPermissions(manifest: AndroidManifestLike): AndroidManifestLike {
-  const existing = manifest.manifest["uses-permission"] ?? [];
+  const existing = (manifest.manifest["uses-permission"] ?? []).filter(
+    (node) =>
+      node.$["android:name"] !== "android.permission.FOREGROUND_SERVICE_MICROPHONE" ||
+      node.$["tools:node"] === "remove",
+  );
   const present = new Set(existing.map((node) => node.$["android:name"]));
   const additions = REQUIRED_ANDROID_PERMISSIONS.filter((name) => !present.has(name)).map(
     (name) => ({
       $: { "android:name": name },
     }),
   );
-  if (additions.length === 0) return manifest;
+  if (additions.length === 0 && existing.length === manifest.manifest["uses-permission"]?.length)
+    return manifest;
   return {
     ...manifest,
     manifest: { ...manifest.manifest, "uses-permission": [...existing, ...additions] },