From d89be7bd529a0c045a59bc763d51f03b1947ff99 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 04:45:11 +0000 Subject: [PATCH] Let another app read a whole video in one pinned voice Gecko Video Creator renders through Notes' speech, one chunk per request, and needs what Notes' own render worker does: - the account's voice when none is chosen, where `model` defaults to a fal.ai voice; - the engine pinned after the first chunk, so a Deepgram hiccup partway through cannot read one chunk on fal.ai in a different voice. POST /speech/tts takes an optional `provider` and `voice`. With them, both are resolved by resolve_render_speech, which is now shared with the worker. The response names them in X-Speech-Provider and X-Speech-Voice, for the caller to send back on every later chunk. Without `provider`, a request means exactly what it did. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016fiRJAt34ag62iPiuMkNhm --- backend/app/routers/settings.py | 27 +++++++++++++++-- backend/app/video/worker.py | 29 +++++++++++++----- backend/tests/test_video_voice.py | 49 +++++++++++++++++++++++++++++++ docs/gecko-video-creator.md | 2 +- 4 files changed, 95 insertions(+), 12 deletions(-) diff --git a/backend/app/routers/settings.py b/backend/app/routers/settings.py index 3aaeaa4..0b4f986 100644 --- a/backend/app/routers/settings.py +++ b/backend/app/routers/settings.py @@ -10,7 +10,7 @@ from datetime import datetime, timedelta import fal_client import httpx -from typing import Any, AsyncIterator, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, AsyncIterator, Callable, Dict, List, Literal, Optional, Tuple, Union from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, Response, UploadFile from fastapi.concurrency import run_in_threadpool from fastapi.responses import StreamingResponse @@ -1842,6 +1842,15 @@ class TTSRequest(BaseModel): text: str # The voice name; kept as `model` for backwards-compat with the frontend request shape. model: str = DEFAULT_TTS_VOICE + # Set by an app reading a whole video one chunk at a time (Gecko Video Creator). + # With it, the engine and voice are resolved the way Notes' own render worker + # resolves them (`video.worker.resolve_render_speech`): `voice` null means the + # account's voice, not `model`'s default, and the answer names both in + # X-Speech-Provider / X-Speech-Voice so the caller can pin them for every later + # chunk. Unpinned, a Deepgram hiccup partway through would read one chunk on + # fal.ai in a different voice. Absent, the request means what it always has. + provider: Optional[Literal["auto", "deepgram", "fal"]] = None + voice: Optional[str] = None @router.get("/speech/voices") @@ -2256,8 +2265,20 @@ async def synthesize_speech( session: Session = Depends(get_session), ): user_id = _get_user_id(request) - data, media_type = await synthesize_tts_bytes(session, user_id, payload.text, voice=payload.model) - return Response(content=data, media_type=media_type) + if payload.provider is None: + data, media_type = await synthesize_tts_bytes(session, user_id, payload.text, voice=payload.model) + return Response(content=data, media_type=media_type) + + from app.video.worker import resolve_render_speech + + provider, voice = resolve_render_speech(session, user_id, payload.voice, payload.provider) + data, media_type = await synthesize_tts_bytes( + session, user_id, payload.text, voice=voice, provider_override=provider, + ) + return Response(content=data, media_type=media_type, headers={ + "X-Speech-Provider": provider, + "X-Speech-Voice": voice or "", + }) @router.get("/usage") diff --git a/backend/app/video/worker.py b/backend/app/video/worker.py index c021ab8..725afa3 100644 --- a/backend/app/video/worker.py +++ b/backend/app/video/worker.py @@ -16,7 +16,7 @@ import threading import uuid from datetime import datetime, timedelta -from typing import Optional, Set +from typing import Optional, Set, Tuple from sqlmodel import Session, select @@ -88,6 +88,24 @@ def _render_voice(session: Session, user_id: str, provider: str, chosen: Optiona return load_selected_voice(session, user_id) +def resolve_render_speech( + session: Session, user_id: str, chosen_voice: Optional[str], provider: Optional[str] = None, +) -> Tuple[str, Optional[str]]: + """The engine and voice a whole render is read with, resolved once. + + `provider` "deepgram" or "fal" names the engine; None or "auto" takes the + account's setting, and "auto" there means Deepgram when a key is set. The voice + is `_render_voice`'s. Shared by this worker and by `POST /speech/tts` when + another app (Gecko Video Creator) renders, so both pin the same way. + """ + from app.routers.settings import load_deepgram_api_key, load_speech_config + + tts_provider = provider if provider in ("deepgram", "fal") else load_speech_config(session, user_id)["tts_provider"] + if tts_provider == "auto": + tts_provider = "deepgram" if load_deepgram_api_key(session, user_id) else "fal" + return tts_provider, _render_voice(session, user_id, tts_provider, chosen_voice) + + def _tts_caller(user_id: str, chosen_voice: Optional[str] = None): """A sync `text -> mp3 bytes` callable for the renderer. @@ -97,9 +115,7 @@ def _tts_caller(user_id: str, chosen_voice: Optional[str] = None): this runs on a worker thread, not in a request. `chosen_voice` is the video dialog's voice (`RenderOptions.voice`); None means the account's. """ - from app.routers.settings import ( - load_deepgram_api_key, load_speech_config, synthesize_tts_bytes, - ) + from app.routers.settings import synthesize_tts_bytes with Session(engine) as session: # Resolve "auto" to a concrete provider once, up front, and pin it for @@ -109,10 +125,7 @@ def _tts_caller(user_id: str, chosen_voice: Optional[str] = None): # that one chunk, handing back a video that audibly changes voice # mid-way. Pinned, the same hiccup fails the render instead, which is # a render worth retrying rather than a video worth keeping. - tts_provider = load_speech_config(session, user_id)["tts_provider"] - if tts_provider == "auto": - tts_provider = "deepgram" if load_deepgram_api_key(session, user_id) else "fal" - voice = _render_voice(session, user_id, tts_provider, chosen_voice) + tts_provider, voice = resolve_render_speech(session, user_id, chosen_voice) def call(text: str) -> bytes: with Session(engine) as session: diff --git a/backend/tests/test_video_voice.py b/backend/tests/test_video_voice.py index 8d233d5..6618e58 100644 --- a/backend/tests/test_video_voice.py +++ b/backend/tests/test_video_voice.py @@ -107,3 +107,52 @@ def test_a_render_is_read_in_the_dialogs_voice_on_the_engine_it_pins( worker._tts_caller(USER, chosen)("A chunk of narration.") assert heard == [expected] + + +# ── POST /speech/tts for another app's render (Gecko Video Creator) ────────── + +def _speak(session, **body): + from types import SimpleNamespace + + request = SimpleNamespace(state=SimpleNamespace(user_id=USER)) + payload = settings.TTSRequest(text="A chunk of narration.", **body) + return asyncio.run(settings.synthesize_speech(payload, request, session)) + + +def test_another_apps_render_is_read_in_the_accounts_voice_and_told_which(session, monkeypatch): + _set(session, "tts_model", "Rex") + _set(session, settings._SPEECH_CONFIG, + {"tts_provider": "auto", "deepgram_tts_model": "flux-haley-en"}) + heard = _fake_providers(monkeypatch) + + response = _speak(session, provider="auto") + + # "auto" with a Deepgram key: Deepgram, in the account's Flux voice rather than + # `model`'s fal.ai default. + assert heard == [("deepgram", "flux-haley-en")] + assert response.headers["X-Speech-Provider"] == "deepgram" + # What to send back on every later chunk, so the video keeps one voice. + assert response.headers["X-Speech-Voice"] == "Rex" + + +def test_a_pinned_engine_and_voice_are_kept(session, monkeypatch): + _set(session, "tts_model", "Rex") + _set(session, settings._SPEECH_CONFIG, + {"tts_provider": "auto", "deepgram_tts_model": "flux-haley-en"}) + heard = _fake_providers(monkeypatch) + + response = _speak(session, provider="fal", voice=FAL_VOICE) + + assert heard == [("fal", FAL_VOICE)] + assert response.headers["X-Speech-Provider"] == "fal" + assert response.headers["X-Speech-Voice"] == FAL_VOICE + + +def test_read_aloud_without_a_provider_is_unchanged(session, monkeypatch): + _set(session, settings._SPEECH_CONFIG, {"tts_provider": "fal"}) + heard = _fake_providers(monkeypatch) + + response = _speak(session, model=FAL_VOICE) + + assert heard == [("fal", FAL_VOICE)] + assert "X-Speech-Provider" not in response.headers diff --git a/docs/gecko-video-creator.md b/docs/gecko-video-creator.md index 039b25b..7ce6f23 100644 --- a/docs/gecko-video-creator.md +++ b/docs/gecko-video-creator.md @@ -938,7 +938,7 @@ or extract. | Note as Markdown for the AI | expose `note_to_markdown`, e.g. `GET /api/notes/{id}?format=markdown` | ➕ small | | "Make video" in Notes | a button / slash-menu item → `https://gvc.geckopico.com/new?note={id}` (replaces today's dialog) | ➕ small | | Cross-origin calls | add `https://gvc.geckopico.com` to `CORS_ORIGIN` and the CSP in [`frontend/nginx.conf`](../frontend/nginx.conf), only where GVC's browser code calls Notes directly (server-side calls with a Bearer token need neither, as in GAM) | ➕ config | -| Speech for GVC's renders | `POST /api/settings/speech/tts`, plus a `provider` field passed through to `synthesize_tts_bytes(provider_override=…)`, so one video can't switch voices partway | ➕ small | +| Speech for GVC's renders | `POST /api/settings/speech/tts` with `provider` (and `voice`, null for the account's): resolved as the render worker resolves them, and named back in `X-Speech-Provider` / `X-Speech-Voice` so GVC pins both for every chunk and one video can't switch voices partway | ✅ | | Put the finished video back in the note | No endpoint can write `Note.content` safely against the editor's autosave (`routers/assets.py:174`), and today's auto-insert relies on Notes' own job registry | ➕ needs design (open decision 11) | **Re-sync.** Every imported segment records the `block_ids` it came from, and every beat