diff --git a/backend/app/routers/settings.py b/backend/app/routers/settings.py index c0d6217..3aaeaa4 100644 --- a/backend/app/routers/settings.py +++ b/backend/app/routers/settings.py @@ -2129,7 +2129,11 @@ async def synthesize_tts_bytes( # ── Deepgram (Flux) path ────────────────────────────────────────────────── if use_deepgram: - dg_voice = speech_cfg["deepgram_tts_model"] or DEFAULT_DEEPGRAM_TTS_MODEL + # A caller may name a Flux voice (the video dialog's voice choice does); + # anything else — read-aloud passes a fal.ai voice name — means the + # account's Deepgram voice. + dg_voice = (voice if voice in _DEEPGRAM_TTS_VOICE_IDS + else speech_cfg["deepgram_tts_model"] or DEFAULT_DEEPGRAM_TTS_MODEL) dg_expressivity = speech_cfg["deepgram_tts_expressivity"] dg_speed = speech_cfg["deepgram_tts_speed"] # Expressivity and speed both change the audio itself, so both are part diff --git a/backend/app/routers/video.py b/backend/app/routers/video.py index e75b6cc..ff2559b 100644 --- a/backend/app/routers/video.py +++ b/backend/app/routers/video.py @@ -9,7 +9,7 @@ import json import uuid -from datetime import datetime +from datetime import datetime, timezone from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Request @@ -236,6 +236,8 @@ def video_spec(payload: VideoRenderRequest, request: Request, session: Session = author=(user.username if user else "") or "", has_audio=F.probe_has_audio, note_id=note.id, + imported_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + selection=bool(payload.selected_content), ) return DataResponse(data=VideoSpecRead( spec=_utf8_safe(imported.spec.model_dump(mode="json", by_alias=True, exclude_none=True)), diff --git a/backend/app/video/__init__.py b/backend/app/video/__init__.py index 08269bf..2e51552 100644 --- a/backend/app/video/__init__.py +++ b/backend/app/video/__init__.py @@ -6,7 +6,10 @@ Module layout: options.py RenderOptions + the aspect/resolution/quality tables - segmenter.py BlockNote JSON -> [Shot] (pure) + spec.py the VideoSpec: a video described as data (pure) + importer.py BlockNote JSON -> VideoSpec (pure) + compiler.py VideoSpec -> [Shot] + RenderOptions (pure) + segmenter.py the Shot model; segment() = compile(import()) (pure) compose.py Pillow: title cards, overlay PNGs, backgrounds narration.py chunking, TTS, silence padding, durations, SRT ffmpeg.py argv + filtergraph builders, run/probe helpers (pure builders) diff --git a/backend/app/video/compiler.py b/backend/app/video/compiler.py index 4b0db21..c725d2e 100644 --- a/backend/app/video/compiler.py +++ b/backend/app/video/compiler.py @@ -16,8 +16,8 @@ from typing import Dict, List, Optional, Tuple from app.video.options import ( - BackgroundSpec, BumperSpec, KenBurnsSpec, MusicSpec, OverlayTextSpec, QuoteSpec, - RenderOptions, WatermarkSpec, WaveformSpec, + BackgroundSpec, BumperSpec, CardTextSpec, CodeSpec, KenBurnsSpec, MusicSpec, + OverlayTextSpec, QuoteSpec, RenderOptions, TransitionSpec, WatermarkSpec, WaveformSpec, ) from app.video.segmenter import ( Segmentation, Shot, ShotKind, _set_apart, resolve_media_path, @@ -110,6 +110,10 @@ def _segment_shots(segment: Segment, spec: VideoSpec, resolve, heading_ms: int) )] if segment.kind == "clip": + if path is None: + # Its file has gone since the import: skipped, as the warning says, + # since a sounded clip is nothing without its own picture and sound. + return [] bumper = segment.role if segment.role in ("intro", "outro") else None return [Shot(kind="video_sound", background=path, chapter=segment.chapter, bumper=bumper, label=segment.label)] @@ -163,6 +167,11 @@ def _overlay_beat(segment: Segment, overlay) -> str: show = overlay.show start = show.from_ if show is not None else None end = show.to if show is not None else None + for anchor in (start, end): + if (isinstance(anchor, AnchorRef) and anchor.beat is not None + and not any(b.id == anchor.beat for b in segment.beats)): + raise UnsupportedSpec(f"overlay {overlay.id!r} in segment {segment.id!r} is anchored " + f"to beat {anchor.beat!r}, which is no longer there") if (isinstance(start, AnchorRef) and isinstance(end, AnchorRef) and start.beat is not None and start.beat == end.beat and start.edge == "start" and end.edge == "end" @@ -194,16 +203,16 @@ def url(key: Optional[str]) -> Optional[str]: type=style.background.type, colors=list(style.background.colors), angle=style.background.angle, url=url(style.background.asset), ), - title_card_text=style.cards.title.model_copy(), - chapter_card_text=style.cards.chapter.model_copy(), - transition=style.transition.model_copy(), + title_card_text=CardTextSpec(**style.cards.title.model_dump()), + chapter_card_text=CardTextSpec(**style.cards.chapter.model_dump()), + transition=TransitionSpec(**style.transition.model_dump()), ken_burns=KenBurnsSpec(effect=style.motion.effect, amount=style.motion.amount, include_cards=style.cards.motion), quotes=QuoteSpec( enabled=any(isinstance(o, QuoteOverlay) for s in spec.segments for o in s.overlays), **style.quote.model_dump(), ), - code=style.code.model_copy(), + code=CodeSpec(**style.code.model_dump()), min_shot_seconds=style.min_segment_seconds, card_seconds=style.cards.seconds, voice=voice.voice, @@ -247,6 +256,10 @@ def _refuse_unsupported(spec: VideoSpec) -> None: if not isinstance(spec.assets[key], NotesAsset): problems.append(f"asset {key!r} is not a file in the note's media " f"({spec.assets[key].from_} assets need Gecko Video Creator)") + for where, key in (("style.background", spec.style.background.asset), + ("chrome.watermark", spec.chrome.watermark.asset if spec.chrome.watermark else None)): + if key is not None and spec.assets[key].type != "image": + problems.append(f"{where}: must be an image") if spec.voice.provider is not None or spec.voice.expressivity is not None: problems.append("voice.provider and voice.expressivity come from the account here") if spec.credits is not None: @@ -286,6 +299,8 @@ def _refuse_unsupported(spec: VideoSpec) -> None: problems.append(f"{where}: a clip carries no narration or overlays") if segment.kind == "clip" and visual is not None and spec.assets[visual.asset].type != "video": problems.append(f"{where}: a clip must be a video") + if segment.kind == "scene" and visual is not None and spec.assets[visual.asset].type == "audio": + problems.append(f"{where}: a scene's picture can't be an audio file") if segment.kind == "card" and segment.overlays: problems.append(f"{where}: overlays on a card") anchored = [] @@ -302,6 +317,15 @@ def _refuse_unsupported(spec: VideoSpec) -> None: if len(anchored) != len(set(anchored)): problems.append(f"{where}: two overlays on one beat") + # The renderer finds the intro and outro by position, to keep the music and + # its drawing off them, so that is the only place either can be. + roles = [s.role for s in spec.segments] + if "intro" in roles[1:] or roles.count("intro") > 1: + problems.append("an intro can only be the first segment") + if "outro" in roles[:-1] or roles.count("outro") > 1: + problems.append("an outro can only be the last segment") + + ids = {s.id for s in spec.segments} content = [s.id for s in spec.segments if s.role not in ("intro", "outro")] if len(spec.audio_tracks) > 1: problems.append("more than one audio track") @@ -313,6 +337,10 @@ def _refuse_unsupported(spec: VideoSpec) -> None: problems.append(f"{where}: trim / loop off") if spec.assets[track.asset].type != "audio": problems.append(f"{where}: must be an audio file") + for anchor in (track.start, track.end): + if isinstance(anchor, AnchorRef) and anchor.segment not in ids: + problems.append(f"{where}: anchored to segment {anchor.segment!r}, " + f"which is no longer there") if not content or not (_at(track.start, spec, content[0], "start") and track.end is not None and _at(track.end, spec, content[-1], "end")): diff --git a/backend/app/video/importer.py b/backend/app/video/importer.py index 4e99875..7966f9f 100644 --- a/backend/app/video/importer.py +++ b/backend/app/video/importer.py @@ -27,10 +27,10 @@ _media_kind, _media_url, _split_attribution, resolve_media_path, ) from app.video.spec import ( - AnchorRef, AudioTrack, Background, Beat, Card, Cards, Chrome, CodeOverlay, Motion, - Narration, NotesAsset, Output, OverlayText, Pauses, QuoteOverlay, QuoteStyle, - Segment, SegmentSource, Span, SpecSource, Style, VideoSpec, Visual, Voice, - Waveform, Watermark, + AnchorRef, AudioTrack, Background, Beat, Card, CardText, Cards, Chrome, CodeOverlay, + CodeStyle, Motion, Narration, NotesAsset, Output, OverlayText, Pauses, QuoteOverlay, + QuoteStyle, Segment, SegmentSource, Span, SpecSource, Style, Transition, VideoSpec, + Visual, Voice, Waveform, Watermark, ) @@ -80,17 +80,21 @@ def import_note( author: str = "", has_audio: Optional[Callable[[str], Any]] = None, note_id: Optional[str] = None, + imported_at: Optional[str] = None, + selection: bool = False, ) -> ImportResult: - """Same arguments as `segment()`, plus the note's id for provenance.""" + """Same arguments as `segment()`, plus provenance: the note's id, when it was + read, and whether `content` is only a selection of it.""" warnings: List[str] = [] - parts = _walk(content, user_id=user_id, media_dir=media_dir, options=options, - note_title=note_title, author=author, has_audio=has_audio, - warnings=warnings) + parts, order = _walk(content, user_id=user_id, media_dir=media_dir, options=options, + note_title=note_title, author=author, has_audio=has_audio, + warnings=warnings) assets = _Assets() - segments = _group(parts, assets, options.heading_pause_ms) + segments = _group(parts, assets, order) spec = VideoSpec( title=note_title, - source=SpecSource(note_id=note_id) if note_id else None, + source=(SpecSource(note_id=note_id, imported_at=imported_at, selection=selection) + if note_id or imported_at else None), output=Output( aspect=options.aspect, resolution=options.resolution, quality=options.quality, fps=options.fps, subtitles=options.subtitles, chapters=options.embed_chapters, @@ -133,7 +137,7 @@ def segment_via_spec( # ── the walk ────────────────────────── def _walk(content, *, user_id, media_dir, options, note_title, author, has_audio, - warnings) -> List[_Part]: + warnings) -> Tuple[List[_Part], Dict[str, int]]: parts: List[_Part] = [] heading_ms = options.heading_pause_ms @@ -153,6 +157,9 @@ def _walk(content, *, user_id, media_dir, options, note_title, author, has_audio )) pending: List[Beat] = [] + # Blocks read in this section that leave no beat (a heading not read + # aloud), recorded on whichever part the section's beats would go to. + pending_blocks: List[str] = [] open_part: Optional[_Part] = None pending_chapter: Optional[str] = None pending_card: Optional[_Part] = None @@ -161,11 +168,15 @@ def spoken() -> str: return narration_text(pending, card=False, heading_pause_ms=heading_ms) def flush(next_part: Optional[_Part]) -> None: - nonlocal pending, open_part, pending_chapter, pending_card + nonlocal pending, pending_blocks, open_part, pending_chapter, pending_card text = spoken() if open_part is None and text: open_part = _Part(kind="still", label="opening") + holder = open_part if open_part is not None else next_part + if holder is not None: + holder.block_ids.extend(pending_blocks) + pending_blocks = [] if open_part is not None: open_part.beats = pending if pending_chapter and not open_part.chapter: @@ -196,7 +207,9 @@ def carry(label: str, **extra: Any) -> _Part: return _Part(kind=kind, label=label, **extra) return _Part(kind=kind, media=open_part.media, path=open_part.path, label=label, **extra) - for block in _flatten(blocks): + flat = _flatten(blocks) + order = {b["id"]: i for i, b in enumerate(flat) if isinstance(b.get("id"), str)} + for block in flat: kind = _classify(block, options) block_id = block.get("id") if isinstance(block.get("id"), str) else None @@ -223,11 +236,12 @@ def carry(label: str, **extra: Any) -> _Part: quoted = _split_attribution(_inline_text(block.get("content"))) if quoted[0]: quote_part = carry("quote", overlay=QuoteOverlay( - id="", text=quoted[0], attribution=quoted[1] or None)) + id="", type="quote", text=quoted[0], attribution=quoted[1] or None)) after = carry("after quote") if spoken() or pending_chapter is not None or pending_card is not None: flush(quote_part) else: + _take_over(open_part, quote_part) open_part = quote_part pending.append(Beat(id="", kind="quote", text=_as_sentence(quoted[0]), block_id=block_id)) flush(after) @@ -236,11 +250,12 @@ def carry(label: str, **extra: Any) -> _Part: if btype == "codeBlock": code = _inline_text(block.get("content")) if code.strip(): - code_part = carry("code", overlay=CodeOverlay(id="", text=code)) + code_part = carry("code", overlay=CodeOverlay(id="", type="code", text=code)) after = carry("after code") if spoken() or pending_chapter is not None or pending_card is not None: flush(code_part) else: + _take_over(open_part, code_part) open_part = code_part # Kept even when it isn't read, so the panel has a beat to # hang on; `speak` decides whether it is heard. @@ -250,6 +265,8 @@ def carry(label: str, **extra: Any) -> _Part: continue if btype == "heading" and not options.read_chapters: + if block_id: + pending_blocks.append(block_id) continue text = _block_narration(block, options) @@ -297,6 +314,8 @@ def carry(label: str, **extra: Any) -> _Part: and part.chapter is None and part.kind == previous.kind and part.path == previous.path): + # Its blocks are still on screen, in the part it continued. + previous.block_ids.extend(part.block_ids) continue trimmed.append(part) parts = trimmed @@ -308,7 +327,14 @@ def carry(label: str, **extra: Any) -> _Part: parts.insert(0, intro) if outro is not None: parts.append(outro) - return parts + return parts, order + + +def _take_over(section: Optional[_Part], part: _Part) -> None: + """A quote or code block replacing a section that had nothing to say keeps + that section's picture, so it keeps the blocks the picture came from too.""" + if section is not None: + part.block_ids = section.block_ids + part.block_ids def _bumper(which, options, user_id, media_dir, warnings) -> Optional[_Part]: @@ -340,7 +366,7 @@ def key(self, url: Optional[str], media_type: str, label: Optional[str] = None) if found is None: found = f"a{len(self._keys) + 1}" self._keys[(url, media_type)] = found - self.registry[found] = NotesAsset(url=url, type=media_type, label=label or None) + self.registry[found] = NotesAsset(from_="notes", url=url, type=media_type, label=label or None) return found @@ -362,7 +388,7 @@ def _joins(segment_parts: List[_Part], part: _Part) -> bool: return last.overlay is not None and part.label == f"after {last.overlay.type}" -def _group(parts: List[_Part], assets: _Assets, heading_ms: int) -> List[Segment]: +def _group(parts: List[_Part], assets: _Assets, order: Dict[str, int]) -> List[Segment]: groups: List[List[_Part]] = [] for part in parts: if groups and _joins(groups[-1], part): @@ -390,7 +416,8 @@ def _group(parts: List[_Part], assets: _Assets, heading_ms: int) -> List[Segment if not part_beats and part.overlay is None and len(group) > 1: # A section with nothing to say, interrupted at once by a quote # or code block, is still its own moment on screen. - part_beats = [Beat(id="", text="", speak=False)] + part_beats = [Beat(id="", text="", speak=False, + block_id=part.block_ids[0] if part.block_ids else None)] for beat in part_beats: beats.append(beat.model_copy(update={"id": f"b{len(beats) + 1}"})) if part.overlay is not None: @@ -412,7 +439,8 @@ def _group(parts: List[_Part], assets: _Assets, heading_ms: int) -> List[Segment narration=Narration(beats=beats) if beats else None, overlays=overlays, chrome=False if first.role in ("intro", "outro") else None, - source=SegmentSource(block_ids=_dedupe(block_ids)) if block_ids else None, + source=(SegmentSource(block_ids=sorted(_dedupe(block_ids), key=order.__getitem__)) + if block_ids else None), )) return segments @@ -437,15 +465,15 @@ def _style(options: RenderOptions, assets: _Assets) -> Style: ), fit=options.fit, motion=Motion(effect=options.ken_burns.effect, amount=options.ken_burns.amount), - transition=options.transition.model_copy(), + transition=Transition(**options.transition.model_dump()), min_segment_seconds=options.min_shot_seconds, cards=Cards( seconds=options.card_seconds, motion=options.ken_burns.include_cards, - title=options.title_card_text.model_copy(), - chapter=options.chapter_card_text.model_copy(), + title=CardText(**options.title_card_text.model_dump()), + chapter=CardText(**options.chapter_card_text.model_dump()), ), quote=QuoteStyle(**options.quotes.model_dump(exclude={"enabled"})), - code=options.code.model_copy(), + code=CodeStyle(**options.code.model_dump()), ) diff --git a/backend/app/video/spec.py b/backend/app/video/spec.py index 55619e6..8524a9c 100644 --- a/backend/app/video/spec.py +++ b/backend/app/video/spec.py @@ -33,8 +33,9 @@ class _Model(BaseModel): # A spec is edited by hand and by an AI, so a misspelt field is an error # rather than something silently ignored. `from` and `in` are Python - # keywords, hence the aliases below and populate_by_name. - model_config = ConfigDict(extra="forbid", populate_by_name=True) + # keywords, hence the aliases below: accepted either way, always written + # as the spec spells them. + model_config = ConfigDict(extra="forbid", populate_by_name=True, serialize_by_alias=True) # ── assets ────────────────────────────────────────────────────────────────── @@ -45,7 +46,7 @@ class _Model(BaseModel): class NotesAsset(_Model): """A file in a note's own /media// directory.""" - from_: Literal["notes"] = Field("notes", alias="from") + from_: Literal["notes"] = Field(alias="from") url: str # How the note used it, which is not always what the extension says: a # `videoFile` block is a video whatever its URL ends in. @@ -56,7 +57,7 @@ class NotesAsset(_Model): class GamAsset(_Model): """An asset or clip in Gecko Asset Manager.""" - from_: Literal["gam"] = Field("gam", alias="from") + from_: Literal["gam"] = Field(alias="from") id: str type: Optional[MediaType] = None label: Optional[str] = None @@ -65,7 +66,7 @@ class GamAsset(_Model): class GeneratedAsset(_Model): """A placeholder for media still to be generated (through GAM).""" - from_: Literal["generate"] = Field("generate", alias="from") + from_: Literal["generate"] = Field(alias="from") type: Literal["image", "video", "music", "sfx"] prompt: str model: Optional[str] = None @@ -78,7 +79,7 @@ class GeneratedAsset(_Model): class SearchAsset(_Model): """A placeholder for footage still to be found in GAM.""" - from_: Literal["search"] = Field("search", alias="from") + from_: Literal["search"] = Field(alias="from") query: str type: Optional[MediaType] = None status: Literal["unresolved", "resolved"] = "unresolved" @@ -129,6 +130,19 @@ class VoiceOverride(_Model): expressivity: Optional[int] = None +class Transition(TransitionSpec): + # The option model's fields and clamps, but strict about unknown keys. + model_config = ConfigDict(extra="forbid") + + +class CardText(CardTextSpec): + model_config = ConfigDict(extra="forbid") + + +class CodeStyle(CodeSpec): + model_config = ConfigDict(extra="forbid") + + class Background(_Model): """Shown behind a segment that has no visual of its own.""" @@ -146,8 +160,8 @@ class Motion(_Model): class Cards(_Model): seconds: float = 3.5 motion: bool = False # Ken Burns on cards (zoom only) - title: CardTextSpec = Field(default_factory=CardTextSpec) - chapter: CardTextSpec = Field(default_factory=CardTextSpec) + title: CardText = Field(default_factory=CardText) + chapter: CardText = Field(default_factory=CardText) class QuoteStyle(_Model): @@ -162,11 +176,11 @@ class Style(_Model): background: Background = Field(default_factory=Background) fit: Fit = "blur" motion: Motion = Field(default_factory=Motion) - transition: TransitionSpec = Field(default_factory=TransitionSpec) + transition: Transition = Field(default_factory=Transition) min_segment_seconds: float = 2.5 cards: Cards = Field(default_factory=Cards) quote: QuoteStyle = Field(default_factory=QuoteStyle) - code: CodeSpec = Field(default_factory=CodeSpec) + code: CodeStyle = Field(default_factory=CodeStyle) class Watermark(_Model): @@ -268,19 +282,19 @@ class _OverlayBase(_Model): class QuoteOverlay(_OverlayBase): - type: Literal["quote"] = "quote" + type: Literal["quote"] text: str attribution: Optional[str] = None class CodeOverlay(_OverlayBase): - type: Literal["code"] = "code" + type: Literal["code"] text: str language: Optional[str] = None class TextOverlay(_OverlayBase): - type: Literal["text"] = "text" + type: Literal["text"] text: str position: Position = "center" size_pct: Optional[float] = None @@ -288,13 +302,13 @@ class TextOverlay(_OverlayBase): class LowerThirdOverlay(_OverlayBase): - type: Literal["lower_third"] = "lower_third" + type: Literal["lower_third"] title: str subtitle: Optional[str] = None class ImageOverlay(_OverlayBase): - type: Literal["image"] = "image" + type: Literal["image"] asset: str position: Position = "center" scale_pct: Optional[float] = None @@ -336,7 +350,7 @@ class Segment(_Model): card: Optional[Card] = None narration: Optional[Narration] = None overlays: List[Overlay] = Field(default_factory=list) - transition_in: Optional[TransitionSpec] = None + transition_in: Optional[Transition] = None duration: Optional[Duration] = None # None: the default, which is on except for an intro or outro. chrome: Optional[bool] = None @@ -371,7 +385,8 @@ class Credits(_Model): class SpecSource(_Model): note_id: Optional[str] = None note_version_id: Optional[str] = None - imported_at: Optional[str] = None + imported_at: Optional[str] = None # ISO 8601; re-sync compares the note against it + selection: bool = False # only part of the note was imported class VideoSpec(_Model): @@ -390,8 +405,10 @@ class VideoSpec(_Model): @model_validator(mode="after") def _well_formed(self) -> "VideoSpec": - """The structural rules (§5.7). Anchors are checked at compile time: - one left pointing at a removed segment is a warning, not an error.""" + """The structural rules (§5.7). Anchors are deliberately not checked + here: an edit may leave one pointing at a removed segment (§5.7), and + what that means is the renderer's call — the Notes compiler refuses + such a spec and says which anchor is orphaned.""" problems: List[str] = [] def unique(ids: List[str], what: str) -> None: diff --git a/backend/app/video/worker.py b/backend/app/video/worker.py index b44672e..c021ab8 100644 --- a/backend/app/video/worker.py +++ b/backend/app/video/worker.py @@ -60,26 +60,48 @@ def _media_dir() -> str: return resolved -def _tts_caller(user_id: str): +def _render_voice(session: Session, user_id: str, provider: str, chosen: Optional[str]) -> Optional[str]: + """The voice to read a render in: the one picked in the video dialog when + it is a voice of the engine doing the reading, otherwise the account's. + + The dialog lists the voices of whichever engine the render will use, but a + choice saved while the account used the other engine can still arrive here; + it falls back to the account's voice rather than to whatever the model + happens to list first. + """ + from app.routers.settings import ( + _DEEPGRAM_TTS_VOICE_IDS, get_voices_for_model, load_selected_voice, load_speech_config, + ) + + if chosen: + if provider == "deepgram" and chosen in _DEEPGRAM_TTS_VOICE_IDS: + return chosen + if provider == "fal": + cfg = load_speech_config(session, user_id) + if chosen in get_voices_for_model(session, cfg["tts_model"], cfg["custom_tts_models"]): + return chosen + # Read-aloud gets the user's chosen voice from the request body; a render + # has no request, so without this the fal.ai path fell back to the selected + # model's first listed voice for every render, silently ignoring whatever + # voice (e.g. a non-default one like "Rex") the user picked in Settings -> + # Speech. (Deepgram ignores a fal.ai name and reads its own setting.) + return load_selected_voice(session, user_id) + + +def _tts_caller(user_id: str, chosen_voice: Optional[str] = None): """A sync `text -> mp3 bytes` callable for the renderer. Wraps the same synthesis core the read-aloud endpoint uses, including its disk cache — which is what makes a preview render followed by a full render bill for the narration only once. Each call opens its own session because - this runs on a worker thread, not in a request. + this runs on a worker thread, not in a request. `chosen_voice` is the video + dialog's voice (`RenderOptions.voice`); None means the account's. """ from app.routers.settings import ( - load_deepgram_api_key, load_selected_voice, load_speech_config, synthesize_tts_bytes, + load_deepgram_api_key, load_speech_config, synthesize_tts_bytes, ) with Session(engine) as session: - # Read-aloud gets the user's chosen voice from the request body; a - # render has no request, so without this the fal.ai path fell back to - # the selected model's first listed voice for every render, silently - # ignoring whatever voice (e.g. a non-default one like "Rex") the user - # picked in Settings -> Speech. - voice = load_selected_voice(session, user_id) - # Resolve "auto" to a concrete provider once, up front, and pin it for # every chunk of this render. Left as "auto" per chunk, a transient # Deepgram hiccup partway through a narration would silently fall back @@ -90,6 +112,7 @@ def _tts_caller(user_id: str): tts_provider = load_speech_config(session, user_id)["tts_provider"] if tts_provider == "auto": tts_provider = "deepgram" if load_deepgram_api_key(session, user_id) else "fal" + voice = _render_voice(session, user_id, tts_provider, chosen_voice) def call(text: str) -> bytes: with Session(engine) as session: @@ -215,7 +238,7 @@ def raise_cancelled() -> None: author=author, options=options, preview=(job.quality == "preview"), - tts=_tts_caller(job.user_id), + tts=_tts_caller(job.user_id, options.voice), progress=progress, max_shots=MAX_SHOTS, max_narration_chars=MAX_NARRATION_CHARS, diff --git a/backend/tests/test_video_spec.py b/backend/tests/test_video_spec.py index 5334672..9074029 100644 --- a/backend/tests/test_video_spec.py +++ b/backend/tests/test_video_spec.py @@ -189,6 +189,8 @@ def test_unreadable_content_imports_as_an_empty_video(tmp_path): root = _media_root(tmp_path) for content in ("", "not json", "{}", "[1, null, \"x\"]"): _assert_same(content, user_id="u1", media_dir=root, options=RenderOptions()) + spec = import_note(content, user_id="u1", media_dir=root, options=RenderOptions()).spec + assert spec.segments == [] and spec.audio_tracks == [] # ── the options survive the trip ──────────────────────────────────────────── @@ -385,6 +387,29 @@ def test_a_quote_is_an_overlay_on_its_own_beat_not_a_segment(tmp_path): assert [b.block_id for b in scene.beats] == ["p1", "q1", "p2"] +def test_a_section_taken_over_by_a_quote_or_code_keeps_its_blocks(tmp_path): + # The quote or code block replaces a section that had nothing to say, but + # it is drawn over that section's picture, so the picture's block is still + # where the segment came from. Re-sync relies on every block being found. + root = _media_root(tmp_path) + text = lambda t: [{"type": "text", "text": t}] + note = json.dumps([ + {"id": "img-a", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "p1", "type": "paragraph", "content": text("Under a.")}, + {"id": "img-b", "type": "image", "props": {"url": "/media/u1/b.png"}}, + {"id": "q1", "type": "quote", "content": text("Salt is gold")}, + {"id": "p2", "type": "paragraph", "content": text("After.")}, + {"id": "img-c", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "c1", "type": "codeBlock", "content": text("print(1)")}, + {"id": "img-d", "type": "image", "props": {"url": "/media/u1/a.png"}}, + ]) + spec = import_note(note, user_id="u1", media_dir=root, + options=RenderOptions(title_card=False, quotes={"enabled": True})).spec + assert [s.source.block_ids for s in spec.segments] == [ + ["img-a", "p1"], ["img-b", "q1", "p2"], ["img-c", "c1", "img-d"], + ] + + def _spec(**changes): base = { "assets": {"a1": {"from": "notes", "url": "/media/u1/a.png", "type": "image"}, @@ -407,6 +432,27 @@ def _spec(**changes): ({"voice": {"provider": "fal"}}, "voice.provider"), ({"audio_tracks": [{"id": "t1", "asset": "a1", "start": "video_start", "end": "video_end"}]}, "audio file"), + ({"assets": {"a1": {"from": "notes", "url": "/media/u1/a.mp3", "type": "audio"}}}, + "can't be an audio file"), + ({"chrome": {"watermark": {"asset": "v1"}}, + "assets": {"a1": {"from": "notes", "url": "/media/u1/a.png", "type": "image"}, + "v1": {"from": "notes", "url": "/media/u1/v.mp4", "type": "video"}}}, + "watermark: must be an image"), + ({"assets": {"a1": {"from": "notes", "url": "/media/u1/a.png", "type": "image"}, + "v1": {"from": "notes", "url": "/media/u1/v.mp4", "type": "video"}}, + "segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "a1"}}, + {"id": "s2", "kind": "clip", "role": "intro", "visual": {"asset": "v1", "audio": "play"}}]}, + "intro can only be the first"), + ({"assets": {"a1": {"from": "notes", "url": "/media/u1/a.png", "type": "image"}, + "m1": {"from": "notes", "url": "/media/u1/m.mp3", "type": "audio"}}, + "audio_tracks": [{"id": "t1", "asset": "m1", "start": {"segment": "gone", "edge": "start"}, + "end": {"segment": "s1", "edge": "end"}}]}, + "segment 'gone', which is no longer there"), + ({"segments": [{"id": "s1", "kind": "scene", "narration": {"beats": [{"id": "b1", "text": "x"}]}, + "overlays": [{"id": "o1", "type": "quote", "text": "x", + "show": {"from": {"beat": "b9", "edge": "start"}, + "to": {"beat": "b9", "edge": "end"}}}]}]}, + "beat 'b9', which is no longer there"), ]) def test_what_the_notes_renderer_cannot_draw_is_refused(change, message): with pytest.raises(UnsupportedSpec, match=message): @@ -419,12 +465,70 @@ def test_what_the_notes_renderer_cannot_draw_is_refused(change, message): ({"segments": [{"id": "s1", "kind": "card"}]}, "card with no card"), ({"segments": [{"id": "s1", "kind": "clip"}]}, "clip with no visual"), ({"segments": [{"id": "s1", "kind": "scene", "colour": "red"}]}, "Extra inputs"), + # A misspelt key inside a nested setting is refused too, not quietly + # dropped in favour of the default. + ({"style": {"transition": {"styl": "dissolve"}}}, "Extra inputs"), + ({"style": {"cards": {"title": {"title_size": 9}}}}, "Extra inputs"), + ({"style": {"code": {"positon": "top"}}}, "Extra inputs"), + ({"segments": [{"id": "s1", "kind": "scene", "transition_in": {"duraton": 2}}]}, "Extra inputs"), + # The tag is what says which kind of asset or overlay this is. + ({"assets": {"a1": {"url": "/media/u1/a.png", "type": "image"}}}, "discriminator"), ]) def test_a_malformed_spec_is_rejected(change, message): with pytest.raises(ValidationError, match=message): _spec(**change) +def test_a_spec_dumps_in_its_own_spelling_and_compact_dumps_load_back(tmp_path): + root = _media_root(tmp_path) + note = json.dumps([ + {"id": "img", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "q1", "type": "quote", "content": [{"type": "text", "text": "Salt is gold"}]}, + ]) + spec = import_note(note, user_id="u1", media_dir=root, + options=RenderOptions(quotes={"enabled": True})).spec + assert '"from":"notes"' in spec.model_dump_json() and "from_" not in spec.model_dump_json() + for compact in (dict(exclude_defaults=True), dict(exclude_unset=True), dict(exclude_none=True)): + assert VideoSpec.model_validate(spec.model_dump(**compact)) == spec, compact + schema = VideoSpec.model_json_schema()["$defs"] + assert "from" in schema["NotesAsset"]["required"] and "type" in schema["QuoteOverlay"]["required"] + + +def test_a_heading_that_is_not_read_still_belongs_to_its_section(tmp_path): + root = _media_root(tmp_path) + text = lambda t: [{"type": "text", "text": t}] + note = json.dumps([ + {"id": "img", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "h1", "type": "heading", "content": text("The market")}, + {"id": "q1", "type": "quote", "content": text("Salt is gold")}, + {"id": "img2", "type": "image", "props": {"url": "/media/u1/b.png"}}, + {"id": "h2", "type": "heading", "content": text("Ruins")}, + {"id": "p1", "type": "paragraph", "content": text("Nothing left.")}, + ]) + spec = import_note(note, user_id="u1", media_dir=root, options=RenderOptions( + title_card=False, read_chapters=False, quotes={"enabled": True})).spec + first, second = spec.segments + assert first.source.block_ids == ["img", "h1", "q1"] + assert second.source.block_ids == ["img2", "h2", "p1"] + # The picture held in silence before the quote says which picture it is. + assert (first.beats[0].text, first.beats[0].speak, first.beats[0].block_id) == ("", False, "img") + + +def test_a_clip_whose_file_has_gone_is_skipped_not_left_without_a_picture(tmp_path): + root = _media_root(tmp_path) + note = json.dumps([ + {"id": "v", "type": "videoFile", "props": {"url": "/media/u1/loud.mp4"}}, + {"id": "p", "type": "paragraph", "content": [{"type": "text", "text": "After the clip."}]}, + ]) + spec = import_note(note, user_id="u1", media_dir=root, options=RenderOptions(title_card=False), + has_audio=lambda path: True).spec + assert [s.kind for s in spec.segments] == ["clip", "scene"] + os.remove(os.path.join(root, "u1", "loud.mp4")) + plan = compile_spec(spec, user_id="u1", media_dir=root).segmentation + assert [s.kind for s in plan.shots] == ["still"] + assert plan.warnings == ["Skipped missing media: /media/u1/loud.mp4"] + + # ── POST /api/video/spec ───────────────────────────────────────────────────── _SPEC_NOTE = [ @@ -458,6 +562,7 @@ def test_the_spec_route_returns_the_note_as_a_spec(notes): data = _spec_route(notes, options={"title_card": False, "quotes": {"enabled": True}}) spec = VideoSpec.model_validate(data.spec) assert spec.source.note_id == "mine" + assert spec.source.imported_at and not spec.source.selection assert spec.title == "Salt" [scene] = spec.segments assert [b.block_id for b in scene.beats] == ["p1", "q1"] @@ -470,6 +575,7 @@ def test_the_spec_route_shapes_the_spec_as_a_render_would(notes): selection = json.dumps(_SPEC_NOTE[2:]) data = _spec_route(notes, options={"title_card": True}, selected_content=selection) spec = VideoSpec.model_validate(data.spec) + assert spec.source.selection assert [s.kind for s in spec.segments] == ["card", "scene"] assert spec.segments[0].card.title == "Salt" # Quotes are off by default, so the quotation is read as plain prose. diff --git a/backend/tests/test_video_voice.py b/backend/tests/test_video_voice.py new file mode 100644 index 0000000..8d233d5 --- /dev/null +++ b/backend/tests/test_video_voice.py @@ -0,0 +1,109 @@ +"""The voice a video is read in: the one picked in the video dialog. + +`RenderOptions.voice` used to be accepted and ignored — every render was read in +the account's voice. Now the worker passes it on when it is a voice of the +engine doing the reading, and the speech core honours a Flux voice id for +Deepgram the way it already honoured a voice name for fal.ai. Providers are +faked; nothing here calls out. +""" + +import asyncio +import json + +import pytest +from sqlmodel import Session, SQLModel, create_engine + +from app.models import UserSetting +from app.routers import settings +from app.video import worker + +USER = "u1" + + +@pytest.fixture +def session(): + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}) + SQLModel.metadata.create_all(engine) + with Session(engine) as s: + yield s + + +def _set(session, key, value): + session.add(UserSetting(user_id=USER, key=key, value=json.dumps(value))) + session.commit() + + +@pytest.fixture +def account(session): + """An account whose read-aloud voice is "Rex" on fal.ai and Haley on Deepgram.""" + _set(session, "tts_model", "Rex") + _set(session, settings._SPEECH_CONFIG, {"deepgram_tts_model": "flux-haley-en"}) + return session + + +FAL_VOICE = settings.FAL_TTS_VOICES[1] + + +@pytest.mark.parametrize("provider, chosen, expected", [ + ("deepgram", "flux-kit-en", "flux-kit-en"), # a Flux voice, read by Deepgram + ("fal", FAL_VOICE, FAL_VOICE), # a voice of the fal.ai model + # A choice saved under the other engine, or none at all: the account's. + ("deepgram", FAL_VOICE, "Rex"), + ("fal", "flux-kit-en", "Rex"), + ("fal", "Nobody", "Rex"), + ("fal", None, "Rex"), + ("deepgram", None, "Rex"), +]) +def test_the_dialogs_voice_is_used_when_it_belongs_to_the_engine(account, provider, chosen, expected): + assert worker._render_voice(account, USER, provider, chosen) == expected + + +def _fake_providers(monkeypatch): + heard = [] + + async def deepgram(api_key, voice, text, expressivity, speed): + heard.append(("deepgram", voice)) + return b"mp3" + + async def fal(session, user_id, api_key, text, voice_hint=None): + heard.append(("fal", voice_hint)) + return b"mp3", "audio/mpeg" + + monkeypatch.setattr(settings, "_deepgram_tts", deepgram) + monkeypatch.setattr(settings, "_fal_tts", fal) + monkeypatch.setattr(settings, "load_deepgram_api_key", lambda s, u: "dg-key") + monkeypatch.setattr(settings, "load_fal_api_key", lambda s, u: "fal-key") + monkeypatch.setattr(settings, "_tts_cache_get", lambda key: None) + monkeypatch.setattr(settings, "_tts_cache_put", lambda key, data: None) + monkeypatch.setattr(settings, "_record_usage", lambda *a, **k: None) + return heard + + +@pytest.mark.parametrize("voice, expected", [ + ("flux-kit-en", "flux-kit-en"), # named by the video dialog + ("Rex", "flux-haley-en"), # read-aloud's fal.ai name: the account's Flux voice + (None, "flux-haley-en"), +]) +def test_deepgram_reads_a_named_flux_voice_and_otherwise_the_accounts(account, monkeypatch, voice, expected): + heard = _fake_providers(monkeypatch) + asyncio.run(settings.synthesize_tts_bytes(account, USER, "Hello.", voice=voice, + provider_override="deepgram")) + assert heard == [("deepgram", expected)] + + +@pytest.mark.parametrize("tts_provider, chosen, expected", [ + ("auto", "flux-kit-en", ("deepgram", "flux-kit-en")), # auto + a Deepgram key: Deepgram + ("auto", None, ("deepgram", "flux-haley-en")), + ("fal", FAL_VOICE, ("fal", FAL_VOICE)), + ("fal", "flux-kit-en", ("fal", "Rex")), # saved under Deepgram: the account's +]) +def test_a_render_is_read_in_the_dialogs_voice_on_the_engine_it_pins( + session, monkeypatch, tts_provider, chosen, expected): + _set(session, "tts_model", "Rex") + _set(session, settings._SPEECH_CONFIG, + {"tts_provider": tts_provider, "deepgram_tts_model": "flux-haley-en"}) + heard = _fake_providers(monkeypatch) + monkeypatch.setattr(worker, "engine", session.get_bind()) + + worker._tts_caller(USER, chosen)("A chunk of narration.") + assert heard == [expected] diff --git a/docs/gecko-video-creator.md b/docs/gecko-video-creator.md index cbd1f9f..2b25b21 100644 --- a/docs/gecko-video-creator.md +++ b/docs/gecko-video-creator.md @@ -142,7 +142,7 @@ narrated text **below** it, up to the next media block, is that shot's narration - **Pause markup** such as `[pause]`, `[pause:long]` and `[pause:1.5s]`, plus ellipses and blank lines, controls the gaps in the speech. -The `Shot` dataclass (`segmenter.py:38`) is the closest thing to a data model today: +The `Shot` dataclass (`segmenter.py`) was the closest thing to a data model: `kind`, `background`, `narration`, `card_title`/`card_subtitle`/`card_kind`, `chapter`, `quote_text`/`quote_attribution`, `code_text`, `bumper`. Everything else (motion, music, transitions, watermark, waveform, voice) is a **global** field on `RenderOptions`. @@ -159,11 +159,14 @@ transitions, watermark, waveform, voice) is a **global** field on `RenderOptions | Structure is **re-inferred every render** | Nothing to edit, version or hand to an AI. The only way to change the video is to change the note | | Media must be in the note's own `/media/` | No library search, no generation inside the video workflow | -Six existing defects should be fixed on the way in rather than carried over: +Seven existing defects should be fixed on the way in rather than carried over: -- **The Voice picker does nothing.** `VideoGenModal.tsx` offers one and writes - `options.voice`, but no backend code reads it. The voice always comes from the account - setting (`video/worker.py:81`). +- ~~**The Voice picker does nothing.**~~ **Fixed.** `VideoGenModal.tsx` offered one and + wrote `options.voice`, but no backend code read it, and it only ever listed fal.ai + voices, so on an account reading with Deepgram no choice could have mattered. Now the + dialog lists the voices of the engine that will read the video, the worker passes the + choice on (`video/worker.py:_render_voice`), and the speech core honours a Flux voice id + for Deepgram. A choice saved under the other engine falls back to the account's voice. - **A heading placed above its image** attaches to the *previous* shot, not the image (pinned by `backend/tests/test_video_segmenter.py`). That's correct for the "text below the picture" model, but surprising. @@ -174,12 +177,17 @@ Six existing defects should be fixed on the way in rather than carried over: scrim, watermark opacity and margin, overlay margin and shadow. - **A chapter mark can vanish.** With headings not read aloud, a heading followed by a picture and then straight away a quote loses its chapter: the mark is carried onto the - picture's shot, and the quote then takes that shot over (`segmenter.py:461`). + picture's shot, and the quote then takes that shot over (the takeover in `importer._walk`, as in the + original walk kept in `tests/segment_reference.py`). - **Pause markers leak into chapter marks.** `Shot.__post_init__` strips `[pause:…]` from drawn fields set when a shot is built, but a section's chapter is set afterwards - (`segmenter.py:365`, `:384`), so `The Road [pause:2s]` lands in the MP4's chapter list + (`compiler._segment_shots`, as in the original walk), so `The Road [pause:2s]` lands in the MP4's chapter list and the "title + chapter" overlay as typed. +- **A sounded clip leaves an empty, muted copy of itself behind** when nothing is said + under it (it ends the note, or another picture follows): a `scene` with no beats over + the same clip, held for `min_segment_seconds`. + Phase 0 reproduces all of these exactly, so its parity tests can hold the spec path to the original output. The spec path is now the only one, so each is a small follow-up fix, made in the importer and in `tests/segment_reference.py` together (or that oracle is @@ -242,7 +250,8 @@ interface VideoSpec { source?: { // where the script came from note_id: string; note_version_id?: string; // the NoteVersion the import read - imported_at: string; // ISO 8601, used for re-sync (§9.1) + imported_at?: string; // ISO 8601, used for re-sync (§9.1) + selection?: boolean; // only part of the note was imported }; output: Output; voice: Voice; @@ -303,7 +312,7 @@ interface Style { type Background = | { type: "gradient"; colors: string[]; angle: number } // ["#1e293b","#0f172a"], 135 | { type: "solid"; colors: [string] } - | { type: "asset"; asset: AssetKey }; + | { type: "image"; asset: AssetKey }; interface Motion { effect: "none" | "zoom_in" | "zoom_out" | "pan_left" | "pan_right" @@ -374,9 +383,12 @@ interface Beat { // one spoken unit, usually one block of t text: string; // supports [pause:…] markup and the ellipsis rules kind?: "prose" | "heading" | "quote" | "code"; // provenance; sets default pauses speak?: boolean; // false = timed but silent (e.g. code with narrate_code off) - hold?: number; // seconds a silent beat lasts + hold?: number; // seconds a silent beat lasts; unset = the minimum (§5.8) pause_after_ms?: number; // overrides the default pause after this beat + block_id?: string; // the note block it was read from, for re-sync } +// A silent beat with empty text holds the picture: the importer writes one where a +// picture shows for a moment before a quote or code block takes over its section. ``` **Why beats exist.** Today a quote has to become its own shot because the only unit of @@ -490,7 +502,8 @@ An asset that is used but has no attribution is flagged before render. before its `start`. These cases are **warnings** (the element is dropped), not errors, so a deleted segment never blocks a render. - Removing a segment that something is anchored to is allowed. The editor and the AI - are told which anchors it orphaned and offered a re-anchor. + are told which anchors it orphaned and offered a re-anchor. The Notes compiler, which + has no editor to ask, refuses such a spec and names the orphaned anchor. ### 5.8 Compiling: spec → timeline @@ -498,13 +511,18 @@ The renderer never reads anchors directly. A compile step resolves them: 1. **Narrate.** Synthesize every spoken beat (cached exactly as today, keyed on provider, voice and text). This yields each beat's duration. -2. **Size segments.** Use `duration.fixed`, or the sum of beats and pauses, raised to - `duration.min` / `min_segment_seconds` / `cards.seconds`. For a clip, use its trimmed - length. -3. **Lay out.** Place segments end to end, overlapping by the length of each blend - transition. -4. **Resolve anchors** into absolute seconds for every overlay span and audio track. -5. **Emit a timeline**: a flat, absolute-time list that the existing ffmpeg stages +2. **Cut scenes at their overlays.** A beat carrying a quote or code overlay is a + *unit* of its own, and the beats between overlays run together as one unit. This is + how Notes has always drawn a quote (as a screen of its own), and Phase 0's parity + tests showed it is what exact parity needs. A segment with no overlays is one unit. +3. **Size units.** Use `duration.fixed`, or the sum of the unit's beats and pauses plus + the segment-end pause, raised to `duration.min` / `min_segment_seconds` / + `cards.seconds`. A silent beat lasts its `hold`, or when it has none, the unit's + minimum. For a clip, use its trimmed length. +4. **Lay out.** Place units end to end, `style.transition` between each pair (a blend + overlaps them by its length), so a quote dips or blends in and out like any cut. +5. **Resolve anchors** into absolute seconds for every overlay span and audio track. +6. **Emit a timeline**: a flat, absolute-time list that the existing ffmpeg stages consume. In effect it's today's `Shot` list plus timed overlay and audio events. The timeline is derived output. It is cached for previews, but never stored as the @@ -777,7 +795,7 @@ spec for any note that renders exactly as Notes renders it now. | Text below a media block | that segment's `narration.beats`, one beat per block | | Text before the first media block ("opening" shot) | `scene` with no `visual` (uses `style.background`) | | Video block without audio | `scene`, `visual: { audio: "mute", loop: true }` | -| Video block with audio (`video_sound` + `video_muted` pair) | a `clip` segment, then a `scene` reusing the same asset muted and looped (only if text follows) | +| Video block with audio (`video_sound` + `video_muted` pair) | a `clip` segment, then a `scene` reusing the same asset muted and looped (even with nothing said under it, a quirk kept for parity, §3.2) | | Missing or remote media | `scene` with no visual, plus an import warning (same as today's warning) | | Heading, `chapter_screens` off | beat `kind: "heading"`; `chapter` set on the segment that carries the mark | | Heading, `chapter_screens` on | `card` segment (`variant: "chapter"`), then a continuation `scene` | @@ -802,15 +820,16 @@ spec for any note that renders exactly as Notes renders it now. | `selected_content` (render a selection) | an import option: import a block range | | `insert_into_note` | not part of the spec. It's an export action ("send back to the note") | -**The one visible difference to watch:** today a quote is its own shot, so a *blend* -transition plays into and out of it. As an overlay it simply appears. The overlay's -`fade` covers the common case. Phase 0's golden tests will show whether exact parity -needs more than that. +**A quote is an overlay, but still its own screen.** Today a quote is its own shot, so a +*blend* transition plays into and out of it. Phase 0 kept that exactly: compiling cuts +a scene at each overlaid beat (§5.8, step 2), so the quote is drawn, timed and +transitioned as before. GVC can later offer an overlay that just appears over a +continuing shot, with its own `fade`, as a separate choice. **Importer options** fix the surprises noted in §3.2 without breaking parity: -- `heading_attaches: "previous" | "next_media"`. The default is `"previous"` (today's - behaviour) in Phase 0. The recommended default in GVC is `"next_media"`, where a - heading above an image opens that image's segment. +- `heading_attaches: "previous" | "next_media"` (**planned**, not in Phase 0, which + always behaves as `"previous"`, today's behaviour). The recommended default in GVC is + `"next_media"`, where a heading above an image opens that image's segment. - `chapter_screens`, `read_chapters`, `quotes`, `narrate_code` stay as **import-time** choices, because they decide the *shape* of the spec. Once imported, the spec just says what it is, and any of them can be changed afterwards by editing. @@ -996,7 +1015,7 @@ bug fixes only, each logged so it can be ported. | Phase | What | Done when | |---|---|---| -| **0 — Spec inside Notes** | **In progress.** Done: the VideoSpec in Pydantic ([`spec.py`](../backend/app/video/spec.py)); an importer that walks a note as the segmenter always has ([`importer.py`](../backend/app/video/importer.py)); a compiler back to the `Shot`s and `RenderOptions` the renderer uses ([`compiler.py`](../backend/app/video/compiler.py)); and `segment()` now going through them, so every render is built from a spec. `POST /api/video/spec` returns that spec for a note (same body as `/estimate`). Still to do: the voice bug and the chapter-mark quirks (§3.2). | [`tests/test_video_spec.py`](../backend/tests/test_video_spec.py): every segmenter test case, a few thousand generated notes and the renderer's ffmpeg commands come out identical to the original walk, kept verbatim as an oracle ([`segment_reference.py`](../backend/tests/segment_reference.py)), with no real encodes | +| **0 — Spec inside Notes** | **In progress.** Done: the VideoSpec in Pydantic ([`spec.py`](../backend/app/video/spec.py)); an importer that walks a note as the segmenter always has ([`importer.py`](../backend/app/video/importer.py)); a compiler back to the `Shot`s and `RenderOptions` the renderer uses ([`compiler.py`](../backend/app/video/compiler.py)); and `segment()` now going through them, so every render is built from a spec. `POST /api/video/spec` returns that spec for a note (same body as `/estimate`). The dialog's Voice choice now reaches the render. Still to do: the chapter-mark quirks (§3.2). | [`tests/test_video_spec.py`](../backend/tests/test_video_spec.py): every segmenter test case, a few thousand generated notes and the renderer's ffmpeg commands come out identical to the original walk, kept verbatim as an oracle ([`segment_reference.py`](../backend/tests/segment_reference.py)), with no real encodes | | **1 — GVC renders what Notes renders** | New repo `davior/gvc` from GAM's skeleton. The engine copied from the Phase 0 merge, SSO, project storage and history, import from Notes over HTTP, speech through Notes, preview and full render, download. Output to GAM and back into the note follow once those are designed. | Any note renders in GVC the same as in Notes | | **2 — The new capabilities** | Timed overlays, multiple audio tracks, trims, per-segment voice/motion/transition, GAM search and picker, `generate` / `search` placeholders, credits | The §6 example renders | | **3 — AI editing** | The operations of §8.2, the suggestions tray, chat, staged review | "Make a video from this note" produces a reviewed, rendered video | @@ -1010,7 +1029,7 @@ already renders, it's wrong, and finding that out costs a branch, not a new app. ## 11. Key touchpoints (for whoever implements this) **In gecko-notes, lifted into GVC or replaced:** -- [`backend/app/video/segmenter.py`](../backend/app/video/segmenter.py): becomes the importer. `Shot`, `segment()`, `_split_attribution`, `_bumper_shot`. +- [`backend/app/video/segmenter.py`](../backend/app/video/segmenter.py): the `Shot` model and the block-reading helpers (`_split_attribution`, …); its walk is now [`importer.py`](../backend/app/video/importer.py) (`_walk`, `_bumper`). - [`backend/app/video/options.py`](../backend/app/video/options.py): `RenderOptions`, the source of every default in §5. - [`backend/app/video/renderer.py`](../backend/app/video/renderer.py): the pipeline; `build_timeline` is the seed of the compiler. - [`backend/app/video/narration.py`](../backend/app/video/narration.py), [`pause_markup.py`](../backend/app/video/pause_markup.py) and its twin [`frontend/src/utils/pauseMarkup.ts`](../frontend/src/utils/pauseMarkup.ts), sharing [`backend/tests/fixtures/pause_cases.json`](../backend/tests/fixtures/pause_cases.json). @@ -1064,6 +1083,19 @@ already renders, it's wrong, and finding that out costs a branch, not a new app. the note and the open editor reconciles it. From another app that needs a Notes endpoint that can't race the editor's autosave, or a different gesture: the note links to the GVC project, and the user inserts the video themselves. +12. **Where limits live (principle 9).** Today the spec stores what it is given and + `RenderOptions` clamps at compile time, so the schema carries no ranges a UI could + read. Before GVC builds sliders: move the ranges into the spec as `Field(ge=…, le=…)` + (refusing out-of-range values instead of clamping them), and generate the UI's limits + from that schema. +13. **Partial overrides.** `transition_in` and a visual's `motion` are whole objects, so + overriding one field restates the rest; `VoiceOverride` is already a true partial. + Decide one rule (whole-object or field-by-field inheritance) before the editor + exposes per-segment overrides. +14. **Addressing edits.** JSON Patch addresses `segments`, `beats`, `overlays` and + `audio_tracks` by list index, so a staged suggestion or a selective undo can hit + the wrong item after a reorder. The §8.2 operations address by id; store those, or + patches resolved by id, rather than raw index paths. --- diff --git a/frontend/src/components/VideoGenModal.tsx b/frontend/src/components/VideoGenModal.tsx index 2a36608..f369888 100644 --- a/frontend/src/components/VideoGenModal.tsx +++ b/frontend/src/components/VideoGenModal.tsx @@ -11,6 +11,7 @@ import { } from '@/api/videoGen' import { useSettingsStore } from '@/stores/settings' import { apiErrorMessage } from '@/utils/format' +import { renderVoices, type VoiceChoice } from '@/utils/renderVoices' const OPTIONS_KEY = 'gecko-video-gen-options' /** The account setting the options are saved under. Saved there rather than only @@ -192,7 +193,8 @@ export default function VideoGenModal({ noteId, noteTitle, diagramImages, select const [options, setOptions] = useState(loadStoredOptions) const [tab, setTab] = useState('format') const [estimate, setEstimate] = useState(null) - const [voices, setVoices] = useState([]) + const [voices, setVoices] = useState([]) + const [accountVoice, setAccountVoice] = useState(null) const [busy, setBusy] = useState<'preview' | 'full' | null>(null) const [uploading, setUploading] = useState(null) const [error, setError] = useState(null) @@ -241,7 +243,13 @@ export default function VideoGenModal({ noteId, noteTitle, diagramImages, select useEffect(() => { let cancelled = false settingsApi.getSpeechSettings() - .then((s) => { if (!cancelled) setVoices(s.voices || []) }) + .then((s) => { + if (cancelled) return + // The voices of the engine that will read this video, not always fal.ai's. + const listed = renderVoices(s, useSettingsStore.getState().voice) + setVoices(listed.choices) + setAccountVoice(listed.accountVoice) + }) .catch(() => { /* the account's default voice is used when this is unavailable */ }) return () => { cancelled = true } }, []) @@ -424,10 +432,13 @@ export default function VideoGenModal({ noteId, noteTitle, diagramImages, select
-
diff --git a/frontend/src/utils/renderVoices.test.ts b/frontend/src/utils/renderVoices.test.ts new file mode 100644 index 0000000..272506d --- /dev/null +++ b/frontend/src/utils/renderVoices.test.ts @@ -0,0 +1,65 @@ +/** + * The video dialog's voice list. + * + * It used to list the fal.ai voices whatever engine would read the video, so + * on an account with a Deepgram key (where "auto" reads with Deepgram) every + * choice in it was ignored. These pin which engine's voices are offered. + */ + +import { describe, expect, it } from 'vitest' + +import type { SpeechSettings } from '@/api/settings' +import { renderVoices } from './renderVoices' + +function speech(overrides: Partial): SpeechSettings { + return { + has_fal_key: true, + tts_models: [], + custom_tts_models: [], + tts_model: 'fal-ai/elevenlabs/tts/eleven-v3', + voices: ['Aria', 'Rex'], + default_voice: 'Aria', + stt_models: [], + stt_model: '', + has_deepgram_key: false, + stt_provider: 'auto', + deepgram_model: '', + deepgram_models: [], + tts_provider: 'auto', + deepgram_tts_model: 'flux-haley-en', + deepgram_tts_models: [ + { id: 'flux-haley-en', label: 'Haley — female, US (Flux)' }, + { id: 'flux-kit-en', label: 'Kit — male, British (Flux)' }, + ], + deepgram_tts_expressivity: 0, + deepgram_tts_speed: 1, + voice_mode_enabled: false, + ...overrides, + } as SpeechSettings +} + +describe('renderVoices', () => { + it('offers the Deepgram voices when auto will read with Deepgram', () => { + const { choices, accountVoice } = renderVoices(speech({ has_deepgram_key: true }), 'Rex') + expect(choices.map((c) => c.id)).toEqual(['flux-haley-en', 'flux-kit-en']) + expect(accountVoice).toBe('Haley — female, US (Flux)') + }) + + it('offers the Deepgram voices when Deepgram is chosen outright', () => { + const { choices } = renderVoices(speech({ tts_provider: 'deepgram' })) + expect(choices[1]).toEqual({ id: 'flux-kit-en', label: 'Kit — male, British (Flux)' }) + }) + + it('offers the fal.ai voices without a Deepgram key, or when fal.ai is chosen', () => { + for (const settings of [speech({}), speech({ tts_provider: 'fal', has_deepgram_key: true })]) { + const { choices, accountVoice } = renderVoices(settings, 'Rex') + expect(choices).toEqual([{ id: 'Aria', label: 'Aria' }, { id: 'Rex', label: 'Rex' }]) + expect(accountVoice).toBe('Rex') + } + }) + + it('names no account voice it cannot vouch for', () => { + expect(renderVoices(speech({}), 'Someone').accountVoice).toBeNull() + expect(renderVoices(speech({ has_deepgram_key: true, deepgram_tts_model: 'gone' })).accountVoice).toBeNull() + }) +}) diff --git a/frontend/src/utils/renderVoices.ts b/frontend/src/utils/renderVoices.ts new file mode 100644 index 0000000..59d877b --- /dev/null +++ b/frontend/src/utils/renderVoices.ts @@ -0,0 +1,33 @@ +import type { SpeechSettings } from '@/api/settings' + +export interface VoiceChoice { + id: string + label: string +} + +export interface RenderVoices { + /** The voices the render can be read in. */ + choices: VoiceChoice[] + /** The voice "Account default" stands for, when it is known. */ + accountVoice: string | null +} + +/** + * The voices a video can be read in: those of the engine that will read it. + * + * Mirrors the render worker: "auto" means Deepgram whenever a Deepgram key is + * set, and fal.ai otherwise. Listing the fal.ai voices regardless — as the + * dialog used to — offered a choice Deepgram never reads. `falVoice` is the + * account's fal.ai voice (the settings store's `voice`), which Deepgram ignores. + */ +export function renderVoices(settings: SpeechSettings, falVoice?: string | null): RenderVoices { + const deepgram = settings.tts_provider === 'deepgram' + || (settings.tts_provider === 'auto' && settings.has_deepgram_key) + if (deepgram) { + const choices = (settings.deepgram_tts_models || []).map((m) => ({ id: m.id, label: m.label })) + const account = choices.find((c) => c.id === settings.deepgram_tts_model) + return { choices, accountVoice: account ? account.label : null } + } + const choices = (settings.voices || []).map((v) => ({ id: v, label: v })) + return { choices, accountVoice: falVoice && settings.voices?.includes(falVoice) ? falVoice : null } +}