From 14a816681d5ed8c2b9f0ce210f4d0705913a1538 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 00:31:18 +0000 Subject: [PATCH] Add the VideoSpec, a note importer and a compiler back to shots Phase 0 of docs/gecko-video-creator.md: the structure the segmenter infers on every render becomes a document. spec.py defines it in Pydantic; importer.py walks a note exactly as segment() does but records beats, overlays and provenance; compiler.py turns a spec back into the Shot list and RenderOptions today's renderer consumes, refusing anything it cannot draw. segment() and the renderer are untouched. tests/test_video_spec.py holds the spec path to today's output: every segmenter test case, 3,000 generated notes and real render() runs (ffmpeg faked) produce identical shots, warnings and ffmpeg commands. The design doc's worked example is validated against the models. The doc also gets the corrections from the repo review: GAM has no origin_* columns yet, read-aloud export keeps some speech code in Notes, inserting a render back into a note needs designing, and where GVC's code lives is recorded. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PwaujUokXQAxnebBArpZuj --- backend/app/video/compiler.py | 333 +++++++++++++++++++++ backend/app/video/importer.py | 476 +++++++++++++++++++++++++++++++ backend/app/video/spec.py | 435 ++++++++++++++++++++++++++++ backend/tests/test_video_spec.py | 411 ++++++++++++++++++++++++++ docs/gecko-video-creator.md | 85 ++++-- 5 files changed, 1720 insertions(+), 20 deletions(-) create mode 100644 backend/app/video/compiler.py create mode 100644 backend/app/video/importer.py create mode 100644 backend/app/video/spec.py create mode 100644 backend/tests/test_video_spec.py diff --git a/backend/app/video/compiler.py b/backend/app/video/compiler.py new file mode 100644 index 0000000..a5932cb --- /dev/null +++ b/backend/app/video/compiler.py @@ -0,0 +1,333 @@ +"""VideoSpec -> the shots and options today's renderer consumes. + +The renderer still works on a flat list of `Shot`s and one `RenderOptions`; +this is the step that produces them from a spec, so the renderer itself needs no +change while the spec becomes the thing that describes a video. A segment is not +always one shot: a quote or a code block is an overlay on one beat of a scene, +and the renderer draws those as a shot of their own, so a scene is split there. + +Anything a spec can say that this renderer cannot do — GAM assets, timed +overlays, per-segment motion or voice, more than one music track — is refused +with `UnsupportedSpec` rather than quietly rendered differently. Those arrive +with Gecko Video Creator's own renderer (docs/gecko-video-creator.md §10). +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + +from app.video.options import ( + BackgroundSpec, BumperSpec, KenBurnsSpec, MusicSpec, OverlayTextSpec, QuoteSpec, + RenderOptions, WatermarkSpec, WaveformSpec, +) +from app.video.segmenter import ( + Segmentation, Shot, ShotKind, _set_apart, resolve_media_path, +) +from app.video.spec import ( + Anchor, AnchorRef, Beat, CodeOverlay, NotesAsset, QuoteOverlay, Segment, VideoSpec, +) + +# RenderOptions fields that decide what a note is *imported* as rather than how +# it renders: whether there is a title card, whether headings become chapter +# screens, and so on. A spec records the outcome — the cards, the beats, the +# bumper segments — so they cannot be read back out of one, and the renderer +# never looks at them. Compiled options leave them at their defaults. +IMPORT_ONLY_OPTIONS = frozenset({ + "title_card", "chapter_screens", "read_chapters", "narrate_code", + "insert_into_note", "diagram_images", "intro", "outro", +}) + + +class UnsupportedSpec(ValueError): + """The spec asks for something the Notes renderer cannot draw.""" + + +@dataclass +class Compiled: + segmentation: Segmentation + options: RenderOptions + # Shot index -> the id of the segment it came from, for diagnostics. + shot_segments: List[str] = field(default_factory=list) + + +def narration_text(beats: List[Beat], *, card: bool, heading_pause_ms: int) -> str: + """What a run of beats sounds like as one shot's narration. + + A heading inside a section is set apart with blank lines, which + `chunk_narration` turns into the heading pause; a card is a section + boundary of its own and needs no marking. + """ + pieces: List[str] = [] + for beat in beats: + if not beat.speak or not beat.text: + continue + text = beat.text + if beat.kind == "heading" and not card and heading_pause_ms > 0: + text = _set_apart(text) + pieces.append(text) + return "\n".join(pieces).strip() + + +def compile_spec(spec: VideoSpec, *, user_id: str, media_dir: str) -> Compiled: + _refuse_unsupported(spec) + warnings: List[str] = [] + shots: List[Shot] = [] + owners: List[str] = [] + paths: Dict[str, Optional[str]] = {} + + def resolve(key: str) -> Optional[str]: + if key not in paths: + asset = spec.assets[key] + paths[key] = resolve_media_path(asset.url, user_id, media_dir) + if paths[key] is None: + warnings.append(f"Skipped missing media: {asset.url[:120]}") + return paths[key] + + heading_ms = spec.voice.pauses.heading_ms + for segment in spec.segments: + for shot in _segment_shots(segment, spec, resolve, heading_ms): + shots.append(shot) + owners.append(segment.id) + + return Compiled( + segmentation=Segmentation(shots=shots, warnings=warnings), + options=_options(spec), + shot_segments=owners, + ) + + +def _segment_shots(segment: Segment, spec: VideoSpec, resolve, heading_ms: int) -> List[Shot]: + path = resolve(segment.visual.asset) if segment.visual is not None else None + + if segment.kind == "card": + return [Shot( + kind="card", + card_title=segment.card.title, + card_subtitle=segment.card.subtitle, + card_kind=segment.card.variant, + chapter=segment.chapter, + narration=narration_text(segment.beats, card=True, heading_pause_ms=heading_ms), + label=segment.label, + )] + + if segment.kind == "clip": + bumper = segment.role if segment.role in ("intro", "outro") else None + return [Shot(kind="video_sound", background=path, chapter=segment.chapter, + bumper=bumper, label=segment.label)] + + kind: ShotKind = "still" + if path is not None and spec.assets[segment.visual.asset].type == "video": + kind = "video_muted" + + # Split the beats where an overlay is: each overlaid beat is a shot of its + # own, and the beats between them run together as one. + overlay_at = {_overlay_beat(segment, o): o for o in segment.overlays} + runs: List[Tuple[Optional[object], List[Beat]]] = [] + for beat in segment.beats: + overlay = overlay_at.get(beat.id) + if overlay is not None: + runs.append((overlay, [beat])) + elif runs and runs[-1][0] is None: + runs[-1][1].append(beat) + else: + runs.append((None, [beat])) + if not runs: + runs.append((None, [])) + + shots: List[Shot] = [] + for index, (overlay, beats) in enumerate(runs): + if index == 0: + label = segment.label + elif overlay is not None: + label = overlay.type + else: + label = f"after {runs[index - 1][0].type}" + shot = Shot( + kind=kind, background=path, + narration=narration_text(beats, card=False, heading_pause_ms=heading_ms), + quote_text=overlay.text if isinstance(overlay, QuoteOverlay) else None, + quote_attribution=overlay.attribution if isinstance(overlay, QuoteOverlay) else None, + code_text=overlay.text if isinstance(overlay, CodeOverlay) else None, + label=label, + ) + if index == 0 and segment.chapter: + # Set after construction, as segment() does for every section, so + # the mark is exactly what it was — pause markers included, which + # nothing strips from a section's chapter today. + shot.chapter = segment.chapter + shots.append(shot) + return shots + + +def _overlay_beat(segment: Segment, overlay) -> str: + """The one beat an overlay spans. Anything else is a timed overlay.""" + show = overlay.show + start = show.from_ if show is not None else None + end = show.to if show is not None else None + if (isinstance(start, AnchorRef) and isinstance(end, AnchorRef) + and start.beat is not None and start.beat == end.beat + and start.edge == "start" and end.edge == "end" + and not start.offset and not end.offset + and start.segment in (None, segment.id) and end.segment in (None, segment.id) + and any(b.id == start.beat for b in segment.beats)): + return start.beat + raise UnsupportedSpec( + f"overlay {overlay.id!r} in segment {segment.id!r} must span exactly one of its beats" + ) + + +def _options(spec: VideoSpec) -> RenderOptions: + style, chrome, voice = spec.style, spec.chrome, spec.voice + + def url(key: Optional[str]) -> Optional[str]: + return spec.assets[key].url if key is not None else None + + options = RenderOptions( + aspect=spec.output.aspect, + resolution=spec.output.resolution, + quality=spec.output.quality, + fps=spec.output.fps, + subtitles=spec.output.subtitles, + embed_chapters=spec.output.chapters, + thumbnail=spec.output.thumbnail, + fit=style.fit, + fallback=BackgroundSpec( + type=style.background.type, colors=list(style.background.colors), + angle=style.background.angle, url=url(style.background.asset), + ), + title_card_text=style.cards.title.model_copy(), + chapter_card_text=style.cards.chapter.model_copy(), + transition=style.transition.model_copy(), + ken_burns=KenBurnsSpec(effect=style.motion.effect, amount=style.motion.amount, + include_cards=style.cards.motion), + quotes=QuoteSpec( + enabled=any(isinstance(o, QuoteOverlay) for s in spec.segments for o in s.overlays), + **style.quote.model_dump(), + ), + code=style.code.model_copy(), + min_shot_seconds=style.min_segment_seconds, + card_seconds=style.cards.seconds, + voice=voice.voice, + speed=voice.speed, + paragraph_pause_ms=voice.pauses.paragraph_ms, + heading_pause_ms=voice.pauses.heading_ms, + shot_end_pause_ms=voice.pauses.segment_end_ms, + ) + if chrome.watermark is not None: + fields = chrome.watermark.model_dump(exclude={"asset"}) + options.watermark = WatermarkSpec(enabled=True, url=url(chrome.watermark.asset), **fields) + if chrome.overlay_text is not None: + options.overlay_text = OverlayTextSpec(enabled=True, **chrome.overlay_text.model_dump()) + if chrome.waveform is not None: + options.waveform = WaveformSpec(enabled=True, **chrome.waveform.model_dump()) + for track in spec.audio_tracks: + options.music = MusicSpec( + enabled=True, url=url(track.asset), volume=track.volume, + duck=track.duck_under_narration is not False, + fade_in=track.fade_in, fade_out=track.fade_out, + ) + for segment in spec.segments: + if segment.role in ("intro", "outro"): + asset = spec.assets[segment.visual.asset] + setattr(options, segment.role, + BumperSpec(enabled=True, url=asset.url, name=asset.label or "")) + return options + + +# ── what the Notes renderer cannot do ─────────────────────────────────────── + +def _refuse_unsupported(spec: VideoSpec) -> None: + problems: List[str] = [] + used = {s.visual.asset for s in spec.segments if s.visual is not None} + used |= {t.asset for t in spec.audio_tracks} + for key in (spec.style.background.asset, + spec.chrome.watermark.asset if spec.chrome.watermark else None): + if key is not None: + used.add(key) + for key in sorted(used): + if not isinstance(spec.assets[key], NotesAsset): + problems.append(f"asset {key!r} is not a file in the note's media " + f"({spec.assets[key].from_} assets need Gecko Video Creator)") + if spec.voice.provider is not None or spec.voice.expressivity is not None: + problems.append("voice.provider and voice.expressivity come from the account here") + if spec.credits is not None: + problems.append("credits") + + for segment in spec.segments: + where = f"segment {segment.id!r}" + bumper = segment.role in ("intro", "outro") + if segment.transition_in is not None: + problems.append(f"{where}: per-segment transition_in") + if segment.duration is not None: + problems.append(f"{where}: duration overrides") + if segment.chrome is not None and segment.chrome != (not bumper): + problems.append(f"{where}: chrome is on for every segment but an intro or outro") + if segment.role == "title" and segment.kind != "card": + problems.append(f"{where}: only a card can be the title") + if bumper and segment.kind != "clip": + problems.append(f"{where}: an intro or outro must be a clip") + if segment.narration is not None and segment.narration.voice is not None: + problems.append(f"{where}: per-segment voice") + for beat in segment.beats: + if beat.hold is not None or beat.pause_after_ms is not None: + problems.append(f"{where}: beat {beat.id!r} hold / pause_after_ms") + visual = segment.visual + if visual is not None: + if visual.fit is not None or visual.motion is not None: + problems.append(f"{where}: per-segment fit or motion") + if visual.trim is not None: + problems.append(f"{where}: trimmed clips") + if not visual.loop: + problems.append(f"{where}: a video that holds its last frame") + if visual.audio != ("play" if segment.kind == "clip" else "mute"): + problems.append(f"{where}: a clip plays its sound, a scene's video is muted") + if segment.kind == "card" and visual is not None: + problems.append(f"{where}: a card draws its own background") + if segment.kind == "clip" and (segment.beats or segment.overlays): + problems.append(f"{where}: a clip carries no narration or overlays") + if segment.kind == "clip" and visual is not None and spec.assets[visual.asset].type != "video": + problems.append(f"{where}: a clip must be a video") + if segment.kind == "card" and segment.overlays: + problems.append(f"{where}: overlays on a card") + anchored = [] + for overlay in segment.overlays: + if not isinstance(overlay, (QuoteOverlay, CodeOverlay)): + problems.append(f"{where}: {overlay.type} overlays") + elif overlay.fade is not None: + problems.append(f"{where}: overlay fades") + else: + try: + anchored.append(_overlay_beat(segment, overlay)) + except UnsupportedSpec as exc: + problems.append(str(exc)) + if len(anchored) != len(set(anchored)): + problems.append(f"{where}: two overlays on one beat") + + content = [s.id for s in spec.segments if s.role not in ("intro", "outro")] + if len(spec.audio_tracks) > 1: + problems.append("more than one audio track") + for track in spec.audio_tracks: + where = f"audio track {track.id!r}" + if track.kind != "music": + problems.append(f"{where}: only a music bed") + if track.trim is not None or track.loop is False: + problems.append(f"{where}: trim / loop off") + if spec.assets[track.asset].type != "audio": + problems.append(f"{where}: must be an audio file") + if not content or not (_at(track.start, spec, content[0], "start") + and track.end is not None + and _at(track.end, spec, content[-1], "end")): + problems.append(f"{where}: the music bed runs from the first section " + f"to the last, around any intro and outro") + if problems: + raise UnsupportedSpec("Not supported by the Notes renderer: " + "; ".join(problems)) + + +def _at(anchor: Anchor, spec: VideoSpec, segment_id: str, edge: str) -> bool: + """Whether `anchor` is exactly the `edge` of segment `segment_id`.""" + if isinstance(anchor, str): + ids = [s.id for s in spec.segments] + index = 0 if anchor == "video_start" else len(ids) - 1 + wanted = "start" if anchor == "video_start" else "end" + return bool(ids) and ids[index] == segment_id and wanted == edge + return (anchor.segment == segment_id and anchor.beat is None + and anchor.edge == edge and not anchor.offset) diff --git a/backend/app/video/importer.py b/backend/app/video/importer.py new file mode 100644 index 0000000..1bafd45 --- /dev/null +++ b/backend/app/video/importer.py @@ -0,0 +1,476 @@ +"""A note (BlockNote JSON) -> a VideoSpec. + +The walk is `segmenter.segment()`'s, step for step, so that compiling what this +produces gives back exactly the shots `segment()` would have made — that +equivalence is what tests/test_video_spec.py holds it to, and what lets +`segment()` later become `compile_spec(import_note(...))`. What it records is +richer: the text under a picture is kept as one beat per block rather than one +joined string, a quote or code block is an overlay on its own beat rather than a +shot of its own, and every segment remembers which blocks it came from. + +Until the switch, the two walks must change together. Anything the import +decides — whether headings become chapter screens, whether quotes are drawn — +is settled here, from `RenderOptions`, and the spec only records the outcome. +""" + +import json +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple + +from app.video.compiler import compile_spec, narration_text +from app.video.options import RenderOptions +from app.video.pause_markup import strip_pause_markup +from app.video.segmenter import ( + Segmentation, ShotKind, _as_sentence, _block_narration, _classify, _flatten, _inline_text, + _media_kind, _media_url, _split_attribution, resolve_media_path, +) +from app.video.spec import ( + AnchorRef, AudioTrack, Background, Beat, Card, Cards, Chrome, CodeOverlay, Motion, + Narration, NotesAsset, Output, OverlayText, Pauses, QuoteOverlay, QuoteStyle, + Segment, SegmentSource, Span, SpecSource, Style, VideoSpec, Visual, Voice, + Waveform, Watermark, +) + + +@dataclass +class ImportResult: + spec: VideoSpec + # Media the note references that could not be used, in the order and + # wording segment() reports them. + warnings: List[str] = field(default_factory=list) + + +@dataclass +class _Part: + """One future shot, while the walk is still deciding what it holds.""" + + kind: ShotKind + media: Optional[Tuple[str, str]] = None # (url, "image" | "video") + media_label: Optional[str] = None # a readable name, when the file has one + path: Optional[str] = None # resolved, for segment()'s comparisons + label: str = "" + beats: List[Beat] = field(default_factory=list) + chapter: Optional[str] = None + card: Optional[Card] = None + role: Optional[str] = None + overlay: Optional[Any] = None # QuoteOverlay | CodeOverlay, not yet anchored + block_ids: List[str] = field(default_factory=list) + + def narration(self, heading_pause_ms: int) -> str: + return narration_text(self.beats, card=self.kind == "card", + heading_pause_ms=heading_pause_ms) + + def panel(self) -> bool: + """Whether segment()'s shot would carry quote_text or code_text.""" + if isinstance(self.overlay, CodeOverlay): + return True + if isinstance(self.overlay, QuoteOverlay): + return bool(strip_pause_markup(self.overlay.text)) + return False + + +def import_note( + content: str, + *, + user_id: str, + media_dir: str, + options: RenderOptions, + note_title: str = "", + author: str = "", + has_audio: Optional[Callable[[str], Any]] = None, + note_id: Optional[str] = None, +) -> ImportResult: + """Same arguments as `segment()`, plus the note's id for provenance.""" + warnings: List[str] = [] + parts = _walk(content, user_id=user_id, media_dir=media_dir, options=options, + note_title=note_title, author=author, has_audio=has_audio, + warnings=warnings) + assets = _Assets() + segments = _group(parts, assets, options.heading_pause_ms) + spec = VideoSpec( + title=note_title, + source=SpecSource(note_id=note_id) if note_id else None, + output=Output( + aspect=options.aspect, resolution=options.resolution, quality=options.quality, + fps=options.fps, subtitles=options.subtitles, chapters=options.embed_chapters, + thumbnail=options.thumbnail, + ), + voice=Voice( + voice=options.voice, speed=options.speed, + pauses=Pauses(paragraph_ms=options.paragraph_pause_ms, + heading_ms=options.heading_pause_ms, + segment_end_ms=options.shot_end_pause_ms), + ), + style=_style(options, assets), + chrome=_chrome(options, assets), + segments=segments, + audio_tracks=_music(options, segments, assets), + assets=assets.registry, + ) + return ImportResult(spec=spec, warnings=warnings) + + +def segment_via_spec( + content: str, + *, + user_id: str, + media_dir: str, + options: RenderOptions, + note_title: str = "", + author: str = "", + has_audio: Optional[Callable[[str], Any]] = None, +) -> Segmentation: + """`segment()` by way of a VideoSpec: same arguments, same shots and warnings.""" + imported = import_note(content, user_id=user_id, media_dir=media_dir, options=options, + note_title=note_title, author=author, has_audio=has_audio) + compiled = compile_spec(imported.spec, user_id=user_id, media_dir=media_dir) + return Segmentation(shots=compiled.segmentation.shots, + warnings=imported.warnings + compiled.segmentation.warnings) + + +# ── the walk (segmenter.segment(), recording beats) ────────────────────────── + +def _walk(content, *, user_id, media_dir, options, note_title, author, has_audio, + warnings) -> List[_Part]: + parts: List[_Part] = [] + heading_ms = options.heading_pause_ms + + try: + blocks = json.loads(content or "[]") + except (ValueError, TypeError): + blocks = [] + if not isinstance(blocks, list): + blocks = [] + + if options.title_card and (note_title or author): + parts.append(_Part( + kind="card", + card=Card(variant="title", title=note_title or "Untitled", subtitle=author), + chapter=note_title or None, + label="title card", + )) + + pending: List[Beat] = [] + open_part: Optional[_Part] = None + pending_chapter: Optional[str] = None + pending_card: Optional[_Part] = None + + def spoken() -> str: + return narration_text(pending, card=False, heading_pause_ms=heading_ms) + + def flush(next_part: Optional[_Part]) -> None: + nonlocal pending, open_part, pending_chapter, pending_card + text = spoken() + + if open_part is None and text: + open_part = _Part(kind="still", label="opening") + if open_part is not None: + open_part.beats = pending + if pending_chapter and not open_part.chapter: + open_part.chapter = pending_chapter + if pending_card is not None: + parts.append(pending_card) + pending_card = None + parts.append(open_part) + elif pending_card is not None: + parts.append(pending_card) + pending_card = None + + carried_chapter = None + if (pending_chapter and open_part is None + and next_part is not None and next_part.kind != "card"): + carried_chapter = pending_chapter + pending = [] + pending_chapter = None + open_part = next_part + if carried_chapter and open_part is not None and not open_part.chapter: + open_part.chapter = carried_chapter + + def carry(label: str, **extra: Any) -> _Part: + """A part over whatever the interrupted section was showing.""" + kind: ShotKind = ("video_muted" if open_part is not None and open_part.kind.startswith("video") + else "still") + if open_part is None: + return _Part(kind=kind, label=label, **extra) + return _Part(kind=kind, media=open_part.media, path=open_part.path, label=label, **extra) + + for block in _flatten(blocks): + kind = _classify(block, options) + block_id = block.get("id") if isinstance(block.get("id"), str) else None + + if kind is None: + btype = block.get("type") + if btype == "heading": + heading = _inline_text(block.get("content")).strip() + if heading: + if options.chapter_screens: + flush(carry("after chapter card")) + beats = ([Beat(id="", kind="heading", text=_as_sentence(heading), block_id=block_id)] + if options.read_chapters else []) + pending_card = _Part( + kind="card", card=Card(variant="chapter", title=heading), + chapter=heading, beats=beats, label="chapter card", + block_ids=[block_id] if block_id else [], + ) + pending_chapter = None + continue + if pending_chapter is None: + pending_chapter = heading + + if btype == "quote" and options.quotes.enabled: + quoted = _split_attribution(_inline_text(block.get("content"))) + if quoted[0]: + quote_part = carry("quote", overlay=QuoteOverlay( + id="", text=quoted[0], attribution=quoted[1] or None)) + after = carry("after quote") + if spoken() or pending_chapter is not None or pending_card is not None: + flush(quote_part) + else: + open_part = quote_part + pending.append(Beat(id="", kind="quote", text=_as_sentence(quoted[0]), block_id=block_id)) + flush(after) + continue + + if btype == "codeBlock": + code = _inline_text(block.get("content")) + if code.strip(): + code_part = carry("code", overlay=CodeOverlay(id="", text=code)) + after = carry("after code") + if spoken() or pending_chapter is not None or pending_card is not None: + flush(code_part) + else: + open_part = code_part + # Kept even when it isn't read, so the panel has a beat to + # hang on; `speak` decides whether it is heard. + pending.append(Beat(id="", kind="code", text=_as_sentence(code), + speak=options.narrate_code, block_id=block_id)) + flush(after) + continue + + if btype == "heading" and not options.read_chapters: + continue + + text = _block_narration(block, options) + if text.strip(): + pending.append(Beat(id="", kind="heading" if btype == "heading" else "prose", + text=_as_sentence(text), block_id=block_id)) + continue + + url = _media_url(block, options) + path = resolve_media_path(url, user_id, media_dir) if url else None + ids = [block_id] if block_id else [] + if path is None: + if url and not url.startswith("/media/"): + warnings.append(f"Skipped remote media: {url[:120]}") + elif url: + warnings.append(f"Skipped missing media: {url[:120]}") + flush(_Part(kind="still", label="fallback", block_ids=ids)) + continue + + if kind == "image": + flush(_Part(kind="still", media=(url, "image"), path=path, block_ids=ids)) + continue + + sounded = bool(has_audio(path)) if callable(has_audio) else False + if sounded: + flush(None) + parts.append(_Part(kind="video_sound", media=(url, "video"), path=path, + label="clip with audio", block_ids=ids)) + open_part = _Part(kind="video_muted", media=(url, "video"), path=path, + label="clip continued, muted") + else: + flush(_Part(kind="video_muted", media=(url, "video"), path=path, block_ids=ids)) + + flush(None) + + parts = [p for p in parts + if p.kind != "still" or p.path is not None or p.narration(heading_ms) + or isinstance(p.overlay, CodeOverlay)] + + trimmed: List[_Part] = [] + for part in parts: + previous = trimmed[-1] if trimmed else None + if (previous is not None and previous.panel() and not part.panel() + and not part.narration(heading_ms) + and part.chapter is None + and part.kind == previous.kind + and part.path == previous.path): + continue + trimmed.append(part) + parts = trimmed + + if parts: + intro = _bumper("intro", options, user_id, media_dir, warnings) + outro = _bumper("outro", options, user_id, media_dir, warnings) + if intro is not None: + parts.insert(0, intro) + if outro is not None: + parts.append(outro) + return parts + + +def _bumper(which, options, user_id, media_dir, warnings) -> Optional[_Part]: + bumper = options.intro if which == "intro" else options.outro + if not bumper.enabled or not bumper.url: + return None + path = resolve_media_path(bumper.url, user_id, media_dir) + if path is None or _media_kind(bumper.url) != "video": + warnings.append(f"The {which} was skipped: that clip could not be read.") + return None + return _Part(kind="video_sound", media=(bumper.url, "video"), path=path, + media_label=bumper.name or None, chapter=which.capitalize(), + role=which, label=which) + + +# ── parts -> segments ─────────────────────────────────────────────────────── + +class _Assets: + """The spec's asset registry: one key per distinct (url, type), in first use order.""" + + def __init__(self) -> None: + self.registry: Dict[str, NotesAsset] = {} + self._keys: Dict[Tuple[str, str], str] = {} + + def key(self, url: Optional[str], media_type: str, label: Optional[str] = None) -> Optional[str]: + if not url: + return None + found = self._keys.get((url, media_type)) + if found is None: + found = f"a{len(self._keys) + 1}" + self._keys[(url, media_type)] = found + self.registry[found] = NotesAsset(url=url, type=media_type, label=label or None) + return found + + +def _joins(segment_parts: List[_Part], part: _Part) -> bool: + """Whether `part` is the next shot of the scene `segment_parts` began. + + A quote or code shot interrupts a section over the same picture, and the + section then resumes over it — that resumption is the only kind of plain + shot that continues a scene. Anything carrying its own chapter mark starts + a segment of its own, since a segment's chapter is marked at its start. + """ + first, last = segment_parts[0], segment_parts[-1] + if first.kind not in ("still", "video_muted") or part.kind != first.kind: + return False + if part.path != first.path or part.chapter is not None or part.role or part.card: + return False + if part.overlay is not None: + return True + return last.overlay is not None and part.label == f"after {last.overlay.type}" + + +def _group(parts: List[_Part], assets: _Assets, heading_ms: int) -> List[Segment]: + groups: List[List[_Part]] = [] + for part in parts: + if groups and _joins(groups[-1], part): + groups[-1].append(part) + else: + groups.append([part]) + + segments: List[Segment] = [] + for index, group in enumerate(groups, start=1): + first = group[0] + segment_kind = {"card": "card", "video_sound": "clip"}.get(first.kind, "scene") + visual = None + if first.media is not None and first.path is not None: + url, media_type = first.media + visual = Visual(asset=assets.key(url, media_type, first.media_label), + audio="play" if segment_kind == "clip" else "mute") + + beats: List[Beat] = [] + overlays = [] + block_ids: List[str] = [] + for part in group: + block_ids.extend(part.block_ids) + block_ids.extend(b.block_id for b in part.beats if b.block_id) + part_beats = list(part.beats) + if not part_beats and part.overlay is None and len(group) > 1: + # A section with nothing to say, interrupted at once by a quote + # or code block, is still its own moment on screen. + part_beats = [Beat(id="", text="", speak=False)] + for beat in part_beats: + beats.append(beat.model_copy(update={"id": f"b{len(beats) + 1}"})) + if part.overlay is not None: + beat_id = beats[-1].id + overlays.append(part.overlay.model_copy(update={ + "id": f"o{len(overlays) + 1}", + "show": Span(from_=AnchorRef(beat=beat_id, edge="start"), + to=AnchorRef(beat=beat_id, edge="end")), + })) + + segments.append(Segment( + id=f"s{index}", + kind=segment_kind, + role=first.role or ("title" if first.card is not None and first.card.variant == "title" else None), + label=first.label, + chapter=first.chapter, + visual=visual, + card=first.card, + narration=Narration(beats=beats) if beats else None, + overlays=overlays, + chrome=False if first.role in ("intro", "outro") else None, + source=SegmentSource(block_ids=_dedupe(block_ids)) if block_ids else None, + )) + return segments + + +def _dedupe(values: List[str]) -> List[str]: + seen, out = set(), [] + for value in values: + if value not in seen: + seen.add(value) + out.append(value) + return out + + +# ── settings ──────────────────────────────────────────────────────────────── + +def _style(options: RenderOptions, assets: _Assets) -> Style: + fallback = options.fallback + return Style( + background=Background( + type=fallback.type, colors=list(fallback.colors), angle=fallback.angle, + asset=assets.key(fallback.url, "image"), + ), + fit=options.fit, + motion=Motion(effect=options.ken_burns.effect, amount=options.ken_burns.amount), + transition=options.transition.model_copy(), + min_segment_seconds=options.min_shot_seconds, + cards=Cards( + seconds=options.card_seconds, motion=options.ken_burns.include_cards, + title=options.title_card_text.model_copy(), + chapter=options.chapter_card_text.model_copy(), + ), + quote=QuoteStyle(**options.quotes.model_dump(exclude={"enabled"})), + code=options.code.model_copy(), + ) + + +def _chrome(options: RenderOptions, assets: _Assets) -> Chrome: + chrome = Chrome() + if options.watermark.enabled: + chrome.watermark = Watermark( + asset=assets.key(options.watermark.url, "image"), + **options.watermark.model_dump(exclude={"enabled", "url"}), + ) + if options.overlay_text.enabled: + chrome.overlay_text = OverlayText(**options.overlay_text.model_dump(exclude={"enabled"})) + if options.waveform.enabled: + chrome.waveform = Waveform(**options.waveform.model_dump(exclude={"enabled"})) + return chrome + + +def _music(options: RenderOptions, segments: List[Segment], assets: _Assets) -> List[AudioTrack]: + """Today's one bed: under everything but the intro and outro.""" + music = options.music + content = [s for s in segments if s.role not in ("intro", "outro")] + if not music.enabled or not music.url or not content: + return [] + return [AudioTrack( + id="m1", kind="music", asset=assets.key(music.url, "audio"), + start=AnchorRef(segment=content[0].id, edge="start"), + end=AnchorRef(segment=content[-1].id, edge="end"), + loop=True, volume=music.volume, duck_under_narration=music.duck, + fade_in=music.fade_in, fade_out=music.fade_out, + )] diff --git a/backend/app/video/spec.py b/backend/app/video/spec.py new file mode 100644 index 0000000..55619e6 --- /dev/null +++ b/backend/app/video/spec.py @@ -0,0 +1,435 @@ +"""The VideoSpec: a video described as data rather than inferred from a note. + +`segmenter` works a video out of a note's block layout every time it renders — +an image starts a shot, the text under it is read over it, a quote becomes its +own shot — and nothing of that structure exists between renders. This module is +that structure made explicit: ordered segments, the narration beats inside them, +overlays anchored to beats, audio tracks anchored to segments, and a registry of +the assets they all point at. It is the document Gecko Video Creator edits; see +docs/gecko-video-creator.md §5 for the design and the reasoning behind it. + +Phase 0 of that plan: `importer` turns a note into one of these, `compiler` +turns one back into the shots and options today's renderer consumes, and +tests/test_video_spec.py proves the round trip changes nothing. + +Pure data. Nothing here touches the filesystem, the database or a provider. +Limits are not repeated here: a spec reaches the renderer through +`RenderOptions`, whose validators clamp exactly as they always have. +""" + +from typing import Annotated, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from app.video.options import ( + Aspect, CardTextSpec, CodeSpec, Fit, KenBurnsEffect, OverlayTextMode, Position, + Quality, QuotePosition, Resolution, SubtitleMode, TransitionSpec, WaveMode, + WavePosition, +) + +SPEC_VERSION = "0.1" + + +class _Model(BaseModel): + # A spec is edited by hand and by an AI, so a misspelt field is an error + # rather than something silently ignored. `from` and `in` are Python + # keywords, hence the aliases below and populate_by_name. + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + +# ── assets ────────────────────────────────────────────────────────────────── + +MediaType = Literal["image", "video", "audio"] + + +class NotesAsset(_Model): + """A file in a note's own /media// directory.""" + + from_: Literal["notes"] = Field("notes", alias="from") + url: str + # How the note used it, which is not always what the extension says: a + # `videoFile` block is a video whatever its URL ends in. + type: MediaType + label: Optional[str] = None + + +class GamAsset(_Model): + """An asset or clip in Gecko Asset Manager.""" + + from_: Literal["gam"] = Field("gam", alias="from") + id: str + type: Optional[MediaType] = None + label: Optional[str] = None + + +class GeneratedAsset(_Model): + """A placeholder for media still to be generated (through GAM).""" + + from_: Literal["generate"] = Field("generate", alias="from") + type: Literal["image", "video", "music", "sfx"] + prompt: str + model: Optional[str] = None + base_assets: List[str] = Field(default_factory=list) + status: Literal["pending", "running", "ready", "failed"] = "pending" + result: Optional[GamAsset] = None + label: Optional[str] = None + + +class SearchAsset(_Model): + """A placeholder for footage still to be found in GAM.""" + + from_: Literal["search"] = Field("search", alias="from") + query: str + type: Optional[MediaType] = None + status: Literal["unresolved", "resolved"] = "unresolved" + result: Optional[GamAsset] = None + label: Optional[str] = None + + +AssetRef = Annotated[ + Union[NotesAsset, GamAsset, GeneratedAsset, SearchAsset], + Field(discriminator="from_"), +] + + +# ── project-wide settings ─────────────────────────────────────────────────── + +class Output(_Model): + aspect: Aspect = "16:9" + resolution: Resolution = "1080p" + quality: Quality = "standard" + fps: int = 30 + subtitles: SubtitleMode = "sidecar" + chapters: bool = True # embed chapter marks in the file + thumbnail: bool = True + + +class Pauses(_Model): + paragraph_ms: int = 350 + heading_ms: int = 800 + segment_end_ms: int = 600 + + +class Voice(_Model): + # None means "whatever the account is set to", which is what Notes renders + # with today; naming one pins it for the whole video. + provider: Optional[Literal["auto", "deepgram", "fal"]] = None + voice: Optional[str] = None + speed: float = 1.0 + expressivity: Optional[int] = None # Deepgram Flux only, -2…2 + pauses: Pauses = Field(default_factory=Pauses) + + +class VoiceOverride(_Model): + """Per-segment voice. Every field left unset inherits from `Voice`.""" + + provider: Optional[Literal["auto", "deepgram", "fal"]] = None + voice: Optional[str] = None + speed: Optional[float] = None + expressivity: Optional[int] = None + + +class Background(_Model): + """Shown behind a segment that has no visual of its own.""" + + type: Literal["gradient", "solid", "image"] = "gradient" + colors: List[str] = Field(default_factory=lambda: ["#1e293b", "#0f172a"]) + angle: int = 135 + asset: Optional[str] = None # used when type == "image" + + +class Motion(_Model): + effect: KenBurnsEffect = "none" # "alternate" only makes sense project-wide + amount: float = 0.12 + + +class Cards(_Model): + seconds: float = 3.5 + motion: bool = False # Ken Burns on cards (zoom only) + title: CardTextSpec = Field(default_factory=CardTextSpec) + chapter: CardTextSpec = Field(default_factory=CardTextSpec) + + +class QuoteStyle(_Model): + position: QuotePosition = "center" + size_pct: float = 4.2 + color: str = "#ffffff" + accent: str = "#818cf8" + scrim: float = 0.55 + + +class Style(_Model): + background: Background = Field(default_factory=Background) + fit: Fit = "blur" + motion: Motion = Field(default_factory=Motion) + transition: TransitionSpec = Field(default_factory=TransitionSpec) + min_segment_seconds: float = 2.5 + cards: Cards = Field(default_factory=Cards) + quote: QuoteStyle = Field(default_factory=QuoteStyle) + code: CodeSpec = Field(default_factory=CodeSpec) + + +class Watermark(_Model): + asset: Optional[str] = None + text: str = "" # empty: "by {title}" + position: Position = "bottom-right" + opacity: float = 0.85 + scale_pct: float = 6.0 + caption_pct: float = 2.3 + margin_pct: int = 4 + + +class OverlayText(_Model): + mode: OverlayTextMode = "fixed" + text: str = "" + position: Position = "bottom-left" + color: str = "#ffffff" + size_pct: float = 3.0 + margin_pct: int = 5 + shadow: bool = True + + +class Waveform(_Model): + mode: WaveMode = "line" + color: str = "#00ff41" + opacity: float = 0.7 + position: WavePosition = "bottom" + height_pct: int = 22 + scrim: float = 0.45 + + +class Chrome(_Model): + """Drawn over every segment that doesn't opt out. Present means on.""" + + watermark: Optional[Watermark] = None + overlay_text: Optional[OverlayText] = None + waveform: Optional[Waveform] = None + + +# ── time ──────────────────────────────────────────────────────────────────── + +class AnchorRef(_Model): + """A moment defined by the structure rather than by a number of seconds.""" + + segment: Optional[str] = None # inside a segment's overlay: that segment + beat: Optional[str] = None + edge: Literal["start", "end"] + offset: float = 0.0 # seconds; negative is before the edge + + +Anchor = Union[Literal["video_start", "video_end"], AnchorRef] + + +class Span(_Model): + from_: Anchor = Field(alias="from") + to: Anchor + + +class Trim(_Model): + in_: float = Field(alias="in") + out: Optional[float] = None + + +# ── segments ──────────────────────────────────────────────────────────────── + +class Visual(_Model): + asset: str + fit: Optional[Fit] = None + motion: Optional[Motion] = None + trim: Optional[Trim] = None + audio: Literal["mute", "play"] = "mute" + loop: bool = True + + +class Beat(_Model): + """One spoken unit — usually one block of the note.""" + + id: str + text: str # [pause:…] markup is kept and honoured + kind: Literal["prose", "heading", "quote", "code"] = "prose" + # False: the beat takes up time without being read — a code block that + # isn't narrated, or an empty `text` that just holds the picture. + speak: bool = True + hold: Optional[float] = None + pause_after_ms: Optional[int] = None + # The note block it was read from, so a re-sync can match it up again. + block_id: Optional[str] = None + + +class Narration(_Model): + beats: List[Beat] = Field(default_factory=list) + voice: Optional[VoiceOverride] = None + + +class _OverlayBase(_Model): + id: str + show: Optional[Span] = None # default: the whole segment + fade: Optional[float] = None + + +class QuoteOverlay(_OverlayBase): + type: Literal["quote"] = "quote" + text: str + attribution: Optional[str] = None + + +class CodeOverlay(_OverlayBase): + type: Literal["code"] = "code" + text: str + language: Optional[str] = None + + +class TextOverlay(_OverlayBase): + type: Literal["text"] = "text" + text: str + position: Position = "center" + size_pct: Optional[float] = None + color: Optional[str] = None + + +class LowerThirdOverlay(_OverlayBase): + type: Literal["lower_third"] = "lower_third" + title: str + subtitle: Optional[str] = None + + +class ImageOverlay(_OverlayBase): + type: Literal["image"] = "image" + asset: str + position: Position = "center" + scale_pct: Optional[float] = None + opacity: Optional[float] = None + + +Overlay = Annotated[ + Union[QuoteOverlay, CodeOverlay, TextOverlay, LowerThirdOverlay, ImageOverlay], + Field(discriminator="type"), +] + + +class Card(_Model): + variant: Literal["title", "chapter"] + title: str + subtitle: Optional[str] = None + + +class Duration(_Model): + min: Optional[float] = None + fixed: Optional[float] = None + + +class SegmentSource(_Model): + """The note blocks a segment was imported from, for re-syncing later.""" + + block_ids: List[str] = Field(default_factory=list) + + +class Segment(_Model): + id: str + # scene: a visual with narration over it. card: a title or chapter screen. + # clip: a video played with its own sound and no narration. + kind: Literal["scene", "card", "clip"] + role: Optional[Literal["title", "intro", "outro"]] = None + label: str = "" + chapter: Optional[str] = None # a chapter mark at this segment's start + visual: Optional[Visual] = None + card: Optional[Card] = None + narration: Optional[Narration] = None + overlays: List[Overlay] = Field(default_factory=list) + transition_in: Optional[TransitionSpec] = None + duration: Optional[Duration] = None + # None: the default, which is on except for an intro or outro. + chrome: Optional[bool] = None + source: Optional[SegmentSource] = None + + @property + def beats(self) -> List[Beat]: + return self.narration.beats if self.narration is not None else [] + + +class AudioTrack(_Model): + id: str + kind: Literal["music", "sfx", "voiceover"] = "music" + asset: str + start: Anchor + end: Optional[Anchor] = None # None: play once, to its natural end + trim: Optional[Trim] = None + loop: Optional[bool] = None # default: on for music + volume: float = 0.18 + duck_under_narration: Optional[bool] = None + fade_in: float = 0.0 + fade_out: float = 0.0 + + +class Credits(_Model): + auto: bool = True + card: bool = True + description: bool = True + extra: List[str] = Field(default_factory=list) + + +class SpecSource(_Model): + note_id: Optional[str] = None + note_version_id: Optional[str] = None + imported_at: Optional[str] = None + + +class VideoSpec(_Model): + spec_version: Literal["0.1"] = SPEC_VERSION + id: Optional[str] = None + title: str = "" + source: Optional[SpecSource] = None + output: Output = Field(default_factory=Output) + voice: Voice = Field(default_factory=Voice) + style: Style = Field(default_factory=Style) + chrome: Chrome = Field(default_factory=Chrome) + assets: Dict[str, AssetRef] = Field(default_factory=dict) + segments: List[Segment] = Field(default_factory=list) + audio_tracks: List[AudioTrack] = Field(default_factory=list) + credits: Optional[Credits] = None + + @model_validator(mode="after") + def _well_formed(self) -> "VideoSpec": + """The structural rules (§5.7). Anchors are checked at compile time: + one left pointing at a removed segment is a warning, not an error.""" + problems: List[str] = [] + + def unique(ids: List[str], what: str) -> None: + seen = set() + for value in ids: + if value in seen: + problems.append(f"duplicate {what} id {value!r}") + seen.add(value) + + def known(key: Optional[str], where: str) -> None: + if key is not None and key not in self.assets: + problems.append(f"{where} refers to unknown asset {key!r}") + + unique([s.id for s in self.segments], "segment") + unique([t.id for t in self.audio_tracks], "audio track") + known(self.style.background.asset, "style.background") + if self.chrome.watermark is not None: + known(self.chrome.watermark.asset, "chrome.watermark") + for key, asset in self.assets.items(): + for base in getattr(asset, "base_assets", []): + known(base, f"asset {key!r}") + for segment in self.segments: + where = f"segment {segment.id!r}" + unique([b.id for b in segment.beats], f"{where} beat") + unique([o.id for o in segment.overlays], f"{where} overlay") + if segment.visual is not None: + known(segment.visual.asset, where) + for overlay in segment.overlays: + if isinstance(overlay, ImageOverlay): + known(overlay.asset, f"{where} overlay {overlay.id!r}") + if segment.kind == "card" and segment.card is None: + problems.append(f"{where} is a card with no card") + if segment.kind != "card" and segment.card is not None: + problems.append(f"{where} has a card but is a {segment.kind}") + if segment.kind == "clip" and segment.visual is None: + problems.append(f"{where} is a clip with no visual") + for track in self.audio_tracks: + known(track.asset, f"audio track {track.id!r}") + if problems: + raise ValueError("; ".join(problems)) + return self diff --git a/backend/tests/test_video_spec.py b/backend/tests/test_video_spec.py new file mode 100644 index 0000000..79317de --- /dev/null +++ b/backend/tests/test_video_spec.py @@ -0,0 +1,411 @@ +"""The VideoSpec round trip: a note imported as a spec and compiled back must be +exactly what `segment()` makes of it — the same shots, field for field, and the +same warnings — and must drive the renderer to the same ffmpeg commands. + +That equivalence is Phase 0 of docs/gecko-video-creator.md: it proves the spec +can describe everything Notes already renders before anything is built on it. +Nothing here encodes video; ffmpeg is faked as in test_video_render_resume. +""" + +import inspect +import json +import os +import random +import re +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import test_video_segmenter as segmenter_cases +from test_video_render_resume import FakeFFmpeg + +from app.video import renderer +from app.video.compiler import IMPORT_ONLY_OPTIONS, UnsupportedSpec, compile_spec +from app.video.importer import import_note, segment_via_spec +from app.video.options import RenderOptions +from app.video.segmenter import segment +from app.video.spec import VideoSpec + +DOC = Path(__file__).resolve().parents[2] / "docs" / "gecko-video-creator.md" + + +def _assert_same(content, **kwargs): + want = segment(content, **kwargs) + got = segment_via_spec(content, **kwargs) + assert got.shots == want.shots + assert got.warnings == want.warnings + return want + + +def _round_trips_as_json(spec: VideoSpec) -> None: + dumped = spec.model_dump_json(by_alias=True, exclude_none=True) + assert VideoSpec.model_validate_json(dumped) == spec + + +# ── every case the segmenter's own tests already cover ────────────────────── + +_SEGMENTER_CASES = sorted( + name for name, fn in inspect.getmembers(segmenter_cases, inspect.isfunction) + if name.startswith("test_") +) + + +def test_the_segmenter_suite_is_all_reused(): + # If that file grows a fixture argument, it would silently stop being run + # below; this keeps the coverage honest. + assert len(_SEGMENTER_CASES) >= 60 + for name in _SEGMENTER_CASES: + assert not inspect.signature(getattr(segmenter_cases, name)).parameters, name + + +@pytest.mark.parametrize("name", _SEGMENTER_CASES) +def test_every_segmenter_case_compiles_back_to_the_same_shots(name, monkeypatch): + calls = [] + + def checked(content, **kwargs): + calls.append(1) + result = _assert_same(content, **kwargs) + _round_trips_as_json(import_note(content, **kwargs).spec) + return result + + monkeypatch.setattr(segmenter_cases, "segment", checked) + case = getattr(segmenter_cases, name) + case() + if "_run(" in inspect.getsource(case) or "segment(" in inspect.getsource(case): + assert calls, f"{name} never reached the patched segment()" + + +# ── a generated corpus, aimed at the walk's corners ───────────────────────── + +_FILES = ("a.png", "b.png", "loud.mp4", "quiet.mp4", "intro.mp4", "outro.mp4", "song.mp3", "logo.png") + + +def _media_root(tmp_path): + (tmp_path / "u1").mkdir(exist_ok=True) + for name in _FILES: + (tmp_path / "u1" / name).write_bytes(b"x") + return str(tmp_path) + + +def _random_text(rng): + words = ["salt", "road", "caravan", "dawn", "market", "well", "camel", "ruin"] + text = " ".join(rng.choice(words) for _ in range(rng.randint(1, 6))) + return rng.choice([ + text, text.capitalize() + ".", text + "...", text + " [pause]", "[pause:long]", + text + " [pause:1.5s] " + text, "", " ", text + "\n" + text, text + ":", + ]) + + +def _inline(rng, text): + if rng.random() < 0.2: + return [{"type": "link", "href": "https://example.com", + "content": [{"type": "text", "text": text}]}] + return [{"type": "text", "text": text}] + + +def _random_block(rng, depth=0): + choice = rng.choice([ + "paragraph", "paragraph", "paragraph", "heading", "heading", "quote", "quote", + "codeBlock", "bulletListItem", "table", "image", "image", "video", "videoFile", + "file", "diagram", "childNote", "audioFile", "divider", + ]) + block = {"id": f"blk{rng.randint(0, 10**6)}", "type": choice} + if choice in ("paragraph", "heading", "bulletListItem", "codeBlock"): + block["content"] = _inline(rng, _random_text(rng)) + elif choice == "quote": + body = _random_text(rng) + body = rng.choice([ + body, f"{body} — Ibn Somebody", f"{body}\n- the guide", f"{body} - an old saying", + f"{body} – 1971.", "[pause]", f"{body} -- unnamed source of very many words here", + ]) + block["content"] = _inline(rng, body) + elif choice == "table": + block["content"] = {"rows": [{"cells": [_inline(rng, _random_text(rng)) for _ in range(2)]} + for _ in range(rng.randint(0, 2))]} + elif choice in ("image", "video", "videoFile", "file"): + block["props"] = {"url": rng.choice([ + "/media/u1/a.png", "/media/u1/b.png", "/media/u1/loud.mp4", "/media/u1/quiet.mp4", + "/media/u1/a.png?v=2", "/media/u1/missing.png", "/media/u2/a.png", + "https://example.com/x.png", "/media/u1/notes.pdf", "", + ])} + elif choice == "diagram": + block["id"] = rng.choice(["d1", "d2"]) + if depth < 2 and rng.random() < 0.15: + block["children"] = [_random_block(rng, depth + 1) for _ in range(rng.randint(1, 3))] + return block + + +def _random_options(rng): + bumper = lambda name: rng.choice([ + {}, {"enabled": True, "url": f"/media/u1/{name}", "name": f"My {name}"}, + {"enabled": True, "url": "/media/u1/missing.mp4"}, {"enabled": True, "url": "/media/u1/a.png"}, + {"enabled": False, "url": f"/media/u1/{name}"}, + ]) + return RenderOptions( + title_card=rng.random() < 0.5, + chapter_screens=rng.random() < 0.4, + read_chapters=rng.random() < 0.7, + narrate_code=rng.random() < 0.5, + heading_pause_ms=rng.choice([0, 800]), + quotes={"enabled": rng.random() < 0.7}, + intro=bumper("intro.mp4"), + outro=bumper("outro.mp4"), + music=rng.choice([{}, {"enabled": True, "url": "/media/u1/song.mp3"}]), + diagram_images=rng.choice([{}, {"d1": "/media/u1/b.png"}, {"d1": "/media/u1/gone.png"}]), + ) + + +def test_a_generated_corpus_compiles_back_to_the_same_shots(tmp_path): + root = _media_root(tmp_path) + rng = random.Random(20261015) + for _ in range(3000): + blocks = [_random_block(rng) for _ in range(rng.randint(0, 12))] + kwargs = dict( + user_id="u1", media_dir=root, options=_random_options(rng), + note_title=rng.choice(["", "The Salt Road", "Road [pause:2s]"]), + author=rng.choice(["", "gecko"]), + has_audio=lambda path: os.path.basename(path) == "loud.mp4", + ) + content = json.dumps(blocks) + try: + _assert_same(content, **kwargs) + except AssertionError: + pytest.fail(f"round trip differs for:\n{json.dumps(blocks, indent=1)}\n{kwargs['options']}") + _round_trips_as_json(import_note(content, **kwargs).spec) + + +def test_unreadable_content_imports_as_an_empty_video(tmp_path): + root = _media_root(tmp_path) + for content in ("", "not json", "{}", "[1, null, \"x\"]"): + _assert_same(content, user_id="u1", media_dir=root, options=RenderOptions()) + + +# ── the options survive the trip ──────────────────────────────────────────── + +_DISABLEABLE = ("waveform", "watermark", "overlay_text", "music") + + +def _effective(options: RenderOptions) -> dict: + """What the renderer can tell apart. A disabled feature's settings are + inert, and the import-only fields are recorded as structure instead.""" + data = options.model_dump(exclude=set(IMPORT_ONLY_OPTIONS)) + data["quotes"].pop("enabled") + if options.music.enabled and not options.music.url: + data["music"]["enabled"] = False + for name in _DISABLEABLE: + if not data[name]["enabled"]: + data[name] = getattr(RenderOptions(), name).model_dump() + return data + + +def _random_render_options(rng): + return RenderOptions( + aspect=rng.choice(["16:9", "9:16", "1:1"]), resolution=rng.choice(["720p", "1080p", "4k"]), + quality=rng.choice(["preview", "standard", "high"]), fps=rng.choice([12, 24, 30, 60, 999]), + fit=rng.choice(["blur", "pad", "crop"]), + fallback=rng.choice([{}, {"type": "solid", "colors": ["#123456"]}, + {"type": "image", "url": "/media/u1/a.png", "angle": 45}, + {"type": "gradient", "url": "/media/u1/b.png"}]), + waveform={"enabled": rng.random() < 0.5, "mode": rng.choice(["line", "point"]), + "height_pct": rng.randint(5, 60), "scrim": rng.random()}, + watermark={"enabled": rng.random() < 0.5, "url": rng.choice([None, "/media/u1/logo.png"]), + "text": rng.choice(["", "geckopico"]), "position": "top-left", + "scale_pct": rng.uniform(0, 40), "margin_pct": rng.randint(0, 9)}, + overlay_text={"enabled": rng.random() < 0.5, "mode": rng.choice(["fixed", "title", "title_chapter"]), + "text": "hello", "shadow": rng.random() < 0.5, "size_pct": rng.uniform(0, 25)}, + title_card_text={"title_pct": rng.uniform(0, 30)}, chapter_card_text={"subtitle_pct": 4.0}, + transition={"style": rng.choice(["none", "fade", "dissolve"]), "duration": rng.uniform(0, 4)}, + ken_burns={"effect": rng.choice(["none", "alternate", "pan_up"]), "amount": rng.uniform(0, 1), + "include_cards": rng.random() < 0.5}, + music=rng.choice([{}, {"enabled": True, "url": "/media/u1/song.mp3", "volume": 0.3, + "duck": False, "fade_in": 0.0, "fade_out": 20}, + {"enabled": True}, {"enabled": False, "url": "/media/u1/song.mp3"}]), + quotes={"enabled": rng.random() < 0.5, "position": "top", "accent": "#000000", "size_pct": 99}, + code={"position": "bottom", "scrim": 0.1}, + subtitles=rng.choice(["off", "sidecar", "soft", "burn"]), + embed_chapters=rng.random() < 0.5, thumbnail=rng.random() < 0.5, + voice=rng.choice([None, "flux-haley-en"]), speed=rng.uniform(0, 3), + paragraph_pause_ms=rng.randint(0, 4000), heading_pause_ms=rng.randint(0, 6000), + shot_end_pause_ms=rng.randint(0, 4000), min_shot_seconds=rng.uniform(0, 40), + card_seconds=rng.uniform(0, 40), + ) + + +def test_render_options_survive_the_round_trip(tmp_path): + root = _media_root(tmp_path) + note = json.dumps([{"id": "1", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "2", "type": "paragraph", "content": [{"type": "text", "text": "Words."}]}]) + rng = random.Random(7) + for _ in range(300): + options = _random_render_options(rng) + spec = import_note(note, user_id="u1", media_dir=root, options=options).spec + compiled = compile_spec(spec, user_id="u1", media_dir=root).options + assert _effective(compiled) == _effective(options) + _round_trips_as_json(spec) + + +def test_bumpers_and_quotes_come_back_as_the_options_that_made_them(tmp_path): + root = _media_root(tmp_path) + options = RenderOptions(intro={"enabled": True, "url": "/media/u1/intro.mp4", "name": "Sting"}, + quotes={"enabled": True}) + note = json.dumps([{"id": "1", "type": "quote", + "content": [{"type": "text", "text": "Worth its weight — A Trader"}]}]) + spec = import_note(note, user_id="u1", media_dir=root, options=options).spec + compiled = compile_spec(spec, user_id="u1", media_dir=root).options + assert compiled.intro == options.intro + assert compiled.quotes.enabled + assert spec.assets[spec.segments[0].visual.asset].label == "Sting" + + +# ── the renderer can't tell the difference ────────────────────────────────── + +def _render_commands(monkeypatch, media, content, options, *, via_spec): + ff = FakeFFmpeg(monkeypatch) + monkeypatch.setattr(renderer.F, "probe_has_audio", lambda path: path.endswith("loud.mp4")) + if via_spec: + imported = import_note(content, user_id="u1", media_dir=media, options=options, + note_title="Salt", author="gecko", + has_audio=lambda path: path.endswith("loud.mp4")) + compiled = compile_spec(imported.spec, user_id="u1", media_dir=media) + plan = compiled.segmentation + plan.warnings[:0] = imported.warnings + monkeypatch.setattr(renderer, "segment", lambda *a, **k: plan) + options = compiled.options + result = renderer.render( + job_id="j", user_id="u1", media_dir=media, note_content=content, + note_title="Salt", author="gecko", options=options, preview=True, + tts=lambda text: b"audio", progress=lambda *a: None, + max_shots=200, max_narration_chars=100_000, + ) + monkeypatch.undo() + # Output names are random per render; everything else must match exactly. + hexes = {result.video_filename.split(".")[0]} + return [[re.sub("|".join(hexes), "OUT", arg) for arg in call] for call in ff.calls], result.warnings + + +_RENDER_NOTE = json.dumps([ + {"id": "1", "type": "paragraph", "content": [{"type": "text", "text": "Before any picture."}]}, + {"id": "2", "type": "heading", "content": [{"type": "text", "text": "The road [pause]"}]}, + {"id": "3", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "4", "type": "paragraph", "content": [{"type": "text", "text": "Under the map."}]}, + {"id": "5", "type": "quote", "content": [{"type": "text", "text": "Salt was gold — A Trader"}]}, + {"id": "6", "type": "paragraph", "content": [{"type": "text", "text": "After the quote."}]}, + {"id": "7", "type": "videoFile", "props": {"url": "/media/u1/loud.mp4"}}, + {"id": "8", "type": "paragraph", "content": [{"type": "text", "text": "Over the clip, muted."}]}, + {"id": "9", "type": "codeBlock", "content": [{"type": "text", "text": "print('salt')"}]}, + {"id": "10", "type": "heading", "content": [{"type": "text", "text": "Ruins"}]}, + {"id": "11", "type": "image", "props": {"url": "/media/u1/missing.png"}}, + {"id": "12", "type": "paragraph", "content": [{"type": "text", "text": "Nothing left."}]}, +]) + + +@pytest.mark.parametrize("overrides", [ + {}, + {"chapter_screens": True, "read_chapters": False, "narrate_code": True}, + # Ken Burns is left to the options round trip above: a drifting still is + # prescaled to ~24 megapixels with Pillow, which costs seconds per shot here. + {"transition": {"style": "dissolve", "duration": 0.5}, + "waveform": {"enabled": True}, "subtitles": "burn"}, + {"intro": {"enabled": True, "url": "/media/u1/intro.mp4"}, + "outro": {"enabled": True, "url": "/media/u1/outro.mp4"}, + "music": {"enabled": True, "url": "/media/u1/song.mp3"}, + "watermark": {"enabled": True, "url": "/media/u1/logo.png"}, + "overlay_text": {"enabled": True, "mode": "title_chapter"}, + "fallback": {"type": "image", "url": "/media/u1/b.png"}}, + {"watermark": {"enabled": False, "url": "/media/u1/logo.png", "text": "hidden"}, + "music": {"enabled": False, "url": "/media/u1/song.mp3"}, "transition": {"style": "fade"}}, +]) +def test_the_renderer_issues_the_same_ffmpeg_commands_either_way(tmp_path, monkeypatch, overrides): + media = _media_root(tmp_path) + options = RenderOptions(quotes={"enabled": True}, **overrides) + want = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=False) + got = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=True) + assert got == want + assert len(want[0]) > 5 + + +def test_music_switched_on_with_no_track_is_not_part_of_the_video(tmp_path, monkeypatch): + # The one thing the trip deliberately drops. A spec's music is a track, and + # a track names its file, so "music on, nothing chosen" has no spec form: + # the video is identical, and only the renderer's note that it skipped the + # music goes. Notes itself still renders from the original options. + media = _media_root(tmp_path) + options = RenderOptions(music={"enabled": True}) + want_calls, want_warnings = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=False) + got_calls, got_warnings = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=True) + assert got_calls == want_calls + assert want_warnings == got_warnings + ["Background music was skipped: that track could not be read."] + + +# ── what a spec says, and what the Notes renderer refuses ─────────────────── + +def test_the_worked_example_in_the_design_doc_is_a_valid_spec(): + example = re.findall(r"```json\n(.*?)```", DOC.read_text(), re.S)[0] + spec = VideoSpec.model_validate_json(example) + assert [s.kind for s in spec.segments] == ["card", "clip", "scene", "scene", "scene", "scene", "clip"] + _round_trips_as_json(spec) + with pytest.raises(UnsupportedSpec, match="Gecko Video Creator"): + compile_spec(spec, user_id="u1", media_dir="/nowhere") + + +def test_a_quote_is_an_overlay_on_its_own_beat_not_a_segment(tmp_path): + root = _media_root(tmp_path) + note = json.dumps([ + {"id": "img", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "p1", "type": "paragraph", "content": [{"type": "text", "text": "Before"}]}, + {"id": "q1", "type": "quote", "content": [{"type": "text", "text": "Salt is gold — A Trader"}]}, + {"id": "p2", "type": "paragraph", "content": [{"type": "text", "text": "After"}]}, + ]) + spec = import_note(note, user_id="u1", media_dir=root, note_id="n1", + options=RenderOptions(title_card=False, quotes={"enabled": True})).spec + assert spec.source.note_id == "n1" + [scene] = spec.segments + assert [(b.id, b.kind, b.text) for b in scene.beats] == [ + ("b1", "prose", "Before."), ("b2", "quote", "Salt is gold."), ("b3", "prose", "After."), + ] + [quote] = scene.overlays + assert (quote.type, quote.text, quote.attribution) == ("quote", "Salt is gold", "A Trader") + assert (quote.show.from_.beat, quote.show.to.beat) == ("b2", "b2") + assert scene.source.block_ids == ["img", "p1", "q1", "p2"] + assert [b.block_id for b in scene.beats] == ["p1", "q1", "p2"] + + +def _spec(**changes): + base = { + "assets": {"a1": {"from": "notes", "url": "/media/u1/a.png", "type": "image"}, + "g1": {"from": "gam", "id": "abc"}}, + "segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "a1"}, + "narration": {"beats": [{"id": "b1", "text": "One."}, {"id": "b2", "text": "Two."}]}}], + } + base.update(changes) + return VideoSpec.model_validate(base) + + +@pytest.mark.parametrize("change, message", [ + ({"segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "g1"}}]}, "not a file in the note"), + ({"segments": [{"id": "s1", "kind": "scene", "transition_in": {"style": "fade"}}]}, "transition_in"), + ({"segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "a1", "trim": {"in": 1}}}]}, "trimmed"), + ({"segments": [{"id": "s1", "kind": "scene", "narration": {"beats": [{"id": "b1", "text": "x"}]}, + "overlays": [{"id": "o1", "type": "lower_third", "title": "Guide"}]}]}, "lower_third"), + ({"segments": [{"id": "s1", "kind": "scene", "narration": {"beats": [{"id": "b1", "text": "x"}]}, + "overlays": [{"id": "o1", "type": "quote", "text": "x"}]}]}, "exactly one of its beats"), + ({"voice": {"provider": "fal"}}, "voice.provider"), + ({"audio_tracks": [{"id": "t1", "asset": "a1", "start": "video_start", "end": "video_end"}]}, + "audio file"), +]) +def test_what_the_notes_renderer_cannot_draw_is_refused(change, message): + with pytest.raises(UnsupportedSpec, match=message): + compile_spec(_spec(**change), user_id="u1", media_dir="/nowhere") + + +@pytest.mark.parametrize("change, message", [ + ({"segments": [{"id": "s1", "kind": "scene"}, {"id": "s1", "kind": "scene"}]}, "duplicate segment"), + ({"segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "nope"}}]}, "unknown asset"), + ({"segments": [{"id": "s1", "kind": "card"}]}, "card with no card"), + ({"segments": [{"id": "s1", "kind": "clip"}]}, "clip with no visual"), + ({"segments": [{"id": "s1", "kind": "scene", "colour": "red"}]}, "Extra inputs"), +]) +def test_a_malformed_spec_is_rejected(change, message): + with pytest.raises(ValidationError, match=message): + _spec(**change) diff --git a/docs/gecko-video-creator.md b/docs/gecko-video-creator.md index ec20311..5079982 100644 --- a/docs/gecko-video-creator.md +++ b/docs/gecko-video-creator.md @@ -1,6 +1,6 @@ # Gecko Video Creator (GVC) — Product & Data Model Spec -**Status:** Draft for discussion +**Status:** Draft for discussion · Phase 0 in progress (§10) **Date:** 2026-10-07 **Scope:** What GVC is, the JSON document a video is described by (the **VideoSpec**), how an AI builds and edits one, and how GVC pulls scripts from Gecko Notes and media @@ -159,7 +159,7 @@ transitions, watermark, waveform, voice) is a **global** field on `RenderOptions | Structure is **re-inferred every render** | Nothing to edit, version or hand to an AI. The only way to change the video is to change the note | | Media must be in the note's own `/media/` | No library search, no generation inside the video workflow | -Four existing defects should be fixed on the way in rather than carried over: +Six existing defects should be fixed on the way in rather than carried over: - **The Voice picker does nothing.** `VideoGenModal.tsx` offers one and writes `options.voice`, but no backend code reads it. The voice always comes from the account @@ -172,6 +172,16 @@ Four existing defects should be fixed on the way in rather than carried over: maxes out at 10 versus 30, and there are a few more. - **Some settings have no UI at all:** `fps`, `fallback.angle`, waveform opacity and scrim, watermark opacity and margin, overlay margin and shadow. +- **A chapter mark can vanish.** With headings not read aloud, a heading followed by a + picture and then straight away a quote loses its chapter: the mark is carried onto the + picture's shot, and the quote then takes that shot over (`segmenter.py:461`). +- **Pause markers leak into chapter marks.** `Shot.__post_init__` strips `[pause:…]` + from drawn fields set when a shot is built, but a section's chapter is set afterwards + (`segmenter.py:365`, `:384`), so `The Road [pause:2s]` lands in the MP4's chapter list + and the "title + chapter" overlay as typed. + +Phase 0 reproduces all of these exactly, so its parity tests can hold the spec path to +today's output; each is a small follow-up fix once the spec path is the only one. --- @@ -214,9 +224,11 @@ Four existing defects should be fixed on the way in rather than carried over: ## 5. The VideoSpec (draft schema) -Written as TypeScript-style types because they're the most readable form. The real -source of truth will be Pydantic (principle 9). Defaults shown are today's -`RenderOptions` defaults. +Written as TypeScript-style types because they're the most readable form. The +canonical definition is now Pydantic, in +[`backend/app/video/spec.py`](../backend/app/video/spec.py) (principle 9). Where the two +differ, the code wins. Defaults shown are today's `RenderOptions` defaults. Not yet +modelled: per-overlay style overrides. ### 5.1 Top level @@ -343,7 +355,7 @@ interface Segment { transition_in?: Transition; // overrides style.transition for the cut INTO this segment duration?: { min?: number; fixed?: number }; // default: automatic chrome?: boolean; // false = no watermark, overlay text or waveform - source?: { note_id: string; block_ids: string[] }; // provenance, for re-sync + source?: { block_ids: string[] }; // provenance, for re-sync (the note is spec.source) } interface Visual { @@ -420,7 +432,9 @@ what the current renderer does around intro/outro bumpers. ```ts type AssetRef = { label?: string } & ( // label: a readable name for people and the AI | { from: "gam"; id: string } // an asset or clip in GAM - | { from: "notes"; url: string } // "/media//.png" from a note + | { from: "notes"; url: string; // "/media//.png" from a note + type: "image" | "video" | "audio" } // as the note used it: a videoFile block + // is a video whatever its extension | { from: "generate"; // to be generated, via GAM type: "image" | "video" | "music" | "sfx"; prompt: string; model?: string; base_assets?: AssetKey[]; @@ -572,7 +586,7 @@ bell sound effect lands on one line. "a_logo": { "from": "gam", "id": "0b6f6c3e-2d1a-4e8b-9c55-7a3e1f0d2b44", "label": "Gecko logo" }, "a_intro": { "from": "gam", "id": "5d2e9a17-6b3c-4f80-a1d4-c8e7f2b0961a", "label": "Channel intro sting" }, "a_outro": { "from": "gam", "id": "e41c7b90-3a2d-4f6e-8b15-92d0c4a7f3e8", "label": "Channel outro" }, - "a_map": { "from": "notes", "url": "/media/9c41e2f0-1b7d-4a3e-8f62-d05a7c3b1e98/2f8a6c1d-4e3b-4b7a-9e0f-5c2d1a8b7e6f.png", "label": "Hand-drawn route map" }, + "a_map": { "from": "notes", "url": "/media/9c41e2f0-1b7d-4a3e-8f62-d05a7c3b1e98/2f8a6c1d-4e3b-4b7a-9e0f-5c2d1a8b7e6f.png", "type": "image", "label": "Hand-drawn route map" }, "a_caravan": { "from": "gam", "id": "8e3b7c21-4d5f-4a90-b6e2-1c7d9f0a3b58", "label": "Archive interview reel, 1971" }, "a_market": { "from": "generate", @@ -620,7 +634,7 @@ bell sound effect lands on one line. { "id": "b3", "text": "One route carried more of it than any other." } ] }, - "source": { "note_id": "6b1e0c2a-4f7d-4c1e-9a55-3f0d2b8e7a10", "block_ids": ["blk-01", "blk-02", "blk-03", "blk-04"] } + "source": { "block_ids": ["blk-01", "blk-02", "blk-03", "blk-04"] } }, { "id": "s_caravan", @@ -888,12 +902,15 @@ or extract. | Media embedded in the note | `GET /api/notes/{id}/assets`; files at `/media/{user}/{uuid}.ext` | ✅ | | Note as Markdown for the AI | expose `note_to_markdown`, e.g. `GET /api/notes/{id}?format=markdown` | ➕ small | | "Make video" in Notes | a button / slash-menu item → `https://gvc.geckopico.com/new?note={id}` (replaces today's dialog) | ➕ small | -| Cross-origin calls | add `https://gvc.geckopico.com` to `CORS_ORIGIN` and the CSP in [`frontend/nginx.conf`](../frontend/nginx.conf) | ➕ config | +| Cross-origin calls | add `https://gvc.geckopico.com` to `CORS_ORIGIN` and the CSP in [`frontend/nginx.conf`](../frontend/nginx.conf), only where GVC's browser code calls Notes directly (server-side calls with a Bearer token need neither, as in GAM) | ➕ config | +| Speech for GVC's renders | `POST /api/settings/speech/tts`, plus a `provider` field passed through to `synthesize_tts_bytes(provider_override=…)`, so one video can't switch voices partway | ➕ small | +| Put the finished video back in the note | No endpoint can write `Note.content` safely against the editor's autosave (`routers/assets.py:174`), and today's auto-insert relies on Notes' own job registry | ➕ needs design (open decision 11) | -**Re-sync.** Every imported segment records the `block_ids` it came from. When the note's +**Re-sync.** Every imported segment records the `block_ids` it came from, and every beat +the block it was read from (`Beat.block_id`). When the note's `modified_at` is newer than the spec's `imported_at`, GVC shows a per-segment diff: - **Changed text** updates the matching beats in place. Overlays anchored to those beats - keep working, because beat IDs follow block IDs. + keep working, because a beat keeps its ID and is matched to its block by `block_id`. - **New blocks** are offered as new beats or segments. - **Deleted blocks** are flagged, not removed. @@ -910,7 +927,7 @@ A segment edited in GVC is never silently overwritten by a re-sync (open decisio | Credits | structured attribution fields on every asset | ✅ (GAM M10) | | Generate images / video / audio | fal.ai through GAM, with prompt and base assets recorded | 🟡 planned (GAM M8) | | Read media bytes for rendering | signed `/media/{key}?exp=…&sig=…` URLs | ✅ (open decision 2) | -| Store the render | upload with `source: "gvc_export"`, `origin_project_id` = spec `id` | 🟡 fields exist on the model; the upload path needs wiring | +| Store the render | upload with `source: "gvc_export"`, `origin_project_id` = spec `id` | 🟡 `gvc_export` is a known source, but `origin_project_id` / `origin_reference_id` are only planned (GAM's `plan-of-attack.md`), not on the model yet; they need a migration, plus the upload path | GVC is what unblocks GAM's M8 and M9. Both were deliberately left waiting for a real consumer. @@ -924,7 +941,11 @@ consumer. - React 18 + Vite + TypeScript + Zustand + Tailwind - Docker Compose + Nginx - served at `gvc.geckopico.com` behind Caddy on the shared `web` network, exactly like - Notes' `docker-compose.prod.yml` + Notes' `docker-compose.prod.yml`. Only the frontend joins `web`, and the backend + service is named `gvc-backend`: Notes and GAM both already put a service called + `backend` on `web`, and each app's nginx proxies to `backend:8000`, so a third would + make that name more ambiguous still. (Check the live host before adding GVC: + `docker compose exec frontend getent hosts backend` should return one address.) **Data:** - `Project`: the current VideoSpec as JSON, plus title, source note and owner @@ -949,15 +970,35 @@ consumer. - the AI chat alongside, with a suggestions tray - a raw JSON tab for power use +### Where the code lives + +Decided after weighing the three options against the code (2026-10-14): **Phase 0 +happens here, in gecko-notes; GVC itself starts in its own repo, `davior/gvc`**, built +from GAM's skeleton (its `config.py`, auth with a shadow user, Alembic, three-job CI, +Python 3.13). The reasons, briefly: + +- About 75% of the video code (3,441 of 4,582 lines) imports nothing from Notes and can + be copied as it is. The rest (`worker.py`, `routers/video.py`, the segmenter's media + seams) has to be rewritten against HTTP wherever GVC lives, so building it inside + Notes saves nothing and makes reaching into Notes' database too easy. +- Notes' repo is set up for one app: its CLAUDE.md deploy steps, CI without path + filters, one `main`, one rollback, and a backup of `notes.db` only. +- The video engine is still changing (six commits in the three weeks before the spec). + Copying it once, after Phase 0 has made it spec-driven, keeps the period with two + copies short. + +From the day the engine is copied into GVC, **new video features in Notes are frozen**: +bug fixes only, each logged so it can be ported. + ### Phases | Phase | What | Done when | |---|---|---| -| **0 — Spec inside Notes** | Define the VideoSpec in Pydantic. `segmenter.py` emits it, a compiler turns it into today's `Shot` list, and `renderer.py` is untouched. Expose `GET /api/video/spec?note_id=` for inspection. | Golden tests: a corpus of notes renders byte-identical (or frame-identical) through the spec path | -| **1 — GVC renders what Notes renders** | New repo `davior/gvc`. SSO, project storage and history, import from Notes, the lifted engine, preview and full render, output to GAM. | Any note renders in GVC the same as in Notes | +| **0 — Spec inside Notes** | **In progress.** First, alongside today's code: the VideoSpec in Pydantic ([`spec.py`](../backend/app/video/spec.py)), an importer that walks a note exactly as `segment()` does ([`importer.py`](../backend/app/video/importer.py)), and a compiler back to today's `Shot`s and `RenderOptions` ([`compiler.py`](../backend/app/video/compiler.py)), with `renderer.py` and `segment()` untouched. Then, separately: `segment()` switches to the spec path, `GET /api/video/spec?note_id=` exposes it, and the voice bug is fixed. | [`tests/test_video_spec.py`](../backend/tests/test_video_spec.py): every segmenter test case, a few thousand generated notes and the renderer's ffmpeg commands come out identical through the spec path, with no real encodes | +| **1 — GVC renders what Notes renders** | New repo `davior/gvc` from GAM's skeleton. The engine copied from the Phase 0 merge, SSO, project storage and history, import from Notes over HTTP, speech through Notes, preview and full render, download. Output to GAM and back into the note follow once those are designed. | Any note renders in GVC the same as in Notes | | **2 — The new capabilities** | Timed overlays, multiple audio tracks, trims, per-segment voice/motion/transition, GAM search and picker, `generate` / `search` placeholders, credits | The §6 example renders | | **3 — AI editing** | The operations of §8.2, the suggestions tray, chat, staged review | "Make a video from this note" produces a reviewed, rendered video | -| **4 — Notes hands over** | Notes' dialog becomes "Open in GVC"; `backend/app/video/` and `VideoGenModal.tsx` are removed from Notes; existing render jobs and outputs keep working | No video code left in Notes | +| **4 — Notes hands over** | Notes' dialog becomes "Open in GVC"; `VideoGenModal.tsx` and the render pipeline are removed from Notes; existing render jobs and outputs keep working. The speech pieces read-aloud export shares (`ffmpeg.py`'s helpers, `narration.py`'s stitching, `pause_markup.py`, imported at `routers/settings.py:26-28`) move to an `app/speech/` package and stay | No render code left in Notes | Phase 0 is the cheap insurance. If the schema can't faithfully describe what Notes already renders, it's wrong, and finding that out costs a branch, not a new app. @@ -982,7 +1023,7 @@ already renders, it's wrong, and finding that out costs a branch, not a new app. **In GAM** (`davior/gam`): - `GET /api/assets`, `GET /api/assets/{id}`, `GET /api/assets/{id}/clips`, `GET /api/search`, signed `/media/{key}`. - M8 (generation) and M9 (GVC surface, embeddable picker): both waiting on GVC. -- `Asset.source = "gvc_export"`, `origin_project_id`, `origin_reference_id`: already on the model. +- `Asset.source = "gvc_export"` is already a known source. `origin_project_id` and `origin_reference_id` are planned in GAM's `plan-of-attack.md` but not on the model yet: a migration GVC needs before it can store renders there. --- @@ -1017,8 +1058,12 @@ already renders, it's wrong, and finding that out costs a branch, not a new app. block a render or only warn? Should the credits card be on by default? 10. **Per-beat voice.** Is a voice per segment enough, or are interview-style videos with several voices within one segment a near-term need? +11. **Putting the render back in the note.** Today the worker appends a video block to + the note and the open editor reconciles it. From another app that needs a Notes + endpoint that can't race the editor's autosave, or a different gesture: the note + links to the GVC project, and the user inserts the video themselves. --- -*This document is a product and data-model spec, not an implementation. No application -behaviour has changed.* +*This document is a product and data-model spec. Phase 0's spec, importer and compiler +are in `backend/app/video/`; nothing a user sees has changed.*