diff --git a/backend/app/video/compiler.py b/backend/app/video/compiler.py new file mode 100644 index 0000000..a5932cb --- /dev/null +++ b/backend/app/video/compiler.py @@ -0,0 +1,333 @@ +"""VideoSpec -> the shots and options today's renderer consumes. + +The renderer still works on a flat list of `Shot`s and one `RenderOptions`; +this is the step that produces them from a spec, so the renderer itself needs no +change while the spec becomes the thing that describes a video. A segment is not +always one shot: a quote or a code block is an overlay on one beat of a scene, +and the renderer draws those as a shot of their own, so a scene is split there. + +Anything a spec can say that this renderer cannot do — GAM assets, timed +overlays, per-segment motion or voice, more than one music track — is refused +with `UnsupportedSpec` rather than quietly rendered differently. Those arrive +with Gecko Video Creator's own renderer (docs/gecko-video-creator.md §10). +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + +from app.video.options import ( + BackgroundSpec, BumperSpec, KenBurnsSpec, MusicSpec, OverlayTextSpec, QuoteSpec, + RenderOptions, WatermarkSpec, WaveformSpec, +) +from app.video.segmenter import ( + Segmentation, Shot, ShotKind, _set_apart, resolve_media_path, +) +from app.video.spec import ( + Anchor, AnchorRef, Beat, CodeOverlay, NotesAsset, QuoteOverlay, Segment, VideoSpec, +) + +# RenderOptions fields that decide what a note is *imported* as rather than how +# it renders: whether there is a title card, whether headings become chapter +# screens, and so on. A spec records the outcome — the cards, the beats, the +# bumper segments — so they cannot be read back out of one, and the renderer +# never looks at them. Compiled options leave them at their defaults. +IMPORT_ONLY_OPTIONS = frozenset({ + "title_card", "chapter_screens", "read_chapters", "narrate_code", + "insert_into_note", "diagram_images", "intro", "outro", +}) + + +class UnsupportedSpec(ValueError): + """The spec asks for something the Notes renderer cannot draw.""" + + +@dataclass +class Compiled: + segmentation: Segmentation + options: RenderOptions + # Shot index -> the id of the segment it came from, for diagnostics. + shot_segments: List[str] = field(default_factory=list) + + +def narration_text(beats: List[Beat], *, card: bool, heading_pause_ms: int) -> str: + """What a run of beats sounds like as one shot's narration. + + A heading inside a section is set apart with blank lines, which + `chunk_narration` turns into the heading pause; a card is a section + boundary of its own and needs no marking. + """ + pieces: List[str] = [] + for beat in beats: + if not beat.speak or not beat.text: + continue + text = beat.text + if beat.kind == "heading" and not card and heading_pause_ms > 0: + text = _set_apart(text) + pieces.append(text) + return "\n".join(pieces).strip() + + +def compile_spec(spec: VideoSpec, *, user_id: str, media_dir: str) -> Compiled: + _refuse_unsupported(spec) + warnings: List[str] = [] + shots: List[Shot] = [] + owners: List[str] = [] + paths: Dict[str, Optional[str]] = {} + + def resolve(key: str) -> Optional[str]: + if key not in paths: + asset = spec.assets[key] + paths[key] = resolve_media_path(asset.url, user_id, media_dir) + if paths[key] is None: + warnings.append(f"Skipped missing media: {asset.url[:120]}") + return paths[key] + + heading_ms = spec.voice.pauses.heading_ms + for segment in spec.segments: + for shot in _segment_shots(segment, spec, resolve, heading_ms): + shots.append(shot) + owners.append(segment.id) + + return Compiled( + segmentation=Segmentation(shots=shots, warnings=warnings), + options=_options(spec), + shot_segments=owners, + ) + + +def _segment_shots(segment: Segment, spec: VideoSpec, resolve, heading_ms: int) -> List[Shot]: + path = resolve(segment.visual.asset) if segment.visual is not None else None + + if segment.kind == "card": + return [Shot( + kind="card", + card_title=segment.card.title, + card_subtitle=segment.card.subtitle, + card_kind=segment.card.variant, + chapter=segment.chapter, + narration=narration_text(segment.beats, card=True, heading_pause_ms=heading_ms), + label=segment.label, + )] + + if segment.kind == "clip": + bumper = segment.role if segment.role in ("intro", "outro") else None + return [Shot(kind="video_sound", background=path, chapter=segment.chapter, + bumper=bumper, label=segment.label)] + + kind: ShotKind = "still" + if path is not None and spec.assets[segment.visual.asset].type == "video": + kind = "video_muted" + + # Split the beats where an overlay is: each overlaid beat is a shot of its + # own, and the beats between them run together as one. + overlay_at = {_overlay_beat(segment, o): o for o in segment.overlays} + runs: List[Tuple[Optional[object], List[Beat]]] = [] + for beat in segment.beats: + overlay = overlay_at.get(beat.id) + if overlay is not None: + runs.append((overlay, [beat])) + elif runs and runs[-1][0] is None: + runs[-1][1].append(beat) + else: + runs.append((None, [beat])) + if not runs: + runs.append((None, [])) + + shots: List[Shot] = [] + for index, (overlay, beats) in enumerate(runs): + if index == 0: + label = segment.label + elif overlay is not None: + label = overlay.type + else: + label = f"after {runs[index - 1][0].type}" + shot = Shot( + kind=kind, background=path, + narration=narration_text(beats, card=False, heading_pause_ms=heading_ms), + quote_text=overlay.text if isinstance(overlay, QuoteOverlay) else None, + quote_attribution=overlay.attribution if isinstance(overlay, QuoteOverlay) else None, + code_text=overlay.text if isinstance(overlay, CodeOverlay) else None, + label=label, + ) + if index == 0 and segment.chapter: + # Set after construction, as segment() does for every section, so + # the mark is exactly what it was — pause markers included, which + # nothing strips from a section's chapter today. + shot.chapter = segment.chapter + shots.append(shot) + return shots + + +def _overlay_beat(segment: Segment, overlay) -> str: + """The one beat an overlay spans. Anything else is a timed overlay.""" + show = overlay.show + start = show.from_ if show is not None else None + end = show.to if show is not None else None + if (isinstance(start, AnchorRef) and isinstance(end, AnchorRef) + and start.beat is not None and start.beat == end.beat + and start.edge == "start" and end.edge == "end" + and not start.offset and not end.offset + and start.segment in (None, segment.id) and end.segment in (None, segment.id) + and any(b.id == start.beat for b in segment.beats)): + return start.beat + raise UnsupportedSpec( + f"overlay {overlay.id!r} in segment {segment.id!r} must span exactly one of its beats" + ) + + +def _options(spec: VideoSpec) -> RenderOptions: + style, chrome, voice = spec.style, spec.chrome, spec.voice + + def url(key: Optional[str]) -> Optional[str]: + return spec.assets[key].url if key is not None else None + + options = RenderOptions( + aspect=spec.output.aspect, + resolution=spec.output.resolution, + quality=spec.output.quality, + fps=spec.output.fps, + subtitles=spec.output.subtitles, + embed_chapters=spec.output.chapters, + thumbnail=spec.output.thumbnail, + fit=style.fit, + fallback=BackgroundSpec( + type=style.background.type, colors=list(style.background.colors), + angle=style.background.angle, url=url(style.background.asset), + ), + title_card_text=style.cards.title.model_copy(), + chapter_card_text=style.cards.chapter.model_copy(), + transition=style.transition.model_copy(), + ken_burns=KenBurnsSpec(effect=style.motion.effect, amount=style.motion.amount, + include_cards=style.cards.motion), + quotes=QuoteSpec( + enabled=any(isinstance(o, QuoteOverlay) for s in spec.segments for o in s.overlays), + **style.quote.model_dump(), + ), + code=style.code.model_copy(), + min_shot_seconds=style.min_segment_seconds, + card_seconds=style.cards.seconds, + voice=voice.voice, + speed=voice.speed, + paragraph_pause_ms=voice.pauses.paragraph_ms, + heading_pause_ms=voice.pauses.heading_ms, + shot_end_pause_ms=voice.pauses.segment_end_ms, + ) + if chrome.watermark is not None: + fields = chrome.watermark.model_dump(exclude={"asset"}) + options.watermark = WatermarkSpec(enabled=True, url=url(chrome.watermark.asset), **fields) + if chrome.overlay_text is not None: + options.overlay_text = OverlayTextSpec(enabled=True, **chrome.overlay_text.model_dump()) + if chrome.waveform is not None: + options.waveform = WaveformSpec(enabled=True, **chrome.waveform.model_dump()) + for track in spec.audio_tracks: + options.music = MusicSpec( + enabled=True, url=url(track.asset), volume=track.volume, + duck=track.duck_under_narration is not False, + fade_in=track.fade_in, fade_out=track.fade_out, + ) + for segment in spec.segments: + if segment.role in ("intro", "outro"): + asset = spec.assets[segment.visual.asset] + setattr(options, segment.role, + BumperSpec(enabled=True, url=asset.url, name=asset.label or "")) + return options + + +# ── what the Notes renderer cannot do ─────────────────────────────────────── + +def _refuse_unsupported(spec: VideoSpec) -> None: + problems: List[str] = [] + used = {s.visual.asset for s in spec.segments if s.visual is not None} + used |= {t.asset for t in spec.audio_tracks} + for key in (spec.style.background.asset, + spec.chrome.watermark.asset if spec.chrome.watermark else None): + if key is not None: + used.add(key) + for key in sorted(used): + if not isinstance(spec.assets[key], NotesAsset): + problems.append(f"asset {key!r} is not a file in the note's media " + f"({spec.assets[key].from_} assets need Gecko Video Creator)") + if spec.voice.provider is not None or spec.voice.expressivity is not None: + problems.append("voice.provider and voice.expressivity come from the account here") + if spec.credits is not None: + problems.append("credits") + + for segment in spec.segments: + where = f"segment {segment.id!r}" + bumper = segment.role in ("intro", "outro") + if segment.transition_in is not None: + problems.append(f"{where}: per-segment transition_in") + if segment.duration is not None: + problems.append(f"{where}: duration overrides") + if segment.chrome is not None and segment.chrome != (not bumper): + problems.append(f"{where}: chrome is on for every segment but an intro or outro") + if segment.role == "title" and segment.kind != "card": + problems.append(f"{where}: only a card can be the title") + if bumper and segment.kind != "clip": + problems.append(f"{where}: an intro or outro must be a clip") + if segment.narration is not None and segment.narration.voice is not None: + problems.append(f"{where}: per-segment voice") + for beat in segment.beats: + if beat.hold is not None or beat.pause_after_ms is not None: + problems.append(f"{where}: beat {beat.id!r} hold / pause_after_ms") + visual = segment.visual + if visual is not None: + if visual.fit is not None or visual.motion is not None: + problems.append(f"{where}: per-segment fit or motion") + if visual.trim is not None: + problems.append(f"{where}: trimmed clips") + if not visual.loop: + problems.append(f"{where}: a video that holds its last frame") + if visual.audio != ("play" if segment.kind == "clip" else "mute"): + problems.append(f"{where}: a clip plays its sound, a scene's video is muted") + if segment.kind == "card" and visual is not None: + problems.append(f"{where}: a card draws its own background") + if segment.kind == "clip" and (segment.beats or segment.overlays): + problems.append(f"{where}: a clip carries no narration or overlays") + if segment.kind == "clip" and visual is not None and spec.assets[visual.asset].type != "video": + problems.append(f"{where}: a clip must be a video") + if segment.kind == "card" and segment.overlays: + problems.append(f"{where}: overlays on a card") + anchored = [] + for overlay in segment.overlays: + if not isinstance(overlay, (QuoteOverlay, CodeOverlay)): + problems.append(f"{where}: {overlay.type} overlays") + elif overlay.fade is not None: + problems.append(f"{where}: overlay fades") + else: + try: + anchored.append(_overlay_beat(segment, overlay)) + except UnsupportedSpec as exc: + problems.append(str(exc)) + if len(anchored) != len(set(anchored)): + problems.append(f"{where}: two overlays on one beat") + + content = [s.id for s in spec.segments if s.role not in ("intro", "outro")] + if len(spec.audio_tracks) > 1: + problems.append("more than one audio track") + for track in spec.audio_tracks: + where = f"audio track {track.id!r}" + if track.kind != "music": + problems.append(f"{where}: only a music bed") + if track.trim is not None or track.loop is False: + problems.append(f"{where}: trim / loop off") + if spec.assets[track.asset].type != "audio": + problems.append(f"{where}: must be an audio file") + if not content or not (_at(track.start, spec, content[0], "start") + and track.end is not None + and _at(track.end, spec, content[-1], "end")): + problems.append(f"{where}: the music bed runs from the first section " + f"to the last, around any intro and outro") + if problems: + raise UnsupportedSpec("Not supported by the Notes renderer: " + "; ".join(problems)) + + +def _at(anchor: Anchor, spec: VideoSpec, segment_id: str, edge: str) -> bool: + """Whether `anchor` is exactly the `edge` of segment `segment_id`.""" + if isinstance(anchor, str): + ids = [s.id for s in spec.segments] + index = 0 if anchor == "video_start" else len(ids) - 1 + wanted = "start" if anchor == "video_start" else "end" + return bool(ids) and ids[index] == segment_id and wanted == edge + return (anchor.segment == segment_id and anchor.beat is None + and anchor.edge == edge and not anchor.offset) diff --git a/backend/app/video/importer.py b/backend/app/video/importer.py new file mode 100644 index 0000000..1bafd45 --- /dev/null +++ b/backend/app/video/importer.py @@ -0,0 +1,476 @@ +"""A note (BlockNote JSON) -> a VideoSpec. + +The walk is `segmenter.segment()`'s, step for step, so that compiling what this +produces gives back exactly the shots `segment()` would have made — that +equivalence is what tests/test_video_spec.py holds it to, and what lets +`segment()` later become `compile_spec(import_note(...))`. What it records is +richer: the text under a picture is kept as one beat per block rather than one +joined string, a quote or code block is an overlay on its own beat rather than a +shot of its own, and every segment remembers which blocks it came from. + +Until the switch, the two walks must change together. Anything the import +decides — whether headings become chapter screens, whether quotes are drawn — +is settled here, from `RenderOptions`, and the spec only records the outcome. +""" + +import json +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple + +from app.video.compiler import compile_spec, narration_text +from app.video.options import RenderOptions +from app.video.pause_markup import strip_pause_markup +from app.video.segmenter import ( + Segmentation, ShotKind, _as_sentence, _block_narration, _classify, _flatten, _inline_text, + _media_kind, _media_url, _split_attribution, resolve_media_path, +) +from app.video.spec import ( + AnchorRef, AudioTrack, Background, Beat, Card, Cards, Chrome, CodeOverlay, Motion, + Narration, NotesAsset, Output, OverlayText, Pauses, QuoteOverlay, QuoteStyle, + Segment, SegmentSource, Span, SpecSource, Style, VideoSpec, Visual, Voice, + Waveform, Watermark, +) + + +@dataclass +class ImportResult: + spec: VideoSpec + # Media the note references that could not be used, in the order and + # wording segment() reports them. + warnings: List[str] = field(default_factory=list) + + +@dataclass +class _Part: + """One future shot, while the walk is still deciding what it holds.""" + + kind: ShotKind + media: Optional[Tuple[str, str]] = None # (url, "image" | "video") + media_label: Optional[str] = None # a readable name, when the file has one + path: Optional[str] = None # resolved, for segment()'s comparisons + label: str = "" + beats: List[Beat] = field(default_factory=list) + chapter: Optional[str] = None + card: Optional[Card] = None + role: Optional[str] = None + overlay: Optional[Any] = None # QuoteOverlay | CodeOverlay, not yet anchored + block_ids: List[str] = field(default_factory=list) + + def narration(self, heading_pause_ms: int) -> str: + return narration_text(self.beats, card=self.kind == "card", + heading_pause_ms=heading_pause_ms) + + def panel(self) -> bool: + """Whether segment()'s shot would carry quote_text or code_text.""" + if isinstance(self.overlay, CodeOverlay): + return True + if isinstance(self.overlay, QuoteOverlay): + return bool(strip_pause_markup(self.overlay.text)) + return False + + +def import_note( + content: str, + *, + user_id: str, + media_dir: str, + options: RenderOptions, + note_title: str = "", + author: str = "", + has_audio: Optional[Callable[[str], Any]] = None, + note_id: Optional[str] = None, +) -> ImportResult: + """Same arguments as `segment()`, plus the note's id for provenance.""" + warnings: List[str] = [] + parts = _walk(content, user_id=user_id, media_dir=media_dir, options=options, + note_title=note_title, author=author, has_audio=has_audio, + warnings=warnings) + assets = _Assets() + segments = _group(parts, assets, options.heading_pause_ms) + spec = VideoSpec( + title=note_title, + source=SpecSource(note_id=note_id) if note_id else None, + output=Output( + aspect=options.aspect, resolution=options.resolution, quality=options.quality, + fps=options.fps, subtitles=options.subtitles, chapters=options.embed_chapters, + thumbnail=options.thumbnail, + ), + voice=Voice( + voice=options.voice, speed=options.speed, + pauses=Pauses(paragraph_ms=options.paragraph_pause_ms, + heading_ms=options.heading_pause_ms, + segment_end_ms=options.shot_end_pause_ms), + ), + style=_style(options, assets), + chrome=_chrome(options, assets), + segments=segments, + audio_tracks=_music(options, segments, assets), + assets=assets.registry, + ) + return ImportResult(spec=spec, warnings=warnings) + + +def segment_via_spec( + content: str, + *, + user_id: str, + media_dir: str, + options: RenderOptions, + note_title: str = "", + author: str = "", + has_audio: Optional[Callable[[str], Any]] = None, +) -> Segmentation: + """`segment()` by way of a VideoSpec: same arguments, same shots and warnings.""" + imported = import_note(content, user_id=user_id, media_dir=media_dir, options=options, + note_title=note_title, author=author, has_audio=has_audio) + compiled = compile_spec(imported.spec, user_id=user_id, media_dir=media_dir) + return Segmentation(shots=compiled.segmentation.shots, + warnings=imported.warnings + compiled.segmentation.warnings) + + +# ── the walk (segmenter.segment(), recording beats) ────────────────────────── + +def _walk(content, *, user_id, media_dir, options, note_title, author, has_audio, + warnings) -> List[_Part]: + parts: List[_Part] = [] + heading_ms = options.heading_pause_ms + + try: + blocks = json.loads(content or "[]") + except (ValueError, TypeError): + blocks = [] + if not isinstance(blocks, list): + blocks = [] + + if options.title_card and (note_title or author): + parts.append(_Part( + kind="card", + card=Card(variant="title", title=note_title or "Untitled", subtitle=author), + chapter=note_title or None, + label="title card", + )) + + pending: List[Beat] = [] + open_part: Optional[_Part] = None + pending_chapter: Optional[str] = None + pending_card: Optional[_Part] = None + + def spoken() -> str: + return narration_text(pending, card=False, heading_pause_ms=heading_ms) + + def flush(next_part: Optional[_Part]) -> None: + nonlocal pending, open_part, pending_chapter, pending_card + text = spoken() + + if open_part is None and text: + open_part = _Part(kind="still", label="opening") + if open_part is not None: + open_part.beats = pending + if pending_chapter and not open_part.chapter: + open_part.chapter = pending_chapter + if pending_card is not None: + parts.append(pending_card) + pending_card = None + parts.append(open_part) + elif pending_card is not None: + parts.append(pending_card) + pending_card = None + + carried_chapter = None + if (pending_chapter and open_part is None + and next_part is not None and next_part.kind != "card"): + carried_chapter = pending_chapter + pending = [] + pending_chapter = None + open_part = next_part + if carried_chapter and open_part is not None and not open_part.chapter: + open_part.chapter = carried_chapter + + def carry(label: str, **extra: Any) -> _Part: + """A part over whatever the interrupted section was showing.""" + kind: ShotKind = ("video_muted" if open_part is not None and open_part.kind.startswith("video") + else "still") + if open_part is None: + return _Part(kind=kind, label=label, **extra) + return _Part(kind=kind, media=open_part.media, path=open_part.path, label=label, **extra) + + for block in _flatten(blocks): + kind = _classify(block, options) + block_id = block.get("id") if isinstance(block.get("id"), str) else None + + if kind is None: + btype = block.get("type") + if btype == "heading": + heading = _inline_text(block.get("content")).strip() + if heading: + if options.chapter_screens: + flush(carry("after chapter card")) + beats = ([Beat(id="", kind="heading", text=_as_sentence(heading), block_id=block_id)] + if options.read_chapters else []) + pending_card = _Part( + kind="card", card=Card(variant="chapter", title=heading), + chapter=heading, beats=beats, label="chapter card", + block_ids=[block_id] if block_id else [], + ) + pending_chapter = None + continue + if pending_chapter is None: + pending_chapter = heading + + if btype == "quote" and options.quotes.enabled: + quoted = _split_attribution(_inline_text(block.get("content"))) + if quoted[0]: + quote_part = carry("quote", overlay=QuoteOverlay( + id="", text=quoted[0], attribution=quoted[1] or None)) + after = carry("after quote") + if spoken() or pending_chapter is not None or pending_card is not None: + flush(quote_part) + else: + open_part = quote_part + pending.append(Beat(id="", kind="quote", text=_as_sentence(quoted[0]), block_id=block_id)) + flush(after) + continue + + if btype == "codeBlock": + code = _inline_text(block.get("content")) + if code.strip(): + code_part = carry("code", overlay=CodeOverlay(id="", text=code)) + after = carry("after code") + if spoken() or pending_chapter is not None or pending_card is not None: + flush(code_part) + else: + open_part = code_part + # Kept even when it isn't read, so the panel has a beat to + # hang on; `speak` decides whether it is heard. + pending.append(Beat(id="", kind="code", text=_as_sentence(code), + speak=options.narrate_code, block_id=block_id)) + flush(after) + continue + + if btype == "heading" and not options.read_chapters: + continue + + text = _block_narration(block, options) + if text.strip(): + pending.append(Beat(id="", kind="heading" if btype == "heading" else "prose", + text=_as_sentence(text), block_id=block_id)) + continue + + url = _media_url(block, options) + path = resolve_media_path(url, user_id, media_dir) if url else None + ids = [block_id] if block_id else [] + if path is None: + if url and not url.startswith("/media/"): + warnings.append(f"Skipped remote media: {url[:120]}") + elif url: + warnings.append(f"Skipped missing media: {url[:120]}") + flush(_Part(kind="still", label="fallback", block_ids=ids)) + continue + + if kind == "image": + flush(_Part(kind="still", media=(url, "image"), path=path, block_ids=ids)) + continue + + sounded = bool(has_audio(path)) if callable(has_audio) else False + if sounded: + flush(None) + parts.append(_Part(kind="video_sound", media=(url, "video"), path=path, + label="clip with audio", block_ids=ids)) + open_part = _Part(kind="video_muted", media=(url, "video"), path=path, + label="clip continued, muted") + else: + flush(_Part(kind="video_muted", media=(url, "video"), path=path, block_ids=ids)) + + flush(None) + + parts = [p for p in parts + if p.kind != "still" or p.path is not None or p.narration(heading_ms) + or isinstance(p.overlay, CodeOverlay)] + + trimmed: List[_Part] = [] + for part in parts: + previous = trimmed[-1] if trimmed else None + if (previous is not None and previous.panel() and not part.panel() + and not part.narration(heading_ms) + and part.chapter is None + and part.kind == previous.kind + and part.path == previous.path): + continue + trimmed.append(part) + parts = trimmed + + if parts: + intro = _bumper("intro", options, user_id, media_dir, warnings) + outro = _bumper("outro", options, user_id, media_dir, warnings) + if intro is not None: + parts.insert(0, intro) + if outro is not None: + parts.append(outro) + return parts + + +def _bumper(which, options, user_id, media_dir, warnings) -> Optional[_Part]: + bumper = options.intro if which == "intro" else options.outro + if not bumper.enabled or not bumper.url: + return None + path = resolve_media_path(bumper.url, user_id, media_dir) + if path is None or _media_kind(bumper.url) != "video": + warnings.append(f"The {which} was skipped: that clip could not be read.") + return None + return _Part(kind="video_sound", media=(bumper.url, "video"), path=path, + media_label=bumper.name or None, chapter=which.capitalize(), + role=which, label=which) + + +# ── parts -> segments ─────────────────────────────────────────────────────── + +class _Assets: + """The spec's asset registry: one key per distinct (url, type), in first use order.""" + + def __init__(self) -> None: + self.registry: Dict[str, NotesAsset] = {} + self._keys: Dict[Tuple[str, str], str] = {} + + def key(self, url: Optional[str], media_type: str, label: Optional[str] = None) -> Optional[str]: + if not url: + return None + found = self._keys.get((url, media_type)) + if found is None: + found = f"a{len(self._keys) + 1}" + self._keys[(url, media_type)] = found + self.registry[found] = NotesAsset(url=url, type=media_type, label=label or None) + return found + + +def _joins(segment_parts: List[_Part], part: _Part) -> bool: + """Whether `part` is the next shot of the scene `segment_parts` began. + + A quote or code shot interrupts a section over the same picture, and the + section then resumes over it — that resumption is the only kind of plain + shot that continues a scene. Anything carrying its own chapter mark starts + a segment of its own, since a segment's chapter is marked at its start. + """ + first, last = segment_parts[0], segment_parts[-1] + if first.kind not in ("still", "video_muted") or part.kind != first.kind: + return False + if part.path != first.path or part.chapter is not None or part.role or part.card: + return False + if part.overlay is not None: + return True + return last.overlay is not None and part.label == f"after {last.overlay.type}" + + +def _group(parts: List[_Part], assets: _Assets, heading_ms: int) -> List[Segment]: + groups: List[List[_Part]] = [] + for part in parts: + if groups and _joins(groups[-1], part): + groups[-1].append(part) + else: + groups.append([part]) + + segments: List[Segment] = [] + for index, group in enumerate(groups, start=1): + first = group[0] + segment_kind = {"card": "card", "video_sound": "clip"}.get(first.kind, "scene") + visual = None + if first.media is not None and first.path is not None: + url, media_type = first.media + visual = Visual(asset=assets.key(url, media_type, first.media_label), + audio="play" if segment_kind == "clip" else "mute") + + beats: List[Beat] = [] + overlays = [] + block_ids: List[str] = [] + for part in group: + block_ids.extend(part.block_ids) + block_ids.extend(b.block_id for b in part.beats if b.block_id) + part_beats = list(part.beats) + if not part_beats and part.overlay is None and len(group) > 1: + # A section with nothing to say, interrupted at once by a quote + # or code block, is still its own moment on screen. + part_beats = [Beat(id="", text="", speak=False)] + for beat in part_beats: + beats.append(beat.model_copy(update={"id": f"b{len(beats) + 1}"})) + if part.overlay is not None: + beat_id = beats[-1].id + overlays.append(part.overlay.model_copy(update={ + "id": f"o{len(overlays) + 1}", + "show": Span(from_=AnchorRef(beat=beat_id, edge="start"), + to=AnchorRef(beat=beat_id, edge="end")), + })) + + segments.append(Segment( + id=f"s{index}", + kind=segment_kind, + role=first.role or ("title" if first.card is not None and first.card.variant == "title" else None), + label=first.label, + chapter=first.chapter, + visual=visual, + card=first.card, + narration=Narration(beats=beats) if beats else None, + overlays=overlays, + chrome=False if first.role in ("intro", "outro") else None, + source=SegmentSource(block_ids=_dedupe(block_ids)) if block_ids else None, + )) + return segments + + +def _dedupe(values: List[str]) -> List[str]: + seen, out = set(), [] + for value in values: + if value not in seen: + seen.add(value) + out.append(value) + return out + + +# ── settings ──────────────────────────────────────────────────────────────── + +def _style(options: RenderOptions, assets: _Assets) -> Style: + fallback = options.fallback + return Style( + background=Background( + type=fallback.type, colors=list(fallback.colors), angle=fallback.angle, + asset=assets.key(fallback.url, "image"), + ), + fit=options.fit, + motion=Motion(effect=options.ken_burns.effect, amount=options.ken_burns.amount), + transition=options.transition.model_copy(), + min_segment_seconds=options.min_shot_seconds, + cards=Cards( + seconds=options.card_seconds, motion=options.ken_burns.include_cards, + title=options.title_card_text.model_copy(), + chapter=options.chapter_card_text.model_copy(), + ), + quote=QuoteStyle(**options.quotes.model_dump(exclude={"enabled"})), + code=options.code.model_copy(), + ) + + +def _chrome(options: RenderOptions, assets: _Assets) -> Chrome: + chrome = Chrome() + if options.watermark.enabled: + chrome.watermark = Watermark( + asset=assets.key(options.watermark.url, "image"), + **options.watermark.model_dump(exclude={"enabled", "url"}), + ) + if options.overlay_text.enabled: + chrome.overlay_text = OverlayText(**options.overlay_text.model_dump(exclude={"enabled"})) + if options.waveform.enabled: + chrome.waveform = Waveform(**options.waveform.model_dump(exclude={"enabled"})) + return chrome + + +def _music(options: RenderOptions, segments: List[Segment], assets: _Assets) -> List[AudioTrack]: + """Today's one bed: under everything but the intro and outro.""" + music = options.music + content = [s for s in segments if s.role not in ("intro", "outro")] + if not music.enabled or not music.url or not content: + return [] + return [AudioTrack( + id="m1", kind="music", asset=assets.key(music.url, "audio"), + start=AnchorRef(segment=content[0].id, edge="start"), + end=AnchorRef(segment=content[-1].id, edge="end"), + loop=True, volume=music.volume, duck_under_narration=music.duck, + fade_in=music.fade_in, fade_out=music.fade_out, + )] diff --git a/backend/app/video/spec.py b/backend/app/video/spec.py new file mode 100644 index 0000000..55619e6 --- /dev/null +++ b/backend/app/video/spec.py @@ -0,0 +1,435 @@ +"""The VideoSpec: a video described as data rather than inferred from a note. + +`segmenter` works a video out of a note's block layout every time it renders — +an image starts a shot, the text under it is read over it, a quote becomes its +own shot — and nothing of that structure exists between renders. This module is +that structure made explicit: ordered segments, the narration beats inside them, +overlays anchored to beats, audio tracks anchored to segments, and a registry of +the assets they all point at. It is the document Gecko Video Creator edits; see +docs/gecko-video-creator.md §5 for the design and the reasoning behind it. + +Phase 0 of that plan: `importer` turns a note into one of these, `compiler` +turns one back into the shots and options today's renderer consumes, and +tests/test_video_spec.py proves the round trip changes nothing. + +Pure data. Nothing here touches the filesystem, the database or a provider. +Limits are not repeated here: a spec reaches the renderer through +`RenderOptions`, whose validators clamp exactly as they always have. +""" + +from typing import Annotated, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from app.video.options import ( + Aspect, CardTextSpec, CodeSpec, Fit, KenBurnsEffect, OverlayTextMode, Position, + Quality, QuotePosition, Resolution, SubtitleMode, TransitionSpec, WaveMode, + WavePosition, +) + +SPEC_VERSION = "0.1" + + +class _Model(BaseModel): + # A spec is edited by hand and by an AI, so a misspelt field is an error + # rather than something silently ignored. `from` and `in` are Python + # keywords, hence the aliases below and populate_by_name. + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + +# ── assets ────────────────────────────────────────────────────────────────── + +MediaType = Literal["image", "video", "audio"] + + +class NotesAsset(_Model): + """A file in a note's own /media// directory.""" + + from_: Literal["notes"] = Field("notes", alias="from") + url: str + # How the note used it, which is not always what the extension says: a + # `videoFile` block is a video whatever its URL ends in. + type: MediaType + label: Optional[str] = None + + +class GamAsset(_Model): + """An asset or clip in Gecko Asset Manager.""" + + from_: Literal["gam"] = Field("gam", alias="from") + id: str + type: Optional[MediaType] = None + label: Optional[str] = None + + +class GeneratedAsset(_Model): + """A placeholder for media still to be generated (through GAM).""" + + from_: Literal["generate"] = Field("generate", alias="from") + type: Literal["image", "video", "music", "sfx"] + prompt: str + model: Optional[str] = None + base_assets: List[str] = Field(default_factory=list) + status: Literal["pending", "running", "ready", "failed"] = "pending" + result: Optional[GamAsset] = None + label: Optional[str] = None + + +class SearchAsset(_Model): + """A placeholder for footage still to be found in GAM.""" + + from_: Literal["search"] = Field("search", alias="from") + query: str + type: Optional[MediaType] = None + status: Literal["unresolved", "resolved"] = "unresolved" + result: Optional[GamAsset] = None + label: Optional[str] = None + + +AssetRef = Annotated[ + Union[NotesAsset, GamAsset, GeneratedAsset, SearchAsset], + Field(discriminator="from_"), +] + + +# ── project-wide settings ─────────────────────────────────────────────────── + +class Output(_Model): + aspect: Aspect = "16:9" + resolution: Resolution = "1080p" + quality: Quality = "standard" + fps: int = 30 + subtitles: SubtitleMode = "sidecar" + chapters: bool = True # embed chapter marks in the file + thumbnail: bool = True + + +class Pauses(_Model): + paragraph_ms: int = 350 + heading_ms: int = 800 + segment_end_ms: int = 600 + + +class Voice(_Model): + # None means "whatever the account is set to", which is what Notes renders + # with today; naming one pins it for the whole video. + provider: Optional[Literal["auto", "deepgram", "fal"]] = None + voice: Optional[str] = None + speed: float = 1.0 + expressivity: Optional[int] = None # Deepgram Flux only, -2…2 + pauses: Pauses = Field(default_factory=Pauses) + + +class VoiceOverride(_Model): + """Per-segment voice. Every field left unset inherits from `Voice`.""" + + provider: Optional[Literal["auto", "deepgram", "fal"]] = None + voice: Optional[str] = None + speed: Optional[float] = None + expressivity: Optional[int] = None + + +class Background(_Model): + """Shown behind a segment that has no visual of its own.""" + + type: Literal["gradient", "solid", "image"] = "gradient" + colors: List[str] = Field(default_factory=lambda: ["#1e293b", "#0f172a"]) + angle: int = 135 + asset: Optional[str] = None # used when type == "image" + + +class Motion(_Model): + effect: KenBurnsEffect = "none" # "alternate" only makes sense project-wide + amount: float = 0.12 + + +class Cards(_Model): + seconds: float = 3.5 + motion: bool = False # Ken Burns on cards (zoom only) + title: CardTextSpec = Field(default_factory=CardTextSpec) + chapter: CardTextSpec = Field(default_factory=CardTextSpec) + + +class QuoteStyle(_Model): + position: QuotePosition = "center" + size_pct: float = 4.2 + color: str = "#ffffff" + accent: str = "#818cf8" + scrim: float = 0.55 + + +class Style(_Model): + background: Background = Field(default_factory=Background) + fit: Fit = "blur" + motion: Motion = Field(default_factory=Motion) + transition: TransitionSpec = Field(default_factory=TransitionSpec) + min_segment_seconds: float = 2.5 + cards: Cards = Field(default_factory=Cards) + quote: QuoteStyle = Field(default_factory=QuoteStyle) + code: CodeSpec = Field(default_factory=CodeSpec) + + +class Watermark(_Model): + asset: Optional[str] = None + text: str = "" # empty: "by {title}" + position: Position = "bottom-right" + opacity: float = 0.85 + scale_pct: float = 6.0 + caption_pct: float = 2.3 + margin_pct: int = 4 + + +class OverlayText(_Model): + mode: OverlayTextMode = "fixed" + text: str = "" + position: Position = "bottom-left" + color: str = "#ffffff" + size_pct: float = 3.0 + margin_pct: int = 5 + shadow: bool = True + + +class Waveform(_Model): + mode: WaveMode = "line" + color: str = "#00ff41" + opacity: float = 0.7 + position: WavePosition = "bottom" + height_pct: int = 22 + scrim: float = 0.45 + + +class Chrome(_Model): + """Drawn over every segment that doesn't opt out. Present means on.""" + + watermark: Optional[Watermark] = None + overlay_text: Optional[OverlayText] = None + waveform: Optional[Waveform] = None + + +# ── time ──────────────────────────────────────────────────────────────────── + +class AnchorRef(_Model): + """A moment defined by the structure rather than by a number of seconds.""" + + segment: Optional[str] = None # inside a segment's overlay: that segment + beat: Optional[str] = None + edge: Literal["start", "end"] + offset: float = 0.0 # seconds; negative is before the edge + + +Anchor = Union[Literal["video_start", "video_end"], AnchorRef] + + +class Span(_Model): + from_: Anchor = Field(alias="from") + to: Anchor + + +class Trim(_Model): + in_: float = Field(alias="in") + out: Optional[float] = None + + +# ── segments ──────────────────────────────────────────────────────────────── + +class Visual(_Model): + asset: str + fit: Optional[Fit] = None + motion: Optional[Motion] = None + trim: Optional[Trim] = None + audio: Literal["mute", "play"] = "mute" + loop: bool = True + + +class Beat(_Model): + """One spoken unit — usually one block of the note.""" + + id: str + text: str # [pause:…] markup is kept and honoured + kind: Literal["prose", "heading", "quote", "code"] = "prose" + # False: the beat takes up time without being read — a code block that + # isn't narrated, or an empty `text` that just holds the picture. + speak: bool = True + hold: Optional[float] = None + pause_after_ms: Optional[int] = None + # The note block it was read from, so a re-sync can match it up again. + block_id: Optional[str] = None + + +class Narration(_Model): + beats: List[Beat] = Field(default_factory=list) + voice: Optional[VoiceOverride] = None + + +class _OverlayBase(_Model): + id: str + show: Optional[Span] = None # default: the whole segment + fade: Optional[float] = None + + +class QuoteOverlay(_OverlayBase): + type: Literal["quote"] = "quote" + text: str + attribution: Optional[str] = None + + +class CodeOverlay(_OverlayBase): + type: Literal["code"] = "code" + text: str + language: Optional[str] = None + + +class TextOverlay(_OverlayBase): + type: Literal["text"] = "text" + text: str + position: Position = "center" + size_pct: Optional[float] = None + color: Optional[str] = None + + +class LowerThirdOverlay(_OverlayBase): + type: Literal["lower_third"] = "lower_third" + title: str + subtitle: Optional[str] = None + + +class ImageOverlay(_OverlayBase): + type: Literal["image"] = "image" + asset: str + position: Position = "center" + scale_pct: Optional[float] = None + opacity: Optional[float] = None + + +Overlay = Annotated[ + Union[QuoteOverlay, CodeOverlay, TextOverlay, LowerThirdOverlay, ImageOverlay], + Field(discriminator="type"), +] + + +class Card(_Model): + variant: Literal["title", "chapter"] + title: str + subtitle: Optional[str] = None + + +class Duration(_Model): + min: Optional[float] = None + fixed: Optional[float] = None + + +class SegmentSource(_Model): + """The note blocks a segment was imported from, for re-syncing later.""" + + block_ids: List[str] = Field(default_factory=list) + + +class Segment(_Model): + id: str + # scene: a visual with narration over it. card: a title or chapter screen. + # clip: a video played with its own sound and no narration. + kind: Literal["scene", "card", "clip"] + role: Optional[Literal["title", "intro", "outro"]] = None + label: str = "" + chapter: Optional[str] = None # a chapter mark at this segment's start + visual: Optional[Visual] = None + card: Optional[Card] = None + narration: Optional[Narration] = None + overlays: List[Overlay] = Field(default_factory=list) + transition_in: Optional[TransitionSpec] = None + duration: Optional[Duration] = None + # None: the default, which is on except for an intro or outro. + chrome: Optional[bool] = None + source: Optional[SegmentSource] = None + + @property + def beats(self) -> List[Beat]: + return self.narration.beats if self.narration is not None else [] + + +class AudioTrack(_Model): + id: str + kind: Literal["music", "sfx", "voiceover"] = "music" + asset: str + start: Anchor + end: Optional[Anchor] = None # None: play once, to its natural end + trim: Optional[Trim] = None + loop: Optional[bool] = None # default: on for music + volume: float = 0.18 + duck_under_narration: Optional[bool] = None + fade_in: float = 0.0 + fade_out: float = 0.0 + + +class Credits(_Model): + auto: bool = True + card: bool = True + description: bool = True + extra: List[str] = Field(default_factory=list) + + +class SpecSource(_Model): + note_id: Optional[str] = None + note_version_id: Optional[str] = None + imported_at: Optional[str] = None + + +class VideoSpec(_Model): + spec_version: Literal["0.1"] = SPEC_VERSION + id: Optional[str] = None + title: str = "" + source: Optional[SpecSource] = None + output: Output = Field(default_factory=Output) + voice: Voice = Field(default_factory=Voice) + style: Style = Field(default_factory=Style) + chrome: Chrome = Field(default_factory=Chrome) + assets: Dict[str, AssetRef] = Field(default_factory=dict) + segments: List[Segment] = Field(default_factory=list) + audio_tracks: List[AudioTrack] = Field(default_factory=list) + credits: Optional[Credits] = None + + @model_validator(mode="after") + def _well_formed(self) -> "VideoSpec": + """The structural rules (§5.7). Anchors are checked at compile time: + one left pointing at a removed segment is a warning, not an error.""" + problems: List[str] = [] + + def unique(ids: List[str], what: str) -> None: + seen = set() + for value in ids: + if value in seen: + problems.append(f"duplicate {what} id {value!r}") + seen.add(value) + + def known(key: Optional[str], where: str) -> None: + if key is not None and key not in self.assets: + problems.append(f"{where} refers to unknown asset {key!r}") + + unique([s.id for s in self.segments], "segment") + unique([t.id for t in self.audio_tracks], "audio track") + known(self.style.background.asset, "style.background") + if self.chrome.watermark is not None: + known(self.chrome.watermark.asset, "chrome.watermark") + for key, asset in self.assets.items(): + for base in getattr(asset, "base_assets", []): + known(base, f"asset {key!r}") + for segment in self.segments: + where = f"segment {segment.id!r}" + unique([b.id for b in segment.beats], f"{where} beat") + unique([o.id for o in segment.overlays], f"{where} overlay") + if segment.visual is not None: + known(segment.visual.asset, where) + for overlay in segment.overlays: + if isinstance(overlay, ImageOverlay): + known(overlay.asset, f"{where} overlay {overlay.id!r}") + if segment.kind == "card" and segment.card is None: + problems.append(f"{where} is a card with no card") + if segment.kind != "card" and segment.card is not None: + problems.append(f"{where} has a card but is a {segment.kind}") + if segment.kind == "clip" and segment.visual is None: + problems.append(f"{where} is a clip with no visual") + for track in self.audio_tracks: + known(track.asset, f"audio track {track.id!r}") + if problems: + raise ValueError("; ".join(problems)) + return self diff --git a/backend/tests/test_video_spec.py b/backend/tests/test_video_spec.py new file mode 100644 index 0000000..79317de --- /dev/null +++ b/backend/tests/test_video_spec.py @@ -0,0 +1,411 @@ +"""The VideoSpec round trip: a note imported as a spec and compiled back must be +exactly what `segment()` makes of it — the same shots, field for field, and the +same warnings — and must drive the renderer to the same ffmpeg commands. + +That equivalence is Phase 0 of docs/gecko-video-creator.md: it proves the spec +can describe everything Notes already renders before anything is built on it. +Nothing here encodes video; ffmpeg is faked as in test_video_render_resume. +""" + +import inspect +import json +import os +import random +import re +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import test_video_segmenter as segmenter_cases +from test_video_render_resume import FakeFFmpeg + +from app.video import renderer +from app.video.compiler import IMPORT_ONLY_OPTIONS, UnsupportedSpec, compile_spec +from app.video.importer import import_note, segment_via_spec +from app.video.options import RenderOptions +from app.video.segmenter import segment +from app.video.spec import VideoSpec + +DOC = Path(__file__).resolve().parents[2] / "docs" / "gecko-video-creator.md" + + +def _assert_same(content, **kwargs): + want = segment(content, **kwargs) + got = segment_via_spec(content, **kwargs) + assert got.shots == want.shots + assert got.warnings == want.warnings + return want + + +def _round_trips_as_json(spec: VideoSpec) -> None: + dumped = spec.model_dump_json(by_alias=True, exclude_none=True) + assert VideoSpec.model_validate_json(dumped) == spec + + +# ── every case the segmenter's own tests already cover ────────────────────── + +_SEGMENTER_CASES = sorted( + name for name, fn in inspect.getmembers(segmenter_cases, inspect.isfunction) + if name.startswith("test_") +) + + +def test_the_segmenter_suite_is_all_reused(): + # If that file grows a fixture argument, it would silently stop being run + # below; this keeps the coverage honest. + assert len(_SEGMENTER_CASES) >= 60 + for name in _SEGMENTER_CASES: + assert not inspect.signature(getattr(segmenter_cases, name)).parameters, name + + +@pytest.mark.parametrize("name", _SEGMENTER_CASES) +def test_every_segmenter_case_compiles_back_to_the_same_shots(name, monkeypatch): + calls = [] + + def checked(content, **kwargs): + calls.append(1) + result = _assert_same(content, **kwargs) + _round_trips_as_json(import_note(content, **kwargs).spec) + return result + + monkeypatch.setattr(segmenter_cases, "segment", checked) + case = getattr(segmenter_cases, name) + case() + if "_run(" in inspect.getsource(case) or "segment(" in inspect.getsource(case): + assert calls, f"{name} never reached the patched segment()" + + +# ── a generated corpus, aimed at the walk's corners ───────────────────────── + +_FILES = ("a.png", "b.png", "loud.mp4", "quiet.mp4", "intro.mp4", "outro.mp4", "song.mp3", "logo.png") + + +def _media_root(tmp_path): + (tmp_path / "u1").mkdir(exist_ok=True) + for name in _FILES: + (tmp_path / "u1" / name).write_bytes(b"x") + return str(tmp_path) + + +def _random_text(rng): + words = ["salt", "road", "caravan", "dawn", "market", "well", "camel", "ruin"] + text = " ".join(rng.choice(words) for _ in range(rng.randint(1, 6))) + return rng.choice([ + text, text.capitalize() + ".", text + "...", text + " [pause]", "[pause:long]", + text + " [pause:1.5s] " + text, "", " ", text + "\n" + text, text + ":", + ]) + + +def _inline(rng, text): + if rng.random() < 0.2: + return [{"type": "link", "href": "https://example.com", + "content": [{"type": "text", "text": text}]}] + return [{"type": "text", "text": text}] + + +def _random_block(rng, depth=0): + choice = rng.choice([ + "paragraph", "paragraph", "paragraph", "heading", "heading", "quote", "quote", + "codeBlock", "bulletListItem", "table", "image", "image", "video", "videoFile", + "file", "diagram", "childNote", "audioFile", "divider", + ]) + block = {"id": f"blk{rng.randint(0, 10**6)}", "type": choice} + if choice in ("paragraph", "heading", "bulletListItem", "codeBlock"): + block["content"] = _inline(rng, _random_text(rng)) + elif choice == "quote": + body = _random_text(rng) + body = rng.choice([ + body, f"{body} — Ibn Somebody", f"{body}\n- the guide", f"{body} - an old saying", + f"{body} – 1971.", "[pause]", f"{body} -- unnamed source of very many words here", + ]) + block["content"] = _inline(rng, body) + elif choice == "table": + block["content"] = {"rows": [{"cells": [_inline(rng, _random_text(rng)) for _ in range(2)]} + for _ in range(rng.randint(0, 2))]} + elif choice in ("image", "video", "videoFile", "file"): + block["props"] = {"url": rng.choice([ + "/media/u1/a.png", "/media/u1/b.png", "/media/u1/loud.mp4", "/media/u1/quiet.mp4", + "/media/u1/a.png?v=2", "/media/u1/missing.png", "/media/u2/a.png", + "https://example.com/x.png", "/media/u1/notes.pdf", "", + ])} + elif choice == "diagram": + block["id"] = rng.choice(["d1", "d2"]) + if depth < 2 and rng.random() < 0.15: + block["children"] = [_random_block(rng, depth + 1) for _ in range(rng.randint(1, 3))] + return block + + +def _random_options(rng): + bumper = lambda name: rng.choice([ + {}, {"enabled": True, "url": f"/media/u1/{name}", "name": f"My {name}"}, + {"enabled": True, "url": "/media/u1/missing.mp4"}, {"enabled": True, "url": "/media/u1/a.png"}, + {"enabled": False, "url": f"/media/u1/{name}"}, + ]) + return RenderOptions( + title_card=rng.random() < 0.5, + chapter_screens=rng.random() < 0.4, + read_chapters=rng.random() < 0.7, + narrate_code=rng.random() < 0.5, + heading_pause_ms=rng.choice([0, 800]), + quotes={"enabled": rng.random() < 0.7}, + intro=bumper("intro.mp4"), + outro=bumper("outro.mp4"), + music=rng.choice([{}, {"enabled": True, "url": "/media/u1/song.mp3"}]), + diagram_images=rng.choice([{}, {"d1": "/media/u1/b.png"}, {"d1": "/media/u1/gone.png"}]), + ) + + +def test_a_generated_corpus_compiles_back_to_the_same_shots(tmp_path): + root = _media_root(tmp_path) + rng = random.Random(20261015) + for _ in range(3000): + blocks = [_random_block(rng) for _ in range(rng.randint(0, 12))] + kwargs = dict( + user_id="u1", media_dir=root, options=_random_options(rng), + note_title=rng.choice(["", "The Salt Road", "Road [pause:2s]"]), + author=rng.choice(["", "gecko"]), + has_audio=lambda path: os.path.basename(path) == "loud.mp4", + ) + content = json.dumps(blocks) + try: + _assert_same(content, **kwargs) + except AssertionError: + pytest.fail(f"round trip differs for:\n{json.dumps(blocks, indent=1)}\n{kwargs['options']}") + _round_trips_as_json(import_note(content, **kwargs).spec) + + +def test_unreadable_content_imports_as_an_empty_video(tmp_path): + root = _media_root(tmp_path) + for content in ("", "not json", "{}", "[1, null, \"x\"]"): + _assert_same(content, user_id="u1", media_dir=root, options=RenderOptions()) + + +# ── the options survive the trip ──────────────────────────────────────────── + +_DISABLEABLE = ("waveform", "watermark", "overlay_text", "music") + + +def _effective(options: RenderOptions) -> dict: + """What the renderer can tell apart. A disabled feature's settings are + inert, and the import-only fields are recorded as structure instead.""" + data = options.model_dump(exclude=set(IMPORT_ONLY_OPTIONS)) + data["quotes"].pop("enabled") + if options.music.enabled and not options.music.url: + data["music"]["enabled"] = False + for name in _DISABLEABLE: + if not data[name]["enabled"]: + data[name] = getattr(RenderOptions(), name).model_dump() + return data + + +def _random_render_options(rng): + return RenderOptions( + aspect=rng.choice(["16:9", "9:16", "1:1"]), resolution=rng.choice(["720p", "1080p", "4k"]), + quality=rng.choice(["preview", "standard", "high"]), fps=rng.choice([12, 24, 30, 60, 999]), + fit=rng.choice(["blur", "pad", "crop"]), + fallback=rng.choice([{}, {"type": "solid", "colors": ["#123456"]}, + {"type": "image", "url": "/media/u1/a.png", "angle": 45}, + {"type": "gradient", "url": "/media/u1/b.png"}]), + waveform={"enabled": rng.random() < 0.5, "mode": rng.choice(["line", "point"]), + "height_pct": rng.randint(5, 60), "scrim": rng.random()}, + watermark={"enabled": rng.random() < 0.5, "url": rng.choice([None, "/media/u1/logo.png"]), + "text": rng.choice(["", "geckopico"]), "position": "top-left", + "scale_pct": rng.uniform(0, 40), "margin_pct": rng.randint(0, 9)}, + overlay_text={"enabled": rng.random() < 0.5, "mode": rng.choice(["fixed", "title", "title_chapter"]), + "text": "hello", "shadow": rng.random() < 0.5, "size_pct": rng.uniform(0, 25)}, + title_card_text={"title_pct": rng.uniform(0, 30)}, chapter_card_text={"subtitle_pct": 4.0}, + transition={"style": rng.choice(["none", "fade", "dissolve"]), "duration": rng.uniform(0, 4)}, + ken_burns={"effect": rng.choice(["none", "alternate", "pan_up"]), "amount": rng.uniform(0, 1), + "include_cards": rng.random() < 0.5}, + music=rng.choice([{}, {"enabled": True, "url": "/media/u1/song.mp3", "volume": 0.3, + "duck": False, "fade_in": 0.0, "fade_out": 20}, + {"enabled": True}, {"enabled": False, "url": "/media/u1/song.mp3"}]), + quotes={"enabled": rng.random() < 0.5, "position": "top", "accent": "#000000", "size_pct": 99}, + code={"position": "bottom", "scrim": 0.1}, + subtitles=rng.choice(["off", "sidecar", "soft", "burn"]), + embed_chapters=rng.random() < 0.5, thumbnail=rng.random() < 0.5, + voice=rng.choice([None, "flux-haley-en"]), speed=rng.uniform(0, 3), + paragraph_pause_ms=rng.randint(0, 4000), heading_pause_ms=rng.randint(0, 6000), + shot_end_pause_ms=rng.randint(0, 4000), min_shot_seconds=rng.uniform(0, 40), + card_seconds=rng.uniform(0, 40), + ) + + +def test_render_options_survive_the_round_trip(tmp_path): + root = _media_root(tmp_path) + note = json.dumps([{"id": "1", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "2", "type": "paragraph", "content": [{"type": "text", "text": "Words."}]}]) + rng = random.Random(7) + for _ in range(300): + options = _random_render_options(rng) + spec = import_note(note, user_id="u1", media_dir=root, options=options).spec + compiled = compile_spec(spec, user_id="u1", media_dir=root).options + assert _effective(compiled) == _effective(options) + _round_trips_as_json(spec) + + +def test_bumpers_and_quotes_come_back_as_the_options_that_made_them(tmp_path): + root = _media_root(tmp_path) + options = RenderOptions(intro={"enabled": True, "url": "/media/u1/intro.mp4", "name": "Sting"}, + quotes={"enabled": True}) + note = json.dumps([{"id": "1", "type": "quote", + "content": [{"type": "text", "text": "Worth its weight — A Trader"}]}]) + spec = import_note(note, user_id="u1", media_dir=root, options=options).spec + compiled = compile_spec(spec, user_id="u1", media_dir=root).options + assert compiled.intro == options.intro + assert compiled.quotes.enabled + assert spec.assets[spec.segments[0].visual.asset].label == "Sting" + + +# ── the renderer can't tell the difference ────────────────────────────────── + +def _render_commands(monkeypatch, media, content, options, *, via_spec): + ff = FakeFFmpeg(monkeypatch) + monkeypatch.setattr(renderer.F, "probe_has_audio", lambda path: path.endswith("loud.mp4")) + if via_spec: + imported = import_note(content, user_id="u1", media_dir=media, options=options, + note_title="Salt", author="gecko", + has_audio=lambda path: path.endswith("loud.mp4")) + compiled = compile_spec(imported.spec, user_id="u1", media_dir=media) + plan = compiled.segmentation + plan.warnings[:0] = imported.warnings + monkeypatch.setattr(renderer, "segment", lambda *a, **k: plan) + options = compiled.options + result = renderer.render( + job_id="j", user_id="u1", media_dir=media, note_content=content, + note_title="Salt", author="gecko", options=options, preview=True, + tts=lambda text: b"audio", progress=lambda *a: None, + max_shots=200, max_narration_chars=100_000, + ) + monkeypatch.undo() + # Output names are random per render; everything else must match exactly. + hexes = {result.video_filename.split(".")[0]} + return [[re.sub("|".join(hexes), "OUT", arg) for arg in call] for call in ff.calls], result.warnings + + +_RENDER_NOTE = json.dumps([ + {"id": "1", "type": "paragraph", "content": [{"type": "text", "text": "Before any picture."}]}, + {"id": "2", "type": "heading", "content": [{"type": "text", "text": "The road [pause]"}]}, + {"id": "3", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "4", "type": "paragraph", "content": [{"type": "text", "text": "Under the map."}]}, + {"id": "5", "type": "quote", "content": [{"type": "text", "text": "Salt was gold — A Trader"}]}, + {"id": "6", "type": "paragraph", "content": [{"type": "text", "text": "After the quote."}]}, + {"id": "7", "type": "videoFile", "props": {"url": "/media/u1/loud.mp4"}}, + {"id": "8", "type": "paragraph", "content": [{"type": "text", "text": "Over the clip, muted."}]}, + {"id": "9", "type": "codeBlock", "content": [{"type": "text", "text": "print('salt')"}]}, + {"id": "10", "type": "heading", "content": [{"type": "text", "text": "Ruins"}]}, + {"id": "11", "type": "image", "props": {"url": "/media/u1/missing.png"}}, + {"id": "12", "type": "paragraph", "content": [{"type": "text", "text": "Nothing left."}]}, +]) + + +@pytest.mark.parametrize("overrides", [ + {}, + {"chapter_screens": True, "read_chapters": False, "narrate_code": True}, + # Ken Burns is left to the options round trip above: a drifting still is + # prescaled to ~24 megapixels with Pillow, which costs seconds per shot here. + {"transition": {"style": "dissolve", "duration": 0.5}, + "waveform": {"enabled": True}, "subtitles": "burn"}, + {"intro": {"enabled": True, "url": "/media/u1/intro.mp4"}, + "outro": {"enabled": True, "url": "/media/u1/outro.mp4"}, + "music": {"enabled": True, "url": "/media/u1/song.mp3"}, + "watermark": {"enabled": True, "url": "/media/u1/logo.png"}, + "overlay_text": {"enabled": True, "mode": "title_chapter"}, + "fallback": {"type": "image", "url": "/media/u1/b.png"}}, + {"watermark": {"enabled": False, "url": "/media/u1/logo.png", "text": "hidden"}, + "music": {"enabled": False, "url": "/media/u1/song.mp3"}, "transition": {"style": "fade"}}, +]) +def test_the_renderer_issues_the_same_ffmpeg_commands_either_way(tmp_path, monkeypatch, overrides): + media = _media_root(tmp_path) + options = RenderOptions(quotes={"enabled": True}, **overrides) + want = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=False) + got = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=True) + assert got == want + assert len(want[0]) > 5 + + +def test_music_switched_on_with_no_track_is_not_part_of_the_video(tmp_path, monkeypatch): + # The one thing the trip deliberately drops. A spec's music is a track, and + # a track names its file, so "music on, nothing chosen" has no spec form: + # the video is identical, and only the renderer's note that it skipped the + # music goes. Notes itself still renders from the original options. + media = _media_root(tmp_path) + options = RenderOptions(music={"enabled": True}) + want_calls, want_warnings = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=False) + got_calls, got_warnings = _render_commands(monkeypatch, media, _RENDER_NOTE, options, via_spec=True) + assert got_calls == want_calls + assert want_warnings == got_warnings + ["Background music was skipped: that track could not be read."] + + +# ── what a spec says, and what the Notes renderer refuses ─────────────────── + +def test_the_worked_example_in_the_design_doc_is_a_valid_spec(): + example = re.findall(r"```json\n(.*?)```", DOC.read_text(), re.S)[0] + spec = VideoSpec.model_validate_json(example) + assert [s.kind for s in spec.segments] == ["card", "clip", "scene", "scene", "scene", "scene", "clip"] + _round_trips_as_json(spec) + with pytest.raises(UnsupportedSpec, match="Gecko Video Creator"): + compile_spec(spec, user_id="u1", media_dir="/nowhere") + + +def test_a_quote_is_an_overlay_on_its_own_beat_not_a_segment(tmp_path): + root = _media_root(tmp_path) + note = json.dumps([ + {"id": "img", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "p1", "type": "paragraph", "content": [{"type": "text", "text": "Before"}]}, + {"id": "q1", "type": "quote", "content": [{"type": "text", "text": "Salt is gold — A Trader"}]}, + {"id": "p2", "type": "paragraph", "content": [{"type": "text", "text": "After"}]}, + ]) + spec = import_note(note, user_id="u1", media_dir=root, note_id="n1", + options=RenderOptions(title_card=False, quotes={"enabled": True})).spec + assert spec.source.note_id == "n1" + [scene] = spec.segments + assert [(b.id, b.kind, b.text) for b in scene.beats] == [ + ("b1", "prose", "Before."), ("b2", "quote", "Salt is gold."), ("b3", "prose", "After."), + ] + [quote] = scene.overlays + assert (quote.type, quote.text, quote.attribution) == ("quote", "Salt is gold", "A Trader") + assert (quote.show.from_.beat, quote.show.to.beat) == ("b2", "b2") + assert scene.source.block_ids == ["img", "p1", "q1", "p2"] + assert [b.block_id for b in scene.beats] == ["p1", "q1", "p2"] + + +def _spec(**changes): + base = { + "assets": {"a1": {"from": "notes", "url": "/media/u1/a.png", "type": "image"}, + "g1": {"from": "gam", "id": "abc"}}, + "segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "a1"}, + "narration": {"beats": [{"id": "b1", "text": "One."}, {"id": "b2", "text": "Two."}]}}], + } + base.update(changes) + return VideoSpec.model_validate(base) + + +@pytest.mark.parametrize("change, message", [ + ({"segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "g1"}}]}, "not a file in the note"), + ({"segments": [{"id": "s1", "kind": "scene", "transition_in": {"style": "fade"}}]}, "transition_in"), + ({"segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "a1", "trim": {"in": 1}}}]}, "trimmed"), + ({"segments": [{"id": "s1", "kind": "scene", "narration": {"beats": [{"id": "b1", "text": "x"}]}, + "overlays": [{"id": "o1", "type": "lower_third", "title": "Guide"}]}]}, "lower_third"), + ({"segments": [{"id": "s1", "kind": "scene", "narration": {"beats": [{"id": "b1", "text": "x"}]}, + "overlays": [{"id": "o1", "type": "quote", "text": "x"}]}]}, "exactly one of its beats"), + ({"voice": {"provider": "fal"}}, "voice.provider"), + ({"audio_tracks": [{"id": "t1", "asset": "a1", "start": "video_start", "end": "video_end"}]}, + "audio file"), +]) +def test_what_the_notes_renderer_cannot_draw_is_refused(change, message): + with pytest.raises(UnsupportedSpec, match=message): + compile_spec(_spec(**change), user_id="u1", media_dir="/nowhere") + + +@pytest.mark.parametrize("change, message", [ + ({"segments": [{"id": "s1", "kind": "scene"}, {"id": "s1", "kind": "scene"}]}, "duplicate segment"), + ({"segments": [{"id": "s1", "kind": "scene", "visual": {"asset": "nope"}}]}, "unknown asset"), + ({"segments": [{"id": "s1", "kind": "card"}]}, "card with no card"), + ({"segments": [{"id": "s1", "kind": "clip"}]}, "clip with no visual"), + ({"segments": [{"id": "s1", "kind": "scene", "colour": "red"}]}, "Extra inputs"), +]) +def test_a_malformed_spec_is_rejected(change, message): + with pytest.raises(ValidationError, match=message): + _spec(**change) diff --git a/docs/gecko-video-creator.md b/docs/gecko-video-creator.md index ec20311..5079982 100644 --- a/docs/gecko-video-creator.md +++ b/docs/gecko-video-creator.md @@ -1,6 +1,6 @@ # Gecko Video Creator (GVC) — Product & Data Model Spec -**Status:** Draft for discussion +**Status:** Draft for discussion · Phase 0 in progress (§10) **Date:** 2026-10-07 **Scope:** What GVC is, the JSON document a video is described by (the **VideoSpec**), how an AI builds and edits one, and how GVC pulls scripts from Gecko Notes and media @@ -159,7 +159,7 @@ transitions, watermark, waveform, voice) is a **global** field on `RenderOptions | Structure is **re-inferred every render** | Nothing to edit, version or hand to an AI. The only way to change the video is to change the note | | Media must be in the note's own `/media/` | No library search, no generation inside the video workflow | -Four existing defects should be fixed on the way in rather than carried over: +Six existing defects should be fixed on the way in rather than carried over: - **The Voice picker does nothing.** `VideoGenModal.tsx` offers one and writes `options.voice`, but no backend code reads it. The voice always comes from the account @@ -172,6 +172,16 @@ Four existing defects should be fixed on the way in rather than carried over: maxes out at 10 versus 30, and there are a few more. - **Some settings have no UI at all:** `fps`, `fallback.angle`, waveform opacity and scrim, watermark opacity and margin, overlay margin and shadow. +- **A chapter mark can vanish.** With headings not read aloud, a heading followed by a + picture and then straight away a quote loses its chapter: the mark is carried onto the + picture's shot, and the quote then takes that shot over (`segmenter.py:461`). +- **Pause markers leak into chapter marks.** `Shot.__post_init__` strips `[pause:…]` + from drawn fields set when a shot is built, but a section's chapter is set afterwards + (`segmenter.py:365`, `:384`), so `The Road [pause:2s]` lands in the MP4's chapter list + and the "title + chapter" overlay as typed. + +Phase 0 reproduces all of these exactly, so its parity tests can hold the spec path to +today's output; each is a small follow-up fix once the spec path is the only one. --- @@ -214,9 +224,11 @@ Four existing defects should be fixed on the way in rather than carried over: ## 5. The VideoSpec (draft schema) -Written as TypeScript-style types because they're the most readable form. The real -source of truth will be Pydantic (principle 9). Defaults shown are today's -`RenderOptions` defaults. +Written as TypeScript-style types because they're the most readable form. The +canonical definition is now Pydantic, in +[`backend/app/video/spec.py`](../backend/app/video/spec.py) (principle 9). Where the two +differ, the code wins. Defaults shown are today's `RenderOptions` defaults. Not yet +modelled: per-overlay style overrides. ### 5.1 Top level @@ -343,7 +355,7 @@ interface Segment { transition_in?: Transition; // overrides style.transition for the cut INTO this segment duration?: { min?: number; fixed?: number }; // default: automatic chrome?: boolean; // false = no watermark, overlay text or waveform - source?: { note_id: string; block_ids: string[] }; // provenance, for re-sync + source?: { block_ids: string[] }; // provenance, for re-sync (the note is spec.source) } interface Visual { @@ -420,7 +432,9 @@ what the current renderer does around intro/outro bumpers. ```ts type AssetRef = { label?: string } & ( // label: a readable name for people and the AI | { from: "gam"; id: string } // an asset or clip in GAM - | { from: "notes"; url: string } // "/media//.png" from a note + | { from: "notes"; url: string; // "/media//.png" from a note + type: "image" | "video" | "audio" } // as the note used it: a videoFile block + // is a video whatever its extension | { from: "generate"; // to be generated, via GAM type: "image" | "video" | "music" | "sfx"; prompt: string; model?: string; base_assets?: AssetKey[]; @@ -572,7 +586,7 @@ bell sound effect lands on one line. "a_logo": { "from": "gam", "id": "0b6f6c3e-2d1a-4e8b-9c55-7a3e1f0d2b44", "label": "Gecko logo" }, "a_intro": { "from": "gam", "id": "5d2e9a17-6b3c-4f80-a1d4-c8e7f2b0961a", "label": "Channel intro sting" }, "a_outro": { "from": "gam", "id": "e41c7b90-3a2d-4f6e-8b15-92d0c4a7f3e8", "label": "Channel outro" }, - "a_map": { "from": "notes", "url": "/media/9c41e2f0-1b7d-4a3e-8f62-d05a7c3b1e98/2f8a6c1d-4e3b-4b7a-9e0f-5c2d1a8b7e6f.png", "label": "Hand-drawn route map" }, + "a_map": { "from": "notes", "url": "/media/9c41e2f0-1b7d-4a3e-8f62-d05a7c3b1e98/2f8a6c1d-4e3b-4b7a-9e0f-5c2d1a8b7e6f.png", "type": "image", "label": "Hand-drawn route map" }, "a_caravan": { "from": "gam", "id": "8e3b7c21-4d5f-4a90-b6e2-1c7d9f0a3b58", "label": "Archive interview reel, 1971" }, "a_market": { "from": "generate", @@ -620,7 +634,7 @@ bell sound effect lands on one line. { "id": "b3", "text": "One route carried more of it than any other." } ] }, - "source": { "note_id": "6b1e0c2a-4f7d-4c1e-9a55-3f0d2b8e7a10", "block_ids": ["blk-01", "blk-02", "blk-03", "blk-04"] } + "source": { "block_ids": ["blk-01", "blk-02", "blk-03", "blk-04"] } }, { "id": "s_caravan", @@ -888,12 +902,15 @@ or extract. | Media embedded in the note | `GET /api/notes/{id}/assets`; files at `/media/{user}/{uuid}.ext` | ✅ | | Note as Markdown for the AI | expose `note_to_markdown`, e.g. `GET /api/notes/{id}?format=markdown` | ➕ small | | "Make video" in Notes | a button / slash-menu item → `https://gvc.geckopico.com/new?note={id}` (replaces today's dialog) | ➕ small | -| Cross-origin calls | add `https://gvc.geckopico.com` to `CORS_ORIGIN` and the CSP in [`frontend/nginx.conf`](../frontend/nginx.conf) | ➕ config | +| Cross-origin calls | add `https://gvc.geckopico.com` to `CORS_ORIGIN` and the CSP in [`frontend/nginx.conf`](../frontend/nginx.conf), only where GVC's browser code calls Notes directly (server-side calls with a Bearer token need neither, as in GAM) | ➕ config | +| Speech for GVC's renders | `POST /api/settings/speech/tts`, plus a `provider` field passed through to `synthesize_tts_bytes(provider_override=…)`, so one video can't switch voices partway | ➕ small | +| Put the finished video back in the note | No endpoint can write `Note.content` safely against the editor's autosave (`routers/assets.py:174`), and today's auto-insert relies on Notes' own job registry | ➕ needs design (open decision 11) | -**Re-sync.** Every imported segment records the `block_ids` it came from. When the note's +**Re-sync.** Every imported segment records the `block_ids` it came from, and every beat +the block it was read from (`Beat.block_id`). When the note's `modified_at` is newer than the spec's `imported_at`, GVC shows a per-segment diff: - **Changed text** updates the matching beats in place. Overlays anchored to those beats - keep working, because beat IDs follow block IDs. + keep working, because a beat keeps its ID and is matched to its block by `block_id`. - **New blocks** are offered as new beats or segments. - **Deleted blocks** are flagged, not removed. @@ -910,7 +927,7 @@ A segment edited in GVC is never silently overwritten by a re-sync (open decisio | Credits | structured attribution fields on every asset | ✅ (GAM M10) | | Generate images / video / audio | fal.ai through GAM, with prompt and base assets recorded | 🟡 planned (GAM M8) | | Read media bytes for rendering | signed `/media/{key}?exp=…&sig=…` URLs | ✅ (open decision 2) | -| Store the render | upload with `source: "gvc_export"`, `origin_project_id` = spec `id` | 🟡 fields exist on the model; the upload path needs wiring | +| Store the render | upload with `source: "gvc_export"`, `origin_project_id` = spec `id` | 🟡 `gvc_export` is a known source, but `origin_project_id` / `origin_reference_id` are only planned (GAM's `plan-of-attack.md`), not on the model yet; they need a migration, plus the upload path | GVC is what unblocks GAM's M8 and M9. Both were deliberately left waiting for a real consumer. @@ -924,7 +941,11 @@ consumer. - React 18 + Vite + TypeScript + Zustand + Tailwind - Docker Compose + Nginx - served at `gvc.geckopico.com` behind Caddy on the shared `web` network, exactly like - Notes' `docker-compose.prod.yml` + Notes' `docker-compose.prod.yml`. Only the frontend joins `web`, and the backend + service is named `gvc-backend`: Notes and GAM both already put a service called + `backend` on `web`, and each app's nginx proxies to `backend:8000`, so a third would + make that name more ambiguous still. (Check the live host before adding GVC: + `docker compose exec frontend getent hosts backend` should return one address.) **Data:** - `Project`: the current VideoSpec as JSON, plus title, source note and owner @@ -949,15 +970,35 @@ consumer. - the AI chat alongside, with a suggestions tray - a raw JSON tab for power use +### Where the code lives + +Decided after weighing the three options against the code (2026-10-14): **Phase 0 +happens here, in gecko-notes; GVC itself starts in its own repo, `davior/gvc`**, built +from GAM's skeleton (its `config.py`, auth with a shadow user, Alembic, three-job CI, +Python 3.13). The reasons, briefly: + +- About 75% of the video code (3,441 of 4,582 lines) imports nothing from Notes and can + be copied as it is. The rest (`worker.py`, `routers/video.py`, the segmenter's media + seams) has to be rewritten against HTTP wherever GVC lives, so building it inside + Notes saves nothing and makes reaching into Notes' database too easy. +- Notes' repo is set up for one app: its CLAUDE.md deploy steps, CI without path + filters, one `main`, one rollback, and a backup of `notes.db` only. +- The video engine is still changing (six commits in the three weeks before the spec). + Copying it once, after Phase 0 has made it spec-driven, keeps the period with two + copies short. + +From the day the engine is copied into GVC, **new video features in Notes are frozen**: +bug fixes only, each logged so it can be ported. + ### Phases | Phase | What | Done when | |---|---|---| -| **0 — Spec inside Notes** | Define the VideoSpec in Pydantic. `segmenter.py` emits it, a compiler turns it into today's `Shot` list, and `renderer.py` is untouched. Expose `GET /api/video/spec?note_id=` for inspection. | Golden tests: a corpus of notes renders byte-identical (or frame-identical) through the spec path | -| **1 — GVC renders what Notes renders** | New repo `davior/gvc`. SSO, project storage and history, import from Notes, the lifted engine, preview and full render, output to GAM. | Any note renders in GVC the same as in Notes | +| **0 — Spec inside Notes** | **In progress.** First, alongside today's code: the VideoSpec in Pydantic ([`spec.py`](../backend/app/video/spec.py)), an importer that walks a note exactly as `segment()` does ([`importer.py`](../backend/app/video/importer.py)), and a compiler back to today's `Shot`s and `RenderOptions` ([`compiler.py`](../backend/app/video/compiler.py)), with `renderer.py` and `segment()` untouched. Then, separately: `segment()` switches to the spec path, `GET /api/video/spec?note_id=` exposes it, and the voice bug is fixed. | [`tests/test_video_spec.py`](../backend/tests/test_video_spec.py): every segmenter test case, a few thousand generated notes and the renderer's ffmpeg commands come out identical through the spec path, with no real encodes | +| **1 — GVC renders what Notes renders** | New repo `davior/gvc` from GAM's skeleton. The engine copied from the Phase 0 merge, SSO, project storage and history, import from Notes over HTTP, speech through Notes, preview and full render, download. Output to GAM and back into the note follow once those are designed. | Any note renders in GVC the same as in Notes | | **2 — The new capabilities** | Timed overlays, multiple audio tracks, trims, per-segment voice/motion/transition, GAM search and picker, `generate` / `search` placeholders, credits | The §6 example renders | | **3 — AI editing** | The operations of §8.2, the suggestions tray, chat, staged review | "Make a video from this note" produces a reviewed, rendered video | -| **4 — Notes hands over** | Notes' dialog becomes "Open in GVC"; `backend/app/video/` and `VideoGenModal.tsx` are removed from Notes; existing render jobs and outputs keep working | No video code left in Notes | +| **4 — Notes hands over** | Notes' dialog becomes "Open in GVC"; `VideoGenModal.tsx` and the render pipeline are removed from Notes; existing render jobs and outputs keep working. The speech pieces read-aloud export shares (`ffmpeg.py`'s helpers, `narration.py`'s stitching, `pause_markup.py`, imported at `routers/settings.py:26-28`) move to an `app/speech/` package and stay | No render code left in Notes | Phase 0 is the cheap insurance. If the schema can't faithfully describe what Notes already renders, it's wrong, and finding that out costs a branch, not a new app. @@ -982,7 +1023,7 @@ already renders, it's wrong, and finding that out costs a branch, not a new app. **In GAM** (`davior/gam`): - `GET /api/assets`, `GET /api/assets/{id}`, `GET /api/assets/{id}/clips`, `GET /api/search`, signed `/media/{key}`. - M8 (generation) and M9 (GVC surface, embeddable picker): both waiting on GVC. -- `Asset.source = "gvc_export"`, `origin_project_id`, `origin_reference_id`: already on the model. +- `Asset.source = "gvc_export"` is already a known source. `origin_project_id` and `origin_reference_id` are planned in GAM's `plan-of-attack.md` but not on the model yet: a migration GVC needs before it can store renders there. --- @@ -1017,8 +1058,12 @@ already renders, it's wrong, and finding that out costs a branch, not a new app. block a render or only warn? Should the credits card be on by default? 10. **Per-beat voice.** Is a voice per segment enough, or are interview-style videos with several voices within one segment a near-term need? +11. **Putting the render back in the note.** Today the worker appends a video block to + the note and the open editor reconciles it. From another app that needs a Notes + endpoint that can't race the editor's autosave, or a different gesture: the note + links to the GVC project, and the user inserts the video themselves. --- -*This document is a product and data-model spec, not an implementation. No application -behaviour has changed.* +*This document is a product and data-model spec. Phase 0's spec, importer and compiler +are in `backend/app/video/`; nothing a user sees has changed.*