diff --git a/backend/app/assistant/clips.py b/backend/app/assistant/clips.py new file mode 100644 index 0000000..d60a3ca --- /dev/null +++ b/backend/app/assistant/clips.py @@ -0,0 +1,136 @@ +"""The clip-picking pass: one video's whole transcript read against one idea. + +A separate model call, not more reading in the turn's own conversation, for two reasons. +Everything the turn reads stays in its conversation and is sent again every round, so +reading five hour-long interviews there costs five transcripts times every round left; +here each is read once and only the chosen passages come back. And a call with nothing +to do but answer "where in this does someone make this point best" judges the passage, +where the turn's model is busy writing a video. + +The picker names lines by number, never by time. The times come from the transcript, so +a cut can't land on a moment the picker made up. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple + +if TYPE_CHECKING: + # tools.py imports this module, so only for the annotations. + from app.assistant.tools import Sentence + +# What one picker call reads: about 15k tokens. A longer transcript is read in windows +# and the best passages of all of them kept. +MAX_PICK_CHARS = 60_000 +# Lines each window repeats from the one before, so a passage that straddles the edge +# is whole in one of them. +WINDOW_OVERLAP = 6 +# Longer than this is a scene, not a clip; the picker is told so, and held to it. +MAX_CLIP_SECONDS = 90.0 + +SYSTEM = """\ +You pick clips for a video editor. You are given one video's transcript, one sentence \ +per numbered line with its [start-end] seconds, and the idea a scene has to put across. + +Find the passages where the speaker demonstrates that idea best: a complete thought, \ +said clearly, that makes sense to a viewer who hasn't seen the rest of the video. Prefer \ +the clearest, most concrete or most striking statement to the first or longest mention. \ +A passage is one or more consecutive lines, usually 5-40 seconds and never over 90. Start \ +it where the thought starts, not mid-argument, and end it where the thought lands. + +Answer with JSON only: an array, best first, of \ +{"first": line, "last": line, "score": 1-10, "why": "one short sentence"}. \ +score is how well the passage makes the point (10: exactly this, said perfectly; 5: \ +touches on it; below 4: don't include it). An empty array when nothing makes the point.""" + + +@dataclass +class Clip: + first: int + last: int + score: float + why: str + + +def windows(lines: Sequence[Sentence]) -> List[Tuple[int, int]]: + """[start, end) line ranges of at most MAX_PICK_CHARS each, overlapping a little.""" + out: List[Tuple[int, int]] = [] + start = 0 + while start < len(lines): + size, end = 0, start + while end < len(lines) and (end == start or size + len(lines[end].text) + 20 + <= MAX_PICK_CHARS): + size += len(lines[end].text) + 20 + end += 1 + out.append((start, end)) + if end >= len(lines): + break + start = max(start + 1, end - WINDOW_OVERLAP) + return out + + +def prompt_for(idea: str, count: int, lines: Sequence[Sentence], window: Tuple[int, int], + name: Optional[str] = None) -> str: + start, end = window + numbered = "\n".join(f"{i} [{lines[i].start:.1f}-{lines[i].end:.1f}] {lines[i].text}" + for i in range(start, end)) + part = (f" (lines {start}-{end - 1} of {len(lines)}; the rest is read separately)" + if start > 0 or end < len(lines) else "") + return (f"The idea: {idea.strip()}\n\n" + f"Up to {count} passages.\n\n" + f"Transcript of {name or 'the video'}{part}:\n{numbered}") + + +def _array(text: str) -> List[Any]: + """The JSON array in a reply, which a model may wrap in prose or a code fence.""" + match = re.search(r"\[.*\]", text or "", re.DOTALL) + if not match: + return [] + try: + found = json.loads(match.group(0)) + except ValueError: + return [] + return found if isinstance(found, list) else [] + + +def parse(text: str, lines: Sequence[Sentence], window: Tuple[int, int]) -> List[Clip]: + """The picker's passages that name real lines in its window and fit the rules. + + Anything else is dropped rather than repaired: a line number out of range is a + passage the picker didn't read, and stretching or clamping it would cut something + nobody chose. + """ + start, end = window + out: List[Clip] = [] + for item in _array(text): + if not isinstance(item, dict): + continue + try: + first, last = int(item["first"]), int(item["last"]) + score = float(item.get("score", 0)) + except (KeyError, TypeError, ValueError): + continue + if not (start <= first <= last < end): + continue + if lines[last].end - lines[first].start > MAX_CLIP_SECONDS or score < 4: + continue + out.append(Clip(first, last, min(score, 10.0), str(item.get("why") or "")[:200])) + return out + + +def best(found: Sequence[Clip], count: int) -> List[Clip]: + """The highest-scoring passages, no two overlapping, in the order they were said. + + Overlap happens where windows share lines, and when a picker offers the same + thought cut two ways; either way the viewer would see it twice. + """ + kept: List[Clip] = [] + for clip in sorted(found, key=lambda c: (-c.score, c.first)): + if all(clip.last < k.first or clip.first > k.last for k in kept): + kept.append(clip) + if len(kept) == count: + break + return sorted(kept, key=lambda c: c.first) diff --git a/backend/app/assistant/loop.py b/backend/app/assistant/loop.py index c535fb3..e80c78d 100644 --- a/backend/app/assistant/loop.py +++ b/backend/app/assistant/loop.py @@ -76,10 +76,12 @@ def run(provider: ChatProvider, working: Working, *, system: Sequence[str], raise Stopped() if call.name == "search_assets": on_step("search", f"Searching the library: {call.arguments.get('query', '')!r}") - elif call.name == "read_transcript": + elif call.name in ("read_transcript", "find_clips"): gam_id = call.arguments.get("gam_id", "") seen = working.hits.get(gam_id) - on_step("search", f"Reading the transcript of {seen.name if seen else gam_id}") + doing = "Reading the transcript of" if call.name == "read_transcript" \ + else "Picking clips from" + on_step("search", f"{doing} {seen.name if seen else gam_id}") result = T.execute(working, call) results.append(result) if call.name in T.READS: diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py index ca76de0..a008c2e 100644 --- a/backend/app/assistant/prompt.py +++ b/backend/app/assistant/prompt.py @@ -109,14 +109,15 @@ per video, the best match for your words, which is often not the best way to make the \ point. So for each scene that will show someone speaking: 1. Search, in a few phrasings, to find the candidate videos. -2. `read_transcript` the most promising ones (several at once), all of each where it's \ -short, and around `said_at` where it's long. A hit with `more_moments` matched elsewhere \ -too; a video that matched only by name has no `said_at`, so read it. -3. Choose the passage that best demonstrates the idea: a complete thought, said clearly, \ -that makes sense on its own, usually 5-40 s. Prefer the clearest, most concrete line to \ -the first mention. One video can give several clips to different scenes. -4. `set_visual` with `trim` {"in": first line's start, "out": last line's end}, taken \ -from the transcript lines, never estimated. +2. `find_clips` on each promising video (several at once), with the idea the scene has to \ +put across in a sentence. It reads the whole video and returns its best passages, scored, \ +each with a ready `trim`. A hit with `more_moments` matched elsewhere too, and a video that \ +matched only by name has no `said_at`: both are worth it. +3. Compare the passages across videos and choose the one that best demonstrates the idea. \ +One video can give several clips to different scenes; give each scene its own idea. +4. `set_visual` with that passage's `trim`, as given. +Use `read_transcript` to check what is said either side of a passage, or when \ +`find_clips` isn't available; then cut from the transcript lines, never from an estimate. Choosing music: search with `type` audio for the sound the scene needs, in the words a \ music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \ diff --git a/backend/app/assistant/tools.py b/backend/app/assistant/tools.py index be0367a..4988c26 100644 --- a/backend/app/assistant/tools.py +++ b/backend/app/assistant/tools.py @@ -26,8 +26,9 @@ from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Tuple +from app.assistant import clips from app.gam.client import GamClient, GamError -from app.providers.base import ToolCall, ToolResult, ToolSpec +from app.providers.base import ProviderError, ToolCall, ToolResult, ToolSpec from app.schemas_gam import GamTranscript from app.render.estimate import estimate from app.services import note_media as note_files @@ -160,6 +161,15 @@ class Working: # Transcripts fetched this turn by gam_id, so reading one and then cutting from it # asks GAM once. transcripts: Dict[str, GamTranscript] = field(default_factory=dict) + # One model call with no tools: (system, prompt) -> the reply's text, its spend + # recorded against the turn. Set by the worker; None where there is no provider, + # and then find_clips says to read the transcript instead. + ask: Optional[Callable[[str, str], str]] = None + # Calls the clip-picking pass may make in one turn (a long video takes several), + # and what it found, so asking the same thing twice pays once. + max_picks: int = 16 + picks: int = 0 + picked: Dict[Tuple[str, str, int], str] = field(default_factory=dict) def __post_init__(self) -> None: self.data = copy.deepcopy(self.data) @@ -381,6 +391,61 @@ def read_transcript(w: Working, gam_id: str, start: Optional[float] = None, return head + "\n" + "\n".join(out) + tail +def find_clips(w: Working, gam_id: str, idea: str, count: int = 3) -> str: + if w.ask is None: + raise ToolFailed("Clip picking isn't available in this turn; use read_transcript.") + if not (idea or "").strip(): + raise ToolFailed("Say the idea the clip has to put across.") + try: + count = max(1, min(5, int(count))) + except (TypeError, ValueError): + raise ToolFailed("count must be a whole number from 1 to 5.") + key = (gam_id, idea.strip().lower(), count) + if key in w.picked: + return w.picked[key] + lines = sentences(w.transcript(gam_id)) + if not lines: + return ("This asset has no transcript: it hasn't been transcribed in the library, or " + "nothing is said in it.") + spans = clips.windows(lines) + left = w.max_picks - w.picks + if len(spans) > left: + raise ToolFailed(f"Clip picking has {left} of its {w.max_picks} calls left this turn " + f"and this video needs {len(spans)}. Use read_transcript around a " + "search hit's said_at instead.") + seen = w.hits.get(gam_id) + found: List[clips.Clip] = [] + for span in spans: + w.picks += 1 + try: + text = w.ask(clips.SYSTEM, clips.prompt_for(idea, count, lines, span, + seen.name if seen else None)) + except ProviderError as exc: + raise ToolFailed(f"The clip-picking pass failed ({exc}). Use read_transcript " + "and choose the passage yourself.") + found += clips.parse(text, lines, span) + chosen = clips.best(found, count) + if not chosen: + result = ("Nothing in this video makes that point well. Try another video, or " + "describe the idea differently.") + else: + duration = seen.duration if seen else None + rows = [] + for clip in chosen: + first, last = lines[clip.first], lines[clip.last] + said = " ".join(lines[i].text for i in range(clip.first, clip.last + 1)) + rows.append({ + "trim": _padded(first.start, last.end, lines, clip.first, clip.last, duration), + "seconds": round(last.end - first.start, 1), + "score": clip.score, + "why": clip.why, + "says": said if len(said) <= 400 else said[:400] + "…", + }) + result = _compact({"gam_id": gam_id, "idea": idea.strip(), "clips": rows}) + w.picked[key] = result + return result + + def replace_spec(w: Working, spec: Dict[str, Any]) -> str: if not isinstance(spec, dict): raise ToolFailed("spec must be a VideoSpec object.") @@ -523,30 +588,35 @@ def remove_segment(w: Working, segment_id: str) -> str: return result +def _padded(a: float, b: float, lines: List[Sentence], first: Optional[int], + last: Optional[int], duration: Optional[float]) -> Optional[Dict[str, float]]: + """[a, b] as a trim, padded so a cut doesn't clip a word, but never into the + sentences either side of lines first..last.""" + floor = lines[first - 1].end if first else 0.0 + ceiling = lines[last + 1].start if last is not None and last + 1 < len(lines) else None + a = max(floor, a - MOMENT_PAD) + b = b + MOMENT_PAD if ceiling is None else min(b + MOMENT_PAD, max(b, ceiling)) + if duration: + b = min(b, duration) + return {"in": round(a, 1), "out": round(b, 1)} if b > a else None + + def _moment_trim(hit: Hit, lines: List[Sentence] = ()) -> Optional[Dict[str, float]]: """The found moment as a cut, grown to the whole sentences it falls in when the - transcript is to hand, and padded without reaching into the next sentence.""" + transcript is to hand.""" if hit.start is None: return None a = hit.start b = hit.end if hit.end is not None else hit.start + DEFAULT_MOMENT_SECONDS - floor, ceiling = 0.0, None touching = [i for i, s in enumerate(lines) if s.end > a and s.start < b] + first = last = None if touching: first, last = touching[0], touching[-1] if a - lines[first].start <= MAX_SENTENCE_STRETCH: a = min(a, lines[first].start) if lines[last].end - b <= MAX_SENTENCE_STRETCH: b = max(b, lines[last].end) - if first > 0: - floor = lines[first - 1].end - if last + 1 < len(lines): - ceiling = lines[last + 1].start - a = max(floor, a - MOMENT_PAD) - b = b + MOMENT_PAD if ceiling is None else min(b + MOMENT_PAD, max(b, ceiling)) - if hit.duration: - b = min(b, hit.duration) - return {"in": round(a, 1), "out": round(b, 1)} if b > a else None + return _padded(a, b, lines, first, last, hit.duration) def set_visual(w: Working, segment_id: str, gam_id: Optional[str] = None, @@ -809,6 +879,17 @@ def run_estimate(w: Working) -> str: "transcript comes in pages and says where to carry on.", {"type": "object", "properties": {"gam_id": _STR, "start": _NUM, "end": _NUM}, "required": ["gam_id"]}), read_transcript), + (ToolSpec("find_clips", + "Read a whole library video's transcript, in a separate pass, and return the " + "passages that best put across one idea, best scored, each with the trim to " + "pass to set_visual, what is said and why it fits. Use it on each candidate " + "video from search_assets before choosing a clip: it sees all of the video, " + "where a search shows one moment. idea: what the scene needs said, in a " + "sentence. count: passages wanted (1-5, default 3).", + {"type": "object", "properties": { + "gam_id": _STR, "idea": _STR, + "count": {"type": "integer", "minimum": 1, "maximum": 5}}, + "required": ["gam_id", "idea"]}), find_clips), (ToolSpec("get_asset", "Details of one library asset by gam_id.", {"type": "object", "properties": {"gam_id": _STR}, "required": ["gam_id"]}), get_asset), @@ -912,7 +993,8 @@ def run_estimate(w: Working) -> str: _HANDLERS = {spec.name: handler for spec, handler in TOOLS} # Read-only tools, which neither change the video nor count as an edit in the steps. -READS = {"get_spec", "search_assets", "read_transcript", "get_asset", "estimate"} +READS = {"get_spec", "search_assets", "read_transcript", "find_clips", "get_asset", + "estimate"} def execute(w: Working, call: ToolCall) -> ToolResult: diff --git a/backend/app/assistant/worker.py b/backend/app/assistant/worker.py index c93c36f..ab8d125 100644 --- a/backend/app/assistant/worker.py +++ b/backend/app/assistant/worker.py @@ -122,6 +122,7 @@ def on_reply(reply: Reply) -> None: attached = material(attachments, sees_images=sees_images) images = images_of(chat_id, attachments) if sees_images else [] working = Working(base, gam=gam, max_generations=settings.assistant_max_generations, + max_picks=settings.assistant_max_clip_picks, note_media=media_ids(attachments)) def generate(data, key, image_prompt, base_ids, outputs): @@ -135,6 +136,16 @@ def generate(data, key, image_prompt, base_ids, outputs): return row.model_label, row.estimate working.generate = generate + + def ask(pick_system: str, pick_prompt: str) -> str: + # The clip-picking pass: the turn's own provider, a fresh conversation with no + # tools, its spend on the turn like any other call. + reply = provider.conversation(system=[pick_system], tools=[], history=[], + prompt=pick_prompt).ask() + on_reply(reply) + return reply.text + + working.ask = ask system = [REFERENCE, attached, context(base, focus=focus, from_scratch=project_id is None)] outcome = loop.run(provider, working, system=system, history=history, prompt=prompt, diff --git a/backend/app/config.py b/backend/app/config.py index 92457ed..a32b88f 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -98,6 +98,11 @@ class Settings(BaseSettings): # Pictures one request may generate on its own (each is a paid call to fal.ai through # GAM). Past it the assistant leaves placeholders for the user to generate. assistant_max_generations: int = 6 + # Calls the clip-picking pass may make in one request. Each reads up to about 15k + # tokens of transcript, once (an hour of speech is one or two), so this is roughly + # how many candidate videos a request can weigh before it falls back to reading + # transcripts itself. + assistant_max_clip_picks: int = 16 # Material attached to a conversation (files, notes, web pages), as text. Each turn # reads all of it on every round, so the limits are about cost as much as context # length: 150k characters is roughly 40k tokens, cached after the first round. diff --git a/backend/tests/test_assistant_tools.py b/backend/tests/test_assistant_tools.py index 7f72fe3..fd7e6f6 100644 --- a/backend/tests/test_assistant_tools.py +++ b/backend/tests/test_assistant_tools.py @@ -1,7 +1,10 @@ """The assistant's operations on a working copy of a spec: pure, with GAM faked.""" +import json + import pytest +from app.assistant import clips from app.assistant import tools as T from app.assistant.tools import Working from app.gam.client import GamClient @@ -354,3 +357,85 @@ def test_only_the_ids_of_attached_notes_can_be_imported(siblings): "caption": "", "after": ""}} result = run(w, "import_note_media", media_ids=["/etc/passwd"]) assert result.is_error and "No such note file" in result.content + + +def picker(*answers): + """A clip-picking pass that answers from a list, recording what it was asked.""" + asked = [] + + def ask(system, prompt): + asked.append(prompt) + return answers[len(asked) - 1] + ask.asked = asked + return ask + + +def test_find_clips_reads_the_whole_video_and_returns_ready_cuts(gam, siblings): + river_transcript(siblings) + w = Working(spec(), gam=gam) + run(w, "search_assets", query="water") + w.ask = picker('Here you go:\n```json\n[{"first": 1, "last": 2, "score": 9, ' + '"why": "Says the river dried up and that nobody planned for it."},' + '{"first": 0, "last": 0, "score": 3, "why": "weak"}]\n```') + found = run(w, "find_clips", gam_id="r", idea="The river ran dry without warning") + assert not found.is_error, found.content + result = json.loads(found.content) + # Lines 1-2 run 36.5-52.0; padded into the gap before, the video's end after. The + # weak passage is dropped. + assert [c["trim"] for c in result["clips"]] == [{"in": 36.0, "out": 52.5}] + assert result["clips"][0]["says"].startswith("Back then the river ran clear") + # The picker saw every line, numbered, and the idea. + prompt = w.ask.asked[0] + assert "The idea: The river ran dry without warning" in prompt + assert "0 [30.0-36.0] We came here" in prompt and "2 [48.0-52.0] Nobody" in prompt + + # Asked again, it isn't paid for again. + assert run(w, "find_clips", gam_id="r", idea="the river ran dry without warning ").content \ + == found.content + assert len(w.ask.asked) == 1 and w.picks == 1 + + +def test_find_clips_never_cuts_what_the_picker_did_not_read(gam, siblings): + river_transcript(siblings) + w = Working(spec(), gam=gam) + w.ask = picker('[{"first": 2, "last": 9, "score": 10}, {"first": "x"}, ' + '{"first": 2, "last": 1, "score": 8}]') + assert "Nothing in this video" in run(w, "find_clips", gam_id="r", idea="water").content + + +def test_find_clips_reads_a_long_video_in_windows_and_keeps_the_best(gam, siblings, monkeypatch): + river_transcript(siblings) + monkeypatch.setattr(clips, "MAX_PICK_CHARS", 90) + monkeypatch.setattr(clips, "WINDOW_OVERLAP", 1) + w = Working(spec(), gam=gam) + spans = clips.windows(T.sentences(w.transcript("r"))) + assert spans[0][0] == 0 and spans[-1][1] == 3 and len(spans) > 1 + answers = ['[{"first": %d, "last": %d, "score": %d}]' % (s, s, 5 + i) + for i, (s, _) in enumerate(spans)] + w.ask = picker(*answers) + result = json.loads(run(w, "find_clips", gam_id="r", idea="water", count=1).content) + assert len(result["clips"]) == 1 and result["clips"][0]["score"] == 5 + len(spans) - 1 + assert w.picks == len(spans) + assert "lines" in w.ask.asked[0] and "read separately" in w.ask.asked[0] + + +def test_find_clips_stops_at_its_budget_and_says_what_to_do_instead(gam, siblings): + river_transcript(siblings) + w = Working(spec(), gam=gam, max_picks=0) + w.ask = picker() + refused = run(w, "find_clips", gam_id="r", idea="water") + assert refused.is_error and "read_transcript" in refused.content + + +def test_find_clips_without_a_picker_or_a_transcript_says_so(gam, siblings): + w = Working(spec(), gam=gam) + assert run(w, "find_clips", gam_id="r", idea="water").is_error + w.ask = picker() + assert "no transcript" in run(w, "find_clips", gam_id="r", idea="water").content + assert run(w, "find_clips", gam_id="r", idea="water", count="a few").is_error + + +def test_overlapping_passages_are_kept_once_in_the_order_said(): + found = [clips.Clip(4, 6, 7, ""), clips.Clip(5, 8, 9, ""), clips.Clip(0, 1, 6, ""), + clips.Clip(10, 10, 8, "")] + assert [(c.first, c.last) for c in clips.best(found, 3)] == [(0, 1), (5, 8), (10, 10)] diff --git a/backend/tests/test_assistant_turns.py b/backend/tests/test_assistant_turns.py index a373f70..32cdfca 100644 --- a/backend/tests/test_assistant_turns.py +++ b/backend/tests/test_assistant_turns.py @@ -14,7 +14,7 @@ from app.models.usage import UsageEvent from tests.conftest import TEST_USER_ID from tests.fake_provider import ScriptedProvider, call, says -from tests.fake_siblings import gam_asset, hit +from tests.fake_siblings import gam_asset, hit, transcript @pytest.fixture @@ -334,3 +334,38 @@ def test_a_draft_generates_its_pictures_but_never_its_video( made = auth_client.post(f"/api/assistant/turns/{turn['id']}/new-project").json()["data"] listed = auth_client.get(f"/api/projects/{made['project_id']}/generations").json()["data"] assert [g["asset_key"] for g in listed] == ["a_dam"] + + +def test_clips_are_picked_in_a_pass_of_their_own_paid_for_on_the_turn( + auth_client, session, provider, library, engine_for_worker): + project = project_with(auth_client, VALID) + library.transcripts["r"] = transcript( + "r", (30.0, 36.0, "We came here in the spring."), + (36.5, 47.5, "Back then the river ran clear all year, but now the water is gone.")) + provider.script += [ + says("", call("search_assets", query="water", type="video")), + says("", call("find_clips", gam_id="r", idea="The river is gone")), + # The picking pass: its own conversation, answered from the same script. + says('[{"first": 1, "last": 1, "score": 9, "why": "Says it plainly."}]'), + says("", call("set_visual", segment_id="s_valley", gam_id="r", + trim={"in": 36.0, "out": 48.0})), + says("The valley now cuts to the river interview saying the water is gone."), + ] + turn = start(auth_client, prompt="Find a better clip for the valley", + project_id=project["id"]) + worker._run(turn["id"]) + done = auth_client.get(f"/api/assistant/turns/{turn['id']}").json()["data"] + assert done["status"] == "done", done + + picked = provider.results()[1] + assert '"trim":{"in":36.0,"out":48.0}' in picked.content and "Says it plainly" in picked.content + # The pass was a conversation of its own, the last one started: no tools, no history, + # the picker's instructions and the numbered transcript. + assert provider.started["tools"] == [] and provider.started["history"] == [] + assert "You pick clips" in provider.started["system"][0] + assert "1 [36.5-47.5] Back then" in provider.started["prompt"] + assert any(s["text"] == "Picking clips from River interview" for s in done["steps"]) + assert done["proposed_spec"]["segments"][1]["visual"]["trim"] == {"in": 36.0, "out": 48.0} + # Five model calls, the pass among them, all on the turn's bill. + rows = session.query(UsageEvent).all() + assert len(rows) == 5 and all(r.turn_id == turn["id"] for r in rows)