From 0f7ba0e4e069a413c2ee1b18f3b81ed6d9107c97 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 08:14:11 +0000 Subject: [PATCH] Pick clips in a pass of their own, one video at a time find_clips hands one video's whole transcript, numbered by sentence, and the idea a scene needs to a separate model call with no tools, and gives the turn back only the best passages: a ready trim, what is said, a score and why. The turn's own conversation no longer has to carry every transcript it weighs on every round, and the picker judges passages with nothing else to do. The picker names lines, never times, so a cut can't land on a moment it made up; a passage naming lines it wasn't shown is dropped. Long videos are read in overlapping windows of about 15k tokens and the best of all kept. Calls are capped per request (ASSISTANT_MAX_CLIP_PICKS, 16), cached per video and idea, and billed to the turn like any other call. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V1geD2rbaPtRa9JcN8sZH8 --- backend/app/assistant/clips.py | 136 ++++++++++++++++++++++++++ backend/app/assistant/loop.py | 6 +- backend/app/assistant/prompt.py | 17 ++-- backend/app/assistant/tools.py | 108 +++++++++++++++++--- backend/app/assistant/worker.py | 11 +++ backend/app/config.py | 5 + backend/tests/test_assistant_tools.py | 85 ++++++++++++++++ backend/tests/test_assistant_turns.py | 37 ++++++- 8 files changed, 381 insertions(+), 24 deletions(-) create mode 100644 backend/app/assistant/clips.py diff --git a/backend/app/assistant/clips.py b/backend/app/assistant/clips.py new file mode 100644 index 0000000..d60a3ca --- /dev/null +++ b/backend/app/assistant/clips.py @@ -0,0 +1,136 @@ +"""The clip-picking pass: one video's whole transcript read against one idea. + +A separate model call, not more reading in the turn's own conversation, for two reasons. +Everything the turn reads stays in its conversation and is sent again every round, so +reading five hour-long interviews there costs five transcripts times every round left; +here each is read once and only the chosen passages come back. And a call with nothing +to do but answer "where in this does someone make this point best" judges the passage, +where the turn's model is busy writing a video. + +The picker names lines by number, never by time. The times come from the transcript, so +a cut can't land on a moment the picker made up. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple + +if TYPE_CHECKING: + # tools.py imports this module, so only for the annotations. + from app.assistant.tools import Sentence + +# What one picker call reads: about 15k tokens. A longer transcript is read in windows +# and the best passages of all of them kept. +MAX_PICK_CHARS = 60_000 +# Lines each window repeats from the one before, so a passage that straddles the edge +# is whole in one of them. +WINDOW_OVERLAP = 6 +# Longer than this is a scene, not a clip; the picker is told so, and held to it. +MAX_CLIP_SECONDS = 90.0 + +SYSTEM = """\ +You pick clips for a video editor. You are given one video's transcript, one sentence \ +per numbered line with its [start-end] seconds, and the idea a scene has to put across. + +Find the passages where the speaker demonstrates that idea best: a complete thought, \ +said clearly, that makes sense to a viewer who hasn't seen the rest of the video. Prefer \ +the clearest, most concrete or most striking statement to the first or longest mention. \ +A passage is one or more consecutive lines, usually 5-40 seconds and never over 90. Start \ +it where the thought starts, not mid-argument, and end it where the thought lands. + +Answer with JSON only: an array, best first, of \ +{"first": line, "last": line, "score": 1-10, "why": "one short sentence"}. \ +score is how well the passage makes the point (10: exactly this, said perfectly; 5: \ +touches on it; below 4: don't include it). An empty array when nothing makes the point.""" + + +@dataclass +class Clip: + first: int + last: int + score: float + why: str + + +def windows(lines: Sequence[Sentence]) -> List[Tuple[int, int]]: + """[start, end) line ranges of at most MAX_PICK_CHARS each, overlapping a little.""" + out: List[Tuple[int, int]] = [] + start = 0 + while start < len(lines): + size, end = 0, start + while end < len(lines) and (end == start or size + len(lines[end].text) + 20 + <= MAX_PICK_CHARS): + size += len(lines[end].text) + 20 + end += 1 + out.append((start, end)) + if end >= len(lines): + break + start = max(start + 1, end - WINDOW_OVERLAP) + return out + + +def prompt_for(idea: str, count: int, lines: Sequence[Sentence], window: Tuple[int, int], + name: Optional[str] = None) -> str: + start, end = window + numbered = "\n".join(f"{i} [{lines[i].start:.1f}-{lines[i].end:.1f}] {lines[i].text}" + for i in range(start, end)) + part = (f" (lines {start}-{end - 1} of {len(lines)}; the rest is read separately)" + if start > 0 or end < len(lines) else "") + return (f"The idea: {idea.strip()}\n\n" + f"Up to {count} passages.\n\n" + f"Transcript of {name or 'the video'}{part}:\n{numbered}") + + +def _array(text: str) -> List[Any]: + """The JSON array in a reply, which a model may wrap in prose or a code fence.""" + match = re.search(r"\[.*\]", text or "", re.DOTALL) + if not match: + return [] + try: + found = json.loads(match.group(0)) + except ValueError: + return [] + return found if isinstance(found, list) else [] + + +def parse(text: str, lines: Sequence[Sentence], window: Tuple[int, int]) -> List[Clip]: + """The picker's passages that name real lines in its window and fit the rules. + + Anything else is dropped rather than repaired: a line number out of range is a + passage the picker didn't read, and stretching or clamping it would cut something + nobody chose. + """ + start, end = window + out: List[Clip] = [] + for item in _array(text): + if not isinstance(item, dict): + continue + try: + first, last = int(item["first"]), int(item["last"]) + score = float(item.get("score", 0)) + except (KeyError, TypeError, ValueError): + continue + if not (start <= first <= last < end): + continue + if lines[last].end - lines[first].start > MAX_CLIP_SECONDS or score < 4: + continue + out.append(Clip(first, last, min(score, 10.0), str(item.get("why") or "")[:200])) + return out + + +def best(found: Sequence[Clip], count: int) -> List[Clip]: + """The highest-scoring passages, no two overlapping, in the order they were said. + + Overlap happens where windows share lines, and when a picker offers the same + thought cut two ways; either way the viewer would see it twice. + """ + kept: List[Clip] = [] + for clip in sorted(found, key=lambda c: (-c.score, c.first)): + if all(clip.last < k.first or clip.first > k.last for k in kept): + kept.append(clip) + if len(kept) == count: + break + return sorted(kept, key=lambda c: c.first) diff --git a/backend/app/assistant/loop.py b/backend/app/assistant/loop.py index c535fb3..e80c78d 100644 --- a/backend/app/assistant/loop.py +++ b/backend/app/assistant/loop.py @@ -76,10 +76,12 @@ def run(provider: ChatProvider, working: Working, *, system: Sequence[str], raise Stopped() if call.name == "search_assets": on_step("search", f"Searching the library: {call.arguments.get('query', '')!r}") - elif call.name == "read_transcript": + elif call.name in ("read_transcript", "find_clips"): gam_id = call.arguments.get("gam_id", "") seen = working.hits.get(gam_id) - on_step("search", f"Reading the transcript of {seen.name if seen else gam_id}") + doing = "Reading the transcript of" if call.name == "read_transcript" \ + else "Picking clips from" + on_step("search", f"{doing} {seen.name if seen else gam_id}") result = T.execute(working, call) results.append(result) if call.name in T.READS: diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py index ca76de0..a008c2e 100644 --- a/backend/app/assistant/prompt.py +++ b/backend/app/assistant/prompt.py @@ -109,14 +109,15 @@ per video, the best match for your words, which is often not the best way to make the \ point. So for each scene that will show someone speaking: 1. Search, in a few phrasings, to find the candidate videos. -2. `read_transcript` the most promising ones (several at once), all of each where it's \ -short, and around `said_at` where it's long. A hit with `more_moments` matched elsewhere \ -too; a video that matched only by name has no `said_at`, so read it. -3. Choose the passage that best demonstrates the idea: a complete thought, said clearly, \ -that makes sense on its own, usually 5-40 s. Prefer the clearest, most concrete line to \ -the first mention. One video can give several clips to different scenes. -4. `set_visual` with `trim` {"in": first line's start, "out": last line's end}, taken \ -from the transcript lines, never estimated. +2. `find_clips` on each promising video (several at once), with the idea the scene has to \ +put across in a sentence. It reads the whole video and returns its best passages, scored, \ +each with a ready `trim`. A hit with `more_moments` matched elsewhere too, and a video that \ +matched only by name has no `said_at`: both are worth it. +3. Compare the passages across videos and choose the one that best demonstrates the idea. \ +One video can give several clips to different scenes; give each scene its own idea. +4. `set_visual` with that passage's `trim`, as given. +Use `read_transcript` to check what is said either side of a passage, or when \ +`find_clips` isn't available; then cut from the transcript lines, never from an estimate. Choosing music: search with `type` audio for the sound the scene needs, in the words a \ music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \ diff --git a/backend/app/assistant/tools.py b/backend/app/assistant/tools.py index be0367a..4988c26 100644 --- a/backend/app/assistant/tools.py +++ b/backend/app/assistant/tools.py @@ -26,8 +26,9 @@ from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Tuple +from app.assistant import clips from app.gam.client import GamClient, GamError -from app.providers.base import ToolCall, ToolResult, ToolSpec +from app.providers.base import ProviderError, ToolCall, ToolResult, ToolSpec from app.schemas_gam import GamTranscript from app.render.estimate import estimate from app.services import note_media as note_files @@ -160,6 +161,15 @@ class Working: # Transcripts fetched this turn by gam_id, so reading one and then cutting from it # asks GAM once. transcripts: Dict[str, GamTranscript] = field(default_factory=dict) + # One model call with no tools: (system, prompt) -> the reply's text, its spend + # recorded against the turn. Set by the worker; None where there is no provider, + # and then find_clips says to read the transcript instead. + ask: Optional[Callable[[str, str], str]] = None + # Calls the clip-picking pass may make in one turn (a long video takes several), + # and what it found, so asking the same thing twice pays once. + max_picks: int = 16 + picks: int = 0 + picked: Dict[Tuple[str, str, int], str] = field(default_factory=dict) def __post_init__(self) -> None: self.data = copy.deepcopy(self.data) @@ -381,6 +391,61 @@ def read_transcript(w: Working, gam_id: str, start: Optional[float] = None, return head + "\n" + "\n".join(out) + tail +def find_clips(w: Working, gam_id: str, idea: str, count: int = 3) -> str: + if w.ask is None: + raise ToolFailed("Clip picking isn't available in this turn; use read_transcript.") + if not (idea or "").strip(): + raise ToolFailed("Say the idea the clip has to put across.") + try: + count = max(1, min(5, int(count))) + except (TypeError, ValueError): + raise ToolFailed("count must be a whole number from 1 to 5.") + key = (gam_id, idea.strip().lower(), count) + if key in w.picked: + return w.picked[key] + lines = sentences(w.transcript(gam_id)) + if not lines: + return ("This asset has no transcript: it hasn't been transcribed in the library, or " + "nothing is said in it.") + spans = clips.windows(lines) + left = w.max_picks - w.picks + if len(spans) > left: + raise ToolFailed(f"Clip picking has {left} of its {w.max_picks} calls left this turn " + f"and this video needs {len(spans)}. Use read_transcript around a " + "search hit's said_at instead.") + seen = w.hits.get(gam_id) + found: List[clips.Clip] = [] + for span in spans: + w.picks += 1 + try: + text = w.ask(clips.SYSTEM, clips.prompt_for(idea, count, lines, span, + seen.name if seen else None)) + except ProviderError as exc: + raise ToolFailed(f"The clip-picking pass failed ({exc}). Use read_transcript " + "and choose the passage yourself.") + found += clips.parse(text, lines, span) + chosen = clips.best(found, count) + if not chosen: + result = ("Nothing in this video makes that point well. Try another video, or " + "describe the idea differently.") + else: + duration = seen.duration if seen else None + rows = [] + for clip in chosen: + first, last = lines[clip.first], lines[clip.last] + said = " ".join(lines[i].text for i in range(clip.first, clip.last + 1)) + rows.append({ + "trim": _padded(first.start, last.end, lines, clip.first, clip.last, duration), + "seconds": round(last.end - first.start, 1), + "score": clip.score, + "why": clip.why, + "says": said if len(said) <= 400 else said[:400] + "…", + }) + result = _compact({"gam_id": gam_id, "idea": idea.strip(), "clips": rows}) + w.picked[key] = result + return result + + def replace_spec(w: Working, spec: Dict[str, Any]) -> str: if not isinstance(spec, dict): raise ToolFailed("spec must be a VideoSpec object.") @@ -523,30 +588,35 @@ def remove_segment(w: Working, segment_id: str) -> str: return result +def _padded(a: float, b: float, lines: List[Sentence], first: Optional[int], + last: Optional[int], duration: Optional[float]) -> Optional[Dict[str, float]]: + """[a, b] as a trim, padded so a cut doesn't clip a word, but never into the + sentences either side of lines first..last.""" + floor = lines[first - 1].end if first else 0.0 + ceiling = lines[last + 1].start if last is not None and last + 1 < len(lines) else None + a = max(floor, a - MOMENT_PAD) + b = b + MOMENT_PAD if ceiling is None else min(b + MOMENT_PAD, max(b, ceiling)) + if duration: + b = min(b, duration) + return {"in": round(a, 1), "out": round(b, 1)} if b > a else None + + def _moment_trim(hit: Hit, lines: List[Sentence] = ()) -> Optional[Dict[str, float]]: """The found moment as a cut, grown to the whole sentences it falls in when the - transcript is to hand, and padded without reaching into the next sentence.""" + transcript is to hand.""" if hit.start is None: return None a = hit.start b = hit.end if hit.end is not None else hit.start + DEFAULT_MOMENT_SECONDS - floor, ceiling = 0.0, None touching = [i for i, s in enumerate(lines) if s.end > a and s.start < b] + first = last = None if touching: first, last = touching[0], touching[-1] if a - lines[first].start <= MAX_SENTENCE_STRETCH: a = min(a, lines[first].start) if lines[last].end - b <= MAX_SENTENCE_STRETCH: b = max(b, lines[last].end) - if first > 0: - floor = lines[first - 1].end - if last + 1 < len(lines): - ceiling = lines[last + 1].start - a = max(floor, a - MOMENT_PAD) - b = b + MOMENT_PAD if ceiling is None else min(b + MOMENT_PAD, max(b, ceiling)) - if hit.duration: - b = min(b, hit.duration) - return {"in": round(a, 1), "out": round(b, 1)} if b > a else None + return _padded(a, b, lines, first, last, hit.duration) def set_visual(w: Working, segment_id: str, gam_id: Optional[str] = None, @@ -809,6 +879,17 @@ def run_estimate(w: Working) -> str: "transcript comes in pages and says where to carry on.", {"type": "object", "properties": {"gam_id": _STR, "start": _NUM, "end": _NUM}, "required": ["gam_id"]}), read_transcript), + (ToolSpec("find_clips", + "Read a whole library video's transcript, in a separate pass, and return the " + "passages that best put across one idea, best scored, each with the trim to " + "pass to set_visual, what is said and why it fits. Use it on each candidate " + "video from search_assets before choosing a clip: it sees all of the video, " + "where a search shows one moment. idea: what the scene needs said, in a " + "sentence. count: passages wanted (1-5, default 3).", + {"type": "object", "properties": { + "gam_id": _STR, "idea": _STR, + "count": {"type": "integer", "minimum": 1, "maximum": 5}}, + "required": ["gam_id", "idea"]}), find_clips), (ToolSpec("get_asset", "Details of one library asset by gam_id.", {"type": "object", "properties": {"gam_id": _STR}, "required": ["gam_id"]}), get_asset), @@ -912,7 +993,8 @@ def run_estimate(w: Working) -> str: _HANDLERS = {spec.name: handler for spec, handler in TOOLS} # Read-only tools, which neither change the video nor count as an edit in the steps. -READS = {"get_spec", "search_assets", "read_transcript", "get_asset", "estimate"} +READS = {"get_spec", "search_assets", "read_transcript", "find_clips", "get_asset", + "estimate"} def execute(w: Working, call: ToolCall) -> ToolResult: diff --git a/backend/app/assistant/worker.py b/backend/app/assistant/worker.py index c93c36f..ab8d125 100644 --- a/backend/app/assistant/worker.py +++ b/backend/app/assistant/worker.py @@ -122,6 +122,7 @@ def on_reply(reply: Reply) -> None: attached = material(attachments, sees_images=sees_images) images = images_of(chat_id, attachments) if sees_images else [] working = Working(base, gam=gam, max_generations=settings.assistant_max_generations, + max_picks=settings.assistant_max_clip_picks, note_media=media_ids(attachments)) def generate(data, key, image_prompt, base_ids, outputs): @@ -135,6 +136,16 @@ def generate(data, key, image_prompt, base_ids, outputs): return row.model_label, row.estimate working.generate = generate + + def ask(pick_system: str, pick_prompt: str) -> str: + # The clip-picking pass: the turn's own provider, a fresh conversation with no + # tools, its spend on the turn like any other call. + reply = provider.conversation(system=[pick_system], tools=[], history=[], + prompt=pick_prompt).ask() + on_reply(reply) + return reply.text + + working.ask = ask system = [REFERENCE, attached, context(base, focus=focus, from_scratch=project_id is None)] outcome = loop.run(provider, working, system=system, history=history, prompt=prompt, diff --git a/backend/app/config.py b/backend/app/config.py index 92457ed..a32b88f 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -98,6 +98,11 @@ class Settings(BaseSettings): # Pictures one request may generate on its own (each is a paid call to fal.ai through # GAM). Past it the assistant leaves placeholders for the user to generate. assistant_max_generations: int = 6 + # Calls the clip-picking pass may make in one request. Each reads up to about 15k + # tokens of transcript, once (an hour of speech is one or two), so this is roughly + # how many candidate videos a request can weigh before it falls back to reading + # transcripts itself. + assistant_max_clip_picks: int = 16 # Material attached to a conversation (files, notes, web pages), as text. Each turn # reads all of it on every round, so the limits are about cost as much as context # length: 150k characters is roughly 40k tokens, cached after the first round. diff --git a/backend/tests/test_assistant_tools.py b/backend/tests/test_assistant_tools.py index 7f72fe3..fd7e6f6 100644 --- a/backend/tests/test_assistant_tools.py +++ b/backend/tests/test_assistant_tools.py @@ -1,7 +1,10 @@ """The assistant's operations on a working copy of a spec: pure, with GAM faked.""" +import json + import pytest +from app.assistant import clips from app.assistant import tools as T from app.assistant.tools import Working from app.gam.client import GamClient @@ -354,3 +357,85 @@ def test_only_the_ids_of_attached_notes_can_be_imported(siblings): "caption": "", "after": ""}} result = run(w, "import_note_media", media_ids=["/etc/passwd"]) assert result.is_error and "No such note file" in result.content + + +def picker(*answers): + """A clip-picking pass that answers from a list, recording what it was asked.""" + asked = [] + + def ask(system, prompt): + asked.append(prompt) + return answers[len(asked) - 1] + ask.asked = asked + return ask + + +def test_find_clips_reads_the_whole_video_and_returns_ready_cuts(gam, siblings): + river_transcript(siblings) + w = Working(spec(), gam=gam) + run(w, "search_assets", query="water") + w.ask = picker('Here you go:\n```json\n[{"first": 1, "last": 2, "score": 9, ' + '"why": "Says the river dried up and that nobody planned for it."},' + '{"first": 0, "last": 0, "score": 3, "why": "weak"}]\n```') + found = run(w, "find_clips", gam_id="r", idea="The river ran dry without warning") + assert not found.is_error, found.content + result = json.loads(found.content) + # Lines 1-2 run 36.5-52.0; padded into the gap before, the video's end after. The + # weak passage is dropped. + assert [c["trim"] for c in result["clips"]] == [{"in": 36.0, "out": 52.5}] + assert result["clips"][0]["says"].startswith("Back then the river ran clear") + # The picker saw every line, numbered, and the idea. + prompt = w.ask.asked[0] + assert "The idea: The river ran dry without warning" in prompt + assert "0 [30.0-36.0] We came here" in prompt and "2 [48.0-52.0] Nobody" in prompt + + # Asked again, it isn't paid for again. + assert run(w, "find_clips", gam_id="r", idea="the river ran dry without warning ").content \ + == found.content + assert len(w.ask.asked) == 1 and w.picks == 1 + + +def test_find_clips_never_cuts_what_the_picker_did_not_read(gam, siblings): + river_transcript(siblings) + w = Working(spec(), gam=gam) + w.ask = picker('[{"first": 2, "last": 9, "score": 10}, {"first": "x"}, ' + '{"first": 2, "last": 1, "score": 8}]') + assert "Nothing in this video" in run(w, "find_clips", gam_id="r", idea="water").content + + +def test_find_clips_reads_a_long_video_in_windows_and_keeps_the_best(gam, siblings, monkeypatch): + river_transcript(siblings) + monkeypatch.setattr(clips, "MAX_PICK_CHARS", 90) + monkeypatch.setattr(clips, "WINDOW_OVERLAP", 1) + w = Working(spec(), gam=gam) + spans = clips.windows(T.sentences(w.transcript("r"))) + assert spans[0][0] == 0 and spans[-1][1] == 3 and len(spans) > 1 + answers = ['[{"first": %d, "last": %d, "score": %d}]' % (s, s, 5 + i) + for i, (s, _) in enumerate(spans)] + w.ask = picker(*answers) + result = json.loads(run(w, "find_clips", gam_id="r", idea="water", count=1).content) + assert len(result["clips"]) == 1 and result["clips"][0]["score"] == 5 + len(spans) - 1 + assert w.picks == len(spans) + assert "lines" in w.ask.asked[0] and "read separately" in w.ask.asked[0] + + +def test_find_clips_stops_at_its_budget_and_says_what_to_do_instead(gam, siblings): + river_transcript(siblings) + w = Working(spec(), gam=gam, max_picks=0) + w.ask = picker() + refused = run(w, "find_clips", gam_id="r", idea="water") + assert refused.is_error and "read_transcript" in refused.content + + +def test_find_clips_without_a_picker_or_a_transcript_says_so(gam, siblings): + w = Working(spec(), gam=gam) + assert run(w, "find_clips", gam_id="r", idea="water").is_error + w.ask = picker() + assert "no transcript" in run(w, "find_clips", gam_id="r", idea="water").content + assert run(w, "find_clips", gam_id="r", idea="water", count="a few").is_error + + +def test_overlapping_passages_are_kept_once_in_the_order_said(): + found = [clips.Clip(4, 6, 7, ""), clips.Clip(5, 8, 9, ""), clips.Clip(0, 1, 6, ""), + clips.Clip(10, 10, 8, "")] + assert [(c.first, c.last) for c in clips.best(found, 3)] == [(0, 1), (5, 8), (10, 10)] diff --git a/backend/tests/test_assistant_turns.py b/backend/tests/test_assistant_turns.py index a373f70..32cdfca 100644 --- a/backend/tests/test_assistant_turns.py +++ b/backend/tests/test_assistant_turns.py @@ -14,7 +14,7 @@ from app.models.usage import UsageEvent from tests.conftest import TEST_USER_ID from tests.fake_provider import ScriptedProvider, call, says -from tests.fake_siblings import gam_asset, hit +from tests.fake_siblings import gam_asset, hit, transcript @pytest.fixture @@ -334,3 +334,38 @@ def test_a_draft_generates_its_pictures_but_never_its_video( made = auth_client.post(f"/api/assistant/turns/{turn['id']}/new-project").json()["data"] listed = auth_client.get(f"/api/projects/{made['project_id']}/generations").json()["data"] assert [g["asset_key"] for g in listed] == ["a_dam"] + + +def test_clips_are_picked_in_a_pass_of_their_own_paid_for_on_the_turn( + auth_client, session, provider, library, engine_for_worker): + project = project_with(auth_client, VALID) + library.transcripts["r"] = transcript( + "r", (30.0, 36.0, "We came here in the spring."), + (36.5, 47.5, "Back then the river ran clear all year, but now the water is gone.")) + provider.script += [ + says("", call("search_assets", query="water", type="video")), + says("", call("find_clips", gam_id="r", idea="The river is gone")), + # The picking pass: its own conversation, answered from the same script. + says('[{"first": 1, "last": 1, "score": 9, "why": "Says it plainly."}]'), + says("", call("set_visual", segment_id="s_valley", gam_id="r", + trim={"in": 36.0, "out": 48.0})), + says("The valley now cuts to the river interview saying the water is gone."), + ] + turn = start(auth_client, prompt="Find a better clip for the valley", + project_id=project["id"]) + worker._run(turn["id"]) + done = auth_client.get(f"/api/assistant/turns/{turn['id']}").json()["data"] + assert done["status"] == "done", done + + picked = provider.results()[1] + assert '"trim":{"in":36.0,"out":48.0}' in picked.content and "Says it plainly" in picked.content + # The pass was a conversation of its own, the last one started: no tools, no history, + # the picker's instructions and the numbered transcript. + assert provider.started["tools"] == [] and provider.started["history"] == [] + assert "You pick clips" in provider.started["system"][0] + assert "1 [36.5-47.5] Back then" in provider.started["prompt"] + assert any(s["text"] == "Picking clips from River interview" for s in done["steps"]) + assert done["proposed_spec"]["segments"][1]["visual"]["trim"] == {"in": 36.0, "out": 48.0} + # Five model calls, the pass among them, all on the turn's bill. + rows = session.query(UsageEvent).all() + assert len(rows) == 5 and all(r.turn_id == turn["id"] for r in rows)