Pick clips in a pass of their own, one video at a time

backend/app/assistant/clips.py

@@ -0,0 +1,136 @@
+"""The clip-picking pass: one video's whole transcript read against one idea.
+
+A separate model call, not more reading in the turn's own conversation, for two reasons.
+Everything the turn reads stays in its conversation and is sent again every round, so
+reading five hour-long interviews there costs five transcripts times every round left;
+here each is read once and only the chosen passages come back. And a call with nothing
+to do but answer "where in this does someone make this point best" judges the passage,
+where the turn's model is busy writing a video.
+
+The picker names lines by number, never by time. The times come from the transcript, so
+a cut can't land on a moment the picker made up.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple
+
+if TYPE_CHECKING:
+    # tools.py imports this module, so only for the annotations.
+    from app.assistant.tools import Sentence
+
+# What one picker call reads: about 15k tokens. A longer transcript is read in windows
+# and the best passages of all of them kept.
+MAX_PICK_CHARS = 60_000
+# Lines each window repeats from the one before, so a passage that straddles the edge
+# is whole in one of them.
+WINDOW_OVERLAP = 6
+# Longer than this is a scene, not a clip; the picker is told so, and held to it.
+MAX_CLIP_SECONDS = 90.0
+
+SYSTEM = """\
+You pick clips for a video editor. You are given one video's transcript, one sentence \
+per numbered line with its [start-end] seconds, and the idea a scene has to put across.
+
+Find the passages where the speaker demonstrates that idea best: a complete thought, \
+said clearly, that makes sense to a viewer who hasn't seen the rest of the video. Prefer \
+the clearest, most concrete or most striking statement to the first or longest mention. \
+A passage is one or more consecutive lines, usually 5-40 seconds and never over 90. Start \
+it where the thought starts, not mid-argument, and end it where the thought lands.
+
+Answer with JSON only: an array, best first, of \
+{"first": line, "last": line, "score": 1-10, "why": "one short sentence"}. \
+score is how well the passage makes the point (10: exactly this, said perfectly; 5: \
+touches on it; below 4: don't include it). An empty array when nothing makes the point."""
+
+
+@dataclass
+class Clip:
+    first: int
+    last: int
+    score: float
+    why: str
+
+
+def windows(lines: Sequence[Sentence]) -> List[Tuple[int, int]]:
+    """[start, end) line ranges of at most MAX_PICK_CHARS each, overlapping a little."""
+    out: List[Tuple[int, int]] = []
+    start = 0
+    while start < len(lines):
+        size, end = 0, start
+        while end < len(lines) and (end == start or size + len(lines[end].text) + 20
+                                    <= MAX_PICK_CHARS):
+            size += len(lines[end].text) + 20
+            end += 1
+        out.append((start, end))
+        if end >= len(lines):
+            break
+        start = max(start + 1, end - WINDOW_OVERLAP)
+    return out
+
+
+def prompt_for(idea: str, count: int, lines: Sequence[Sentence], window: Tuple[int, int],
+               name: Optional[str] = None) -> str:
+    start, end = window
+    numbered = "\n".join(f"{i} [{lines[i].start:.1f}-{lines[i].end:.1f}] {lines[i].text}"
+                         for i in range(start, end))
+    part = (f" (lines {start}-{end - 1} of {len(lines)}; the rest is read separately)"
+            if start > 0 or end < len(lines) else "")
+    return (f"The idea: {idea.strip()}\n\n"
+            f"Up to {count} passages.\n\n"
+            f"Transcript of {name or 'the video'}{part}:\n{numbered}")
+
+
+def _array(text: str) -> List[Any]:
+    """The JSON array in a reply, which a model may wrap in prose or a code fence."""
+    match = re.search(r"\[.*\]", text or "", re.DOTALL)
+    if not match:
+        return []
+    try:
+        found = json.loads(match.group(0))
+    except ValueError:
+        return []
+    return found if isinstance(found, list) else []
+
+
+def parse(text: str, lines: Sequence[Sentence], window: Tuple[int, int]) -> List[Clip]:
+    """The picker's passages that name real lines in its window and fit the rules.
+
+    Anything else is dropped rather than repaired: a line number out of range is a
+    passage the picker didn't read, and stretching or clamping it would cut something
+    nobody chose.
+    """
+    start, end = window
+    out: List[Clip] = []
+    for item in _array(text):
+        if not isinstance(item, dict):
+            continue
+        try:
+            first, last = int(item["first"]), int(item["last"])
+            score = float(item.get("score", 0))
+        except (KeyError, TypeError, ValueError):
+            continue
+        if not (start <= first <= last < end):
+            continue
+        if lines[last].end - lines[first].start > MAX_CLIP_SECONDS or score < 4:
+            continue
+        out.append(Clip(first, last, min(score, 10.0), str(item.get("why") or "")[:200]))
+    return out
+
+
+def best(found: Sequence[Clip], count: int) -> List[Clip]:
+    """The highest-scoring passages, no two overlapping, in the order they were said.
+
+    Overlap happens where windows share lines, and when a picker offers the same
+    thought cut two ways; either way the viewer would see it twice.
+    """
+    kept: List[Clip] = []
+    for clip in sorted(found, key=lambda c: (-c.score, c.first)):
+        if all(clip.last < k.first or clip.first > k.last for k in kept):
+            kept.append(clip)
+        if len(kept) == count:
+            break
+    return sorted(kept, key=lambda c: c.first)

backend/app/assistant/loop.py

@@ -76,10 +76,12 @@ def run(provider: ChatProvider, working: Working, *, system: Sequence[str],
                 raise Stopped()
             if call.name == "search_assets":
                 on_step("search", f"Searching the library: {call.arguments.get('query', '')!r}")
-            elif call.name == "read_transcript":
+            elif call.name in ("read_transcript", "find_clips"):
                 gam_id = call.arguments.get("gam_id", "")
                 seen = working.hits.get(gam_id)
-                on_step("search", f"Reading the transcript of {seen.name if seen else gam_id}")
+                doing = "Reading the transcript of" if call.name == "read_transcript" \
+                    else "Picking clips from"
+                on_step("search", f"{doing} {seen.name if seen else gam_id}")
             result = T.execute(working, call)
             results.append(result)
             if call.name in T.READS:

backend/app/assistant/prompt.py

@@ -109,14 +109,15 @@
 per video, the best match for your words, which is often not the best way to make the \
 point. So for each scene that will show someone speaking:
 1. Search, in a few phrasings, to find the candidate videos.
-2. `read_transcript` the most promising ones (several at once), all of each where it's \
-short, and around `said_at` where it's long. A hit with `more_moments` matched elsewhere \
-too; a video that matched only by name has no `said_at`, so read it.
-3. Choose the passage that best demonstrates the idea: a complete thought, said clearly, \
-that makes sense on its own, usually 5-40 s. Prefer the clearest, most concrete line to \
-the first mention. One video can give several clips to different scenes.
-4. `set_visual` with `trim` {"in": first line's start, "out": last line's end}, taken \
-from the transcript lines, never estimated.
+2. `find_clips` on each promising video (several at once), with the idea the scene has to \
+put across in a sentence. It reads the whole video and returns its best passages, scored, \
+each with a ready `trim`. A hit with `more_moments` matched elsewhere too, and a video that \
+matched only by name has no `said_at`: both are worth it.
+3. Compare the passages across videos and choose the one that best demonstrates the idea. \
+One video can give several clips to different scenes; give each scene its own idea.
+4. `set_visual` with that passage's `trim`, as given.
+Use `read_transcript` to check what is said either side of a passage, or when \
+`find_clips` isn't available; then cut from the transcript lines, never from an estimate.
 
 Choosing music: search with `type` audio for the sound the scene needs, in the words a \
 music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \

backend/app/assistant/tools.py

@@ -26,8 +26,9 @@
 from dataclasses import dataclass, field
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
+from app.assistant import clips
 from app.gam.client import GamClient, GamError
-from app.providers.base import ToolCall, ToolResult, ToolSpec
+from app.providers.base import ProviderError, ToolCall, ToolResult, ToolSpec
 from app.schemas_gam import GamTranscript
 from app.render.estimate import estimate
 from app.services import note_media as note_files
@@ -160,6 +161,15 @@ class Working:
     # Transcripts fetched this turn by gam_id, so reading one and then cutting from it
     # asks GAM once.
     transcripts: Dict[str, GamTranscript] = field(default_factory=dict)
+    # One model call with no tools: (system, prompt) -> the reply's text, its spend
+    # recorded against the turn. Set by the worker; None where there is no provider,
+    # and then find_clips says to read the transcript instead.
+    ask: Optional[Callable[[str, str], str]] = None
+    # Calls the clip-picking pass may make in one turn (a long video takes several),
+    # and what it found, so asking the same thing twice pays once.
+    max_picks: int = 16
+    picks: int = 0
+    picked: Dict[Tuple[str, str, int], str] = field(default_factory=dict)
 
     def __post_init__(self) -> None:
         self.data = copy.deepcopy(self.data)
@@ -381,6 +391,61 @@ def read_transcript(w: Working, gam_id: str, start: Optional[float] = None,
     return head + "\n" + "\n".join(out) + tail
 
 
+def find_clips(w: Working, gam_id: str, idea: str, count: int = 3) -> str:
+    if w.ask is None:
+        raise ToolFailed("Clip picking isn't available in this turn; use read_transcript.")
+    if not (idea or "").strip():
+        raise ToolFailed("Say the idea the clip has to put across.")
+    try:
+        count = max(1, min(5, int(count)))
+    except (TypeError, ValueError):
+        raise ToolFailed("count must be a whole number from 1 to 5.")
+    key = (gam_id, idea.strip().lower(), count)
+    if key in w.picked:
+        return w.picked[key]
+    lines = sentences(w.transcript(gam_id))
+    if not lines:
+        return ("This asset has no transcript: it hasn't been transcribed in the library, or "
+                "nothing is said in it.")
+    spans = clips.windows(lines)
+    left = w.max_picks - w.picks
+    if len(spans) > left:
+        raise ToolFailed(f"Clip picking has {left} of its {w.max_picks} calls left this turn "
+                         f"and this video needs {len(spans)}. Use read_transcript around a "
+                         "search hit's said_at instead.")
+    seen = w.hits.get(gam_id)
+    found: List[clips.Clip] = []
+    for span in spans:
+        w.picks += 1
+        try:
+            text = w.ask(clips.SYSTEM, clips.prompt_for(idea, count, lines, span,
+                                                        seen.name if seen else None))
+        except ProviderError as exc:
+            raise ToolFailed(f"The clip-picking pass failed ({exc}). Use read_transcript "
+                             "and choose the passage yourself.")
+        found += clips.parse(text, lines, span)
+    chosen = clips.best(found, count)
+    if not chosen:
+        result = ("Nothing in this video makes that point well. Try another video, or "
+                  "describe the idea differently.")
+    else:
+        duration = seen.duration if seen else None
+        rows = []
+        for clip in chosen:
+            first, last = lines[clip.first], lines[clip.last]
+            said = " ".join(lines[i].text for i in range(clip.first, clip.last + 1))
+            rows.append({
+                "trim": _padded(first.start, last.end, lines, clip.first, clip.last, duration),
+                "seconds": round(last.end - first.start, 1),
+                "score": clip.score,
+                "why": clip.why,
+                "says": said if len(said) <= 400 else said[:400] + "…",
+            })
+        result = _compact({"gam_id": gam_id, "idea": idea.strip(), "clips": rows})
+    w.picked[key] = result
+    return result
+
+
 def replace_spec(w: Working, spec: Dict[str, Any]) -> str:
     if not isinstance(spec, dict):
         raise ToolFailed("spec must be a VideoSpec object.")
@@ -523,30 +588,35 @@ def remove_segment(w: Working, segment_id: str) -> str:
     return result
 
 
+def _padded(a: float, b: float, lines: List[Sentence], first: Optional[int],
+            last: Optional[int], duration: Optional[float]) -> Optional[Dict[str, float]]:
+    """[a, b] as a trim, padded so a cut doesn't clip a word, but never into the
+    sentences either side of lines first..last."""
+    floor = lines[first - 1].end if first else 0.0
+    ceiling = lines[last + 1].start if last is not None and last + 1 < len(lines) else None
+    a = max(floor, a - MOMENT_PAD)
+    b = b + MOMENT_PAD if ceiling is None else min(b + MOMENT_PAD, max(b, ceiling))
+    if duration:
+        b = min(b, duration)
+    return {"in": round(a, 1), "out": round(b, 1)} if b > a else None
+
+
 def _moment_trim(hit: Hit, lines: List[Sentence] = ()) -> Optional[Dict[str, float]]:
     """The found moment as a cut, grown to the whole sentences it falls in when the
-    transcript is to hand, and padded without reaching into the next sentence."""
+    transcript is to hand."""
     if hit.start is None:
         return None
     a = hit.start
     b = hit.end if hit.end is not None else hit.start + DEFAULT_MOMENT_SECONDS
-    floor, ceiling = 0.0, None
     touching = [i for i, s in enumerate(lines) if s.end > a and s.start < b]
+    first = last = None
     if touching:
         first, last = touching[0], touching[-1]
         if a - lines[first].start <= MAX_SENTENCE_STRETCH:
             a = min(a, lines[first].start)
         if lines[last].end - b <= MAX_SENTENCE_STRETCH:
             b = max(b, lines[last].end)
-        if first > 0:
-            floor = lines[first - 1].end
-        if last + 1 < len(lines):
-            ceiling = lines[last + 1].start
-    a = max(floor, a - MOMENT_PAD)
-    b = b + MOMENT_PAD if ceiling is None else min(b + MOMENT_PAD, max(b, ceiling))
-    if hit.duration:
-        b = min(b, hit.duration)
-    return {"in": round(a, 1), "out": round(b, 1)} if b > a else None
+    return _padded(a, b, lines, first, last, hit.duration)
 
 
 def set_visual(w: Working, segment_id: str, gam_id: Optional[str] = None,
@@ -809,6 +879,17 @@ def run_estimate(w: Working) -> str:
               "transcript comes in pages and says where to carry on.",
               {"type": "object", "properties": {"gam_id": _STR, "start": _NUM, "end": _NUM},
                "required": ["gam_id"]}), read_transcript),
+    (ToolSpec("find_clips",
+              "Read a whole library video's transcript, in a separate pass, and return the "
+              "passages that best put across one idea, best scored, each with the trim to "
+              "pass to set_visual, what is said and why it fits. Use it on each candidate "
+              "video from search_assets before choosing a clip: it sees all of the video, "
+              "where a search shows one moment. idea: what the scene needs said, in a "
+              "sentence. count: passages wanted (1-5, default 3).",
+              {"type": "object", "properties": {
+                  "gam_id": _STR, "idea": _STR,
+                  "count": {"type": "integer", "minimum": 1, "maximum": 5}},
+               "required": ["gam_id", "idea"]}), find_clips),
     (ToolSpec("get_asset", "Details of one library asset by gam_id.",
               {"type": "object", "properties": {"gam_id": _STR}, "required": ["gam_id"]}),
      get_asset),
@@ -912,7 +993,8 @@ def run_estimate(w: Working) -> str:
 _HANDLERS = {spec.name: handler for spec, handler in TOOLS}
 
 # Read-only tools, which neither change the video nor count as an edit in the steps.
-READS = {"get_spec", "search_assets", "read_transcript", "get_asset", "estimate"}
+READS = {"get_spec", "search_assets", "read_transcript", "find_clips", "get_asset",
+         "estimate"}
 
 
 def execute(w: Working, call: ToolCall) -> ToolResult:

backend/app/assistant/worker.py

@@ -122,6 +122,7 @@ def on_reply(reply: Reply) -> None:
         attached = material(attachments, sees_images=sees_images)
         images = images_of(chat_id, attachments) if sees_images else []
         working = Working(base, gam=gam, max_generations=settings.assistant_max_generations,
+                          max_picks=settings.assistant_max_clip_picks,
                           note_media=media_ids(attachments))
 
         def generate(data, key, image_prompt, base_ids, outputs):
@@ -135,6 +136,16 @@ def generate(data, key, image_prompt, base_ids, outputs):
                 return row.model_label, row.estimate
 
         working.generate = generate
+
+        def ask(pick_system: str, pick_prompt: str) -> str:
+            # The clip-picking pass: the turn's own provider, a fresh conversation with no
+            # tools, its spend on the turn like any other call.
+            reply = provider.conversation(system=[pick_system], tools=[], history=[],
+                                          prompt=pick_prompt).ask()
+            on_reply(reply)
+            return reply.text
+
+        working.ask = ask
         system = [REFERENCE, attached,
                   context(base, focus=focus, from_scratch=project_id is None)]
         outcome = loop.run(provider, working, system=system, history=history, prompt=prompt,

backend/app/config.py

@@ -98,6 +98,11 @@ class Settings(BaseSettings):
     # Pictures one request may generate on its own (each is a paid call to fal.ai through
     # GAM). Past it the assistant leaves placeholders for the user to generate.
     assistant_max_generations: int = 6
+    # Calls the clip-picking pass may make in one request. Each reads up to about 15k
+    # tokens of transcript, once (an hour of speech is one or two), so this is roughly
+    # how many candidate videos a request can weigh before it falls back to reading
+    # transcripts itself.
+    assistant_max_clip_picks: int = 16
     # Material attached to a conversation (files, notes, web pages), as text. Each turn
     # reads all of it on every round, so the limits are about cost as much as context
     # length: 150k characters is roughly 40k tokens, cached after the first round.