From d1c92edd5ac5e40456e5b492fdd02525d8f0539b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 11:56:01 +0000 Subject: [PATCH] Tell music from voice recordings when choosing a soundtrack GAM now records what an audio file is (davior/gam#49). GVC reads it: - GamAssetRead.audio_kind: music, speech or sfx; null when not known or against a GAM without it, and for any value GVC does not recognise. - search_assets and get_asset include it as "kind", and the prompt says to use music (or a plainly musical unknown) for a soundtrack, never speech, with sfx for sound effects. - The Sound picker leads its summary line with "Speech" or "Sound effect", so a voice memo stands out in a list of tracks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DCL4jm6ogZBBhnjEfBBaXU --- backend/app/assistant/prompt.py | 4 ++++ backend/app/assistant/tools.py | 9 +++++++-- backend/app/gam/client.py | 1 + backend/app/schemas_gam.py | 4 ++++ backend/tests/test_assistant_tools.py | 14 ++++++++++++++ backend/tests/test_gam_api.py | 11 +++++++++++ docs/plan-of-attack.md | 1 + frontend/src/api/gam.ts | 16 ++++++++++++++++ .../src/components/FootageSearch.test.tsx | 19 +++++++++++++++++++ frontend/src/components/FootageSearch.tsx | 10 +++++----- 10 files changed, 82 insertions(+), 7 deletions(-) diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py index d0e2652..0e3fa98 100644 --- a/backend/app/assistant/prompt.py +++ b/backend/app/assistant/prompt.py @@ -108,6 +108,10 @@ Choosing music: search with `type` audio for the sound the scene needs, in the words a \ music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \ "uplifting acoustic guitar", "slow piano", "high energy"), not for the video's subject. \ +An audio hit's `kind` says what the file is: `music`, `speech` (a voice recording) or \ +`sfx` (a sound effect); no `kind` means not known. For a soundtrack use `music`, or an \ +unknown file whose name and `music` make it plainly a track; never `speech`. A sound \ +effect is `sfx`; extra voice is `speech`. An audio hit's `music` says what is known about the track: - Under narration, prefer `"vocals":"instrumental"`. Sung words fight the voiceover; \ a track with no `vocals` is unknown, so say so if you use it. diff --git a/backend/app/assistant/tools.py b/backend/app/assistant/tools.py index 9e63fcc..9c275e1 100644 --- a/backend/app/assistant/tools.py +++ b/backend/app/assistant/tools.py @@ -245,6 +245,8 @@ def search_assets(w: Working, query: str, type: Optional[str] = None, row["credit"] = asset.credit if asset.width and asset.height: row["size"] = f"{asset.width}x{asset.height}" + if asset.audio_kind: + row["kind"] = asset.audio_kind if asset.music: row["music"] = _music_row(asset.music) rows.append(row) @@ -267,6 +269,8 @@ def get_asset(w: Working, gam_id: str) -> str: "duration": asset.duration_seconds, "width": asset.width, "height": asset.height, "credit": asset.credit, "clip_of": asset.parent_asset_id} + if asset.audio_kind: + details["kind"] = asset.audio_kind if asset.music: details["music"] = _music_row(asset.music) return _compact(details) @@ -663,8 +667,9 @@ def run_estimate(w: Working) -> str: "Search the user's asset library (GAM) by name, description and what is SAID in " "videos. A hit on speech carries said_at [start, end] seconds: the moment to cut " "to. Use type to restrict to video, image or audio. An audio hit may carry " - "music: its genres, moods, instruments, bpm, key, energy and vocals, which " - "are also searchable, so search music by the sound you want.", + "kind (music, speech or sfx) and music: its genres, moods, instruments, bpm, " + "key, energy and vocals, which are also searchable, so search music by the " + "sound you want.", {"type": "object", "properties": { "query": _STR, "type": {"type": "string", "enum": ["video", "image", "audio"]}, "limit": {"type": "integer", "minimum": 1, "maximum": MAX_SEARCH_HITS}}, diff --git a/backend/app/gam/client.py b/backend/app/gam/client.py index 1358820..6ff46e2 100644 --- a/backend/app/gam/client.py +++ b/backend/app/gam/client.py @@ -58,6 +58,7 @@ def to_asset(raw: Dict[str, Any]) -> GamAssetRead: out_point=raw.get("out_point"), missing=bool(raw.get("missing")), music=_music(raw), + audio_kind=raw.get("audio_kind") if raw.get("audio_kind") in ("music", "speech", "sfx") else None, ) diff --git a/backend/app/schemas_gam.py b/backend/app/schemas_gam.py index 19baf51..c4f9167 100644 --- a/backend/app/schemas_gam.py +++ b/backend/app/schemas_gam.py @@ -51,6 +51,10 @@ class GamAssetRead(BaseModel): out_point: Optional[float] = None missing: bool = False music: Optional[GamMusic] = None + # What GAM says an audio file is: music | speech | sfx, null when not known (or GAM + # predates gam#49). Kept beside `music`, not in it: a voice memo has a kind and no + # profile, and the kind is what keeps it out of a soundtrack. + audio_kind: Optional[str] = None class GamHit(BaseModel): diff --git a/backend/tests/test_assistant_tools.py b/backend/tests/test_assistant_tools.py index 05d81d4..b0d1a87 100644 --- a/backend/tests/test_assistant_tools.py +++ b/backend/tests/test_assistant_tools.py @@ -144,6 +144,20 @@ def test_a_track_hit_says_what_it_sounds_like(gam, siblings): assert '"music"' not in untagged +def test_a_hit_says_whether_it_is_music_or_a_voice(gam, siblings): + siblings.hits = [ + hit(gam_asset("m1", "Theme", "audio", audio_kind="music", music_genres=["Ambient"])), + hit(gam_asset("v1", "Voice memo", "audio", audio_kind="speech")), + hit(gam_asset("u1", "Unknown", "audio")), + ] + w = Working(spec(), gam=gam) + found = run(w, "search_assets", query="calm", type="audio").content + + assert '"name":"Theme","type":"audio","kind":"music","music":{"genres":["Ambient"]}' in found + assert '"name":"Voice memo","type":"audio","kind":"speech"' in found + assert '"kind"' not in found.split('"Unknown"')[1] + + def test_get_asset_includes_the_music_profile(gam, siblings): siblings.assets["m1"] = gam_asset("m1", "Caravan theme", "audio", music_vocals=True) w = Working(spec(), gam=gam) diff --git a/backend/tests/test_gam_api.py b/backend/tests/test_gam_api.py index e4683c5..800259e 100644 --- a/backend/tests/test_gam_api.py +++ b/backend/tests/test_gam_api.py @@ -58,6 +58,17 @@ def test_a_track_gam_knows_nothing_about_has_no_profile(auth_client, siblings): assert [h["asset"]["music"] for h in data] == [None, None] +def test_a_track_carries_what_kind_of_audio_it_is(auth_client, siblings): + siblings.hits = [ + hit(gam_asset("m1", "Theme", "audio", audio_kind="music")), + hit(gam_asset("v1", "Voice memo", "audio", audio_kind="speech")), + hit(gam_asset("o1", "From an old GAM", "audio")), + hit(gam_asset("x1", "Odd value", "audio", audio_kind="podcast")), + ] + data = auth_client.get("/api/gam/search", params={"q": "x"}).json()["data"] + assert [h["asset"]["audio_kind"] for h in data] == ["music", "speech", None, None] + + def test_documents_are_not_offered_for_a_video(auth_client, siblings): siblings.hits = [hit(gam_asset("d1", "Report", "document")), hit(gam_asset("i1", "Map", "image"))] body = auth_client.get("/api/gam/search", params={"q": "x"}).json() diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md index ef0238a..0efd9a4 100644 --- a/docs/plan-of-attack.md +++ b/docs/plan-of-attack.md @@ -350,6 +350,7 @@ Specified here, applied in GAM's repository, not from this one. | GAM-4 | M9: documented read API, `/picker?embed=1` with `postMessage({ assetId, inPoint, outPoint })` | Picking assets and moments by hand in the editor | | GAM-5 | `CORS_ORIGIN` / CSP for `gvc.geckopico.com`, only where GVC's browser code calls GAM directly | Server-side calls with a Bearer token need neither | | GAM-6 | A music profile on audio assets (genres, moods, instruments, BPM, key, vocals, energy), read from ID3/Vorbis/MP4 tags and searchable. **[davior/gam#46](https://github.com/davior/gam/pull/46).** GVC reads it as `GamAssetRead.music`; against a GAM without it, `music` is null and tracks are chosen by name as before | The assistant choosing a soundtrack by how it sounds, and the Sound picker saying so | +| GAM-7 | `audio_kind` (music / speech / sfx) on audio assets. **[davior/gam#49](https://github.com/davior/gam/pull/49).** GVC reads it as `GamAssetRead.audio_kind`; null against a GAM without it | Keeping voice recordings out of a soundtrack, in the assistant and the Sound picker | GAM-1 and GAM-2 come first. They're small, and M1 below depends on them. diff --git a/frontend/src/api/gam.ts b/frontend/src/api/gam.ts index ace782c..8d0d211 100644 --- a/frontend/src/api/gam.ts +++ b/frontend/src/api/gam.ts @@ -25,6 +25,8 @@ export interface GamAsset { /** What a track sounds like (GAM's music profile). Null when GAM knows nothing about * it, or is older than the profile. Optional for test literals that predate it. */ music?: GamMusic | null + /** What GAM says the audio is. Null when not known, or from a GAM before gam#49. */ + audio_kind?: 'music' | 'speech' | 'sfx' | null } export interface GamMusic { @@ -99,3 +101,17 @@ export const gamApi = { .then((r) => r.data.data.assets) }, } + +const KIND_LABELS = { music: 'Music', speech: 'Speech', sfx: 'Sound effect' } as const + +/** + * The line under an audio hit: what kind of audio it is, then what it sounds like. + * "Speech" leads on purpose: a voice memo in a list of soundtracks should be the first + * thing a person notices about it. A music track with a profile needs no "Music" label. + */ +export function audioSummary(asset: Pick): string { + const sound = musicSummary(asset.music) + const kind = asset.audio_kind ? KIND_LABELS[asset.audio_kind] : '' + if (asset.audio_kind === 'music' && sound) return sound + return [kind, sound].filter(Boolean).join(' · ') +} diff --git a/frontend/src/components/FootageSearch.test.tsx b/frontend/src/components/FootageSearch.test.tsx index 9603dfb..636ee2c 100644 --- a/frontend/src/components/FootageSearch.test.tsx +++ b/frontend/src/components/FootageSearch.test.tsx @@ -2,6 +2,7 @@ import { render, screen } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { afterEach, describe, expect, it, vi } from 'vitest' import { + audioSummary, gamApi, musicSummary, type GamAsset, @@ -184,6 +185,24 @@ describe('FootageSearch', () => { ).toBe('') }) + it('puts the kind of audio first, so a voice memo stands out', () => { + expect(audioSummary({ audio_kind: 'speech', music: null })).toBe('Speech') + expect(audioSummary({ audio_kind: 'sfx', music: null })).toBe('Sound effect') + expect(audioSummary({ audio_kind: 'music', music: null })).toBe('Music') + const music = { + genres: ['Ambient'], + moods: [], + instruments: [], + bpm: null, + key: null, + vocals: null, + energy: null, + } + expect(audioSummary({ audio_kind: 'music', music })).toBe('Ambient') + expect(audioSummary({ audio_kind: null, music })).toBe('Ambient') + expect(audioSummary({ audio_kind: null, music: null })).toBe('') + }) + it('shows what went wrong when the library cannot be reached', async () => { vi.spyOn(gamApi, 'search').mockRejectedValue( new Error('Could not reach the asset library') diff --git a/frontend/src/components/FootageSearch.tsx b/frontend/src/components/FootageSearch.tsx index 457fad0..cd3b050 100644 --- a/frontend/src/components/FootageSearch.tsx +++ b/frontend/src/components/FootageSearch.tsx @@ -3,7 +3,7 @@ import { Film, Image as ImageIcon, Search, X } from 'lucide-react' import { apiErrorMessage } from '@/api/client' import { gamApi, - musicSummary, + audioSummary, type GamHit, type GamSearch, type MediaKind, @@ -194,9 +194,9 @@ export default function FootageSearch({ {h.end_time !== null && `–${clock(h.end_time)}`} )} - {musicSummary(h.asset.music) && ( + {audioSummary(h.asset) && ( - {musicSummary(h.asset.music)} + {audioSummary(h.asset)} )} {h.snippet && } @@ -246,9 +246,9 @@ export default function FootageSearch({ ? 'Uses the whole video' : 'A still picture'}

- {musicSummary(selected.asset.music) && ( + {audioSummary(selected.asset) && (

- {musicSummary(selected.asset.music)} + {audioSummary(selected.asset)}

)} {selected.asset.credit && (