Tell music from voice recordings when choosing a soundtrack

backend/app/assistant/prompt.py

@@ -108,6 +108,10 @@
 Choosing music: search with `type` audio for the sound the scene needs, in the words a \
 music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \
 "uplifting acoustic guitar", "slow piano", "high energy"), not for the video's subject. \
+An audio hit's `kind` says what the file is: `music`, `speech` (a voice recording) or \
+`sfx` (a sound effect); no `kind` means not known. For a soundtrack use `music`, or an \
+unknown file whose name and `music` make it plainly a track; never `speech`. A sound \
+effect is `sfx`; extra voice is `speech`.
 An audio hit's `music` says what is known about the track:
 - Under narration, prefer `"vocals":"instrumental"`. Sung words fight the voiceover; \
 a track with no `vocals` is unknown, so say so if you use it.

backend/app/assistant/tools.py

@@ -245,6 +245,8 @@ def search_assets(w: Working, query: str, type: Optional[str] = None,
             row["credit"] = asset.credit
         if asset.width and asset.height:
             row["size"] = f"{asset.width}x{asset.height}"
+        if asset.audio_kind:
+            row["kind"] = asset.audio_kind
         if asset.music:
             row["music"] = _music_row(asset.music)
         rows.append(row)
@@ -267,6 +269,8 @@ def get_asset(w: Working, gam_id: str) -> str:
                "duration": asset.duration_seconds, "width": asset.width,
                "height": asset.height, "credit": asset.credit,
                "clip_of": asset.parent_asset_id}
+    if asset.audio_kind:
+        details["kind"] = asset.audio_kind
     if asset.music:
         details["music"] = _music_row(asset.music)
     return _compact(details)
@@ -663,8 +667,9 @@ def run_estimate(w: Working) -> str:
               "Search the user's asset library (GAM) by name, description and what is SAID in "
               "videos. A hit on speech carries said_at [start, end] seconds: the moment to cut "
               "to. Use type to restrict to video, image or audio. An audio hit may carry "
-              "music: its genres, moods, instruments, bpm, key, energy and vocals, which "
-              "are also searchable, so search music by the sound you want.",
+              "kind (music, speech or sfx) and music: its genres, moods, instruments, bpm, "
+              "key, energy and vocals, which are also searchable, so search music by the "
+              "sound you want.",
               {"type": "object", "properties": {
                   "query": _STR, "type": {"type": "string", "enum": ["video", "image", "audio"]},
                   "limit": {"type": "integer", "minimum": 1, "maximum": MAX_SEARCH_HITS}},

backend/app/gam/client.py

@@ -58,6 +58,7 @@ def to_asset(raw: Dict[str, Any]) -> GamAssetRead:
         out_point=raw.get("out_point"),
         missing=bool(raw.get("missing")),
         music=_music(raw),
+        audio_kind=raw.get("audio_kind") if raw.get("audio_kind") in ("music", "speech", "sfx") else None,
     )
 
 

backend/app/schemas_gam.py

@@ -51,6 +51,10 @@ class GamAssetRead(BaseModel):
     out_point: Optional[float] = None
     missing: bool = False
     music: Optional[GamMusic] = None
+    # What GAM says an audio file is: music | speech | sfx, null when not known (or GAM
+    # predates gam#49). Kept beside `music`, not in it: a voice memo has a kind and no
+    # profile, and the kind is what keeps it out of a soundtrack.
+    audio_kind: Optional[str] = None
 
 
 class GamHit(BaseModel):

backend/tests/test_assistant_tools.py

@@ -144,6 +144,20 @@ def test_a_track_hit_says_what_it_sounds_like(gam, siblings):
     assert '"music"' not in untagged
 
 
+def test_a_hit_says_whether_it_is_music_or_a_voice(gam, siblings):
+    siblings.hits = [
+        hit(gam_asset("m1", "Theme", "audio", audio_kind="music", music_genres=["Ambient"])),
+        hit(gam_asset("v1", "Voice memo", "audio", audio_kind="speech")),
+        hit(gam_asset("u1", "Unknown", "audio")),
+    ]
+    w = Working(spec(), gam=gam)
+    found = run(w, "search_assets", query="calm", type="audio").content
+
+    assert '"name":"Theme","type":"audio","kind":"music","music":{"genres":["Ambient"]}' in found
+    assert '"name":"Voice memo","type":"audio","kind":"speech"' in found
+    assert '"kind"' not in found.split('"Unknown"')[1]
+
+
 def test_get_asset_includes_the_music_profile(gam, siblings):
     siblings.assets["m1"] = gam_asset("m1", "Caravan theme", "audio", music_vocals=True)
     w = Working(spec(), gam=gam)

backend/tests/test_gam_api.py

@@ -58,6 +58,17 @@ def test_a_track_gam_knows_nothing_about_has_no_profile(auth_client, siblings):
     assert [h["asset"]["music"] for h in data] == [None, None]
 
 
+def test_a_track_carries_what_kind_of_audio_it_is(auth_client, siblings):
+    siblings.hits = [
+        hit(gam_asset("m1", "Theme", "audio", audio_kind="music")),
+        hit(gam_asset("v1", "Voice memo", "audio", audio_kind="speech")),
+        hit(gam_asset("o1", "From an old GAM", "audio")),
+        hit(gam_asset("x1", "Odd value", "audio", audio_kind="podcast")),
+    ]
+    data = auth_client.get("/api/gam/search", params={"q": "x"}).json()["data"]
+    assert [h["asset"]["audio_kind"] for h in data] == ["music", "speech", None, None]
+
+
 def test_documents_are_not_offered_for_a_video(auth_client, siblings):
     siblings.hits = [hit(gam_asset("d1", "Report", "document")), hit(gam_asset("i1", "Map", "image"))]
     body = auth_client.get("/api/gam/search", params={"q": "x"}).json()

docs/plan-of-attack.md

@@ -350,6 +350,7 @@ Specified here, applied in GAM's repository, not from this one.
 | GAM-4 | M9: documented read API, `/picker?embed=1` with `postMessage({ assetId, inPoint, outPoint })` | Picking assets and moments by hand in the editor |
 | GAM-5 | `CORS_ORIGIN` / CSP for `gvc.geckopico.com`, only where GVC's browser code calls GAM directly | Server-side calls with a Bearer token need neither |
 | GAM-6 | A music profile on audio assets (genres, moods, instruments, BPM, key, vocals, energy), read from ID3/Vorbis/MP4 tags and searchable. **[davior/gam#46](https://github.com/davior/gam/pull/46).** GVC reads it as `GamAssetRead.music`; against a GAM without it, `music` is null and tracks are chosen by name as before | The assistant choosing a soundtrack by how it sounds, and the Sound picker saying so |
+| GAM-7 | `audio_kind` (music / speech / sfx) on audio assets. **[davior/gam#49](https://github.com/davior/gam/pull/49).** GVC reads it as `GamAssetRead.audio_kind`; null against a GAM without it | Keeping voice recordings out of a soundtrack, in the assistant and the Sound picker |
 
 GAM-1 and GAM-2 come first. They're small, and M1 below depends on them.
 

frontend/src/api/gam.ts

@@ -25,6 +25,8 @@ export interface GamAsset {
   /** What a track sounds like (GAM's music profile). Null when GAM knows nothing about
    *  it, or is older than the profile. Optional for test literals that predate it. */
   music?: GamMusic | null
+  /** What GAM says the audio is. Null when not known, or from a GAM before gam#49. */
+  audio_kind?: 'music' | 'speech' | 'sfx' | null
 }
 
 export interface GamMusic {
@@ -99,3 +101,17 @@ export const gamApi = {
       .then((r) => r.data.data.assets)
   },
 }
+
+const KIND_LABELS = { music: 'Music', speech: 'Speech', sfx: 'Sound effect' } as const
+
+/**
+ * The line under an audio hit: what kind of audio it is, then what it sounds like.
+ * "Speech" leads on purpose: a voice memo in a list of soundtracks should be the first
+ * thing a person notices about it. A music track with a profile needs no "Music" label.
+ */
+export function audioSummary(asset: Pick<GamAsset, 'audio_kind' | 'music'>): string {
+  const sound = musicSummary(asset.music)
+  const kind = asset.audio_kind ? KIND_LABELS[asset.audio_kind] : ''
+  if (asset.audio_kind === 'music' && sound) return sound
+  return [kind, sound].filter(Boolean).join(' · ')
+}

frontend/src/components/FootageSearch.test.tsx

@@ -2,6 +2,7 @@ import { render, screen } from '@testing-library/react'
 import userEvent from '@testing-library/user-event'
 import { afterEach, describe, expect, it, vi } from 'vitest'
 import {
+  audioSummary,
   gamApi,
   musicSummary,
   type GamAsset,
@@ -184,6 +185,24 @@ describe('FootageSearch', () => {
     ).toBe('')
   })
 
+  it('puts the kind of audio first, so a voice memo stands out', () => {
+    expect(audioSummary({ audio_kind: 'speech', music: null })).toBe('Speech')
+    expect(audioSummary({ audio_kind: 'sfx', music: null })).toBe('Sound effect')
+    expect(audioSummary({ audio_kind: 'music', music: null })).toBe('Music')
+    const music = {
+      genres: ['Ambient'],
+      moods: [],
+      instruments: [],
+      bpm: null,
+      key: null,
+      vocals: null,
+      energy: null,
+    }
+    expect(audioSummary({ audio_kind: 'music', music })).toBe('Ambient')
+    expect(audioSummary({ audio_kind: null, music })).toBe('Ambient')
+    expect(audioSummary({ audio_kind: null, music: null })).toBe('')
+  })
+
   it('shows what went wrong when the library cannot be reached', async () => {
     vi.spyOn(gamApi, 'search').mockRejectedValue(
       new Error('Could not reach the asset library')

frontend/src/components/FootageSearch.tsx

@@ -3,7 +3,7 @@ import { Film, Image as ImageIcon, Search, X } from 'lucide-react'
 import { apiErrorMessage } from '@/api/client'
 import {
   gamApi,
-  musicSummary,
+  audioSummary,
   type GamHit,
   type GamSearch,
   type MediaKind,
@@ -194,9 +194,9 @@ export default function FootageSearch({
                         {h.end_time !== null && `–${clock(h.end_time)}`}
                       </span>
                     )}
-                    {musicSummary(h.asset.music) && (
+                    {audioSummary(h.asset) && (
                       <span className="block truncate text-xs text-gray-500">
-                        {musicSummary(h.asset.music)}
+                        {audioSummary(h.asset)}
                       </span>
                     )}
                     {h.snippet && <Snippet text={h.snippet} />}
@@ -246,9 +246,9 @@ export default function FootageSearch({
                       ? 'Uses the whole video'
                       : 'A still picture'}
               </p>
-              {musicSummary(selected.asset.music) && (
+              {audioSummary(selected.asset) && (
                 <p className="text-xs text-gray-600 dark:text-gray-400">
-                  {musicSummary(selected.asset.music)}
+                  {audioSummary(selected.asset)}
                 </p>
               )}
               {selected.asset.credit && (