Read GAM's music profile and use it to choose soundtracks

backend/app/assistant/prompt.py

@@ -105,6 +105,19 @@
 `add_placeholder` with a clear description of what to find or generate. Never invent a \
 gam_id or an asset key that isn't in the registry.
 
+Choosing music: search with `type` audio for the sound the scene needs, in the words a \
+music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \
+"uplifting acoustic guitar", "slow piano", "high energy"), not for the video's subject. \
+An audio hit's `music` says what is known about the track:
+- Under narration, prefer `"vocals":"instrumental"`. Sung words fight the voiceover; \
+a track with no `vocals` is unknown, so say so if you use it.
+- Match `energy` and `moods` to the section: low and calm under explanation, higher at \
+an opening, a reveal or an ending. When the mood changes, change the track (see the \
+anchors above) rather than stretching one across both.
+- Tracks that share a `bpm` and `key` crossfade cleanly into each other.
+- A hit with no `music` is known only by its name; judge from that, and tell the user \
+they can describe the track in GAM's Music tab so it is found by its sound next time.
+
 ## Generating pictures
 `generate_image` makes a still with AI (through the user's asset library) and puts it in \
 the video; it fills in by itself a few seconds later. It costs money, so:

backend/app/assistant/tools.py

@@ -198,6 +198,27 @@ def get_spec(w: Working, segment_id: Optional[str] = None) -> str:
     return _compact(w.data)
 
 
+def _music_row(music) -> Dict[str, Any]:
+    """A track's profile with the empty parts left out, so a hit costs no more tokens
+    than what is actually known about it. Vocals are spelled as the word that matters
+    when choosing music under narration, and left out entirely when not known: a missing
+    key is what tells the model it cannot assume either way."""
+    row: Dict[str, Any] = {}
+    for name in ("genres", "moods", "instruments"):
+        terms = getattr(music, name)
+        if terms:
+            row[name] = terms
+    if music.bpm is not None:
+        row["bpm"] = round(music.bpm)
+    if music.key:
+        row["key"] = music.key
+    if music.energy:
+        row["energy"] = music.energy
+    if music.vocals is not None:
+        row["vocals"] = "yes" if music.vocals else "instrumental"
+    return row
+
+
 def search_assets(w: Working, query: str, type: Optional[str] = None,
                   limit: int = 8) -> str:
     if w.gam is None:
@@ -224,6 +245,8 @@ def search_assets(w: Working, query: str, type: Optional[str] = None,
             row["credit"] = asset.credit
         if asset.width and asset.height:
             row["size"] = f"{asset.width}x{asset.height}"
+        if asset.music:
+            row["music"] = _music_row(asset.music)
         rows.append(row)
     if not rows:
         return (f"Nothing in the library matches {query!r}. Try other words, or leave a "
@@ -240,10 +263,13 @@ def get_asset(w: Working, gam_id: str) -> str:
         raise ToolFailed(f"The asset library failed: {exc.message}")
     if asset is None:
         raise ToolFailed(f"No asset {gam_id!r} in the library.")
-    return _compact({"gam_id": asset.id, "name": asset.name, "type": asset.asset_type,
-                     "duration": asset.duration_seconds, "width": asset.width,
-                     "height": asset.height, "credit": asset.credit,
-                     "clip_of": asset.parent_asset_id})
+    details = {"gam_id": asset.id, "name": asset.name, "type": asset.asset_type,
+               "duration": asset.duration_seconds, "width": asset.width,
+               "height": asset.height, "credit": asset.credit,
+               "clip_of": asset.parent_asset_id}
+    if asset.music:
+        details["music"] = _music_row(asset.music)
+    return _compact(details)
 
 
 def replace_spec(w: Working, spec: Dict[str, Any]) -> str:
@@ -636,7 +662,9 @@ def run_estimate(w: Working) -> str:
     (ToolSpec("search_assets",
               "Search the user's asset library (GAM) by name, description and what is SAID in "
               "videos. A hit on speech carries said_at [start, end] seconds: the moment to cut "
-              "to. Use type to restrict to video, image or audio.",
+              "to. Use type to restrict to video, image or audio. An audio hit may carry "
+              "music: its genres, moods, instruments, bpm, key, energy and vocals, which "
+              "are also searchable, so search music by the sound you want.",
               {"type": "object", "properties": {
                   "query": _STR, "type": {"type": "string", "enum": ["video", "image", "audio"]},
                   "limit": {"type": "integer", "minimum": 1, "maximum": MAX_SEARCH_HITS}},

backend/app/gam/client.py

@@ -18,7 +18,7 @@
 
 from app import outbound
 from app.config import settings
-from app.schemas_gam import GamAssetRead, GamHit, GamSearch
+from app.schemas_gam import GamAssetRead, GamHit, GamMusic, GamSearch
 
 logger = logging.getLogger(__name__)
 
@@ -57,9 +57,37 @@ def to_asset(raw: Dict[str, Any]) -> GamAssetRead:
         in_point=raw.get("in_point"),
         out_point=raw.get("out_point"),
         missing=bool(raw.get("missing")),
+        music=_music(raw),
     )
 
 
+def _music(raw: Dict[str, Any]) -> Optional[GamMusic]:
+    """GAM's flat `music_*` fields as one object, or None when none of them says anything.
+
+    None rather than an empty profile so "GAM knows nothing about this track" and "GAM
+    is older than the profile" read the same, and nothing downstream has to tell an
+    empty object from a missing one.
+    """
+    def terms(name: str) -> List[str]:
+        value = raw.get(name)
+        return [t for t in value if isinstance(t, str)] if isinstance(value, list) else []
+
+    music = GamMusic(
+        genres=terms("music_genres"),
+        moods=terms("music_moods"),
+        instruments=terms("music_instruments"),
+        bpm=raw.get("music_bpm"),
+        key=raw.get("music_key"),
+        vocals=raw.get("music_vocals"),
+        energy=raw.get("music_energy"),
+    )
+    if not (music.genres or music.moods or music.instruments) and all(
+        value is None for value in (music.bpm, music.key, music.vocals, music.energy)
+    ):
+        return None
+    return music
+
+
 class GamClient:
     def __init__(self, token: Optional[str]):
         headers = {"Authorization": f"Bearer {token}"} if token else {}

backend/app/schemas_gam.py

@@ -9,6 +9,25 @@
 from pydantic import BaseModel, Field
 
 
+class GamMusic(BaseModel):
+    """What a track sounds like, from GAM's music profile (gam#46).
+
+    The one thing that lets a soundtrack be chosen for a scene rather than by its title.
+    Null on the asset when GAM knows nothing about it, or is older than the profile.
+    """
+
+    genres: List[str] = Field(default_factory=list)
+    moods: List[str] = Field(default_factory=list)
+    instruments: List[str] = Field(default_factory=list)
+    bpm: Optional[float] = None
+    # As GAM stores it: "Am", "A minor" and Camelot "8A" are all left as written.
+    key: Optional[str] = None
+    # Null is "not known", which is not False ("instrumental"). Treating the two as one
+    # would put songs with words under narration.
+    vocals: Optional[bool] = None
+    energy: Optional[str] = None  # low | medium | high
+
+
 class GamAssetRead(BaseModel):
     id: str
     name: str
@@ -31,6 +50,7 @@ class GamAssetRead(BaseModel):
     in_point: Optional[float] = None
     out_point: Optional[float] = None
     missing: bool = False
+    music: Optional[GamMusic] = None
 
 
 class GamHit(BaseModel):

backend/tests/test_assistant_tools.py

@@ -125,6 +125,31 @@ def test_a_search_hit_becomes_the_visual_with_its_moment_as_the_trim(gam, siblin
         "from": "gam", "id": "r", "type": "video", "label": "River interview"}
 
 
+def test_a_track_hit_says_what_it_sounds_like(gam, siblings):
+    siblings.hits = [
+        hit(gam_asset("m1", "Caravan theme", "audio", music_genres=["Ambient"],
+                      music_moods=["Tense"], music_instruments=[], music_bpm=91.6,
+                      music_key=None, music_vocals=False, music_energy="low")),
+        hit(gam_asset("m2", "Untagged song", "audio")),
+    ]
+    w = Working(spec(), gam=gam)
+    found = run(w, "search_assets", query="tense ambient", type="audio").content
+
+    # Only what is known: no empty instruments, no null key, and vocals as the word that
+    # matters under narration.
+    assert ('"music":{"genres":["Ambient"],"moods":["Tense"],"bpm":92,"energy":"low",'
+            '"vocals":"instrumental"}') in found
+    # A track with no profile carries no music key at all, rather than an empty one.
+    untagged = found.split('"Untagged song"')[1]
+    assert '"music"' not in untagged
+
+
+def test_get_asset_includes_the_music_profile(gam, siblings):
+    siblings.assets["m1"] = gam_asset("m1", "Caravan theme", "audio", music_vocals=True)
+    w = Working(spec(), gam=gam)
+    assert '"music":{"vocals":"yes"}' in run(w, "get_asset", gam_id="m1").content
+
+
 def test_an_empty_search_says_to_try_again_or_leave_a_placeholder(gam, siblings):
     siblings.hits = []
     w = Working(spec(), gam=gam)

backend/tests/test_gam_api.py

@@ -36,6 +36,28 @@ def test_a_gam_without_end_times_still_answers(auth_client, siblings):
     assert top["start_time"] == 10.0 and top["end_time"] is None
 
 
+def test_a_track_carries_its_music_profile(auth_client, siblings):
+    track = gam_asset("m1", "Caravan theme", "audio", music_genres=["Ambient"],
+                      music_moods=["Tense"], music_instruments=[], music_bpm=92.0,
+                      music_key="Am", music_vocals=False, music_energy="low")
+    siblings.hits = [hit(track)]
+
+    asset = auth_client.get("/api/gam/search", params={"q": "tense"}).json()["data"][0]["asset"]
+
+    assert asset["music"] == {"genres": ["Ambient"], "moods": ["Tense"], "instruments": [],
+                              "bpm": 92.0, "key": "Am", "vocals": False, "energy": "low"}
+
+
+def test_a_track_gam_knows_nothing_about_has_no_profile(auth_client, siblings):
+    """An empty profile and a GAM older than gam#46 read the same: null."""
+    siblings.hits = [
+        hit(gam_asset("m1", "Untagged", "audio", music_genres=[], music_vocals=None)),
+        hit(gam_asset("m2", "From an old GAM", "audio")),
+    ]
+    data = auth_client.get("/api/gam/search", params={"q": "x"}).json()["data"]
+    assert [h["asset"]["music"] for h in data] == [None, None]
+
+
 def test_documents_are_not_offered_for_a_video(auth_client, siblings):
     siblings.hits = [hit(gam_asset("d1", "Report", "document")), hit(gam_asset("i1", "Map", "image"))]
     body = auth_client.get("/api/gam/search", params={"q": "x"}).json()

docs/plan-of-attack.md

@@ -349,6 +349,7 @@ Specified here, applied in GAM's repository, not from this one.
 | GAM-3 | `origin_project_id` / `origin_reference_id` on `Asset` (migration) + upload with `source: "gvc_export"`. **[davior/gam#37](https://github.com/davior/gam/pull/37).** Against a GAM without it, a save lands as an ordinary upload | Filing renders back into the library |
 | GAM-4 | M9: documented read API, `/picker?embed=1` with `postMessage({ assetId, inPoint, outPoint })` | Picking assets and moments by hand in the editor |
 | GAM-5 | `CORS_ORIGIN` / CSP for `gvc.geckopico.com`, only where GVC's browser code calls GAM directly | Server-side calls with a Bearer token need neither |
+| GAM-6 | A music profile on audio assets (genres, moods, instruments, BPM, key, vocals, energy), read from ID3/Vorbis/MP4 tags and searchable. **[davior/gam#46](https://github.com/davior/gam/pull/46).** GVC reads it as `GamAssetRead.music`; against a GAM without it, `music` is null and tracks are chosen by name as before | The assistant choosing a soundtrack by how it sounds, and the Sound picker saying so |
 
 GAM-1 and GAM-2 come first. They're small, and M1 below depends on them.
 

frontend/src/api/gam.ts

@@ -22,6 +22,37 @@ export interface GamAsset {
   in_point: number | null
   out_point: number | null
   missing: boolean
+  /** What a track sounds like (GAM's music profile). Null when GAM knows nothing about
+   *  it, or is older than the profile. Optional for test literals that predate it. */
+  music?: GamMusic | null
+}
+
+export interface GamMusic {
+  genres: string[]
+  moods: string[]
+  instruments: string[]
+  bpm: number | null
+  key: string | null
+  /** Null is "not known", which is not false ("instrumental"). */
+  vocals: boolean | null
+  energy: 'low' | 'medium' | 'high' | string | null
+}
+
+/**
+ * One line saying what a track sounds like: "Ambient, Tense · 92 BPM · Am · Low energy ·
+ * Instrumental". Empty when nothing is known, so the caller can leave the line out.
+ */
+export function musicSummary(music: GamMusic | null | undefined): string {
+  if (!music) return ''
+  const parts: string[] = []
+  const words = [...music.genres, ...music.moods, ...music.instruments]
+  if (words.length) parts.push(words.join(', '))
+  if (music.bpm !== null) parts.push(`${Math.round(music.bpm)} BPM`)
+  if (music.key) parts.push(music.key)
+  if (music.energy)
+    parts.push(`${music.energy[0].toUpperCase()}${music.energy.slice(1)} energy`)
+  if (music.vocals !== null) parts.push(music.vocals ? 'Vocals' : 'Instrumental')
+  return parts.join(' · ')
 }
 
 export interface GamHit {

frontend/src/components/FootageSearch.test.tsx

@@ -1,7 +1,13 @@
 import { render, screen } from '@testing-library/react'
 import userEvent from '@testing-library/user-event'
 import { afterEach, describe, expect, it, vi } from 'vitest'
-import { gamApi, type GamAsset, type GamHit, type GamSearch } from '@/api/gam'
+import {
+  gamApi,
+  musicSummary,
+  type GamAsset,
+  type GamHit,
+  type GamSearch,
+} from '@/api/gam'
 import FootageSearch from '@/components/FootageSearch'
 
 function asset(overrides: Partial<GamAsset> = {}): GamAsset {
@@ -127,6 +133,57 @@ describe('FootageSearch', () => {
     expect(screen.getByText('Nothing in the library matches.')).toBeInTheDocument()
   })
 
+  it('says what a track sounds like, in the list and once picked', async () => {
+    const track: GamHit = {
+      asset: asset({
+        id: 'm1',
+        name: 'Caravan theme',
+        asset_type: 'audio',
+        thumb_url: null,
+        music: {
+          genres: ['Ambient'],
+          moods: ['Tense'],
+          instruments: [],
+          bpm: 92,
+          key: 'Am',
+          vocals: false,
+          energy: 'low',
+        },
+      }),
+      score: 0.03,
+      snippet: '',
+      start_time: null,
+      end_time: null,
+    }
+    vi.spyOn(gamApi, 'search').mockResolvedValue(result([track]))
+    render(
+      <FootageSearch title="Music" purpose="track" onPick={vi.fn()} onClose={vi.fn()} />
+    )
+
+    await userEvent.type(screen.getByLabelText('Search the asset library'), 'tense')
+    await userEvent.click(screen.getByRole('button', { name: 'Search' }))
+
+    const line = 'Ambient, Tense · 92 BPM · Am · Low energy · Instrumental'
+    expect(await screen.findByText(line)).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: /Caravan theme/ }))
+    expect(screen.getAllByText(line)).toHaveLength(2)
+  })
+
+  it('leaves the line out for a track nothing is known about', () => {
+    expect(musicSummary(null)).toBe('')
+    expect(
+      musicSummary({
+        genres: [],
+        moods: [],
+        instruments: [],
+        bpm: null,
+        key: null,
+        vocals: null,
+        energy: null,
+      })
+    ).toBe('')
+  })
+
   it('shows what went wrong when the library cannot be reached', async () => {
     vi.spyOn(gamApi, 'search').mockRejectedValue(
       new Error('Could not reach the asset library')

frontend/src/components/FootageSearch.tsx

@@ -1,7 +1,13 @@
 import { useState, type FormEvent, type ReactNode } from 'react'
 import { Film, Image as ImageIcon, Search, X } from 'lucide-react'
 import { apiErrorMessage } from '@/api/client'
-import { gamApi, type GamHit, type GamSearch, type MediaKind } from '@/api/gam'
+import {
+  gamApi,
+  musicSummary,
+  type GamHit,
+  type GamSearch,
+  type MediaKind,
+} from '@/api/gam'
 import { clock, momentTrim, type MomentSize, type PlaceMode } from '@/spec/footage'
 
 /**
@@ -188,6 +194,11 @@ export default function FootageSearch({
                         {h.end_time !== null && `–${clock(h.end_time)}`}
                       </span>
                     )}
+                    {musicSummary(h.asset.music) && (
+                      <span className="block truncate text-xs text-gray-500">
+                        {musicSummary(h.asset.music)}
+                      </span>
+                    )}
                     {h.snippet && <Snippet text={h.snippet} />}
                   </span>
                 </button>
@@ -235,6 +246,11 @@ export default function FootageSearch({
                       ? 'Uses the whole video'
                       : 'A still picture'}
               </p>
+              {musicSummary(selected.asset.music) && (
+                <p className="text-xs text-gray-600 dark:text-gray-400">
+                  {musicSummary(selected.asset.music)}
+                </p>
+              )}
               {selected.asset.credit && (
                 <p className="text-xs text-gray-500">{selected.asset.credit}</p>
               )}