diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py index 07ba105..d0e2652 100644 --- a/backend/app/assistant/prompt.py +++ b/backend/app/assistant/prompt.py @@ -105,6 +105,19 @@ `add_placeholder` with a clear description of what to find or generate. Never invent a \ gam_id or an asset key that isn't in the registry. +Choosing music: search with `type` audio for the sound the scene needs, in the words a \ +music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \ +"uplifting acoustic guitar", "slow piano", "high energy"), not for the video's subject. \ +An audio hit's `music` says what is known about the track: +- Under narration, prefer `"vocals":"instrumental"`. Sung words fight the voiceover; \ +a track with no `vocals` is unknown, so say so if you use it. +- Match `energy` and `moods` to the section: low and calm under explanation, higher at \ +an opening, a reveal or an ending. When the mood changes, change the track (see the \ +anchors above) rather than stretching one across both. +- Tracks that share a `bpm` and `key` crossfade cleanly into each other. +- A hit with no `music` is known only by its name; judge from that, and tell the user \ +they can describe the track in GAM's Music tab so it is found by its sound next time. + ## Generating pictures `generate_image` makes a still with AI (through the user's asset library) and puts it in \ the video; it fills in by itself a few seconds later. It costs money, so: diff --git a/backend/app/assistant/tools.py b/backend/app/assistant/tools.py index 5297361..9e63fcc 100644 --- a/backend/app/assistant/tools.py +++ b/backend/app/assistant/tools.py @@ -198,6 +198,27 @@ def get_spec(w: Working, segment_id: Optional[str] = None) -> str: return _compact(w.data) +def _music_row(music) -> Dict[str, Any]: + """A track's profile with the empty parts left out, so a hit costs no more tokens + than what is actually known about it. Vocals are spelled as the word that matters + when choosing music under narration, and left out entirely when not known: a missing + key is what tells the model it cannot assume either way.""" + row: Dict[str, Any] = {} + for name in ("genres", "moods", "instruments"): + terms = getattr(music, name) + if terms: + row[name] = terms + if music.bpm is not None: + row["bpm"] = round(music.bpm) + if music.key: + row["key"] = music.key + if music.energy: + row["energy"] = music.energy + if music.vocals is not None: + row["vocals"] = "yes" if music.vocals else "instrumental" + return row + + def search_assets(w: Working, query: str, type: Optional[str] = None, limit: int = 8) -> str: if w.gam is None: @@ -224,6 +245,8 @@ def search_assets(w: Working, query: str, type: Optional[str] = None, row["credit"] = asset.credit if asset.width and asset.height: row["size"] = f"{asset.width}x{asset.height}" + if asset.music: + row["music"] = _music_row(asset.music) rows.append(row) if not rows: return (f"Nothing in the library matches {query!r}. Try other words, or leave a " @@ -240,10 +263,13 @@ def get_asset(w: Working, gam_id: str) -> str: raise ToolFailed(f"The asset library failed: {exc.message}") if asset is None: raise ToolFailed(f"No asset {gam_id!r} in the library.") - return _compact({"gam_id": asset.id, "name": asset.name, "type": asset.asset_type, - "duration": asset.duration_seconds, "width": asset.width, - "height": asset.height, "credit": asset.credit, - "clip_of": asset.parent_asset_id}) + details = {"gam_id": asset.id, "name": asset.name, "type": asset.asset_type, + "duration": asset.duration_seconds, "width": asset.width, + "height": asset.height, "credit": asset.credit, + "clip_of": asset.parent_asset_id} + if asset.music: + details["music"] = _music_row(asset.music) + return _compact(details) def replace_spec(w: Working, spec: Dict[str, Any]) -> str: @@ -636,7 +662,9 @@ def run_estimate(w: Working) -> str: (ToolSpec("search_assets", "Search the user's asset library (GAM) by name, description and what is SAID in " "videos. A hit on speech carries said_at [start, end] seconds: the moment to cut " - "to. Use type to restrict to video, image or audio.", + "to. Use type to restrict to video, image or audio. An audio hit may carry " + "music: its genres, moods, instruments, bpm, key, energy and vocals, which " + "are also searchable, so search music by the sound you want.", {"type": "object", "properties": { "query": _STR, "type": {"type": "string", "enum": ["video", "image", "audio"]}, "limit": {"type": "integer", "minimum": 1, "maximum": MAX_SEARCH_HITS}}, diff --git a/backend/app/gam/client.py b/backend/app/gam/client.py index 31b78e6..1358820 100644 --- a/backend/app/gam/client.py +++ b/backend/app/gam/client.py @@ -18,7 +18,7 @@ from app import outbound from app.config import settings -from app.schemas_gam import GamAssetRead, GamHit, GamSearch +from app.schemas_gam import GamAssetRead, GamHit, GamMusic, GamSearch logger = logging.getLogger(__name__) @@ -57,9 +57,37 @@ def to_asset(raw: Dict[str, Any]) -> GamAssetRead: in_point=raw.get("in_point"), out_point=raw.get("out_point"), missing=bool(raw.get("missing")), + music=_music(raw), ) +def _music(raw: Dict[str, Any]) -> Optional[GamMusic]: + """GAM's flat `music_*` fields as one object, or None when none of them says anything. + + None rather than an empty profile so "GAM knows nothing about this track" and "GAM + is older than the profile" read the same, and nothing downstream has to tell an + empty object from a missing one. + """ + def terms(name: str) -> List[str]: + value = raw.get(name) + return [t for t in value if isinstance(t, str)] if isinstance(value, list) else [] + + music = GamMusic( + genres=terms("music_genres"), + moods=terms("music_moods"), + instruments=terms("music_instruments"), + bpm=raw.get("music_bpm"), + key=raw.get("music_key"), + vocals=raw.get("music_vocals"), + energy=raw.get("music_energy"), + ) + if not (music.genres or music.moods or music.instruments) and all( + value is None for value in (music.bpm, music.key, music.vocals, music.energy) + ): + return None + return music + + class GamClient: def __init__(self, token: Optional[str]): headers = {"Authorization": f"Bearer {token}"} if token else {} diff --git a/backend/app/schemas_gam.py b/backend/app/schemas_gam.py index dd3d076..19baf51 100644 --- a/backend/app/schemas_gam.py +++ b/backend/app/schemas_gam.py @@ -9,6 +9,25 @@ from pydantic import BaseModel, Field +class GamMusic(BaseModel): + """What a track sounds like, from GAM's music profile (gam#46). + + The one thing that lets a soundtrack be chosen for a scene rather than by its title. + Null on the asset when GAM knows nothing about it, or is older than the profile. + """ + + genres: List[str] = Field(default_factory=list) + moods: List[str] = Field(default_factory=list) + instruments: List[str] = Field(default_factory=list) + bpm: Optional[float] = None + # As GAM stores it: "Am", "A minor" and Camelot "8A" are all left as written. + key: Optional[str] = None + # Null is "not known", which is not False ("instrumental"). Treating the two as one + # would put songs with words under narration. + vocals: Optional[bool] = None + energy: Optional[str] = None # low | medium | high + + class GamAssetRead(BaseModel): id: str name: str @@ -31,6 +50,7 @@ class GamAssetRead(BaseModel): in_point: Optional[float] = None out_point: Optional[float] = None missing: bool = False + music: Optional[GamMusic] = None class GamHit(BaseModel): diff --git a/backend/tests/test_assistant_tools.py b/backend/tests/test_assistant_tools.py index c1da5d0..05d81d4 100644 --- a/backend/tests/test_assistant_tools.py +++ b/backend/tests/test_assistant_tools.py @@ -125,6 +125,31 @@ def test_a_search_hit_becomes_the_visual_with_its_moment_as_the_trim(gam, siblin "from": "gam", "id": "r", "type": "video", "label": "River interview"} +def test_a_track_hit_says_what_it_sounds_like(gam, siblings): + siblings.hits = [ + hit(gam_asset("m1", "Caravan theme", "audio", music_genres=["Ambient"], + music_moods=["Tense"], music_instruments=[], music_bpm=91.6, + music_key=None, music_vocals=False, music_energy="low")), + hit(gam_asset("m2", "Untagged song", "audio")), + ] + w = Working(spec(), gam=gam) + found = run(w, "search_assets", query="tense ambient", type="audio").content + + # Only what is known: no empty instruments, no null key, and vocals as the word that + # matters under narration. + assert ('"music":{"genres":["Ambient"],"moods":["Tense"],"bpm":92,"energy":"low",' + '"vocals":"instrumental"}') in found + # A track with no profile carries no music key at all, rather than an empty one. + untagged = found.split('"Untagged song"')[1] + assert '"music"' not in untagged + + +def test_get_asset_includes_the_music_profile(gam, siblings): + siblings.assets["m1"] = gam_asset("m1", "Caravan theme", "audio", music_vocals=True) + w = Working(spec(), gam=gam) + assert '"music":{"vocals":"yes"}' in run(w, "get_asset", gam_id="m1").content + + def test_an_empty_search_says_to_try_again_or_leave_a_placeholder(gam, siblings): siblings.hits = [] w = Working(spec(), gam=gam) diff --git a/backend/tests/test_gam_api.py b/backend/tests/test_gam_api.py index c33338d..e4683c5 100644 --- a/backend/tests/test_gam_api.py +++ b/backend/tests/test_gam_api.py @@ -36,6 +36,28 @@ def test_a_gam_without_end_times_still_answers(auth_client, siblings): assert top["start_time"] == 10.0 and top["end_time"] is None +def test_a_track_carries_its_music_profile(auth_client, siblings): + track = gam_asset("m1", "Caravan theme", "audio", music_genres=["Ambient"], + music_moods=["Tense"], music_instruments=[], music_bpm=92.0, + music_key="Am", music_vocals=False, music_energy="low") + siblings.hits = [hit(track)] + + asset = auth_client.get("/api/gam/search", params={"q": "tense"}).json()["data"][0]["asset"] + + assert asset["music"] == {"genres": ["Ambient"], "moods": ["Tense"], "instruments": [], + "bpm": 92.0, "key": "Am", "vocals": False, "energy": "low"} + + +def test_a_track_gam_knows_nothing_about_has_no_profile(auth_client, siblings): + """An empty profile and a GAM older than gam#46 read the same: null.""" + siblings.hits = [ + hit(gam_asset("m1", "Untagged", "audio", music_genres=[], music_vocals=None)), + hit(gam_asset("m2", "From an old GAM", "audio")), + ] + data = auth_client.get("/api/gam/search", params={"q": "x"}).json()["data"] + assert [h["asset"]["music"] for h in data] == [None, None] + + def test_documents_are_not_offered_for_a_video(auth_client, siblings): siblings.hits = [hit(gam_asset("d1", "Report", "document")), hit(gam_asset("i1", "Map", "image"))] body = auth_client.get("/api/gam/search", params={"q": "x"}).json() diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md index 2478e61..ef0238a 100644 --- a/docs/plan-of-attack.md +++ b/docs/plan-of-attack.md @@ -349,6 +349,7 @@ Specified here, applied in GAM's repository, not from this one. | GAM-3 | `origin_project_id` / `origin_reference_id` on `Asset` (migration) + upload with `source: "gvc_export"`. **[davior/gam#37](https://github.com/davior/gam/pull/37).** Against a GAM without it, a save lands as an ordinary upload | Filing renders back into the library | | GAM-4 | M9: documented read API, `/picker?embed=1` with `postMessage({ assetId, inPoint, outPoint })` | Picking assets and moments by hand in the editor | | GAM-5 | `CORS_ORIGIN` / CSP for `gvc.geckopico.com`, only where GVC's browser code calls GAM directly | Server-side calls with a Bearer token need neither | +| GAM-6 | A music profile on audio assets (genres, moods, instruments, BPM, key, vocals, energy), read from ID3/Vorbis/MP4 tags and searchable. **[davior/gam#46](https://github.com/davior/gam/pull/46).** GVC reads it as `GamAssetRead.music`; against a GAM without it, `music` is null and tracks are chosen by name as before | The assistant choosing a soundtrack by how it sounds, and the Sound picker saying so | GAM-1 and GAM-2 come first. They're small, and M1 below depends on them. diff --git a/frontend/src/api/gam.ts b/frontend/src/api/gam.ts index c4141eb..ace782c 100644 --- a/frontend/src/api/gam.ts +++ b/frontend/src/api/gam.ts @@ -22,6 +22,37 @@ export interface GamAsset { in_point: number | null out_point: number | null missing: boolean + /** What a track sounds like (GAM's music profile). Null when GAM knows nothing about + * it, or is older than the profile. Optional for test literals that predate it. */ + music?: GamMusic | null +} + +export interface GamMusic { + genres: string[] + moods: string[] + instruments: string[] + bpm: number | null + key: string | null + /** Null is "not known", which is not false ("instrumental"). */ + vocals: boolean | null + energy: 'low' | 'medium' | 'high' | string | null +} + +/** + * One line saying what a track sounds like: "Ambient, Tense · 92 BPM · Am · Low energy · + * Instrumental". Empty when nothing is known, so the caller can leave the line out. + */ +export function musicSummary(music: GamMusic | null | undefined): string { + if (!music) return '' + const parts: string[] = [] + const words = [...music.genres, ...music.moods, ...music.instruments] + if (words.length) parts.push(words.join(', ')) + if (music.bpm !== null) parts.push(`${Math.round(music.bpm)} BPM`) + if (music.key) parts.push(music.key) + if (music.energy) + parts.push(`${music.energy[0].toUpperCase()}${music.energy.slice(1)} energy`) + if (music.vocals !== null) parts.push(music.vocals ? 'Vocals' : 'Instrumental') + return parts.join(' · ') } export interface GamHit { diff --git a/frontend/src/components/FootageSearch.test.tsx b/frontend/src/components/FootageSearch.test.tsx index 90df5db..9603dfb 100644 --- a/frontend/src/components/FootageSearch.test.tsx +++ b/frontend/src/components/FootageSearch.test.tsx @@ -1,7 +1,13 @@ import { render, screen } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { afterEach, describe, expect, it, vi } from 'vitest' -import { gamApi, type GamAsset, type GamHit, type GamSearch } from '@/api/gam' +import { + gamApi, + musicSummary, + type GamAsset, + type GamHit, + type GamSearch, +} from '@/api/gam' import FootageSearch from '@/components/FootageSearch' function asset(overrides: Partial = {}): GamAsset { @@ -127,6 +133,57 @@ describe('FootageSearch', () => { expect(screen.getByText('Nothing in the library matches.')).toBeInTheDocument() }) + it('says what a track sounds like, in the list and once picked', async () => { + const track: GamHit = { + asset: asset({ + id: 'm1', + name: 'Caravan theme', + asset_type: 'audio', + thumb_url: null, + music: { + genres: ['Ambient'], + moods: ['Tense'], + instruments: [], + bpm: 92, + key: 'Am', + vocals: false, + energy: 'low', + }, + }), + score: 0.03, + snippet: '', + start_time: null, + end_time: null, + } + vi.spyOn(gamApi, 'search').mockResolvedValue(result([track])) + render( + + ) + + await userEvent.type(screen.getByLabelText('Search the asset library'), 'tense') + await userEvent.click(screen.getByRole('button', { name: 'Search' })) + + const line = 'Ambient, Tense · 92 BPM · Am · Low energy · Instrumental' + expect(await screen.findByText(line)).toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: /Caravan theme/ })) + expect(screen.getAllByText(line)).toHaveLength(2) + }) + + it('leaves the line out for a track nothing is known about', () => { + expect(musicSummary(null)).toBe('') + expect( + musicSummary({ + genres: [], + moods: [], + instruments: [], + bpm: null, + key: null, + vocals: null, + energy: null, + }) + ).toBe('') + }) + it('shows what went wrong when the library cannot be reached', async () => { vi.spyOn(gamApi, 'search').mockRejectedValue( new Error('Could not reach the asset library') diff --git a/frontend/src/components/FootageSearch.tsx b/frontend/src/components/FootageSearch.tsx index 3f84456..457fad0 100644 --- a/frontend/src/components/FootageSearch.tsx +++ b/frontend/src/components/FootageSearch.tsx @@ -1,7 +1,13 @@ import { useState, type FormEvent, type ReactNode } from 'react' import { Film, Image as ImageIcon, Search, X } from 'lucide-react' import { apiErrorMessage } from '@/api/client' -import { gamApi, type GamHit, type GamSearch, type MediaKind } from '@/api/gam' +import { + gamApi, + musicSummary, + type GamHit, + type GamSearch, + type MediaKind, +} from '@/api/gam' import { clock, momentTrim, type MomentSize, type PlaceMode } from '@/spec/footage' /** @@ -188,6 +194,11 @@ export default function FootageSearch({ {h.end_time !== null && `–${clock(h.end_time)}`} )} + {musicSummary(h.asset.music) && ( + + {musicSummary(h.asset.music)} + + )} {h.snippet && } @@ -235,6 +246,11 @@ export default function FootageSearch({ ? 'Uses the whole video' : 'A still picture'}

+ {musicSummary(selected.asset.music) && ( +

+ {musicSummary(selected.asset.music)} +

+ )} {selected.asset.credit && (

{selected.asset.credit}

)}