diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py
index d0e2652..0e3fa98 100644
--- a/backend/app/assistant/prompt.py
+++ b/backend/app/assistant/prompt.py
@@ -108,6 +108,10 @@
Choosing music: search with `type` audio for the sound the scene needs, in the words a \
music library uses (a genre, a mood, an instrument, an energy: "tense ambient", \
"uplifting acoustic guitar", "slow piano", "high energy"), not for the video's subject. \
+An audio hit's `kind` says what the file is: `music`, `speech` (a voice recording) or \
+`sfx` (a sound effect); no `kind` means not known. For a soundtrack use `music`, or an \
+unknown file whose name and `music` make it plainly a track; never `speech`. A sound \
+effect is `sfx`; extra voice is `speech`.
An audio hit's `music` says what is known about the track:
- Under narration, prefer `"vocals":"instrumental"`. Sung words fight the voiceover; \
a track with no `vocals` is unknown, so say so if you use it.
diff --git a/backend/app/assistant/tools.py b/backend/app/assistant/tools.py
index 9e63fcc..9c275e1 100644
--- a/backend/app/assistant/tools.py
+++ b/backend/app/assistant/tools.py
@@ -245,6 +245,8 @@ def search_assets(w: Working, query: str, type: Optional[str] = None,
row["credit"] = asset.credit
if asset.width and asset.height:
row["size"] = f"{asset.width}x{asset.height}"
+ if asset.audio_kind:
+ row["kind"] = asset.audio_kind
if asset.music:
row["music"] = _music_row(asset.music)
rows.append(row)
@@ -267,6 +269,8 @@ def get_asset(w: Working, gam_id: str) -> str:
"duration": asset.duration_seconds, "width": asset.width,
"height": asset.height, "credit": asset.credit,
"clip_of": asset.parent_asset_id}
+ if asset.audio_kind:
+ details["kind"] = asset.audio_kind
if asset.music:
details["music"] = _music_row(asset.music)
return _compact(details)
@@ -663,8 +667,9 @@ def run_estimate(w: Working) -> str:
"Search the user's asset library (GAM) by name, description and what is SAID in "
"videos. A hit on speech carries said_at [start, end] seconds: the moment to cut "
"to. Use type to restrict to video, image or audio. An audio hit may carry "
- "music: its genres, moods, instruments, bpm, key, energy and vocals, which "
- "are also searchable, so search music by the sound you want.",
+ "kind (music, speech or sfx) and music: its genres, moods, instruments, bpm, "
+ "key, energy and vocals, which are also searchable, so search music by the "
+ "sound you want.",
{"type": "object", "properties": {
"query": _STR, "type": {"type": "string", "enum": ["video", "image", "audio"]},
"limit": {"type": "integer", "minimum": 1, "maximum": MAX_SEARCH_HITS}},
diff --git a/backend/app/gam/client.py b/backend/app/gam/client.py
index 1358820..6ff46e2 100644
--- a/backend/app/gam/client.py
+++ b/backend/app/gam/client.py
@@ -58,6 +58,7 @@ def to_asset(raw: Dict[str, Any]) -> GamAssetRead:
out_point=raw.get("out_point"),
missing=bool(raw.get("missing")),
music=_music(raw),
+ audio_kind=raw.get("audio_kind") if raw.get("audio_kind") in ("music", "speech", "sfx") else None,
)
diff --git a/backend/app/schemas_gam.py b/backend/app/schemas_gam.py
index 19baf51..c4f9167 100644
--- a/backend/app/schemas_gam.py
+++ b/backend/app/schemas_gam.py
@@ -51,6 +51,10 @@ class GamAssetRead(BaseModel):
out_point: Optional[float] = None
missing: bool = False
music: Optional[GamMusic] = None
+ # What GAM says an audio file is: music | speech | sfx, null when not known (or GAM
+ # predates gam#49). Kept beside `music`, not in it: a voice memo has a kind and no
+ # profile, and the kind is what keeps it out of a soundtrack.
+ audio_kind: Optional[str] = None
class GamHit(BaseModel):
diff --git a/backend/tests/test_assistant_tools.py b/backend/tests/test_assistant_tools.py
index 05d81d4..b0d1a87 100644
--- a/backend/tests/test_assistant_tools.py
+++ b/backend/tests/test_assistant_tools.py
@@ -144,6 +144,20 @@ def test_a_track_hit_says_what_it_sounds_like(gam, siblings):
assert '"music"' not in untagged
+def test_a_hit_says_whether_it_is_music_or_a_voice(gam, siblings):
+ siblings.hits = [
+ hit(gam_asset("m1", "Theme", "audio", audio_kind="music", music_genres=["Ambient"])),
+ hit(gam_asset("v1", "Voice memo", "audio", audio_kind="speech")),
+ hit(gam_asset("u1", "Unknown", "audio")),
+ ]
+ w = Working(spec(), gam=gam)
+ found = run(w, "search_assets", query="calm", type="audio").content
+
+ assert '"name":"Theme","type":"audio","kind":"music","music":{"genres":["Ambient"]}' in found
+ assert '"name":"Voice memo","type":"audio","kind":"speech"' in found
+ assert '"kind"' not in found.split('"Unknown"')[1]
+
+
def test_get_asset_includes_the_music_profile(gam, siblings):
siblings.assets["m1"] = gam_asset("m1", "Caravan theme", "audio", music_vocals=True)
w = Working(spec(), gam=gam)
diff --git a/backend/tests/test_gam_api.py b/backend/tests/test_gam_api.py
index e4683c5..800259e 100644
--- a/backend/tests/test_gam_api.py
+++ b/backend/tests/test_gam_api.py
@@ -58,6 +58,17 @@ def test_a_track_gam_knows_nothing_about_has_no_profile(auth_client, siblings):
assert [h["asset"]["music"] for h in data] == [None, None]
+def test_a_track_carries_what_kind_of_audio_it_is(auth_client, siblings):
+ siblings.hits = [
+ hit(gam_asset("m1", "Theme", "audio", audio_kind="music")),
+ hit(gam_asset("v1", "Voice memo", "audio", audio_kind="speech")),
+ hit(gam_asset("o1", "From an old GAM", "audio")),
+ hit(gam_asset("x1", "Odd value", "audio", audio_kind="podcast")),
+ ]
+ data = auth_client.get("/api/gam/search", params={"q": "x"}).json()["data"]
+ assert [h["asset"]["audio_kind"] for h in data] == ["music", "speech", None, None]
+
+
def test_documents_are_not_offered_for_a_video(auth_client, siblings):
siblings.hits = [hit(gam_asset("d1", "Report", "document")), hit(gam_asset("i1", "Map", "image"))]
body = auth_client.get("/api/gam/search", params={"q": "x"}).json()
diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md
index ef0238a..0efd9a4 100644
--- a/docs/plan-of-attack.md
+++ b/docs/plan-of-attack.md
@@ -350,6 +350,7 @@ Specified here, applied in GAM's repository, not from this one.
| GAM-4 | M9: documented read API, `/picker?embed=1` with `postMessage({ assetId, inPoint, outPoint })` | Picking assets and moments by hand in the editor |
| GAM-5 | `CORS_ORIGIN` / CSP for `gvc.geckopico.com`, only where GVC's browser code calls GAM directly | Server-side calls with a Bearer token need neither |
| GAM-6 | A music profile on audio assets (genres, moods, instruments, BPM, key, vocals, energy), read from ID3/Vorbis/MP4 tags and searchable. **[davior/gam#46](https://github.com/davior/gam/pull/46).** GVC reads it as `GamAssetRead.music`; against a GAM without it, `music` is null and tracks are chosen by name as before | The assistant choosing a soundtrack by how it sounds, and the Sound picker saying so |
+| GAM-7 | `audio_kind` (music / speech / sfx) on audio assets. **[davior/gam#49](https://github.com/davior/gam/pull/49).** GVC reads it as `GamAssetRead.audio_kind`; null against a GAM without it | Keeping voice recordings out of a soundtrack, in the assistant and the Sound picker |
GAM-1 and GAM-2 come first. They're small, and M1 below depends on them.
diff --git a/frontend/src/api/gam.ts b/frontend/src/api/gam.ts
index ace782c..8d0d211 100644
--- a/frontend/src/api/gam.ts
+++ b/frontend/src/api/gam.ts
@@ -25,6 +25,8 @@ export interface GamAsset {
/** What a track sounds like (GAM's music profile). Null when GAM knows nothing about
* it, or is older than the profile. Optional for test literals that predate it. */
music?: GamMusic | null
+ /** What GAM says the audio is. Null when not known, or from a GAM before gam#49. */
+ audio_kind?: 'music' | 'speech' | 'sfx' | null
}
export interface GamMusic {
@@ -99,3 +101,17 @@ export const gamApi = {
.then((r) => r.data.data.assets)
},
}
+
+const KIND_LABELS = { music: 'Music', speech: 'Speech', sfx: 'Sound effect' } as const
+
+/**
+ * The line under an audio hit: what kind of audio it is, then what it sounds like.
+ * "Speech" leads on purpose: a voice memo in a list of soundtracks should be the first
+ * thing a person notices about it. A music track with a profile needs no "Music" label.
+ */
+export function audioSummary(asset: Pick
- {musicSummary(selected.asset.music)} + {audioSummary(selected.asset)}
)} {selected.asset.credit && (