Add a music profile to audio assets, read from ID3/Vorbis/MP4 tags

backend/alembic/versions/20261026_0900_add_music_profile.py

@@ -0,0 +1,52 @@
+"""add the music profile to asset
+
+What a track sounds like (genres, moods, instruments, tempo, key, vocals, energy), so
+Video Creator can find a soundtrack that fits a scene. See app/music.py.
+
+All nullable and all plain ALTER on SQLite, so the upgrade does not recreate `asset`.
+The downgrade's drops do, hence the PRAGMA dance `a8e1c0f4d2b6` explains.
+
+Split from `d71e4b9a2c86`, which rebuilds the search index, for the reason `3f7a21c9d4e5`
+and `9c2e08b4a1f7` were split: that half can fail and this half cannot.
+
+Revision ID: c3a8f15e7b20
+Revises: 5b9e3d71c2a4
+Create Date: 2026-10-26 09:00:00.000000+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+revision: str = 'c3a8f15e7b20'
+down_revision: Union[str, None] = '5b9e3d71c2a4'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table('asset', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('music_genres', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+        batch_op.add_column(sa.Column('music_moods', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+        batch_op.add_column(sa.Column('music_instruments', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+        batch_op.add_column(sa.Column('music_bpm', sa.Float(), nullable=True))
+        batch_op.add_column(sa.Column('music_key', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+        batch_op.add_column(sa.Column('music_vocals', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('music_energy', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+
+
+def downgrade() -> None:
+    with op.get_context().autocommit_block():
+        op.execute('PRAGMA foreign_keys=OFF')
+    with op.batch_alter_table('asset', schema=None) as batch_op:
+        batch_op.drop_column('music_energy')
+        batch_op.drop_column('music_vocals')
+        batch_op.drop_column('music_key')
+        batch_op.drop_column('music_bpm')
+        batch_op.drop_column('music_instruments')
+        batch_op.drop_column('music_moods')
+        batch_op.drop_column('music_genres')
+    with op.get_context().autocommit_block():
+        op.execute('PRAGMA foreign_keys=ON')

backend/alembic/versions/20261026_0901_rebuild_asset_fts_with_music.py

@@ -0,0 +1,139 @@
+"""rebuild asset_fts with a music column
+
+A music profile nobody can search for is no use to the one thing that wants it: Video
+Creator looking for "tense slow piano". So it reaches the keyword index as `music_text`.
+
+FTS5 has no ALTER TABLE ADD COLUMN, so this drops and recreates the table and then
+repopulates it, exactly as `9c2e08b4a1f7` did for attribution; that migration's
+docstring explains why the repopulate is not optional. The new column is appended last
+because `search_assets` addresses `name` by column index in snippet().
+
+The repopulated text is composed to match `app/search/fts.py::index_asset` and
+`app/music.py::index_text`, so the first ordinary edit of an asset does not silently
+rewrite its index entry into something different. The music columns were only added by
+the revision before, so on a real upgrade they are all NULL and `music_text` comes out
+empty. It is composed properly anyway, so a downgrade and re-upgrade of a library that
+has profiles does not lose them from the index.
+
+Revision ID: d71e4b9a2c86
+Revises: c3a8f15e7b20
+Create Date: 2026-10-26 09:01:00.000000+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = 'd71e4b9a2c86'
+down_revision: Union[str, None] = 'c3a8f15e7b20'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+# Literal copies, not imports from app/search/fts.py: a migration has to describe the
+# schema as it was when it was written. test_migrations.py asserts they still agree.
+ASSET_FTS_DDL = """
+CREATE VIRTUAL TABLE asset_fts USING fts5(
+    asset_id UNINDEXED,
+    user_id UNINDEXED,
+    name,
+    description,
+    summary,
+    tags_text,
+    attribution_text,
+    music_text,
+    tokenize='porter unicode61'
+)
+"""
+
+_TAGS_TEXT = """
+    COALESCE(
+        (SELECT group_concat(t.name, ' ')
+           FROM assettag at
+           JOIN tag t ON t.id = at.tag_id
+          WHERE at.asset_id = a.id),
+        ''
+    )"""
+
+_ATTRIBUTION_TEXT = """
+    TRIM(
+        COALESCE(a.creator, '') || ' ' ||
+        COALESCE(a.publisher, '') || ' ' ||
+        COALESCE(a.source_title, '') || ' ' ||
+        COALESCE(a.license, '') || ' ' ||
+        COALESCE(a.credit_line, '') || ' ' ||
+        COALESCE(a.source_url, '')
+    )"""
+
+# json_each over a NULL or non-array yields no rows, so an unprofiled asset gets ''.
+# json_valid guards a malformed value, which `music.decode_terms` also reads as empty.
+_MUSIC_TEXT = """
+    TRIM(
+        COALESCE((SELECT group_concat(j.value, ' ') FROM json_each(
+            CASE WHEN json_valid(a.music_genres) THEN a.music_genres END) j), '') || ' ' ||
+        COALESCE((SELECT group_concat(j.value, ' ') FROM json_each(
+            CASE WHEN json_valid(a.music_moods) THEN a.music_moods END) j), '') || ' ' ||
+        COALESCE((SELECT group_concat(j.value, ' ') FROM json_each(
+            CASE WHEN json_valid(a.music_instruments) THEN a.music_instruments END) j), '') || ' ' ||
+        CASE a.music_vocals WHEN 1 THEN 'vocals' WHEN 0 THEN 'instrumental' ELSE '' END || ' ' ||
+        CASE WHEN a.music_energy IN ('low', 'medium', 'high')
+             THEN a.music_energy || ' energy' ELSE '' END
+    )"""
+
+REPOPULATE = f"""
+INSERT INTO asset_fts (
+    asset_id, user_id, name, description, summary, tags_text, attribution_text, music_text
+)
+SELECT
+    a.id,
+    a.user_id,
+    COALESCE(a.name, ''),
+    COALESCE(a.description, ''),
+    COALESCE(a.summary, ''),
+    {_TAGS_TEXT},
+    {_ATTRIBUTION_TEXT},
+    {_MUSIC_TEXT}
+FROM asset a
+"""
+
+# The previous shape, for downgrade. Same reasoning: literal, not imported.
+ASSET_FTS_DDL_WITHOUT_MUSIC = """
+CREATE VIRTUAL TABLE asset_fts USING fts5(
+    asset_id UNINDEXED,
+    user_id UNINDEXED,
+    name,
+    description,
+    summary,
+    tags_text,
+    attribution_text,
+    tokenize='porter unicode61'
+)
+"""
+
+REPOPULATE_WITHOUT_MUSIC = f"""
+INSERT INTO asset_fts (
+    asset_id, user_id, name, description, summary, tags_text, attribution_text
+)
+SELECT
+    a.id,
+    a.user_id,
+    COALESCE(a.name, ''),
+    COALESCE(a.description, ''),
+    COALESCE(a.summary, ''),
+    {_TAGS_TEXT},
+    {_ATTRIBUTION_TEXT}
+FROM asset a
+"""
+
+
+def upgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS asset_fts")
+    op.execute(ASSET_FTS_DDL)
+    op.execute(REPOPULATE)
+
+
+def downgrade() -> None:
+    # Repopulating for the same reason as the upgrade: an emptied index is a silent,
+    # data-shaped loss.
+    op.execute("DROP TABLE IF EXISTS asset_fts")
+    op.execute(ASSET_FTS_DDL_WITHOUT_MUSIC)
+    op.execute(REPOPULATE_WITHOUT_MUSIC)

backend/app/enrichment/embed.py

@@ -13,6 +13,7 @@
 
 from sqlmodel import Session, col, select
 
+from app import music
 from app.embeddings import EmbeddingError, build_embedder
 from app.models.asset import Asset
 from app.models.embedding import OWNER_ASSET, OWNER_SEGMENT
@@ -58,7 +59,14 @@ def run(session: Session, asset: Asset, progress: Progress) -> int:
 
 def _embed_asset_text(session: Session, asset: Asset, embedder) -> int:
     """One vector for what the asset *is*, as opposed to what is said in it."""
-    parts = [asset.name or "", asset.description or "", asset.summary or ""]
+    # The music profile rides along so "tense slow piano" finds a track by meaning, not
+    # only by the exact words the keyword index holds. Empty for everything but music.
+    parts = [
+        asset.name or "",
+        asset.description or "",
+        asset.summary or "",
+        music.index_text(asset),
+    ]
     text = "\n".join(part for part in parts if part.strip()).strip()
     if not text:
         return 0

backend/app/enrichment/harvest_attribution.py

@@ -9,6 +9,10 @@
 one progress bar, one line in the activity feed rather than a wall of near-identical
 ones.
 
+It fills an audio file's music profile from the same tag block at the same time — genre,
+tempo, key and mood — so a library of songs uploaded before that existed catches up in
+the same pass rather than needing a job of its own.
+
 Safe to run repeatedly. It only ever fills fields that are blank, so a second run is a
 no-op over everything the first one filled and over everything the user has since
 corrected by hand — there is no "already harvested" flag to keep, and no way for this to
@@ -79,7 +83,7 @@ def run(session: Session, user_id: str, progress: Progress) -> HarvestResult:
             continue
 
         before = {name: getattr(asset, name, None) for name in found}
-        asset_service.apply_embedded_attribution(session, asset, found)
+        asset_service.apply_embedded_fields(session, asset, found)
         session.refresh(asset)
         if any(getattr(asset, name, None) != before[name] for name in found):
             attributed += 1

backend/app/enrichment/import_url.py

@@ -36,7 +36,7 @@
 from app.enrichment.transcribe import store_transcript
 from app.ingest import captions as caption_parser
 from app.ingest import thumbnails, ytdlp
-from app.ingest.filetypes import SOURCE_URL, TYPE_IMAGE
+from app.ingest.filetypes import SOURCE_URL, TYPE_AUDIO, TYPE_IMAGE
 from app.ingest.ytdlp import ImportedMetadata, UrlImportError
 from app.jobs.runner import readable_error, set_fields
 from app.models.asset import Asset
@@ -324,6 +324,10 @@ def _apply_metadata(session: Session, asset: Asset, meta: ImportedMetadata) -> N
     changes["retrieved_at"] = utcnow()
     if meta.description:
         changes["description"] = meta.description
+    # A genre describes the music only when the import is music. A video's "genre" is
+    # the programme's, which is not what the music profile records.
+    if asset.asset_type == TYPE_AUDIO:
+        changes.update(meta.music)
     try:
         asset_service.apply_embedded_metadata(session, asset, changes)
     except Exception:  # noqa: BLE001 - the video is in; losing its metadata must not undo that

backend/app/ingest/embedded_metadata.py

@@ -20,6 +20,11 @@
   `title` is frequently boilerplate from an encoder, and silently renaming somebody's
   asset on upload is a worse failure than leaving a field blank.
 - `description` / `summary` — not attribution, and owned by the enrichment path.
+
+An audio file's tags also carry a music profile — genre, tempo, key, mood — which is read
+here too, by `_music_from_tags`. Same standing as the attribution beside it: what the
+file says, filling blanks, stamped `embedded`. Only for audio: a video's `genre` is the
+film's ("Documentary"), not a description of its soundtrack.
 """
 
 from __future__ import annotations
@@ -30,6 +35,7 @@
 from pathlib import Path
 from typing import Any, Mapping, Optional
 
+from app import music
 from app.ingest.filetypes import TYPE_AUDIO, TYPE_DOCUMENT, TYPE_IMAGE, TYPE_VIDEO, extension_of
 
 logger = logging.getLogger(__name__)
@@ -62,6 +68,18 @@
     "source_url": ("purl", "url", "wxxx", "comment"),
 }
 
+# Music tags, by the key ffprobe reports (lowercased in `probe.py`). ffprobe names the
+# ID3 frames it knows (TCON is `genre`) and passes the rest through under their frame id,
+# so TBPM, TKEY and TMOO arrive as `tbpm`, `tkey`, `tmoo`. The plain names are the Vorbis
+# comment spellings FLAC and Ogg use, and what ffmpeg itself writes into a TXXX frame;
+# `tmpo` is the MP4 tempo atom.
+_MUSIC_TAG_SOURCES: dict[str, tuple[str, ...]] = {
+    "music_genres": ("genre",),
+    "music_moods": ("tmoo", "mood"),
+    "music_bpm": ("tbpm", "bpm", "tmpo", "tempo"),
+    "music_key": ("tkey", "initialkey", "initial_key", "key"),
+}
+
 
 def harvest(
     path: Path,
@@ -76,7 +94,10 @@ def harvest(
     never clobber something a person typed.
     """
     try:
-        if asset_type in (TYPE_VIDEO, TYPE_AUDIO):
+        if asset_type == TYPE_AUDIO:
+            tags = probe_tags or {}
+            return {**_from_container_tags(tags), **_music_from_tags(tags)}
+        if asset_type == TYPE_VIDEO:
             return _from_container_tags(probe_tags or {})
         if asset_type == TYPE_IMAGE:
             return _from_image(path)
@@ -128,6 +149,52 @@ def _from_container_tags(tags: Mapping[str, str]) -> dict[str, Any]:
     return found
 
 
+def _music_from_tags(tags: Mapping[str, str]) -> dict[str, Any]:
+    """A music profile out of an audio file's tag block, in column form.
+
+    Vocals are only ever inferred from evidence a writer left on purpose: a lyrics frame
+    means words are sung, and a genre of "Instrumental" (ID3v1 genre 33) means none are.
+    Anything else leaves it unknown — guessing "instrumental" for an untagged track would
+    put songs under every search for music without words.
+    """
+    found: dict[str, Any] = {}
+
+    for key in _MUSIC_TAG_SOURCES["music_genres"]:
+        genres = music.split_terms(tags.get(key))
+        if genres:
+            found["music_genres"] = music.encode_terms(genres)
+            if any(genre.lower() == "instrumental" for genre in genres):
+                found["music_vocals"] = False
+            break
+
+    for key in _MUSIC_TAG_SOURCES["music_moods"]:
+        moods = music.split_terms(tags.get(key))
+        if moods:
+            found["music_moods"] = music.encode_terms(moods)
+            break
+
+    for key in _MUSIC_TAG_SOURCES["music_bpm"]:
+        bpm = music.parse_bpm(tags.get(key))
+        if bpm is not None:
+            found["music_bpm"] = bpm
+            break
+
+    for key in _MUSIC_TAG_SOURCES["music_key"]:
+        musical_key = music.clean_key(tags.get(key))
+        if musical_key:
+            found["music_key"] = musical_key
+            break
+
+    # ID3 USLT arrives as `lyrics` or `lyrics-eng`; Vorbis as `lyrics` or `unsyncedlyrics`.
+    if any(
+        (name.startswith("lyrics") or name == "unsyncedlyrics") and (value or "").strip()
+        for name, value in tags.items()
+    ):
+        found["music_vocals"] = True
+
+    return found
+
+
 # ─── images ──────────────────────────────────────────────────────────────────
 
 

backend/app/ingest/ytdlp.py

@@ -34,6 +34,7 @@
 from fastapi import HTTPException, status
 from yt_dlp.utils import DownloadCancelled, DownloadError
 
+from app import music
 from app.config import settings
 from app.ingest.embedded_metadata import normalise_partial_date
 from app.jobs.runner import JobCancelled
@@ -517,6 +518,10 @@ class ImportedMetadata:
     # yt-dlp's name for the site ("youtube", "rumble"), for labelling where tags and
     # captions came from.
     extractor: str = ""
+    # Music profile columns, in column form. Only the genre: it is the one music field
+    # sites report (SoundCloud and Bandcamp do; YouTube's `categories` is the site's own
+    # taxonomy, "Music" for every song, and says nothing about which music).
+    music: dict[str, Any] = field(default_factory=dict)
 
     @property
     def source_url(self) -> Optional[str]:
@@ -566,13 +571,19 @@ def put(name: str, value: Any) -> None:
     if published:
         attribution["published_date"] = published
 
+    genres = info.get("genres")
+    if not isinstance(genres, (list, tuple)):
+        genres = music.split_terms(info.get("genre"))
+    encoded_genres = music.encode_terms(genres)
+
     return ImportedMetadata(
         title=title[:255],
         description=description,
         attribution=attribution,
         tags=_tags(info.get("tags")),
         chapters=_chapters(info.get("chapters")),
         extractor=_text(info.get("extractor_key") or info.get("extractor")).lower(),
+        music={"music_genres": encoded_genres} if encoded_genres else {},
     )