Add audio kind (music / speech / sound effect) and skip transcribing music

backend/alembic/versions/20261026_1200_add_audio_kind.py

@@ -0,0 +1,40 @@
+"""add audio_kind to asset
+
+What an audio file is: music, speech or a sound effect, NULL when not known. It decides
+whether an upload is transcribed (music and sound effects are not) and lets Video
+Creator keep voice recordings out of a soundtrack search. See app/music.py.
+
+Existing rows stay NULL, which is treated exactly as before this column existed:
+transcribed and analysed. The "Read file tags" library run fills it from tags.
+
+Revision ID: e4b2c9d17a35
+Revises: d71e4b9a2c86
+Create Date: 2026-10-26 12:00:00.000000+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+revision: str = 'e4b2c9d17a35'
+down_revision: Union[str, None] = 'd71e4b9a2c86'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table('asset', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('audio_kind', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+
+
+def downgrade() -> None:
+    # The drop recreates `asset` on SQLite; see `a8e1c0f4d2b6` for why foreign keys are
+    # switched off around it.
+    with op.get_context().autocommit_block():
+        op.execute('PRAGMA foreign_keys=OFF')
+    with op.batch_alter_table('asset', schema=None) as batch_op:
+        batch_op.drop_column('audio_kind')
+    with op.get_context().autocommit_block():
+        op.execute('PRAGMA foreign_keys=ON')

backend/app/enrichment/page_import.py

@@ -350,6 +350,9 @@ def report(received: int, total: Optional[int]) -> None:
                 filename=f"{stem[:200]}{path.suffix.lower()}",
                 name=track.title,
                 source=SOURCE_URL,
+                # A music library's track is music: never sent to Deepgram, and offered
+                # to a soundtrack search as what it is.
+                audio_kind=music.AUDIO_KIND_MUSIC,
             )
         except asset_service.UnsupportedFile as exc:
             raise PageImportError("The site sent a file the library does not accept") from exc
@@ -376,6 +379,9 @@ def apply_track_metadata(
     changes: dict[str, Any] = {
         "source_url": track.source_url or track.audio_url,
         "retrieved_at": utcnow(),
+        # Already set on a fresh import; this is for the copy found by checksum, which
+        # was perhaps uploaded by hand and never said what it was.
+        "audio_kind": music.AUDIO_KIND_MUSIC,
     }
     for name in ("creator", "publisher", "source_title", "license", "credit_line", "description"):
         value = getattr(track, name)

backend/app/ingest/embedded_metadata.py

@@ -192,6 +192,12 @@ def _music_from_tags(tags: Mapping[str, str]) -> dict[str, Any]:
     ):
         found["music_vocals"] = True
 
+    # Decided here, at upload, because transcription is queued the moment the file lands
+    # and the kind is what decides whether it is.
+    kind = music.kind_from_tags(music.decode_terms(found.get("music_genres")), found.get("music_bpm"))
+    if kind:
+        found["audio_kind"] = kind
+
     return found
 
 

backend/app/models/asset.py

@@ -147,6 +147,12 @@ class Asset(SQLModel, table=True):
     # "instrumental", or every untagged song would match a search for music without words.
     music_vocals: Optional[bool] = None
     music_energy: Optional[str] = None  # low | medium | high
+    # What the audio is: music | speech | sfx, NULL when not known. Decides whether an
+    # upload is transcribed (music and sound effects are not) and lets Video Creator keep
+    # voice recordings out of a soundtrack search. Set from the file's tags, by a music
+    # page import, or by hand; never guessed from the sound, since a podcast with a music
+    # bed has a beat and a key and would lose its transcript. See app/music.py.
+    audio_kind: Optional[str] = None
 
     # ─── AI generation (M8) ──────────────────────────────────────────────────
     # What made this file, kept so it can be made again. Specified in

backend/app/music.py

@@ -45,6 +45,27 @@
 MIN_BPM = 20.0
 MAX_BPM = 300.0
 
+# What an audio file is, which decides what GAM does with it. `audio_kind` on the asset;
+# NULL is "not known" and is treated as it always was (transcribed, analysed).
+AUDIO_KIND_MUSIC = "music"
+AUDIO_KIND_SPEECH = "speech"
+AUDIO_KIND_SFX = "sfx"
+AUDIO_KINDS: tuple[str, ...] = (AUDIO_KIND_MUSIC, AUDIO_KIND_SPEECH, AUDIO_KIND_SFX)
+
+# Kinds that are not transcribed on upload. Transcribing a song bills Deepgram per
+# minute for, at best, its lyrics; a sound effect has nothing to say at all. Unknown is
+# deliberately not here: losing a talk's transcript is the costly mistake, so only a
+# confident "music" or "sound effect" skips it. The Transcribe button still works.
+UNTRANSCRIBED_KINDS = frozenset({AUDIO_KIND_MUSIC, AUDIO_KIND_SFX})
+
+# Genres that tag a recording of speech rather than music. ID3v1's own list has "Speech"
+# (101) and "Audiobook"; podcast tools write "Podcast". A file tagged with one of these is
+# speech however else it is tagged.
+SPEECH_GENRES = frozenset(
+    {"speech", "spoken word", "spoken", "podcast", "audiobook", "audio book", "talk",
+     "lecture", "interview", "news", "comedy", "books & spoken"}
+)
+
 MAX_TERMS = 20
 MAX_TERM_LENGTH = 60
 # Long enough for "F# minor", "Abm", Camelot "12B" or Open Key "10d"; anything longer is
@@ -182,3 +203,17 @@ def index_text(asset: Any) -> str:
         parts.append(f"{energy} energy")
 
     return " ".join(parts)
+
+
+def kind_from_tags(genres: list[str], bpm: Optional[float]) -> Optional[str]:
+    """What a file's own tags say it is, or None when they say nothing either way.
+
+    A speech genre wins. Otherwise a genre or a tempo means music: nobody tags a voice
+    memo with a BPM, and a genre on a non-speech file is a music genre. Untagged files
+    stay unknown, which keeps them transcribed.
+    """
+    if any(genre.strip().lower() in SPEECH_GENRES for genre in genres):
+        return AUDIO_KIND_SPEECH
+    if genres or bpm is not None:
+        return AUDIO_KIND_MUSIC
+    return None

backend/app/routers/assets.py

@@ -369,7 +369,15 @@ def update_asset(
     """
     asset = _owned(asset_id, user.id, session)
     changes = music.to_columns(payload.model_dump(exclude_unset=True))
+    if changes.get("audio_kind") and asset.asset_type != "audio":
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={"code": "not_audio", "message": "Only audio files have an audio kind"},
+        )
+    kind_before = asset.audio_kind
     asset = service.apply_metadata(session, asset, changes)
+    if "audio_kind" in changes and asset.audio_kind != kind_before:
+        service.follow_kind_change(session, asset)
     # Same reason as the read above, and it bites harder here: the library store replaces
     # its copy with whatever this returns, so a response with empty tags makes an
     # asset's tags disappear from the grid after a rename until the page is reloaded.

backend/app/schemas_assets.py

@@ -86,6 +86,8 @@ class AssetRead(BaseModel):
     # Null is "not known", which is different from False, "instrumental".
     music_vocals: Optional[bool] = None
     music_energy: Optional[str] = None
+    # music | speech | sfx, null when not known. Decides transcription; see app/music.py.
+    audio_kind: Optional[str] = None
 
     # Batch-loaded for a listing, never per row: sixty assets a page each asking for
     # their own tags is sixty queries that grow with the page.
@@ -170,6 +172,7 @@ class AssetUpdate(BaseModel):
     music_key: Optional[str] = Field(default=None, max_length=music.MAX_KEY_LENGTH)
     music_vocals: Optional[bool] = None
     music_energy: Optional[Literal["low", "medium", "high"]] = None
+    audio_kind: Optional[Literal["music", "speech", "sfx"]] = None
 
     @field_validator("music_genres", "music_moods", "music_instruments")
     @classmethod

backend/app/services/assets.py

@@ -110,6 +110,7 @@ def ingest_file(
     name: Optional[str] = None,
     source: str = SOURCE_UPLOAD,
     content_type: Optional[str] = None,
+    audio_kind: Optional[str] = None,
 ) -> Asset:
     """`ingest_upload` for bytes that are already a file on disk.
 
@@ -121,6 +122,10 @@ def ingest_file(
 
     `name` overrides the filename-derived default. An import knows the real title, and
     deriving it back out of a filename would lose anything the filename could not hold.
+
+    `audio_kind` is for a caller that knows what the audio is before it lands, a music
+    page import: set on the row before transcription is decided, which is the moment
+    after cataloguing, so a song from a music library is never sent to Deepgram.
     """
     asset_type = _require_supported(filename)
     stored = storage.write_file(new_key(user_id, extension_of(filename)), source_path)
@@ -134,6 +139,7 @@ def ingest_file(
         source=source,
         content_type=content_type,
         name=name,
+        audio_kind=audio_kind,
     )
 
 
@@ -157,6 +163,7 @@ def _catalogue(
     name: Optional[str] = None,
     origin_project_id: Optional[str] = None,
     origin_reference_id: Optional[str] = None,
+    audio_kind: Optional[str] = None,
 ) -> Asset:
     """Everything that happens once the bytes are safely stored."""
     extension = extension_of(filename)
@@ -173,6 +180,7 @@ def _catalogue(
         checksum_sha256=stored.sha256,
         origin_project_id=origin_project_id,
         origin_reference_id=origin_reference_id,
+        audio_kind=audio_kind if asset_type == TYPE_AUDIO else None,
     )
     session.add(asset)
     session.commit()
@@ -272,7 +280,9 @@ def apply_embedded_fields(session: Session, asset: Asset, embedded: dict) -> Non
 
 
 # What a harvest may fill. Not `name` or `description`: see ingest/embedded_metadata.py.
-_EMBEDDED_FIELDS = frozenset(attribution.ATTRIBUTION_FIELDS) | frozenset(music.MUSIC_FIELDS)
+_EMBEDDED_FIELDS = (
+    frozenset(attribution.ATTRIBUTION_FIELDS) | frozenset(music.MUSIC_FIELDS) | {"audio_kind"}
+)
 
 
 def _is_unset(value) -> bool:
@@ -310,6 +320,11 @@ def _chain_transcription(session: Session, asset: Asset) -> None:
     if asset.source == SOURCE_AI:
         return
 
+    # Music and sound effects, by their tags, a music page import, or a person's say-so.
+    # Unknown is still transcribed: see `music.UNTRANSCRIBED_KINDS`.
+    if asset.audio_kind in music.UNTRANSCRIBED_KINDS:
+        return
+
     try:
         if not load_deepgram_key(session, asset.user_id):
             logger.info(
@@ -326,6 +341,19 @@ def _chain_transcription(session: Session, asset: Asset) -> None:
         logger.warning("Could not queue transcription for asset %s", asset.id, exc_info=True)
 
 
+def follow_kind_change(session: Session, asset: Asset) -> None:
+    """Catch up on what an audio file's new kind says should have happened at upload.
+
+    Marked speech: transcribe it, if it never was. Marked music: measure it. Neither
+    undoes anything; a transcript already made stays, since the person may want lyrics
+    searchable, and the guards inside each chain keep a second job from being queued.
+    """
+    if asset.audio_kind == music.AUDIO_KIND_SPEECH and asset.transcript_status in (None, "none"):
+        _chain_transcription(session, asset)
+    elif asset.audio_kind == music.AUDIO_KIND_MUSIC:
+        _chain_music_analysis(session, asset)
+
+
 def _chain_music_analysis(session: Session, asset: Asset) -> None:
     """Measure an audio file's tempo, key and energy the moment it lands.
 
@@ -343,7 +371,7 @@ def _chain_music_analysis(session: Session, asset: Asset) -> None:
     from app.jobs import enrichment as enrichment_jobs
     from app.models.job import KIND_ANALYZE_MUSIC
 
-    if not analysable(asset):
+    if not analysable(asset) or asset.audio_kind in (music.AUDIO_KIND_SPEECH, music.AUDIO_KIND_SFX):
         return
     try:
         if enrichment_jobs.active_job(session, asset.id, KIND_ANALYZE_MUSIC) is not None:
@@ -953,6 +981,7 @@ def to_read_model(
         music_key=asset.music_key,
         music_vocals=asset.music_vocals,
         music_energy=asset.music_energy,
+        audio_kind=asset.audio_kind,
         tags=[
             AssetTagRead(id=t.id, name=t.name, category_id=t.category_id)
             for t in (asset_tags or [])