From 0c457e8f2a1cf914000ed64526c8ff2dda1f3226 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 11:18:31 +0000 Subject: [PATCH 1/2] Import music tracks from a library's web page Paste a page (incompetech, Mixkit, or any page listing tracks), review what it holds, tick what you want, and import each as its own job with the page's details on it. - Read best source first: incompetech's pieces.json (the file its page loads), schema.org MusicRecording JSON-LD, Mixkit's card links for moods, instruments and tags, then the AI provider (held to the audio URLs actually on the page), then bare audio links. - ingest/web.py: SSRF-checked on every redirect, capped while streaming, uncompressed audio only, non-audio responses refused. - Page metadata lands stamped "embedded": credits (incompetech's exact CC BY wording), genres, moods, instruments, BPM and tags. Tempo, key and energy the page left out are then measured from the sound. - Tracks already in the library are marked and not downloaded twice; at most 50 per import. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DCL4jm6ogZBBhnjEfBBaXU --- backend/app/enrichment/page_import.py | 345 +++++++++++++ backend/app/ingest/page_tracks.py | 476 ++++++++++++++++++ backend/app/ingest/web.py | 220 ++++++++ backend/app/jobs/enrichment.py | 17 + backend/app/jobs/registry.py | 12 +- backend/app/models/job.py | 5 + backend/app/routers/imports.py | 131 ++++- .../fixtures/pages/incompetech_music.html | 17 + .../fixtures/pages/incompetech_pieces.json | 10 + backend/tests/fixtures/pages/mixkit_mood.html | 32 ++ backend/tests/fixtures/pages/plain_links.html | 10 + backend/tests/test_page_import.py | 360 +++++++++++++ docs/page-import.md | 58 +++ frontend/src/api/imports.ts | 54 ++ frontend/src/components/ActivityIndicator.tsx | 1 + frontend/src/components/AddPanel.tsx | 6 +- frontend/src/components/PageImport.test.tsx | 154 ++++++ frontend/src/components/PageImport.tsx | 309 ++++++++++++ 18 files changed, 2211 insertions(+), 6 deletions(-) create mode 100644 backend/app/enrichment/page_import.py create mode 100644 backend/app/ingest/page_tracks.py create mode 100644 backend/app/ingest/web.py create mode 100644 backend/tests/fixtures/pages/incompetech_music.html create mode 100644 backend/tests/fixtures/pages/incompetech_pieces.json create mode 100644 backend/tests/fixtures/pages/mixkit_mood.html create mode 100644 backend/tests/fixtures/pages/plain_links.html create mode 100644 backend/tests/test_page_import.py create mode 100644 docs/page-import.md create mode 100644 frontend/src/components/PageImport.test.tsx create mode 100644 frontend/src/components/PageImport.tsx diff --git a/backend/app/enrichment/page_import.py b/backend/app/enrichment/page_import.py new file mode 100644 index 0000000..fe238a1 --- /dev/null +++ b/backend/app/enrichment/page_import.py @@ -0,0 +1,345 @@ +"""Importing tracks from a music library's web page: preview, then one job per track. + +Two steps, because a page can list a thousand tracks and the person wants twelve. The +preview reads the page and returns what it found, synchronously, for a review list; the +import queues one job per ticked track, as a playlist import does, so a failure on track +eight costs track eight and each one is cancellable. + +What is found is described in `ingest/page_tracks.py`. This module adds the parts that +need a session: the AI fallback (a provider, and usage to record), recognising what is +already in the library, and writing what the page said onto the imported asset. + +Only ticked tracks are downloaded, one at a time on the enrichment queue. A music +library's licence is about using its tracks, and some forbid bulk downloading; one person +importing what they chose, at the pace of one worker, is the use those licences describe. +""" + +from __future__ import annotations + +import json +import logging +import re +import tempfile +from dataclasses import dataclass, fields +from pathlib import Path +from typing import Any, Callable, Optional +from urllib.parse import urljoin + +from sqlmodel import Session, col, select + +from app import music +from app.clock import utcnow +from app.ingest import page_tracks, web +from app.ingest.embedded_metadata import normalise_partial_date +from app.ingest.filetypes import SOURCE_URL +from app.ingest.page_tracks import PageExtraction, PageTrack +from app.models.asset import Asset +from app.models.job import EnrichmentJob +from app.providers import build_provider +from app.providers.base import ProviderError +from app.services import assets as asset_service +from app.services import tags as tag_service +from app.storage import build_storage, capacity +from app.usage import events as usage_events + +logger = logging.getLogger(__name__) + +Progress = Callable[..., None] + +# One import request's worth. A review list of fifty ticked tracks is already an +# afternoon of listening; more at once is a bulk download, not a choice. +MAX_TRACKS_PER_IMPORT = 50 + +# The longest page the model is shown, in characters, and the most tracks it may return. +AI_PAGE_CHARS = 60_000 +AI_MAX_TRACKS = 100 + +AI_SYSTEM = ( + "You read a web page from a music library and list the music tracks on it as JSON. " + "You only report what the page states. You never invent a URL." +) + + +class PageImportError(Exception): + """The page could not be read, or an import job could not run. Worded for the user.""" + + +# ─── preview ───────────────────────────────────────────────────────────────── + + +def preview(session: Session, user_id: str, url: str, *, use_ai: bool = True) -> PageExtraction: + """Read a page and return the tracks on it, best source first.""" + try: + final_url, page = web.fetch_text(url) + extraction = page_tracks.extract_structured(final_url, page, web.fetch_text) + except web.WebFetchError as exc: + raise PageImportError(str(exc)) from exc + + if extraction is None and use_ai: + provider = build_provider(session, user_id) + if provider is not None: + extraction = _ask_ai(session, user_id, provider, final_url, page) + + if extraction is None or not extraction.tracks: + links = page_tracks.extract_links(final_url, page) + if links.tracks: + if extraction is not None and extraction.method == page_tracks.METHOD_AI: + links.note = "The AI found no tracks it could tie to a file, so these are the bare links." + elif use_ai: + links.note = ( + "No AI provider is configured, so these are the bare audio links. " + "Add one in Settings to read titles and details from the page." + ) + extraction = links + else: + extraction = PageExtraction( + final_url, + page_tracks.METHOD_LINKS, + [], + "links", + note=( + "No audio files were found on this page. If the site builds its track " + "list with JavaScript after loading, the files are not in the page itself." + ), + ) + return extraction + + +def already_imported(session: Session, user_id: str, tracks: list[PageTrack]) -> dict[str, str]: + """{audio_url: asset id} for the tracks this library already has.""" + keys: dict[str, str] = {} + for track in tracks: + keys[track.source_url or track.audio_url] = track.audio_url + keys[track.audio_url] = track.audio_url + if not keys: + return {} + rows = session.exec( + select(Asset.source_url, Asset.id).where( + Asset.user_id == user_id, col(Asset.source_url).in_(list(keys)) + ) + ).all() + return {keys[source_url]: asset_id for source_url, asset_id in rows} + + +# ─── the AI fallback ───────────────────────────────────────────────────────── + + +def _ask_ai(session: Session, user_id: str, provider, page_url: str, page: str) -> Optional[PageExtraction]: + """Ask the provider to list the tracks, holding it to the audio URLs the page has. + + The grounding rule is the point. A model asked for download links will produce + plausible ones, and a plausible URL that is not on the page is either a 404 or a file + nobody chose. Every URL it returns is checked against the page's own audio URLs, and + an entry whose URL is not among them is dropped. + """ + candidates = page_tracks.candidate_audio_urls(page_url, page) + if not candidates: + return None + + prompt = ( + f"Page URL: {page_url}\n\n" + "Audio file URLs found on the page (the only URLs you may use):\n" + + "\n".join(candidates[:400]) + + "\n\nPage text, with link targets written as [link: …]:\n" + + page_tracks.page_text(page, AI_PAGE_CHARS) + + "\n\nList each music track on this page as a JSON array, one object per track, " + "with these keys, leaving out any the page does not state:\n" + ' "audio_url" (exactly one of the URLs above), "title", "artist", "album", ' + '"description", "duration" (seconds), "bpm", "genres" (list), "moods" (list), ' + '"instruments" (list), "tags" (list), "license", "credit" (attribution text the ' + 'page asks for), "page_url" (the track\'s own page, if linked), "published" ' + "(YYYY, YYYY-MM or YYYY-MM-DD).\n" + "Answer with the JSON array only." + ) + try: + completion = provider.complete(prompt, system=AI_SYSTEM) + except ProviderError as exc: + raise PageImportError(f"The AI provider could not read the page: {exc}") from exc + usage_events.record_completion(session, None, user_id, completion) + + allowed = set(candidates) + tracks = [t for t in (_track_from_ai(page_url, item) for item in _json_array(completion.text)) if t] + tracks = [t for t in tracks if t.audio_url in allowed] + unique = list({t.audio_url: t for t in tracks}.values())[:AI_MAX_TRACKS] + return PageExtraction(page_url, page_tracks.METHOD_AI, unique, "AI") + + +def _json_array(text: str) -> list[Any]: + """The first JSON array in a reply, tolerating a code fence or a sentence around it.""" + start, end = text.find("["), text.rfind("]") + if start < 0 or end <= start: + return [] + try: + value = json.loads(text[start : end + 1]) + except ValueError: + return [] + return value if isinstance(value, list) else [] + + +def _track_from_ai(page_url: str, item: Any) -> Optional[PageTrack]: + if not isinstance(item, dict): + return None + audio_url = item.get("audio_url") + title = _short(item.get("title"), 255) + if not isinstance(audio_url, str) or not title: + return None + audio_url = urljoin(page_url, audio_url.strip()) + track_page = item.get("page_url") + duration = item.get("duration") + return PageTrack( + audio_url=audio_url, + title=title, + creator=_short(item.get("artist"), 500), + source_title=_short(item.get("album"), 500), + description=_short(item.get("description"), 2000), + duration_seconds=float(duration) if isinstance(duration, (int, float)) and duration > 0 else None, + published_date=_partial_date(item.get("published")), + license=_short(item.get("license"), 500), + credit_line=_short(item.get("credit"), 2000), + source_url=urljoin(page_url, track_page) if isinstance(track_page, str) and track_page else audio_url, + genres=_list(item.get("genres")), + moods=_list(item.get("moods")), + instruments=_list(item.get("instruments")), + tags=_list(item.get("tags")), + bpm=music.parse_bpm(item.get("bpm")), + ) + + +def _short(value: Any, limit: int) -> Optional[str]: + if not isinstance(value, str): + return None + text = " ".join(value.split()) + return text[:limit] or None + + +def _list(value: Any) -> list[str]: + if isinstance(value, list): + return music.clean_terms(v for v in value if isinstance(v, str)) + if isinstance(value, str): + return music.split_terms(value) + return [] + + +def _partial_date(value: Any) -> Optional[str]: + return normalise_partial_date(value) if isinstance(value, str) else None + + +# ─── the import job ────────────────────────────────────────────────────────── + + +def encode_track(track: PageTrack) -> str: + return json.dumps(track.to_dict(), sort_keys=True) + + +def decode_track(payload: Optional[str]) -> PageTrack: + try: + data = json.loads(payload or "{}") + except ValueError: + data = {} + if not isinstance(data, dict): + data = {} + data = data.get("track", data) + known = {f.name for f in fields(PageTrack)} + values = {k: v for k, v in data.items() if k in known} + if not isinstance(values.get("audio_url"), str) or not isinstance(values.get("title"), str): + raise PageImportError("This import job has no track to fetch") + return PageTrack(**values) + + +@dataclass +class TrackImportResult: + detail: str + created_asset_id: Optional[str] = None + + +def run_track(session: Session, job: EnrichmentJob, progress: Progress) -> TrackImportResult: + """Download one track and catalogue it with what its page said.""" + track = decode_track(job.payload) + + existing = already_imported(session, job.user_id, [track]) + if track.audio_url in existing: + return TrackImportResult("Already in your library", existing[track.audio_url]) + + if not capacity.has_room(): + raise PageImportError(capacity.DISK_FULL_MESSAGE) + + with tempfile.TemporaryDirectory(prefix="gam-track-") as workdir: + progress("Downloading", 5, "") + + def report(received: int, total: Optional[int]) -> None: + if total: + progress("Downloading", 5 + int(80 * received / total), "") + + try: + path = web.download_audio(track.audio_url, Path(workdir), "track", on_progress=report) + except web.WebFetchError as exc: + raise PageImportError(str(exc)) from exc + + progress("Saving", 88, "") + stem = re.sub(r"[\\/]+", "-", track.title).strip() or "track" + try: + asset = asset_service.ingest_file( + session, + build_storage(), + user_id=job.user_id, + source_path=path, + filename=f"{stem[:200]}{path.suffix.lower()}", + name=track.title, + source=SOURCE_URL, + ) + except asset_service.UnsupportedFile as exc: + raise PageImportError("The site sent a file the library does not accept") from exc + + progress("Recording what the page said", 95, "") + tag_count = apply_track_metadata(session, asset, track) + detail = "Imported" + if tag_count: + detail += f" · {tag_count} tag{'' if tag_count == 1 else 's'}" + return TrackImportResult(detail, asset.id) + + +def apply_track_metadata(session: Session, asset: Asset, track: PageTrack) -> int: + """What the page said about the track, stamped `embedded`, and its tags attached. + + `embedded` for the same reason an imported video's site metadata is: a statement from + the source that nobody here checked. Written over whatever the file's own tags said: + the site describing its own track is the better authority. Returns the tag count. + """ + changes: dict[str, Any] = { + "source_url": track.source_url or track.audio_url, + "retrieved_at": utcnow(), + } + for name in ("creator", "publisher", "source_title", "license", "credit_line", "description"): + value = getattr(track, name) + if value: + changes[name] = value + # Normalised here, not trusted: the track came back from the browser, and the date + # filters compare this column as text, which only works for ISO partial dates. + published = normalise_partial_date(track.published_date) if track.published_date else None + if published: + changes["published_date"] = published + if track.genres: + changes["music_genres"] = music.encode_terms(track.genres) + if track.moods: + changes["music_moods"] = music.encode_terms(track.moods) + if track.instruments: + changes["music_instruments"] = music.encode_terms(track.instruments) + if track.bpm is not None: + changes["music_bpm"] = track.bpm + + try: + asset_service.apply_embedded_metadata(session, asset, changes) + except Exception: # noqa: BLE001 - the track is in; losing its metadata must not undo that + logger.warning("Could not store page metadata for %s", asset.id, exc_info=True) + + attached = 0 + try: + for name in track.tags: + tag = tag_service.get_or_create(session, asset.user_id, name) + if tag is not None and tag_service.attach(session, asset.id, tag.id): + attached += 1 + if attached: + asset_service.reindex_ids(session, [asset.id]) + except Exception: # noqa: BLE001 - same + logger.warning("Could not attach page tags to %s", asset.id, exc_info=True) + return attached diff --git a/backend/app/ingest/page_tracks.py b/backend/app/ingest/page_tracks.py new file mode 100644 index 0000000..d871154 --- /dev/null +++ b/backend/app/ingest/page_tracks.py @@ -0,0 +1,476 @@ +"""Finding the tracks on a web page, with what the page says about each. + +For music libraries that list their tracks on a page: a mood, a genre, a search. Three +ways in, tried in order of how much they can be trusted: + +1. **What the site publishes for machines.** schema.org `MusicRecording` data (JSON-LD), + which Mixkit and many others put in the page for search engines, and incompetech's + `pieces.json`, the file its own page loads. Both are the site's structured statement + of what each track is, so nothing has to be guessed from layout. +2. **The AI provider**, for a page with neither (`enrichment/page_import.py`). It reads + the page's text and is only allowed to name audio URLs that actually appear in it. +3. **Bare audio links**, when there is no AI provider: every `.mp3` the page links to, + named by its link text. Little metadata, but something to tick. + +Pure apart from `fetch_text`, which is passed in, so the adapters are tested on saved +pages with no network. + +A site-specific adapter exists only where the structured data leaves out something worth +having: Mixkit's JSON-LD has one genre per track, while its cards also link each track's +moods, instruments and tags. Each adapter names its site, and a site that changes its +layout breaks that adapter alone; the JSON-LD underneath keeps working. +""" + +from __future__ import annotations + +import html as html_lib +import json +import re +from dataclasses import asdict, dataclass, field +from typing import Any, Callable, Iterable, Optional +from urllib.parse import quote, unquote, urljoin, urlsplit + +from app import music +from app.ingest.embedded_metadata import normalise_partial_date +from app.ingest.filetypes import AUDIO_EXTENSIONS, extension_of + +METHOD_STRUCTURED = "structured" +METHOD_AI = "ai" +METHOD_LINKS = "links" + +# How many a single review list shows. Incompetech's catalogue page lists all of it, +# well over a thousand; the review is filtered client-side, and this only stops a +# pathological page from producing an unbounded response. +MAX_TRACKS = 2500 +MAX_TEXT = 500 +MAX_DESCRIPTION = 2000 + +FetchText = Callable[[str], tuple[str, str]] + + +@dataclass +class PageTrack: + """One track as a page describes it. Every field but the URL and title optional.""" + + audio_url: str + title: str + creator: Optional[str] = None + publisher: Optional[str] = None + source_title: Optional[str] = None + description: Optional[str] = None + duration_seconds: Optional[float] = None + published_date: Optional[str] = None + license: Optional[str] = None + credit_line: Optional[str] = None + # Where to cite, and how a re-import is recognised. The track's own page when the + # site has one, else the audio file itself. + source_url: Optional[str] = None + genres: list[str] = field(default_factory=list) + moods: list[str] = field(default_factory=list) + instruments: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + bpm: Optional[float] = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class PageExtraction: + page_url: str + method: str + tracks: list[PageTrack] + # Which adapter or source produced them, for the review screen: "incompetech", + # "mixkit", "schema.org", "AI", "links". + source: str + note: str = "" + + +# ─── entry point ───────────────────────────────────────────────────────────── + + +def extract_structured(page_url: str, page: str, fetch_text: FetchText) -> Optional[PageExtraction]: + """Tracks the site states outright. None when it states none.""" + host = (urlsplit(page_url).hostname or "").lower() + + if _is_host(host, "incompetech.com") and "pieces.json" in page: + tracks = _incompetech(page_url, page, fetch_text) + if tracks: + return PageExtraction(page_url, METHOD_STRUCTURED, tracks, "incompetech") + + tracks = _json_ld_tracks(page_url, page) + if tracks: + source = "schema.org" + if _is_host(host, "mixkit.co"): + _enrich_mixkit(page_url, page, tracks) + source = "mixkit" + return PageExtraction(page_url, METHOD_STRUCTURED, tracks[:MAX_TRACKS], source) + + return None + + +def extract_links(page_url: str, page: str) -> PageExtraction: + """Every audio file the page links to, named by its link text. The last resort.""" + tracks: list[PageTrack] = [] + seen: set[str] = set() + for href, text in _audio_anchors(page_url, page): + if href in seen: + continue + seen.add(href) + tracks.append( + PageTrack(audio_url=href, title=text or _title_from_url(href), source_url=href) + ) + return PageExtraction(page_url, METHOD_LINKS, tracks[:MAX_TRACKS], "links") + + +def candidate_audio_urls(page_url: str, page: str) -> list[str]: + """Every audio URL the page contains, anywhere: links, players, scripts, data attributes. + + What the AI path may choose from. A model that names a URL not in this list has + invented it, and is not believed. + """ + found: list[str] = [] + seen: set[str] = set() + pattern = r"""(?:https?:)?//[^\s"'<>()\\]+?\.(?:mp3|ogg|oga|opus|wav|flac|m4a|aac)(?:\?[^\s"'<>()\\]*)?(?=["'\s<>)\\]|$)""" + for match in re.finditer(pattern, html_lib.unescape(page), re.I): + url = urljoin(page_url, match.group(0)) + if url not in seen: + seen.add(url) + found.append(url) + for href, _text in _audio_anchors(page_url, page): + if href not in seen: + seen.add(href) + found.append(href) + return found + + +def page_text(page: str, limit: int = 60_000) -> str: + """The page as readable text for a model, with every link's target kept inline. + + Scripts and styles go, except JSON-LD and inline JSON, which often hold exactly the + track list a model needs. Link targets are kept as "text [link: url]" so the model can + tie a title to its file; angle brackets would not survive the tag stripping below. + """ + body = re.sub(r"<(style|noscript|svg)[^>]*>.*?", " ", page, flags=re.S | re.I) + body = re.sub(r"]*json)[^>]*>.*?", " ", body, flags=re.S | re.I) + body = re.sub( + r"]*href\s*=\s*[\"']([^\"']+)[\"'][^>]*>(.*?)", + lambda m: f" {m.group(2)} [link: {m.group(1)}] ", + body, + flags=re.S | re.I, + ) + body = re.sub( + r"<(?:audio|source)\b[^>]*src\s*=\s*[\"']([^\"']+)[\"'][^>]*>", + lambda m: f" [audio: {m.group(1)}] ", + body, + flags=re.I, + ) + body = re.sub(r"|", "\n", body, flags=re.I) + body = re.sub(r"<[^>]+>", " ", body) + body = html_lib.unescape(body) + body = re.sub(r"[ \t]+", " ", body) + body = re.sub(r"\n\s*\n+", "\n", body) + return body.strip()[:limit] + + +# ─── incompetech ───────────────────────────────────────────────────────────── + +# Kevin MacLeod's catalogue. The page renders nothing itself: its script fetches +# `pieces.json` beside it and builds the list in the browser. The genre and album names +# are lookup tables in that same script, read out of the page rather than copied here, +# so a genre added there is picked up without a change here. +# +# The credit is the attribution code the page shows under every track, verbatim: it is +# what the licence asks for, so it is stored as the credit line rather than composed. +_INCOMPETECH_CREATOR = "Kevin MacLeod" +_INCOMPETECH_PUBLISHER = "incompetech.com" +_INCOMPETECH_LICENSE = "CC BY 4.0" +_INCOMPETECH_CREDIT = ( + '"{title}" Kevin MacLeod (incompetech.com) ' + "Licensed under Creative Commons: By Attribution 4.0 License " + "http://creativecommons.org/licenses/by/4.0/" +) +_INCOMPETECH_MP3_PATH = "/music/royalty-free/mp3-royaltyfree/" + + +def _incompetech(page_url: str, page: str, fetch_text: FetchText) -> list[PageTrack]: + genres = { + m.group(1): m.group(2) + for m in re.finditer(r'"id"\s*:\s*"?(\d+)"?\s*,\s*"genre"\s*:\s*"([^"]+)"', page) + } + albums = { + m.group(3): m.group(1) + for m in re.finditer( + r'"collection_name"\s*:\s*"([^"]+)"\s*,\s*"collection_category"\s*:\s*"([^"]+)"' + r'\s*,\s*"code"\s*:\s*"(\d+)"', + page, + ) + } + + pieces_url, raw = fetch_text(urljoin(page_url, "pieces.json")) + try: + pieces = json.loads(raw) + except ValueError: + return [] + if isinstance(pieces, dict): + pieces = pieces.get("pieces") or pieces.get("data") or [] + if not isinstance(pieces, list): + return [] + + tracks: list[PageTrack] = [] + for piece in pieces: + if not isinstance(piece, dict): + continue + filename = _text(piece.get("filename")) + title = _text(piece.get("title")) + if not filename or not title or extension_of(filename) not in AUDIO_EXTENSIONS: + continue + audio_url = urljoin(pieces_url, _INCOMPETECH_MP3_PATH + quote(filename)) + genre = genres.get(str(piece.get("genre") or "")) + album = albums.get(str(piece.get("collection") or "")) + tracks.append( + PageTrack( + audio_url=audio_url, + title=title[:255], + creator=_INCOMPETECH_CREATOR, + publisher=_INCOMPETECH_PUBLISHER, + source_title=album if album and album != "not in a collection" else None, + description=_plain(piece.get("description"), MAX_DESCRIPTION), + duration_seconds=_clock_seconds(piece.get("length")), + published_date=normalise_partial_date(_text(piece.get("uploaded"))), + license=_INCOMPETECH_LICENSE, + credit_line=_INCOMPETECH_CREDIT.format(title=title), + source_url=audio_url, + genres=[genre] if genre and genre != "Unclassifiable" else [], + moods=music.split_terms(piece.get("feel")), + instruments=music.split_terms(piece.get("instruments")), + bpm=music.parse_bpm(piece.get("bpm")), + ) + ) + if len(tracks) >= MAX_TRACKS: + break + return tracks + + +# ─── schema.org JSON-LD ────────────────────────────────────────────────────── + + +def _json_ld_tracks(page_url: str, page: str) -> list[PageTrack]: + tracks: list[PageTrack] = [] + seen: set[str] = set() + for block in re.findall( + r"]*type\s*=\s*[\"']application/ld\+json[\"'][^>]*>(.*?)", + page, + flags=re.S | re.I, + ): + try: + data = json.loads(html_lib.unescape(block.strip())) + except ValueError: + continue + for node in _walk(data): + if not _is_type(node, "MusicRecording", "AudioObject"): + continue + track = _from_recording(page_url, node) + if track and track.audio_url not in seen: + seen.add(track.audio_url) + tracks.append(track) + return tracks + + +def _walk(node: Any) -> Iterable[dict]: + if isinstance(node, dict): + yield node + for value in node.values(): + yield from _walk(value) + elif isinstance(node, list): + for value in node: + yield from _walk(value) + + +def _is_type(node: dict, *names: str) -> bool: + kind = node.get("@type") + kinds = kind if isinstance(kind, list) else [kind] + return any(k in names for k in kinds if isinstance(k, str)) + + +def _from_recording(page_url: str, node: dict) -> Optional[PageTrack]: + audio = node.get("audio") if isinstance(node.get("audio"), dict) else {} + candidates = [node.get("contentUrl"), audio.get("contentUrl"), node.get("url")] + audio_url = next( + ( + urljoin(page_url, c) + for c in candidates + if isinstance(c, str) and extension_of(urlsplit(c).path) in AUDIO_EXTENSIONS + ), + None, + ) + title = _text(node.get("name")) + if not audio_url or not title: + return None + + # The track's own page when the data names one. Otherwise the file itself, not the + # listing it was found on: a listing is shared by every track on it, and the source + # URL is also what recognises a track already imported. + page_link = node.get("url") + track_page = ( + urljoin(page_url, page_link) + if isinstance(page_link, str) and urljoin(page_url, page_link) != audio_url + else audio_url + ) + album = node.get("inAlbum") + licence_url = _text(node.get("license")) + notice = _text(node.get("copyrightNotice")) + return PageTrack( + audio_url=audio_url, + title=title[:255], + creator=_name_of(node.get("byArtist")) or _name_of(node.get("creator")), + publisher=_name_of(node.get("publisher")), + source_title=_name_of(album) if album else None, + description=_plain(node.get("description"), MAX_DESCRIPTION), + duration_seconds=_iso_duration(node.get("duration") or audio.get("duration")), + published_date=normalise_partial_date(_text(node.get("datePublished"))), + license=(f"{notice} ({licence_url})" if notice and licence_url else notice or licence_url) + or None, + source_url=track_page, + genres=_terms(node.get("genre")), + tags=_terms(node.get("keywords")), + ) + + +def _name_of(value: Any) -> Optional[str]: + if isinstance(value, str): + return _text(value)[:MAX_TEXT] or None + if isinstance(value, dict): + name = value.get("name") + return _text(name)[:MAX_TEXT] or None if isinstance(name, str) else None + if isinstance(value, list): + names = [n for n in (_name_of(v) for v in value) if n] + return ", ".join(names)[:MAX_TEXT] or None + return None + + +def _terms(value: Any) -> list[str]: + if isinstance(value, list): + return music.clean_terms(v for v in value if isinstance(v, str)) + if isinstance(value, str): + return music.split_terms(value) + return [] + + +# ─── mixkit ────────────────────────────────────────────────────────────────── + +# Each card is an audio player keyed by its MP3, followed by links to the track's genre, +# moods, instruments and tags. The JSON-LD has the genre only, so the rest comes from +# here, matched to the JSON-LD track by that same MP3 URL. +_MIXKIT_CARD = re.compile(r'data-audio-player-preview-url-value="([^"]+)"') +_MIXKIT_LINK = re.compile(r'class="meta-links__link"[^>]*href="([^"]+)"[^>]*>([^<]+)') + + +def _enrich_mixkit(page_url: str, page: str, tracks: list[PageTrack]) -> None: + by_url = {t.audio_url: t for t in tracks} + cards = list(_MIXKIT_CARD.finditer(page)) + for index, card in enumerate(cards): + track = by_url.get(urljoin(page_url, html_lib.unescape(card.group(1)))) + if track is None: + continue + end = cards[index + 1].start() if index + 1 < len(cards) else len(page) + for link in _MIXKIT_LINK.finditer(page, card.end(), end): + kind, label = _mixkit_kind(link.group(1)), html_lib.unescape(link.group(2)).strip() + if kind == "mood": + track.moods = music.clean_terms([*track.moods, label]) + elif kind == "instrument": + track.instruments = music.clean_terms([*track.instruments, label]) + elif kind == "tag": + track.tags = music.clean_terms([*track.tags, label]) + elif kind == "genre": + track.genres = music.clean_terms([*track.genres, label]) + + # A listing filtered to one mood or instrument says so of every track on it. + kind, label = _mixkit_page_filter(page_url) + for track in tracks: + track.publisher = track.publisher or "Mixkit" + if kind == "mood": + track.moods = music.clean_terms([*track.moods, label]) + elif kind == "instrument": + track.instruments = music.clean_terms([*track.instruments, label]) + + +def _mixkit_kind(href: str) -> Optional[str]: + parts = [p for p in urlsplit(href).path.split("/") if p] + if len(parts) >= 3 and parts[0] == "free-stock-music" and parts[1] in ("mood", "instrument", "tag"): + return parts[1] + if len(parts) == 2 and parts[0] == "free-stock-music": + return "genre" + return None + + +def _mixkit_page_filter(page_url: str) -> tuple[Optional[str], str]: + parts = [p for p in urlsplit(page_url).path.split("/") if p] + if len(parts) >= 3 and parts[0] == "free-stock-music" and parts[1] in ("mood", "instrument"): + return parts[1], unquote(parts[2]).replace("-", " ").title() + return None, "" + + +# ─── plain links ───────────────────────────────────────────────────────────── + + +def _audio_anchors(page_url: str, page: str) -> Iterable[tuple[str, str]]: + for match in re.finditer( + r"<(?:a\b[^>]*href|audio\b[^>]*src|source\b[^>]*src)\s*=\s*[\"']([^\"']+)[\"'][^>]*>(.*?)(?=<)", + page, + flags=re.S | re.I, + ): + href = urljoin(page_url, html_lib.unescape(match.group(1))) + if extension_of(urlsplit(href).path) not in AUDIO_EXTENSIONS: + continue + text = _plain(match.group(2), MAX_TEXT) or "" + yield href, text + + +def _title_from_url(url: str) -> str: + stem = unquote(urlsplit(url).path.rsplit("/", 1)[-1]).rsplit(".", 1)[0] + return re.sub(r"[-_]+", " ", stem).strip()[:255] or "Untitled track" + + +# ─── small parsers ─────────────────────────────────────────────────────────── + + +def _text(value: Any) -> str: + return " ".join(value.split()) if isinstance(value, str) else "" + + +def _plain(value: Any, limit: int) -> Optional[str]: + if not isinstance(value, str): + return None + text = html_lib.unescape(re.sub(r"<[^>]+>", " ", value)) + text = " ".join(text.split()) + return text[:limit] or None + + +def _iso_duration(value: Any) -> Optional[float]: + """ISO 8601 `PT1M53S` (schema.org's form) in seconds.""" + if not isinstance(value, str): + return None + match = re.fullmatch(r"P(?:T(?:(\d+(?:\.\d+)?)H)?(?:(\d+(?:\.\d+)?)M)?(?:(\d+(?:\.\d+)?)S)?)", value.strip()) + if not match or not any(match.groups()): + return None + hours, minutes, seconds = (float(g) if g else 0.0 for g in match.groups()) + return hours * 3600 + minutes * 60 + seconds + + +def _clock_seconds(value: Any) -> Optional[float]: + """`h:mm:ss` or `m:ss` in seconds.""" + if not isinstance(value, str) or not value.strip(): + return None + try: + parts = [float(p) for p in value.strip().split(":")] + except ValueError: + return None + total = 0.0 + for part in parts: + total = total * 60 + part + return total or None + + +def _is_host(host: str, domain: str) -> bool: + return host == domain or host.endswith("." + domain) diff --git a/backend/app/ingest/web.py b/backend/app/ingest/web.py new file mode 100644 index 0000000..c05b4e7 --- /dev/null +++ b/backend/app/ingest/web.py @@ -0,0 +1,220 @@ +"""Fetching a web page, or one audio file named on it, on the user's behalf. + +For importing tracks from a page (`ingest/page_tracks.py`). The same rules as +`generation/download.py`, which is fal-specific (its errors, its retry policy, image and +video only), restated here for a page and a track rather than shared through a flag: + +- **SSRF, on every hop.** The URL is the user's, and the track URLs are the page's. A + public address that redirects inward is the classic way past a check made only on the + first, so redirects are followed by hand and each target is checked. +- **Capped while it arrives,** against `Content-Length` first when there is one, then + counted, because a check made after the write has already filled the disk. +- **Uncompressed bodies only.** httpx decodes gzip and brotli with no output limit, so a + small compressed body could become gigabytes in memory before the cap saw a byte. A + page is the exception, being small and almost always sent compressed: it is decoded, + but read through a cap on the decoded size. +""" + +from __future__ import annotations + +import logging +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Callable, Iterator, Optional +from urllib.parse import urljoin, urlsplit + +import httpx +from fastapi import HTTPException + +from app.ingest.filetypes import AUDIO_EXTENSIONS, extension_of +from app.jobs.runner import readable_error +from app.safe_url import require_safe_external_url + +logger = logging.getLogger(__name__) + +MAX_REDIRECTS = 5 +TIMEOUT = httpx.Timeout(30.0, connect=10.0) +# A listing page with a hundred tracks and their markup is a few hundred kilobytes, and +# incompetech's whole catalogue as JSON about one megabyte. +MAX_PAGE_BYTES = 8 * 1024 * 1024 +# A long, high-bitrate track. Anything bigger is not a soundtrack. +MAX_TRACK_BYTES = 200 * 1024 * 1024 +MAX_TRACK_SECONDS = 15 * 60 + +# Says what is asking, so a site that wants to can tell this apart from a browser and +# from a crawler. It is one person's library fetching the tracks they picked. +USER_AGENT = "GeckoAssetManager/1.0 (+personal media library; one request per chosen track)" + +monotonic = time.monotonic + +# Bytes received, and the total when the server said. +Progress = Callable[[int, Optional[int]], None] + +_AUDIO_TYPES = { + "audio/mpeg": ".mp3", + "audio/mp3": ".mp3", + "audio/ogg": ".ogg", + "audio/opus": ".opus", + "audio/wav": ".wav", + "audio/x-wav": ".wav", + "audio/wave": ".wav", + "audio/flac": ".flac", + "audio/x-flac": ".flac", + "audio/aac": ".aac", + "audio/mp4": ".m4a", + "audio/x-m4a": ".m4a", +} + + +class WebFetchError(Exception): + """A page or a track could not be fetched. The message is worded for the user.""" + + +def require_safe(url: str) -> None: + try: + require_safe_external_url(url) + except HTTPException as exc: + raise WebFetchError(f"That address is not one this app will fetch: {readable_error(exc)}") from exc + + +def _client(accept_encoding: str) -> httpx.Client: + return httpx.Client( + timeout=TIMEOUT, + follow_redirects=False, + headers={"User-Agent": USER_AGENT, "Accept-Encoding": accept_encoding}, + ) + + +@contextmanager +def _get(client: httpx.Client, url: str) -> Iterator[tuple[str, httpx.Response]]: + """Follow redirects by hand, checking each hop. Yields (final url, open response).""" + target = url + for _hop in range(MAX_REDIRECTS + 1): + require_safe(target) + with client.stream("GET", target) as response: + if response.is_redirect: + target = urljoin(target, response.headers["location"]) + continue + yield target, response + return + raise WebFetchError("The address redirected too many times") + + +def _status_error(what: str, response: httpx.Response) -> WebFetchError: + if response.status_code in (401, 403): + return WebFetchError(f"The site refused to send the {what} (HTTP {response.status_code})") + if response.status_code == 404: + return WebFetchError(f"The site has no such {what} (HTTP 404)") + return WebFetchError(f"Could not fetch the {what} (HTTP {response.status_code})") + + +def _content_length(response: httpx.Response) -> Optional[int]: + raw = response.headers.get("content-length") + try: + return int(raw) if raw is not None else None + except ValueError: + return None + + +def fetch_text(url: str, *, max_bytes: int = MAX_PAGE_BYTES) -> tuple[str, str]: + """A page (or a JSON file) as text. Returns (the final URL after redirects, text). + + The final URL matters: relative links on the page resolve against it, not against + what was typed. + """ + try: + with _client("gzip, deflate") as client, _get(client, url) as (final, response): + if response.status_code >= 400: + raise _status_error("page", response) + chunks: list[bytes] = [] + received = 0 + # iter_bytes decodes as it goes, so this counts what the page becomes, not + # what was sent: the cap holds against a compression bomb too. + for chunk in response.iter_bytes(): + received += len(chunk) + if received > max_bytes: + raise WebFetchError("The page is too large to read") + chunks.append(chunk) + encoding = response.encoding or "utf-8" + except httpx.TimeoutException as exc: + raise WebFetchError("The site took too long to answer") from exc + except httpx.RequestError as exc: + # The type, not the text: a RequestError's message carries the full URL. + raise WebFetchError(f"Could not reach the site ({type(exc).__name__})") from exc + + return final, b"".join(chunks).decode(encoding, errors="replace") + + +def download_audio( + url: str, + workdir: Path, + stem: str, + *, + max_bytes: int = MAX_TRACK_BYTES, + on_progress: Optional[Progress] = None, +) -> Path: + """Fetch one track into `workdir/` and return the path. + + The extension comes from the URL path when it names an audio type, else from the + response's `Content-Type`. Anything that is neither is refused: a link that answers + with an HTML page (a login wall, a "download starting" interstitial) must not land in + the library as a broken track. + """ + report = on_progress or (lambda _received, _total: None) + try: + with _client("identity") as client, _get(client, url) as (final, response): + if response.status_code >= 400: + raise _status_error("track", response) + encoding = response.headers.get("content-encoding", "").strip().lower() + if encoding not in ("", "identity"): + raise WebFetchError("The site sent the track compressed, which is refused") + + extension = _audio_extension(final, response.headers.get("content-type")) + if extension is None: + raise WebFetchError("That link is not an audio file") + + total = _content_length(response) + if total is not None and total > max_bytes: + raise _too_large(max_bytes) + + destination = workdir / f"{stem}{extension}" + _stream_to(response, destination, max_bytes=max_bytes, total=total, report=report) + return destination + except httpx.TimeoutException as exc: + raise WebFetchError("The track did not download in time") from exc + except httpx.RequestError as exc: + raise WebFetchError(f"Could not download the track ({type(exc).__name__})") from exc + + +def _audio_extension(url: str, content_type: Optional[str]) -> Optional[str]: + from_path = extension_of(urlsplit(url).path) + if from_path in AUDIO_EXTENSIONS: + return from_path + media_type = (content_type or "").split(";", 1)[0].strip().lower() + return _AUDIO_TYPES.get(media_type) + + +def _stream_to( + response: httpx.Response, destination: Path, *, max_bytes: int, total: Optional[int], report: Progress +) -> None: + received = 0 + started = monotonic() + try: + with open(destination, "wb") as handle: + for chunk in response.iter_bytes(): + received += len(chunk) + if received > max_bytes: + raise _too_large(max_bytes) + if monotonic() - started > MAX_TRACK_SECONDS: + raise WebFetchError("The track did not download in time") + handle.write(chunk) + report(received, total) + except BaseException: + # BaseException, so a cancel raised from `report` cleans up too. + destination.unlink(missing_ok=True) + raise + + +def _too_large(max_bytes: int) -> WebFetchError: + return WebFetchError(f"The track is larger than the {max_bytes // (1024 * 1024)} MB limit") diff --git a/backend/app/jobs/enrichment.py b/backend/app/jobs/enrichment.py index d8b184c..38e620a 100644 --- a/backend/app/jobs/enrichment.py +++ b/backend/app/jobs/enrichment.py @@ -38,6 +38,8 @@ from app.enrichment.extract_text import run as run_extract_text from app.enrichment.generate_all import run as run_generate_all from app.enrichment.import_url import run as run_import_url +from app.enrichment.page_import import PageImportError +from app.enrichment.page_import import run_track as run_import_track from app.enrichment.source import NoSourceMaterial from app.enrichment.subvideo import SubvideoExtractionError from app.enrichment.summarize import run as run_summarize @@ -72,6 +74,7 @@ GENERATION_JOB_KINDS, KIND_GENERATE, KIND_GENERATE_ALL, + KIND_IMPORT_TRACK, KIND_IMPORT_URL, KIND_SUMMARIZE, KIND_TRANSCRIBE, @@ -351,6 +354,15 @@ def _run(job_id: str, q: JobQueue) -> None: # finished row, which is the first thing to look at when an import # produced something unexpected. job.payload = _with_created_asset(job.payload, imported.created_asset_id) + elif job.kind == KIND_IMPORT_TRACK: + imported_track = run_import_track(session, job, progress) + detail = imported_track.detail + if imported_track.created_asset_id: + # Merged rather than replacing, as for a URL import: the track's URL + # and what the page said stay readable on the finished row. + job.payload = _with_created_asset( + job.payload, imported_track.created_asset_id + ) elif job.kind == KIND_GENERATE: # The created ids are already in the payload — the job writes each as it # is made, so a restart mid-way knows what not to make again. @@ -446,6 +458,11 @@ def _run(job_id: str, q: JobQueue) -> None: set_fields(session, job, status="error", stage="", error_message=message) logger.info("Enrichment job %s failed: %s", job_id, message) + except PageImportError as exc: + message = str(exc) + set_fields(session, job, status="error", stage="", error_message=message) + logger.info("Enrichment job %s failed: %s", job_id, message) + except UrlImportError as exc: message = str(exc) set_fields(session, job, status="error", stage="", error_message=message) diff --git a/backend/app/jobs/registry.py b/backend/app/jobs/registry.py index 1bb19e3..0dd8462 100644 --- a/backend/app/jobs/registry.py +++ b/backend/app/jobs/registry.py @@ -21,7 +21,13 @@ from sqlmodel import Session, col, select from app.jobs.runner import ACTIVE_STATUSES, JobQueue, is_stale -from app.models.job import KIND_EXTRACT_SUBVIDEO, KIND_GENERATE, KIND_IMPORT_URL, EnrichmentJob +from app.models.job import ( + KIND_EXTRACT_SUBVIDEO, + KIND_GENERATE, + KIND_IMPORT_TRACK, + KIND_IMPORT_URL, + EnrichmentJob, +) from app.schemas_jobs import ActivityJobRead @@ -48,7 +54,9 @@ def __init__( # when the job was queued, so `asset_id` cannot carry them: for an extraction it stays # pointed at the source, and an import or a generation has no asset at all until it is # done. -_CREATES_AN_ASSET = frozenset({KIND_EXTRACT_SUBVIDEO, KIND_IMPORT_URL, KIND_GENERATE}) +_CREATES_AN_ASSET = frozenset( + {KIND_EXTRACT_SUBVIDEO, KIND_IMPORT_URL, KIND_IMPORT_TRACK, KIND_GENERATE} +) def _result_asset_ids(job: EnrichmentJob) -> List[str]: diff --git a/backend/app/models/job.py b/backend/app/models/job.py index 914dfa9..e9511ce 100644 --- a/backend/app/models/job.py +++ b/backend/app/models/job.py @@ -73,6 +73,10 @@ # The same over every audio file in a library, for the ones uploaded before it existed. # Library-wide for the reasons `KIND_HARVEST_ATTRIBUTION` is. KIND_ANALYZE_MUSIC_LIBRARY = "analyze_music_library" +# One track chosen from a music library's page (enrichment/page_import.py): downloaded, +# catalogued, and given what the page said about it. A library kind for the reason an +# import is: the asset does not exist until it has run. The track rides in `payload`. +KIND_IMPORT_TRACK = "import_track" # Kinds that run on the generation queue rather than the enrichment one. A generation # is minutes of waiting on fal, and on the shared queue at concurrency 1 a video would @@ -105,6 +109,7 @@ KIND_HARVEST_ATTRIBUTION, KIND_ANALYZE_MUSIC_LIBRARY, KIND_IMPORT_URL, + KIND_IMPORT_TRACK, KIND_GENERATE, } ) diff --git a/backend/app/routers/imports.py b/backend/app/routers/imports.py index 5194b4c..66a1d38 100644 --- a/backend/app/routers/imports.py +++ b/backend/app/routers/imports.py @@ -8,18 +8,23 @@ from __future__ import annotations -from fastapi import APIRouter, Depends, status +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel, Field from sqlmodel import Session from app.auth import CurrentUser from app.database import get_session +from app.enrichment import page_import from app.enrichment.import_url import ImportRequest +from app.ingest.page_tracks import PageTrack from app.ingest.ytdlp import MAX_URL, normalise_url from app.jobs import enrichment as enrichment_jobs from app.jobs.registry import KINDS -from app.models.job import KIND_IMPORT_URL +from app.models.job import KIND_IMPORT_TRACK, KIND_IMPORT_URL from app.schemas import DataResponse +from app.safe_url import require_safe_external_url from app.schemas_jobs import ActivityJobRead from app.storage import capacity @@ -70,3 +75,125 @@ def start_url_import( session, user.id, KIND_IMPORT_URL, payload=request.encode(), asset_name=url[:255] ) return DataResponse(data=KINDS["enrichment"].to_activity(job)) + + +# ─── tracks from a music library's page ────────────────────────────────────── + + +class PageTrackBody(BaseModel): + """One track, as the preview found it and the person corrected it.""" + + audio_url: str = Field(min_length=1, max_length=MAX_URL) + title: str = Field(min_length=1, max_length=255) + creator: Optional[str] = Field(default=None, max_length=500) + publisher: Optional[str] = Field(default=None, max_length=500) + source_title: Optional[str] = Field(default=None, max_length=500) + description: Optional[str] = Field(default=None, max_length=20_000) + duration_seconds: Optional[float] = None + published_date: Optional[str] = Field(default=None, max_length=10) + license: Optional[str] = Field(default=None, max_length=500) + credit_line: Optional[str] = Field(default=None, max_length=2_000) + source_url: Optional[str] = Field(default=None, max_length=MAX_URL) + genres: List[str] = Field(default_factory=list, max_length=20) + moods: List[str] = Field(default_factory=list, max_length=20) + instruments: List[str] = Field(default_factory=list, max_length=20) + tags: List[str] = Field(default_factory=list, max_length=50) + bpm: Optional[float] = None + + +class PageTrackRead(PageTrackBody): + # The asset this library already holds for the track, so the review list can show it + # as imported rather than offering it twice. + existing_asset_id: Optional[str] = None + + +class PagePreviewRequest(BaseModel): + url: str = Field(min_length=1, max_length=MAX_URL) + # Off skips the AI fallback, which costs a little per page. + use_ai: bool = True + + +class PagePreview(BaseModel): + page_url: str + # structured | ai | links: how much the details can be trusted. + method: str + source: str + note: str = "" + tracks: List[PageTrackRead] + + +class PageImportRequest(BaseModel): + tracks: List[PageTrackBody] = Field(min_length=1, max_length=page_import.MAX_TRACKS_PER_IMPORT) + + +def _page_error(exc: Exception) -> HTTPException: + return HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "page_unreadable", "message": str(exc)}, + ) + + +@router.post("/import-page/preview", response_model=DataResponse[PagePreview]) +def preview_page_import( + payload: PagePreviewRequest, + user: CurrentUser, + session: Session = Depends(get_session), +) -> DataResponse[PagePreview]: + """Read a page and list the tracks on it, for the person to choose from. + + Synchronous, unlike a URL import: the answer is a list to review, not work to watch, + and reading a page (and, without structured data, asking the AI about it) takes + seconds rather than minutes. Nothing is downloaded here. + """ + url = normalise_url(payload.url) + try: + extraction = page_import.preview(session, user.id, url, use_ai=payload.use_ai) + except page_import.PageImportError as exc: + raise _page_error(exc) from exc + + existing = page_import.already_imported(session, user.id, extraction.tracks) + return DataResponse( + data=PagePreview( + page_url=extraction.page_url, + method=extraction.method, + source=extraction.source, + note=extraction.note, + tracks=[ + PageTrackRead(**t.to_dict(), existing_asset_id=existing.get(t.audio_url)) + for t in extraction.tracks + ], + ) + ) + + +@router.post( + "/import-page", + response_model=DataResponse[List[ActivityJobRead]], + status_code=status.HTTP_202_ACCEPTED, +) +def start_page_import( + payload: PageImportRequest, + user: CurrentUser, + session: Session = Depends(get_session), +) -> DataResponse[List[ActivityJobRead]]: + """Queue one download per chosen track. + + The tracks come back from the browser, so every audio URL is checked again here: the + review list is editable, and a URL in it is the user's, not the page's. + """ + capacity.require_room() + for track in payload.tracks: + require_safe_external_url(track.audio_url) + + jobs = [] + for body in payload.tracks: + track = PageTrack(**body.model_dump()) + job = enrichment_jobs.submit_library( + session, + user.id, + KIND_IMPORT_TRACK, + payload=page_import.encode_track(track), + asset_name=track.title, + ) + jobs.append(KINDS["enrichment"].to_activity(job)) + return DataResponse(data=jobs) diff --git a/backend/tests/fixtures/pages/incompetech_music.html b/backend/tests/fixtures/pages/incompetech_music.html new file mode 100644 index 0000000..c483e56 --- /dev/null +++ b/backend/tests/fixtures/pages/incompetech_music.html @@ -0,0 +1,17 @@ + + +
+ diff --git a/backend/tests/fixtures/pages/incompetech_pieces.json b/backend/tests/fixtures/pages/incompetech_pieces.json new file mode 100644 index 0000000..21a5257 --- /dev/null +++ b/backend/tests/fixtures/pages/incompetech_pieces.json @@ -0,0 +1,10 @@ +[ + {"title": "Cipher", "bpm": "128", "genre": "7", "length": "0:3:05", + "description": "Dark pulsing electronica.", "instruments": "Synths, Drums", + "feel": "Dark, Driving, Mysterious", "filename": "Cipher.mp3", "collection": "12", + "isrc": "USUAN1100301", "uploaded": "2011-04-04"}, + {"title": "Gymnopedie No. 1", "bpm": "", "genre": "24", "length": "0:3:06", + "description": "Satie.", "instruments": "Piano", "feel": "Calm, Relaxed", + "filename": "Gymnopedie No 1.mp3", "collection": "999", "uploaded": "2010"}, + {"title": "Broken entry with no file", "genre": "22"} +] diff --git a/backend/tests/fixtures/pages/mixkit_mood.html b/backend/tests/fixtures/pages/mixkit_mood.html new file mode 100644 index 0000000..4f0c889 --- /dev/null +++ b/backend/tests/fixtures/pages/mixkit_mood.html @@ -0,0 +1,32 @@ + + + + + +
+
+

I'm Still Waitin'

+

by Michael Ramir C.

+
+ Folk Pop + Warm + Acoustic Guitar + Road Trip +
+
+
+
+

Bigger than Texas

+
+ Folk + Relaxed + Percussion +
+
+ diff --git a/backend/tests/fixtures/pages/plain_links.html b/backend/tests/fixtures/pages/plain_links.html new file mode 100644 index 0000000..506bfb9 --- /dev/null +++ b/backend/tests/fixtures/pages/plain_links.html @@ -0,0 +1,10 @@ + + +

Some free music

+ + + diff --git a/backend/tests/test_page_import.py b/backend/tests/test_page_import.py new file mode 100644 index 0000000..8d118f8 --- /dev/null +++ b/backend/tests/test_page_import.py @@ -0,0 +1,360 @@ +"""Importing tracks from a music library's page: finding them, choosing, downloading. + +The pages are trimmed copies of real ones (`tests/fixtures/pages/`), kept to the +structure each adapter reads. The network is replaced at `ingest/web.py`: the page +fetch returns a fixture, and a track download writes a real MP3, so the import runs +through the same ingest an upload does. +""" + +import json +import shutil +from pathlib import Path + +import httpx +import pytest +from sqlmodel import select + +from app import music +from app.enrichment import page_import +from app.ingest import page_tracks, web +from app.models.asset import PROVENANCE_EMBEDDED, Asset +from app.models.job import KIND_IMPORT_TRACK, EnrichmentJob +from app.providers.base import Completion, Usage + +PAGES = Path(__file__).parent / "fixtures" / "pages" +SAMPLE_MP3 = Path(__file__).parent / "fixtures" / "sample_audio.mp3" + +MIXKIT_URL = "https://mixkit.co/free-stock-music/mood/carefree/?page=2" +INCOMPETECH_URL = "https://incompetech.com/music/royalty-free/music.html" +PLAIN_URL = "https://music.example.org/free/" + + +def _read(name: str) -> str: + return (PAGES / name).read_text(encoding="utf-8") + + +@pytest.fixture +def pages(monkeypatch): + """Serve the fixture pages in place of the network. Returns the URLs asked for.""" + served = { + MIXKIT_URL: _read("mixkit_mood.html"), + INCOMPETECH_URL: _read("incompetech_music.html"), + "https://incompetech.com/music/royalty-free/pieces.json": _read("incompetech_pieces.json"), + PLAIN_URL: _read("plain_links.html"), + } + asked: list[str] = [] + + def fetch_text(url, **_kwargs): + asked.append(url) + if url not in served: + raise web.WebFetchError("The site has no such page (HTTP 404)") + return url, served[url] + + monkeypatch.setattr(web, "fetch_text", fetch_text) + return asked + + +def _no_structure(url, **_kwargs): + raise AssertionError(f"unexpected fetch of {url}") + + +# ─── finding the tracks ────────────────────────────────────────────────────── + + +def test_mixkit_tracks_come_from_its_structured_data_and_cards(): + found = page_tracks.extract_structured(MIXKIT_URL, _read("mixkit_mood.html"), _no_structure) + + assert found.source == "mixkit" and found.method == page_tracks.METHOD_STRUCTURED + first, second = found.tracks + assert first.title == "I'm Still Waitin'" + assert first.audio_url == "https://assets.mixkit.co/music/910/910.mp3" + assert first.creator == "Michael Ramir C." + assert first.publisher == "Mixkit" + assert first.duration_seconds == 158 + assert first.published_date == "2020-01-28" + assert first.genres == ["Folk Pop"] + # The card's mood, plus the mood the listing itself is filtered to. + assert first.moods == ["Warm", "Carefree"] + assert first.instruments == ["Acoustic Guitar"] + assert first.tags == ["Road Trip"] + assert "Mixkit Stock Music Free License" in first.license + # No page of its own, so the file is the source, and what recognises a re-import. + assert first.source_url == first.audio_url + # Matched by MP3, not by position: the second card's links stay with the second track. + assert second.genres == ["Country", "Folk"] + assert second.instruments == ["Percussion"] + + +def test_incompetech_tracks_come_from_the_file_its_page_loads(): + asked = [] + + def fetch(url): + asked.append(url) + return url, _read("incompetech_pieces.json") + + found = page_tracks.extract_structured(INCOMPETECH_URL, _read("incompetech_music.html"), fetch) + + assert asked == ["https://incompetech.com/music/royalty-free/pieces.json"] + assert found.source == "incompetech" + cipher, gymnopedie = found.tracks # the entry with no file is skipped + assert cipher.audio_url == ( + "https://incompetech.com/music/royalty-free/mp3-royaltyfree/Cipher.mp3" + ) + assert cipher.creator == "Kevin MacLeod" + assert cipher.genres == ["Electronica"] + assert cipher.source_title == "Hard Electronic" + assert cipher.moods == ["Dark", "Driving", "Mysterious"] + assert cipher.instruments == ["Synths", "Drums"] + assert cipher.bpm == 128 + assert cipher.duration_seconds == 185 + assert cipher.description == "Dark pulsing electronica." + # The licence's own attribution wording, verbatim. + assert cipher.credit_line.startswith('"Cipher" Kevin MacLeod (incompetech.com) Licensed') + assert cipher.license == "CC BY 4.0" + # A space in the filename is encoded; "Unclassifiable" is no genre; no album. + assert gymnopedie.audio_url.endswith("/Gymnopedie%20No%201.mp3") + assert gymnopedie.genres == [] and gymnopedie.source_title is None + assert gymnopedie.bpm is None and gymnopedie.published_date == "2010" + + +def test_a_page_with_no_structure_falls_back_to_its_audio_links(): + found = page_tracks.extract_links(PLAIN_URL, _read("plain_links.html")) + assert [(t.title, t.audio_url) for t in found.tracks] == [ + ("Morning Light", "https://music.example.org/files/Morning%20Light.mp3"), + ("Night Drive", "https://cdn.example.org/tracks/night_drive.ogg"), + ("ambient bed", "https://music.example.org/files/ambient-bed.wav"), + ] + + +def test_page_text_keeps_link_targets_for_a_model(): + text = page_tracks.page_text(_read("plain_links.html")) + assert "Morning Light [link: /files/Morning%20Light.mp3]" in text + assert "[audio: /files/ambient-bed.wav]" in text + assert "") + + _serve(monkeypatch, handler) + with pytest.raises(web.WebFetchError, match="not an audio file"): + web.download_audio("https://music.example.org/download?id=4", tmp_path, "t") + assert list(tmp_path.iterdir()) == [] + + +def test_a_track_over_the_cap_is_refused_and_removed(monkeypatch, tmp_path): + def handler(request): + return httpx.Response(200, headers={"content-type": "audio/mpeg"}, content=b"x" * 2048) + + _serve(monkeypatch, handler) + with pytest.raises(web.WebFetchError, match="larger than"): + web.download_audio("https://music.example.org/t.mp3", tmp_path, "t", max_bytes=1024) + assert list(tmp_path.iterdir()) == [] + + +def test_a_track_downloads_with_its_type_from_the_response(monkeypatch, tmp_path): + def handler(request): + return httpx.Response(200, headers={"content-type": "audio/ogg"}, content=b"OggS...") + + _serve(monkeypatch, handler) + path = web.download_audio("https://music.example.org/stream?id=4", tmp_path, "t") + assert path.name == "t.ogg" and path.read_bytes() == b"OggS..." + + +def test_a_page_is_read_through_its_redirect(monkeypatch): + def handler(request): + if request.url.path == "/old": + return httpx.Response(301, headers={"location": "/new"}) + return httpx.Response(200, headers={"content-type": "text/html"}, text="

hi

") + + _serve(monkeypatch, handler) + final, text = web.fetch_text("https://music.example.org/old") + assert final == "https://music.example.org/new" and text == "

hi

" diff --git a/docs/page-import.md b/docs/page-import.md new file mode 100644 index 0000000..efe7f4d --- /dev/null +++ b/docs/page-import.md @@ -0,0 +1,58 @@ +# Import tracks from a page + +**Status:** built. Step 3 of making the library a soundtrack source for Gecko Video +Creator, after the music profile and analysis of the sound (`music-profile.md`). + +Paste a page from a music library into **Add → "Or paste a page of music tracks"** and +press **Find tracks**. GAM reads the page and lists its tracks with their details; +filter, tick the ones you want (up to 50 at a time) and press **Import**. Each becomes +its own background job: downloaded, catalogued like an upload, given what the page said +about it, then measured for whatever the page left out. + +## How a page is read + +Best source first. The review list says which one was used. + +| Source | When | What it gives | +|---|---|---| +| **incompetech** | `incompetech.com` pages | The page builds its list in the browser from `pieces.json` beside it; GAM reads that file. Title, genre, album, feel (→ moods), instruments, BPM, length, description, upload date, and the licence's exact attribution text as the credit line. Genre and album names come from the lookup tables in the page's own script. | +| **schema.org** | Any page with `MusicRecording` JSON-LD | Title, artist, genre, duration, date, licence, keywords. What sites publish for search engines, so no layout is guessed. | +| **mixkit** | `mixkit.co` (on top of its JSON-LD) | Each card's mood, instrument and tag links, matched to the track by its MP3 URL, plus the mood or instrument the listing itself is filtered to. | +| **AI** | A page with none of the above, when an AI provider is configured and "Ask the AI" is ticked | The model reads the page's text and may only name audio URLs found on the page; anything else it returns is dropped. Costs one completion per page, recorded in Usage. | +| **links** | Otherwise | Every audio file linked or embedded, titled by its link text. | + +A page that builds its list with JavaScript from somewhere other than incompetech's +known file has no audio URLs in it for any of these to find; the review says so. + +## What lands on the asset + +Everything from the page is stamped `embedded` in `field_provenance`, the standing an +imported video's site metadata has: a statement from the source nobody here checked. + +| Asset field | From | +|---|---| +| `name` | title | +| `creator`, `publisher`, `source_title` | artist, site, album | +| `license`, `credit_line` | as the page states them; incompetech's credit is its own attribution wording | +| `source_url` | the track's own page when there is one, else the audio file. Also what recognises a re-import: a track already in the library is marked "In library" and not downloaded twice. | +| `retrieved_at` | now | +| `music_genres`, `music_moods`, `music_instruments`, `music_bpm` | the page's | +| tags | the page's tags (Mixkit's "Road Trip", "Lifestyle") | + +Tempo, key and energy the page did not give are measured from the sound afterwards, by +the analysis every audio upload gets. + +## Downloading + +`backend/app/ingest/web.py`: an SSRF check on every redirect hop, uncompressed bodies +only, a 200 MB cap enforced while the bytes arrive, and a response that is not audio +(a login wall, an interstitial) refused rather than catalogued. Requests identify +themselves as GAM. One track per job on the shared enrichment queue, so downloads run one +at a time. + +## Licences + +Only ticked tracks are fetched, at the pace of one worker: one person importing what they +chose, which is the use these sites' licences describe. Mixkit's licence is its own (not +Creative Commons) and forbids redistributing the files; incompetech's tracks are CC BY 4.0 +and need the credit GVC's end credits carry. diff --git a/frontend/src/api/imports.ts b/frontend/src/api/imports.ts index 2492dc6..a3a875f 100644 --- a/frontend/src/api/imports.ts +++ b/frontend/src/api/imports.ts @@ -21,11 +21,65 @@ export interface UrlImportRequest { chapters_as_clips?: boolean } +/** One track a music library's page lists, as the preview found it. */ +export interface PageTrack { + audio_url: string + title: string + creator: string | null + publisher: string | null + source_title: string | null + description: string | null + duration_seconds: number | null + published_date: string | null + license: string | null + credit_line: string | null + source_url: string | null + genres: string[] + moods: string[] + instruments: string[] + tags: string[] + bpm: number | null + /** Set when the library already holds this track. */ + existing_asset_id?: string | null +} + +export interface PagePreview { + page_url: string + /** How the tracks were found, and so how far their details can be trusted: + * `structured` is the site's own data, `ai` a model's reading of the page, `links` + * the bare audio links. */ + method: 'structured' | 'ai' | 'links' + /** The adapter or source: "incompetech", "mixkit", "schema.org", "AI", "links". */ + source: string + note: string + tracks: PageTrack[] +} + +/** The most tracks one import may queue; the server refuses more. */ +export const MAX_TRACKS_PER_IMPORT = 50 + interface DataResponse { data: T } export const importsApi = { + /** Read a music library's page and list its tracks. Downloads nothing. */ + previewPage(url: string, useAi = true): Promise { + return client + .post>('/assets/import-page/preview', { + url, + use_ai: useAi, + }) + .then((r) => r.data.data) + }, + + /** Queue one download per chosen track. Returns the jobs, one per track. */ + importTracks(tracks: PageTrack[]): Promise { + return client + .post>('/assets/import-page', { tracks }) + .then((r) => r.data.data) + }, + fromUrl(request: UrlImportRequest): Promise { return client .post>('/assets/import', request) diff --git a/frontend/src/components/ActivityIndicator.tsx b/frontend/src/components/ActivityIndicator.tsx index c2f679a..201c0d2 100644 --- a/frontend/src/components/ActivityIndicator.tsx +++ b/frontend/src/components/ActivityIndicator.tsx @@ -20,6 +20,7 @@ const LABELS: Record = { autotag: 'Suggesting tags', extract_text: 'Reading text', import_url: 'Importing', + import_track: 'Importing a track', generate: 'Generating', harvest_attribution: 'Reading file metadata', analyze_music: 'Analysing music', diff --git a/frontend/src/components/AddPanel.tsx b/frontend/src/components/AddPanel.tsx index 908dec5..5334c4f 100644 --- a/frontend/src/components/AddPanel.tsx +++ b/frontend/src/components/AddPanel.tsx @@ -1,5 +1,6 @@ import { X } from 'lucide-react' import GenerateForm from '@/components/GenerateForm' +import PageImport from '@/components/PageImport' import UploadZone from '@/components/UploadZone' import UrlImport from '@/components/UrlImport' @@ -11,8 +12,8 @@ interface Props { } /** - * The three ways into the library — files, links, and M8's text → image generation — - * folded behind the library's + button. Generating from existing images starts from + * The ways into the library — files, links, pages of music tracks, and M8's text → image + * generation — folded behind the library's + button. Generating from existing images starts from * those images instead (the selection bar, an asset's Generate tab), since that is where * they are picked. * @@ -55,6 +56,7 @@ export default function AddPanel({ open, onClose }: Props) {
+
diff --git a/frontend/src/components/PageImport.test.tsx b/frontend/src/components/PageImport.test.tsx new file mode 100644 index 0000000..b4f0642 --- /dev/null +++ b/frontend/src/components/PageImport.test.tsx @@ -0,0 +1,154 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import PageImport from '@/components/PageImport' +import { importsApi, type PagePreview, type PageTrack } from '@/api/imports' +import type { ActivityJob } from '@/api/transcripts' +import { useActivityStore } from '@/stores/activity' + +function track(overrides: Partial = {}): PageTrack { + return { + audio_url: 'https://assets.mixkit.co/music/910/910.mp3', + title: "I'm Still Waitin'", + creator: 'Michael Ramir C.', + publisher: 'Mixkit', + source_title: null, + description: null, + duration_seconds: 158, + published_date: '2020-01-28', + license: 'Mixkit Stock Music Free License', + credit_line: null, + source_url: null, + genres: ['Folk Pop'], + moods: ['Warm', 'Carefree'], + instruments: ['Acoustic Guitar'], + tags: ['Road Trip'], + bpm: null, + existing_asset_id: null, + ...overrides, + } +} + +const texas = track({ + audio_url: 'https://assets.mixkit.co/music/1053/1053.mp3', + title: 'Bigger than Texas', + genres: ['Country'], + moods: ['Relaxed'], + instruments: ['Percussion'], + bpm: 96, +}) +const imported = track({ + audio_url: 'https://assets.mixkit.co/music/903/903.mp3', + title: 'Farewell to You', + existing_asset_id: 'a9', +}) + +function preview(tracks: PageTrack[], overrides: Partial = {}): PagePreview { + return { + page_url: 'https://mixkit.co/free-stock-music/mood/carefree/', + method: 'structured', + source: 'mixkit', + note: '', + tracks, + ...overrides, + } +} + +const refresh = vi.fn().mockResolvedValue(undefined) + +beforeEach(() => { + refresh.mockClear() + useActivityStore.setState({ refresh }) +}) + +afterEach(() => vi.restoreAllMocks()) + +async function find(tracks: PageTrack[], overrides: Partial = {}) { + const spy = vi + .spyOn(importsApi, 'previewPage') + .mockResolvedValue(preview(tracks, overrides)) + render() + await userEvent.type( + screen.getByLabelText('Page of tracks'), + 'https://mixkit.co/free-stock-music/mood/carefree/' + ) + await userEvent.click(screen.getByRole('button', { name: 'Find tracks' })) + return spy +} + +describe('PageImport', () => { + it('lists what the page holds, with where the details came from', async () => { + const spy = await find([track(), texas]) + + expect(spy).toHaveBeenCalledWith( + 'https://mixkit.co/free-stock-music/mood/carefree/', + true + ) + expect(await screen.findByText(/2 tracks found, from Mixkit/)).toBeInTheDocument() + expect( + screen.getByText('Folk Pop, Warm, Carefree, Acoustic Guitar') + ).toBeInTheDocument() + expect(screen.getByText('Country, Relaxed, Percussion · 96 BPM')).toBeInTheDocument() + }) + + it('imports only the ticked tracks', async () => { + const start = vi + .spyOn(importsApi, 'importTracks') + .mockResolvedValue([{ id: 'j1' } as ActivityJob]) + await find([track(), texas]) + + await userEvent.click( + await screen.findByRole('checkbox', { name: 'Bigger than Texas' }) + ) + await userEvent.click(screen.getByRole('button', { name: 'Import 1 track' })) + + expect(start).toHaveBeenCalledWith([texas]) + expect(await screen.findByText(/Queued 1 track\./)).toBeInTheDocument() + expect(refresh).toHaveBeenCalled() + }) + + it('filters by details, and ticks only what is shown', async () => { + const start = vi.spyOn(importsApi, 'importTracks').mockResolvedValue([]) + await find([track(), texas]) + + await userEvent.type( + await screen.findByLabelText('Filter tracks'), + 'relaxed percussion' + ) + expect(screen.queryByText("I'm Still Waitin'")).not.toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: 'Tick shown' })) + await userEvent.click(screen.getByRole('button', { name: 'Import 1 track' })) + + expect(start).toHaveBeenCalledWith([texas]) + }) + + it('will not offer a track the library already has', async () => { + await find([imported]) + expect( + await screen.findByRole('checkbox', { name: 'Farewell to You' }) + ).toBeDisabled() + expect(screen.getByText('In library')).toBeInTheDocument() + }) + + it('says when the details are only the bare links', async () => { + await find([track({ genres: [], moods: [], instruments: [] })], { + method: 'links', + source: 'links', + note: 'No AI provider is configured, so these are the bare audio links.', + }) + expect(await screen.findByText(/No AI provider is configured/)).toBeInTheDocument() + expect(screen.getByText(/titles only/)).toBeInTheDocument() + }) + + it('skips the AI when asked to', async () => { + const spy = vi.spyOn(importsApi, 'previewPage').mockResolvedValue(preview([])) + render() + await userEvent.click(screen.getByLabelText(/Ask the AI/)) + await userEvent.type( + screen.getByLabelText('Page of tracks'), + 'https://example.org/music' + ) + await userEvent.click(screen.getByRole('button', { name: 'Find tracks' })) + expect(spy).toHaveBeenCalledWith('https://example.org/music', false) + }) +}) diff --git a/frontend/src/components/PageImport.tsx b/frontend/src/components/PageImport.tsx new file mode 100644 index 0000000..5ab1111 --- /dev/null +++ b/frontend/src/components/PageImport.tsx @@ -0,0 +1,309 @@ +import { useMemo, useState, type FormEvent } from 'react' +import { ExternalLink, ListMusic, Loader2 } from 'lucide-react' +import { apiErrorMessage } from '@/api/client' +import { + importsApi, + MAX_TRACKS_PER_IMPORT, + type PagePreview, + type PageTrack, +} from '@/api/imports' +import { useActivityStore } from '@/stores/activity' + +/** + * Choose tracks from a music library's page — incompetech, Mixkit, anything that lists + * music — and import the chosen ones with what the page says about each. + * + * Two steps where `UrlImport` has one, because a page can list a thousand tracks and the + * person wants a dozen: Find reads the page and lists them, nothing is downloaded until + * Import, and only what is ticked is. The filter is what makes incompetech's whole + * catalogue (one page) usable; it searches the details too, so "dark piano" narrows to + * tracks whose moods and instruments say so. + * + * Details are corrected after import, in the asset's Music and Source tabs, rather than + * here: the review is for choosing, and an edit form per row would bury the list. + */ + +const SOURCE_LABELS: Record = { + incompetech: "incompetech's catalogue data", + mixkit: "Mixkit's track data", + 'schema.org': "the page's structured track data", + AI: 'the AI reading the page (check the details)', + links: 'the audio links on the page (titles only)', +} + +function clock(seconds: number | null): string { + if (!seconds) return '' + const total = Math.round(seconds) + return `${Math.floor(total / 60)}:${String(total % 60).padStart(2, '0')}` +} + +function summary(track: PageTrack): string { + const words = [...track.genres, ...track.moods, ...track.instruments] + const parts = [] + if (words.length) parts.push(words.join(', ')) + if (track.bpm) parts.push(`${Math.round(track.bpm)} BPM`) + return parts.join(' · ') +} + +function matches(track: PageTrack, query: string): boolean { + const haystack = [ + track.title, + track.creator ?? '', + track.source_title ?? '', + ...track.genres, + ...track.moods, + ...track.instruments, + ...track.tags, + ] + .join(' ') + .toLowerCase() + return query + .toLowerCase() + .split(/\s+/) + .filter(Boolean) + .every((word) => haystack.includes(word)) +} + +export default function PageImport() { + const refreshActivity = useActivityStore((s) => s.refresh) + + const [url, setUrl] = useState('') + const [useAi, setUseAi] = useState(true) + const [finding, setFinding] = useState(false) + const [importing, setImporting] = useState(false) + const [preview, setPreview] = useState(null) + const [filter, setFilter] = useState('') + const [chosen, setChosen] = useState>(new Set()) + const [error, setError] = useState(null) + const [queued, setQueued] = useState(null) + + const shown = useMemo( + () => (preview ? preview.tracks.filter((t) => matches(t, filter)) : []), + [preview, filter] + ) + const full = chosen.size >= MAX_TRACKS_PER_IMPORT + + const find = async (event: FormEvent) => { + event.preventDefault() + const link = url.trim() + if (!link || finding) return + setFinding(true) + setError(null) + setQueued(null) + try { + const found = await importsApi.previewPage(link, useAi) + setPreview(found) + setChosen(new Set()) + setFilter('') + } catch (err) { + setPreview(null) + setError(apiErrorMessage(err, 'Could not read that page')) + } finally { + setFinding(false) + } + } + + const toggle = (track: PageTrack) => + setChosen((previous) => { + const next = new Set(previous) + if (next.has(track.audio_url)) next.delete(track.audio_url) + else if (next.size < MAX_TRACKS_PER_IMPORT) next.add(track.audio_url) + return next + }) + + const chooseShown = () => + setChosen((previous) => { + const next = new Set(previous) + for (const track of shown) { + if (next.size >= MAX_TRACKS_PER_IMPORT) break + if (!track.existing_asset_id) next.add(track.audio_url) + } + return next + }) + + const startImport = async () => { + if (!preview || chosen.size === 0 || importing) return + setImporting(true) + setError(null) + try { + const tracks = preview.tracks.filter((t) => chosen.has(t.audio_url)) + const jobs = await importsApi.importTracks(tracks) + setQueued(jobs.length) + setChosen(new Set()) + void refreshActivity() + } catch (err) { + setError(apiErrorMessage(err, 'Could not start the import')) + } finally { + setImporting(false) + } + } + + return ( +
+
void find(e)} className="flex items-center gap-2"> +
+ ) +} From 1ca72cc31cb404fe382eb04633d985b86aabcea8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 11:29:56 +0000 Subject: [PATCH 2/2] Catch duplicate tracks on page import by file, batch and title The URL alone missed the same file under another address and files uploaded by hand. Now: - Checksum: after downloading, the file's SHA-256 is matched against the library. A copy already here is kept, the download discarded, and the page's details only fill its blanks (corrections stay); its source URL is recorded so the URL check catches it next time. - A track ticked twice in one import is one job. - The review list flags "Maybe in library" when the title (and artist, when both are known) matches an audio file already here: a different file of the same piece. Still tickable; "Tick shown" skips it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DCL4jm6ogZBBhnjEfBBaXU --- backend/app/enrichment/page_import.py | 87 +++++++++++++++++++-- backend/app/routers/imports.py | 17 +++- backend/tests/test_page_import.py | 76 ++++++++++++++++++ docs/page-import.md | 14 +++- frontend/src/api/imports.ts | 3 + frontend/src/components/PageImport.test.tsx | 14 ++++ frontend/src/components/PageImport.tsx | 24 +++++- 7 files changed, 224 insertions(+), 11 deletions(-) diff --git a/backend/app/enrichment/page_import.py b/backend/app/enrichment/page_import.py index fe238a1..cdf81d2 100644 --- a/backend/app/enrichment/page_import.py +++ b/backend/app/enrichment/page_import.py @@ -16,6 +16,7 @@ from __future__ import annotations +import hashlib import json import logging import re @@ -25,13 +26,13 @@ from typing import Any, Callable, Optional from urllib.parse import urljoin -from sqlmodel import Session, col, select +from sqlmodel import Session, col, func, select from app import music from app.clock import utcnow from app.ingest import page_tracks, web from app.ingest.embedded_metadata import normalise_partial_date -from app.ingest.filetypes import SOURCE_URL +from app.ingest.filetypes import SOURCE_URL, TYPE_AUDIO from app.ingest.page_tracks import PageExtraction, PageTrack from app.models.asset import Asset from app.models.job import EnrichmentJob @@ -121,6 +122,57 @@ def already_imported(session: Session, user_id: str, tracks: list[PageTrack]) -> return {keys[source_url]: asset_id for source_url, asset_id in rows} +def similar_in_library(session: Session, user_id: str, tracks: list[PageTrack]) -> dict[str, str]: + """{audio_url: asset id} for tracks whose title (and artist, when both are known) + match an audio file already in the library. + + A warning, not a refusal. The same piece downloaded from somewhere else is a + different file, so neither its URL nor its checksum can recognise it; its name can. + Names also collide by accident ("Intro"), which is why this only flags. + """ + titles = {t.title.strip().lower() for t in tracks if t.title.strip()} + if not titles: + return {} + rows = session.exec( + select(Asset.id, Asset.name, Asset.creator).where( + Asset.user_id == user_id, + Asset.asset_type == TYPE_AUDIO, + col(Asset.storage_key).is_not(None), + func.lower(Asset.name).in_(titles), + ) + ).all() + by_title: dict[str, list[tuple[str, Optional[str]]]] = {} + for asset_id, name, creator in rows: + by_title.setdefault(name.strip().lower(), []).append((asset_id, creator)) + + found: dict[str, str] = {} + for track in tracks: + for asset_id, creator in by_title.get(track.title.strip().lower(), []): + if track.creator and creator and track.creator.strip().lower() != creator.strip().lower(): + continue + found[track.audio_url] = asset_id + break + return found + + +def _same_file(session: Session, user_id: str, path: Path) -> Optional[Asset]: + """An asset in this library with exactly these bytes, if there is one. + + The checksum every stored file already carries is what catches a duplicate the URL + cannot: the same MP3 from a mirror, a different link to one file, or a track + uploaded by hand before it was imported. + """ + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return session.exec( + select(Asset).where( + Asset.user_id == user_id, Asset.checksum_sha256 == digest.hexdigest() + ) + ).first() + + # ─── the AI fallback ───────────────────────────────────────────────────────── @@ -275,6 +327,18 @@ def report(received: int, total: Optional[int]) -> None: except web.WebFetchError as exc: raise PageImportError(str(exc)) from exc + progress("Checking for a copy already here", 86, "") + duplicate = _same_file(session, job.user_id, path) + if duplicate is not None: + # The download is thrown away with the temp directory. What the page said + # still fills the copy's blanks, so a track uploaded bare gains its credits + # and its profile, and its source URL is recorded for next time. + apply_track_metadata(session, duplicate, track, fill_blanks_only=True) + return TrackImportResult( + f"Already in your library as “{duplicate.name}” (the same file)", + duplicate.id, + ) + progress("Saving", 88, "") stem = re.sub(r"[\\/]+", "-", track.title).strip() or "track" try: @@ -298,12 +362,16 @@ def report(received: int, total: Optional[int]) -> None: return TrackImportResult(detail, asset.id) -def apply_track_metadata(session: Session, asset: Asset, track: PageTrack) -> int: +def apply_track_metadata( + session: Session, asset: Asset, track: PageTrack, *, fill_blanks_only: bool = False +) -> int: """What the page said about the track, stamped `embedded`, and its tags attached. `embedded` for the same reason an imported video's site metadata is: a statement from - the source that nobody here checked. Written over whatever the file's own tags said: - the site describing its own track is the better authority. Returns the tag count. + the source that nobody here checked. On a fresh import it is written over whatever + the file's own tags said: the site describing its own track is the better authority. + On a duplicate (`fill_blanks_only`) the asset may hold a person's corrections, so it + only fills what is empty. Returns the tag count. """ changes: dict[str, Any] = { "source_url": track.source_url or track.audio_url, @@ -327,8 +395,15 @@ def apply_track_metadata(session: Session, asset: Asset, track: PageTrack) -> in if track.bpm is not None: changes["music_bpm"] = track.bpm + if fill_blanks_only: + changes = { + name: value + for name, value in changes.items() + if getattr(asset, name, None) in (None, "") and name != "retrieved_at" + } try: - asset_service.apply_embedded_metadata(session, asset, changes) + if changes: + asset_service.apply_embedded_metadata(session, asset, changes) except Exception: # noqa: BLE001 - the track is in; losing its metadata must not undo that logger.warning("Could not store page metadata for %s", asset.id, exc_info=True) diff --git a/backend/app/routers/imports.py b/backend/app/routers/imports.py index 66a1d38..9682d39 100644 --- a/backend/app/routers/imports.py +++ b/backend/app/routers/imports.py @@ -105,6 +105,9 @@ class PageTrackRead(PageTrackBody): # The asset this library already holds for the track, so the review list can show it # as imported rather than offering it twice. existing_asset_id: Optional[str] = None + # An audio file with the same title (and artist) is already here: probably the same + # piece from somewhere else. Flagged, not refused, since titles collide. + similar_asset_id: Optional[str] = None class PagePreviewRequest(BaseModel): @@ -152,6 +155,7 @@ def preview_page_import( raise _page_error(exc) from exc existing = page_import.already_imported(session, user.id, extraction.tracks) + similar = page_import.similar_in_library(session, user.id, extraction.tracks) return DataResponse( data=PagePreview( page_url=extraction.page_url, @@ -159,7 +163,13 @@ def preview_page_import( source=extraction.source, note=extraction.note, tracks=[ - PageTrackRead(**t.to_dict(), existing_asset_id=existing.get(t.audio_url)) + PageTrackRead( + **t.to_dict(), + existing_asset_id=existing.get(t.audio_url), + similar_asset_id=( + None if t.audio_url in existing else similar.get(t.audio_url) + ), + ) for t in extraction.tracks ], ) @@ -185,8 +195,11 @@ def start_page_import( for track in payload.tracks: require_safe_external_url(track.audio_url) + # The same track ticked twice (a page that lists it under two moods) is one job. + unique = list({body.audio_url: body for body in payload.tracks}.values()) + jobs = [] - for body in payload.tracks: + for body in unique: track = PageTrack(**body.model_dump()) job = enrichment_jobs.submit_library( session, diff --git a/backend/tests/test_page_import.py b/backend/tests/test_page_import.py index 8d118f8..b147922 100644 --- a/backend/tests/test_page_import.py +++ b/backend/tests/test_page_import.py @@ -275,6 +275,82 @@ def test_a_track_already_in_the_library_is_not_downloaded_twice( assert session.get(EnrichmentJob, second["id"]).detail == "Already in your library" +def test_the_same_file_from_another_url_is_not_imported_twice( + library, session, downloads, monkeypatch +): + """A mirror, or a second link to one file: the URL differs, the bytes do not.""" + first = {"audio_url": "https://a.example.org/song.mp3", "title": "Song"} + second = { + "audio_url": "https://mirror.example.net/other-name.mp3", + "title": "Song (mirror)", + "creator": "Jane Doe", + "moods": ["Calm"], + } + job = library.post("/api/assets/import-page", json={"tracks": [first]}).json()["data"][0] + _run_job(job["id"], session, monkeypatch) + job = library.post("/api/assets/import-page", json={"tracks": [second]}).json()["data"][0] + _run_job(job["id"], session, monkeypatch) + + [asset] = session.exec(select(Asset).where(Asset.asset_type == "audio")).all() + finished = session.get(EnrichmentJob, job["id"]) + assert finished.status == "done" + assert finished.detail == "Already in your library as \u201cSong\u201d (the same file)" + # The page's details fill the copy's blanks, and the job points at it. + assert asset.creator == "Jane Doe" and music.decode_terms(asset.music_moods) == ["Calm"] + assert library.get("/api/activity").json()["data"][0]["result_asset_id"] == asset.id + + +def test_a_file_uploaded_by_hand_is_recognised_and_keeps_its_corrections( + library, session, downloads, monkeypatch +): + files = [("files", ("mine.mp3", SAMPLE_MP3.read_bytes(), "audio/mpeg"))] + mine = library.post("/api/assets", files=files).json()["created"][0] + library.patch(f"/api/assets/{mine['id']}", json={"creator": "Who I say it is"}) + + track = {"audio_url": "https://a.example.org/x.mp3", "title": "X", "creator": "The site"} + job = library.post("/api/assets/import-page", json={"tracks": [track]}).json()["data"][0] + _run_job(job["id"], session, monkeypatch) + + assert len(session.exec(select(Asset).where(Asset.asset_type == "audio")).all()) == 1 + stored = session.get(Asset, mine["id"]) + assert stored.creator == "Who I say it is" + # A blank is still filled, so the next import recognises it by URL alone. + assert stored.source_url == "https://a.example.org/x.mp3" + + +def test_the_same_track_ticked_twice_is_one_job(library, session): + track = {"audio_url": "https://a.example.org/song.mp3", "title": "Song"} + jobs = library.post("/api/assets/import-page", json={"tracks": [track, track]}).json()["data"] + assert len(jobs) == 1 + + +def test_a_track_with_a_familiar_title_is_flagged_in_the_review(library, session, pages, monkeypatch): + """Same piece, different file (downloaded elsewhere): only its name can tell.""" + files = [("files", ("x.mp3", SAMPLE_MP3.read_bytes(), "audio/mpeg"))] + mine = library.post("/api/assets", files=files).json()["created"][0] + library.patch(f"/api/assets/{mine['id']}", json={"name": "Bigger Than Texas"}) + + tracks = library.post( + "/api/assets/import-page/preview", json={"url": MIXKIT_URL} + ).json()["data"]["tracks"] + by_title = {t["title"]: t for t in tracks} + assert by_title["Bigger than Texas"]["similar_asset_id"] == mine["id"] + assert by_title["Bigger than Texas"]["existing_asset_id"] is None + assert by_title["I'm Still Waitin'"]["similar_asset_id"] is None + + +def test_a_familiar_title_by_another_artist_is_not_flagged(library, session, pages): + files = [("files", ("x.mp3", SAMPLE_MP3.read_bytes(), "audio/mpeg"))] + mine = library.post("/api/assets", files=files).json()["created"][0] + library.patch( + f"/api/assets/{mine['id']}", json={"name": "Bigger than Texas", "creator": "Someone Else"} + ) + tracks = library.post( + "/api/assets/import-page/preview", json={"url": MIXKIT_URL} + ).json()["data"]["tracks"] + assert all(t["similar_asset_id"] is None for t in tracks) + + def test_a_date_from_the_browser_is_normalised_not_trusted(library, session, downloads, monkeypatch): track = {"audio_url": "https://music.example.org/a.mp3", "title": "A", "published_date": "summer"} job = library.post("/api/assets/import-page", json={"tracks": [track]}).json()["data"][0] diff --git a/docs/page-import.md b/docs/page-import.md index efe7f4d..316cb07 100644 --- a/docs/page-import.md +++ b/docs/page-import.md @@ -34,7 +34,7 @@ imported video's site metadata has: a statement from the source nobody here chec | `name` | title | | `creator`, `publisher`, `source_title` | artist, site, album | | `license`, `credit_line` | as the page states them; incompetech's credit is its own attribution wording | -| `source_url` | the track's own page when there is one, else the audio file. Also what recognises a re-import: a track already in the library is marked "In library" and not downloaded twice. | +| `source_url` | the track's own page when there is one, else the audio file. Also what the URL duplicate check matches (below). | | `retrieved_at` | now | | `music_genres`, `music_moods`, `music_instruments`, `music_bpm` | the page's | | tags | the page's tags (Mixkit's "Road Trip", "Lifestyle") | @@ -42,6 +42,18 @@ imported video's site metadata has: a statement from the source nobody here chec Tempo, key and energy the page did not give are measured from the sound afterwards, by the analysis every audio upload gets. +## Duplicates + +Caught three ways, because each misses something the others catch: + +| Check | When | Catches | Then | +|---|---|---|---| +| **URL** | In the review list, and again in the job | A track imported from this address before | Shown "In library", cannot be ticked; a job finds it and points at the copy | +| **Checksum** | In the job, after downloading | The same file at another URL (a mirror, a second link), or one uploaded by hand | The download is discarded; the page's details only fill the copy's blanks (a person's corrections stay), and its source URL is recorded so the URL check catches it next time | +| **Title and artist** | In the review list | The same piece downloaded from somewhere else, a different file | Shown "Maybe in library". Still tickable, since titles collide ("Intro"), but "Tick shown" skips it | + +A track ticked twice in one import is one job. + ## Downloading `backend/app/ingest/web.py`: an SSRF check on every redirect hop, uncompressed bodies diff --git a/frontend/src/api/imports.ts b/frontend/src/api/imports.ts index a3a875f..553240d 100644 --- a/frontend/src/api/imports.ts +++ b/frontend/src/api/imports.ts @@ -41,6 +41,9 @@ export interface PageTrack { bpm: number | null /** Set when the library already holds this track. */ existing_asset_id?: string | null + /** Set when an audio file with the same title (and artist) is already in the library: + * probably the same piece from elsewhere. A warning; it can still be imported. */ + similar_asset_id?: string | null } export interface PagePreview { diff --git a/frontend/src/components/PageImport.test.tsx b/frontend/src/components/PageImport.test.tsx index b4f0642..d5bfb39 100644 --- a/frontend/src/components/PageImport.test.tsx +++ b/frontend/src/components/PageImport.test.tsx @@ -130,6 +130,20 @@ describe('PageImport', () => { expect(screen.getByText('In library')).toBeInTheDocument() }) + it('flags a familiar title but still lets it be chosen, one at a time', async () => { + const start = vi.spyOn(importsApi, 'importTracks').mockResolvedValue([]) + const familiar = { ...texas, similar_asset_id: 'a5' } + await find([track(), familiar]) + + expect(await screen.findByText('Maybe in library')).toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: 'Tick shown' })) + await userEvent.click(screen.getByRole('button', { name: 'Import 1 track' })) + expect(start).toHaveBeenLastCalledWith([track()]) + + await userEvent.click(screen.getByRole('checkbox', { name: 'Bigger than Texas' })) + expect(screen.getByRole('checkbox', { name: 'Bigger than Texas' })).toBeChecked() + }) + it('says when the details are only the bare links', async () => { await find([track({ genres: [], moods: [], instruments: [] })], { method: 'links', diff --git a/frontend/src/components/PageImport.tsx b/frontend/src/components/PageImport.tsx index 5ab1111..b9e17dc 100644 --- a/frontend/src/components/PageImport.tsx +++ b/frontend/src/components/PageImport.tsx @@ -19,6 +19,12 @@ import { useActivityStore } from '@/stores/activity' * catalogue (one page) usable; it searches the details too, so "dark piano" narrows to * tracks whose moods and instruments say so. * + * Duplicates are caught three ways. "In library" is a track already imported from this + * URL and cannot be ticked. "Maybe in library" is a title (and artist) the library + * already has, probably the same piece from elsewhere; it can still be ticked. And the + * import job itself compares the downloaded file's checksum with the library, so the + * same file under another URL, or one uploaded by hand, is never stored twice. + * * Details are corrected after import, in the asset's Music and Source tabs, rather than * here: the review is for choosing, and an edit form per row would bury the list. */ @@ -116,7 +122,9 @@ export default function PageImport() { const next = new Set(previous) for (const track of shown) { if (next.size >= MAX_TRACKS_PER_IMPORT) break - if (!track.existing_asset_id) next.add(track.audio_url) + // Skips the likely duplicates too: ticking one of those is a choice to make one + // at a time, not in bulk. + if (!track.existing_asset_id && !track.similar_asset_id) next.add(track.audio_url) } return next }) @@ -217,6 +225,7 @@ export default function PageImport() { > {shown.map((track) => { const inLibrary = Boolean(track.existing_asset_id) + const maybeInLibrary = !inLibrary && Boolean(track.similar_asset_id) const ticked = chosen.has(track.audio_url) return (
  • @@ -244,7 +253,18 @@ export default function PageImport() { )} - {inLibrary ? 'In library' : clock(track.duration_seconds)} + {inLibrary ? ( + 'In library' + ) : maybeInLibrary ? ( + + Maybe in library + + ) : ( + clock(track.duration_seconds) + )} {summary(track) && (