diff --git a/backend/app/enrichment/page_import.py b/backend/app/enrichment/page_import.py
new file mode 100644
index 0000000..cdf81d2
--- /dev/null
+++ b/backend/app/enrichment/page_import.py
@@ -0,0 +1,420 @@
+"""Importing tracks from a music library's web page: preview, then one job per track.
+
+Two steps, because a page can list a thousand tracks and the person wants twelve. The
+preview reads the page and returns what it found, synchronously, for a review list; the
+import queues one job per ticked track, as a playlist import does, so a failure on track
+eight costs track eight and each one is cancellable.
+
+What is found is described in `ingest/page_tracks.py`. This module adds the parts that
+need a session: the AI fallback (a provider, and usage to record), recognising what is
+already in the library, and writing what the page said onto the imported asset.
+
+Only ticked tracks are downloaded, one at a time on the enrichment queue. A music
+library's licence is about using its tracks, and some forbid bulk downloading; one person
+importing what they chose, at the pace of one worker, is the use those licences describe.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+import tempfile
+from dataclasses import dataclass, fields
+from pathlib import Path
+from typing import Any, Callable, Optional
+from urllib.parse import urljoin
+
+from sqlmodel import Session, col, func, select
+
+from app import music
+from app.clock import utcnow
+from app.ingest import page_tracks, web
+from app.ingest.embedded_metadata import normalise_partial_date
+from app.ingest.filetypes import SOURCE_URL, TYPE_AUDIO
+from app.ingest.page_tracks import PageExtraction, PageTrack
+from app.models.asset import Asset
+from app.models.job import EnrichmentJob
+from app.providers import build_provider
+from app.providers.base import ProviderError
+from app.services import assets as asset_service
+from app.services import tags as tag_service
+from app.storage import build_storage, capacity
+from app.usage import events as usage_events
+
+logger = logging.getLogger(__name__)
+
+Progress = Callable[..., None]
+
+# One import request's worth. A review list of fifty ticked tracks is already an
+# afternoon of listening; more at once is a bulk download, not a choice.
+MAX_TRACKS_PER_IMPORT = 50
+
+# The longest page the model is shown, in characters, and the most tracks it may return.
+AI_PAGE_CHARS = 60_000
+AI_MAX_TRACKS = 100
+
+AI_SYSTEM = (
+ "You read a web page from a music library and list the music tracks on it as JSON. "
+ "You only report what the page states. You never invent a URL."
+)
+
+
+class PageImportError(Exception):
+ """The page could not be read, or an import job could not run. Worded for the user."""
+
+
+# ─── preview ─────────────────────────────────────────────────────────────────
+
+
+def preview(session: Session, user_id: str, url: str, *, use_ai: bool = True) -> PageExtraction:
+ """Read a page and return the tracks on it, best source first."""
+ try:
+ final_url, page = web.fetch_text(url)
+ extraction = page_tracks.extract_structured(final_url, page, web.fetch_text)
+ except web.WebFetchError as exc:
+ raise PageImportError(str(exc)) from exc
+
+ if extraction is None and use_ai:
+ provider = build_provider(session, user_id)
+ if provider is not None:
+ extraction = _ask_ai(session, user_id, provider, final_url, page)
+
+ if extraction is None or not extraction.tracks:
+ links = page_tracks.extract_links(final_url, page)
+ if links.tracks:
+ if extraction is not None and extraction.method == page_tracks.METHOD_AI:
+ links.note = "The AI found no tracks it could tie to a file, so these are the bare links."
+ elif use_ai:
+ links.note = (
+ "No AI provider is configured, so these are the bare audio links. "
+ "Add one in Settings to read titles and details from the page."
+ )
+ extraction = links
+ else:
+ extraction = PageExtraction(
+ final_url,
+ page_tracks.METHOD_LINKS,
+ [],
+ "links",
+ note=(
+ "No audio files were found on this page. If the site builds its track "
+ "list with JavaScript after loading, the files are not in the page itself."
+ ),
+ )
+ return extraction
+
+
+def already_imported(session: Session, user_id: str, tracks: list[PageTrack]) -> dict[str, str]:
+ """{audio_url: asset id} for the tracks this library already has."""
+ keys: dict[str, str] = {}
+ for track in tracks:
+ keys[track.source_url or track.audio_url] = track.audio_url
+ keys[track.audio_url] = track.audio_url
+ if not keys:
+ return {}
+ rows = session.exec(
+ select(Asset.source_url, Asset.id).where(
+ Asset.user_id == user_id, col(Asset.source_url).in_(list(keys))
+ )
+ ).all()
+ return {keys[source_url]: asset_id for source_url, asset_id in rows}
+
+
+def similar_in_library(session: Session, user_id: str, tracks: list[PageTrack]) -> dict[str, str]:
+ """{audio_url: asset id} for tracks whose title (and artist, when both are known)
+ match an audio file already in the library.
+
+ A warning, not a refusal. The same piece downloaded from somewhere else is a
+ different file, so neither its URL nor its checksum can recognise it; its name can.
+ Names also collide by accident ("Intro"), which is why this only flags.
+ """
+ titles = {t.title.strip().lower() for t in tracks if t.title.strip()}
+ if not titles:
+ return {}
+ rows = session.exec(
+ select(Asset.id, Asset.name, Asset.creator).where(
+ Asset.user_id == user_id,
+ Asset.asset_type == TYPE_AUDIO,
+ col(Asset.storage_key).is_not(None),
+ func.lower(Asset.name).in_(titles),
+ )
+ ).all()
+ by_title: dict[str, list[tuple[str, Optional[str]]]] = {}
+ for asset_id, name, creator in rows:
+ by_title.setdefault(name.strip().lower(), []).append((asset_id, creator))
+
+ found: dict[str, str] = {}
+ for track in tracks:
+ for asset_id, creator in by_title.get(track.title.strip().lower(), []):
+ if track.creator and creator and track.creator.strip().lower() != creator.strip().lower():
+ continue
+ found[track.audio_url] = asset_id
+ break
+ return found
+
+
+def _same_file(session: Session, user_id: str, path: Path) -> Optional[Asset]:
+ """An asset in this library with exactly these bytes, if there is one.
+
+ The checksum every stored file already carries is what catches a duplicate the URL
+ cannot: the same MP3 from a mirror, a different link to one file, or a track
+ uploaded by hand before it was imported.
+ """
+ digest = hashlib.sha256()
+ with open(path, "rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return session.exec(
+ select(Asset).where(
+ Asset.user_id == user_id, Asset.checksum_sha256 == digest.hexdigest()
+ )
+ ).first()
+
+
+# ─── the AI fallback ─────────────────────────────────────────────────────────
+
+
+def _ask_ai(session: Session, user_id: str, provider, page_url: str, page: str) -> Optional[PageExtraction]:
+ """Ask the provider to list the tracks, holding it to the audio URLs the page has.
+
+ The grounding rule is the point. A model asked for download links will produce
+ plausible ones, and a plausible URL that is not on the page is either a 404 or a file
+ nobody chose. Every URL it returns is checked against the page's own audio URLs, and
+ an entry whose URL is not among them is dropped.
+ """
+ candidates = page_tracks.candidate_audio_urls(page_url, page)
+ if not candidates:
+ return None
+
+ prompt = (
+ f"Page URL: {page_url}\n\n"
+ "Audio file URLs found on the page (the only URLs you may use):\n"
+ + "\n".join(candidates[:400])
+ + "\n\nPage text, with link targets written as [link: …]:\n"
+ + page_tracks.page_text(page, AI_PAGE_CHARS)
+ + "\n\nList each music track on this page as a JSON array, one object per track, "
+ "with these keys, leaving out any the page does not state:\n"
+ ' "audio_url" (exactly one of the URLs above), "title", "artist", "album", '
+ '"description", "duration" (seconds), "bpm", "genres" (list), "moods" (list), '
+ '"instruments" (list), "tags" (list), "license", "credit" (attribution text the '
+ 'page asks for), "page_url" (the track\'s own page, if linked), "published" '
+ "(YYYY, YYYY-MM or YYYY-MM-DD).\n"
+ "Answer with the JSON array only."
+ )
+ try:
+ completion = provider.complete(prompt, system=AI_SYSTEM)
+ except ProviderError as exc:
+ raise PageImportError(f"The AI provider could not read the page: {exc}") from exc
+ usage_events.record_completion(session, None, user_id, completion)
+
+ allowed = set(candidates)
+ tracks = [t for t in (_track_from_ai(page_url, item) for item in _json_array(completion.text)) if t]
+ tracks = [t for t in tracks if t.audio_url in allowed]
+ unique = list({t.audio_url: t for t in tracks}.values())[:AI_MAX_TRACKS]
+ return PageExtraction(page_url, page_tracks.METHOD_AI, unique, "AI")
+
+
+def _json_array(text: str) -> list[Any]:
+ """The first JSON array in a reply, tolerating a code fence or a sentence around it."""
+ start, end = text.find("["), text.rfind("]")
+ if start < 0 or end <= start:
+ return []
+ try:
+ value = json.loads(text[start : end + 1])
+ except ValueError:
+ return []
+ return value if isinstance(value, list) else []
+
+
+def _track_from_ai(page_url: str, item: Any) -> Optional[PageTrack]:
+ if not isinstance(item, dict):
+ return None
+ audio_url = item.get("audio_url")
+ title = _short(item.get("title"), 255)
+ if not isinstance(audio_url, str) or not title:
+ return None
+ audio_url = urljoin(page_url, audio_url.strip())
+ track_page = item.get("page_url")
+ duration = item.get("duration")
+ return PageTrack(
+ audio_url=audio_url,
+ title=title,
+ creator=_short(item.get("artist"), 500),
+ source_title=_short(item.get("album"), 500),
+ description=_short(item.get("description"), 2000),
+ duration_seconds=float(duration) if isinstance(duration, (int, float)) and duration > 0 else None,
+ published_date=_partial_date(item.get("published")),
+ license=_short(item.get("license"), 500),
+ credit_line=_short(item.get("credit"), 2000),
+ source_url=urljoin(page_url, track_page) if isinstance(track_page, str) and track_page else audio_url,
+ genres=_list(item.get("genres")),
+ moods=_list(item.get("moods")),
+ instruments=_list(item.get("instruments")),
+ tags=_list(item.get("tags")),
+ bpm=music.parse_bpm(item.get("bpm")),
+ )
+
+
+def _short(value: Any, limit: int) -> Optional[str]:
+ if not isinstance(value, str):
+ return None
+ text = " ".join(value.split())
+ return text[:limit] or None
+
+
+def _list(value: Any) -> list[str]:
+ if isinstance(value, list):
+ return music.clean_terms(v for v in value if isinstance(v, str))
+ if isinstance(value, str):
+ return music.split_terms(value)
+ return []
+
+
+def _partial_date(value: Any) -> Optional[str]:
+ return normalise_partial_date(value) if isinstance(value, str) else None
+
+
+# ─── the import job ──────────────────────────────────────────────────────────
+
+
+def encode_track(track: PageTrack) -> str:
+ return json.dumps(track.to_dict(), sort_keys=True)
+
+
+def decode_track(payload: Optional[str]) -> PageTrack:
+ try:
+ data = json.loads(payload or "{}")
+ except ValueError:
+ data = {}
+ if not isinstance(data, dict):
+ data = {}
+ data = data.get("track", data)
+ known = {f.name for f in fields(PageTrack)}
+ values = {k: v for k, v in data.items() if k in known}
+ if not isinstance(values.get("audio_url"), str) or not isinstance(values.get("title"), str):
+ raise PageImportError("This import job has no track to fetch")
+ return PageTrack(**values)
+
+
+@dataclass
+class TrackImportResult:
+ detail: str
+ created_asset_id: Optional[str] = None
+
+
+def run_track(session: Session, job: EnrichmentJob, progress: Progress) -> TrackImportResult:
+ """Download one track and catalogue it with what its page said."""
+ track = decode_track(job.payload)
+
+ existing = already_imported(session, job.user_id, [track])
+ if track.audio_url in existing:
+ return TrackImportResult("Already in your library", existing[track.audio_url])
+
+ if not capacity.has_room():
+ raise PageImportError(capacity.DISK_FULL_MESSAGE)
+
+ with tempfile.TemporaryDirectory(prefix="gam-track-") as workdir:
+ progress("Downloading", 5, "")
+
+ def report(received: int, total: Optional[int]) -> None:
+ if total:
+ progress("Downloading", 5 + int(80 * received / total), "")
+
+ try:
+ path = web.download_audio(track.audio_url, Path(workdir), "track", on_progress=report)
+ except web.WebFetchError as exc:
+ raise PageImportError(str(exc)) from exc
+
+ progress("Checking for a copy already here", 86, "")
+ duplicate = _same_file(session, job.user_id, path)
+ if duplicate is not None:
+ # The download is thrown away with the temp directory. What the page said
+ # still fills the copy's blanks, so a track uploaded bare gains its credits
+ # and its profile, and its source URL is recorded for next time.
+ apply_track_metadata(session, duplicate, track, fill_blanks_only=True)
+ return TrackImportResult(
+ f"Already in your library as “{duplicate.name}” (the same file)",
+ duplicate.id,
+ )
+
+ progress("Saving", 88, "")
+ stem = re.sub(r"[\\/]+", "-", track.title).strip() or "track"
+ try:
+ asset = asset_service.ingest_file(
+ session,
+ build_storage(),
+ user_id=job.user_id,
+ source_path=path,
+ filename=f"{stem[:200]}{path.suffix.lower()}",
+ name=track.title,
+ source=SOURCE_URL,
+ )
+ except asset_service.UnsupportedFile as exc:
+ raise PageImportError("The site sent a file the library does not accept") from exc
+
+ progress("Recording what the page said", 95, "")
+ tag_count = apply_track_metadata(session, asset, track)
+ detail = "Imported"
+ if tag_count:
+ detail += f" · {tag_count} tag{'' if tag_count == 1 else 's'}"
+ return TrackImportResult(detail, asset.id)
+
+
+def apply_track_metadata(
+ session: Session, asset: Asset, track: PageTrack, *, fill_blanks_only: bool = False
+) -> int:
+ """What the page said about the track, stamped `embedded`, and its tags attached.
+
+ `embedded` for the same reason an imported video's site metadata is: a statement from
+ the source that nobody here checked. On a fresh import it is written over whatever
+ the file's own tags said: the site describing its own track is the better authority.
+ On a duplicate (`fill_blanks_only`) the asset may hold a person's corrections, so it
+ only fills what is empty. Returns the tag count.
+ """
+ changes: dict[str, Any] = {
+ "source_url": track.source_url or track.audio_url,
+ "retrieved_at": utcnow(),
+ }
+ for name in ("creator", "publisher", "source_title", "license", "credit_line", "description"):
+ value = getattr(track, name)
+ if value:
+ changes[name] = value
+ # Normalised here, not trusted: the track came back from the browser, and the date
+ # filters compare this column as text, which only works for ISO partial dates.
+ published = normalise_partial_date(track.published_date) if track.published_date else None
+ if published:
+ changes["published_date"] = published
+ if track.genres:
+ changes["music_genres"] = music.encode_terms(track.genres)
+ if track.moods:
+ changes["music_moods"] = music.encode_terms(track.moods)
+ if track.instruments:
+ changes["music_instruments"] = music.encode_terms(track.instruments)
+ if track.bpm is not None:
+ changes["music_bpm"] = track.bpm
+
+ if fill_blanks_only:
+ changes = {
+ name: value
+ for name, value in changes.items()
+ if getattr(asset, name, None) in (None, "") and name != "retrieved_at"
+ }
+ try:
+ if changes:
+ asset_service.apply_embedded_metadata(session, asset, changes)
+ except Exception: # noqa: BLE001 - the track is in; losing its metadata must not undo that
+ logger.warning("Could not store page metadata for %s", asset.id, exc_info=True)
+
+ attached = 0
+ try:
+ for name in track.tags:
+ tag = tag_service.get_or_create(session, asset.user_id, name)
+ if tag is not None and tag_service.attach(session, asset.id, tag.id):
+ attached += 1
+ if attached:
+ asset_service.reindex_ids(session, [asset.id])
+ except Exception: # noqa: BLE001 - same
+ logger.warning("Could not attach page tags to %s", asset.id, exc_info=True)
+ return attached
diff --git a/backend/app/ingest/page_tracks.py b/backend/app/ingest/page_tracks.py
new file mode 100644
index 0000000..d871154
--- /dev/null
+++ b/backend/app/ingest/page_tracks.py
@@ -0,0 +1,476 @@
+"""Finding the tracks on a web page, with what the page says about each.
+
+For music libraries that list their tracks on a page: a mood, a genre, a search. Three
+ways in, tried in order of how much they can be trusted:
+
+1. **What the site publishes for machines.** schema.org `MusicRecording` data (JSON-LD),
+ which Mixkit and many others put in the page for search engines, and incompetech's
+ `pieces.json`, the file its own page loads. Both are the site's structured statement
+ of what each track is, so nothing has to be guessed from layout.
+2. **The AI provider**, for a page with neither (`enrichment/page_import.py`). It reads
+ the page's text and is only allowed to name audio URLs that actually appear in it.
+3. **Bare audio links**, when there is no AI provider: every `.mp3` the page links to,
+ named by its link text. Little metadata, but something to tick.
+
+Pure apart from `fetch_text`, which is passed in, so the adapters are tested on saved
+pages with no network.
+
+A site-specific adapter exists only where the structured data leaves out something worth
+having: Mixkit's JSON-LD has one genre per track, while its cards also link each track's
+moods, instruments and tags. Each adapter names its site, and a site that changes its
+layout breaks that adapter alone; the JSON-LD underneath keeps working.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import json
+import re
+from dataclasses import asdict, dataclass, field
+from typing import Any, Callable, Iterable, Optional
+from urllib.parse import quote, unquote, urljoin, urlsplit
+
+from app import music
+from app.ingest.embedded_metadata import normalise_partial_date
+from app.ingest.filetypes import AUDIO_EXTENSIONS, extension_of
+
+METHOD_STRUCTURED = "structured"
+METHOD_AI = "ai"
+METHOD_LINKS = "links"
+
+# How many a single review list shows. Incompetech's catalogue page lists all of it,
+# well over a thousand; the review is filtered client-side, and this only stops a
+# pathological page from producing an unbounded response.
+MAX_TRACKS = 2500
+MAX_TEXT = 500
+MAX_DESCRIPTION = 2000
+
+FetchText = Callable[[str], tuple[str, str]]
+
+
+@dataclass
+class PageTrack:
+ """One track as a page describes it. Every field but the URL and title optional."""
+
+ audio_url: str
+ title: str
+ creator: Optional[str] = None
+ publisher: Optional[str] = None
+ source_title: Optional[str] = None
+ description: Optional[str] = None
+ duration_seconds: Optional[float] = None
+ published_date: Optional[str] = None
+ license: Optional[str] = None
+ credit_line: Optional[str] = None
+ # Where to cite, and how a re-import is recognised. The track's own page when the
+ # site has one, else the audio file itself.
+ source_url: Optional[str] = None
+ genres: list[str] = field(default_factory=list)
+ moods: list[str] = field(default_factory=list)
+ instruments: list[str] = field(default_factory=list)
+ tags: list[str] = field(default_factory=list)
+ bpm: Optional[float] = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+@dataclass
+class PageExtraction:
+ page_url: str
+ method: str
+ tracks: list[PageTrack]
+ # Which adapter or source produced them, for the review screen: "incompetech",
+ # "mixkit", "schema.org", "AI", "links".
+ source: str
+ note: str = ""
+
+
+# ─── entry point ─────────────────────────────────────────────────────────────
+
+
+def extract_structured(page_url: str, page: str, fetch_text: FetchText) -> Optional[PageExtraction]:
+ """Tracks the site states outright. None when it states none."""
+ host = (urlsplit(page_url).hostname or "").lower()
+
+ if _is_host(host, "incompetech.com") and "pieces.json" in page:
+ tracks = _incompetech(page_url, page, fetch_text)
+ if tracks:
+ return PageExtraction(page_url, METHOD_STRUCTURED, tracks, "incompetech")
+
+ tracks = _json_ld_tracks(page_url, page)
+ if tracks:
+ source = "schema.org"
+ if _is_host(host, "mixkit.co"):
+ _enrich_mixkit(page_url, page, tracks)
+ source = "mixkit"
+ return PageExtraction(page_url, METHOD_STRUCTURED, tracks[:MAX_TRACKS], source)
+
+ return None
+
+
+def extract_links(page_url: str, page: str) -> PageExtraction:
+ """Every audio file the page links to, named by its link text. The last resort."""
+ tracks: list[PageTrack] = []
+ seen: set[str] = set()
+ for href, text in _audio_anchors(page_url, page):
+ if href in seen:
+ continue
+ seen.add(href)
+ tracks.append(
+ PageTrack(audio_url=href, title=text or _title_from_url(href), source_url=href)
+ )
+ return PageExtraction(page_url, METHOD_LINKS, tracks[:MAX_TRACKS], "links")
+
+
+def candidate_audio_urls(page_url: str, page: str) -> list[str]:
+ """Every audio URL the page contains, anywhere: links, players, scripts, data attributes.
+
+ What the AI path may choose from. A model that names a URL not in this list has
+ invented it, and is not believed.
+ """
+ found: list[str] = []
+ seen: set[str] = set()
+ pattern = r"""(?:https?:)?//[^\s"'<>()\\]+?\.(?:mp3|ogg|oga|opus|wav|flac|m4a|aac)(?:\?[^\s"'<>()\\]*)?(?=["'\s<>)\\]|$)"""
+ for match in re.finditer(pattern, html_lib.unescape(page), re.I):
+ url = urljoin(page_url, match.group(0))
+ if url not in seen:
+ seen.add(url)
+ found.append(url)
+ for href, _text in _audio_anchors(page_url, page):
+ if href not in seen:
+ seen.add(href)
+ found.append(href)
+ return found
+
+
+def page_text(page: str, limit: int = 60_000) -> str:
+ """The page as readable text for a model, with every link's target kept inline.
+
+ Scripts and styles go, except JSON-LD and inline JSON, which often hold exactly the
+ track list a model needs. Link targets are kept as "text [link: url]" so the model can
+ tie a title to its file; angle brackets would not survive the tag stripping below.
+ """
+ body = re.sub(r"<(style|noscript|svg)[^>]*>.*?\1>", " ", page, flags=re.S | re.I)
+ body = re.sub(r"", " ", body, flags=re.S | re.I)
+ body = re.sub(
+ r"]*href\s*=\s*[\"']([^\"']+)[\"'][^>]*>(.*?)",
+ lambda m: f" {m.group(2)} [link: {m.group(1)}] ",
+ body,
+ flags=re.S | re.I,
+ )
+ body = re.sub(
+ r"<(?:audio|source)\b[^>]*src\s*=\s*[\"']([^\"']+)[\"'][^>]*>",
+ lambda m: f" [audio: {m.group(1)}] ",
+ body,
+ flags=re.I,
+ )
+ body = re.sub(r" |(?:p|div|li|tr|h\d)>", "\n", body, flags=re.I)
+ body = re.sub(r"<[^>]+>", " ", body)
+ body = html_lib.unescape(body)
+ body = re.sub(r"[ \t]+", " ", body)
+ body = re.sub(r"\n\s*\n+", "\n", body)
+ return body.strip()[:limit]
+
+
+# ─── incompetech ─────────────────────────────────────────────────────────────
+
+# Kevin MacLeod's catalogue. The page renders nothing itself: its script fetches
+# `pieces.json` beside it and builds the list in the browser. The genre and album names
+# are lookup tables in that same script, read out of the page rather than copied here,
+# so a genre added there is picked up without a change here.
+#
+# The credit is the attribution code the page shows under every track, verbatim: it is
+# what the licence asks for, so it is stored as the credit line rather than composed.
+_INCOMPETECH_CREATOR = "Kevin MacLeod"
+_INCOMPETECH_PUBLISHER = "incompetech.com"
+_INCOMPETECH_LICENSE = "CC BY 4.0"
+_INCOMPETECH_CREDIT = (
+ '"{title}" Kevin MacLeod (incompetech.com) '
+ "Licensed under Creative Commons: By Attribution 4.0 License "
+ "http://creativecommons.org/licenses/by/4.0/"
+)
+_INCOMPETECH_MP3_PATH = "/music/royalty-free/mp3-royaltyfree/"
+
+
+def _incompetech(page_url: str, page: str, fetch_text: FetchText) -> list[PageTrack]:
+ genres = {
+ m.group(1): m.group(2)
+ for m in re.finditer(r'"id"\s*:\s*"?(\d+)"?\s*,\s*"genre"\s*:\s*"([^"]+)"', page)
+ }
+ albums = {
+ m.group(3): m.group(1)
+ for m in re.finditer(
+ r'"collection_name"\s*:\s*"([^"]+)"\s*,\s*"collection_category"\s*:\s*"([^"]+)"'
+ r'\s*,\s*"code"\s*:\s*"(\d+)"',
+ page,
+ )
+ }
+
+ pieces_url, raw = fetch_text(urljoin(page_url, "pieces.json"))
+ try:
+ pieces = json.loads(raw)
+ except ValueError:
+ return []
+ if isinstance(pieces, dict):
+ pieces = pieces.get("pieces") or pieces.get("data") or []
+ if not isinstance(pieces, list):
+ return []
+
+ tracks: list[PageTrack] = []
+ for piece in pieces:
+ if not isinstance(piece, dict):
+ continue
+ filename = _text(piece.get("filename"))
+ title = _text(piece.get("title"))
+ if not filename or not title or extension_of(filename) not in AUDIO_EXTENSIONS:
+ continue
+ audio_url = urljoin(pieces_url, _INCOMPETECH_MP3_PATH + quote(filename))
+ genre = genres.get(str(piece.get("genre") or ""))
+ album = albums.get(str(piece.get("collection") or ""))
+ tracks.append(
+ PageTrack(
+ audio_url=audio_url,
+ title=title[:255],
+ creator=_INCOMPETECH_CREATOR,
+ publisher=_INCOMPETECH_PUBLISHER,
+ source_title=album if album and album != "not in a collection" else None,
+ description=_plain(piece.get("description"), MAX_DESCRIPTION),
+ duration_seconds=_clock_seconds(piece.get("length")),
+ published_date=normalise_partial_date(_text(piece.get("uploaded"))),
+ license=_INCOMPETECH_LICENSE,
+ credit_line=_INCOMPETECH_CREDIT.format(title=title),
+ source_url=audio_url,
+ genres=[genre] if genre and genre != "Unclassifiable" else [],
+ moods=music.split_terms(piece.get("feel")),
+ instruments=music.split_terms(piece.get("instruments")),
+ bpm=music.parse_bpm(piece.get("bpm")),
+ )
+ )
+ if len(tracks) >= MAX_TRACKS:
+ break
+ return tracks
+
+
+# ─── schema.org JSON-LD ──────────────────────────────────────────────────────
+
+
+def _json_ld_tracks(page_url: str, page: str) -> list[PageTrack]:
+ tracks: list[PageTrack] = []
+ seen: set[str] = set()
+ for block in re.findall(
+ r"",
+ page,
+ flags=re.S | re.I,
+ ):
+ try:
+ data = json.loads(html_lib.unescape(block.strip()))
+ except ValueError:
+ continue
+ for node in _walk(data):
+ if not _is_type(node, "MusicRecording", "AudioObject"):
+ continue
+ track = _from_recording(page_url, node)
+ if track and track.audio_url not in seen:
+ seen.add(track.audio_url)
+ tracks.append(track)
+ return tracks
+
+
+def _walk(node: Any) -> Iterable[dict]:
+ if isinstance(node, dict):
+ yield node
+ for value in node.values():
+ yield from _walk(value)
+ elif isinstance(node, list):
+ for value in node:
+ yield from _walk(value)
+
+
+def _is_type(node: dict, *names: str) -> bool:
+ kind = node.get("@type")
+ kinds = kind if isinstance(kind, list) else [kind]
+ return any(k in names for k in kinds if isinstance(k, str))
+
+
+def _from_recording(page_url: str, node: dict) -> Optional[PageTrack]:
+ audio = node.get("audio") if isinstance(node.get("audio"), dict) else {}
+ candidates = [node.get("contentUrl"), audio.get("contentUrl"), node.get("url")]
+ audio_url = next(
+ (
+ urljoin(page_url, c)
+ for c in candidates
+ if isinstance(c, str) and extension_of(urlsplit(c).path) in AUDIO_EXTENSIONS
+ ),
+ None,
+ )
+ title = _text(node.get("name"))
+ if not audio_url or not title:
+ return None
+
+ # The track's own page when the data names one. Otherwise the file itself, not the
+ # listing it was found on: a listing is shared by every track on it, and the source
+ # URL is also what recognises a track already imported.
+ page_link = node.get("url")
+ track_page = (
+ urljoin(page_url, page_link)
+ if isinstance(page_link, str) and urljoin(page_url, page_link) != audio_url
+ else audio_url
+ )
+ album = node.get("inAlbum")
+ licence_url = _text(node.get("license"))
+ notice = _text(node.get("copyrightNotice"))
+ return PageTrack(
+ audio_url=audio_url,
+ title=title[:255],
+ creator=_name_of(node.get("byArtist")) or _name_of(node.get("creator")),
+ publisher=_name_of(node.get("publisher")),
+ source_title=_name_of(album) if album else None,
+ description=_plain(node.get("description"), MAX_DESCRIPTION),
+ duration_seconds=_iso_duration(node.get("duration") or audio.get("duration")),
+ published_date=normalise_partial_date(_text(node.get("datePublished"))),
+ license=(f"{notice} ({licence_url})" if notice and licence_url else notice or licence_url)
+ or None,
+ source_url=track_page,
+ genres=_terms(node.get("genre")),
+ tags=_terms(node.get("keywords")),
+ )
+
+
+def _name_of(value: Any) -> Optional[str]:
+ if isinstance(value, str):
+ return _text(value)[:MAX_TEXT] or None
+ if isinstance(value, dict):
+ name = value.get("name")
+ return _text(name)[:MAX_TEXT] or None if isinstance(name, str) else None
+ if isinstance(value, list):
+ names = [n for n in (_name_of(v) for v in value) if n]
+ return ", ".join(names)[:MAX_TEXT] or None
+ return None
+
+
+def _terms(value: Any) -> list[str]:
+ if isinstance(value, list):
+ return music.clean_terms(v for v in value if isinstance(v, str))
+ if isinstance(value, str):
+ return music.split_terms(value)
+ return []
+
+
+# ─── mixkit ──────────────────────────────────────────────────────────────────
+
+# Each card is an audio player keyed by its MP3, followed by links to the track's genre,
+# moods, instruments and tags. The JSON-LD has the genre only, so the rest comes from
+# here, matched to the JSON-LD track by that same MP3 URL.
+_MIXKIT_CARD = re.compile(r'data-audio-player-preview-url-value="([^"]+)"')
+_MIXKIT_LINK = re.compile(r'class="meta-links__link"[^>]*href="([^"]+)"[^>]*>([^<]+)')
+
+
+def _enrich_mixkit(page_url: str, page: str, tracks: list[PageTrack]) -> None:
+ by_url = {t.audio_url: t for t in tracks}
+ cards = list(_MIXKIT_CARD.finditer(page))
+ for index, card in enumerate(cards):
+ track = by_url.get(urljoin(page_url, html_lib.unescape(card.group(1))))
+ if track is None:
+ continue
+ end = cards[index + 1].start() if index + 1 < len(cards) else len(page)
+ for link in _MIXKIT_LINK.finditer(page, card.end(), end):
+ kind, label = _mixkit_kind(link.group(1)), html_lib.unescape(link.group(2)).strip()
+ if kind == "mood":
+ track.moods = music.clean_terms([*track.moods, label])
+ elif kind == "instrument":
+ track.instruments = music.clean_terms([*track.instruments, label])
+ elif kind == "tag":
+ track.tags = music.clean_terms([*track.tags, label])
+ elif kind == "genre":
+ track.genres = music.clean_terms([*track.genres, label])
+
+ # A listing filtered to one mood or instrument says so of every track on it.
+ kind, label = _mixkit_page_filter(page_url)
+ for track in tracks:
+ track.publisher = track.publisher or "Mixkit"
+ if kind == "mood":
+ track.moods = music.clean_terms([*track.moods, label])
+ elif kind == "instrument":
+ track.instruments = music.clean_terms([*track.instruments, label])
+
+
+def _mixkit_kind(href: str) -> Optional[str]:
+ parts = [p for p in urlsplit(href).path.split("/") if p]
+ if len(parts) >= 3 and parts[0] == "free-stock-music" and parts[1] in ("mood", "instrument", "tag"):
+ return parts[1]
+ if len(parts) == 2 and parts[0] == "free-stock-music":
+ return "genre"
+ return None
+
+
+def _mixkit_page_filter(page_url: str) -> tuple[Optional[str], str]:
+ parts = [p for p in urlsplit(page_url).path.split("/") if p]
+ if len(parts) >= 3 and parts[0] == "free-stock-music" and parts[1] in ("mood", "instrument"):
+ return parts[1], unquote(parts[2]).replace("-", " ").title()
+ return None, ""
+
+
+# ─── plain links ─────────────────────────────────────────────────────────────
+
+
+def _audio_anchors(page_url: str, page: str) -> Iterable[tuple[str, str]]:
+ for match in re.finditer(
+ r"<(?:a\b[^>]*href|audio\b[^>]*src|source\b[^>]*src)\s*=\s*[\"']([^\"']+)[\"'][^>]*>(.*?)(?=<)",
+ page,
+ flags=re.S | re.I,
+ ):
+ href = urljoin(page_url, html_lib.unescape(match.group(1)))
+ if extension_of(urlsplit(href).path) not in AUDIO_EXTENSIONS:
+ continue
+ text = _plain(match.group(2), MAX_TEXT) or ""
+ yield href, text
+
+
+def _title_from_url(url: str) -> str:
+ stem = unquote(urlsplit(url).path.rsplit("/", 1)[-1]).rsplit(".", 1)[0]
+ return re.sub(r"[-_]+", " ", stem).strip()[:255] or "Untitled track"
+
+
+# ─── small parsers ───────────────────────────────────────────────────────────
+
+
+def _text(value: Any) -> str:
+ return " ".join(value.split()) if isinstance(value, str) else ""
+
+
+def _plain(value: Any, limit: int) -> Optional[str]:
+ if not isinstance(value, str):
+ return None
+ text = html_lib.unescape(re.sub(r"<[^>]+>", " ", value))
+ text = " ".join(text.split())
+ return text[:limit] or None
+
+
+def _iso_duration(value: Any) -> Optional[float]:
+ """ISO 8601 `PT1M53S` (schema.org's form) in seconds."""
+ if not isinstance(value, str):
+ return None
+ match = re.fullmatch(r"P(?:T(?:(\d+(?:\.\d+)?)H)?(?:(\d+(?:\.\d+)?)M)?(?:(\d+(?:\.\d+)?)S)?)", value.strip())
+ if not match or not any(match.groups()):
+ return None
+ hours, minutes, seconds = (float(g) if g else 0.0 for g in match.groups())
+ return hours * 3600 + minutes * 60 + seconds
+
+
+def _clock_seconds(value: Any) -> Optional[float]:
+ """`h:mm:ss` or `m:ss` in seconds."""
+ if not isinstance(value, str) or not value.strip():
+ return None
+ try:
+ parts = [float(p) for p in value.strip().split(":")]
+ except ValueError:
+ return None
+ total = 0.0
+ for part in parts:
+ total = total * 60 + part
+ return total or None
+
+
+def _is_host(host: str, domain: str) -> bool:
+ return host == domain or host.endswith("." + domain)
diff --git a/backend/app/ingest/web.py b/backend/app/ingest/web.py
new file mode 100644
index 0000000..c05b4e7
--- /dev/null
+++ b/backend/app/ingest/web.py
@@ -0,0 +1,220 @@
+"""Fetching a web page, or one audio file named on it, on the user's behalf.
+
+For importing tracks from a page (`ingest/page_tracks.py`). The same rules as
+`generation/download.py`, which is fal-specific (its errors, its retry policy, image and
+video only), restated here for a page and a track rather than shared through a flag:
+
+- **SSRF, on every hop.** The URL is the user's, and the track URLs are the page's. A
+ public address that redirects inward is the classic way past a check made only on the
+ first, so redirects are followed by hand and each target is checked.
+- **Capped while it arrives,** against `Content-Length` first when there is one, then
+ counted, because a check made after the write has already filled the disk.
+- **Uncompressed bodies only.** httpx decodes gzip and brotli with no output limit, so a
+ small compressed body could become gigabytes in memory before the cap saw a byte. A
+ page is the exception, being small and almost always sent compressed: it is decoded,
+ but read through a cap on the decoded size.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Callable, Iterator, Optional
+from urllib.parse import urljoin, urlsplit
+
+import httpx
+from fastapi import HTTPException
+
+from app.ingest.filetypes import AUDIO_EXTENSIONS, extension_of
+from app.jobs.runner import readable_error
+from app.safe_url import require_safe_external_url
+
+logger = logging.getLogger(__name__)
+
+MAX_REDIRECTS = 5
+TIMEOUT = httpx.Timeout(30.0, connect=10.0)
+# A listing page with a hundred tracks and their markup is a few hundred kilobytes, and
+# incompetech's whole catalogue as JSON about one megabyte.
+MAX_PAGE_BYTES = 8 * 1024 * 1024
+# A long, high-bitrate track. Anything bigger is not a soundtrack.
+MAX_TRACK_BYTES = 200 * 1024 * 1024
+MAX_TRACK_SECONDS = 15 * 60
+
+# Says what is asking, so a site that wants to can tell this apart from a browser and
+# from a crawler. It is one person's library fetching the tracks they picked.
+USER_AGENT = "GeckoAssetManager/1.0 (+personal media library; one request per chosen track)"
+
+monotonic = time.monotonic
+
+# Bytes received, and the total when the server said.
+Progress = Callable[[int, Optional[int]], None]
+
+_AUDIO_TYPES = {
+ "audio/mpeg": ".mp3",
+ "audio/mp3": ".mp3",
+ "audio/ogg": ".ogg",
+ "audio/opus": ".opus",
+ "audio/wav": ".wav",
+ "audio/x-wav": ".wav",
+ "audio/wave": ".wav",
+ "audio/flac": ".flac",
+ "audio/x-flac": ".flac",
+ "audio/aac": ".aac",
+ "audio/mp4": ".m4a",
+ "audio/x-m4a": ".m4a",
+}
+
+
+class WebFetchError(Exception):
+ """A page or a track could not be fetched. The message is worded for the user."""
+
+
+def require_safe(url: str) -> None:
+ try:
+ require_safe_external_url(url)
+ except HTTPException as exc:
+ raise WebFetchError(f"That address is not one this app will fetch: {readable_error(exc)}") from exc
+
+
+def _client(accept_encoding: str) -> httpx.Client:
+ return httpx.Client(
+ timeout=TIMEOUT,
+ follow_redirects=False,
+ headers={"User-Agent": USER_AGENT, "Accept-Encoding": accept_encoding},
+ )
+
+
+@contextmanager
+def _get(client: httpx.Client, url: str) -> Iterator[tuple[str, httpx.Response]]:
+ """Follow redirects by hand, checking each hop. Yields (final url, open response)."""
+ target = url
+ for _hop in range(MAX_REDIRECTS + 1):
+ require_safe(target)
+ with client.stream("GET", target) as response:
+ if response.is_redirect:
+ target = urljoin(target, response.headers["location"])
+ continue
+ yield target, response
+ return
+ raise WebFetchError("The address redirected too many times")
+
+
+def _status_error(what: str, response: httpx.Response) -> WebFetchError:
+ if response.status_code in (401, 403):
+ return WebFetchError(f"The site refused to send the {what} (HTTP {response.status_code})")
+ if response.status_code == 404:
+ return WebFetchError(f"The site has no such {what} (HTTP 404)")
+ return WebFetchError(f"Could not fetch the {what} (HTTP {response.status_code})")
+
+
+def _content_length(response: httpx.Response) -> Optional[int]:
+ raw = response.headers.get("content-length")
+ try:
+ return int(raw) if raw is not None else None
+ except ValueError:
+ return None
+
+
+def fetch_text(url: str, *, max_bytes: int = MAX_PAGE_BYTES) -> tuple[str, str]:
+ """A page (or a JSON file) as text. Returns (the final URL after redirects, text).
+
+ The final URL matters: relative links on the page resolve against it, not against
+ what was typed.
+ """
+ try:
+ with _client("gzip, deflate") as client, _get(client, url) as (final, response):
+ if response.status_code >= 400:
+ raise _status_error("page", response)
+ chunks: list[bytes] = []
+ received = 0
+ # iter_bytes decodes as it goes, so this counts what the page becomes, not
+ # what was sent: the cap holds against a compression bomb too.
+ for chunk in response.iter_bytes():
+ received += len(chunk)
+ if received > max_bytes:
+ raise WebFetchError("The page is too large to read")
+ chunks.append(chunk)
+ encoding = response.encoding or "utf-8"
+ except httpx.TimeoutException as exc:
+ raise WebFetchError("The site took too long to answer") from exc
+ except httpx.RequestError as exc:
+ # The type, not the text: a RequestError's message carries the full URL.
+ raise WebFetchError(f"Could not reach the site ({type(exc).__name__})") from exc
+
+ return final, b"".join(chunks).decode(encoding, errors="replace")
+
+
+def download_audio(
+ url: str,
+ workdir: Path,
+ stem: str,
+ *,
+ max_bytes: int = MAX_TRACK_BYTES,
+ on_progress: Optional[Progress] = None,
+) -> Path:
+ """Fetch one track into `workdir/` and return the path.
+
+ The extension comes from the URL path when it names an audio type, else from the
+ response's `Content-Type`. Anything that is neither is refused: a link that answers
+ with an HTML page (a login wall, a "download starting" interstitial) must not land in
+ the library as a broken track.
+ """
+ report = on_progress or (lambda _received, _total: None)
+ try:
+ with _client("identity") as client, _get(client, url) as (final, response):
+ if response.status_code >= 400:
+ raise _status_error("track", response)
+ encoding = response.headers.get("content-encoding", "").strip().lower()
+ if encoding not in ("", "identity"):
+ raise WebFetchError("The site sent the track compressed, which is refused")
+
+ extension = _audio_extension(final, response.headers.get("content-type"))
+ if extension is None:
+ raise WebFetchError("That link is not an audio file")
+
+ total = _content_length(response)
+ if total is not None and total > max_bytes:
+ raise _too_large(max_bytes)
+
+ destination = workdir / f"{stem}{extension}"
+ _stream_to(response, destination, max_bytes=max_bytes, total=total, report=report)
+ return destination
+ except httpx.TimeoutException as exc:
+ raise WebFetchError("The track did not download in time") from exc
+ except httpx.RequestError as exc:
+ raise WebFetchError(f"Could not download the track ({type(exc).__name__})") from exc
+
+
+def _audio_extension(url: str, content_type: Optional[str]) -> Optional[str]:
+ from_path = extension_of(urlsplit(url).path)
+ if from_path in AUDIO_EXTENSIONS:
+ return from_path
+ media_type = (content_type or "").split(";", 1)[0].strip().lower()
+ return _AUDIO_TYPES.get(media_type)
+
+
+def _stream_to(
+ response: httpx.Response, destination: Path, *, max_bytes: int, total: Optional[int], report: Progress
+) -> None:
+ received = 0
+ started = monotonic()
+ try:
+ with open(destination, "wb") as handle:
+ for chunk in response.iter_bytes():
+ received += len(chunk)
+ if received > max_bytes:
+ raise _too_large(max_bytes)
+ if monotonic() - started > MAX_TRACK_SECONDS:
+ raise WebFetchError("The track did not download in time")
+ handle.write(chunk)
+ report(received, total)
+ except BaseException:
+ # BaseException, so a cancel raised from `report` cleans up too.
+ destination.unlink(missing_ok=True)
+ raise
+
+
+def _too_large(max_bytes: int) -> WebFetchError:
+ return WebFetchError(f"The track is larger than the {max_bytes // (1024 * 1024)} MB limit")
diff --git a/backend/app/jobs/enrichment.py b/backend/app/jobs/enrichment.py
index d8b184c..38e620a 100644
--- a/backend/app/jobs/enrichment.py
+++ b/backend/app/jobs/enrichment.py
@@ -38,6 +38,8 @@
from app.enrichment.extract_text import run as run_extract_text
from app.enrichment.generate_all import run as run_generate_all
from app.enrichment.import_url import run as run_import_url
+from app.enrichment.page_import import PageImportError
+from app.enrichment.page_import import run_track as run_import_track
from app.enrichment.source import NoSourceMaterial
from app.enrichment.subvideo import SubvideoExtractionError
from app.enrichment.summarize import run as run_summarize
@@ -72,6 +74,7 @@
GENERATION_JOB_KINDS,
KIND_GENERATE,
KIND_GENERATE_ALL,
+ KIND_IMPORT_TRACK,
KIND_IMPORT_URL,
KIND_SUMMARIZE,
KIND_TRANSCRIBE,
@@ -351,6 +354,15 @@ def _run(job_id: str, q: JobQueue) -> None:
# finished row, which is the first thing to look at when an import
# produced something unexpected.
job.payload = _with_created_asset(job.payload, imported.created_asset_id)
+ elif job.kind == KIND_IMPORT_TRACK:
+ imported_track = run_import_track(session, job, progress)
+ detail = imported_track.detail
+ if imported_track.created_asset_id:
+ # Merged rather than replacing, as for a URL import: the track's URL
+ # and what the page said stay readable on the finished row.
+ job.payload = _with_created_asset(
+ job.payload, imported_track.created_asset_id
+ )
elif job.kind == KIND_GENERATE:
# The created ids are already in the payload — the job writes each as it
# is made, so a restart mid-way knows what not to make again.
@@ -446,6 +458,11 @@ def _run(job_id: str, q: JobQueue) -> None:
set_fields(session, job, status="error", stage="", error_message=message)
logger.info("Enrichment job %s failed: %s", job_id, message)
+ except PageImportError as exc:
+ message = str(exc)
+ set_fields(session, job, status="error", stage="", error_message=message)
+ logger.info("Enrichment job %s failed: %s", job_id, message)
+
except UrlImportError as exc:
message = str(exc)
set_fields(session, job, status="error", stage="", error_message=message)
diff --git a/backend/app/jobs/registry.py b/backend/app/jobs/registry.py
index 1bb19e3..0dd8462 100644
--- a/backend/app/jobs/registry.py
+++ b/backend/app/jobs/registry.py
@@ -21,7 +21,13 @@
from sqlmodel import Session, col, select
from app.jobs.runner import ACTIVE_STATUSES, JobQueue, is_stale
-from app.models.job import KIND_EXTRACT_SUBVIDEO, KIND_GENERATE, KIND_IMPORT_URL, EnrichmentJob
+from app.models.job import (
+ KIND_EXTRACT_SUBVIDEO,
+ KIND_GENERATE,
+ KIND_IMPORT_TRACK,
+ KIND_IMPORT_URL,
+ EnrichmentJob,
+)
from app.schemas_jobs import ActivityJobRead
@@ -48,7 +54,9 @@ def __init__(
# when the job was queued, so `asset_id` cannot carry them: for an extraction it stays
# pointed at the source, and an import or a generation has no asset at all until it is
# done.
-_CREATES_AN_ASSET = frozenset({KIND_EXTRACT_SUBVIDEO, KIND_IMPORT_URL, KIND_GENERATE})
+_CREATES_AN_ASSET = frozenset(
+ {KIND_EXTRACT_SUBVIDEO, KIND_IMPORT_URL, KIND_IMPORT_TRACK, KIND_GENERATE}
+)
def _result_asset_ids(job: EnrichmentJob) -> List[str]:
diff --git a/backend/app/models/job.py b/backend/app/models/job.py
index 914dfa9..e9511ce 100644
--- a/backend/app/models/job.py
+++ b/backend/app/models/job.py
@@ -73,6 +73,10 @@
# The same over every audio file in a library, for the ones uploaded before it existed.
# Library-wide for the reasons `KIND_HARVEST_ATTRIBUTION` is.
KIND_ANALYZE_MUSIC_LIBRARY = "analyze_music_library"
+# One track chosen from a music library's page (enrichment/page_import.py): downloaded,
+# catalogued, and given what the page said about it. A library kind for the reason an
+# import is: the asset does not exist until it has run. The track rides in `payload`.
+KIND_IMPORT_TRACK = "import_track"
# Kinds that run on the generation queue rather than the enrichment one. A generation
# is minutes of waiting on fal, and on the shared queue at concurrency 1 a video would
@@ -105,6 +109,7 @@
KIND_HARVEST_ATTRIBUTION,
KIND_ANALYZE_MUSIC_LIBRARY,
KIND_IMPORT_URL,
+ KIND_IMPORT_TRACK,
KIND_GENERATE,
}
)
diff --git a/backend/app/routers/imports.py b/backend/app/routers/imports.py
index 5194b4c..9682d39 100644
--- a/backend/app/routers/imports.py
+++ b/backend/app/routers/imports.py
@@ -8,18 +8,23 @@
from __future__ import annotations
-from fastapi import APIRouter, Depends, status
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel, Field
from sqlmodel import Session
from app.auth import CurrentUser
from app.database import get_session
+from app.enrichment import page_import
from app.enrichment.import_url import ImportRequest
+from app.ingest.page_tracks import PageTrack
from app.ingest.ytdlp import MAX_URL, normalise_url
from app.jobs import enrichment as enrichment_jobs
from app.jobs.registry import KINDS
-from app.models.job import KIND_IMPORT_URL
+from app.models.job import KIND_IMPORT_TRACK, KIND_IMPORT_URL
from app.schemas import DataResponse
+from app.safe_url import require_safe_external_url
from app.schemas_jobs import ActivityJobRead
from app.storage import capacity
@@ -70,3 +75,138 @@ def start_url_import(
session, user.id, KIND_IMPORT_URL, payload=request.encode(), asset_name=url[:255]
)
return DataResponse(data=KINDS["enrichment"].to_activity(job))
+
+
+# ─── tracks from a music library's page ──────────────────────────────────────
+
+
+class PageTrackBody(BaseModel):
+ """One track, as the preview found it and the person corrected it."""
+
+ audio_url: str = Field(min_length=1, max_length=MAX_URL)
+ title: str = Field(min_length=1, max_length=255)
+ creator: Optional[str] = Field(default=None, max_length=500)
+ publisher: Optional[str] = Field(default=None, max_length=500)
+ source_title: Optional[str] = Field(default=None, max_length=500)
+ description: Optional[str] = Field(default=None, max_length=20_000)
+ duration_seconds: Optional[float] = None
+ published_date: Optional[str] = Field(default=None, max_length=10)
+ license: Optional[str] = Field(default=None, max_length=500)
+ credit_line: Optional[str] = Field(default=None, max_length=2_000)
+ source_url: Optional[str] = Field(default=None, max_length=MAX_URL)
+ genres: List[str] = Field(default_factory=list, max_length=20)
+ moods: List[str] = Field(default_factory=list, max_length=20)
+ instruments: List[str] = Field(default_factory=list, max_length=20)
+ tags: List[str] = Field(default_factory=list, max_length=50)
+ bpm: Optional[float] = None
+
+
+class PageTrackRead(PageTrackBody):
+ # The asset this library already holds for the track, so the review list can show it
+ # as imported rather than offering it twice.
+ existing_asset_id: Optional[str] = None
+ # An audio file with the same title (and artist) is already here: probably the same
+ # piece from somewhere else. Flagged, not refused, since titles collide.
+ similar_asset_id: Optional[str] = None
+
+
+class PagePreviewRequest(BaseModel):
+ url: str = Field(min_length=1, max_length=MAX_URL)
+ # Off skips the AI fallback, which costs a little per page.
+ use_ai: bool = True
+
+
+class PagePreview(BaseModel):
+ page_url: str
+ # structured | ai | links: how much the details can be trusted.
+ method: str
+ source: str
+ note: str = ""
+ tracks: List[PageTrackRead]
+
+
+class PageImportRequest(BaseModel):
+ tracks: List[PageTrackBody] = Field(min_length=1, max_length=page_import.MAX_TRACKS_PER_IMPORT)
+
+
+def _page_error(exc: Exception) -> HTTPException:
+ return HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail={"code": "page_unreadable", "message": str(exc)},
+ )
+
+
+@router.post("/import-page/preview", response_model=DataResponse[PagePreview])
+def preview_page_import(
+ payload: PagePreviewRequest,
+ user: CurrentUser,
+ session: Session = Depends(get_session),
+) -> DataResponse[PagePreview]:
+ """Read a page and list the tracks on it, for the person to choose from.
+
+ Synchronous, unlike a URL import: the answer is a list to review, not work to watch,
+ and reading a page (and, without structured data, asking the AI about it) takes
+ seconds rather than minutes. Nothing is downloaded here.
+ """
+ url = normalise_url(payload.url)
+ try:
+ extraction = page_import.preview(session, user.id, url, use_ai=payload.use_ai)
+ except page_import.PageImportError as exc:
+ raise _page_error(exc) from exc
+
+ existing = page_import.already_imported(session, user.id, extraction.tracks)
+ similar = page_import.similar_in_library(session, user.id, extraction.tracks)
+ return DataResponse(
+ data=PagePreview(
+ page_url=extraction.page_url,
+ method=extraction.method,
+ source=extraction.source,
+ note=extraction.note,
+ tracks=[
+ PageTrackRead(
+ **t.to_dict(),
+ existing_asset_id=existing.get(t.audio_url),
+ similar_asset_id=(
+ None if t.audio_url in existing else similar.get(t.audio_url)
+ ),
+ )
+ for t in extraction.tracks
+ ],
+ )
+ )
+
+
+@router.post(
+ "/import-page",
+ response_model=DataResponse[List[ActivityJobRead]],
+ status_code=status.HTTP_202_ACCEPTED,
+)
+def start_page_import(
+ payload: PageImportRequest,
+ user: CurrentUser,
+ session: Session = Depends(get_session),
+) -> DataResponse[List[ActivityJobRead]]:
+ """Queue one download per chosen track.
+
+ The tracks come back from the browser, so every audio URL is checked again here: the
+ review list is editable, and a URL in it is the user's, not the page's.
+ """
+ capacity.require_room()
+ for track in payload.tracks:
+ require_safe_external_url(track.audio_url)
+
+ # The same track ticked twice (a page that lists it under two moods) is one job.
+ unique = list({body.audio_url: body for body in payload.tracks}.values())
+
+ jobs = []
+ for body in unique:
+ track = PageTrack(**body.model_dump())
+ job = enrichment_jobs.submit_library(
+ session,
+ user.id,
+ KIND_IMPORT_TRACK,
+ payload=page_import.encode_track(track),
+ asset_name=track.title,
+ )
+ jobs.append(KINDS["enrichment"].to_activity(job))
+ return DataResponse(data=jobs)
diff --git a/backend/tests/fixtures/pages/incompetech_music.html b/backend/tests/fixtures/pages/incompetech_music.html
new file mode 100644
index 0000000..c483e56
--- /dev/null
+++ b/backend/tests/fixtures/pages/incompetech_music.html
@@ -0,0 +1,17 @@
+
+
+
+
diff --git a/backend/tests/fixtures/pages/incompetech_pieces.json b/backend/tests/fixtures/pages/incompetech_pieces.json
new file mode 100644
index 0000000..21a5257
--- /dev/null
+++ b/backend/tests/fixtures/pages/incompetech_pieces.json
@@ -0,0 +1,10 @@
+[
+ {"title": "Cipher", "bpm": "128", "genre": "7", "length": "0:3:05",
+ "description": "Dark pulsing electronica.", "instruments": "Synths, Drums",
+ "feel": "Dark, Driving, Mysterious", "filename": "Cipher.mp3", "collection": "12",
+ "isrc": "USUAN1100301", "uploaded": "2011-04-04"},
+ {"title": "Gymnopedie No. 1", "bpm": "", "genre": "24", "length": "0:3:06",
+ "description": "Satie.", "instruments": "Piano", "feel": "Calm, Relaxed",
+ "filename": "Gymnopedie No 1.mp3", "collection": "999", "uploaded": "2010"},
+ {"title": "Broken entry with no file", "genre": "22"}
+]
diff --git a/backend/tests/fixtures/pages/mixkit_mood.html b/backend/tests/fixtures/pages/mixkit_mood.html
new file mode 100644
index 0000000..4f0c889
--- /dev/null
+++ b/backend/tests/fixtures/pages/mixkit_mood.html
@@ -0,0 +1,32 @@
+
+
+
+
+
+