diff --git a/backend/app/enrichment/page_import.py b/backend/app/enrichment/page_import.py index ef5f1bb..4f56f50 100644 --- a/backend/app/enrichment/page_import.py +++ b/backend/app/enrichment/page_import.py @@ -23,7 +23,7 @@ import tempfile from dataclasses import dataclass, fields from pathlib import Path -from typing import Any, Callable, Optional +from typing import Any, BinaryIO, Callable, Optional from urllib.parse import urljoin from sqlmodel import Session, col, func, select @@ -32,7 +32,7 @@ from app.clock import utcnow from app.ingest import page_tracks, web from app.ingest.embedded_metadata import normalise_partial_date -from app.ingest.filetypes import SOURCE_URL, TYPE_AUDIO +from app.ingest.filetypes import AUDIO_EXTENSIONS, SOURCE_UPLOAD, SOURCE_URL, TYPE_AUDIO, extension_of from app.ingest.page_tracks import PageExtraction, PageTrack from app.models.asset import Asset from app.models.job import EnrichmentJob @@ -107,11 +107,12 @@ def preview(session: Session, user_id: str, url: str, *, use_ai: bool = True) -> def already_imported(session: Session, user_id: str, tracks: list[PageTrack]) -> dict[str, str]: - """{audio_url: asset id} for the tracks this library already has.""" + """{track.key: asset id} for the tracks this library already has.""" keys: dict[str, str] = {} for track in tracks: - keys[track.source_url or track.audio_url] = track.audio_url - keys[track.audio_url] = track.audio_url + for url in (track.source_url, track.audio_url): + if url: + keys[url] = track.key if not keys: return {} rows = session.exec( @@ -123,7 +124,7 @@ def already_imported(session: Session, user_id: str, tracks: list[PageTrack]) -> def similar_in_library(session: Session, user_id: str, tracks: list[PageTrack]) -> dict[str, str]: - """{audio_url: asset id} for tracks whose title (and artist, when both are known) + """{track.key: asset id} for tracks whose title (and artist, when both are known) match an audio file already in the library. A warning, not a refusal. The same piece downloaded from somewhere else is a @@ -150,7 +151,7 @@ def similar_in_library(session: Session, user_id: str, tracks: list[PageTrack]) for asset_id, creator in by_title.get(track.title.strip().lower(), []): if track.creator and creator and track.creator.strip().lower() != creator.strip().lower(): continue - found[track.audio_url] = asset_id + found[track.key] = asset_id break return found @@ -309,8 +310,8 @@ def run_track(session: Session, job: EnrichmentJob, progress: Progress) -> Track track = decode_track(job.payload) existing = already_imported(session, job.user_id, [track]) - if track.audio_url in existing: - return TrackImportResult("Already in your library", existing[track.audio_url]) + if track.key in existing: + return TrackImportResult("Already in your library", existing[track.key]) if not capacity.has_room(): raise PageImportError(capacity.DISK_FULL_MESSAGE) @@ -424,3 +425,125 @@ def apply_track_metadata( except Exception: # noqa: BLE001 - same logger.warning("Could not attach page tags to %s", asset.id, exc_info=True) return attached + + +# ─── a saved page and the files downloaded from it ────────────────────────── + + +SAVED_STATUS_IMPORTED = "imported" +SAVED_STATUS_DUPLICATE = "duplicate" +SAVED_STATUS_UNMATCHED = "unmatched" +SAVED_STATUS_REJECTED = "rejected" + + +@dataclass +class SavedFileResult: + filename: str + status: str + asset_id: Optional[str] = None + track_title: Optional[str] = None + message: str = "" + + +def import_saved( + session: Session, + user_id: str, + extraction: PageExtraction, + files: list[tuple[str, BinaryIO]], +) -> list[SavedFileResult]: + """Catalogue files the person downloaded, with what their saved page said of each. + + For sites that refuse GAM (see `page_tracks.extract_saved`). Nothing is fetched: the + person brought the page and the files, and this only matches one to the other and + files them, the way an upload is filed. A file that matches no track on the page is + still imported, as music without the page's details; the person chose to upload it. + The same three duplicate checks as a live import apply, the checksum one here. + """ + matches = page_tracks.match_files(extraction.tracks, [name for name, _ in files]) + results: list[SavedFileResult] = [] + with tempfile.TemporaryDirectory(prefix="gam-saved-") as workdir: + for index, (name, stream) in enumerate(files): + extension = extension_of(name) + if extension not in AUDIO_EXTENSIONS: + results.append(SavedFileResult(name, SAVED_STATUS_REJECTED, message="Not an audio file")) + continue + if not capacity.has_room(): + results.append( + SavedFileResult(name, SAVED_STATUS_REJECTED, message=capacity.DISK_FULL_MESSAGE) + ) + continue + + path = Path(workdir) / f"{index}{extension}" + if not _spool(stream, path, web.MAX_TRACK_BYTES): + results.append( + SavedFileResult( + name, + SAVED_STATUS_REJECTED, + message=f"Larger than the {web.MAX_TRACK_BYTES // (1024 * 1024)} MB limit", + ) + ) + continue + + track = matches.get(name) + duplicate = _same_file(session, user_id, path) + if duplicate is not None: + if track is not None: + apply_track_metadata(session, duplicate, track, fill_blanks_only=True) + results.append( + SavedFileResult( + name, + SAVED_STATUS_DUPLICATE, + asset_id=duplicate.id, + track_title=track.title if track else None, + message=f"Already in your library as \u201c{duplicate.name}\u201d", + ) + ) + continue + + try: + asset = asset_service.ingest_file( + session, + build_storage(), + user_id=user_id, + source_path=path, + filename=name, + name=track.title if track else None, + source=SOURCE_UPLOAD, + # The person brought it from a music library's page: music, so not + # sent to Deepgram, matched or not. + audio_kind=music.AUDIO_KIND_MUSIC, + ) + except asset_service.UnsupportedFile: + results.append(SavedFileResult(name, SAVED_STATUS_REJECTED, message="Not an audio file")) + continue + + if track is None: + results.append( + SavedFileResult( + name, + SAVED_STATUS_UNMATCHED, + asset_id=asset.id, + message="Imported, but not found on the page, so it has no details from it", + ) + ) + continue + apply_track_metadata(session, asset, track) + results.append( + SavedFileResult(name, SAVED_STATUS_IMPORTED, asset_id=asset.id, track_title=track.title) + ) + return results + + +def _spool(stream: BinaryIO, target: Path, max_bytes: int) -> bool: + """Copy an upload to disk, giving up past `max_bytes`. False when it was too large.""" + written = 0 + with open(target, "wb") as handle: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + written += len(chunk) + if written > max_bytes: + break + handle.write(chunk) + if written > max_bytes: + target.unlink(missing_ok=True) + return False + return True diff --git a/backend/app/ingest/page_tracks.py b/backend/app/ingest/page_tracks.py index d871154..3eb1ed4 100644 --- a/backend/app/ingest/page_tracks.py +++ b/backend/app/ingest/page_tracks.py @@ -50,9 +50,14 @@ @dataclass class PageTrack: - """One track as a page describes it. Every field but the URL and title optional.""" + """One track as a page describes it. Every field but the title optional. - audio_url: str + `audio_url` is always set on a live page import, which downloads from it. A saved page + can lack one (Pixabay's download is a button, not a link): the file then comes from + the person, matched to the track by `match_id` or title (`match_files`). + """ + + audio_url: Optional[str] title: str creator: Optional[str] = None publisher: Optional[str] = None @@ -70,6 +75,14 @@ class PageTrack: instruments: list[str] = field(default_factory=list) tags: list[str] = field(default_factory=list) bpm: Optional[float] = None + # A site's own id for the track, when its downloaded files carry it in their names + # (Pixabay's do: `artist-title-604094.mp3`). What matches a saved page to the files. + match_id: Optional[str] = None + + @property + def key(self) -> str: + """What identifies this track within one page, with or without an audio URL.""" + return self.audio_url or self.source_url or self.title def to_dict(self) -> dict[str, Any]: return asdict(self) @@ -173,6 +186,171 @@ def page_text(page: str, limit: int = 60_000) -> str: return body.strip()[:limit] +def extract_saved(page: str) -> PageExtraction: + """Tracks from a page the person saved in their own browser. Never touches the network. + + For sites that refuse automated requests (Pixabay answers a server with 403): the + person opens the page, saves it, and downloads the files themselves, so the only + thing GAM does with the site is read what they handed it. The page's own address is + recovered from the saved file, since adapters key on the host and on the path. + """ + page_url = saved_page_url(page) or "" + host = (urlsplit(page_url).hostname or "").lower() + + if _is_host(host, "pixabay.com"): + tracks = _pixabay(page_url, page) + if tracks: + return PageExtraction(page_url, METHOD_STRUCTURED, tracks[:MAX_TRACKS], "pixabay") + + try: + found = extract_structured(page_url, page, _no_network) + except SavedPageNeedsMore as exc: + return PageExtraction(page_url, METHOD_STRUCTURED, [], "saved page", note=str(exc)) + if found is not None: + return found + links = extract_links(page_url, page) + if not links.tracks: + links.note = "No tracks were found in the saved page." + return links + + +class SavedPageNeedsMore(Exception): + """An adapter needed a second file the saved page only points to.""" + + +def _no_network(url: str) -> tuple[str, str]: + name = urlsplit(url).path.rsplit("/", 1)[-1] or url + raise SavedPageNeedsMore( + f"This page loads its track list from {name}. Use the page link instead of a saved copy." + ) + + +def saved_page_url(page: str) -> Optional[str]: + """Where a saved page came from: its canonical link, its og:url, or the comment + Chrome and Edge write at the top of a saved page.""" + for pattern in ( + r"]*rel=[\"']canonical[\"'][^>]*href=[\"']([^\"']+)[\"']", + r"]*href=[\"']([^\"']+)[\"'][^>]*rel=[\"']canonical[\"']", + r"]*property=[\"']og:url[\"'][^>]*content=[\"']([^\"']+)[\"']", + r"", + ): + match = re.search(pattern, page, re.I) + if match and match.group(1).startswith(("http://", "https://")): + return html_lib.unescape(match.group(1)) + return None + + +# ─── matching files to tracks ──────────────────────────────────────────────── + + +def _normalise(text: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", text.lower()).strip() + + +def match_files(tracks: list[PageTrack], filenames: list[str]) -> dict[str, Optional[PageTrack]]: + """Which track each downloaded file is. One file per track, one track per file. + + By the site's id in the filename first (Pixabay names a download + `artist-title-604094.mp3`), which cannot be wrong. Then by title contained in the + filename, longest title first, so a file named for "Tension Riser" is not taken by + the track called "Riser". A file matching nothing maps to None. + """ + result: dict[str, Optional[PageTrack]] = {name: None for name in filenames} + taken: set[int] = set() + by_id = {t.match_id: i for i, t in enumerate(tracks) if t.match_id} + + for name in filenames: + stem = name.rsplit(".", 1)[0] + for number in re.findall(r"\d+", stem): + index = by_id.get(number) + if index is not None and index not in taken: + result[name] = tracks[index] + taken.add(index) + break + + titled = sorted( + ((i, _normalise(t.title)) for i, t in enumerate(tracks) if _normalise(t.title)), + key=lambda pair: len(pair[1]), + reverse=True, + ) + for name in filenames: + if result[name] is not None: + continue + stem = f" {_normalise(name.rsplit('.', 1)[0])} " + for index, title in titled: + if index not in taken and f" {title} " in stem: + result[name] = tracks[index] + taken.add(index) + break + return result + + +# ─── pixabay ───────────────────────────────────────────────────────────────── + +# A saved copy of a Pixabay music listing, as the browser rendered it: one row per track. +# The class names carry build hashes (`audioRow--et3ac`) that change every release, so +# only their stable prefixes are matched. There is no audio URL in a row, since +# downloading is a button, so the person's own downloads are matched to these rows. +_PIXABAY_ROW = re.compile(r"class=\"audioRow--") +_PIXABAY_TITLE = re.compile(r"]*class=\"title--[^\"]*\"[^>]*>([^<]*)") +_PIXABAY_ARTIST = re.compile(r"class=\"name--[^\"]*\"[^>]*>([^<]*)") +_PIXABAY_DURATION = re.compile(r"class=\"duration--[^\"]*\"[^>]*>([^<]*)<") +_PIXABAY_GENRE = re.compile(r"class=\"theme--[^\"]*\"[^>]*>([^<]*)") +_PIXABAY_TAGS = re.compile(r"class=\"tags--[^\"]*\"[^>]*>(.*?)", re.S) +_PIXABAY_LICENSE = "Pixabay Content License (https://pixabay.com/service/license-summary/)" + + +def _pixabay(page_url: str, page: str) -> list[PageTrack]: + starts = [m.start() for m in _PIXABAY_ROW.finditer(page)] + [len(page)] + kind, label = _pixabay_page_filter(page_url) + tracks: list[PageTrack] = [] + for start, end in zip(starts, starts[1:]): + row = page[start:end] + title_match = _PIXABAY_TITLE.search(row) + if not title_match: + continue + link = html_lib.unescape(title_match.group(1)) + title = _plain(title_match.group(2), 255) + if not title: + continue + artist = _PIXABAY_ARTIST.search(row) + duration = _PIXABAY_DURATION.search(row) + genre = _PIXABAY_GENRE.search(row) + tag_block = _PIXABAY_TAGS.search(row) + tags = ( + [html_lib.unescape(t) for t in re.findall(r">([^<]+)", tag_block.group(1))] + if tag_block + else [] + ) + id_match = re.search(r"-(\d+)/?$", link) + track = PageTrack( + audio_url=None, + title=title, + creator=_plain(artist.group(1), MAX_TEXT) if artist else None, + publisher="Pixabay", + duration_seconds=_clock_seconds(duration.group(1)) if duration else None, + license=_PIXABAY_LICENSE, + source_url=urljoin("https://pixabay.com/", link), + genres=music.clean_terms([html_lib.unescape(genre.group(1))]) if genre else [], + tags=music.clean_terms(tags), + match_id=id_match.group(1) if id_match else None, + ) + if kind == "mood": + track.moods = music.clean_terms([*track.moods, label]) + elif kind == "genre": + track.genres = music.clean_terms([*track.genres, label]) + tracks.append(track) + return tracks + + +def _pixabay_page_filter(page_url: str) -> tuple[Optional[str], str]: + """`/music/search/mood/mysterious/` says every track listed is mysterious.""" + parts = [p for p in urlsplit(page_url).path.split("/") if p] + if len(parts) >= 4 and parts[:2] == ["music", "search"] and parts[2] in ("mood", "genre"): + return parts[2], unquote(parts[3]).replace("-", " ").title() + return None, "" + + # ─── incompetech ───────────────────────────────────────────────────────────── # Kevin MacLeod's catalogue. The page renders nothing itself: its script fetches diff --git a/backend/app/routers/imports.py b/backend/app/routers/imports.py index 9682d39..8a9e4fb 100644 --- a/backend/app/routers/imports.py +++ b/backend/app/routers/imports.py @@ -10,7 +10,7 @@ from typing import List, Optional -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status from pydantic import BaseModel, Field from sqlmodel import Session @@ -18,7 +18,8 @@ from app.database import get_session from app.enrichment import page_import from app.enrichment.import_url import ImportRequest -from app.ingest.page_tracks import PageTrack +from app.ingest import page_tracks, web +from app.ingest.page_tracks import PageExtraction, PageTrack from app.ingest.ytdlp import MAX_URL, normalise_url from app.jobs import enrichment as enrichment_jobs from app.jobs.registry import KINDS @@ -80,10 +81,12 @@ def start_url_import( # ─── tracks from a music library's page ────────────────────────────────────── -class PageTrackBody(BaseModel): - """One track, as the preview found it and the person corrected it.""" +class PageTrackFields(BaseModel): + """One track, as a page describes it.""" - audio_url: str = Field(min_length=1, max_length=MAX_URL) + # Optional here because a saved page may have no audio link (Pixabay's download is a + # button); `PageTrackBody` requires it, since a live import downloads from it. + audio_url: Optional[str] = Field(default=None, max_length=MAX_URL) title: str = Field(min_length=1, max_length=255) creator: Optional[str] = Field(default=None, max_length=500) publisher: Optional[str] = Field(default=None, max_length=500) @@ -99,9 +102,16 @@ class PageTrackBody(BaseModel): instruments: List[str] = Field(default_factory=list, max_length=20) tags: List[str] = Field(default_factory=list, max_length=50) bpm: Optional[float] = None + match_id: Optional[str] = Field(default=None, max_length=64) + + +class PageTrackBody(PageTrackFields): + """One track to download, as the preview found it and the person corrected it.""" + + audio_url: str = Field(min_length=1, max_length=MAX_URL) -class PageTrackRead(PageTrackBody): +class PageTrackRead(PageTrackFields): # The asset this library already holds for the track, so the review list can show it # as imported rather than offering it twice. existing_asset_id: Optional[str] = None @@ -154,25 +164,25 @@ def preview_page_import( except page_import.PageImportError as exc: raise _page_error(exc) from exc - existing = page_import.already_imported(session, user.id, extraction.tracks) - similar = page_import.similar_in_library(session, user.id, extraction.tracks) - return DataResponse( - data=PagePreview( - page_url=extraction.page_url, - method=extraction.method, - source=extraction.source, - note=extraction.note, - tracks=[ - PageTrackRead( - **t.to_dict(), - existing_asset_id=existing.get(t.audio_url), - similar_asset_id=( - None if t.audio_url in existing else similar.get(t.audio_url) - ), - ) - for t in extraction.tracks - ], - ) + return DataResponse(data=_preview_of(session, user.id, extraction)) + + +def _preview_of(session: Session, user_id: str, extraction: PageExtraction) -> PagePreview: + existing = page_import.already_imported(session, user_id, extraction.tracks) + similar = page_import.similar_in_library(session, user_id, extraction.tracks) + return PagePreview( + page_url=extraction.page_url, + method=extraction.method, + source=extraction.source, + note=extraction.note, + tracks=[ + PageTrackRead( + **t.to_dict(), + existing_asset_id=existing.get(t.key), + similar_asset_id=None if t.key in existing else similar.get(t.key), + ) + for t in extraction.tracks + ], ) @@ -210,3 +220,77 @@ def start_page_import( ) jobs.append(KINDS["enrichment"].to_activity(job)) return DataResponse(data=jobs) + + +# ─── from a saved page ─────────────────────────────────────────────────────── + + +class SavedFileRead(BaseModel): + filename: str + # imported | duplicate | unmatched | rejected + status: str + asset_id: Optional[str] = None + track_title: Optional[str] = None + message: str = "" + + +class SavedImport(BaseModel): + page_url: str + source: str + results: List[SavedFileRead] + + +def _read_page(page: UploadFile) -> str: + """The saved page as text, refusing one too large to be a page.""" + data = page.file.read(web.MAX_PAGE_BYTES + 1) + if len(data) > web.MAX_PAGE_BYTES: + raise _page_error(Exception("The saved page is too large to read")) + return data.decode("utf-8", errors="replace") + + +@router.post("/import-page/saved/preview", response_model=DataResponse[PagePreview]) +def preview_saved_page( + user: CurrentUser, + page: UploadFile = File(...), + session: Session = Depends(get_session), +) -> DataResponse[PagePreview]: + """List the tracks on a page the person saved in their own browser. + + For sites that refuse GAM's own request (Pixabay). Reads only the uploaded file. + """ + extraction = page_tracks.extract_saved(_read_page(page)) + return DataResponse(data=_preview_of(session, user.id, extraction)) + + +@router.post("/import-page/saved", response_model=DataResponse[SavedImport]) +def import_saved_page( + user: CurrentUser, + page: UploadFile = File(...), + files: List[UploadFile] = File(...), + session: Session = Depends(get_session), +) -> DataResponse[SavedImport]: + """Catalogue downloaded tracks with what their saved page says about each. + + Synchronous, as an upload is: the bytes are already here, and there is nothing to + wait on but writing them. + """ + if len(files) > page_import.MAX_TRACKS_PER_IMPORT: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "too_many_files", + "message": f"Import at most {page_import.MAX_TRACKS_PER_IMPORT} files at a time", + }, + ) + capacity.require_room() + extraction = page_tracks.extract_saved(_read_page(page)) + results = page_import.import_saved( + session, user.id, extraction, [(f.filename or "track", f.file) for f in files] + ) + return DataResponse( + data=SavedImport( + page_url=extraction.page_url, + source=extraction.source, + results=[SavedFileRead(**r.__dict__) for r in results], + ) + ) diff --git a/backend/tests/fixtures/pages/pixabay_saved.html b/backend/tests/fixtures/pages/pixabay_saved.html new file mode 100644 index 0000000..6730ec6 --- /dev/null +++ b/backend/tests/fixtures/pages/pixabay_saved.html @@ -0,0 +1,11 @@ + + + +
Investigationleberch
4:24
Crime Scene
Investigation
+
Dark AmbientAtlasAudio
1:34
Ambient
Dark Ambient
+
Riserleberch
2:04
Ambient
RiserRiser Music
+
Tension Riserleberch
2:01
Ambient
Tension Riser
+ diff --git a/backend/tests/test_saved_page_import.py b/backend/tests/test_saved_page_import.py new file mode 100644 index 0000000..48857aa --- /dev/null +++ b/backend/tests/test_saved_page_import.py @@ -0,0 +1,192 @@ +"""Importing tracks from a page the person saved, with the files they downloaded. + +For sites that refuse GAM's own requests (Pixabay answers a server with 403). Nothing +here touches the network: the tests hand in a saved page and some audio files, the way +the person would. +""" + +import json +import shutil +from pathlib import Path + +from sqlmodel import select + +from app import music +from app.ingest import page_tracks +from app.models.asset import PROVENANCE_EMBEDDED, Asset +from app.models.job import KIND_ANALYZE_MUSIC, KIND_TRANSCRIBE, EnrichmentJob +from app.settings_store import DEEPGRAM_API_KEY, set_setting + +PAGES = Path(__file__).parent / "fixtures" / "pages" +FIXTURES = Path(__file__).parent / "fixtures" +TEST_USER = "user-under-test" + + +def _page(name: str) -> str: + return (PAGES / name).read_text(encoding="utf-8") + + +def _audio(tmp_path: Path, name: str, source: str = "sample_audio.mp3") -> Path: + target = tmp_path / name + shutil.copy(FIXTURES / source, target) + return target + + +def _post(client, page: str, files: list[Path]): + upload = [("page", ("saved.html", (PAGES / page).read_bytes(), "text/html"))] + upload += [("files", (f.name, f.read_bytes(), "audio/mpeg")) for f in files] + return client.post("/api/assets/import-page/saved", files=upload) + + +# ─── reading the saved page ────────────────────────────────────────────────── + + +def test_a_saved_pixabay_page_gives_each_track_and_its_id(): + found = page_tracks.extract_saved(_page("pixabay_saved.html")) + + assert found.source == "pixabay" + assert found.page_url == "https://pixabay.com/music/search/mood/mysterious/?pagi=2" + first = found.tracks[0] + assert first.title == "Investigation" + assert first.audio_url is None + assert first.match_id == "604094" + assert first.creator == "leberch" and first.publisher == "Pixabay" + assert first.source_url == "https://pixabay.com/music/crime-scene-investigation-604094/" + assert first.duration_seconds == 264 + assert first.genres == ["Crime Scene"] + # The listing is the "mysterious" mood, so every track on it is. + assert first.moods == ["Mysterious"] + assert first.tags == ["Investigation"] + assert "Pixabay Content License" in first.license + + +def test_a_saved_mixkit_page_works_without_the_network(): + """A saved copy has no canonical link here, so the JSON-LD still has to carry it.""" + found = page_tracks.extract_saved(_page("mixkit_mood.html")) + assert [t.title for t in found.tracks] == ["I'm Still Waitin'", "Bigger than Texas"] + + +def test_a_saved_incompetech_page_says_what_it_needs(): + page = _page("incompetech_music.html").replace( + "", + '', + ) + found = page_tracks.extract_saved(page) + assert found.tracks == [] + assert "pieces.json" in found.note + + +def test_the_page_address_comes_from_the_comment_chrome_writes(): + page = "\n" + assert page_tracks.saved_page_url(page) == "https://example.org/music/list.html" + + +# ─── matching files to tracks ──────────────────────────────────────────────── + + +def test_files_match_by_id_then_by_title(): + tracks = page_tracks.extract_saved(_page("pixabay_saved.html")).tracks + matched = page_tracks.match_files( + tracks, + [ + "leberch-investigation-604094.mp3", # by Pixabay's id + "Dark Ambient.mp3", # by title + "my tension riser copy.mp3", # the longer title, not "Riser" + "riser.mp3", + "something else.mp3", + ], + ) + titles = {name: (t.title if t else None) for name, t in matched.items()} + assert titles == { + "leberch-investigation-604094.mp3": "Investigation", + "Dark Ambient.mp3": "Dark Ambient", + "my tension riser copy.mp3": "Tension Riser", + "riser.mp3": "Riser", + "something else.mp3": None, + } + + +def test_one_track_is_matched_to_one_file(): + tracks = page_tracks.extract_saved(_page("pixabay_saved.html")).tracks + matched = page_tracks.match_files(tracks, ["Dark Ambient.mp3", "Dark Ambient (1).mp3"]) + assert [t and t.title for t in matched.values()] == ["Dark Ambient", None] + + +# ─── the API ───────────────────────────────────────────────────────────────── + + +def test_the_preview_lists_the_tracks(library): + response = library.post( + "/api/assets/import-page/saved/preview", + files=[("page", ("saved.html", (PAGES / "pixabay_saved.html").read_bytes(), "text/html"))], + ) + body = response.json()["data"] + assert body["source"] == "pixabay" and len(body["tracks"]) == 4 + assert body["tracks"][0]["match_id"] == "604094" + assert body["tracks"][0]["audio_url"] is None + + +def test_downloaded_files_are_imported_with_their_page_details(library, session, tmp_path): + set_setting(session, TEST_USER, DEEPGRAM_API_KEY, "dg-test-key") + from tests import music_synth as synth + + # A real-length track, so the analysis is queued for what the page left out. + track = synth.write_mp3( + synth.song(100, 0, "major", seconds=12), tmp_path / "leberch-investigation-604094.mp3" + ) + response = _post(library, "pixabay_saved.html", [track]) + + [result] = response.json()["data"]["results"] + assert result["status"] == "imported" and result["track_title"] == "Investigation" + asset = session.get(Asset, result["asset_id"]) + assert asset.name == "Investigation" + assert asset.creator == "leberch" and asset.publisher == "Pixabay" + assert asset.source_url == "https://pixabay.com/music/crime-scene-investigation-604094/" + assert music.decode_terms(asset.music_moods) == ["Mysterious"] + assert json.loads(asset.field_provenance)["music_genres"] == PROVENANCE_EMBEDDED + assert asset.audio_kind == "music" + + jobs = { + j.kind + for j in session.exec(select(EnrichmentJob).where(EnrichmentJob.asset_id == asset.id)).all() + } + assert KIND_TRANSCRIBE not in jobs and KIND_ANALYZE_MUSIC in jobs + + +def test_a_file_not_on_the_page_is_imported_without_details(library, session, tmp_path): + response = _post(library, "pixabay_saved.html", [_audio(tmp_path, "my own.mp3")]) + [result] = response.json()["data"]["results"] + assert result["status"] == "unmatched" + asset = session.get(Asset, result["asset_id"]) + assert asset.audio_kind == "music" and asset.creator is None + + +def test_a_file_already_in_the_library_is_not_stored_twice(library, session, tmp_path): + files = [("files", ("mine.mp3", (FIXTURES / "sample_audio.mp3").read_bytes(), "audio/mpeg"))] + mine = library.post("/api/assets", files=files).json()["created"][0] + + response = _post(library, "pixabay_saved.html", [_audio(tmp_path, "Dark Ambient.mp3")]) + + [result] = response.json()["data"]["results"] + assert result["status"] == "duplicate" and result["asset_id"] == mine["id"] + assert len(session.exec(select(Asset)).all()) == 1 + # The page still fills the copy's blanks. + assert session.get(Asset, mine["id"]).creator == "AtlasAudio" + + +def test_a_file_that_is_not_audio_is_refused(library, session, tmp_path): + notes = tmp_path / "notes.txt" + notes.write_text("not a track") + [result] = _post(library, "pixabay_saved.html", [notes]).json()["data"]["results"] + assert result["status"] == "rejected" + assert session.exec(select(Asset)).all() == [] + + +def test_too_many_files_at_once_are_refused(library, tmp_path): + from app.enrichment import page_import + + track = _audio(tmp_path, "t.mp3") + response = _post(library, "pixabay_saved.html", [track] * (page_import.MAX_TRACKS_PER_IMPORT + 1)) + assert response.status_code == 400 + assert response.json()["detail"]["code"] == "too_many_files" diff --git a/docs/page-import.md b/docs/page-import.md index 316cb07..4ad3c01 100644 --- a/docs/page-import.md +++ b/docs/page-import.md @@ -54,6 +54,45 @@ Caught three ways, because each misses something the others catch: A track ticked twice in one import is one job. +## Saved pages (sites that block GAM) + +Some sites refuse requests that do not come from a browser: Pixabay answers GAM's fetch +with 403 and its terms rule out automated access. GAM does not disguise itself to get +past that. The person does the part that needs a browser instead, under **Add → "Or use +a saved page"**: + +1. Open the page and save it (File → Save Page As). +2. Download the tracks with the site's own Download button. +3. Choose the saved page and the downloaded files. GAM lists the page's tracks, shows + which file matched which, and imports them. + +GAM never contacts the site. `page_tracks.extract_saved` reads only the uploaded page: + +- **The page's address** comes from its canonical link, `og:url`, or the + `` comment Chrome writes, since adapters key on host and path. +- **Pixabay:** one rendered row per track. Its class names carry build hashes + (`audioRow--et3ac`), so only their stable prefixes are matched. Each row gives title, + artist, track page, duration, genre and tags, plus the numeric ID at the end of the + track link. The listing's `/mood//` or `/genre//` applies to every track. + Credited "Pixabay" under the Pixabay Content License. +- **Other sites:** the same structured adapters as a live import, without the network, so + a saved Mixkit page works too. incompetech's page only points to `pieces.json`, so a + saved copy says to use the link instead. + +**Matching files to tracks** (`page_tracks.match_files`, mirrored in the browser by +`utils/matchFiles.ts` for the preview): by the site's ID in the filename first (Pixabay +names a download `artist-title-604094.mp3`), then by title contained in the filename, +longest title first ("Tension Riser" before "Riser"), with one file per track. + +**Importing** (`page_import.import_saved`), synchronously like an upload: +- Every file is stored as **music**, so it is never transcribed, and is queued for + tempo/key/energy analysis. +- A matched file gets the page's details, stamped `embedded`. +- An unmatched file is still imported and reported as "not on the page". +- A file whose checksum is already in the library is not stored again; the page only + fills that copy's blanks. +- Non-audio files are refused, and the limit is 50 files per import. + ## Downloading `backend/app/ingest/web.py`: an SSRF check on every redirect hop, uncompressed bodies diff --git a/frontend/src/api/imports.ts b/frontend/src/api/imports.ts index 553240d..0cd4262 100644 --- a/frontend/src/api/imports.ts +++ b/frontend/src/api/imports.ts @@ -58,6 +58,30 @@ export interface PagePreview { tracks: PageTrack[] } +/** A track read from a page the person saved. It may have no audio URL (Pixabay's + * download is a button); `match_id` is the site's id, which its downloaded filenames + * carry. */ +export type SavedPageTrack = Omit & { + audio_url: string | null + match_id: string | null +} + +export type SavedPagePreview = Omit & { tracks: SavedPageTrack[] } + +export interface SavedFileResult { + filename: string + status: 'imported' | 'duplicate' | 'unmatched' | 'rejected' + asset_id: string | null + track_title: string | null + message: string +} + +export interface SavedImport { + page_url: string + source: string + results: SavedFileResult[] +} + /** The most tracks one import may queue; the server refuses more. */ export const MAX_TRACKS_PER_IMPORT = 50 @@ -65,7 +89,39 @@ interface DataResponse { data: T } +function multipart(page: File, files: File[] = []): FormData { + const body = new FormData() + body.append('page', page) + files.forEach((file) => body.append('files', file)) + return body +} + +// Content-Type left to the browser, which must add the multipart boundary itself. +const MULTIPART = { headers: { 'Content-Type': undefined } } + export const importsApi = { + /** Read a page the person saved in their browser. Uploads only the page. */ + previewSavedPage(page: File): Promise { + return client + .post>( + '/assets/import-page/saved/preview', + multipart(page), + MULTIPART + ) + .then((r) => r.data.data) + }, + + /** Import downloaded tracks with the details their saved page gives. */ + importSavedPage(page: File, files: File[]): Promise { + return client + .post>( + '/assets/import-page/saved', + multipart(page, files), + MULTIPART + ) + .then((r) => r.data.data) + }, + /** Read a music library's page and list its tracks. Downloads nothing. */ previewPage(url: string, useAi = true): Promise { return client diff --git a/frontend/src/components/AddPanel.tsx b/frontend/src/components/AddPanel.tsx index 5334c4f..a6c1c25 100644 --- a/frontend/src/components/AddPanel.tsx +++ b/frontend/src/components/AddPanel.tsx @@ -1,6 +1,7 @@ import { X } from 'lucide-react' import GenerateForm from '@/components/GenerateForm' import PageImport from '@/components/PageImport' +import SavedPageImport from '@/components/SavedPageImport' import UploadZone from '@/components/UploadZone' import UrlImport from '@/components/UrlImport' @@ -57,6 +58,7 @@ export default function AddPanel({ open, onClose }: Props) { + diff --git a/frontend/src/components/SavedPageImport.test.tsx b/frontend/src/components/SavedPageImport.test.tsx new file mode 100644 index 0000000..2c007ef --- /dev/null +++ b/frontend/src/components/SavedPageImport.test.tsx @@ -0,0 +1,132 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import SavedPageImport from '@/components/SavedPageImport' +import { importsApi, type SavedPagePreview, type SavedPageTrack } from '@/api/imports' +import { useLibraryStore } from '@/stores/library' +import { matchFiles } from '@/utils/matchFiles' + +function track(overrides: Partial = {}): SavedPageTrack { + return { + audio_url: null, + title: 'Investigation', + creator: 'leberch', + publisher: 'Pixabay', + source_title: null, + description: null, + duration_seconds: 264, + published_date: null, + license: null, + credit_line: null, + source_url: 'https://pixabay.com/music/crime-scene-investigation-604094/', + genres: ['Crime Scene'], + moods: ['Mysterious'], + instruments: [], + tags: ['Investigation'], + bpm: null, + match_id: '604094', + ...overrides, + } +} + +const riser = track({ title: 'Riser', match_id: '590947' }) +const tensionRiser = track({ title: 'Tension Riser', match_id: '590425' }) + +function preview(tracks: SavedPageTrack[]): SavedPagePreview { + return { + page_url: 'https://pixabay.com/music/search/mood/mysterious/', + method: 'structured', + source: 'pixabay', + note: '', + tracks, + } +} + +const load = vi.fn().mockResolvedValue(undefined) + +beforeEach(() => { + load.mockClear() + useLibraryStore.setState({ load }) +}) + +afterEach(() => vi.restoreAllMocks()) + +const savedPage = new File([''], 'page.html', { type: 'text/html' }) +const audio = (name: string) => new File(['x'], name, { type: 'audio/mpeg' }) + +describe('matchFiles', () => { + it('matches by id first, then by the longest title', () => { + const matched = matchFiles( + [track(), riser, tensionRiser], + [ + 'leberch-investigation-604094.mp3', + 'my tension riser.mp3', + 'riser.mp3', + 'other.mp3', + ] + ) + expect([...matched.values()].map((t) => t?.title ?? null)).toEqual([ + 'Investigation', + 'Tension Riser', + 'Riser', + null, + ]) + }) +}) + +describe('SavedPageImport', () => { + it('shows which track each file is before uploading, then imports them', async () => { + vi.spyOn(importsApi, 'previewSavedPage').mockResolvedValue(preview([track(), riser])) + const start = vi.spyOn(importsApi, 'importSavedPage').mockResolvedValue({ + page_url: 'https://pixabay.com/music/search/mood/mysterious/', + source: 'pixabay', + results: [ + { + filename: 'leberch-investigation-604094.mp3', + status: 'imported', + asset_id: 'a1', + track_title: 'Investigation', + message: '', + }, + { + filename: 'other.mp3', + status: 'unmatched', + asset_id: 'a2', + track_title: null, + message: 'Imported, but not found on the page', + }, + ], + }) + const user = userEvent.setup() + render() + + await user.upload(screen.getByLabelText('Saved page'), savedPage) + expect(await screen.findByText(/2 tracks on the page/)).toBeInTheDocument() + + const files = [audio('leberch-investigation-604094.mp3'), audio('other.mp3')] + await user.upload(screen.getByLabelText('Downloaded tracks'), files) + expect(screen.getByText('→ Investigation · leberch')).toBeInTheDocument() + expect(screen.getByText('Not on the page')).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Import 2 files' })) + + expect(start).toHaveBeenCalledWith(savedPage, files) + expect(await screen.findByText('Imported')).toBeInTheDocument() + expect( + screen.getByText(/other\.mp3 — Imported, but not found on the page/) + ).toBeInTheDocument() + expect(load).toHaveBeenCalled() + }) + + it('says when the page cannot be read', async () => { + vi.spyOn(importsApi, 'previewSavedPage').mockRejectedValue( + new Error('The saved page is too large to read') + ) + const user = userEvent.setup() + render() + await user.upload(screen.getByLabelText('Saved page'), savedPage) + expect(await screen.findByRole('alert')).toHaveTextContent( + 'The saved page is too large to read' + ) + }) +}) diff --git a/frontend/src/components/SavedPageImport.tsx b/frontend/src/components/SavedPageImport.tsx new file mode 100644 index 0000000..dd67d55 --- /dev/null +++ b/frontend/src/components/SavedPageImport.tsx @@ -0,0 +1,215 @@ +import { useMemo, useState } from 'react' +import { FileDown, Loader2 } from 'lucide-react' +import { apiErrorMessage } from '@/api/client' +import { + importsApi, + MAX_TRACKS_PER_IMPORT, + type SavedFileResult, + type SavedPagePreview, +} from '@/api/imports' +import { useLibraryStore } from '@/stores/library' +import { matchFiles } from '@/utils/matchFiles' + +/** + * Tracks from a site that refuses GAM's own requests — Pixabay answers a server with 403. + * + * The person does the part that needs a browser: open the page, save it, download the + * tracks they want. GAM reads the saved page for each track's details and matches the + * downloads to it, and never contacts the site. Matching is shown before uploading, from + * a mirror of the server's rule, so a wrong or missing match is seen while it can still be + * fixed by renaming a file; the server's own match is the one that counts. + */ + +const RESULT_LABELS: Record = { + imported: 'Imported', + duplicate: 'Already in library', + unmatched: 'Not on the page', + rejected: 'Not imported', +} + +export default function SavedPageImport() { + const reload = useLibraryStore((s) => s.load) + + const [page, setPage] = useState(null) + const [files, setFiles] = useState([]) + const [preview, setPreview] = useState(null) + const [reading, setReading] = useState(false) + const [importing, setImporting] = useState(false) + const [results, setResults] = useState(null) + const [error, setError] = useState(null) + + const matches = useMemo( + () => + preview + ? matchFiles( + preview.tracks, + files.map((f) => f.name) + ) + : new Map(), + [preview, files] + ) + const tooMany = files.length > MAX_TRACKS_PER_IMPORT + + const choosePage = async (file: File | undefined) => { + setPage(file ?? null) + setPreview(null) + setResults(null) + setError(null) + if (!file) return + setReading(true) + try { + setPreview(await importsApi.previewSavedPage(file)) + } catch (err) { + setError(apiErrorMessage(err, 'Could not read that saved page')) + } finally { + setReading(false) + } + } + + const startImport = async () => { + if (!page || files.length === 0 || tooMany || importing) return + setImporting(true) + setError(null) + try { + const done = await importsApi.importSavedPage(page, files) + setResults(done.results) + setFiles([]) + void reload() + } catch (err) { + setError(apiErrorMessage(err, 'Could not import the files')) + } finally { + setImporting(false) + } + } + + return ( +
+

+

+
    +
  1. Open the page in your browser and save it (File → Save Page As).
    2. +
  2. Download the tracks you want with the site’s Download button.
    3. +
  3. Choose both here. GAM fills in each track’s details from the page.
    4. +
+ +
+ + +
+ + {reading && ( +

+ + Reading the page… +

+ )} + {preview && ( +

+ {preview.tracks.length} track{preview.tracks.length === 1 ? '' : 's'} on the + page + {preview.note ? `. ${preview.note}` : '.'} +

+ )} + + {preview && files.length > 0 && ( +
    + {files.map((file) => { + const track = matches.get(file.name) + return ( +
  • + + {file.name} + + {track ? ( + + → {track.title} + {track.creator ? ` · ${track.creator}` : ''} + + ) : ( + + Not on the page + + )} +
    • + ) + })} +
+ )} + + {tooMany && ( +

+ Choose at most {MAX_TRACKS_PER_IMPORT} files at a time. +

+ )} + + {page && files.length > 0 && ( +
+ +
+ )} + + {results && ( +
    + {results.map((result, index) => ( +
  • + + {RESULT_LABELS[result.status]} + + + {' '} + · {result.track_title ?? result.filename} + {result.message ? ` — ${result.message}` : ''} + +
    • + ))} +
+ )} + + {error && ( +

+ {error} +

+ )} +
+ ) +} diff --git a/frontend/src/utils/matchFiles.ts b/frontend/src/utils/matchFiles.ts new file mode 100644 index 0000000..d00a878 --- /dev/null +++ b/frontend/src/utils/matchFiles.ts @@ -0,0 +1,64 @@ +/** + * Which saved-page track each downloaded file is, for showing before upload. + * + * A mirror of `page_tracks.match_files` on the server, which is the one that counts: by + * the site's id in the filename first (Pixabay names a download `artist-title-604094.mp3`), + * then by title contained in the filename, longest title first, one file per track. + */ + +interface Matchable { + title: string + match_id: string | null +} + +function normalise(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() +} + +function stem(filename: string): string { + const dot = filename.lastIndexOf('.') + return dot > 0 ? filename.slice(0, dot) : filename +} + +export function matchFiles( + tracks: T[], + filenames: string[] +): Map { + const result = new Map(filenames.map((name) => [name, null])) + const taken = new Set() + const byId = new Map() + tracks.forEach((track, index) => { + if (track.match_id) byId.set(track.match_id, index) + }) + + for (const name of filenames) { + for (const number of stem(name).match(/\d+/g) ?? []) { + const index = byId.get(number) + if (index !== undefined && !taken.has(index)) { + result.set(name, tracks[index]) + taken.add(index) + break + } + } + } + + const titled = tracks + .map((track, index) => ({ index, title: normalise(track.title) })) + .filter((entry) => entry.title) + .sort((a, b) => b.title.length - a.title.length) + for (const name of filenames) { + if (result.get(name)) continue + const padded = ` ${normalise(stem(name))} ` + for (const { index, title } of titled) { + if (!taken.has(index) && padded.includes(` ${title} `)) { + result.set(name, tracks[index]) + taken.add(index) + break + } + } + } + return result +}