From d1e85afe57e5ae79f666f84ed16f46cef2fd37f8 Mon Sep 17 00:00:00 2001 From: Matt Conflitti Date: Tue, 17 Nov 2026 15:47:47 -0500 Subject: [PATCH 1/4] Add optional data dictionaries and refresh Connect pin previews --- .github/workflows/ci.yml | 17 + examples/data-dict/README.md | 44 +++ examples/data-dict/create_bundle.py | 86 +++++ examples/data-dict/data-dict.yaml | 25 ++ pins/boards.py | 102 ++++-- pins/data_dict.py | 159 +++++++++ pins/rsconnect/html/index.html | 170 ++++++---- pins/rsconnect/html/pin.css | 102 ++++++ pins/rsconnect/html/pin.js | 45 +++ pins/rsconnect/html/theme.js | 29 ++ .../index.html | 155 ++++----- .../pin.css | 102 ++++++ .../pin.js | 45 +++ .../theme.js | 29 ++ pins/tests/test_connect_preview.py | 125 +++++++ pins/tests/test_data_dict.py | 313 ++++++++++++++++++ pyproject.toml | 1 + 17 files changed, 1377 insertions(+), 172 deletions(-) create mode 100644 examples/data-dict/README.md create mode 100644 examples/data-dict/create_bundle.py create mode 100644 examples/data-dict/data-dict.yaml create mode 100644 pins/data_dict.py create mode 100644 pins/rsconnect/html/pin.css create mode 100644 pins/rsconnect/html/pin.js create mode 100644 pins/rsconnect/html/theme.js create mode 100644 pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.css create mode 100644 pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.js create mode 100644 pins/tests/_snapshots/test_board_pin_write_rsc_index_html/theme.js create mode 100644 pins/tests/test_connect_preview.py create mode 100644 pins/tests/test_data_dict.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d24baee9..0ac04ed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,23 @@ jobs: # https://github.com/browsertron/pytest-parallel/issues/93 no_proxy: "*" + test-data-dict: + name: "Test data dictionary CLI" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e '.[data-dict]' pytest + - name: Run data dictionary tests + run: pytest pins/tests/test_data_dict.py + env: + PINS_REQUIRE_DATA_DICT_CLI: "1" + test-connect: name: "Test Posit Connect" runs-on: ubuntu-latest diff --git a/examples/data-dict/README.md b/examples/data-dict/README.md new file mode 100644 index 00000000..183a09e0 --- /dev/null +++ b/examples/data-dict/README.md @@ -0,0 +1,44 @@ +# Deploy a data dictionary pin + +This example prepares a pin bundle from synthetic data for Posit Connect. +The publisher runs the `data-dict` CLI while preparing the bundle and stores the rendered dictionary with the pin. +The combined entrypoint keeps the standard pin preview available alongside the dictionary view. +The `create_bundle.py` script uses `CONNECT_SERVER`, which defaults to `https://connect.example.com`, so it can build the bundle offline. +The rsconnect-python package publishes the bundle through the saved account named by `CONNECT_ACCOUNT`. + +Run from the repository root: + +```{.bash filename="Terminal"} +uv run --no-project --with-editable '.[data-dict]' python examples/data-dict/create_bundle.py /tmp/pins-python-data-dict-bundle --dictionary examples/data-dict/data-dict.yaml +uv tool run --from rsconnect-python rsconnect list +env -u CONNECT_SERVER uv tool run --from rsconnect-python rsconnect deploy manifest /tmp/pins-python-data-dict-bundle/manifest.json --name "$CONNECT_ACCOUNT" --new --title 'Pins Python data dictionary' --no-metadata +``` + +The output directory must be empty. +Set `CONNECT_ACCOUNT` to a saved account name before publishing, and register the account with `rsconnect login` if needed. +The deployment uses the server's default private content access. + +To build a regular pin preview without a dictionary, run: + +```{.bash filename="Terminal"} +uv run --no-project --with-editable . --with pyarrow python examples/data-dict/create_bundle.py /tmp/pins-python-preview-bundle --no-dictionary +``` + +Supply `--dictionary /path/to/data-dict.yaml` to attach an authored dictionary. +Omit `--dictionary` to generate a draft. +The dictionary must omit table `source` fields so rendering does not depend on files on the publisher's machine. +Attaching an authored dictionary to a folder board does not require the CLI. +Drafting and Connect rendering require the optional dependency or a `data-dict` executable on `PATH`. +For a pin with one Parquet file and one dictionary table, rendering profiles the pinned data and embeds column statistics in the dictionary view. +The stored dictionary omits source paths; the renderer adds the bundle-relative Parquet path to a temporary copy. +You can also supply YAML text in `metadata["data_dict"]`, using one dictionary input per call. + +For an authenticated Connect board, use the public API: + +```{.python filename="example.py"} +board.pin_write(data, name="username/pin-name", type="parquet", data_dict=True) +board.pin_write(data, name="username/pin-name", type="parquet", data_dict="data-dict.yaml", force_identical_write=True) +``` + +Dictionary-only changes need `force_identical_write=True` because pins compares data hashes when deciding whether to write. +Generated dictionaries contain `todo` notes for human decisions; review the draft before relying on it. diff --git a/examples/data-dict/create_bundle.py b/examples/data-dict/create_bundle.py new file mode 100644 index 00000000..bfb88787 --- /dev/null +++ b/examples/data-dict/create_bundle.py @@ -0,0 +1,86 @@ +"""Build a pin bundle from this checkout for deployment with rsconnect-python.""" + +import argparse +import json +import os +import tempfile +from pathlib import Path + +import pandas as pd +import yaml + +from pins import board_connect, board_folder +from pins.rsconnect.fs import PinBundleManifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + dictionary_options = parser.add_mutually_exclusive_group() + dictionary_options.add_argument("--dictionary", type=Path) + dictionary_options.add_argument("--no-dictionary", action="store_true") + args = parser.parse_args() + if args.no_dictionary: + data_dict = None + elif args.dictionary is not None: + data_dict = args.dictionary + else: + data_dict = True + output = args.output.resolve() + if output.exists() and any(output.iterdir()): + parser.error("The output directory must be empty.") + + data = pd.DataFrame( + {"id": [1, 2, 3], "value": [10.0, 20.0, 30.0], "group": ["a", "b", "a"]} + ) + with tempfile.TemporaryDirectory() as temporary: + local = board_folder(temporary) + meta = local.pin_write( + data, + "dictionary-example", + type="parquet", + metadata={"example_language": "Python"}, + data_dict=data_dict, + ) + pd.testing.assert_frame_equal(local.pin_read("dictionary-example"), data) + assert meta.user["example_language"] == "Python" + dictionary = meta.user.get("data_dict") + if dictionary is not None: + assert all( + "source" not in table for table in yaml.safe_load(dictionary)["tables"] + ) + + # Prepare the normal Connect bundle offline; the deployment CLI handles OAuth. + connect = board_connect( + server_url=os.environ.get("CONNECT_SERVER", "https://connect.example.com"), + api_key="unused-offline", + cache=None, + ) + output.mkdir(parents=True, exist_ok=True) + prepared = connect.prepare_pin_version( + output, + data, + "example/data-dict-python", + type="parquet", + title="Pins Python data dictionary", + metadata=meta.user, + ) + PinBundleManifest.add_manifest_to_directory(output) + uploaded = yaml.safe_load((output / "data.txt").read_text()) + if dictionary is None: + assert "data_dict" not in uploaded["user"] + else: + assert uploaded["user"]["data_dict"] == dictionary + pd.testing.assert_frame_equal(pd.read_parquet(output / prepared.file), data) + manifest = json.loads((output / "manifest.json").read_text()) + assert manifest["metadata"]["primary_html"] == "index.html" + if dictionary is not None: + assert "Data dictionary" in (output / "index.html").read_text() + assert "unused-offline" not in "".join( + p.read_text() for p in output.glob("*.txt") + ) + print(output / "manifest.json") + + +if __name__ == "__main__": + main() diff --git a/examples/data-dict/data-dict.yaml b/examples/data-dict/data-dict.yaml new file mode 100644 index 00000000..6a173395 --- /dev/null +++ b/examples/data-dict/data-dict.yaml @@ -0,0 +1,25 @@ +$version: 0.1.0 +$learn_more: https://data-dict.tidyverse.org/ +label: Pins sample data +description: >- + Three synthetic observations used to demonstrate publishing a data dictionary + alongside a Parquet pin from R and Python. +tables: + - name: data + columns: + - name: id + type: number(id) + description: Unique identifier for a synthetic observation. + constraints: [primary_key] + examples: [1, 2, 3] + - name: value + type: number + description: Synthetic numeric value used to demonstrate a numeric column. + constraints: [required] + range: [10, 30] + examples: [10, 20, 30] + - name: group + type: string + description: Synthetic grouping label; a and b identify the two example groups. + constraints: [required] + examples: [a, b] diff --git a/pins/boards.py b/pins/boards.py index d19b19d0..f1b8cf67 100644 --- a/pins/boards.py +++ b/pins/boards.py @@ -3,6 +3,7 @@ import contextlib import functools import inspect +import json import logging import re import shutil @@ -12,6 +13,7 @@ from io import IOBase from pathlib import Path from typing import Any, Protocol +from urllib.parse import quote from importlib_resources import files from importlib_resources.abc import Traversable @@ -233,7 +235,11 @@ def _pin_store( created: datetime | None = None, *, force_identical_write: bool = False, + data_dict: bool | str | Path | None = None, ) -> Meta: + from pins.data_dict import prepare_metadata + + metadata = prepare_metadata(metadata, data_dict, x, upload=type == "file") _type = type if _type == "feather": warn_deprecated( @@ -295,8 +301,13 @@ def _pin_store( if last_hash == meta.pin_hash: msg = ( f'The hash of pin "{name}" has not changed. Your pin will not ' - f"be stored.", + "be stored." ) + if meta.user and "data_dict" in meta.user: + msg += ( + " To save dictionary or metadata changes, use " + "force_identical_write=True." + ) inform(log=_log, msg=msg) return last_meta @@ -349,6 +360,7 @@ def pin_write( created: datetime | None = None, *, force_identical_write: bool = False, + data_dict: bool | str | Path | None = None, ) -> Meta: """Write a pin object to the board. @@ -370,6 +382,11 @@ def pin_write( metadata: A dictionary containing additional metadata to store with the pin. This gets stored on the Meta.user field. + data_dict: + Optional YAML file path, or True to draft a dictionary from a DataFrame. + Stores YAML in metadata["data_dict"]. For Connect, the publisher renders + dictionary.html and index.html keeps the pin overview and dictionary view. + Requires the optional data-dict CLI for drafting and rendering. versioned: Whether the pin should be versioned. Defaults to versioning. created: @@ -404,6 +421,7 @@ def pin_write( versioned, created, force_identical_write=force_identical_write, + data_dict=data_dict, ) def pin_download(self, name, version=None, hash=None) -> Sequence[str]: @@ -463,6 +481,9 @@ def pin_upload( title=None, description=None, metadata=None, + *, + data_dict: bool | str | Path | None = None, + force_identical_write: bool = False, ): """Write a pin based on paths to one or more files. @@ -485,6 +506,12 @@ def pin_upload( metadata: A dictionary containing additional metadata to store with the pin. This gets stored on the Meta.user field. + data_dict: + Optional YAML file path, or True to draft from Parquet files. + For Connect, the publisher renders dictionary.html; index.html keeps + the pin overview and adds the dictionary view. + force_identical_write: + Store a new version even when only the metadata changes. """ if isinstance(paths, (list, tuple)): @@ -500,6 +527,8 @@ def pin_upload( title=title, description=description, metadata=metadata, + data_dict=data_dict, + force_identical_write=force_identical_write, ) def pin_version_delete(self, name: str, version: str): @@ -755,6 +784,13 @@ def _create_meta( # file is saved locally in order to hash, calc size file_names = save_data(x, p_obj, type, apply_suffix) + metadata_name = self.meta_factory.get_meta_name() + payloads = [file_names] if isinstance(file_names, str) else file_names + if any( + Path(file).name.casefold() == metadata_name.casefold() for file in payloads + ): + raise PinsError(f"Generating pin metadata would overwrite {metadata_name}.") + meta = self.meta_factory.create( pin_dir_path, file_names, @@ -1222,46 +1258,56 @@ def prepare_pin_version(self, pin_dir_path, x, name: str | None, *args, **kwargs meta = super()._create_meta(pin_dir_path, x, short_name, *args, **kwargs) meta.name = name - # copy in files needed by index.html ---------------------------------- - crnt_files = set([meta.file] if isinstance(meta.file, str) else meta.file) - to_add = [str(p) for p in self.html_assets_dir.rglob("*")] - overlap = set(to_add) & crnt_files + has_dictionary = bool(meta.user and "data_dict" in meta.user) + all_files = [meta.file] if isinstance(meta.file, str) else meta.file + reserved = { + str(path.relative_to(self.html_assets_dir)) + for path in self.html_assets_dir.rglob("*") + } + reserved.update({"data.txt", "manifest.json"}) + if has_dictionary: + reserved.add("dictionary.html") + reserved = {file.casefold() for file in reserved} + overlap = {file for file in all_files if file.casefold() in reserved} if overlap: raise PinsError( - f"Generating an index.html would overwrite these files: {overlap}" + f"Generating a pin preview would overwrite these files: {overlap}" + ) + if has_dictionary: + from pins.data_dict import render + + render( + meta.user["data_dict"], + pin_dir_path, + meta.file, + filename="dictionary.html", ) - - # recursively copy all assets into prepped pin version dir shutil.copytree(self.html_assets_dir, pin_dir_path, dirs_exist_ok=True) - # render index.html ------------------------------------------------ - - all_files = [meta.file] if isinstance(meta.file, str) else meta.file - pin_files = ", ".join(f"""{file}""" for file in all_files) - + preview = json.loads(create_adaptor(x).data_preview) + if preview.get("columns"): + preview["options"] = { + "rows": {"min": min(10, max(1, len(preview["data"]))), "max": 10} + } context = { "date": meta.version.created.replace(microsecond=0), "pin_name": self.path_to_pin(name), - "pin_files": pin_files, + "pin_name_literal": json.dumps(self.path_to_pin(name), ensure_ascii=False), + "pin_files": [ + {"name": file, "url": quote(file, safe="")} for file in all_files + ], "pin_metadata": meta, "board_deparse": board_deparse(self), - "data_preview": create_adaptor(x).data_preview, + "data_preview": json.dumps(preview).replace("<", "\\u003c"), + "has_preview": bool(preview.get("columns")), + "has_dictionary": has_dictionary, + "show_r": meta.type not in ["joblib"], + "read_method": "pin_download" if meta.type == "file" else "pin_read", } - - # do not show r code if not round-trip friendly - if meta.type in ["joblib"]: - context["show_r_style"] = "display:none" - else: - context["show_r_style"] = "" - - # render html template ---- - from jinja2 import Environment - env = Environment() + env = Environment(autoescape=True, trim_blocks=True, lstrip_blocks=True) template = env.from_string(self.html_template.read_text()) - rendered = template.render(context) - (Path(pin_dir_path) / "index.html").write_text(rendered) - + (Path(pin_dir_path) / "index.html").write_text(rendered, encoding="utf-8") return meta diff --git a/pins/data_dict.py b/pins/data_dict.py new file mode 100644 index 00000000..b223726b --- /dev/null +++ b/pins/data_dict.py @@ -0,0 +1,159 @@ +"""Optional data dictionary generation and Connect rendering.""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from importlib.metadata import PackageNotFoundError, distribution +from pathlib import Path + +import pandas as pd +import yaml + +from pins.errors import PinsError + + +def _cli(): + try: + package = distribution("data-dict-yaml") + for file in package.files or []: + if file.name in ("data-dict", "data-dict.exe"): + binary = package.locate_file(file) + if binary.is_file(): + return str(binary) + except PackageNotFoundError: + pass + binary = shutil.which("data-dict") + if binary: + return binary + raise PinsError( + "Data dictionary support requires the data-dict CLI. " + "Install pins[data-dict] or put data-dict on PATH." + ) + + +def _run(args): + try: + subprocess.run( + [_cli(), *map(str, args)], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as error: + raise PinsError(f"data-dict failed: {error.stderr.strip()}") from error + except (OSError, subprocess.TimeoutExpired) as error: + raise PinsError(f"Could not run data-dict: {error}") from error + + +def _parse(text): + if not isinstance(text, str): + raise PinsError("metadata['data_dict'] must contain YAML text.") + try: + document = yaml.safe_load(text) + except yaml.YAMLError as error: + raise PinsError(f"Invalid data dictionary YAML: {error}") from error + if not isinstance(document, dict) or not isinstance(document.get("tables"), list): + raise PinsError("A data dictionary must contain a tables list.") + if not document["tables"] or any( + not isinstance(table, dict) for table in document["tables"] + ): + raise PinsError("A data dictionary must contain at least one table mapping.") + return document + + +def _parse_portable(text): + document = _parse(text) + if any("source" in table for table in document["tables"]): + raise PinsError("Pin data dictionaries must omit table source fields.") + return document + + +def _draft(paths, output): + paths = [Path(path).resolve() for path in paths] + _run(["draft", *paths, "-o", output]) + return _parse(output.read_text(encoding="utf-8")) + + +def prepare_metadata(metadata, data_dict, x, *, upload=False): + """Attach portable YAML without changing the caller's metadata.""" + if data_dict is None and (metadata is None or "data_dict" not in metadata): + return metadata + result = dict(metadata or {}) + if data_dict is not None and "data_dict" in result: + raise PinsError("Supply data_dict or metadata['data_dict'], not both.") + if data_dict is True: + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "data-dict.yaml" + if upload: + paths = [x] if isinstance(x, (str, Path)) else list(x) + if not paths or any(Path(p).suffix.lower() != ".parquet" for p in paths): + raise PinsError( + "data_dict=True for pin_upload requires Parquet files." + ) + else: + if not isinstance(x, pd.DataFrame): + raise PinsError("data_dict=True for pin_write requires a DataFrame.") + paths = [Path(temporary) / "data.parquet"] + try: + x.to_parquet(paths[0], engine="pyarrow", index=False) + except ImportError as error: + raise PinsError( + "Drafting a dictionary requires pins[data-dict]." + ) from error + document = _draft(paths, output) + for table in document["tables"]: + table.pop("source", None) + result["data_dict"] = yaml.safe_dump(document, sort_keys=False) + elif isinstance(data_dict, (str, Path)): + path = Path(data_dict) + try: + result["data_dict"] = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as error: + raise PinsError( + f"Could not read data dictionary file {path}: {error}. " + "Supply raw YAML in metadata['data_dict'] instead." + ) from error + elif data_dict is not None: + raise PinsError("data_dict must be None, True, or a YAML file path.") + _parse_portable(result["data_dict"]) + return result + + +def render(text, directory, files=(), *, filename="index.html"): + """Render a self-contained index without bundling temporary YAML.""" + document = _parse_portable(text) + index = Path(directory) / filename + if index.exists() or index.is_symlink(): + raise PinsError(f"Generating a data dictionary would overwrite {filename}.") + files = [files] if isinstance(files, str) else list(files) + # shortcut: Profiles cover one Parquet file and one table; add explicit mapping for multiple tables. + if ( + len(files) == 1 + and Path(files[0]).suffix.lower() == ".parquet" + and len(document["tables"]) == 1 + ): + document["tables"][0]["source"] = {"parquet": files[0]} + text = yaml.safe_dump(document, sort_keys=False) + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", suffix=".yaml", dir=directory, delete=False + ) as temporary: + dictionary = Path(temporary.name) + try: + dictionary.write_text(text, encoding="utf-8") + _run(["render", dictionary, "-o", index]) + if not index.is_file(): + raise PinsError(f"data-dict did not produce {filename}.") + if filename == "dictionary.html": + html = index.read_text(encoding="utf-8") + integration = '' + index.write_text( + html.replace("", integration + "", 1), encoding="utf-8" + ) + except Exception: + index.unlink(missing_ok=True) + raise + finally: + dictionary.unlink(missing_ok=True) diff --git a/pins/rsconnect/html/index.html b/pins/rsconnect/html/index.html index 019be237..a555a338 100644 --- a/pins/rsconnect/html/index.html +++ b/pins/rsconnect/html/index.html @@ -1,83 +1,113 @@ - + + + + + {{ pin_name }} · Data pin - + - - + + + - -
-

{{pin_name}}

- {% if pin_metadata %} -

- {% if date %}Last updated from Python: {{ date }} •{% endif %} - Format: {{ pin_metadata.type }} • - API: v{{ pin_metadata.api_version }} -

-

{{ pin_metadata.description }}

-

Download data: {{ pin_files }}

-
- Raw metadata -
{{ pin_metadata.to_pin_yaml() }}
-
- {% endif %} -
- -
-

Python Code

- -
from pins import board_connect
+    
+    
+
+
+ + Posit Connect Shared data +
+ Data pin +
+
+
+
+

Published from Python

+

{{ pin_name }}

+ {% if pin_metadata.description %}

{{ pin_metadata.description }}

{% endif %} +
+
Format
{{ pin_metadata.type }}
+ {% if date %}
Last updated
{{ date }} UTC
{% endif %} +
Pin API
v{{ pin_metadata.api_version }}
+
Content hash
{{ pin_metadata.pin_hash }}
+
+
+ +
+
+
+ {% if has_preview %} +
+

Data preview

Up to 100 rows from this version

+
+
+ +
+
+
+ {% endif %} +
+

Use this data

Connect with your account, then read the pin.

+
+

Python

+
from pins import board_connect
 from dotenv import load_dotenv
 load_dotenv()
 
-board = {{board_deparse}}
-board.pin_read("{{pin_name}}")
+board = {{ board_deparse }} +board.{{ read_method }}({{ pin_name_literal }})
+ {% if show_r %} +

R

+
library(pins)
 
-    
-    
- -
-

R Code

-
library(pins)
-
-board <- board_connect(auth = "envvar")
-pin_read(board, "{{pin_name}}")
+board <- board_connect(auth = "envvar") +{{ read_method }}(board, {{ pin_name_literal }})
+ {% endif %} + +
+ + + + +
+
+

Pin metadata

+

{{ pin_metadata.title }}

+
- -
-

Preview (up to 100 rows)

-
- -
-
+
+ {% if has_dictionary %} +
+
+
+

Data dictionary

Explore column definitions, constraints, and available profiles.

+ Open full page ↗ +
+ +
+
+ {% endif %} + + diff --git a/pins/rsconnect/html/pin.css b/pins/rsconnect/html/pin.css new file mode 100644 index 00000000..2c2b2ecd --- /dev/null +++ b/pins/rsconnect/html/pin.css @@ -0,0 +1,102 @@ +:root { + color-scheme: light; + --page: #f5f7fa; + --surface: #ffffff; + --soft: #f0f4f8; + --ink: #172b3a; + --muted: #536778; + --line: #dce4eb; + --accent: #176b87; + --accent-soft: #e8f3f7; + --code: #f7f9fc; + font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} +* { box-sizing: border-box; } +body { margin: 0; background: var(--page); color: var(--ink); font-size: 15px; line-height: 1.6; } +a { color: var(--accent); text-underline-offset: 3px; } +button, a { -webkit-tap-highlight-color: transparent; } +button, input { font: inherit; } +:focus-visible { outline: 3px solid var(--accent); outline-offset: 4px; } +[hidden] { display: none !important; } +.skip-link { position: absolute; top: -80px; left: 16px; padding: 10px 16px; z-index: 10; background: var(--surface); } +.skip-link:focus { top: 16px; } +.topbar { background: var(--surface); border-bottom: 1px solid var(--line); } +.topbar-inner { max-width: 1240px; margin: auto; padding: 18px 32px; display: flex; align-items: center; justify-content: space-between; gap: 16px; } +.brand { display: flex; align-items: center; gap: 12px; font-weight: 650; letter-spacing: -.02em; } +.brand-mark { width: 30px; height: 30px; color: var(--accent); } +.brand small { font-size: 13px; font-weight: 450; color: var(--muted); margin-left: 12px; letter-spacing: 0; } +.badge { display: inline-flex; align-items: center; border: 1px solid var(--line); border-radius: 999px; padding: 3px 11px; color: var(--muted); font-size: 12px; font-weight: 600; } +main { max-width: 1240px; margin: auto; padding: 36px 32px 48px; } +.hero { margin-bottom: 26px; } +.eyebrow { color: var(--accent); font-size: 12px; font-weight: 700; letter-spacing: .12em; text-transform: uppercase; margin: 0 0 8px; } +h1 { font-size: clamp(25px, 3.4vw, 38px); line-height: 1.2; letter-spacing: -.035em; margin: 0 0 12px; overflow-wrap: anywhere; } +.description { color: var(--muted); max-width: 850px; font-size: 16px; margin: 0; white-space: pre-wrap; overflow-wrap: anywhere; } +.facts { display: flex; gap: 28px; flex-wrap: wrap; margin: 24px 0 0; padding: 0; } +.facts div { min-width: 90px; } +dt { color: var(--muted); font-size: 12px; } +dd { margin: 2px 0 0; font-size: 14px; font-weight: 600; overflow-wrap: anywhere; } +.views { display: flex; gap: 26px; border-bottom: 1px solid var(--line); margin-bottom: 24px; } +.views a { padding: 10px 0 13px; text-decoration: none; color: var(--muted); font-weight: 600; border-bottom: 3px solid transparent; margin-bottom: -1px; } +.views a[aria-current="page"] { color: var(--accent); border-bottom-color: var(--accent); } +.overview-grid { display: grid; grid-template-columns: minmax(0, 1fr) 290px; gap: 22px; align-items: start; } +.stack { display: grid; gap: 22px; min-width: 0; } +.card { background: var(--surface); border: 1px solid var(--line); border-radius: 14px; overflow: hidden; } +.card-head { display: flex; justify-content: space-between; align-items: center; gap: 16px; padding: 19px 22px; border-bottom: 1px solid var(--line); } +h2 { font-size: 17px; letter-spacing: -.02em; margin: 0; line-height: 1.4; } +h3 { font-size: 14px; margin: 0; } +.card-head p, .helper { font-size: 13px; color: var(--muted); margin: 3px 0 0; } +.card-body { padding: 20px 22px; } +.files { display: grid; gap: 10px; } +.file-link { display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 12px 14px; border: 1px solid var(--line); border-radius: 9px; text-decoration: none; font-size: 13px; overflow-wrap: anywhere; } +.file-link:hover { background: var(--accent-soft); border-color: var(--accent); } +.file-link span { min-width: 0; } +.file-link svg { flex-shrink: 0; } +.code-block { padding: 0 22px 22px; } +.code-label { display: flex; align-items: center; justify-content: space-between; padding: 18px 0 10px; } +.copy { border: 1px solid var(--line); background: var(--surface); color: var(--muted); border-radius: 6px; padding: 4px 10px; font-size: 12px; cursor: pointer; } +.copy:hover { color: var(--accent); border-color: var(--accent); } +pre { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12.5px; line-height: 1.7; margin: 0; overflow: auto; tab-size: 2; } +pre.pin-code { padding: 15px 17px; background: var(--code); border: 1px solid var(--line); border-radius: 9px; } +.pin-code code.hljs { background: transparent; padding: 0; } +.metadata { padding: 16px 22px; } +.metadata pre { margin-top: 0; color: var(--muted); white-space: pre-wrap; overflow-wrap: anywhere; } +.metadata-download { margin: 12px 0 0; font-size: 13px; } +.related-links { display: grid; gap: 8px; font-size: 13px; overflow-wrap: anywhere; } +.table-wrap { padding: 12px 14px; } +.pagedtable { padding: 0; } +.pagedtable table { border-bottom-color: var(--line); font-weight: 400; font-size: 13px; } +.pagedtable th { padding: 10px 12px; color: var(--muted); font-weight: 600; border-bottom-color: var(--line); background: var(--code); } +.pagedtable td { padding: 10px 12px; border-bottom: 1px solid var(--line); } +.pagedtable .even { background: var(--code); } +.pagedtable-footer { padding: 14px 8px; font-size: 12px; } +.pagedtable-info, .pagedtable-index-separator-left, .pagedtable-index-separator-right { color: var(--muted); } +.pagedtable-index-current, a.pagedtable-index-current:hover { color: var(--ink); } +.dictionary-toolbar { display: flex; justify-content: space-between; align-items: center; gap: 20px; padding: 18px 22px; border-bottom: 1px solid var(--line); } +.dictionary-toolbar p { margin: 2px 0 0; color: var(--muted); font-size: 13px; } +.dictionary-toolbar a { font-size: 13px; white-space: nowrap; } +.dictionary-frame { display: block; width: 100%; height: max(620px, 78vh); border: 0; background: var(--surface); } +.footer { color: var(--muted); font-size: 12px; margin: 28px 0 0; } +@media (max-width: 820px) { + .overview-grid { grid-template-columns: 1fr; } + .sidebar { grid-row: 1; } + .sidebar .files { grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); } +} +@media (max-width: 540px) { + main { padding: 24px 16px 32px; } + .topbar-inner { padding: 14px 16px; } + .brand small { display: none; } + .facts { gap: 16px 24px; } + .card-head, .card-body, .dictionary-toolbar { padding: 16px; } + .code-block { padding: 0 16px 16px; } + .dictionary-toolbar { align-items: flex-start; flex-direction: column; gap: 8px; } + .dictionary-frame { height: 76vh; min-height: 520px; } +} + +:root[data-theme="dark"] { color-scheme: dark; --page: #101820; --surface: #17232d; --soft: #21313e; --ink: #e4edf4; --muted: #a5b7c5; --line: #314553; --accent: #79c5e1; --accent-soft: #203d4a; --code: #131f29; } +[data-theme="dark"] .hljs, [data-theme="dark"] .hljs-subst, [data-theme="dark"] .hljs-title, [data-theme="dark"] .hljs-tag { color: var(--ink); } +[data-theme="dark"] .hljs-keyword, [data-theme="dark"] .hljs-selector-tag, [data-theme="dark"] .hljs-built_in { color: #9dc7ff; } +[data-theme="dark"] .hljs-string, [data-theme="dark"] .hljs-number, [data-theme="dark"] .hljs-literal { color: #9ad8b9; } +[data-theme="dark"] .hljs-comment { color: var(--muted); } +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { transition: none !important; animation: none !important; } +} diff --git a/pins/rsconnect/html/pin.js b/pins/rsconnect/html/pin.js new file mode 100644 index 00000000..14fcfc49 --- /dev/null +++ b/pins/rsconnect/html/pin.js @@ -0,0 +1,45 @@ +"use strict"; + +const viewLinks = document.querySelectorAll("[data-pin-view]"); +const viewPanels = document.querySelectorAll("[data-pin-panel]"); + +function showPinView() { + const requested = window.location.hash.slice(1); + const view = Array.from(viewPanels).some(panel => panel.id === requested) + ? requested : "overview"; + viewPanels.forEach(panel => { panel.hidden = panel.id !== view; }); + viewLinks.forEach(link => { + if (link.dataset.pinView === view) link.setAttribute("aria-current", "page"); + else link.removeAttribute("aria-current"); + }); + window.dispatchEvent(new Event("resize")); +} + +if (viewLinks.length) { + window.addEventListener("load", showPinView); + window.addEventListener("hashchange", showPinView); +} + +document.querySelectorAll("[data-copy-code]").forEach(button => { + button.hidden = false; + button.addEventListener("click", async () => { + const code = document.getElementById(button.dataset.copyCode); + try { + await navigator.clipboard.writeText(code.textContent); + button.textContent = "Copied"; + } catch { + const range = document.createRange(); + range.selectNodeContents(code); + const selection = window.getSelection(); + selection.removeAllRanges(); + selection.addRange(range); + button.textContent = "Selected — copy to clipboard"; + } + }); +}); + +if (window.hljs) { + hljs.registerLanguage("python", highlight_python); + hljs.registerLanguage("r", highlight_r); + document.querySelectorAll(".pin-code code").forEach(code => hljs.highlightBlock(code)); +} diff --git a/pins/rsconnect/html/theme.js b/pins/rsconnect/html/theme.js new file mode 100644 index 00000000..b5fe4a17 --- /dev/null +++ b/pins/rsconnect/html/theme.js @@ -0,0 +1,29 @@ +"use strict"; + +(() => { + let parentRoot; + try { + if (window.parent !== window) parentRoot = window.parent.document.documentElement; + } catch {} + + function updateTheme() { + let theme = "light"; + try { + theme = parentRoot + ? (parentRoot.dataset.theme === "dark" || parentRoot.classList.contains("dark") ? "dark" : "light") + : (window.localStorage.getItem("connect.theme") === "dark" ? "dark" : "light"); + } catch {} + document.documentElement.dataset.theme = theme; + } + + updateTheme(); + if (parentRoot) { + new MutationObserver(updateTheme).observe(parentRoot, { + attributes: true, attributeFilter: ["class", "data-theme"] + }); + } else { + window.addEventListener("storage", event => { + if (event.key === "connect.theme" || event.key === null) updateTheme(); + }); + } +})(); diff --git a/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/index.html b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/index.html index 48efbfc8..2477da56 100644 --- a/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/index.html +++ b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/index.html @@ -1,47 +1,87 @@ - + + + + + derek/test_rsc_pin · Data pin - + - - + + + + +
+
+
+ + Posit Connect Shared data +
+ Data pin +
+
+
+
+

Published from Python

+

derek/test_rsc_pin

+

some description

+
Format
csv
+
Last updated
2020-01-13 23:58:59 UTC
Pin API
v1
+
Content hash
a6cf5331bf3de6c6
+
+
+ +
+
+
+
+

Data preview

Up to 100 rows from this version

+
+
+ +
+
+
+
+

Use this data

Connect with your account, then read the pin.

+
+

Python

+
from pins import board_connect
+from dotenv import load_dotenv
+load_dotenv()
+
+board = board_connect(server_url='http://localhost:3939')
+board.pin_read("derek/test_rsc_pin")
+

R

+
library(pins)
 
-    
-

derek/test_rsc_pin

- -

- Last updated from Python: 2020-01-13 23:58:59 • - Format: csv • - API: v1 -

-

some description

-

Download data: test_rsc_pin.csv

-
- Raw metadata -
api_version: 1
+board <- board_connect(auth = "envvar")
+pin_read(board, "derek/test_rsc_pin")
+
+
+
+ +
+
+
+
+

Pin metadata

+

some pin

+
- -
-

Python Code

- -
from pins import board_connect
-from dotenv import load_dotenv
-load_dotenv()
-
-board = board_connect(server_url='http://localhost:3939')
-board.pin_read("derek/test_rsc_pin")
- - -
- -
-

R Code

-
library(pins)
-
-board <- board_connect(auth = "envvar")
-pin_read(board, "derek/test_rsc_pin")
+
- -
-

Preview (up to 100 rows)

-
- -
-
+
+ +
\ No newline at end of file diff --git a/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.css b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.css new file mode 100644 index 00000000..2c2b2ecd --- /dev/null +++ b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.css @@ -0,0 +1,102 @@ +:root { + color-scheme: light; + --page: #f5f7fa; + --surface: #ffffff; + --soft: #f0f4f8; + --ink: #172b3a; + --muted: #536778; + --line: #dce4eb; + --accent: #176b87; + --accent-soft: #e8f3f7; + --code: #f7f9fc; + font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} +* { box-sizing: border-box; } +body { margin: 0; background: var(--page); color: var(--ink); font-size: 15px; line-height: 1.6; } +a { color: var(--accent); text-underline-offset: 3px; } +button, a { -webkit-tap-highlight-color: transparent; } +button, input { font: inherit; } +:focus-visible { outline: 3px solid var(--accent); outline-offset: 4px; } +[hidden] { display: none !important; } +.skip-link { position: absolute; top: -80px; left: 16px; padding: 10px 16px; z-index: 10; background: var(--surface); } +.skip-link:focus { top: 16px; } +.topbar { background: var(--surface); border-bottom: 1px solid var(--line); } +.topbar-inner { max-width: 1240px; margin: auto; padding: 18px 32px; display: flex; align-items: center; justify-content: space-between; gap: 16px; } +.brand { display: flex; align-items: center; gap: 12px; font-weight: 650; letter-spacing: -.02em; } +.brand-mark { width: 30px; height: 30px; color: var(--accent); } +.brand small { font-size: 13px; font-weight: 450; color: var(--muted); margin-left: 12px; letter-spacing: 0; } +.badge { display: inline-flex; align-items: center; border: 1px solid var(--line); border-radius: 999px; padding: 3px 11px; color: var(--muted); font-size: 12px; font-weight: 600; } +main { max-width: 1240px; margin: auto; padding: 36px 32px 48px; } +.hero { margin-bottom: 26px; } +.eyebrow { color: var(--accent); font-size: 12px; font-weight: 700; letter-spacing: .12em; text-transform: uppercase; margin: 0 0 8px; } +h1 { font-size: clamp(25px, 3.4vw, 38px); line-height: 1.2; letter-spacing: -.035em; margin: 0 0 12px; overflow-wrap: anywhere; } +.description { color: var(--muted); max-width: 850px; font-size: 16px; margin: 0; white-space: pre-wrap; overflow-wrap: anywhere; } +.facts { display: flex; gap: 28px; flex-wrap: wrap; margin: 24px 0 0; padding: 0; } +.facts div { min-width: 90px; } +dt { color: var(--muted); font-size: 12px; } +dd { margin: 2px 0 0; font-size: 14px; font-weight: 600; overflow-wrap: anywhere; } +.views { display: flex; gap: 26px; border-bottom: 1px solid var(--line); margin-bottom: 24px; } +.views a { padding: 10px 0 13px; text-decoration: none; color: var(--muted); font-weight: 600; border-bottom: 3px solid transparent; margin-bottom: -1px; } +.views a[aria-current="page"] { color: var(--accent); border-bottom-color: var(--accent); } +.overview-grid { display: grid; grid-template-columns: minmax(0, 1fr) 290px; gap: 22px; align-items: start; } +.stack { display: grid; gap: 22px; min-width: 0; } +.card { background: var(--surface); border: 1px solid var(--line); border-radius: 14px; overflow: hidden; } +.card-head { display: flex; justify-content: space-between; align-items: center; gap: 16px; padding: 19px 22px; border-bottom: 1px solid var(--line); } +h2 { font-size: 17px; letter-spacing: -.02em; margin: 0; line-height: 1.4; } +h3 { font-size: 14px; margin: 0; } +.card-head p, .helper { font-size: 13px; color: var(--muted); margin: 3px 0 0; } +.card-body { padding: 20px 22px; } +.files { display: grid; gap: 10px; } +.file-link { display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 12px 14px; border: 1px solid var(--line); border-radius: 9px; text-decoration: none; font-size: 13px; overflow-wrap: anywhere; } +.file-link:hover { background: var(--accent-soft); border-color: var(--accent); } +.file-link span { min-width: 0; } +.file-link svg { flex-shrink: 0; } +.code-block { padding: 0 22px 22px; } +.code-label { display: flex; align-items: center; justify-content: space-between; padding: 18px 0 10px; } +.copy { border: 1px solid var(--line); background: var(--surface); color: var(--muted); border-radius: 6px; padding: 4px 10px; font-size: 12px; cursor: pointer; } +.copy:hover { color: var(--accent); border-color: var(--accent); } +pre { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12.5px; line-height: 1.7; margin: 0; overflow: auto; tab-size: 2; } +pre.pin-code { padding: 15px 17px; background: var(--code); border: 1px solid var(--line); border-radius: 9px; } +.pin-code code.hljs { background: transparent; padding: 0; } +.metadata { padding: 16px 22px; } +.metadata pre { margin-top: 0; color: var(--muted); white-space: pre-wrap; overflow-wrap: anywhere; } +.metadata-download { margin: 12px 0 0; font-size: 13px; } +.related-links { display: grid; gap: 8px; font-size: 13px; overflow-wrap: anywhere; } +.table-wrap { padding: 12px 14px; } +.pagedtable { padding: 0; } +.pagedtable table { border-bottom-color: var(--line); font-weight: 400; font-size: 13px; } +.pagedtable th { padding: 10px 12px; color: var(--muted); font-weight: 600; border-bottom-color: var(--line); background: var(--code); } +.pagedtable td { padding: 10px 12px; border-bottom: 1px solid var(--line); } +.pagedtable .even { background: var(--code); } +.pagedtable-footer { padding: 14px 8px; font-size: 12px; } +.pagedtable-info, .pagedtable-index-separator-left, .pagedtable-index-separator-right { color: var(--muted); } +.pagedtable-index-current, a.pagedtable-index-current:hover { color: var(--ink); } +.dictionary-toolbar { display: flex; justify-content: space-between; align-items: center; gap: 20px; padding: 18px 22px; border-bottom: 1px solid var(--line); } +.dictionary-toolbar p { margin: 2px 0 0; color: var(--muted); font-size: 13px; } +.dictionary-toolbar a { font-size: 13px; white-space: nowrap; } +.dictionary-frame { display: block; width: 100%; height: max(620px, 78vh); border: 0; background: var(--surface); } +.footer { color: var(--muted); font-size: 12px; margin: 28px 0 0; } +@media (max-width: 820px) { + .overview-grid { grid-template-columns: 1fr; } + .sidebar { grid-row: 1; } + .sidebar .files { grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); } +} +@media (max-width: 540px) { + main { padding: 24px 16px 32px; } + .topbar-inner { padding: 14px 16px; } + .brand small { display: none; } + .facts { gap: 16px 24px; } + .card-head, .card-body, .dictionary-toolbar { padding: 16px; } + .code-block { padding: 0 16px 16px; } + .dictionary-toolbar { align-items: flex-start; flex-direction: column; gap: 8px; } + .dictionary-frame { height: 76vh; min-height: 520px; } +} + +:root[data-theme="dark"] { color-scheme: dark; --page: #101820; --surface: #17232d; --soft: #21313e; --ink: #e4edf4; --muted: #a5b7c5; --line: #314553; --accent: #79c5e1; --accent-soft: #203d4a; --code: #131f29; } +[data-theme="dark"] .hljs, [data-theme="dark"] .hljs-subst, [data-theme="dark"] .hljs-title, [data-theme="dark"] .hljs-tag { color: var(--ink); } +[data-theme="dark"] .hljs-keyword, [data-theme="dark"] .hljs-selector-tag, [data-theme="dark"] .hljs-built_in { color: #9dc7ff; } +[data-theme="dark"] .hljs-string, [data-theme="dark"] .hljs-number, [data-theme="dark"] .hljs-literal { color: #9ad8b9; } +[data-theme="dark"] .hljs-comment { color: var(--muted); } +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { transition: none !important; animation: none !important; } +} diff --git a/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.js b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.js new file mode 100644 index 00000000..14fcfc49 --- /dev/null +++ b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/pin.js @@ -0,0 +1,45 @@ +"use strict"; + +const viewLinks = document.querySelectorAll("[data-pin-view]"); +const viewPanels = document.querySelectorAll("[data-pin-panel]"); + +function showPinView() { + const requested = window.location.hash.slice(1); + const view = Array.from(viewPanels).some(panel => panel.id === requested) + ? requested : "overview"; + viewPanels.forEach(panel => { panel.hidden = panel.id !== view; }); + viewLinks.forEach(link => { + if (link.dataset.pinView === view) link.setAttribute("aria-current", "page"); + else link.removeAttribute("aria-current"); + }); + window.dispatchEvent(new Event("resize")); +} + +if (viewLinks.length) { + window.addEventListener("load", showPinView); + window.addEventListener("hashchange", showPinView); +} + +document.querySelectorAll("[data-copy-code]").forEach(button => { + button.hidden = false; + button.addEventListener("click", async () => { + const code = document.getElementById(button.dataset.copyCode); + try { + await navigator.clipboard.writeText(code.textContent); + button.textContent = "Copied"; + } catch { + const range = document.createRange(); + range.selectNodeContents(code); + const selection = window.getSelection(); + selection.removeAllRanges(); + selection.addRange(range); + button.textContent = "Selected — copy to clipboard"; + } + }); +}); + +if (window.hljs) { + hljs.registerLanguage("python", highlight_python); + hljs.registerLanguage("r", highlight_r); + document.querySelectorAll(".pin-code code").forEach(code => hljs.highlightBlock(code)); +} diff --git a/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/theme.js b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/theme.js new file mode 100644 index 00000000..b5fe4a17 --- /dev/null +++ b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/theme.js @@ -0,0 +1,29 @@ +"use strict"; + +(() => { + let parentRoot; + try { + if (window.parent !== window) parentRoot = window.parent.document.documentElement; + } catch {} + + function updateTheme() { + let theme = "light"; + try { + theme = parentRoot + ? (parentRoot.dataset.theme === "dark" || parentRoot.classList.contains("dark") ? "dark" : "light") + : (window.localStorage.getItem("connect.theme") === "dark" ? "dark" : "light"); + } catch {} + document.documentElement.dataset.theme = theme; + } + + updateTheme(); + if (parentRoot) { + new MutationObserver(updateTheme).observe(parentRoot, { + attributes: true, attributeFilter: ["class", "data-theme"] + }); + } else { + window.addEventListener("storage", event => { + if (event.key === "connect.theme" || event.key === null) updateTheme(); + }); + } +})(); diff --git a/pins/tests/test_connect_preview.py b/pins/tests/test_connect_preview.py new file mode 100644 index 00000000..2a449d17 --- /dev/null +++ b/pins/tests/test_connect_preview.py @@ -0,0 +1,125 @@ +import json +from html.parser import HTMLParser + +import pandas as pd +import pytest + +from pins import board_connect +from pins import data_dict as dd +from pins.errors import PinsError + + +class Preview(HTMLParser): + def __init__(self, html): + super().__init__() + self.links = [] + self.codes = {} + self.preview = "" + self._code = None + self._preview = False + self.panels = {} + self._panel = None + self._sections = 0 + self.feed(html) + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == "section": + if "data-pin-panel" in attrs: + self._panel = attrs["id"] + self.panels[self._panel] = "" + if self._panel: + self._sections += 1 + if tag == "a": + self.links.append(attrs.get("href")) + if tag == "code": + self._code = attrs.get("id") + self.codes[self._code] = "" + if tag == "script" and "data-pagedtable-source" in attrs: + self._preview = True + + def handle_endtag(self, tag): + if tag == "section" and self._panel: + self._sections -= 1 + if not self._sections: + self._panel = None + if tag == "code": + self._code = None + if tag == "script": + self._preview = False + + def handle_data(self, text): + if self._panel: + self.panels[self._panel] += text + if self._code: + self.codes[self._code] += text + if self._preview: + self.preview += text + + +def test_standard_preview_preserves_data_and_escapes_content(tmp_path, monkeypatch): + monkeypatch.setattr(dd, "_cli", lambda: pytest.fail("Unexpected CLI lookup")) + attack = '' + name = 'owner/data"' + frame = pd.DataFrame({"value": [attack]}) + connect = board_connect( + server_url="https://connect.example.com", api_key="offline", cache=None + ) + meta = connect.prepare_pin_version( + tmp_path, frame, name, type="csv", title="Sample pin", description=attack + ) + html = (tmp_path / "index.html").read_text() + page = Preview(html) + assert "#metadata" in page.links + assert "Download metadata" in page.panels["metadata"] + assert "Download metadata" not in page.panels["overview"] + assert attack not in html + assert "Data preview" in html and "Download metadata" in html + assert "' in html + assert "#theme-toggle{display:none}" in html + embedded = json.loads( + re.search(r']*id="dict"[^>]*>(.*?)', html, re.S)[1] + ) + table = embedded["tables"][0] + profiles = {column["name"]: column["profile"] for column in table["columns"]} + assert table["rows"] == 3 + assert table["source"]["parquet"] == meta.file + assert profiles["id"]["distinct"]["count"] == 3 + assert profiles["value"]["missing"] == 1 + assert profiles["value"]["range"] == {"min": 10.0, "max": 30.0} + assert sum(bucket["count"] for bucket in profiles["value"]["histogram"]["bins"]) == 2 + assert profiles["group"]["common_values"]["values"] == [ + {"value": "a", "count": 2}, + {"value": "b", "count": 1}, + ] + assert yaml.safe_load((output / "data.txt").read_text())["user"] == metadata + assert "source" not in yaml.safe_load(metadata["data_dict"])["tables"][0] + assert not list(output.glob("*.yaml")) + assert (output / "pin.css").exists() and (output / "pin.js").exists() + assert str(output) not in html + pd.testing.assert_frame_equal(pd.read_parquet(output / meta.file), frame) + + +def test_cli_falls_back_to_path(monkeypatch): + def missing(_): + raise dd.PackageNotFoundError + + monkeypatch.setattr(dd, "distribution", missing) + monkeypatch.setattr(dd.shutil, "which", lambda _: "/usr/local/bin/data-dict") + assert dd._cli() == "/usr/local/bin/data-dict" + + +def test_required_cli_fails_when_missing(monkeypatch): + monkeypatch.setenv("PINS_REQUIRE_DATA_DICT_CLI", "1") + + def missing(): + raise PinsError("data-dict is missing") + + monkeypatch.setattr(dd, "_cli", missing) + with pytest.raises(PinsError, match="data-dict is missing"): + _require_cli() + + +@pytest.mark.parametrize("file_bytes", [None, b"\xff"]) +def test_data_dict_file_errors_are_pins_errors(tmp_path, file_bytes): + path = tmp_path / "dictionary.yaml" + if file_bytes is not None: + path.write_bytes(file_bytes) + + with pytest.raises(PinsError) as error: + dd.prepare_metadata(None, path, None) + + assert str(path) in str(error.value) + assert "metadata['data_dict']" in str(error.value) + + +def test_draft_uses_absolute_uploaded_paths(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + arguments = [] + + def draft(args): + arguments.extend(args) + Path(args[-1]).write_text(DICTIONARY, encoding="utf-8") + + monkeypatch.setattr(dd, "_run", draft) + metadata = dd.prepare_metadata(None, True, "-data.parquet", upload=True) + + assert arguments[0] == "draft" + assert arguments[1] == (tmp_path / "-data.parquet").resolve() + assert "source" not in yaml.safe_load(metadata["data_dict"])["tables"][0] + + +@pytest.mark.parametrize( + "error", + [OSError("not executable"), dd.subprocess.TimeoutExpired("data-dict", 300)], +) +def test_run_os_and_timeout_errors_are_pins_errors(monkeypatch, error): + monkeypatch.setattr(dd, "_cli", lambda: "data-dict") + + def fail(*args, **kwargs): + raise error + + monkeypatch.setattr(dd.subprocess, "run", fail) + with pytest.raises(PinsError, match="Could not run data-dict"): + dd._run(["render"]) + + +def test_render_write_failure_cleans_temporary_yaml(tmp_path, monkeypatch): + temporary_files = [] + + def fail_write(path, *args, **kwargs): + temporary_files.append(path) + raise OSError("disk full") + + monkeypatch.setattr(Path, "write_text", fail_write) + with pytest.raises(OSError, match="disk full"): + dd.render(DICTIONARY, tmp_path) + + assert len(temporary_files) == 1 + assert not temporary_files[0].exists() + assert not (tmp_path / "index.html").exists() + + +def test_render_failure_cleans_temporary_yaml_and_leaves_index_absent( + tmp_path, monkeypatch +): + def fail(args): + Path(args[-1]).write_text("partial render", encoding="utf-8") + raise PinsError("renderer failed") + + monkeypatch.setattr(dd, "_run", fail) + with pytest.raises(PinsError, match="renderer failed"): + dd.render(DICTIONARY, tmp_path) + + assert not list(tmp_path.glob("*.yaml")) + assert not (tmp_path / "index.html").exists() + + +@pytest.mark.parametrize( + ("files", "dictionary", "expected_source"), + [ + (["one.parquet", "two.parquet"], DICTIONARY, None), + (["one.parquet"], MULTI_TABLE_DICTIONARY, None), + (["DATA.PARQUET"], DICTIONARY, {"parquet": "DATA.PARQUET"}), + (["one.csv"], DICTIONARY, None), + ], +) +def test_render_profiles_only_one_parquet_table( + tmp_path, monkeypatch, files, dictionary, expected_source +): + rendered = {} + + def render_cli(args): + rendered["document"] = yaml.safe_load(Path(args[1]).read_text(encoding="utf-8")) + Path(args[3]).write_text("rendered", encoding="utf-8") + + monkeypatch.setattr(dd, "_run", render_cli) + dd.render(dictionary, tmp_path, files) + + tables = rendered["document"]["tables"] + if expected_source is None: + assert all("source" not in table for table in tables) + else: + assert tables[0]["source"] == expected_source + assert (tmp_path / "index.html").is_file() + + +def test_render_preserves_existing_dangling_symlink(tmp_path): + index = tmp_path / "dictionary.html" + try: + index.symlink_to(tmp_path / "missing.html") + except OSError: + pytest.skip("Symlinks are unavailable on this filesystem") + with pytest.raises(PinsError, match="overwrite"): + dd.render( + "tables: [{name: data, columns: []}]", tmp_path, filename="dictionary.html" + ) + assert index.is_symlink() diff --git a/pyproject.toml b/pyproject.toml index 57cd1405..1e3f0cb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ ] [project.optional-dependencies] +data-dict = ["data-dict-yaml>=0.0.3,<0.0.4", "pyarrow"] aws = ["s3fs"] azure = ["adlfs>=2024.4.1"] check = [ From 1eae50f485eca74836034e46bcd7e5f0416fbbbb Mon Sep 17 00:00:00 2001 From: Matt Conflitti Date: Tue, 17 Nov 2026 16:47:28 -0500 Subject: [PATCH 2/4] Fix optional dictionary CLI support and protect rendered previews --- .github/workflows/ci.yml | 39 ++++-- examples/data-dict/README.md | 6 + pins/boards.py | 37 ++++- pins/data_dict.py | 87 ++++++++++-- pins/rsconnect/html/dictionary.js | 28 ++++ .../dictionary.js | 28 ++++ pins/tests/test_connect_preview.py | 14 +- pins/tests/test_data_dict.py | 130 +++++++++++++++++- pins/tests/test_data_dict_security.py | 122 ++++++++++++++++ 9 files changed, 455 insertions(+), 36 deletions(-) create mode 100644 pins/rsconnect/html/dictionary.js create mode 100644 pins/tests/_snapshots/test_board_pin_write_rsc_index_html/dictionary.js create mode 100644 pins/tests/test_data_dict_security.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ac04ed4..e77337dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,11 +21,11 @@ jobs: fail-fast: false matrix: python: ["3.9", "3.10", "3.11", "3.12", "3.13"] - os: ["ubuntu-latest"] + os: ["ubuntu-24.04"] pytest_opts: ["--workers 4 --tests-per-worker 1"] requirements: [""] include: - - os: "ubuntu-latest" + - os: "ubuntu-24.04" python: "3.9" requirements: "requirements/minimum.txt" - os: "macos-latest" @@ -38,8 +38,8 @@ jobs: # ignore doctests pytest_opts: "-k pins/tests" steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python }} - name: Install dependencies @@ -81,7 +81,11 @@ jobs: test-data-dict: name: "Test data dictionary CLI" - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-latest, windows-latest] steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 @@ -91,8 +95,13 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e '.[data-dict]' pytest - - name: Run data dictionary tests - run: pytest pins/tests/test_data_dict.py + - name: Run data dictionary tests and offline bundle example + shell: bash + run: | + pytest pins/tests/test_data_dict.py pins/tests/test_connect_preview.py pins/tests/test_data_dict_security.py + python examples/data-dict/create_bundle.py \ + "$RUNNER_TEMP/pins-python-data-dict-bundle" \ + --dictionary examples/data-dict/data-dict.yaml env: PINS_REQUIRE_DATA_DICT_CLI: "1" @@ -101,8 +110,8 @@ jobs: runs-on: ubuntu-latest if: ${{ !github.event.pull_request.head.repo.fork }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.10" - name: Install dependencies @@ -129,8 +138,8 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.pull_request.head.repo.fork }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.10" - name: Install dependencies @@ -148,8 +157,8 @@ jobs: name: "Build Docs" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.10" - name: Install dependencies @@ -251,8 +260,8 @@ jobs: if: github.event_name == 'release' needs: [build-docs, tests] steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.10" - name: "Build Package" diff --git a/examples/data-dict/README.md b/examples/data-dict/README.md index 183a09e0..c36a2b7a 100644 --- a/examples/data-dict/README.md +++ b/examples/data-dict/README.md @@ -42,3 +42,9 @@ board.pin_write(data, name="username/pin-name", type="parquet", data_dict="data- Dictionary-only changes need `force_identical_write=True` because pins compares data hashes when deciding whether to write. Generated dictionaries contain `todo` notes for human decisions; review the draft before relying on it. + +The CLI is optional and the supported version is `0.0.3`. +Ordinary pins do not require it. +Generated dictionary pages block executable links and remove links with unsafe URL schemes. +Dictionary-only edits require the force option; skipped writes do not draft or render a dictionary. +Install the optional extra with `python -m pip install "pins[data-dict]"`, or provide an executable on `PATH`. diff --git a/pins/boards.py b/pins/boards.py index f1b8cf67..8ee798f1 100644 --- a/pins/boards.py +++ b/pins/boards.py @@ -239,7 +239,13 @@ def _pin_store( ) -> Meta: from pins.data_dict import prepare_metadata - metadata = prepare_metadata(metadata, data_dict, x, upload=type == "file") + if data_dict is True and metadata and "data_dict" in metadata: + raise PinsError("Supply data_dict or metadata['data_dict'], not both.") + if data_dict is not True: + metadata = prepare_metadata(metadata, data_dict, x, upload=type == "file") + dictionary_requested = data_dict is True or bool( + metadata and "data_dict" in metadata + ) _type = type if _type == "feather": warn_deprecated( @@ -281,7 +287,10 @@ def _pin_store( # create all pin data (e.g. data.txt, save object) to get the metadata. # For unversioned boards, this also will delete the most recent pin version, # ready for it to be replaced with a new one. - meta = self.prepare_pin_version( + prepare = ( + self._create_meta if dictionary_requested else self.prepare_pin_version + ) + meta = prepare( tmp_dir, x, pin_name, @@ -303,7 +312,7 @@ def _pin_store( f'The hash of pin "{name}" has not changed. Your pin will not ' "be stored." ) - if meta.user and "data_dict" in meta.user: + if data_dict is True or (meta.user and "data_dict" in meta.user): msg += ( " To save dictionary or metadata changes, use " "force_identical_write=True." @@ -311,6 +320,17 @@ def _pin_store( inform(log=_log, msg=msg) return last_meta + if data_dict is True: + meta.user = prepare_metadata( + metadata, data_dict, x, upload=_type == "file" + ) + with (Path(tmp_dir) / self.meta_factory.get_meta_name()).open( + "w" + ) as stream: + meta.to_pin_yaml(stream) + if dictionary_requested: + self._finalize_pin_version(tmp_dir, x, name, meta, versioned) + # move pin to destination ---- # create pin version folder dst_pin_path = self.construct_path([pin_name]) @@ -741,11 +761,12 @@ def prepare_pin_version( object_name, ) - # handle unversioned boards - version_setup(self, name, meta.version, versioned) - + self._finalize_pin_version(pin_dir_path, x, name, meta, versioned) return meta + def _finalize_pin_version(self, pin_dir_path, x, name, meta, versioned): + version_setup(self, name, meta.version, versioned) + def _create_meta( self, pin_dir_path, @@ -1247,7 +1268,7 @@ def path_to_deploy_version(self, name: str, version: str): def user_name(self): return self.fs.api.get_user()["username"] - def prepare_pin_version(self, pin_dir_path, x, name: str | None, *args, **kwargs): + def _create_meta(self, pin_dir_path, x, name: str | None, *args, **kwargs): # RSC pin names can have form /, but this will try to # create the object in a directory named . So we grab just # the part. @@ -1257,7 +1278,9 @@ def prepare_pin_version(self, pin_dir_path, x, name: str | None, *args, **kwargs # user passed, but guessing people want the long name? meta = super()._create_meta(pin_dir_path, x, short_name, *args, **kwargs) meta.name = name + return meta + def _finalize_pin_version(self, pin_dir_path, x, name, meta, versioned): has_dictionary = bool(meta.user and "data_dict" in meta.user) all_files = [meta.file] if isinstance(meta.file, str) else meta.file reserved = { diff --git a/pins/data_dict.py b/pins/data_dict.py index b223726b..1ee9c7c4 100644 --- a/pins/data_dict.py +++ b/pins/data_dict.py @@ -2,6 +2,10 @@ from __future__ import annotations +import base64 +import hashlib +import os +import re import shutil import subprocess import tempfile @@ -15,18 +19,41 @@ def _cli(): + bundled = None try: package = distribution("data-dict-yaml") for file in package.files or []: if file.name in ("data-dict", "data-dict.exe"): binary = package.locate_file(file) if binary.is_file(): - return str(binary) + bundled = binary + if os.access(binary, os.X_OK): + return str(binary) + break except PackageNotFoundError: pass + binary = shutil.which("data-dict") if binary: return binary + + if bundled is not None and os.name == "posix": + try: + bundled.chmod(bundled.stat().st_mode | 0o100) + except OSError as error: + raise PinsError( + f"The bundled data-dict CLI is not executable and its user " + f"execute permission could not be added: {error}. " + "Install a working data-dict CLI on PATH." + ) from error + if os.access(bundled, os.X_OK): + return str(bundled) + + if bundled is not None: + raise PinsError( + "The bundled data-dict CLI is not executable. " + "Install a working data-dict CLI on PATH." + ) raise PinsError( "Data dictionary support requires the data-dict CLI. " "Install pins[data-dict] or put data-dict on PATH." @@ -34,9 +61,27 @@ def _cli(): def _run(args): + cli = _cli() try: + version = ( + subprocess.run( + [cli, "--version"], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + .stdout.strip() + .split() + ) + version = version[-1] if version else "" + if version != "0.0.3": + raise PinsError( + f"Unsupported data-dict version {version or 'unknown'}; " + "pins requires data-dict 0.0.3." + ) subprocess.run( - [_cli(), *map(str, args)], + [cli, *map(str, args)], check=True, capture_output=True, text=True, @@ -122,6 +167,34 @@ def prepare_metadata(metadata, data_dict, x, *, upload=False): return result +def _secure_html(html, *, theme=False): + head = re.search(r"]*>", html, re.I) + if head is None or "" not in html: + raise PinsError("data-dict produced HTML without a complete head element.") + security = (Path(__file__).parent / "rsconnect/html/dictionary.js").read_text( + encoding="utf-8" + ) + integration = f"" + if theme: + integration += ( + '' + ) + html = html.replace("", integration + "", 1) + hashes = [ + "'sha256-" + + base64.b64encode(hashlib.sha256(script.encode("utf-8")).digest()).decode( + "ascii" + ) + + "'" + for script in re.findall(r"]*>(.*?)", html, re.I | re.S) + ] + policy = ( + "script-src 'self' " + " ".join(hashes) + "; object-src 'none'; base-uri 'none'" + ) + meta = '' + return html[: head.end()] + meta + html[head.end() :] + + def render(text, directory, files=(), *, filename="index.html"): """Render a self-contained index without bundling temporary YAML.""" document = _parse_portable(text) @@ -146,12 +219,10 @@ def render(text, directory, files=(), *, filename="index.html"): _run(["render", dictionary, "-o", index]) if not index.is_file(): raise PinsError(f"data-dict did not produce {filename}.") - if filename == "dictionary.html": - html = index.read_text(encoding="utf-8") - integration = '' - index.write_text( - html.replace("", integration + "", 1), encoding="utf-8" - ) + html = _secure_html( + index.read_text(encoding="utf-8"), theme=filename == "dictionary.html" + ) + index.write_text(html, encoding="utf-8") except Exception: index.unlink(missing_ok=True) raise diff --git a/pins/rsconnect/html/dictionary.js b/pins/rsconnect/html/dictionary.js new file mode 100644 index 00000000..0f2398b0 --- /dev/null +++ b/pins/rsconnect/html/dictionary.js @@ -0,0 +1,28 @@ +"use strict"; + +(() => { + function safeLink(anchor) { + const href = anchor.getAttribute("href"); + if (!href || href.startsWith("#")) return true; + try { + return ["http:", "https:", "mailto:"].includes(new URL(href, document.baseURI).protocol); + } catch { + return false; + } + } + function sanitizeLinks() { + for (const anchor of document.querySelectorAll("a[href]")) { + if (!safeLink(anchor)) anchor.removeAttribute("href"); + } + } + for (const event of ["click", "auxclick"]) { + document.addEventListener(event, event => { + const anchor = event.target.closest && event.target.closest("a[href]"); + if (anchor && !safeLink(anchor)) event.preventDefault(); + }, true); + } + new MutationObserver(sanitizeLinks).observe(document.documentElement, { + childList: true, subtree: true, attributes: true, attributeFilter: ["href"] + }); + sanitizeLinks(); +})(); diff --git a/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/dictionary.js b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/dictionary.js new file mode 100644 index 00000000..0f2398b0 --- /dev/null +++ b/pins/tests/_snapshots/test_board_pin_write_rsc_index_html/dictionary.js @@ -0,0 +1,28 @@ +"use strict"; + +(() => { + function safeLink(anchor) { + const href = anchor.getAttribute("href"); + if (!href || href.startsWith("#")) return true; + try { + return ["http:", "https:", "mailto:"].includes(new URL(href, document.baseURI).protocol); + } catch { + return false; + } + } + function sanitizeLinks() { + for (const anchor of document.querySelectorAll("a[href]")) { + if (!safeLink(anchor)) anchor.removeAttribute("href"); + } + } + for (const event of ["click", "auxclick"]) { + document.addEventListener(event, event => { + const anchor = event.target.closest && event.target.closest("a[href]"); + if (anchor && !safeLink(anchor)) event.preventDefault(); + }, true); + } + new MutationObserver(sanitizeLinks).observe(document.documentElement, { + childList: true, subtree: true, attributes: true, attributeFilter: ["href"] + }); + sanitizeLinks(); +})(); diff --git a/pins/tests/test_connect_preview.py b/pins/tests/test_connect_preview.py index 2a449d17..39719f0e 100644 --- a/pins/tests/test_connect_preview.py +++ b/pins/tests/test_connect_preview.py @@ -60,13 +60,18 @@ def handle_data(self, text): def test_standard_preview_preserves_data_and_escapes_content(tmp_path, monkeypatch): monkeypatch.setattr(dd, "_cli", lambda: pytest.fail("Unexpected CLI lookup")) attack = '' - name = 'owner/data"' + name = "owner/data&'s dataset" frame = pd.DataFrame({"value": [attack]}) connect = board_connect( server_url="https://connect.example.com", api_key="offline", cache=None ) meta = connect.prepare_pin_version( - tmp_path, frame, name, type="csv", title="Sample pin", description=attack + tmp_path, + frame, + name, + type="csv", + title="Sample & important pin", + description="Publisher's & shared description", ) html = (tmp_path / "index.html").read_text() page = Preview(html) @@ -76,9 +81,10 @@ def test_standard_preview_preserves_data_and_escapes_content(tmp_path, monkeypat assert attack not in html assert "Data preview" in html and "Download metadata" in html assert "rendered", encoding="utf-8" + ) monkeypatch.setattr(dd, "_run", render_cli) dd.render(dictionary, tmp_path, files) diff --git a/pins/tests/test_data_dict_security.py b/pins/tests/test_data_dict_security.py new file mode 100644 index 00000000..7d35c276 --- /dev/null +++ b/pins/tests/test_data_dict_security.py @@ -0,0 +1,122 @@ +import base64 +import hashlib +from html.parser import HTMLParser + +import pandas as pd +import pytest + +from pins import board_connect, board_folder +from pins import data_dict as dd +from pins.errors import PinsError +from pins.tests.test_data_dict import DICTIONARY, _require_cli + + +class PolicyParser(HTMLParser): + def __init__(self, html): + super().__init__() + self.policy = "" + self.scripts = [] + self._script = None + self.feed(html) + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == "meta" and attrs.get("http-equiv") == "Content-Security-Policy": + self.policy = attrs["content"] + if tag == "script": + self._script = "" + + def handle_data(self, data): + if self._script is not None: + self._script += data + + def handle_endtag(self, tag): + if tag == "script" and self._script is not None: + if self._script: + self.scripts.append(self._script) + self._script = None + + +def assert_policy(html): + page = PolicyParser(html) + assert "script-src 'self'" in page.policy + assert "'unsafe-inline'" not in page.policy + assert "object-src 'none'" in page.policy and "base-uri 'none'" in page.policy + for script in page.scripts: + expected = base64.b64encode(hashlib.sha256(script.encode()).digest()).decode() + assert f"'sha256-{expected}'" in page.policy + assert "javascript:alert(1)" not in page.scripts + + +def test_renderer_protects_inline_scripts_without_allowing_inline_urls(): + html = '' + protected = dd._secure_html(html, theme=True) + assert_policy(protected) + assert '' in protected + + +def test_incomplete_renderer_html_fails_and_removes_output(tmp_path, monkeypatch): + def render(args): + args[-1].write_text("missing end") + + monkeypatch.setattr(dd, "_run", render) + with pytest.raises(PinsError, match="complete head"): + dd.render(DICTIONARY, tmp_path) + assert not list(tmp_path.iterdir()) + + +def test_real_cli_applies_policy_to_dictionary_prose(tmp_path): + _require_cli() + text = DICTIONARY.replace( + " columns:", + ' description: "[click](javascript:alert%281%29)"\n columns:', + ) + text = text.replace("type: integer", "type: string\n examples: [sample]") + dd.render(text, tmp_path, filename="dictionary.html") + html = (tmp_path / "dictionary.html").read_text() + assert_policy(html) + assert "MutationObserver" in html + + +def test_identical_write_does_not_draft_or_render(tmp_path, monkeypatch, capsys): + frame = pd.DataFrame({"id": [1, 2]}) + board = board_folder(tmp_path / "board", versioned=False) + board.pin_write(frame, "data", type="csv") + monkeypatch.setattr(dd, "_cli", lambda: pytest.fail("Skipped write consulted CLI")) + board.pin_write(frame, "data", type="csv", data_dict=True) + assert "force_identical_write=True" in capsys.readouterr().err + pd.testing.assert_frame_equal(board.pin_read("data"), frame) + + +def test_connect_identical_dictionary_write_avoids_render(tmp_path, monkeypatch): + frame = pd.DataFrame({"id": [1, 2]}) + board = board_connect(server_url="https://example.com", api_key="offline", cache=None) + old = board._create_meta( + tmp_path, frame, "owner/data", type="csv", metadata={"data_dict": DICTIONARY} + ) + board.__dict__["user_name"] = "owner" + monkeypatch.setattr(board, "pin_exists", lambda name: True) + monkeypatch.setattr(board, "pin_meta", lambda name: old) + monkeypatch.setattr( + dd, "render", lambda *args, **kwargs: pytest.fail("Skipped write rendered") + ) + assert ( + board._pin_store( + frame, "owner/data", type="csv", metadata={"data_dict": DICTIONARY} + ) + == old + ) + + +def test_ordinary_writes_preserve_preparation_override(tmp_path, monkeypatch): + board = board_folder(tmp_path / "board") + prepare = board.prepare_pin_version + calls = [] + + def override(*args, **kwargs): + calls.append(True) + return prepare(*args, **kwargs) + + monkeypatch.setattr(board, "prepare_pin_version", override) + board.pin_write(pd.DataFrame({"id": [1]}), "data", type="csv") + assert calls == [True] From 4a9f92d8333164ae1561e5bb9722961bfc5f7074 Mon Sep 17 00:00:00 2001 From: Matt Conflitti Date: Tue, 17 Nov 2026 16:49:52 -0500 Subject: [PATCH 3/4] Verify renderer escaping and document dictionary board hooks --- pins/boards.py | 7 +++++++ pins/tests/test_data_dict_security.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/pins/boards.py b/pins/boards.py index 8ee798f1..5b7a494e 100644 --- a/pins/boards.py +++ b/pins/boards.py @@ -748,6 +748,13 @@ def prepare_pin_version( created: datetime | None = None, object_name: str | list[str] | None = None, ): + """Prepare a complete version for direct bundle creation. + + Dictionary writes call `_create_meta` and `_finalize_pin_version` + separately so identical-data writes skip drafting and rendering. + Custom board implementations should extend those hooks to support + dictionary writes. + """ meta = self._create_meta( pin_dir_path, x, diff --git a/pins/tests/test_data_dict_security.py b/pins/tests/test_data_dict_security.py index 7d35c276..fc6920d7 100644 --- a/pins/tests/test_data_dict_security.py +++ b/pins/tests/test_data_dict_security.py @@ -4,6 +4,7 @@ import pandas as pd import pytest +import yaml from pins import board_connect, board_folder from pins import data_dict as dd @@ -67,15 +68,21 @@ def render(args): def test_real_cli_applies_policy_to_dictionary_prose(tmp_path): _require_cli() - text = DICTIONARY.replace( - " columns:", - ' description: "[click](javascript:alert%281%29)"\n columns:', - ) - text = text.replace("type: integer", "type: string\n examples: [sample]") + document = yaml.safe_load(DICTIONARY) + attack = "" + document["label"] = attack + document["description"] = attack + table = document["tables"][0] + table["description"] = attack + " [click](javascript:alert%281%29)" + column = table["columns"][0] + column.update(type="string", label=attack, description=attack, examples=[attack]) + text = yaml.safe_dump(document) dd.render(text, tmp_path, filename="dictionary.html") html = (tmp_path / "dictionary.html").read_text() assert_policy(html) assert "MutationObserver" in html + assert "" not in html + assert "" not in html def test_identical_write_does_not_draft_or_render(tmp_path, monkeypatch, capsys): From f6430b6e8cbe6a4e9503dd452e4a10b2aa68217e Mon Sep 17 00:00:00 2001 From: Matt Conflitti Date: Tue, 17 Nov 2026 16:52:31 -0500 Subject: [PATCH 4/4] Read generated pin HTML explicitly as UTF-8 on Windows --- examples/data-dict/create_bundle.py | 10 ++++++---- pins/boards.py | 2 +- pins/tests/test_connect_preview.py | 5 +++-- pins/tests/test_data_dict.py | 18 ++++++++++++------ pins/tests/test_data_dict_security.py | 2 +- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/examples/data-dict/create_bundle.py b/examples/data-dict/create_bundle.py index bfb88787..cdb3fba9 100644 --- a/examples/data-dict/create_bundle.py +++ b/examples/data-dict/create_bundle.py @@ -66,18 +66,20 @@ def main(): metadata=meta.user, ) PinBundleManifest.add_manifest_to_directory(output) - uploaded = yaml.safe_load((output / "data.txt").read_text()) + uploaded = yaml.safe_load((output / "data.txt").read_text(encoding="utf-8")) if dictionary is None: assert "data_dict" not in uploaded["user"] else: assert uploaded["user"]["data_dict"] == dictionary pd.testing.assert_frame_equal(pd.read_parquet(output / prepared.file), data) - manifest = json.loads((output / "manifest.json").read_text()) + manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) assert manifest["metadata"]["primary_html"] == "index.html" if dictionary is not None: - assert "Data dictionary" in (output / "index.html").read_text() + assert "Data dictionary" in (output / "index.html").read_text( + encoding="utf-8" + ) assert "unused-offline" not in "".join( - p.read_text() for p in output.glob("*.txt") + p.read_text(encoding="utf-8") for p in output.glob("*.txt") ) print(output / "manifest.json") diff --git a/pins/boards.py b/pins/boards.py index 5b7a494e..c0d7b3e8 100644 --- a/pins/boards.py +++ b/pins/boards.py @@ -1337,7 +1337,7 @@ def _finalize_pin_version(self, pin_dir_path, x, name, meta, versioned): from jinja2 import Environment env = Environment(autoescape=True, trim_blocks=True, lstrip_blocks=True) - template = env.from_string(self.html_template.read_text()) + template = env.from_string(self.html_template.read_text(encoding="utf-8")) rendered = template.render(context) (Path(pin_dir_path) / "index.html").write_text(rendered, encoding="utf-8") return meta diff --git a/pins/tests/test_connect_preview.py b/pins/tests/test_connect_preview.py index 39719f0e..0f929ceb 100644 --- a/pins/tests/test_connect_preview.py +++ b/pins/tests/test_connect_preview.py @@ -73,8 +73,9 @@ def test_standard_preview_preserves_data_and_escapes_content(tmp_path, monkeypat title="Sample & important pin", description="Publisher's & shared description", ) - html = (tmp_path / "index.html").read_text() + html = (tmp_path / "index.html").read_text(encoding="utf-8") page = Preview(html) + assert " · Data pin" in html assert "#metadata" in page.links assert "Download metadata" in page.panels["metadata"] assert "Download metadata" not in page.panels["overview"] @@ -128,4 +129,4 @@ def test_preview_rejects_payload_collisions(tmp_path, monkeypatch, filename): metadata=metadata, object_name=[source.name], ) - assert source.read_text() == "original payload" + assert source.read_text(encoding="utf-8") == "original payload" diff --git a/pins/tests/test_data_dict.py b/pins/tests/test_data_dict.py index 3c9cbb3b..188194a5 100644 --- a/pins/tests/test_data_dict.py +++ b/pins/tests/test_data_dict.py @@ -99,8 +99,11 @@ def test_draft_and_connect_entrypoint(tmp_path): prepared = connect.prepare_pin_version( output, frame, "owner/data", type="csv", metadata=meta.user ) - assert "Data dictionary" in (output / "index.html").read_text() - assert yaml.safe_load((output / "data.txt").read_text())["user"] == meta.user + assert "Data dictionary" in (output / "index.html").read_text(encoding="utf-8") + assert ( + yaml.safe_load((output / "data.txt").read_text(encoding="utf-8"))["user"] + == meta.user + ) pd.testing.assert_frame_equal(pd.read_csv(output / prepared.file), frame) assert not (output / "lib").exists() parquet = tmp_path / "upload.parquet" @@ -125,7 +128,7 @@ def test_unsupported_generation_and_collision(tmp_path): index.write_text("publisher's original") with pytest.raises(PinsError, match="overwrite index.html"): dd.render(DICTIONARY, tmp_path) - assert index.read_text() == "publisher's original" + assert index.read_text(encoding="utf-8") == "publisher's original" def test_missing_cli(monkeypatch): @@ -237,10 +240,10 @@ def test_connect_parquet_profiles(tmp_path): meta = connect.prepare_pin_version( output, frame, "owner/data", type="parquet", metadata=metadata ) - overview = (output / "index.html").read_text() + overview = (output / "index.html").read_text(encoding="utf-8") assert 'src="dictionary.html"' in overview assert "Download metadata" in overview and "Use this data" in overview - html = (output / "dictionary.html").read_text() + html = (output / "dictionary.html").read_text(encoding="utf-8") assert '' in html assert "#theme-toggle{display:none}" in html embedded = json.loads( @@ -258,7 +261,10 @@ def test_connect_parquet_profiles(tmp_path): {"value": "a", "count": 2}, {"value": "b", "count": 1}, ] - assert yaml.safe_load((output / "data.txt").read_text())["user"] == metadata + assert ( + yaml.safe_load((output / "data.txt").read_text(encoding="utf-8"))["user"] + == metadata + ) assert "source" not in yaml.safe_load(metadata["data_dict"])["tables"][0] assert not list(output.glob("*.yaml")) assert (output / "pin.css").exists() and (output / "pin.js").exists() diff --git a/pins/tests/test_data_dict_security.py b/pins/tests/test_data_dict_security.py index fc6920d7..7e11daf9 100644 --- a/pins/tests/test_data_dict_security.py +++ b/pins/tests/test_data_dict_security.py @@ -78,7 +78,7 @@ def test_real_cli_applies_policy_to_dictionary_prose(tmp_path): column.update(type="string", label=attack, description=attack, examples=[attack]) text = yaml.safe_dump(document) dd.render(text, tmp_path, filename="dictionary.html") - html = (tmp_path / "dictionary.html").read_text() + html = (tmp_path / "dictionary.html").read_text(encoding="utf-8") assert_policy(html) assert "MutationObserver" in html assert "" not in html