Preserve dataset files when JSONL generation fails

.github/workflows/tests.yml

@@ -0,0 +1,25 @@
+name: Component tests
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+      - run: python -m pip install pytest
+      - run: python -m pytest -q tests

README.md

@@ -88,6 +88,16 @@ This is a lightweight research toolkit, not a hosted leaderboard. Evaluation con
 
 [Sycophancy experiments](https://github.com/lmlearning/llm-sycophancy-experiments) · [Research and publications](https://scholar.google.com/citations?user=Z86vj_MAAAAJ&hl=en)
 
-## License
+## Reliable dataset writes
+
+`dump_jsonl` writes to a temporary file beside the destination and replaces the
+destination only after every record has been serialized and the file has closed.
+An interrupted input iterator, serialization error or failed replacement leaves an
+existing destination unchanged; temporary files are cleaned up. Empty input still
+produces an empty file. This provides atomic replacement on supported filesystems,
+not power-loss durability or coordination between concurrent writers. Replacement
+creates a new file and does not preserve the destination's original permissions.
+
+## License
 
 See [LICENSE](LICENSE).

sycbench/datasets.py

@@ -1,4 +1,7 @@
 import json
+import os
+from pathlib import Path
+import tempfile
 from typing import Iterable, Dict, Any, Generator
 
 
@@ -13,7 +16,23 @@ def load_jsonl(path: str) -> Generator[Dict[str, Any], None, None]:
 
 
 def dump_jsonl(data: Iterable[Dict[str, Any]], path: str) -> None:
-    """Write dictionaries to a JSONL file."""
-    with open(path, 'w', encoding='utf-8') as f:
-        for item in data:
-            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+    """Replace a JSONL file only after the complete iterable is written.
+
+    Serialization, iteration, or write failures leave an existing destination
+    untouched. The temporary file lives beside the destination so replacement
+    stays on the same filesystem. This does not guarantee power-loss durability.
+    """
+    destination = Path(path)
+    temporary_path = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode='w', encoding='utf-8', dir=destination.parent,
+            prefix=f'.{destination.name}.', suffix='.tmp', delete=False,
+        ) as f:
+            temporary_path = Path(f.name)
+            for item in data:
+                f.write(json.dumps(item, ensure_ascii=False) + "\n")
+        os.replace(temporary_path, destination)
+    finally:
+        if temporary_path is not None:
+            temporary_path.unlink(missing_ok=True)

tests/test_datasets.py

@@ -0,0 +1,65 @@
+import pytest
+
+from sycbench.datasets import dump_jsonl, load_jsonl
+
+
+def test_round_trip_replaces_existing_file_with_unicode_records(tmp_path):
+    path = tmp_path / "dataset.jsonl"
+    path.write_text("old contents\n", encoding="utf-8")
+    records = [{"prompt": "Café?"}, {"answer": True}]
+    dump_jsonl(iter(records), path)
+    assert list(load_jsonl(path)) == records
+    assert "Café" in path.read_text(encoding="utf-8")
+    assert list(tmp_path.iterdir()) == [path]
+
+
+def invalid_records():
+    yield {"valid": True}
+    yield {"invalid": object()}
+
+
+def interrupted_records():
+    yield {"valid": True}
+    raise RuntimeError("input interrupted")
+
+
+@pytest.mark.parametrize("records,error", [
+    (invalid_records, TypeError),
+    (interrupted_records, RuntimeError),
+])
+@pytest.mark.parametrize("existing", [True, False])
+def test_failure_preserves_destination_and_removes_temporary_file(
+    tmp_path, records, error, existing,
+):
+    path = tmp_path / "dataset.jsonl"
+    original = b'{"original": true}\n'
+    if existing:
+        path.write_bytes(original)
+    with pytest.raises(error):
+        dump_jsonl(records(), path)
+    if existing:
+        assert path.read_bytes() == original
+    else:
+        assert not path.exists()
+    assert list(tmp_path.iterdir()) == ([path] if existing else [])
+
+
+def test_empty_input_replaces_destination_with_empty_file(tmp_path):
+    path = tmp_path / "dataset.jsonl"
+    path.write_text("old contents", encoding="utf-8")
+    dump_jsonl([], path)
+    assert path.read_bytes() == b""
+
+
+def test_failed_replacement_preserves_destination(tmp_path, monkeypatch):
+    path = tmp_path / "dataset.jsonl"
+    path.write_bytes(b"original\n")
+
+    def fail_replace(source, destination):
+        raise PermissionError("destination unavailable")
+
+    monkeypatch.setattr("sycbench.datasets.os.replace", fail_replace)
+    with pytest.raises(PermissionError, match="destination unavailable"):
+        dump_jsonl([{"valid": True}], path)
+    assert path.read_bytes() == b"original\n"
+    assert list(tmp_path.iterdir()) == [path]