From 5f4818471573a95320027305f4e4fdf2429ec4fe Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 11:25:19 +0200 Subject: [PATCH 01/17] test: give conformance events a DTSTART RFC 5545 requires DTSTART on a VEVENT outside a scheduling message, and Radicale refuses one without it (HTTP 400). The suite made events from a summary alone, so a strict server failed tests that were not about validity. minimal() builds the smallest valid item of each kind. Prompt: read the [untracked handover document from yesterday] and the ROADMAP and continue working on this project Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/_async/conformance.py | 50 ++++++++++++++++----------- src/calendaring/_sync/conformance.py | 50 ++++++++++++++++----------- 2 files changed, 60 insertions(+), 40 deletions(-) diff --git a/src/calendaring/_async/conformance.py b/src/calendaring/_async/conformance.py index 350b9c8..3867e75 100644 --- a/src/calendaring/_async/conformance.py +++ b/src/calendaring/_async/conformance.py @@ -122,6 +122,16 @@ async def snapshot(collection: AsyncCollection) -> list[tuple[str | None, str | Component.JOURNAL: (Journal, "add_journal", "journals", AsyncJournal, Feature.COMPONENT_JOURNAL), } +# What a component needs to be valid: RFC 5545 requires DTSTART on a VEVENT +# outside a scheduling message, and a server may refuse one without (Radicale). +REQUIRED: dict[Component, dict[str, Any]] = {Component.EVENT: {"start": at(1)}} + + +def minimal(kind: Component, summary: str, **properties: Any) -> Item: + """The smallest valid item of a kind, with ``properties`` as ``new()`` takes them.""" + return KINDS[kind][0].new(summary=summary, **REQUIRED.get(kind, {}), **properties) + + # A feature that is about one property, and a value of it to store. PROPERTY_SAMPLES: dict[Feature, tuple[str, Any, dict[str, str]]] = { Feature.CATEGORIES: ("CATEGORIES", ["work"], {}), @@ -392,14 +402,16 @@ async def test_missing_collection_raises(self, backend: AsyncBackend) -> None: @pytest.mark.parametrize("kind", list(KINDS)) async def test_add_returns_a_bound_item(self, collection: AsyncCollection, kind: Component) -> None: - data_class, add_name, list_name, bound_class, feature = KINDS[kind] - item = data_class.new(summary="conformance") + _, add_name, list_name, bound_class, feature = KINDS[kind] + item = minimal(kind, "conformance") if not writable(collection): await self.assert_read_only(collection) return if not holds(collection, kind): await self.assert_unsupported(collection, feature, lambda: collection.add(item)) - await self.assert_unsupported(collection, feature, lambda: getattr(collection, add_name)(summary="x")) + await self.assert_unsupported( + collection, feature, lambda: getattr(collection, add_name)(summary="x", **REQUIRED.get(kind, {})) + ) assert await getattr(collection, list_name)() == [] return stored = await collection.add(item) @@ -413,7 +425,7 @@ async def test_add_returns_a_bound_item(self, collection: AsyncCollection, kind: assert str(stored.component["SUMMARY"]) == "conformance" assert item.native_id is None, "add() changed its argument" assert item.etag is None, "add() changed its argument" - shorthand = await getattr(collection, add_name)(summary="shorthand") + shorthand = await getattr(collection, add_name)(summary="shorthand", **REQUIRED.get(kind, {})) assert isinstance(shorthand, bound_class) assert {i.native_id for i in await getattr(collection, list_name)()} >= {stored.native_id, shorthand.native_id} @@ -486,24 +498,24 @@ async def test_reads_are_bound_and_consistent(self, collection: AsyncCollection) async def test_get_finds_the_exact_uid(self, backend: AsyncBackend, collection: AsyncCollection) -> None: stored = [] - for kind, (data_class, _, _, bound_class, _) in KINDS.items(): + for kind, (_, _, _, bound_class, _) in KINDS.items(): if holds(collection, kind) and writable(collection): - item = data_class.new(summary=f"get {kind.value}") + item = minimal(kind, f"get {kind.value}") added = await collection.add(item) if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: assert added.uid == item.uid, f"identity.client-uid: the {kind.value} UID was replaced" - stored.append((added, bound_class)) + stored.append((added, bound_class, kind)) if not stored: pytest.skip("nothing can be stored here") - for added, bound_class in stored: + for added, bound_class, _ in stored: found = await collection.get(added.uid) assert isinstance(found, bound_class) assert found.native_id == added.native_id if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: - kind_class = stored[0][0].as_data().__class__ - suffix = f"{stored[0][0].uid}@conformance.invalid" - longer = await collection.add(kind_class.new(summary="longer", uid=f"11-{suffix}")) - shorter = await collection.add(kind_class.new(summary="shorter", uid=f"1-{suffix}")) + first, _, first_kind = stored[0] + suffix = f"{first.uid}@conformance.invalid" + longer = await collection.add(minimal(first_kind, "longer", uid=f"11-{suffix}")) + shorter = await collection.add(minimal(first_kind, "shorter", uid=f"1-{suffix}")) assert (await collection.get(shorter.uid)).native_id == shorter.native_id assert (await collection.get(longer.uid)).native_id == longer.native_id with pytest.raises(NotFoundError): @@ -523,7 +535,7 @@ async def test_add_with_native_id_keeps_the_uid(self, collection: AsyncCollectio if not writable(collection): return # test_read_only_rejects_every_write checked the writes kind = next(k for k in KINDS if holds(collection, k)) - item = KINDS[kind][0].new(summary="named and identified") + item = minimal(kind, "named and identified") name = f"conformance-{uuid.uuid4().hex[:12]}.ics" stored = await collection.add(item, native_id=name) assert stored.uid == item.uid @@ -547,15 +559,14 @@ async def test_a_listing_shows_a_save_made_elsewhere(self, collection: AsyncColl async def test_read_paths_on_seeded_items(self, collection: AsyncCollection) -> None: """What a read-only backend must pass too: everything here is stored through ``seed()``.""" kind = next(k for k in KINDS if holds(collection, k)) - new = KINDS[kind][0].new - [first] = await self.seed(collection, [new(summary="seeded first")]) + [first] = await self.seed(collection, [minimal(kind, "seeded first")]) assert (await collection.get(first.uid)).native_id == first.native_id assert (await collection.get_by_native_id(nid(first))).uid == first.uid assert (await collection.reload(first)).etag == first.etag if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: return start = await collection.changes() - [second] = await self.seed(collection, [new(summary="seeded second")]) + [second] = await self.seed(collection, [minimal(kind, "seeded second")]) since = await collection.changes(start.token) assert nid(second) in {nid(i) for i in since.changed} assert nid(first) not in {nid(i) for i in since.changed} @@ -640,7 +651,7 @@ async def test_read_only_rejects_every_write(self, backend: AsyncBackend, collec kinds = [k for k in KINDS if holds(collection, k)] if not kinds: pytest.fail("a read-only collection that holds nothing") - [stored] = await self.seed(collection, [KINDS[kinds[0]][0].new(summary="seeded")]) + [stored] = await self.seed(collection, [minimal(kinds[0], "seeded")]) other = await self.open_collection(backend, "conformance-other") for call in ( lambda: collection.add(Task.new(summary="x")), @@ -832,8 +843,7 @@ def fields(items: Sequence[AsyncItem]) -> list[tuple[Any, ...]]: async def test_listings_are_not_cut_at_a_page(self, collection: AsyncCollection) -> None: kind = next(k for k in KINDS if holds(collection, k)) - new = KINDS[kind][0].new - seeded = await self.seed(collection, [new(summary=f"paged {n}") for n in range(self.page_size_hint)]) + seeded = await self.seed(collection, [minimal(kind, f"paged {n}") for n in range(self.page_size_hint)]) mine = {nid(i) for i in seeded} assert {nid(i) for i in stored_by(await collection.search(), seeded)} == mine assert {nid(i) async for i in collection.iter_search() if i.native_id in mine} == mine @@ -969,7 +979,7 @@ async def test_changes_full_listing(self, collection: AsyncCollection) -> None: await self.assert_unsupported(collection, Feature.CHANGES, lambda: collection.changes()) return kind = next(k for k in KINDS if holds(collection, k)) - seeded = await self.seed(collection, [KINDS[kind][0].new(summary=s) for s in ("one", "two")]) + seeded = await self.seed(collection, [minimal(kind, s) for s in ("one", "two")]) changes = await collection.changes() assert {i.native_id for i in changes.changed} >= {i.native_id for i in seeded} assert changes.deleted == [] diff --git a/src/calendaring/_sync/conformance.py b/src/calendaring/_sync/conformance.py index 3586023..689617b 100644 --- a/src/calendaring/_sync/conformance.py +++ b/src/calendaring/_sync/conformance.py @@ -122,6 +122,16 @@ def snapshot(collection: Collection) -> list[tuple[str | None, str | None]]: Component.JOURNAL: (Journal, "add_journal", "journals", SyncJournal, Feature.COMPONENT_JOURNAL), } +# What a component needs to be valid: RFC 5545 requires DTSTART on a VEVENT +# outside a scheduling message, and a server may refuse one without (Radicale). +REQUIRED: dict[Component, dict[str, Any]] = {Component.EVENT: {"start": at(1)}} + + +def minimal(kind: Component, summary: str, **properties: Any) -> Item: + """The smallest valid item of a kind, with ``properties`` as ``new()`` takes them.""" + return KINDS[kind][0].new(summary=summary, **REQUIRED.get(kind, {}), **properties) + + # A feature that is about one property, and a value of it to store. PROPERTY_SAMPLES: dict[Feature, tuple[str, Any, dict[str, str]]] = { Feature.CATEGORIES: ("CATEGORIES", ["work"], {}), @@ -392,14 +402,16 @@ def test_missing_collection_raises(self, backend: Backend) -> None: @pytest.mark.parametrize("kind", list(KINDS)) def test_add_returns_a_bound_item(self, collection: Collection, kind: Component) -> None: - data_class, add_name, list_name, bound_class, feature = KINDS[kind] - item = data_class.new(summary="conformance") + _, add_name, list_name, bound_class, feature = KINDS[kind] + item = minimal(kind, "conformance") if not writable(collection): self.assert_read_only(collection) return if not holds(collection, kind): self.assert_unsupported(collection, feature, lambda: collection.add(item)) - self.assert_unsupported(collection, feature, lambda: getattr(collection, add_name)(summary="x")) + self.assert_unsupported( + collection, feature, lambda: getattr(collection, add_name)(summary="x", **REQUIRED.get(kind, {})) + ) assert getattr(collection, list_name)() == [] return stored = collection.add(item) @@ -413,7 +425,7 @@ def test_add_returns_a_bound_item(self, collection: Collection, kind: Component) assert str(stored.component["SUMMARY"]) == "conformance" assert item.native_id is None, "add() changed its argument" assert item.etag is None, "add() changed its argument" - shorthand = getattr(collection, add_name)(summary="shorthand") + shorthand = getattr(collection, add_name)(summary="shorthand", **REQUIRED.get(kind, {})) assert isinstance(shorthand, bound_class) assert {i.native_id for i in getattr(collection, list_name)()} >= {stored.native_id, shorthand.native_id} @@ -486,24 +498,24 @@ def test_reads_are_bound_and_consistent(self, collection: Collection) -> None: def test_get_finds_the_exact_uid(self, backend: Backend, collection: Collection) -> None: stored = [] - for kind, (data_class, _, _, bound_class, _) in KINDS.items(): + for kind, (_, _, _, bound_class, _) in KINDS.items(): if holds(collection, kind) and writable(collection): - item = data_class.new(summary=f"get {kind.value}") + item = minimal(kind, f"get {kind.value}") added = collection.add(item) if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: assert added.uid == item.uid, f"identity.client-uid: the {kind.value} UID was replaced" - stored.append((added, bound_class)) + stored.append((added, bound_class, kind)) if not stored: pytest.skip("nothing can be stored here") - for added, bound_class in stored: + for added, bound_class, _ in stored: found = collection.get(added.uid) assert isinstance(found, bound_class) assert found.native_id == added.native_id if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: - kind_class = stored[0][0].as_data().__class__ - suffix = f"{stored[0][0].uid}@conformance.invalid" - longer = collection.add(kind_class.new(summary="longer", uid=f"11-{suffix}")) - shorter = collection.add(kind_class.new(summary="shorter", uid=f"1-{suffix}")) + first, _, first_kind = stored[0] + suffix = f"{first.uid}@conformance.invalid" + longer = collection.add(minimal(first_kind, "longer", uid=f"11-{suffix}")) + shorter = collection.add(minimal(first_kind, "shorter", uid=f"1-{suffix}")) assert (collection.get(shorter.uid)).native_id == shorter.native_id assert (collection.get(longer.uid)).native_id == longer.native_id with pytest.raises(NotFoundError): @@ -523,7 +535,7 @@ def test_add_with_native_id_keeps_the_uid(self, collection: Collection) -> None: if not writable(collection): return # test_read_only_rejects_every_write checked the writes kind = next(k for k in KINDS if holds(collection, k)) - item = KINDS[kind][0].new(summary="named and identified") + item = minimal(kind, "named and identified") name = f"conformance-{uuid.uuid4().hex[:12]}.ics" stored = collection.add(item, native_id=name) assert stored.uid == item.uid @@ -547,15 +559,14 @@ def test_a_listing_shows_a_save_made_elsewhere(self, collection: Collection) -> def test_read_paths_on_seeded_items(self, collection: Collection) -> None: """What a read-only backend must pass too: everything here is stored through ``seed()``.""" kind = next(k for k in KINDS if holds(collection, k)) - new = KINDS[kind][0].new - [first] = self.seed(collection, [new(summary="seeded first")]) + [first] = self.seed(collection, [minimal(kind, "seeded first")]) assert (collection.get(first.uid)).native_id == first.native_id assert (collection.get_by_native_id(nid(first))).uid == first.uid assert (collection.reload(first)).etag == first.etag if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: return start = collection.changes() - [second] = self.seed(collection, [new(summary="seeded second")]) + [second] = self.seed(collection, [minimal(kind, "seeded second")]) since = collection.changes(start.token) assert nid(second) in {nid(i) for i in since.changed} assert nid(first) not in {nid(i) for i in since.changed} @@ -640,7 +651,7 @@ def test_read_only_rejects_every_write(self, backend: Backend, collection: Colle kinds = [k for k in KINDS if holds(collection, k)] if not kinds: pytest.fail("a read-only collection that holds nothing") - [stored] = self.seed(collection, [KINDS[kinds[0]][0].new(summary="seeded")]) + [stored] = self.seed(collection, [minimal(kinds[0], "seeded")]) other = self.open_collection(backend, "conformance-other") for call in ( lambda: collection.add(Task.new(summary="x")), @@ -832,8 +843,7 @@ def fields(items: Sequence[SyncItem]) -> list[tuple[Any, ...]]: def test_listings_are_not_cut_at_a_page(self, collection: Collection) -> None: kind = next(k for k in KINDS if holds(collection, k)) - new = KINDS[kind][0].new - seeded = self.seed(collection, [new(summary=f"paged {n}") for n in range(self.page_size_hint)]) + seeded = self.seed(collection, [minimal(kind, f"paged {n}") for n in range(self.page_size_hint)]) mine = {nid(i) for i in seeded} assert {nid(i) for i in stored_by(collection.search(), seeded)} == mine assert {nid(i) for i in collection.iter_search() if i.native_id in mine} == mine @@ -969,7 +979,7 @@ def test_changes_full_listing(self, collection: Collection) -> None: self.assert_unsupported(collection, Feature.CHANGES, lambda: collection.changes()) return kind = next(k for k in KINDS if holds(collection, k)) - seeded = self.seed(collection, [KINDS[kind][0].new(summary=s) for s in ("one", "two")]) + seeded = self.seed(collection, [minimal(kind, s) for s in ("one", "two")]) changes = collection.changes() assert {i.native_id for i in changes.changed} >= {i.native_id for i in seeded} assert changes.deleted == [] From 55ac82f038f735de5837575540f908ee83c58940 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 11:25:28 +0200 Subject: [PATCH 02/17] feat: add the CalDAV backend over caldav (2.1) The backend never calls caldav directly. _caldav_io.py holds AsyncDAV and its sync twin DAV, which resolve caldav's "X | Coroutine" returns once each, so a missing await in the generated code is a type error. Writes are raw PUT/DELETE with If-None-Match or If-Match, since caldav's save() sends no If-None-Match and bumps SEQUENCE. The capability table comes from caldav's server hints, mapped as API_DESIGN section 5.1 says; for a server caldav has no hints for, what the server must store is UNKNOWN. Search, changes and known-changes are still emulated client-side. The conformance suite passes in both modes against embedded Xandikos and Radicale. Prompt: read the tmp-handover* document and the ROADMAP and continue working on this project [tmp-handover*: an untracked session-handover note] Followup-Prompt: (...) then proceed with 2.1. [roadmap item 2.1, the CalDAV backend] Prompts (comments made inline in the document while reviewing it): > (on the review finding that the capability table claimed FULL for server-dependent features on every server, option 1: UNKNOWN when unhinted) we'll go for #1 Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/__init__.py | 7 + src/calendaring/_async/caldav.py | 167 ++++++++++++++++ src/calendaring/_caldav_features.py | 121 ++++++++++++ src/calendaring/_caldav_io.py | 252 ++++++++++++++++++++++++ src/calendaring/_sync/caldav.py | 167 ++++++++++++++++ src/calendaring/caldav.py | 6 + tests/_async/test_caldav_conformance.py | 38 ++++ tests/_sync/test_caldav_conformance.py | 38 ++++ tests/conftest.py | 46 +++++ tests/test_caldav_io.py | 29 +++ tools/generate_sync.py | 4 + 11 files changed, 875 insertions(+) create mode 100644 src/calendaring/_async/caldav.py create mode 100644 src/calendaring/_caldav_features.py create mode 100644 src/calendaring/_caldav_io.py create mode 100644 src/calendaring/_sync/caldav.py create mode 100644 src/calendaring/caldav.py create mode 100644 tests/_async/test_caldav_conformance.py create mode 100644 tests/_sync/test_caldav_conformance.py create mode 100644 tests/conftest.py create mode 100644 tests/test_caldav_io.py diff --git a/src/calendaring/__init__.py b/src/calendaring/__init__.py index ed526a8..e71299d 100644 --- a/src/calendaring/__init__.py +++ b/src/calendaring/__init__.py @@ -6,6 +6,8 @@ out is bound to it (``SyncTask`` / ``AsyncTask`` and so on). """ +from importlib.util import find_spec as _find_spec + try: from ._version import __version__ except ImportError: @@ -53,6 +55,11 @@ # Register the reference backend's kind ("memory") in both modes. from . import memory as _memory # noqa: E402, F401 isort: skip +# The CalDAV backend ("caldav"), when the caldav extra is installed. Checked rather than +# caught, so that an ImportError inside calendaring's own modules is not mistaken for it. +if _find_spec("caldav") is not None: + from . import caldav as _caldav # noqa: E402, F401 isort: skip + __all__ = [ "AlreadyExistsError", "AmbiguousError", diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py new file mode 100644 index 0000000..ba86e1a --- /dev/null +++ b/src/calendaring/_async/caldav.py @@ -0,0 +1,167 @@ +# Hand-written source; tools/generate_sync.py derives ../_sync/caldav.py from this file. +"""The CalDAV backend (roadmap 2.1), over the caldav library. + +It never calls caldav directly: ``AsyncDAV`` in ``_caldav_io.py`` is the +honestly typed per-mode layer, so a missing ``await`` here is a type error. +A ``native_id`` is the object's name within its calendar (the last segment +of its href, unquoted), so ``add(native_id=...)`` names the resource. +""" + +from __future__ import annotations + +import re +import uuid +from typing import Any, ClassVar +from urllib.parse import quote, unquote + +import icalendar +from icalendar_searcher import Searcher + +from .._caldav_features import backend_capabilities, collection_capabilities +from .._caldav_io import AsyncDAV, Calendar +from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError +from ..items import _items_of, _parse +from ..loss import LossPolicy +from ..types import Component, StoredObject +from .backend import AsyncBackend +from .collection import AsyncCollection + +__all__ = ["AsyncCalDAVBackend", "AsyncCalDAVCollection"] + +# The search flag caldav takes for each component. +SEARCH_FLAG = {Component.EVENT: "event", Component.TASK: "todo", Component.JOURNAL: "journal"} + +# A UID made of these is used as the resource name, as caldav does; anything +# else gets a random one, so no server has to agree on how to escape it. +SAFE_NAME = re.compile(r"^[A-Za-z0-9._@-]{1,200}$") + + +def _name_of(href: object) -> str: + return unquote(str(href).rstrip("/").rsplit("/", 1)[-1]) + + +class AsyncCalDAVCollection(AsyncCollection): + _backend: AsyncCalDAVBackend + + def __init__( + self, backend: AsyncCalDAVBackend, calendar: Calendar, *, name: str | None, components: frozenset[Component] + ) -> None: + self.calendar = calendar + self.url = str(calendar.url).rstrip("/") + "/" + super().__init__( + backend, + id=_name_of(self.url), + name=name, + components=components, + capabilities=collection_capabilities(backend._dav.client.features, components), + native=calendar, + ) + + def _href(self, native_id: str) -> str: + return self.url + quote(native_id, safe="") + + def _stored(self, obj: Any) -> StoredObject: + data = obj.data + wire = data.encode() if isinstance(data, str) else bytes(data) + return StoredObject(_name_of(obj.url), obj.etag, _parse(wire), wire, obj) + + async def _query(self, searcher: Searcher | None) -> list[StoredObject]: + # Everything of each kind the calendar holds: the generic layer filters (A3). + found: dict[str, StoredObject] = {} + for kind in sorted(self.components, key=lambda c: c.value): + flags: dict[str, Any] = {SEARCH_FLAG[kind]: True} + if kind is Component.TASK: + flags["include_completed"] = True + for obj in await self._backend._dav.search(self.calendar, **flags): + stored = self._stored(obj) + found.setdefault(stored.native_id, stored) + return list(found.values()) + + async def _fetch(self, native_id: str) -> StoredObject: + for obj in await self._backend._dav.multiget(self.calendar, [self._href(native_id)]): + if obj.data is not None: # caldav returns a missing href as an object without data + return self._stored(obj) + raise NotFoundError(f"no object {native_id!r}") + + async def _fetch_many(self, native_ids: Any) -> list[StoredObject]: + hrefs = [self._href(n) for n in native_ids] + if not hrefs: + return [] + return [self._stored(o) for o in await self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] + + async def _find_uid(self, uid: str) -> StoredObject: + return self._stored(await self._backend._dav.by_uid(self.calendar, uid)) + + async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + data = wire if wire is not None else calendar.to_ical() + uid = str(_items_of(calendar)[0].get("UID", "")) + name = native_id or (f"{uid}.ics" if SAFE_NAME.match(uid) else f"{uuid.uuid4()}.ics") + try: + _, etag = await self._backend._dav.put(self._href(name), data, create=True, etag=None) + except AlreadyExistsError as exc: + if native_id is None: + exc.taken = "uid" # the name was derived from the UID + raise + if etag is None: + return await self._fetch(name) + return StoredObject(name, etag, _parse(data)) + + async def _update( + self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None + ) -> StoredObject: + data = wire if wire is not None else calendar.to_ical() + _, new_etag = await self._backend._dav.put(self._href(native_id), data, create=False, etag=etag) + if new_etag is None: + return await self._fetch(native_id) + return StoredObject(native_id, new_etag, _parse(data)) + + async def _delete(self, native_id: str, *, etag: str | None) -> None: + await self._backend._dav.delete(self._href(native_id), etag=etag) + + async def _set_name(self, name: str) -> None: + await self._backend._dav.set_display_name(self.calendar, name) + + async def _delete_collection(self) -> None: + await self._backend._dav.delete_calendar(self.calendar) + + +class AsyncCalDAVBackend(AsyncBackend): + """``AsyncBackend.connect("https://...", kind="caldav", username=..., password=...)``.""" + + kind = "caldav" + collection_class: ClassVar[type[AsyncCalDAVCollection]] = AsyncCalDAVCollection + + def __init__(self, dav: AsyncDAV, *, id: str, loss: LossPolicy | None = None, verify: bool | None = None) -> None: + self._dav = dav + super().__init__( + id=id, capabilities=backend_capabilities(dav.client.features), native=dav.client, loss=loss, verify=verify + ) + + @classmethod + async def _connect(cls, url: str, **options: Any) -> AsyncCalDAVBackend: + if options.pop("session", None) is not None: + raise ConfigurationError("the caldav backend cannot use a caller's session yet") + token = options.pop("token", None) + if token is not None: + if options.get("password") is not None: + raise ConfigurationError("give a password or a token, not both") + options.update(password=token, auth_type="bearer") + loss = options.pop("loss", None) + verify = options.pop("verify", None) + backend_id = options.pop("id", None) or url + return cls(AsyncDAV(url, **options), id=backend_id, loss=loss, verify=verify) + + async def _wrap(self, calendar: Calendar) -> AsyncCalDAVCollection: + components = frozenset( + Component(c) for c in await self._dav.supported_components(calendar) if c in {k.value for k in Component} + ) + return self.collection_class(self, calendar, name=await self._dav.display_name(calendar), components=components) + + async def collections(self) -> list[AsyncCollection]: + return [await self._wrap(c) for c in await self._dav.calendars()] + + async def _create_collection(self, name: str, components: frozenset[Component]) -> AsyncCollection: + return await self._wrap(await self._dav.make_calendar(name, sorted(c.value for c in components))) + + async def close(self) -> None: + await self._dav.close() diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py new file mode 100644 index 0000000..de07fdd --- /dev/null +++ b/src/calendaring/_caldav_features.py @@ -0,0 +1,121 @@ +"""The CalDAV backend's capability declaration, derived from caldav's server hints (API_DESIGN.md §5.1). + +caldav's ``FeatureSet`` describes servers; this table describes what the +backend does with one. Its levels map as §5.1 says: ``full``/``quirk`` → +``FULL``; ``unsupported``, ``broken``, ``ungraceful`` → ``UNSUPPORTED``; +``fragile``, ``unknown`` → ``UNKNOWN``. For a server caldav has no hints +for, every server-dependent feature is ``UNKNOWN``. Most of caldav's hints +describe workarounds it applies itself, so few map to a feature; what a +hinted server stores beyond those is taken from caldav's save-load tests +having passed (``STORED``). What the backend does client-side whatever the +server says is declared here directly. +""" + +from __future__ import annotations + +from typing import Any + +from .capabilities import BACKEND_FEATURES, Capabilities, Capability, Feature, Support +from .types import Component + +__all__ = ["backend_capabilities", "collection_capabilities"] + +F, E, U, X = Support.FULL, Support.EMULATED, Support.UNSUPPORTED, Support.UNKNOWN + +LEVELS = { + "full": F, + "quirk": F, + "unsupported": U, + "broken": U, + "ungraceful": U, + "fragile": X, + "unknown": X, +} + +# Feature -> the caldav feature its level comes from. +FROM_SERVER = { + Feature.CREATE_COLLECTION: "create-calendar", + Feature.DELETE_COLLECTION: "delete-calendar", + Feature.COLLECTION_SET_NAME: "create-calendar.set-displayname", + Feature.RECURRENCE: "save-load.event.recurrences", + Feature.TASK_RELATIONS_PARENT: "save-load.icalendar.related-to", + Feature.TASK_RELATIONS_DEPENDS_ON: "save-load.icalendar.related-to", +} + +COMPONENT_FROM_SERVER = { + Component.EVENT: (Feature.COMPONENT_EVENT, "save-load.event"), + Component.TASK: (Feature.COMPONENT_TASK, "save-load.todo"), + Component.JOURNAL: (Feature.COMPONENT_JOURNAL, "save-load.journal"), +} + +_PENDING = "pending recurring-ical-events issue 292 (API_DESIGN.md Q3)" + +# What the backend does the same way on every server, for now. +FIXED: dict[Feature, Any] = { + **dict.fromkeys(Feature, F), + Feature.SEARCH_SERVER_SIDE: {"level": E, "note": "everything is fetched and filtered client-side"}, + Feature.SEARCH_TIME_RANGE: E, + Feature.SEARCH_TEXT: E, + Feature.CHANGES: {"level": E, "method": "etag-listing"}, + Feature.CHANGES_KNOWN: E, + Feature.MOVE: {"level": E, "note": "added to the target, then deleted here"}, + Feature.IDENTITY_FOREIGN_ID: {"level": U, "note": "the UID is native"}, + Feature.RECURRENCE_EDIT_OCCURRENCE: {"level": U, "note": _PENDING}, + Feature.RECURRENCE_EDIT_THIS_AND_FUTURE: U, + Feature.TASK_COMPLETE_RECURRING: {"level": U, "note": _PENDING}, +} + + +def _hinted(features: Any) -> bool: + """Whether caldav knows anything about this server; without hints its defaults are guesses.""" + return bool(getattr(features, "_server_features", None)) + + +def _level(features: Any, name: str) -> Support: + if not _hinted(features): + return X + return LEVELS.get(str(features.is_supported(name, str)), X) + + +# What the server must store, or honour, as sent: FULL on a hinted server, UNKNOWN on another. +STORED = frozenset( + { + Feature.WRITE, + Feature.WRITE_WIRE, + Feature.WRITE_CONDITIONAL, + Feature.IDENTITY_CLIENT_UID, + Feature.IDENTITY_CLIENT_NATIVE_ID, + Feature.PROPERTIES_PASSTHROUGH, + Feature.CATEGORIES, + Feature.ATTENDEES, + Feature.TASK_STATUS, + Feature.TASK_PRIORITY, + Feature.TASK_PERCENT_COMPLETE, + Feature.TASK_START, + Feature.TASK_DUE, + Feature.TASK_COMPLETED, + Feature.TASK_PLANNED, + Feature.TASK_ESTIMATE, + } +) + + +def _table(features: Any) -> dict[Feature, Any]: + table = dict(FIXED) + if not _hinted(features): + table.update(dict.fromkeys(STORED, X)) + for feature, name in FROM_SERVER.items(): + table[feature] = _level(features, name) + return table + + +def backend_capabilities(features: Any) -> Capabilities: + return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES}) + + +def collection_capabilities(features: Any, components: frozenset[Component]) -> Capabilities: + """The server's table, narrowed to what the calendar says it holds.""" + table = {f: v for f, v in _table(features).items() if f not in BACKEND_FEATURES} + for kind, (feature, name) in COMPONENT_FROM_SERVER.items(): + table[feature] = _level(features, name) if kind in components else U + return Capabilities({f: Capability.coerce(v) for f, v in table.items()}) diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py new file mode 100644 index 0000000..5454973 --- /dev/null +++ b/src/calendaring/_caldav_io.py @@ -0,0 +1,252 @@ +"""The hand-written per-mode layer over caldav (roadmap 2.1, 0.2 *What is not settled*, 5). + +caldav's domain objects are dual-mode: with an async client a method returns +a coroutine, with a sync one the value, and the annotation says +``X | Coroutine``. Neither type checker can tell a missing ``await`` from a +used value through that, so the generated backend never calls caldav itself. +It calls ``AsyncDAV`` here, whose twin ``DAV`` the generator's replacement map +substitutes. Each wrapper resolves the union once: ``_awaited`` / +``_returned`` also fail loudly if the client is of the wrong mode. + +Writes go through the client's ``put()`` / ``delete()`` with the +preconditions this library needs (``If-None-Match: *`` to create, +``If-Match`` to update or delete), since ``CalendarObjectResource.save()`` +sends no ``If-None-Match`` and bumps ``SEQUENCE``. Those return the status +instead of raising, and ``_status()`` maps it. +""" + +from __future__ import annotations + +import inspect +from collections.abc import Coroutine, Iterator, Mapping +from contextlib import contextmanager +from typing import Any, TypeVar, cast + +from caldav.aio import AsyncDAVClient +from caldav.calendarobjectresource import CalendarObjectResource +from caldav.collection import Calendar +from caldav.davclient import DAVClient +from caldav.elements.dav import DisplayName +from caldav.lib import error as dav_error +from caldav.lib.url import URL + +from .errors import ( + AlreadyExistsError, + AuthenticationError, + AuthorizationError, + BackendError, + CalendaringError, + ConflictError, + NotFoundError, + RateLimitError, + TransportError, +) + +__all__ = ["DAV", "AsyncDAV", "Calendar", "WriteResult"] + +T = TypeVar("T") + +CONTENT_TYPE = {"Content-Type": "text/calendar; charset=utf-8"} + +WriteResult = tuple[int, str | None] # HTTP status, and the ETag the server returned, if any + + +async def _awaited(value: T | Coroutine[Any, Any, T]) -> T: + if not inspect.iscoroutine(value): + raise TypeError(f"caldav returned {type(value).__name__} where an async client returns a coroutine") + return await cast("Coroutine[Any, Any, T]", value) + + +def _returned(value: T | Coroutine[Any, Any, T]) -> T: + if inspect.iscoroutine(value): + value.close() + raise TypeError("caldav returned a coroutine to a sync client") + return cast(T, value) + + +@contextmanager +def translated() -> Iterator[None]: + """caldav's exceptions as this library's (API_DESIGN.md §6), with caldav's as ``__cause__``.""" + try: + yield + except CalendaringError: + raise + except dav_error.NotFoundError as exc: + raise NotFoundError(str(exc)) from exc + except dav_error.AuthorizationError as exc: + status = getattr(exc, "status", None) or getattr(getattr(exc, "response", None), "status", None) + raise (AuthenticationError if status == 401 else AuthorizationError)(str(exc)) from exc + except (dav_error.ETagMismatchError, dav_error.ScheduleTagMismatchError) as exc: + raise ConflictError(str(exc)) from exc + except dav_error.RateLimitError as exc: + raise RateLimitError(str(exc), retry_after=getattr(exc, "retry_after", None)) from exc + except dav_error.DAVError as exc: + raise BackendError(str(exc)) from exc + except OSError as exc: + raise TransportError(str(exc)) from exc + + +def _status(status: int, what: str, *, creating: bool = False, body: bytes | str = b"") -> None: + """Raise for a write's status; 2xx passes.""" + if 200 <= status < 300: + return + if creating and status == 412: + raise AlreadyExistsError(f"{what}: the name is taken (412)", taken="native_id") + if creating and status in (403, 409) and "no-uid-conflict" in (body if isinstance(body, str) else body.decode()): + raise AlreadyExistsError(f"{what}: another object holds the UID (RFC 4791 §5.3.2.1)", taken="uid") + if status == 412: + raise ConflictError(f"{what}: changed since it was read (412)") + if status == 404: + raise NotFoundError(f"{what}: not found (404)") + if status == 401: + raise AuthenticationError(f"{what}: 401") + if status == 403: + raise AuthorizationError(f"{what}: 403") + if status == 429: + raise RateLimitError(f"{what}: 429") + raise BackendError(f"{what}: HTTP {status}") + + +def _write_headers(*, create: bool, etag: str | None) -> dict[str, str]: + headers = dict(CONTENT_TYPE) + if create: + headers["If-None-Match"] = "*" + elif etag is not None: + headers["If-Match"] = etag + return headers + + +def _body(response: Any) -> bytes | str: + return getattr(response, "_raw", b"") or b"" # DAVResponse keeps the body only privately + + +def _etag(headers: Mapping[str, str]) -> str | None: + """The ETag of a write's response, if the server sent a strong one (RFC 4791 §5.3.4).""" + for name, value in headers.items(): + if name.lower() == "etag": + return None if value.startswith("W/") else value + return None + + +class AsyncDAV: + """caldav, async, typed honestly.""" + + def __init__(self, url: str, **options: Any) -> None: + with translated(): + self.client = AsyncDAVClient(url=url, **options) + + async def close(self) -> None: + with translated(): + await self.client.close() + + async def calendars(self) -> list[Calendar]: + with translated(): + principal = await self.client.get_principal() + return await _awaited(principal.get_calendars()) + + async def make_calendar(self, name: str, components: list[str]) -> Calendar: + with translated(): + principal = await self.client.get_principal() + return await _awaited(principal.make_calendar(name=name, supported_calendar_component_set=components)) + + async def delete_calendar(self, calendar: Calendar) -> None: + with translated(): + await _awaited(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated + + async def supported_components(self, calendar: Calendar) -> list[str]: + with translated(): + return [str(c) for c in await _awaited(calendar.get_supported_components())] + + async def display_name(self, calendar: Calendar) -> str | None: + with translated(): + return await _awaited(calendar.get_display_name()) + + async def set_display_name(self, calendar: Calendar, name: str) -> None: + with translated(): + await _awaited(calendar.set_properties([DisplayName(name)])) + + async def search(self, calendar: Calendar, **kwargs: Any) -> list[CalendarObjectResource]: + with translated(): + return list(await _awaited(calendar.search(**kwargs))) + + async def by_uid(self, calendar: Calendar, uid: str) -> CalendarObjectResource: + with translated(): + return await _awaited(calendar.get_object_by_uid(uid)) + + async def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectResource]: + # multiget() is annotated Iterable[_CC], but returns a coroutine to an async client. + with translated(): + found: Any = calendar.multiget([URL.objectify(u) for u in urls]) + return list(await _awaited(cast(Any, found))) + + async def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: + with translated(): + response = await self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) + _status(response.status, f"PUT {url}", creating=create, body=_body(response)) + return response.status, _etag(response.headers) + + async def delete(self, url: str, *, etag: str | None) -> None: + with translated(): + response = await self.client.delete(url, None if etag is None else {"If-Match": etag}) + _status(response.status, f"DELETE {url}") + + +class DAV: + """caldav, sync, typed honestly: ``AsyncDAV``'s twin, method for method.""" + + def __init__(self, url: str, **options: Any) -> None: + with translated(): + self.client = DAVClient(url=url, **options) + + def close(self) -> None: + with translated(): + self.client.close() + + def calendars(self) -> list[Calendar]: + with translated(): + principal = self.client.get_principal() + return _returned(principal.get_calendars()) + + def make_calendar(self, name: str, components: list[str]) -> Calendar: + with translated(): + principal = self.client.get_principal() + return _returned(principal.make_calendar(name=name, supported_calendar_component_set=components)) + + def delete_calendar(self, calendar: Calendar) -> None: + with translated(): + _returned(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated + + def supported_components(self, calendar: Calendar) -> list[str]: + with translated(): + return [str(c) for c in _returned(calendar.get_supported_components())] + + def display_name(self, calendar: Calendar) -> str | None: + with translated(): + return _returned(calendar.get_display_name()) + + def set_display_name(self, calendar: Calendar, name: str) -> None: + with translated(): + _returned(calendar.set_properties([DisplayName(name)])) + + def search(self, calendar: Calendar, **kwargs: Any) -> list[CalendarObjectResource]: + with translated(): + return list(_returned(calendar.search(**kwargs))) + + def by_uid(self, calendar: Calendar, uid: str) -> CalendarObjectResource: + with translated(): + return _returned(calendar.get_object_by_uid(uid)) + + def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectResource]: + with translated(): + return list(_returned(calendar.multiget([URL.objectify(u) for u in urls]))) + + def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: + with translated(): + response = self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) + _status(response.status, f"PUT {url}", creating=create, body=_body(response)) + return response.status, _etag(response.headers) + + def delete(self, url: str, *, etag: str | None) -> None: + with translated(): + response = self.client.delete(url, None if etag is None else {"If-Match": etag}) + _status(response.status, f"DELETE {url}") diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py new file mode 100644 index 0000000..807fc86 --- /dev/null +++ b/src/calendaring/_sync/caldav.py @@ -0,0 +1,167 @@ +# Code generated by tools/generate_sync.py from _async/; DO NOT EDIT. +"""The CalDAV backend (roadmap 2.1), over the caldav library. + +It never calls caldav directly: ``AsyncDAV`` in ``_caldav_io.py`` is the +honestly typed per-mode layer, so a missing ``await`` here is a type error. +A ``native_id`` is the object's name within its calendar (the last segment +of its href, unquoted), so ``add(native_id=...)`` names the resource. +""" + +from __future__ import annotations + +import re +import uuid +from typing import Any, ClassVar +from urllib.parse import quote, unquote + +import icalendar +from icalendar_searcher import Searcher + +from .._caldav_features import backend_capabilities, collection_capabilities +from .._caldav_io import DAV, Calendar +from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError +from ..items import _items_of, _parse +from ..loss import LossPolicy +from ..types import Component, StoredObject +from .backend import Backend +from .collection import Collection + +__all__ = ["CalDAVBackend", "CalDAVCollection"] + +# The search flag caldav takes for each component. +SEARCH_FLAG = {Component.EVENT: "event", Component.TASK: "todo", Component.JOURNAL: "journal"} + +# A UID made of these is used as the resource name, as caldav does; anything +# else gets a random one, so no server has to agree on how to escape it. +SAFE_NAME = re.compile(r"^[A-Za-z0-9._@-]{1,200}$") + + +def _name_of(href: object) -> str: + return unquote(str(href).rstrip("/").rsplit("/", 1)[-1]) + + +class CalDAVCollection(Collection): + _backend: CalDAVBackend + + def __init__( + self, backend: CalDAVBackend, calendar: Calendar, *, name: str | None, components: frozenset[Component] + ) -> None: + self.calendar = calendar + self.url = str(calendar.url).rstrip("/") + "/" + super().__init__( + backend, + id=_name_of(self.url), + name=name, + components=components, + capabilities=collection_capabilities(backend._dav.client.features, components), + native=calendar, + ) + + def _href(self, native_id: str) -> str: + return self.url + quote(native_id, safe="") + + def _stored(self, obj: Any) -> StoredObject: + data = obj.data + wire = data.encode() if isinstance(data, str) else bytes(data) + return StoredObject(_name_of(obj.url), obj.etag, _parse(wire), wire, obj) + + def _query(self, searcher: Searcher | None) -> list[StoredObject]: + # Everything of each kind the calendar holds: the generic layer filters (A3). + found: dict[str, StoredObject] = {} + for kind in sorted(self.components, key=lambda c: c.value): + flags: dict[str, Any] = {SEARCH_FLAG[kind]: True} + if kind is Component.TASK: + flags["include_completed"] = True + for obj in self._backend._dav.search(self.calendar, **flags): + stored = self._stored(obj) + found.setdefault(stored.native_id, stored) + return list(found.values()) + + def _fetch(self, native_id: str) -> StoredObject: + for obj in self._backend._dav.multiget(self.calendar, [self._href(native_id)]): + if obj.data is not None: # caldav returns a missing href as an object without data + return self._stored(obj) + raise NotFoundError(f"no object {native_id!r}") + + def _fetch_many(self, native_ids: Any) -> list[StoredObject]: + hrefs = [self._href(n) for n in native_ids] + if not hrefs: + return [] + return [self._stored(o) for o in self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] + + def _find_uid(self, uid: str) -> StoredObject: + return self._stored(self._backend._dav.by_uid(self.calendar, uid)) + + def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + data = wire if wire is not None else calendar.to_ical() + uid = str(_items_of(calendar)[0].get("UID", "")) + name = native_id or (f"{uid}.ics" if SAFE_NAME.match(uid) else f"{uuid.uuid4()}.ics") + try: + _, etag = self._backend._dav.put(self._href(name), data, create=True, etag=None) + except AlreadyExistsError as exc: + if native_id is None: + exc.taken = "uid" # the name was derived from the UID + raise + if etag is None: + return self._fetch(name) + return StoredObject(name, etag, _parse(data)) + + def _update( + self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None + ) -> StoredObject: + data = wire if wire is not None else calendar.to_ical() + _, new_etag = self._backend._dav.put(self._href(native_id), data, create=False, etag=etag) + if new_etag is None: + return self._fetch(native_id) + return StoredObject(native_id, new_etag, _parse(data)) + + def _delete(self, native_id: str, *, etag: str | None) -> None: + self._backend._dav.delete(self._href(native_id), etag=etag) + + def _set_name(self, name: str) -> None: + self._backend._dav.set_display_name(self.calendar, name) + + def _delete_collection(self) -> None: + self._backend._dav.delete_calendar(self.calendar) + + +class CalDAVBackend(Backend): + """``AsyncBackend.connect("https://...", kind="caldav", username=..., password=...)``.""" + + kind = "caldav" + collection_class: ClassVar[type[CalDAVCollection]] = CalDAVCollection + + def __init__(self, dav: DAV, *, id: str, loss: LossPolicy | None = None, verify: bool | None = None) -> None: + self._dav = dav + super().__init__( + id=id, capabilities=backend_capabilities(dav.client.features), native=dav.client, loss=loss, verify=verify + ) + + @classmethod + def _connect(cls, url: str, **options: Any) -> CalDAVBackend: + if options.pop("session", None) is not None: + raise ConfigurationError("the caldav backend cannot use a caller's session yet") + token = options.pop("token", None) + if token is not None: + if options.get("password") is not None: + raise ConfigurationError("give a password or a token, not both") + options.update(password=token, auth_type="bearer") + loss = options.pop("loss", None) + verify = options.pop("verify", None) + backend_id = options.pop("id", None) or url + return cls(DAV(url, **options), id=backend_id, loss=loss, verify=verify) + + def _wrap(self, calendar: Calendar) -> CalDAVCollection: + components = frozenset( + Component(c) for c in self._dav.supported_components(calendar) if c in {k.value for k in Component} + ) + return self.collection_class(self, calendar, name=self._dav.display_name(calendar), components=components) + + def collections(self) -> list[Collection]: + return [self._wrap(c) for c in self._dav.calendars()] + + def _create_collection(self, name: str, components: frozenset[Component]) -> Collection: + return self._wrap(self._dav.make_calendar(name, sorted(c.value for c in components))) + + def close(self) -> None: + self._dav.close() diff --git a/src/calendaring/caldav.py b/src/calendaring/caldav.py new file mode 100644 index 0000000..33ffb67 --- /dev/null +++ b/src/calendaring/caldav.py @@ -0,0 +1,6 @@ +"""The CalDAV backend, in both modes (roadmap 2.1). Needs ``pip install calendaring[caldav]``.""" + +from ._async.caldav import AsyncCalDAVBackend, AsyncCalDAVCollection +from ._sync.caldav import CalDAVBackend, CalDAVCollection + +__all__ = ["AsyncCalDAVBackend", "AsyncCalDAVCollection", "CalDAVBackend", "CalDAVCollection"] diff --git a/tests/_async/test_caldav_conformance.py b/tests/_async/test_caldav_conformance.py new file mode 100644 index 0000000..f0c2659 --- /dev/null +++ b/tests/_async/test_caldav_conformance.py @@ -0,0 +1,38 @@ +# Hand-written source; tools/generate_sync.py derives ../_sync/ from this file. +"""The caldav backend passes the conformance suite against embedded Xandikos and Radicale.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from calendaring import AsyncBackend +from calendaring.testing import AsyncConformance + + +class CalDAVConformance(AsyncConformance): + # A CalDAV server is probed, not known in advance (§5.1). + allow_unknown = True + server: Any + + async def open_backend(self) -> AsyncBackend: + return await AsyncBackend.connect( + self.server.url, + kind="caldav", + username=self.server.username, + password=self.server.password, + features=self.server.config["features"], + ) + + +class TestXandikos(CalDAVConformance): + @pytest.fixture(autouse=True) + def _server(self, xandikos_server: Any) -> None: + self.server = xandikos_server + + +class TestRadicale(CalDAVConformance): + @pytest.fixture(autouse=True) + def _server(self, radicale_server: Any) -> None: + self.server = radicale_server diff --git a/tests/_sync/test_caldav_conformance.py b/tests/_sync/test_caldav_conformance.py new file mode 100644 index 0000000..b1eaa5f --- /dev/null +++ b/tests/_sync/test_caldav_conformance.py @@ -0,0 +1,38 @@ +# Code generated by tools/generate_sync.py from _async/; DO NOT EDIT. +"""The caldav backend passes the conformance suite against embedded Xandikos and Radicale.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from calendaring import Backend +from calendaring.testing import SyncConformance + + +class CalDAVConformance(SyncConformance): + # A CalDAV server is probed, not known in advance (§5.1). + allow_unknown = True + server: Any + + def open_backend(self) -> Backend: + return Backend.connect( + self.server.url, + kind="caldav", + username=self.server.username, + password=self.server.password, + features=self.server.config["features"], + ) + + +class TestXandikos(CalDAVConformance): + @pytest.fixture(autouse=True) + def _server(self, xandikos_server: Any) -> None: + self.server = xandikos_server + + +class TestRadicale(CalDAVConformance): + @pytest.fixture(autouse=True) + def _server(self, radicale_server: Any) -> None: + self.server = radicale_server diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9077ff4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,46 @@ +"""Embedded CalDAV servers for the caldav backend's conformance tests (roadmap 2.1). + +Each is started once per session, and skipped when it is not installed +(``pip install -e .[caldav,test-servers]``). +""" + +from __future__ import annotations + +import socket +from collections.abc import Iterator +from typing import Any + +import pytest + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("localhost", 0)) + return int(s.getsockname()[1]) + + +def serve(module: str, factory: str, config: dict[str, Any]) -> Iterator[Any]: + pytest.importorskip(module) + testing = pytest.importorskip("caldav.testing") + server = getattr(testing, factory)(config) + server.start() + yield server + # stop() leaves these open (RadicaleServer one end of its socketpair, + # XandikosServer its thread's event loop): https://github.com/python-caldav/caldav/issues/797 + leftovers = [getattr(server, name, None) for name in ("shutdown_socket_out", "xapp_loop")] + server.stop() + for leftover in leftovers: + if leftover is not None: + leftover.close() + + +@pytest.fixture(scope="session") +def xandikos_server() -> Iterator[Any]: + yield from serve("xandikos", "XandikosServer", {"port": free_port()}) + + +@pytest.fixture(scope="session") +def radicale_server() -> Iterator[Any]: + # RadicaleServer does not pass its port on to Radicale, which then + # listens on its default, 5232: https://github.com/python-caldav/caldav/issues/797 + yield from serve("radicale", "RadicaleServer", {}) diff --git a/tests/test_caldav_io.py b/tests/test_caldav_io.py new file mode 100644 index 0000000..dc663db --- /dev/null +++ b/tests/test_caldav_io.py @@ -0,0 +1,29 @@ +"""The caldav shim's two modes match, method for method (roadmap 2.1).""" + +from __future__ import annotations + +import inspect + +import pytest + +pytest.importorskip("caldav") + +from calendaring._caldav_io import DAV, AsyncDAV # noqa: E402 + + +def methods(cls: type) -> dict[str, inspect.Signature]: + return { + name: inspect.signature(member) + for name, member in vars(cls).items() + if inspect.isfunction(member) and (not name.startswith("_") or name == "__init__") + } + + +def test_both_modes_have_the_same_methods_and_signatures() -> None: + assert methods(AsyncDAV) == methods(DAV) + + +@pytest.mark.parametrize("name", sorted(set(methods(AsyncDAV)) - {"__init__"})) +def test_only_the_async_mode_is_async(name: str) -> None: + assert inspect.iscoroutinefunction(getattr(AsyncDAV, name)) + assert not inspect.iscoroutinefunction(getattr(DAV, name)) diff --git a/tools/generate_sync.py b/tools/generate_sync.py index e68ffe9..397b7df 100644 --- a/tools/generate_sync.py +++ b/tools/generate_sync.py @@ -41,8 +41,12 @@ "AsyncCollection": "Collection", "AsyncMemoryBackend": "MemoryBackend", "AsyncMemoryCollection": "MemoryCollection", + "AsyncCalDAVBackend": "CalDAVBackend", + "AsyncCalDAVCollection": "CalDAVCollection", # the per-mode layer (calendaring/_concurrency.py) "agather": "gather", + # the caldav bridge (calendaring/_caldav_io.py) + "AsyncDAV": "DAV", } RULE = unasync.Rule("/_async/", "/_sync/", additional_replacements=REPLACEMENTS) From 5d4b13f9d9ee998c22b7a95533d0552cd04cf9d3 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 11:30:09 +0200 Subject: [PATCH 03/17] feat: native change detection for CalDAV (2.1) changes(known=...) now lists ETags with one Depth: 1 PROPFIND instead of downloading every object, so changes.known is FULL. changes(token) uses the server's sync-collection report (RFC 6578) when caldav's hints say sync-token and its deletions work and an unknown token is refused, not answered with a full listing (Stalwart, OX). A truncated report, or one without a new token, falls back to the etag listing on the first call and is a SyncTokenError later, as is an expired token. A known= token stays on the etag listing. Prompt: read the tmp-handover* document and the ROADMAP and continue working on this project [tmp-handover*: an untracked session-handover note] Followup-Prompt: (...) then proceed with 2.1. [roadmap item 2.1, the CalDAV backend] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/_async/caldav.py | 54 ++++++++++++++++++- src/calendaring/_async/conformance.py | 4 ++ src/calendaring/_caldav_features.py | 15 +++++- src/calendaring/_caldav_io.py | 78 ++++++++++++++++++++++----- src/calendaring/_sync/caldav.py | 54 ++++++++++++++++++- src/calendaring/_sync/conformance.py | 4 ++ tests/_async/test_caldav_backend.py | 65 ++++++++++++++++++++++ tests/_sync/test_caldav_backend.py | 65 ++++++++++++++++++++++ 8 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 tests/_async/test_caldav_backend.py create mode 100644 tests/_sync/test_caldav_backend.py diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index ba86e1a..d658213 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -9,8 +9,12 @@ from __future__ import annotations +import base64 +import binascii +import json import re import uuid +from collections.abc import Mapping from typing import Any, ClassVar from urllib.parse import quote, unquote @@ -19,10 +23,11 @@ from .._caldav_features import backend_capabilities, collection_capabilities from .._caldav_io import AsyncDAV, Calendar -from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError +from ..capabilities import Feature +from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError, SyncTokenError from ..items import _items_of, _parse from ..loss import LossPolicy -from ..types import Component, StoredObject +from ..types import Component, StoredObject, SyncToken from .backend import AsyncBackend from .collection import AsyncCollection @@ -89,6 +94,51 @@ async def _fetch_many(self, native_ids: Any) -> list[StoredObject]: return [] return [self._stored(o) for o in await self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] + async def _list_etags(self) -> dict[str, str]: + return {_name_of(href): etag for href, etag in (await self._backend._dav.etags(self.url)).items()} + + async def _changes( + self, token: SyncToken | None, known: Mapping[str, str] | None + ) -> tuple[list[StoredObject], list[str], SyncToken]: + """The server's sync-collection report (RFC 6578) where it has one; else the inherited etag diff. + + A token from the etag diff (``changes(known=...)``, or a server that + could not report everything at once) stays on the etag diff. + """ + if self.capabilities.details(Feature.CHANGES).get("method") != "sync-token" or known is not None: + return await super()._changes(token, known) + if token is not None: + try: + server_token: str | None = self._decode_server_token(token) + except SyncTokenError: + return await super()._changes(token, None) # an etag-listing token, from changes(known=...) + else: + server_token = None + hrefs, gone, new_token, incomplete = await self._backend._dav.sync(self.calendar, server_token) + if new_token is None or incomplete: + # caldav follows a truncated report itself; one it gave up on cannot be made complete here. + if server_token is None: + return await super()._changes(None, None) + raise SyncTokenError("the server's sync report was incomplete; start again with changes()") + deleted = {_name_of(href) for href in gone} + changed = [name for name in dict.fromkeys(_name_of(href) for href in hrefs) if name not in deleted] + return await self._fetch_many(changed), sorted(deleted), self._encode_server_token(new_token) + + def _encode_server_token(self, server_token: str) -> SyncToken: + raw = json.dumps({"collection": self.id, "sync-token": server_token}, sort_keys=True).encode() + return SyncToken(base64.urlsafe_b64encode(raw).decode()) + + def _decode_server_token(self, token: SyncToken) -> str: + try: + data = json.loads(base64.urlsafe_b64decode(token.encode())) + issued_by = data["collection"] + server_token = data["sync-token"] + except (binascii.Error, ValueError, TypeError, KeyError) as exc: + raise SyncTokenError(f"not a token this collection issued: {token!r}") from exc + if issued_by != self.id or not isinstance(server_token, str): + raise SyncTokenError(f"a token from another collection: {issued_by!r}") + return server_token + async def _find_uid(self, uid: str) -> StoredObject: return self._stored(await self._backend._dav.by_uid(self.calendar, uid)) diff --git a/src/calendaring/_async/conformance.py b/src/calendaring/_async/conformance.py index 3867e75..c0413b3 100644 --- a/src/calendaring/_async/conformance.py +++ b/src/calendaring/_async/conformance.py @@ -1024,6 +1024,10 @@ async def test_changes_against_known(self, collection: AsyncCollection) -> None: assert result.deleted == [drop.native_id] with pytest.raises(ValueError, match="known"): await collection.changes(result.token, known=known) + later = await collection.add_task(summary="later") + following = await collection.changes(result.token) # a known= token is a token like any other + assert {i.native_id for i in following.changed} == {later.native_id} + assert following.deleted == [] async def test_a_bad_token_raises(self, collection: AsyncCollection) -> None: if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index de07fdd..bbfd216 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -57,7 +57,6 @@ Feature.SEARCH_TIME_RANGE: E, Feature.SEARCH_TEXT: E, Feature.CHANGES: {"level": E, "method": "etag-listing"}, - Feature.CHANGES_KNOWN: E, Feature.MOVE: {"level": E, "note": "added to the target, then deleted here"}, Feature.IDENTITY_FOREIGN_ID: {"level": U, "note": "the UID is native"}, Feature.RECURRENCE_EDIT_OCCURRENCE: {"level": U, "note": _PENDING}, @@ -109,6 +108,19 @@ def _table(features: Any) -> dict[Feature, Any]: return table +def _changes(features: Any) -> dict[str, Any]: + """A native change feed where the server has a reliable one; else the etag listing, which always works. + + Reliable includes refusing a token it no longer honours, with an error of any kind: a server + that answers one with a full listing (``broken``: Stalwart, OX) would look like a report where + everything changed and nothing was deleted. + """ + refuses = str(features.is_supported("sync-token.invalid-token", str)) != "broken" + if _level(features, "sync-token") is F and _level(features, "sync-token.delete") is F and refuses: + return {"level": F, "method": "sync-token"} + return {"level": E, "method": "etag-listing"} + + def backend_capabilities(features: Any) -> Capabilities: return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES}) @@ -116,6 +128,7 @@ def backend_capabilities(features: Any) -> Capabilities: def collection_capabilities(features: Any, components: frozenset[Component]) -> Capabilities: """The server's table, narrowed to what the calendar says it holds.""" table = {f: v for f, v in _table(features).items() if f not in BACKEND_FEATURES} + table[Feature.CHANGES] = _changes(features) for kind, (feature, name) in COMPONENT_FROM_SERVER.items(): table[feature] = _level(features, name) if kind in components else U return Capabilities({f: Capability.coerce(v) for f, v in table.items()}) diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py index 5454973..3abce90 100644 --- a/src/calendaring/_caldav_io.py +++ b/src/calendaring/_caldav_io.py @@ -26,7 +26,7 @@ from caldav.calendarobjectresource import CalendarObjectResource from caldav.collection import Calendar from caldav.davclient import DAVClient -from caldav.elements.dav import DisplayName +from caldav.elements.dav import DisplayName, GetEtag from caldav.lib import error as dav_error from caldav.lib.url import URL @@ -39,15 +39,27 @@ ConflictError, NotFoundError, RateLimitError, + SyncTokenError, TransportError, ) -__all__ = ["DAV", "AsyncDAV", "Calendar", "WriteResult"] +__all__ = ["DAV", "AsyncDAV", "Calendar", "SyncResult", "WriteResult"] T = TypeVar("T") CONTENT_TYPE = {"Content-Type": "text/calendar; charset=utf-8"} +# A Depth: 1 PROPFIND for the ETag of every object in a calendar (RFC 4791 §8.2.1.1 lists hrefs this way). +ETAG_PROPFIND = ( + '' +) + +VALID_SYNC_TOKEN = "{DAV:}valid-sync-token" # RFC 6578 §3.2 + +# A sync-collection report: hrefs changed, hrefs removed, the new token (None if the server gave +# none), and whether it is incomplete (truncated beyond what caldav itself follows, RFC 6578 §3.6). +SyncResult = tuple[list[str], list[str], str | None, bool] + WriteResult = tuple[int, str | None] # HTTP status, and the ETag the server returned, if any @@ -71,21 +83,28 @@ def translated() -> Iterator[None]: yield except CalendaringError: raise - except dav_error.NotFoundError as exc: - raise NotFoundError(str(exc)) from exc - except dav_error.AuthorizationError as exc: - status = getattr(exc, "status", None) or getattr(getattr(exc, "response", None), "status", None) - raise (AuthenticationError if status == 401 else AuthorizationError)(str(exc)) from exc - except (dav_error.ETagMismatchError, dav_error.ScheduleTagMismatchError) as exc: - raise ConflictError(str(exc)) from exc - except dav_error.RateLimitError as exc: - raise RateLimitError(str(exc), retry_after=getattr(exc, "retry_after", None)) from exc except dav_error.DAVError as exc: - raise BackendError(str(exc)) from exc + raise _translate(exc) from exc except OSError as exc: raise TransportError(str(exc)) from exc +def _translate(exc: dav_error.DAVError) -> CalendaringError: + message = str(exc) + if VALID_SYNC_TOKEN in getattr(exc, "preconditions", ()): + return SyncTokenError(f"the server no longer honours the sync token: {message}") + if isinstance(exc, dav_error.NotFoundError): + return NotFoundError(message) + if isinstance(exc, (dav_error.ETagMismatchError, dav_error.ScheduleTagMismatchError)): + return ConflictError(message) + if isinstance(exc, dav_error.AuthorizationError): + status = getattr(exc, "status", None) or getattr(getattr(exc, "response", None), "status", None) + return (AuthenticationError if status == 401 else AuthorizationError)(message) + if isinstance(exc, dav_error.RateLimitError): + return RateLimitError(message, retry_after=getattr(exc, "retry_after", None)) + return BackendError(message) + + def _status(status: int, what: str, *, creating: bool = False, body: bytes | str = b"") -> None: """Raise for a write's status; 2xx passes.""" if 200 <= status < 300: @@ -120,6 +139,23 @@ def _body(response: Any) -> bytes | str: return getattr(response, "_raw", b"") or b"" # DAVResponse keeps the body only privately +def _etags(response: Any, url: str) -> dict[str, str]: + """``{href: etag}`` from an ``ETAG_PROPFIND`` response, without the calendar itself.""" + out = {} + for href, props in response.expand_simple_props([GetEtag()]).items(): + etag = props.get(GetEtag.tag) + if etag and str(URL.objectify(href).path).rstrip("/") != str(URL.objectify(url).path).rstrip("/"): + out[str(href)] = etag + return out + + +def _sync_result(found: Any) -> SyncResult: + deleted = {str(u) for u in found.deleted_urls} + changed = [str(o.url) for o in found.objects if str(o.url) not in deleted] + token = None if found.sync_token is None else str(found.sync_token) + return changed, sorted(deleted), token, bool(found.truncated) + + def _etag(headers: Mapping[str, str]) -> str | None: """The ETag of a write's response, if the server sent a strong one (RFC 4791 §5.3.4).""" for name, value in headers.items(): @@ -179,6 +215,15 @@ async def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarOb found: Any = calendar.multiget([URL.objectify(u) for u in urls]) return list(await _awaited(cast(Any, found))) + async def etags(self, url: str) -> dict[str, str]: + with translated(): + return _etags(await self.client.propfind(url, body=ETAG_PROPFIND, depth=1), url) + + async def sync(self, calendar: Calendar, token: str | None) -> SyncResult: + with translated(): + found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) + return _sync_result(await _awaited(found)) + async def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: with translated(): response = await self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) @@ -240,6 +285,15 @@ def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectRe with translated(): return list(_returned(calendar.multiget([URL.objectify(u) for u in urls]))) + def etags(self, url: str) -> dict[str, str]: + with translated(): + return _etags(self.client.propfind(url, ETAG_PROPFIND, depth=1), url) + + def sync(self, calendar: Calendar, token: str | None) -> SyncResult: + with translated(): + found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) + return _sync_result(_returned(found)) + def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: with translated(): response = self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index 807fc86..9d1b4ac 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -9,8 +9,12 @@ from __future__ import annotations +import base64 +import binascii +import json import re import uuid +from collections.abc import Mapping from typing import Any, ClassVar from urllib.parse import quote, unquote @@ -19,10 +23,11 @@ from .._caldav_features import backend_capabilities, collection_capabilities from .._caldav_io import DAV, Calendar -from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError +from ..capabilities import Feature +from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError, SyncTokenError from ..items import _items_of, _parse from ..loss import LossPolicy -from ..types import Component, StoredObject +from ..types import Component, StoredObject, SyncToken from .backend import Backend from .collection import Collection @@ -89,6 +94,51 @@ def _fetch_many(self, native_ids: Any) -> list[StoredObject]: return [] return [self._stored(o) for o in self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] + def _list_etags(self) -> dict[str, str]: + return {_name_of(href): etag for href, etag in (self._backend._dav.etags(self.url)).items()} + + def _changes( + self, token: SyncToken | None, known: Mapping[str, str] | None + ) -> tuple[list[StoredObject], list[str], SyncToken]: + """The server's sync-collection report (RFC 6578) where it has one; else the inherited etag diff. + + A token from the etag diff (``changes(known=...)``, or a server that + could not report everything at once) stays on the etag diff. + """ + if self.capabilities.details(Feature.CHANGES).get("method") != "sync-token" or known is not None: + return super()._changes(token, known) + if token is not None: + try: + server_token: str | None = self._decode_server_token(token) + except SyncTokenError: + return super()._changes(token, None) # an etag-listing token, from changes(known=...) + else: + server_token = None + hrefs, gone, new_token, incomplete = self._backend._dav.sync(self.calendar, server_token) + if new_token is None or incomplete: + # caldav follows a truncated report itself; one it gave up on cannot be made complete here. + if server_token is None: + return super()._changes(None, None) + raise SyncTokenError("the server's sync report was incomplete; start again with changes()") + deleted = {_name_of(href) for href in gone} + changed = [name for name in dict.fromkeys(_name_of(href) for href in hrefs) if name not in deleted] + return self._fetch_many(changed), sorted(deleted), self._encode_server_token(new_token) + + def _encode_server_token(self, server_token: str) -> SyncToken: + raw = json.dumps({"collection": self.id, "sync-token": server_token}, sort_keys=True).encode() + return SyncToken(base64.urlsafe_b64encode(raw).decode()) + + def _decode_server_token(self, token: SyncToken) -> str: + try: + data = json.loads(base64.urlsafe_b64decode(token.encode())) + issued_by = data["collection"] + server_token = data["sync-token"] + except (binascii.Error, ValueError, TypeError, KeyError) as exc: + raise SyncTokenError(f"not a token this collection issued: {token!r}") from exc + if issued_by != self.id or not isinstance(server_token, str): + raise SyncTokenError(f"a token from another collection: {issued_by!r}") + return server_token + def _find_uid(self, uid: str) -> StoredObject: return self._stored(self._backend._dav.by_uid(self.calendar, uid)) diff --git a/src/calendaring/_sync/conformance.py b/src/calendaring/_sync/conformance.py index 689617b..d66c8ce 100644 --- a/src/calendaring/_sync/conformance.py +++ b/src/calendaring/_sync/conformance.py @@ -1024,6 +1024,10 @@ def test_changes_against_known(self, collection: Collection) -> None: assert result.deleted == [drop.native_id] with pytest.raises(ValueError, match="known"): collection.changes(result.token, known=known) + later = collection.add_task(summary="later") + following = collection.changes(result.token) # a known= token is a token like any other + assert {i.native_id for i in following.changed} == {later.native_id} + assert following.deleted == [] def test_a_bad_token_raises(self, collection: Collection) -> None: if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py new file mode 100644 index 0000000..7365e51 --- /dev/null +++ b/tests/_async/test_caldav_backend.py @@ -0,0 +1,65 @@ +# Hand-written source; tools/generate_sync.py derives ../_sync/ from this file. +"""CalDAV backend behaviour the conformance suite cannot set up: servers misbehaving (roadmap 2.1).""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from calendaring import AsyncBackend, AsyncCollection, Component, ConfigurationError, SyncTokenError + +pytest.importorskip("caldav") + + +@pytest.fixture +async def collection(xandikos_server: Any) -> AsyncIterator[AsyncCollection]: + backend = await AsyncBackend.connect( + xandikos_server.url, + kind="caldav", + username=xandikos_server.username, + password=xandikos_server.password, + features=xandikos_server.config["features"], + ) + async with backend: + made = await backend.create_collection("backend-tests", components=(Component.TASK,)) + yield made + await made.delete_collection() + + +async def incomplete(*args: Any) -> tuple[list[str], list[str], str | None, bool]: + """A sync report caldav gave up on: no token, truncated (Stalwart's initial sync).""" + return [], [], None, True + + +async def test_an_incomplete_initial_sync_falls_back_to_listing_etags( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = await collection.add_task(summary="listed anyway") + monkeypatch.setattr(collection._backend._dav, "sync", incomplete) # type: ignore[attr-defined] + result = await collection.changes() + assert [i.native_id for i in result.changed] == [stored.native_id] + later = await collection.add_task(summary="later") + assert [i.native_id for i in (await collection.changes(result.token)).changed] == [later.native_id] + + +async def test_an_incomplete_incremental_sync_asks_to_start_again( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + token = (await collection.changes()).token + monkeypatch.setattr(collection._backend._dav, "sync", incomplete) # type: ignore[attr-defined] + with pytest.raises(SyncTokenError): + await collection.changes(token) + + +async def test_a_token_is_bearer_auth() -> None: + backend = await AsyncBackend.connect("https://dav.example.com/", kind="caldav", token="s3cret") + async with backend: + assert backend.native.auth_type == "bearer" # type: ignore[attr-defined] + assert backend.native.password in ("s3cret", b"s3cret") # type: ignore[attr-defined] # bytes in sync mode + + +async def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None: + with pytest.raises(ConfigurationError, match="no_such_option"): + await AsyncBackend.connect("https://dav.example.com/", kind="caldav", no_such_option=1) diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py new file mode 100644 index 0000000..7f29b61 --- /dev/null +++ b/tests/_sync/test_caldav_backend.py @@ -0,0 +1,65 @@ +# Code generated by tools/generate_sync.py from _async/; DO NOT EDIT. +"""CalDAV backend behaviour the conformance suite cannot set up: servers misbehaving (roadmap 2.1).""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +import pytest + +from calendaring import Backend, Collection, Component, ConfigurationError, SyncTokenError + +pytest.importorskip("caldav") + + +@pytest.fixture +def collection(xandikos_server: Any) -> Iterator[Collection]: + backend = Backend.connect( + xandikos_server.url, + kind="caldav", + username=xandikos_server.username, + password=xandikos_server.password, + features=xandikos_server.config["features"], + ) + with backend: + made = backend.create_collection("backend-tests", components=(Component.TASK,)) + yield made + made.delete_collection() + + +def incomplete(*args: Any) -> tuple[list[str], list[str], str | None, bool]: + """A sync report caldav gave up on: no token, truncated (Stalwart's initial sync).""" + return [], [], None, True + + +def test_an_incomplete_initial_sync_falls_back_to_listing_etags( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = collection.add_task(summary="listed anyway") + monkeypatch.setattr(collection._backend._dav, "sync", incomplete) # type: ignore[attr-defined] + result = collection.changes() + assert [i.native_id for i in result.changed] == [stored.native_id] + later = collection.add_task(summary="later") + assert [i.native_id for i in (collection.changes(result.token)).changed] == [later.native_id] + + +def test_an_incomplete_incremental_sync_asks_to_start_again( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + token = (collection.changes()).token + monkeypatch.setattr(collection._backend._dav, "sync", incomplete) # type: ignore[attr-defined] + with pytest.raises(SyncTokenError): + collection.changes(token) + + +def test_a_token_is_bearer_auth() -> None: + backend = Backend.connect("https://dav.example.com/", kind="caldav", token="s3cret") + with backend: + assert backend.native.auth_type == "bearer" # type: ignore[attr-defined] + assert backend.native.password in ("s3cret", b"s3cret") # type: ignore[attr-defined] # bytes in sync mode + + +def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None: + with pytest.raises(ConfigurationError, match="no_such_option"): + Backend.connect("https://dav.example.com/", kind="caldav", no_such_option=1) From c341a1724ac035039a052c6e166b77e7b5e4ca91 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 11:31:19 +0200 Subject: [PATCH 04/17] feat: narrow CalDAV searches on the server (2.1) A search now asks the server only for the component kinds it can match. Event searches also pass their time range, but only where caldav's hints say the server handles open ranges and matches recurring events by their occurrences. Task and journal searches fetch everything, since servers disagree most on RFC 4791's time-range rules for VTODO, and an under-match loses data. The generic layer still filters every result, and new unit tests pin the hint mapping. The range is not widened here: caldav already widens it for CalDAV's UTC-only ranges, and a code comment says so. Prompt: read the tmp-handover* document and the ROADMAP and continue working on this project [tmp-handover*: an untracked session-handover note] Followup-Prompt: (...) then proceed with 2.1. [roadmap item 2.1, the CalDAV backend] Prompts (comments made inline in the document while reviewing it): > (on the review finding that a server-side time range could under-match floating, all-day or naive bounds) Performance-wise this seems silly - we'll end up asking the caldav server for +/- 2 days, filter it in the caldav library to be correct +/- 1 day, and then filter once more to make it correct. "The +/- 1 day and then filter"-logic in the caldav libary (sic) is there due to a design flaw in the caldav protocol (search range has to be in UTC-time) I think we should trust the backend logic to do the right thing. (This should be recorded somewhere, perhaps as code comments in the calendaring code) Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/_async/caldav.py | 16 ++++- src/calendaring/_caldav_features.py | 21 ++++++- src/calendaring/_sync/caldav.py | 16 ++++- tests/test_caldav_features.py | 91 +++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 tests/test_caldav_features.py diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index d658213..36d69b7 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -71,17 +71,29 @@ def _stored(self, obj: Any) -> StoredObject: return StoredObject(_name_of(obj.url), obj.etag, _parse(wire), wire, obj) async def _query(self, searcher: Searcher | None) -> list[StoredObject]: - # Everything of each kind the calendar holds: the generic layer filters (A3). + """Each kind the search can match, narrowed on the server only where that cannot under-match (A3).""" found: dict[str, StoredObject] = {} - for kind in sorted(self.components, key=lambda c: c.value): + for kind in sorted(self._kinds(searcher), key=lambda c: c.value): flags: dict[str, Any] = {SEARCH_FLAG[kind]: True} if kind is Component.TASK: flags["include_completed"] = True + # The range goes to caldav as given. CalDAV takes a range in UTC only, so floating + # times, all-day events and naive bounds need a wider window; caldav asks the server + # for a day more on each side and filters the result itself. Widening again here + # would cost requests and gain nothing: that is the backend's job, and it does it. + if searcher is not None and kind.value in self.capabilities.details(Feature.SEARCH_TIME_RANGE).get( + "components", () + ): + flags.update({k: v for k, v in (("start", searcher.start), ("end", searcher.end)) if v is not None}) for obj in await self._backend._dav.search(self.calendar, **flags): stored = self._stored(obj) found.setdefault(stored.native_id, stored) return list(found.values()) + def _kinds(self, searcher: Searcher | None) -> frozenset[Component]: + wanted = {kind for kind, flag in SEARCH_FLAG.items() if searcher is not None and getattr(searcher, flag, None)} + return self.components & wanted if wanted else self.components + async def _fetch(self, native_id: str) -> StoredObject: for obj in await self._backend._dav.multiget(self.calendar, [self._href(native_id)]): if obj.data is not None: # caldav returns a missing href as an object without data diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index bbfd216..7bf95dc 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -53,7 +53,7 @@ # What the backend does the same way on every server, for now. FIXED: dict[Feature, Any] = { **dict.fromkeys(Feature, F), - Feature.SEARCH_SERVER_SIDE: {"level": E, "note": "everything is fetched and filtered client-side"}, + Feature.SEARCH_SERVER_SIDE: {"level": E, "note": "the server narrows by component, and events by time range"}, Feature.SEARCH_TIME_RANGE: E, Feature.SEARCH_TEXT: E, Feature.CHANGES: {"level": E, "method": "etag-listing"}, @@ -121,6 +121,24 @@ def _changes(features: Any) -> dict[str, Any]: return {"level": E, "method": "etag-listing"} +def _time_range(features: Any) -> dict[str, Any]: + """Events only, and only where the server matches recurrences and open ranges as RFC 4791 says. + + Tasks are always filtered client-side: RFC 4791 §9.9's rules for a + ``VTODO`` are where servers disagree most, and an under-match loses data. + """ + needed = ( + "search.time-range.event", + "search.time-range.accurate", + "search.time-range.open.start", + "search.time-range.open.end", + "search.recurrences.includes-implicit.event", + ) + if all(_level(features, name) is F for name in needed): + return {"level": E, "components": ["VEVENT"]} + return {"level": E, "components": []} + + def backend_capabilities(features: Any) -> Capabilities: return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES}) @@ -129,6 +147,7 @@ def collection_capabilities(features: Any, components: frozenset[Component]) -> """The server's table, narrowed to what the calendar says it holds.""" table = {f: v for f, v in _table(features).items() if f not in BACKEND_FEATURES} table[Feature.CHANGES] = _changes(features) + table[Feature.SEARCH_TIME_RANGE] = _time_range(features) for kind, (feature, name) in COMPONENT_FROM_SERVER.items(): table[feature] = _level(features, name) if kind in components else U return Capabilities({f: Capability.coerce(v) for f, v in table.items()}) diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index 9d1b4ac..d2d5752 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -71,17 +71,29 @@ def _stored(self, obj: Any) -> StoredObject: return StoredObject(_name_of(obj.url), obj.etag, _parse(wire), wire, obj) def _query(self, searcher: Searcher | None) -> list[StoredObject]: - # Everything of each kind the calendar holds: the generic layer filters (A3). + """Each kind the search can match, narrowed on the server only where that cannot under-match (A3).""" found: dict[str, StoredObject] = {} - for kind in sorted(self.components, key=lambda c: c.value): + for kind in sorted(self._kinds(searcher), key=lambda c: c.value): flags: dict[str, Any] = {SEARCH_FLAG[kind]: True} if kind is Component.TASK: flags["include_completed"] = True + # The range goes to caldav as given. CalDAV takes a range in UTC only, so floating + # times, all-day events and naive bounds need a wider window; caldav asks the server + # for a day more on each side and filters the result itself. Widening again here + # would cost requests and gain nothing: that is the backend's job, and it does it. + if searcher is not None and kind.value in self.capabilities.details(Feature.SEARCH_TIME_RANGE).get( + "components", () + ): + flags.update({k: v for k, v in (("start", searcher.start), ("end", searcher.end)) if v is not None}) for obj in self._backend._dav.search(self.calendar, **flags): stored = self._stored(obj) found.setdefault(stored.native_id, stored) return list(found.values()) + def _kinds(self, searcher: Searcher | None) -> frozenset[Component]: + wanted = {kind for kind, flag in SEARCH_FLAG.items() if searcher is not None and getattr(searcher, flag, None)} + return self.components & wanted if wanted else self.components + def _fetch(self, native_id: str) -> StoredObject: for obj in self._backend._dav.multiget(self.calendar, [self._href(native_id)]): if obj.data is not None: # caldav returns a missing href as an object without data diff --git a/tests/test_caldav_features.py b/tests/test_caldav_features.py new file mode 100644 index 0000000..337a314 --- /dev/null +++ b/tests/test_caldav_features.py @@ -0,0 +1,91 @@ +"""The CalDAV backend's capability table follows caldav's hints as API_DESIGN.md §5.1 says (roadmap 2.1).""" + +from __future__ import annotations + +from typing import Any + +import pytest + +pytest.importorskip("caldav") + +from caldav.compatibility_hints import FeatureSet # noqa: E402 + +from calendaring._caldav_features import backend_capabilities, collection_capabilities # noqa: E402 +from calendaring.capabilities import Feature, Support # noqa: E402 +from calendaring.types import Component # noqa: E402 + +ALL = frozenset(Component) + + +def features(table: dict[str, Any] | None = None) -> Any: + return FeatureSet(table) # type: ignore[no-untyped-call] # caldav leaves it unannotated + + +def hinted(**levels: str) -> Any: + """A server caldav has hints for: ``sync_token="unsupported"`` sets ``sync-token``.""" + return features({name.replace("_", "-"): {"support": level} for name, level in levels.items()}) + + +def test_an_unhinted_server_is_unknown_not_unsupported() -> None: + table = collection_capabilities(features(), ALL) + assert table.level(Feature.RECURRENCE) is Support.UNKNOWN + assert table.level(Feature.COMPONENT_TASK) is Support.UNKNOWN + assert backend_capabilities(features()).level(Feature.CREATE_COLLECTION) is Support.UNKNOWN + + +def test_what_the_server_stores_is_unknown_without_hints() -> None: + unhinted = collection_capabilities(features(), ALL) + known = collection_capabilities(hinted(save_load="full"), ALL) + for feature in (Feature.PROPERTIES_PASSTHROUGH, Feature.CATEGORIES, Feature.WRITE_CONDITIONAL, Feature.TASK_DUE): + assert unhinted.level(feature) is Support.UNKNOWN, feature + assert known.level(feature) is Support.FULL, feature + + +def test_what_the_backend_does_itself_is_declared_whatever_the_server() -> None: + table = collection_capabilities(features(), ALL) + assert table.level(Feature.MOVE) is Support.EMULATED + assert table.level(Feature.CHANGES_KNOWN) is Support.FULL + assert table.details(Feature.CHANGES) == {"method": "etag-listing"} + assert table.details(Feature.SEARCH_TIME_RANGE) == {"components": []} + + +@pytest.mark.parametrize( + ("level", "expected"), + [ + ("full", Support.FULL), + ("quirk", Support.FULL), + ("unsupported", Support.UNSUPPORTED), + ("broken", Support.UNSUPPORTED), + ("ungraceful", Support.UNSUPPORTED), + ("fragile", Support.UNKNOWN), + ], +) +def test_levels_map_as_section_5_1_says(level: str, expected: Support) -> None: + features = hinted(**{"create-calendar": level, "save-load": "full"}) + assert backend_capabilities(features).level(Feature.CREATE_COLLECTION) is expected + + +def test_a_calendar_without_a_component_cannot_hold_it() -> None: + table = collection_capabilities(hinted(save_load="full"), frozenset({Component.TASK})) + assert table.level(Feature.COMPONENT_TASK) is Support.FULL + assert table.level(Feature.COMPONENT_EVENT) is Support.UNSUPPORTED + + +def test_sync_token_is_used_only_where_deletions_are_reported_too() -> None: + assert collection_capabilities(hinted(sync_token="full"), ALL).details(Feature.CHANGES) == {"method": "sync-token"} + broken_delete = hinted(**{"sync-token": "full", "sync-token.delete": "unsupported"}) + assert collection_capabilities(broken_delete, ALL).details(Feature.CHANGES) == {"method": "etag-listing"} + + +def test_sync_token_is_not_used_where_an_expired_token_is_answered_with_a_listing() -> None: + # Stalwart, OX: 207 and everything for a token they never issued, so deletions would be lost. + broken = hinted(**{"sync-token": "full", "sync-token.invalid-token": "broken"}) + assert collection_capabilities(broken, ALL).details(Feature.CHANGES) == {"method": "etag-listing"} + + +def test_time_range_narrows_events_only_where_recurrences_match_too() -> None: + assert collection_capabilities(hinted(search="full"), ALL).details(Feature.SEARCH_TIME_RANGE) == { + "components": ["VEVENT"] + } + no_implicit = hinted(**{"search": "full", "search.recurrences.includes-implicit.event": "unsupported"}) + assert collection_capabilities(no_implicit, ALL).details(Feature.SEARCH_TIME_RANGE) == {"components": []} From 2f2b49e210ab9f48d28f449d8cc4b135d122ef03 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 11:35:49 +0200 Subject: [PATCH 05/17] fix: verify ignores harmless server rewrites verify=True compared alarms first come first served, property lines as text, so a sent alarm matching two stored ones could take the one another sent alarm needed, and a server's harmless rewrites inside an alarm counted as losses. Subcomponents are now matched every way, and compared as the top level is. CATEGORIES compare as a set however a server splits them, and on a single event an aware date-time stored in UTC instead of with its TZID compares as the same instant. In a recurring series it does not, since the occurrences would shift at a DST change, and a floating time converted to UTC is a difference too. As at the top level, only what was sent is compared: properties or alarms a server adds are no difference. Prompt: read the [untracked handover document from yesterday] and the ROADMAP and continue working on this project Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/_mapping.py | 100 ++++++++++++++++++++++-------- tests/test_verify.py | 118 ++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 25 deletions(-) create mode 100644 tests/test_verify.py diff --git a/src/calendaring/_mapping.py b/src/calendaring/_mapping.py index e887cbe..40d98fa 100644 --- a/src/calendaring/_mapping.py +++ b/src/calendaring/_mapping.py @@ -64,6 +64,9 @@ "SIBLING": Feature.TASK_RELATIONS_PARENT, } +# Single date-time properties a server may store in UTC instead of with their TZID. +INSTANTS = frozenset({"DTSTART", "DTEND", "DUE", "COMPLETED", "RECURRENCE-ID", X_PLANNED_START, X_PLANNED_END}) + # Properties servers legitimately change, and identity a backend may assign (§5.3). NOT_VERIFIED = frozenset({"DTSTAMP", "SEQUENCE", "LAST-MODIFIED", "UID"}) @@ -211,7 +214,10 @@ def apply() -> None: def property_lines(component: icalendar.Component, name: str) -> list[bytes]: - """One property's content lines, unfolded and sorted, for an order-insensitive comparison.""" + """One property's content lines, unfolded and sorted, for an order-insensitive comparison. + + Only the component's own: a ``VALARM``'s ``DESCRIPTION`` is not its event's. + """ out = [] for line in _lines(component): head = line.split(b":", 1)[0].split(b";", 1)[0] @@ -220,6 +226,61 @@ def property_lines(component: icalendar.Component, name: str) -> list[bytes]: return sorted(out) +def _comparable(component: icalendar.Component, name: str) -> list[Any]: + """What ``verify`` compares for a property, so that a server's harmless rewrites compare equal. + + ``CATEGORIES`` is a set however it is split over lines and commas, and + an aware date-time on a single event is an instant, whether it is stored + with its TZID or in UTC. A floating time is not an instant, and in a + recurring series a TZID is not either, so converting one is a change. + """ + if name == "CATEGORIES": + return sorted({str(c) for value in _values(component, name) for c in getattr(value, "cats", [value])}) + if name in INSTANTS and not isinstance(component.get(name), list) and not _recurs(component): + value = component.decoded(name) + if isinstance(value, datetime) and value.tzinfo is not None: + return [value.astimezone(UTC)] + return list(property_lines(component, name)) + + +def _recurs(component: icalendar.Component) -> bool: + """A series' times are local times: moved to UTC, its occurrences shift at a DST change.""" + return "RRULE" in component or "RDATE" in component + + +def _stored_as_sent(sent: icalendar.Component, stored: icalendar.Component) -> bool: + """Whether ``stored`` holds everything ``sent`` did, as the top level is compared: what a server adds is fine.""" + if sent.name != stored.name: + return False + if any( + _comparable(sent, name) != _comparable(stored, name) for name in dict.fromkeys(sent) if name not in NOT_VERIFIED + ): + return False + return _matched(sent.subcomponents, stored.subcomponents) + + +def _matched(sent: list[icalendar.Component], stored: list[icalendar.Component]) -> bool: + """Each sent subcomponent has a stored one of its own, in any order. + + Tried every way, not first come first served: a sent alarm that matches two stored + ones must not take the one another sent alarm needs. A component has few. + """ + if not sent: + return True + first, rest = sent[0], sent[1:] + return any( + _stored_as_sent(first, other) and _matched(rest, stored[:i] + stored[i + 1 :]) for i, other in enumerate(stored) + ) + + +def _subcomponents(component: icalendar.Component, kind: str) -> list[icalendar.Component]: + return [c for c in component.subcomponents if c.name == kind] + + +def _ical(components: list[icalendar.Component]) -> str: + return b"".join(c.to_ical() for c in components).decode() + + def _by_occurrence(calendar: icalendar.Calendar) -> dict[str | None, icalendar.Component]: out = {} for component in calendar.subcomponents: @@ -242,29 +303,18 @@ def differences(sent: icalendar.Calendar, stored: icalendar.Calendar) -> list[Lo for name in dict.fromkeys(component): if name in NOT_VERIFIED: continue - mine = property_lines(component, name) - theirs = property_lines(other, name) - if mine != theirs: + if _comparable(component, name) != _comparable(other, name): + mine, theirs = property_lines(component, name), property_lines(other, name) out.append(Loss(name, b"\n".join(mine).decode(), b"\n".join(theirs).decode() or None)) - for kind in dict.fromkeys(str(c.name) for c in component.subcomponents): - candidates = [c for c in other.subcomponents if c.name == kind] - missing = [] - for sent_part in (c for c in component.subcomponents if c.name == kind): - match = next((c for c in candidates if _covers(c, sent_part)), None) - if match is None: - missing.append(sent_part) - else: - candidates.remove(match) - if missing: - sent_text = b"".join(c.to_ical() for c in missing).decode() - out.append(Loss(kind, sent_text, b"".join(c.to_ical() for c in candidates).decode() or None)) + for kind in dict.fromkeys(str(c.name) for c in [*component.subcomponents, *other.subcomponents]): + mine_subs = _subcomponents(component, kind) + theirs_subs = _subcomponents(other, kind) + if not _matched(mine_subs, theirs_subs): + # Name what was not kept: the sent ones nothing stored matches, and the stored ones that match + # nothing sent. Where each matches but they cannot be paired off, that is all of them. + lost = [m for m in mine_subs if not any(_stored_as_sent(m, t) for t in theirs_subs)] + instead = [t for t in theirs_subs if not any(_stored_as_sent(m, t) for m in mine_subs)] + if not lost: + lost, instead = mine_subs, theirs_subs + out.append(Loss(kind, _ical(lost), _ical(instead) or None)) return out - - -def _covers(stored: icalendar.Component, sent: icalendar.Component) -> bool: - """Whether ``stored`` (an alarm) holds every property of ``sent`` as sent; servers may add their own.""" - return all( - property_lines(stored, name) == property_lines(sent, name) - for name in dict.fromkeys(sent) - if name not in NOT_VERIFIED - ) diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100644 index 0000000..9d1b2d7 --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,118 @@ +"""What ``verify=True`` counts as a difference (roadmap 2.1): servers' harmless rewrites are not, a lost alarm is.""" + +from __future__ import annotations + +import icalendar + +from calendaring._mapping import differences + + +def calendar(*event_lines: str, extra: str = "") -> icalendar.Calendar: + lines = [ + "BEGIN:VCALENDAR", + "VERSION:2.0", + "PRODID:-//test//EN", + "BEGIN:VEVENT", + "UID:verify-1", + "DTSTAMP:20261019T100000Z", + "SUMMARY:meeting", + *event_lines, + "END:VEVENT", + "END:VCALENDAR", + ] + return icalendar.Calendar.from_ical("\r\n".join(lines) + "\r\n" + extra) + + +ALARM = ("BEGIN:VALARM", "ACTION:DISPLAY", "DESCRIPTION:wake up", "TRIGGER:-PT15M", "END:VALARM") + + +def names(sent: icalendar.Calendar, stored: icalendar.Calendar) -> list[str]: + return [loss.property for loss in differences(sent, stored)] + + +def test_identical_data_has_no_differences() -> None: + assert names(calendar("DTSTART:20261020T100000Z", *ALARM), calendar("DTSTART:20261020T100000Z", *ALARM)) == [] + + +def test_a_changed_alarm_is_a_difference() -> None: + changed = ("BEGIN:VALARM", "ACTION:DISPLAY", "DESCRIPTION:wake up", "TRIGGER:-PT5M", "END:VALARM") + assert names(calendar("DTSTART:20261020T100000Z", *ALARM), calendar("DTSTART:20261020T100000Z", *changed)) == [ + "VALARM" + ] + + +def test_an_alarm_description_is_not_the_events() -> None: + sent = calendar("DESCRIPTION:agenda", "DTSTART:20261020T100000Z", *ALARM) + stored = calendar("DESCRIPTION:agenda", "DTSTART:20261020T100000Z", *ALARM) + assert names(sent, stored) == [] + assert names(sent, calendar("DTSTART:20261020T100000Z", *ALARM)) == ["DESCRIPTION"] + + +def test_alarms_compare_in_any_order() -> None: + other = ("BEGIN:VALARM", "ACTION:AUDIO", "TRIGGER:-PT1H", "END:VALARM") + sent = calendar("DTSTART:20261020T100000Z", *ALARM, *other) + stored = calendar("DTSTART:20261020T100000Z", *other, *ALARM) + assert names(sent, stored) == [] + + +def test_split_categories_are_the_same_categories() -> None: + assert names(calendar("CATEGORIES:work,home"), calendar("CATEGORIES:work", "CATEGORIES:home")) == [] + assert names(calendar("CATEGORIES:work,home"), calendar("CATEGORIES:home,work")) == [] + assert names(calendar("CATEGORIES:work,home"), calendar("CATEGORIES:work")) == ["CATEGORIES"] + + +def test_quoted_and_unquoted_parameters_are_the_same() -> None: + sent = calendar('ATTENDEE;CN="Alice Example":mailto:alice@example.com') + stored = calendar("ATTENDEE;CN=Alice Example:mailto:alice@example.com") + assert names(sent, stored) == [] + + +def test_a_time_converted_to_utc_is_the_same_instant() -> None: + sent = calendar("DTSTART;TZID=Europe/Oslo:20261020T100000", extra="") + assert names(sent, calendar("DTSTART:20261020T080000Z")) == [] + assert names(sent, calendar("DTSTART:20261020T090000Z")) == ["DTSTART"] + + +def test_a_floating_time_converted_to_utc_is_a_difference() -> None: + assert names(calendar("DTSTART:20261020T100000"), calendar("DTSTART:20261020T100000Z")) == ["DTSTART"] + + +def test_a_recurring_series_moved_to_utc_is_a_difference() -> None: + """Every occurrence after a DST change would move by an hour.""" + sent = calendar("DTSTART;TZID=Europe/Oslo:20261020T100000", "RRULE:FREQ=WEEKLY;COUNT=30") + stored = calendar("DTSTART:20261020T080000Z", "RRULE:FREQ=WEEKLY;COUNT=30") + assert names(sent, stored) == ["DTSTART"] + + +def test_a_property_a_server_adds_to_an_alarm_is_no_difference() -> None: + stored_alarm = (*ALARM[:-1], "ACKNOWLEDGED:20261020T090000Z", "X-MOZ-LASTACK:20261020T090000Z", "END:VALARM") + assert ( + names(calendar("DTSTART:20261020T100000Z", *ALARM), calendar("DTSTART:20261020T100000Z", *stored_alarm)) == [] + ) + + +def test_an_alarm_a_server_adds_is_no_difference() -> None: + assert names(calendar("DTSTART:20261020T100000Z"), calendar("DTSTART:20261020T100000Z", *ALARM)) == [] + + +def test_alarms_are_matched_so_that_each_finds_its_own() -> None: + short = ("BEGIN:VALARM", "ACTION:DISPLAY", "TRIGGER:-PT15M", "END:VALARM") + sent = calendar("DTSTART:20261020T100000Z", *short, *ALARM) + # Stored in the other order: the short alarm also matches the full one, which the full one needs. + stored = calendar("DTSTART:20261020T100000Z", *ALARM, *short) + assert names(sent, stored) == [] + + +def test_a_loss_names_only_the_alarms_that_were_not_kept() -> None: + other = ("BEGIN:VALARM", "ACTION:AUDIO", "TRIGGER:-PT1H", "END:VALARM") + added = ("BEGIN:VALARM", "ACTION:DISPLAY", "DESCRIPTION:server default", "TRIGGER:-PT5M", "END:VALARM") + sent = calendar("DTSTART:20261020T100000Z", *ALARM, *other) + [dropped] = differences(sent, calendar("DTSTART:20261020T100000Z", *other)) + assert dropped.property == "VALARM" + assert "wake up" in str(dropped.sent) + assert "AUDIO" not in str(dropped.sent) + assert dropped.stored is None + [replaced] = differences(sent, calendar("DTSTART:20261020T100000Z", *other, *added)) + assert "AUDIO" not in str(replaced.sent) + assert "server default" in str(replaced.stored) + assert "AUDIO" not in str(replaced.stored) From df70d7b04177539363f0455a5f818d05825a0076 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 12:52:57 +0200 Subject: [PATCH 06/17] feat: make caldav an optional extra (2.1) caldav pulls in niquests and lxml, which a user of the files or tracker backends does not need, so it is the extra calendaring[caldav]. Without it, kind="caldav" is a ConfigurationError naming the pip command. An http(s) URL without kind= now means CalDAV, until roadmap 2.3's feed-or-CalDAV probe replaces that. The dev extra installs caldav and the embedded Xandikos and Radicale servers, so CI runs the CalDAV conformance suite. Prompt: [caldav should be an optional dependency] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- docs/design/API_DESIGN.md | 2 +- pyproject.toml | 4 ++++ src/calendaring/_async/backend.py | 10 ++++++++-- src/calendaring/_sync/backend.py | 10 ++++++++-- tests/_async/test_config.py | 2 +- tests/_async/test_generic.py | 15 ++++++++++++++- tests/_sync/test_config.py | 2 +- tests/_sync/test_generic.py | 15 ++++++++++++++- 8 files changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 4d0f53b..0c9f8db 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -285,7 +285,7 @@ class AsyncBackend: - `connect` returns the subclass for the kind it picks from the URL when `kind` is not given: `file:` or a path → files; `webcal:` → feed (fetched as `https:`, [prior art - Part 2](PRIOR_ART_AND_DECISIONS.md#part-2-standards)); `http(s):` → probe, which is [roadmap 2.3](ROADMAP.md#23-icalendar-feed-backend-read-only-http)'s "feed or CalDAV" detection; + Part 2](PRIOR_ART_AND_DECISIONS.md#part-2-standards)); `http(s):` → probe, which is [roadmap 2.3](ROADMAP.md#23-icalendar-feed-backend-read-only-http)'s "feed or CalDAV" detection (until then, `http(s):` means CalDAV, so an `https:` feed needs `kind="feed"`); RFC 6764 discovery for a bare domain comes from `caldav`. Trackers cannot be detected and need `kind=`. - `session` is the caller-owned HTTP session that Home Assistant's diff --git a/pyproject.toml b/pyproject.toml index 1681bbe..5a06eb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,9 +30,13 @@ dependencies = [ ] [project.optional-dependencies] +caldav = ["caldav>=3.5"] +# Embedded CalDAV servers the caldav backend's conformance tests run against. +test-servers = ["caldav>=3.5", "xandikos>=0.2", "radicale>=3.2"] # ruff and mypy are pinned exactly; keep ruff in step with .pre-commit-config.yaml. # So is unasync: its output is committed, and the freshness test compares bytes. dev = [ + "calendaring[caldav,test-servers]", "pytest>=8.0", "pytest-asyncio>=1.0", "unasync==0.6.0", diff --git a/src/calendaring/_async/backend.py b/src/calendaring/_async/backend.py index d240125..ec5d6cf 100644 --- a/src/calendaring/_async/backend.py +++ b/src/calendaring/_async/backend.py @@ -20,8 +20,11 @@ __all__ = ["AsyncBackend"] # URL scheme -> backend kind, for connect() without kind= (§2.2). http(s) -# needs the "feed or CalDAV" probe of roadmap 2.3, and trackers need kind=. -SCHEMES = {"": "files", "file": "files", "webcal": "feed", "memory": "memory"} +# is CalDAV until the "feed or CalDAV" probe of roadmap 2.3, and trackers need kind=. +SCHEMES = {"": "files", "file": "files", "webcal": "feed", "memory": "memory", "http": "caldav", "https": "caldav"} + +# Backend kinds that come with an optional extra, and its name. +EXTRAS = {"caldav": "caldav"} class AsyncBackend(ABC): @@ -82,6 +85,9 @@ async def connect( try: klass = kinds[kind] except KeyError: + if kind in EXTRAS: + extra = EXTRAS[kind] + raise ConfigurationError(f"the {kind} backend needs: pip install 'calendaring[{extra}]'") from None raise ConfigurationError(f"no backend of kind {kind!r} (known: {sorted(kinds)})") from None return await klass._connect(url, username=username, password=password, token=token, session=session, **options) diff --git a/src/calendaring/_sync/backend.py b/src/calendaring/_sync/backend.py index 6b6ae5d..3b0d573 100644 --- a/src/calendaring/_sync/backend.py +++ b/src/calendaring/_sync/backend.py @@ -20,8 +20,11 @@ __all__ = ["Backend"] # URL scheme -> backend kind, for connect() without kind= (§2.2). http(s) -# needs the "feed or CalDAV" probe of roadmap 2.3, and trackers need kind=. -SCHEMES = {"": "files", "file": "files", "webcal": "feed", "memory": "memory"} +# is CalDAV until the "feed or CalDAV" probe of roadmap 2.3, and trackers need kind=. +SCHEMES = {"": "files", "file": "files", "webcal": "feed", "memory": "memory", "http": "caldav", "https": "caldav"} + +# Backend kinds that come with an optional extra, and its name. +EXTRAS = {"caldav": "caldav"} class Backend(ABC): @@ -82,6 +85,9 @@ def connect( try: klass = kinds[kind] except KeyError: + if kind in EXTRAS: + extra = EXTRAS[kind] + raise ConfigurationError(f"the {kind} backend needs: pip install 'calendaring[{extra}]'") from None raise ConfigurationError(f"no backend of kind {kind!r} (known: {sorted(kinds)})") from None return klass._connect(url, username=username, password=password, token=token, session=session, **options) diff --git a/tests/_async/test_config.py b/tests/_async/test_config.py index 81d29e1..baf789b 100644 --- a/tests/_async/test_config.py +++ b/tests/_async/test_config.py @@ -98,7 +98,7 @@ async def test_url_must_be_a_string(tmp_path: Path) -> None: async def test_error_does_not_repeat_credentials_in_the_url(tmp_path: Path) -> None: - path = write(tmp_path, {"default": {"url": "https://alice:s3cret@dav.example.com/"}}) + path = write(tmp_path, {"default": {"url": "gopher://alice:s3cret@dav.example.com/"}}) with pytest.raises(ConfigurationError, match="section default") as info: await AsyncWorkspace.from_config(path) assert "s3cret" not in str(info.value) diff --git a/tests/_async/test_generic.py b/tests/_async/test_generic.py index 0d111dd..7bd0ac0 100644 --- a/tests/_async/test_generic.py +++ b/tests/_async/test_generic.py @@ -45,7 +45,7 @@ async def test_connect_picks_the_kind_from_the_url() -> None: assert isinstance(backend, AsyncMemoryBackend) assert backend.id == "from-url" with pytest.raises(ConfigurationError): - await AsyncBackend.connect("https://caldav.example.com/") + await AsyncBackend.connect("gopher://calendar.example.com/") with pytest.raises(ConfigurationError): await AsyncBackend.connect("memory:", kind="no-such-kind") @@ -374,3 +374,16 @@ class Backend(AsyncMemoryBackend): await (await Backend(id="home").collection("default")).changes(token) # Another store under the same label: the default cannot tell them apart. await (await Backend(id="work").collection("default")).changes(token) + + +async def test_a_backend_from_a_missing_extra_says_what_to_install(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(AsyncBackend, "_kinds", {k: v for k, v in AsyncBackend._kinds.items() if k != "caldav"}) + with pytest.raises(ConfigurationError, match=r"pip install 'calendaring\[caldav\]'"): + await AsyncBackend.connect("https://caldav.example.com/", kind="caldav") + + +async def test_http_urls_are_caldav_until_the_feed_probe_exists() -> None: + pytest.importorskip("caldav") + backend = await AsyncBackend.connect("https://caldav.example.com/dav/", username="u", password="p") + async with backend: + assert backend.kind == "caldav" diff --git a/tests/_sync/test_config.py b/tests/_sync/test_config.py index 623712c..152cb76 100644 --- a/tests/_sync/test_config.py +++ b/tests/_sync/test_config.py @@ -98,7 +98,7 @@ def test_url_must_be_a_string(tmp_path: Path) -> None: def test_error_does_not_repeat_credentials_in_the_url(tmp_path: Path) -> None: - path = write(tmp_path, {"default": {"url": "https://alice:s3cret@dav.example.com/"}}) + path = write(tmp_path, {"default": {"url": "gopher://alice:s3cret@dav.example.com/"}}) with pytest.raises(ConfigurationError, match="section default") as info: Workspace.from_config(path) assert "s3cret" not in str(info.value) diff --git a/tests/_sync/test_generic.py b/tests/_sync/test_generic.py index 821235d..5cd77b7 100644 --- a/tests/_sync/test_generic.py +++ b/tests/_sync/test_generic.py @@ -45,7 +45,7 @@ def test_connect_picks_the_kind_from_the_url() -> None: assert isinstance(backend, MemoryBackend) assert backend.id == "from-url" with pytest.raises(ConfigurationError): - Backend.connect("https://caldav.example.com/") + Backend.connect("gopher://calendar.example.com/") with pytest.raises(ConfigurationError): Backend.connect("memory:", kind="no-such-kind") @@ -374,3 +374,16 @@ class Backend(MemoryBackend): (Backend(id="home").collection("default")).changes(token) # Another store under the same label: the default cannot tell them apart. (Backend(id="work").collection("default")).changes(token) + + +def test_a_backend_from_a_missing_extra_says_what_to_install(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Backend, "_kinds", {k: v for k, v in Backend._kinds.items() if k != "caldav"}) + with pytest.raises(ConfigurationError, match=r"pip install 'calendaring\[caldav\]'"): + Backend.connect("https://caldav.example.com/", kind="caldav") + + +def test_http_urls_are_caldav_until_the_feed_probe_exists() -> None: + pytest.importorskip("caldav") + backend = Backend.connect("https://caldav.example.com/dav/", username="u", password="p") + with backend: + assert backend.kind == "caldav" From 2f411fa27f9f5085da6a10560e0a2efa41732646 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 13:51:40 +0200 Subject: [PATCH 07/17] refactor: generate the sync caldav shim (2.1) The shim's sync class DAV was a hand-written copy of AsyncDAV. AsyncDAV now lives in _async/dav.py, and the generator writes DAV from it like the rest of the sync code, with _awaited renamed to _returned and AsyncDAVClient to DAVClient. What both modes share stays in _caldav_io.py. The parity test now guards the generated copy. Prompts (comments made inline in the document while reviewing it): > (on the 2.1 plan, the hand-written caldav shim) This involves a lot of hand-written mapping code, I presume? Any alternatives? > (on the answer proposing to generate the sync half) go for it. Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/_async/caldav.py | 7 +- src/calendaring/_async/dav.py | 104 +++++++++++++++++++ src/calendaring/_caldav_io.py | 166 ++++--------------------------- src/calendaring/_sync/caldav.py | 7 +- src/calendaring/_sync/dav.py | 104 +++++++++++++++++++ tests/test_caldav_io.py | 5 +- tools/generate_sync.py | 4 +- 7 files changed, 240 insertions(+), 157 deletions(-) create mode 100644 src/calendaring/_async/dav.py create mode 100644 src/calendaring/_sync/dav.py diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index 36d69b7..86ca55b 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -1,8 +1,8 @@ # Hand-written source; tools/generate_sync.py derives ../_sync/caldav.py from this file. """The CalDAV backend (roadmap 2.1), over the caldav library. -It never calls caldav directly: ``AsyncDAV`` in ``_caldav_io.py`` is the -honestly typed per-mode layer, so a missing ``await`` here is a type error. +It never calls caldav directly: ``dav.py`` beside it is the honestly typed +per-mode layer (see ``_caldav_io.py``), so a missing ``await`` here is a type error. A ``native_id`` is the object's name within its calendar (the last segment of its href, unquoted), so ``add(native_id=...)`` names the resource. """ @@ -22,7 +22,7 @@ from icalendar_searcher import Searcher from .._caldav_features import backend_capabilities, collection_capabilities -from .._caldav_io import AsyncDAV, Calendar +from .._caldav_io import Calendar from ..capabilities import Feature from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError, SyncTokenError from ..items import _items_of, _parse @@ -30,6 +30,7 @@ from ..types import Component, StoredObject, SyncToken from .backend import AsyncBackend from .collection import AsyncCollection +from .dav import AsyncDAV __all__ = ["AsyncCalDAVBackend", "AsyncCalDAVCollection"] diff --git a/src/calendaring/_async/dav.py b/src/calendaring/_async/dav.py new file mode 100644 index 0000000..6b404de --- /dev/null +++ b/src/calendaring/_async/dav.py @@ -0,0 +1,104 @@ +# Hand-written source; tools/generate_sync.py derives ../_sync/dav.py from this file. +"""The per-mode layer over caldav (roadmap 2.1): the only code that calls it. See ``_caldav_io.py``.""" + +from __future__ import annotations + +from typing import Any, cast + +from .._caldav_io import ( + ETAG_PROPFIND, + URL, + AsyncDAVClient, + Calendar, + CalendarObjectResource, + DisplayName, + SyncResult, + WriteResult, + _awaited, + _body, + _etag, + _etags, + _status, + _sync_result, + _write_headers, + translated, +) +from ..errors import ConfigurationError + +__all__ = ["AsyncDAV"] + + +class AsyncDAV: + """caldav, typed honestly: each call resolves caldav's ``X | Coroutine`` once.""" + + def __init__(self, url: str, **options: Any) -> None: + with translated(): + try: + self.client = AsyncDAVClient(url=url, **options) + except TypeError as exc: # an option the client does not take + raise ConfigurationError(f"caldav: {exc}") from exc + + async def close(self) -> None: + with translated(): + await self.client.close() + + async def calendars(self) -> list[Calendar]: + with translated(): + principal = await self.client.get_principal() + return await _awaited(principal.get_calendars()) + + async def make_calendar(self, name: str, components: list[str]) -> Calendar: + with translated(): + principal = await self.client.get_principal() + return await _awaited(principal.make_calendar(name=name, supported_calendar_component_set=components)) + + async def delete_calendar(self, calendar: Calendar) -> None: + with translated(): + await _awaited(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated + + async def supported_components(self, calendar: Calendar) -> list[str]: + with translated(): + return [str(c) for c in await _awaited(calendar.get_supported_components())] + + async def display_name(self, calendar: Calendar) -> str | None: + with translated(): + return await _awaited(calendar.get_display_name()) + + async def set_display_name(self, calendar: Calendar, name: str) -> None: + with translated(): + await _awaited(calendar.set_properties([DisplayName(name)])) + + async def search(self, calendar: Calendar, **kwargs: Any) -> list[CalendarObjectResource]: + with translated(): + return list(await _awaited(calendar.search(**kwargs))) + + async def by_uid(self, calendar: Calendar, uid: str) -> CalendarObjectResource: + with translated(): + return await _awaited(calendar.get_object_by_uid(uid)) + + async def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectResource]: + # multiget() is annotated Iterable[_CC], but returns a coroutine to an async client. + with translated(): + found: Any = calendar.multiget([URL.objectify(u) for u in urls]) + return list(await _awaited(cast(Any, found))) + + async def etags(self, url: str) -> dict[str, str]: + with translated(): + # Positional: the sync client's second parameter is props=, which also takes an XML body. + return _etags(await self.client.propfind(url, ETAG_PROPFIND, 1), url) + + async def sync(self, calendar: Calendar, token: str | None) -> SyncResult: + with translated(): + found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) + return _sync_result(await _awaited(found)) + + async def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: + with translated(): + response = await self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) + _status(response.status, f"PUT {url}", creating=create, body=_body(response)) + return response.status, _etag(response.headers) + + async def delete(self, url: str, *, etag: str | None) -> None: + with translated(): + response = await self.client.delete(url, None if etag is None else {"If-Match": etag}) + _status(response.status, f"DELETE {url}") diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py index 3abce90..f18565d 100644 --- a/src/calendaring/_caldav_io.py +++ b/src/calendaring/_caldav_io.py @@ -1,12 +1,13 @@ -"""The hand-written per-mode layer over caldav (roadmap 2.1, 0.2 *What is not settled*, 5). +"""The mode-free half of the bridge to caldav (roadmap 2.1, 0.2 *What is not settled*, 5). caldav's domain objects are dual-mode: with an async client a method returns a coroutine, with a sync one the value, and the annotation says ``X | Coroutine``. Neither type checker can tell a missing ``await`` from a -used value through that, so the generated backend never calls caldav itself. -It calls ``AsyncDAV`` here, whose twin ``DAV`` the generator's replacement map -substitutes. Each wrapper resolves the union once: ``_awaited`` / -``_returned`` also fail loudly if the client is of the wrong mode. +used value through that, so the backend never calls caldav itself. It calls +``AsyncDAV`` (``_async/dav.py``), whose sync twin ``DAV`` is generated like +the rest: ``_awaited`` becomes ``_returned`` and ``AsyncDAVClient`` becomes +``DAVClient``. Each wrapper resolves the union once, and fails loudly if the +client is of the wrong mode. What both modes share lives here. Writes go through the client's ``put()`` / ``delete()`` with the preconditions this library needs (``If-None-Match: *`` to create, @@ -43,7 +44,18 @@ TransportError, ) -__all__ = ["DAV", "AsyncDAV", "Calendar", "SyncResult", "WriteResult"] +__all__ = [ + "ETAG_PROPFIND", + "URL", + "AsyncDAVClient", + "Calendar", + "CalendarObjectResource", + "DAVClient", + "DisplayName", + "SyncResult", + "WriteResult", + "translated", +] T = TypeVar("T") @@ -162,145 +174,3 @@ def _etag(headers: Mapping[str, str]) -> str | None: if name.lower() == "etag": return None if value.startswith("W/") else value return None - - -class AsyncDAV: - """caldav, async, typed honestly.""" - - def __init__(self, url: str, **options: Any) -> None: - with translated(): - self.client = AsyncDAVClient(url=url, **options) - - async def close(self) -> None: - with translated(): - await self.client.close() - - async def calendars(self) -> list[Calendar]: - with translated(): - principal = await self.client.get_principal() - return await _awaited(principal.get_calendars()) - - async def make_calendar(self, name: str, components: list[str]) -> Calendar: - with translated(): - principal = await self.client.get_principal() - return await _awaited(principal.make_calendar(name=name, supported_calendar_component_set=components)) - - async def delete_calendar(self, calendar: Calendar) -> None: - with translated(): - await _awaited(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated - - async def supported_components(self, calendar: Calendar) -> list[str]: - with translated(): - return [str(c) for c in await _awaited(calendar.get_supported_components())] - - async def display_name(self, calendar: Calendar) -> str | None: - with translated(): - return await _awaited(calendar.get_display_name()) - - async def set_display_name(self, calendar: Calendar, name: str) -> None: - with translated(): - await _awaited(calendar.set_properties([DisplayName(name)])) - - async def search(self, calendar: Calendar, **kwargs: Any) -> list[CalendarObjectResource]: - with translated(): - return list(await _awaited(calendar.search(**kwargs))) - - async def by_uid(self, calendar: Calendar, uid: str) -> CalendarObjectResource: - with translated(): - return await _awaited(calendar.get_object_by_uid(uid)) - - async def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectResource]: - # multiget() is annotated Iterable[_CC], but returns a coroutine to an async client. - with translated(): - found: Any = calendar.multiget([URL.objectify(u) for u in urls]) - return list(await _awaited(cast(Any, found))) - - async def etags(self, url: str) -> dict[str, str]: - with translated(): - return _etags(await self.client.propfind(url, body=ETAG_PROPFIND, depth=1), url) - - async def sync(self, calendar: Calendar, token: str | None) -> SyncResult: - with translated(): - found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) - return _sync_result(await _awaited(found)) - - async def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: - with translated(): - response = await self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) - _status(response.status, f"PUT {url}", creating=create, body=_body(response)) - return response.status, _etag(response.headers) - - async def delete(self, url: str, *, etag: str | None) -> None: - with translated(): - response = await self.client.delete(url, None if etag is None else {"If-Match": etag}) - _status(response.status, f"DELETE {url}") - - -class DAV: - """caldav, sync, typed honestly: ``AsyncDAV``'s twin, method for method.""" - - def __init__(self, url: str, **options: Any) -> None: - with translated(): - self.client = DAVClient(url=url, **options) - - def close(self) -> None: - with translated(): - self.client.close() - - def calendars(self) -> list[Calendar]: - with translated(): - principal = self.client.get_principal() - return _returned(principal.get_calendars()) - - def make_calendar(self, name: str, components: list[str]) -> Calendar: - with translated(): - principal = self.client.get_principal() - return _returned(principal.make_calendar(name=name, supported_calendar_component_set=components)) - - def delete_calendar(self, calendar: Calendar) -> None: - with translated(): - _returned(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated - - def supported_components(self, calendar: Calendar) -> list[str]: - with translated(): - return [str(c) for c in _returned(calendar.get_supported_components())] - - def display_name(self, calendar: Calendar) -> str | None: - with translated(): - return _returned(calendar.get_display_name()) - - def set_display_name(self, calendar: Calendar, name: str) -> None: - with translated(): - _returned(calendar.set_properties([DisplayName(name)])) - - def search(self, calendar: Calendar, **kwargs: Any) -> list[CalendarObjectResource]: - with translated(): - return list(_returned(calendar.search(**kwargs))) - - def by_uid(self, calendar: Calendar, uid: str) -> CalendarObjectResource: - with translated(): - return _returned(calendar.get_object_by_uid(uid)) - - def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectResource]: - with translated(): - return list(_returned(calendar.multiget([URL.objectify(u) for u in urls]))) - - def etags(self, url: str) -> dict[str, str]: - with translated(): - return _etags(self.client.propfind(url, ETAG_PROPFIND, depth=1), url) - - def sync(self, calendar: Calendar, token: str | None) -> SyncResult: - with translated(): - found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) - return _sync_result(_returned(found)) - - def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: - with translated(): - response = self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) - _status(response.status, f"PUT {url}", creating=create, body=_body(response)) - return response.status, _etag(response.headers) - - def delete(self, url: str, *, etag: str | None) -> None: - with translated(): - response = self.client.delete(url, None if etag is None else {"If-Match": etag}) - _status(response.status, f"DELETE {url}") diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index d2d5752..23fbddd 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -1,8 +1,8 @@ # Code generated by tools/generate_sync.py from _async/; DO NOT EDIT. """The CalDAV backend (roadmap 2.1), over the caldav library. -It never calls caldav directly: ``AsyncDAV`` in ``_caldav_io.py`` is the -honestly typed per-mode layer, so a missing ``await`` here is a type error. +It never calls caldav directly: ``dav.py`` beside it is the honestly typed +per-mode layer (see ``_caldav_io.py``), so a missing ``await`` here is a type error. A ``native_id`` is the object's name within its calendar (the last segment of its href, unquoted), so ``add(native_id=...)`` names the resource. """ @@ -22,7 +22,7 @@ from icalendar_searcher import Searcher from .._caldav_features import backend_capabilities, collection_capabilities -from .._caldav_io import DAV, Calendar +from .._caldav_io import Calendar from ..capabilities import Feature from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError, SyncTokenError from ..items import _items_of, _parse @@ -30,6 +30,7 @@ from ..types import Component, StoredObject, SyncToken from .backend import Backend from .collection import Collection +from .dav import DAV __all__ = ["CalDAVBackend", "CalDAVCollection"] diff --git a/src/calendaring/_sync/dav.py b/src/calendaring/_sync/dav.py new file mode 100644 index 0000000..5fbba6f --- /dev/null +++ b/src/calendaring/_sync/dav.py @@ -0,0 +1,104 @@ +# Code generated by tools/generate_sync.py from _async/; DO NOT EDIT. +"""The per-mode layer over caldav (roadmap 2.1): the only code that calls it. See ``_caldav_io.py``.""" + +from __future__ import annotations + +from typing import Any, cast + +from .._caldav_io import ( + ETAG_PROPFIND, + URL, + DAVClient, + Calendar, + CalendarObjectResource, + DisplayName, + SyncResult, + WriteResult, + _returned, + _body, + _etag, + _etags, + _status, + _sync_result, + _write_headers, + translated, +) +from ..errors import ConfigurationError + +__all__ = ["DAV"] + + +class DAV: + """caldav, typed honestly: each call resolves caldav's ``X | Coroutine`` once.""" + + def __init__(self, url: str, **options: Any) -> None: + with translated(): + try: + self.client = DAVClient(url=url, **options) + except TypeError as exc: # an option the client does not take + raise ConfigurationError(f"caldav: {exc}") from exc + + def close(self) -> None: + with translated(): + self.client.close() + + def calendars(self) -> list[Calendar]: + with translated(): + principal = self.client.get_principal() + return _returned(principal.get_calendars()) + + def make_calendar(self, name: str, components: list[str]) -> Calendar: + with translated(): + principal = self.client.get_principal() + return _returned(principal.make_calendar(name=name, supported_calendar_component_set=components)) + + def delete_calendar(self, calendar: Calendar) -> None: + with translated(): + _returned(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated + + def supported_components(self, calendar: Calendar) -> list[str]: + with translated(): + return [str(c) for c in _returned(calendar.get_supported_components())] + + def display_name(self, calendar: Calendar) -> str | None: + with translated(): + return _returned(calendar.get_display_name()) + + def set_display_name(self, calendar: Calendar, name: str) -> None: + with translated(): + _returned(calendar.set_properties([DisplayName(name)])) + + def search(self, calendar: Calendar, **kwargs: Any) -> list[CalendarObjectResource]: + with translated(): + return list(_returned(calendar.search(**kwargs))) + + def by_uid(self, calendar: Calendar, uid: str) -> CalendarObjectResource: + with translated(): + return _returned(calendar.get_object_by_uid(uid)) + + def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectResource]: + # multiget() is annotated Iterable[_CC], but returns a coroutine to an async client. + with translated(): + found: Any = calendar.multiget([URL.objectify(u) for u in urls]) + return list(_returned(cast(Any, found))) + + def etags(self, url: str) -> dict[str, str]: + with translated(): + # Positional: the sync client's second parameter is props=, which also takes an XML body. + return _etags(self.client.propfind(url, ETAG_PROPFIND, 1), url) + + def sync(self, calendar: Calendar, token: str | None) -> SyncResult: + with translated(): + found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) + return _sync_result(_returned(found)) + + def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: + with translated(): + response = self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) + _status(response.status, f"PUT {url}", creating=create, body=_body(response)) + return response.status, _etag(response.headers) + + def delete(self, url: str, *, etag: str | None) -> None: + with translated(): + response = self.client.delete(url, None if etag is None else {"If-Match": etag}) + _status(response.status, f"DELETE {url}") diff --git a/tests/test_caldav_io.py b/tests/test_caldav_io.py index dc663db..bf36b55 100644 --- a/tests/test_caldav_io.py +++ b/tests/test_caldav_io.py @@ -1,4 +1,4 @@ -"""The caldav shim's two modes match, method for method (roadmap 2.1).""" +"""The caldav shim's generated sync mode still matches the async one, method for method (roadmap 2.1).""" from __future__ import annotations @@ -8,7 +8,8 @@ pytest.importorskip("caldav") -from calendaring._caldav_io import DAV, AsyncDAV # noqa: E402 +from calendaring._async.dav import AsyncDAV # noqa: E402 +from calendaring._sync.dav import DAV # noqa: E402 def methods(cls: type) -> dict[str, inspect.Signature]: diff --git a/tools/generate_sync.py b/tools/generate_sync.py index 397b7df..eba761e 100644 --- a/tools/generate_sync.py +++ b/tools/generate_sync.py @@ -45,8 +45,10 @@ "AsyncCalDAVCollection": "CalDAVCollection", # the per-mode layer (calendaring/_concurrency.py) "agather": "gather", - # the caldav bridge (calendaring/_caldav_io.py) + # the caldav bridge (_async/dav.py over calendaring/_caldav_io.py) "AsyncDAV": "DAV", + "AsyncDAVClient": "DAVClient", + "_awaited": "_returned", } RULE = unasync.Rule("/_async/", "/_sync/", additional_replacements=REPLACEMENTS) From 4cac4c9841cfb80be4bf6f97f4805831f80deb81 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 13:52:16 +0200 Subject: [PATCH 08/17] fix: get() picks the earliest masterless override A stored object may hold overrides without their master (RFC 4791 section 4.1, e.g. an attendee invited to some occurrences only). get(uid) returned whichever the stored object listed first. It now returns the one with the earliest RECURRENCE-ID, and API_DESIGN says that get() then misses the others, which search() returns. Prompts (comments made inline in the document while reviewing it): > (on the 2.1 plan, what get(uid) returns for overrides stored without a master) This problem arises because an Item is defined to either be a full recurrence set including the master, or a single occurrence? > (on the answer: earliest RECURRENCE-ID, or AmbiguousError) Let's give the first one, and document that this may be a problem. Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- docs/design/API_DESIGN.md | 6 +++++- src/calendaring/_async/collection.py | 14 +++++++++++++- src/calendaring/_sync/collection.py | 14 +++++++++++++- tests/_async/test_generic.py | 15 +++++++++++++++ tests/_sync/test_generic.py | 15 +++++++++++++++ 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 0c9f8db..03adcd3 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -656,7 +656,11 @@ does not carry. Occurrences come from three places: backend's choice ([§9](#9-migrating-from-caldav), `server_expand`); - a stored object that holds overrides without their master, which RFC 4791 §4.1 allows, e.g. for an attendee invited to some instances only. The - backend hands out one occurrence item per override. + backend hands out one occurrence item per override. `get(uid)` returns + only the one with the earliest `RECURRENCE-ID` (decided 2026-10-19, + roadmap 2.1), since there is no master to be the whole item. This may + be a problem: a caller expecting `get()` to return everything under a + UID silently misses the other occurrences. `search()` returns them all. Several occurrences of one series are several items. They share a `uid`, and on a stored series they share a `native_id` and an `etag` too. diff --git a/src/calendaring/_async/collection.py b/src/calendaring/_async/collection.py index 3ce2f8c..1538596 100644 --- a/src/calendaring/_async/collection.py +++ b/src/calendaring/_async/collection.py @@ -52,6 +52,17 @@ } +def _occurs_at(item: Item) -> datetime: + """Where an occurrence sorts: its ``RECURRENCE-ID`` as an aware instant (floating counts as UTC).""" + rid = item.component.get("RECURRENCE-ID") + if rid is None: + return datetime.min.replace(tzinfo=UTC) + value = rid.dt + if not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day) + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + class AsyncCollection(ABC): """What holds items: a calendar, a task list, a directory, a repository.""" @@ -449,8 +460,9 @@ async def journals(self, **filters: Any) -> list[AsyncJournal]: return [i for i in await self.search(journal=True, **filters) if isinstance(i, AsyncJournal)] async def get(self, uid: str) -> AsyncItem: + """The item with this UID. Overrides stored without their master: the earliest occurrence (§3.1).""" with self._context(): - return self._one(await self._find_uid(uid)) + return min(self._bound(await self._find_uid(uid)), key=_occurs_at) async def get_by_native_id(self, native_id: str) -> AsyncItem: with self._context(): diff --git a/src/calendaring/_sync/collection.py b/src/calendaring/_sync/collection.py index 3f1131a..9553f35 100644 --- a/src/calendaring/_sync/collection.py +++ b/src/calendaring/_sync/collection.py @@ -52,6 +52,17 @@ } +def _occurs_at(item: Item) -> datetime: + """Where an occurrence sorts: its ``RECURRENCE-ID`` as an aware instant (floating counts as UTC).""" + rid = item.component.get("RECURRENCE-ID") + if rid is None: + return datetime.min.replace(tzinfo=UTC) + value = rid.dt + if not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day) + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + class Collection(ABC): """What holds items: a calendar, a task list, a directory, a repository.""" @@ -449,8 +460,9 @@ def journals(self, **filters: Any) -> list[SyncJournal]: return [i for i in self.search(journal=True, **filters) if isinstance(i, SyncJournal)] def get(self, uid: str) -> SyncItem: + """The item with this UID. Overrides stored without their master: the earliest occurrence (§3.1).""" with self._context(): - return self._one(self._find_uid(uid)) + return min(self._bound(self._find_uid(uid)), key=_occurs_at) def get_by_native_id(self, native_id: str) -> SyncItem: with self._context(): diff --git a/tests/_async/test_generic.py b/tests/_async/test_generic.py index 7bd0ac0..07354d7 100644 --- a/tests/_async/test_generic.py +++ b/tests/_async/test_generic.py @@ -387,3 +387,18 @@ async def test_http_urls_are_caldav_until_the_feed_probe_exists() -> None: backend = await AsyncBackend.connect("https://caldav.example.com/dav/", username="u", password="p") async with backend: assert backend.kind == "caldav" + + +async def test_get_on_overrides_without_a_master_returns_the_earliest() -> None: + backend = AsyncMemoryBackend() + collection = await backend.collection("default") + assert isinstance(collection, AsyncMemoryCollection) + later = Event.new(summary="later", start=at(5, 9), end=at(5, 10), uid="invited") + later.component.add("RECURRENCE-ID", at(5, 9)) + earlier = Event.new(summary="earlier", start=at(2, 9), end=at(2, 10), uid="invited") + earlier.component.add("RECURRENCE-ID", at(2, 9)) + later.icalendar.add_component(earlier.component) + collection.store.put(later.to_ical()) + found = await collection.get("invited") + assert found.is_occurrence + assert str(found.component["SUMMARY"]) == "earlier" diff --git a/tests/_sync/test_generic.py b/tests/_sync/test_generic.py index 5cd77b7..a3210ac 100644 --- a/tests/_sync/test_generic.py +++ b/tests/_sync/test_generic.py @@ -387,3 +387,18 @@ def test_http_urls_are_caldav_until_the_feed_probe_exists() -> None: backend = Backend.connect("https://caldav.example.com/dav/", username="u", password="p") with backend: assert backend.kind == "caldav" + + +def test_get_on_overrides_without_a_master_returns_the_earliest() -> None: + backend = MemoryBackend() + collection = backend.collection("default") + assert isinstance(collection, MemoryCollection) + later = Event.new(summary="later", start=at(5, 9), end=at(5, 10), uid="invited") + later.component.add("RECURRENCE-ID", at(5, 9)) + earlier = Event.new(summary="earlier", start=at(2, 9), end=at(2, 10), uid="invited") + earlier.component.add("RECURRENCE-ID", at(2, 9)) + later.icalendar.add_component(earlier.component) + collection.store.put(later.to_ical()) + found = collection.get("invited") + assert found.is_occurrence + assert str(found.component["SUMMARY"]) == "earlier" From c199144478c4ca12721b4d972cfc8f7ec9470bf6 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 13:53:02 +0200 Subject: [PATCH 09/17] feat: ignore test_* keys in config sections The shared config file carries keys only test tooling reads, such as caldav's test-server selection. Workspace.from_config() passed every key to connect(), so such a section failed to load. Keys starting with test_ are now reserved for test tooling and dropped. Any other unknown key is still a ConfigurationError, so typos are caught. API_DESIGN section 2.2 records the decision, which was left open for the CalDAV backend. Prompts (comments made inline in the document while reviewing it): > (on the 2.1 plan, which config keys a backend ignores) Are there other use-cases except for giving instructions to the test framework? Perhaps we could have some standard like all attributes exclusively read by the test framework should be prepended with `test_` and all of them ignored? > (on the answer recommending a reserved test_ prefix) We'll go for "`test_*`". The caldav code needs to be made both forward- and backward compatible eventually. We don't need to care about backward compatibility in the calendaring package (I may be wrong, but I believe there are very few except for me that have configured test servers in the config file). Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- docs/design/API_DESIGN.md | 19 ++++++++++++------- src/calendaring/_async/workspace.py | 12 ++++++++++-- src/calendaring/_sync/workspace.py | 12 ++++++++++-- tests/_async/test_config.py | 12 ++++++++++++ tests/_sync/test_config.py | 12 ++++++++++++ 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 03adcd3..3451180 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -294,13 +294,18 @@ class AsyncBackend: - Credentials are never required in the URL or in plain-text configuration ([roadmap 1.4](ROADMAP.md#14-configuration-and-credentials)'s keyring work); `connect`'s keyword arguments are for programmatic use. -- Open: which configuration keys a backend ignores. The shared file keeps - keys such as `features`, `calendar_name`, `priority` and - `testing_allowed`, and `Workspace.from_config()` passes every key to - `connect()`; today a key the backend does not take is a - `ConfigurationError` naming the section. Decided with the CalDAV - backend ([roadmap 2.1](ROADMAP.md#21-caldav-backend)), the first one - whose sections carry such keys. +- *Decided 2026-10-19, with the CalDAV backend + ([roadmap 2.1](ROADMAP.md#21-caldav-backend)):* which configuration + keys a backend ignores. Keys starting with `test_` are reserved for + test tooling, and `Workspace.from_config()` drops them before + `connect()`. Every other key reaches the backend, and one it does not + take is a `ConfigurationError` naming the section, so a typo is not + silently ignored. The only such keys found are caldav's test-server + selection, `testing_allowed` and `priority`; caldav is to read + `test_allowed` and `test_priority` too (on + [caldav #786](https://github.com/python-caldav/caldav/issues/786)). + calendaring keeps no compatibility for the old names. `features` is + not a test key: the CalDAV backend passes it to caldav. ### 2.3 Collection diff --git a/src/calendaring/_async/workspace.py b/src/calendaring/_async/workspace.py index 5941463..b5c2986 100644 --- a/src/calendaring/_async/workspace.py +++ b/src/calendaring/_async/workspace.py @@ -37,9 +37,17 @@ def _without_credentials(url: str) -> str: return urlunsplit(parts._replace(netloc=parts.netloc.rpartition("@")[2])) +# Reserved for test tooling: every backend ignores these keys (decided 2026-10-19, roadmap 2.1). +TEST_PREFIX = "test_" + + def _section_params(conn: calendaring_config.Connection) -> tuple[str, str, dict[str, Any]]: - """The section name, its url and the rest of ``connect()``'s keywords, the id included.""" - params = dict(conn.params) + """The section name, its url and the rest of ``connect()``'s keywords, the id included. + + Keys starting with ``test_`` are for test tooling (caldav's test-server + selection, say) and never reach a backend. + """ + params = {k: v for k, v in conn.params.items() if not k.startswith(TEST_PREFIX)} url = params.pop("url", None) if not isinstance(url, str) or not url: raise ConfigurationError(f"section {conn.name}: url must be a non-empty string") diff --git a/src/calendaring/_sync/workspace.py b/src/calendaring/_sync/workspace.py index 878b25e..8e56e9c 100644 --- a/src/calendaring/_sync/workspace.py +++ b/src/calendaring/_sync/workspace.py @@ -37,9 +37,17 @@ def _without_credentials(url: str) -> str: return urlunsplit(parts._replace(netloc=parts.netloc.rpartition("@")[2])) +# Reserved for test tooling: every backend ignores these keys (decided 2026-10-19, roadmap 2.1). +TEST_PREFIX = "test_" + + def _section_params(conn: calendaring_config.Connection) -> tuple[str, str, dict[str, Any]]: - """The section name, its url and the rest of ``connect()``'s keywords, the id included.""" - params = dict(conn.params) + """The section name, its url and the rest of ``connect()``'s keywords, the id included. + + Keys starting with ``test_`` are for test tooling (caldav's test-server + selection, say) and never reach a backend. + """ + params = {k: v for k, v in conn.params.items() if not k.startswith(TEST_PREFIX)} url = params.pop("url", None) if not isinstance(url, str) or not url: raise ConfigurationError(f"section {conn.name}: url must be a non-empty string") diff --git a/tests/_async/test_config.py b/tests/_async/test_config.py index baf789b..c50c8d4 100644 --- a/tests/_async/test_config.py +++ b/tests/_async/test_config.py @@ -46,6 +46,18 @@ async def test_one_backend_per_section_named_after_it(tmp_path: Path) -> None: assert all(b._workspace is workspace for b in workspace.backends) +async def test_keys_for_test_tooling_are_ignored(tmp_path: Path) -> None: + path = write(tmp_path, {"default": {"url": "memory:", "test_allowed": True, "test_priority": 5}}) + [backend] = (await AsyncWorkspace.from_config(path)).backends + assert isinstance(backend, AsyncMemoryBackend) + + +async def test_an_unknown_key_is_still_an_error(tmp_path: Path) -> None: + path = write(tmp_path, {"default": {"url": "memory:", "tset_allowed": True}}) + with pytest.raises(ConfigurationError, match="section default"): + await AsyncWorkspace.from_config(path) + + async def test_section_from_the_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: path = write(tmp_path, {"default": {"url": "memory:"}, "other": {"url": "memory:"}}) monkeypatch.setenv("CALENDARING_CONFIG_SECTION", "other") diff --git a/tests/_sync/test_config.py b/tests/_sync/test_config.py index 152cb76..bc3d5cc 100644 --- a/tests/_sync/test_config.py +++ b/tests/_sync/test_config.py @@ -46,6 +46,18 @@ def test_one_backend_per_section_named_after_it(tmp_path: Path) -> None: assert all(b._workspace is workspace for b in workspace.backends) +def test_keys_for_test_tooling_are_ignored(tmp_path: Path) -> None: + path = write(tmp_path, {"default": {"url": "memory:", "test_allowed": True, "test_priority": 5}}) + [backend] = (Workspace.from_config(path)).backends + assert isinstance(backend, MemoryBackend) + + +def test_an_unknown_key_is_still_an_error(tmp_path: Path) -> None: + path = write(tmp_path, {"default": {"url": "memory:", "tset_allowed": True}}) + with pytest.raises(ConfigurationError, match="section default"): + Workspace.from_config(path) + + def test_section_from_the_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: path = write(tmp_path, {"default": {"url": "memory:"}, "other": {"url": "memory:"}}) monkeypatch.setenv("CALENDARING_CONFIG_SECTION", "other") From 05c27e2aa7379758bcdf9e64c2a5fc84307e86c2 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 14:02:00 +0200 Subject: [PATCH 10/17] feat: delete one occurrence with an EXDATE (Q12) delete() on an occurrence raised UnsupportedError while API_DESIGN Q12 was open. It now adds an EXDATE to the master in the form of its DTSTART, and removes the occurrence's override. The known client bugs with EXDATE came from mismatched forms. A stale etag from a sibling's edit is no conflict: only a change to this occurrence raises ConflictError. Q12 records the decision and why not STATUS:CANCELLED, and the conformance suite tests it on every backend that recurs. Prompts (comments made inline in the document while reviewing it): > (on the 2.1 plan, delete() on an occurrence: EXDATE or STATUS:CANCELLED) I vaguely remember that there was some reason for not using EXDATE, like lack of client support [I later discovered that I was mixing up things - EXRULE is deprecated and should not be used, but EXDATE is fine]. Is there no arguments for/against EXDATE in the issue discussions? Please do some research on client support > (on the research answer) We'll go for EXDATE then. Please draft up a comment to be posted to the 598-issue. Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- docs/design/API_DESIGN.md | 61 ++++++++++++------------ src/calendaring/_async/collection.py | 47 +++++++++++++++++-- src/calendaring/_async/conformance.py | 67 +++++++++++++++++++++++++++ src/calendaring/_sync/collection.py | 47 +++++++++++++++++-- src/calendaring/_sync/conformance.py | 67 +++++++++++++++++++++++++++ src/calendaring/items.py | 40 ++++++++++++++++ tests/_async/test_generic.py | 5 +- tests/_sync/test_generic.py | 5 +- tests/test_items.py | 35 +++++++++++++- 9 files changed, 329 insertions(+), 45 deletions(-) diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 3451180..12a5da2 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -1631,38 +1631,39 @@ marked as not blocking; none blocks [what is in the way of a calendaring adapter](https://github.com/pycalendar/calendaring-sync/pull/19#issuecomment-6114023463); server evidence on calendaring-sync's [`evidence` branch](https://github.com/pycalendar/calendaring-sync/tree/evidence). -12. *Deferred, 2026-10-13:* **Occurrence items: `delete()` and the shared etag** +12. *Decided, 2026-10-19 (roadmap 2.1):* **Occurrence items: `delete()` and the shared etag** ([What an item contains](#what-an-item-contains)). Occurrences of one stored series share its `native_id` and `etag`, so the rules for a whole - object do not carry over. The proposals below are drafts. The author - recalls reasons for preferring `STATUS:CANCELLED` over `EXDATE`, to be - written down when this is settled. - [#398](https://github.com/python-caldav/caldav/issues/398) prefers - `CANCELLED` to raising an error from `delete()`, since a traceback will - reach some end user sooner or later. To be settled before - [roadmap 2.1](ROADMAP.md#21-caldav-backend) implements `delete()` on an - occurrence. - - **`delete(occurrence)`** must not delete the stored series, which - caldav may do today: caldav - [#398](https://github.com/python-caldav/caldav/issues/398) suspects - it, and nobody has tested it. - Two options. **`EXDATE`:** remove that occurrence's override, if - there is one, and add an `EXDATE` to the master. When there is no - master, remove the override, and delete the stored object only when - it was the last one. **`STATUS:CANCELLED`**, which #398 suggests and - the author leans towards: set it on the occurrence's override, - creating the override if needed. The occurrence stays visible as - cancelled, so this is an edit rather than a deletion. - - **Conflicts between siblings.** Saving one occurrence changes the - stored object's etag under its siblings. If a sibling's save then - required the etag it was read with, it would raise `ConflictError` - for an edit nobody made to it. **Proposal:** `save(occurrence)` - already fetches the stored object to merge into, so it raises only if - *that occurrence* changed since it was read, and writes with the - fresh etag. For an overridden occurrence the check is the override's - `SEQUENCE` or `LAST-MODIFIED`. A generated occurrence has no override, - and a concurrent edit of the master (its `RRULE`, its `DTSTART`) - changes it too, so there the check is the master's. + object do not carry over. + - **`delete(occurrence)` writes an `EXDATE`** on the master and removes + that occurrence's override, if there is one. When there is no master, + it removes the override, and deletes the stored object only when it was + the last one. The `EXDATE` takes the master's `DTSTART` form (a date, a + date-time in the same `TZID`, UTC or floating). The `EXDATE` bugs found + in clients (Exchange 2010, Lightning, khal) all came from comparing the + two in different forms. + - **Why not `STATUS:CANCELLED`**, which the author leaned towards, and + which caldav planned in + [#35](https://github.com/python-caldav/caldav/issues/35) and + [#598](https://github.com/python-caldav/caldav/issues/598): + "delete" and "cancel" are different operations, as #35 itself says. + A consumer that does not filter cancelled events shows a cancelled + override as live (Home Assistant until + [core #180947](https://github.com/home-assistant/core/pull/180947)), and + Home Assistant's own occurrence deletion + ([core #176565](https://github.com/home-assistant/core/pull/176565)) + uses `EXDATE`. Cancelling an occurrence is a separate `cancel()`, not + built yet: it writes an override, so it waits for the + `save(occurrence)` merge (Q3). + - **Conflicts between siblings.** Saving or deleting one occurrence + changes the stored object's etag under its siblings. So `delete()` + fetches the stored object, and raises `ConflictError` only if *that + occurrence* changed since it was read: its override, or the + occurrence generated from the master (a changed `RRULE`, `DTSTART` or + `SUMMARY` changes it). It then writes with the fresh etag. + `overwrite=True` skips the check. An occurrence already gone is + `NotFoundError`. `save(occurrence)` will follow the + same rule when the merge (Q3) lands. 13. *Decided, 2026-10-15* **Change tracking** (the reviewer's "borrowing", [PR 5 review](https://github.com/pycalendar/calendaring/pull/5#discussion_r4237509309)). diff --git a/src/calendaring/_async/collection.py b/src/calendaring/_async/collection.py index 1538596..b706c82 100644 --- a/src/calendaring/_async/collection.py +++ b/src/calendaring/_async/collection.py @@ -35,7 +35,7 @@ UnsupportedError, VerificationError, ) -from ..items import Event, Item, Journal, Task, _items_of, _parse, split_stored +from ..items import Event, Item, Journal, Task, _items_of, _parse, split_stored, without_occurrence from ..loss import RAISE, LossPolicy from ..types import BrokenObject, ChangeSet, Component, Scope, StoredObject, SyncToken from .items import BOUND, AsyncEvent, AsyncItem, AsyncJournal, AsyncTask @@ -52,6 +52,20 @@ } +def _occurrence_now(calendar: icalendar.Calendar, item: Item) -> icalendar.Component | None: + """The occurrence ``item`` was, as the stored object has it now: its override, or one generated from the master.""" + wanted = item.component["RECURRENCE-ID"] + for component in _items_of(calendar): + if component.get("RECURRENCE-ID") == wanted: + return component + rid = item.component.decoded("RECURRENCE-ID") + generated: list[icalendar.Component] = recurring_ical_events.of(calendar).between(rid, rid + timedelta(seconds=1)) + for component in generated: + if component.get("RECURRENCE-ID") == wanted: + return component + return None + + def _occurs_at(item: Item) -> datetime: """Where an occurrence sorts: its ``RECURRENCE-ID`` as an aware instant (floating counts as UTC).""" rid = item.component.get("RECURRENCE-ID") @@ -715,10 +729,37 @@ async def delete(self, item: Item | str, *, overwrite: bool = False) -> None: if item.native_id is None: raise ValueError("delete(): the item was never stored") if item.is_occurrence: - # EXDATE or STATUS:CANCELLED is undecided (API_DESIGN.md Q12). - raise UnsupportedError(None, "deleting one occurrence is undecided (API_DESIGN.md Q12)") + await self._delete_occurrence(item, overwrite=overwrite) + return await self._delete(item.native_id, etag=None if overwrite else item.etag) + async def _delete_occurrence(self, item: Item, *, overwrite: bool) -> None: + """An ``EXDATE`` in the master's own form, and the override gone (API_DESIGN.md Q12). + + Siblings share the stored object's etag, so a stale etag is not a + conflict by itself: only a change to this occurrence on the server is. + It is found by comparing the stored occurrence with the item as the + caller holds it, so once the etag moved, a local edit before + ``delete()`` also counts as one; with the etag unchanged nothing + changed on the server, and a local edit is deleted with the rest. + An occurrence already gone is ``NotFoundError``. + """ + assert item.native_id is not None + stored = await self._fetch(item.native_id) + if stored.etag != item.etag: + now = _occurrence_now(stored.calendar, item) + if now is None: + raise NotFoundError(f"the occurrence {item.component.decoded('RECURRENCE-ID')} is gone") + if not overwrite and now != item.component: + raise ConflictError( + f"the occurrence {item.component.decoded('RECURRENCE-ID')} changed since it was read" + ) + calendar = without_occurrence(stored.calendar, item.uid, item.component.decoded("RECURRENCE-ID")) + if not _items_of(calendar): + await self._delete(item.native_id, etag=stored.etag) + return + await self._update(item.native_id, calendar, etag=stored.etag, wire=None) + async def complete( self, task: Task, diff --git a/src/calendaring/_async/conformance.py b/src/calendaring/_async/conformance.py index c0413b3..d31af59 100644 --- a/src/calendaring/_async/conformance.py +++ b/src/calendaring/_async/conformance.py @@ -33,6 +33,7 @@ async def open_backend(self) -> AsyncBackend: from collections.abc import AsyncIterator, Callable, Sequence from datetime import UTC, datetime, timedelta from typing import Any +from zoneinfo import ZoneInfo import icalendar import pytest @@ -953,6 +954,72 @@ async def test_save_one_occurrence(self, collection: AsyncCollection) -> None: again = stored_by(await collection.search(event=True, start=at(2), end=at(4), expand=True), seeded) assert sorted(str(i.component["SUMMARY"]) for i in again) == ["R: daily", "R: moved"] + async def recurring_series(self, collection: AsyncCollection) -> AsyncEvent: + """A stored daily series in Europe/Oslo time, with day 3 moved (an override), or the test ends here.""" + if not writable(collection): + await self.assert_read_only(collection) + pytest.skip("read-only collection: the write paths were checked to raise") + if not holds(collection, Component.EVENT): + pytest.skip("the collection holds no events") + if level(collection, Feature.RECURRENCE) is Support.UNSUPPORTED: + pytest.skip("nothing recurs: test_expanded_search_returns_occurrences checks the refusal") + oslo = ZoneInfo("Europe/Oslo") + start = at(1, 9).astimezone(oslo) + series = Event.new(summary="R: daily", start=start, end=start + timedelta(hours=1)) + series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) + moved = Event.new(summary="R: moved", start=start + timedelta(days=2, hours=5), uid=series.uid) + moved.component["DTEND"] = icalendar.vDDDTypes(start + timedelta(days=2, hours=6)) + moved.component.add("RECURRENCE-ID", start + timedelta(days=2)) + series.icalendar.add_component(moved.component) + series.icalendar.add_missing_timezones() + [stored] = await self.seed(collection, [series]) + assert isinstance(stored, AsyncEvent) + return stored + + async def occurrences(self, collection: AsyncCollection, series: AsyncItem) -> dict[int, AsyncItem]: + """The series' occurrences, by day of the month of their RECURRENCE-ID.""" + found = await collection.search(event=True, start=at(1, 0), end=at(9, 0), expand=True) + return {i.component.decoded("RECURRENCE-ID").day: i for i in found if i.uid == series.uid} + + async def test_delete_one_occurrence(self, collection: AsyncCollection) -> None: + series = await self.recurring_series(collection) + before = await self.occurrences(collection, series) + assert sorted(before) == [1, 2, 3, 4, 5] + await collection.delete(before[2]) + with pytest.raises(NotFoundError): + await collection.delete(before[2]) # already gone + await collection.delete(before[3]) # the moved one + after = await self.occurrences(collection, series) + assert sorted(after) == [1, 4, 5] + stored = await collection.get(series.uid) + assert not stored.is_occurrence, "deleting an occurrence deleted the series" + dtstart = stored.component["DTSTART"] + exdates = stored.component.get("EXDATE") + exdates = exdates if isinstance(exdates, list) else [exdates] + assert sorted(d.dt for e in exdates for d in e.dts) == [ + before[2].component.decoded("RECURRENCE-ID"), + before[3].component.decoded("RECURRENCE-ID"), + ] + assert all(e.params.get("TZID") == dtstart.params.get("TZID") for e in exdates), "EXDATE not in DTSTART's form" + assert not [c for c in stored.icalendar.walk("VEVENT") if "RECURRENCE-ID" in c], "the override was kept" + + async def test_delete_an_occurrence_read_before_a_sibling_was_deleted(self, collection: AsyncCollection) -> None: + series = await self.recurring_series(collection) + read = await self.occurrences(collection, series) + await collection.delete(read[4]) + await collection.delete(read[5]) # its etag is stale, but the occurrence itself is unchanged (Q12) + assert sorted(await self.occurrences(collection, series)) == [1, 2, 3] + + async def test_delete_an_occurrence_changed_since_it_was_read(self, collection: AsyncCollection) -> None: + series = await self.recurring_series(collection) + read = await self.occurrences(collection, series) + series.component["SUMMARY"] = "R: renamed" + await collection.save(series) + with pytest.raises(ConflictError): + await collection.delete(read[2]) + await collection.delete(read[2], overwrite=True) + assert sorted(await self.occurrences(collection, series)) == [1, 3, 4, 5] + # === relatives ==================================================== async def test_relatives(self, collection: AsyncCollection) -> None: diff --git a/src/calendaring/_sync/collection.py b/src/calendaring/_sync/collection.py index 9553f35..288ba10 100644 --- a/src/calendaring/_sync/collection.py +++ b/src/calendaring/_sync/collection.py @@ -35,7 +35,7 @@ UnsupportedError, VerificationError, ) -from ..items import Event, Item, Journal, Task, _items_of, _parse, split_stored +from ..items import Event, Item, Journal, Task, _items_of, _parse, split_stored, without_occurrence from ..loss import RAISE, LossPolicy from ..types import BrokenObject, ChangeSet, Component, Scope, StoredObject, SyncToken from .items import BOUND, SyncEvent, SyncItem, SyncJournal, SyncTask @@ -52,6 +52,20 @@ } +def _occurrence_now(calendar: icalendar.Calendar, item: Item) -> icalendar.Component | None: + """The occurrence ``item`` was, as the stored object has it now: its override, or one generated from the master.""" + wanted = item.component["RECURRENCE-ID"] + for component in _items_of(calendar): + if component.get("RECURRENCE-ID") == wanted: + return component + rid = item.component.decoded("RECURRENCE-ID") + generated: list[icalendar.Component] = recurring_ical_events.of(calendar).between(rid, rid + timedelta(seconds=1)) + for component in generated: + if component.get("RECURRENCE-ID") == wanted: + return component + return None + + def _occurs_at(item: Item) -> datetime: """Where an occurrence sorts: its ``RECURRENCE-ID`` as an aware instant (floating counts as UTC).""" rid = item.component.get("RECURRENCE-ID") @@ -715,10 +729,37 @@ def delete(self, item: Item | str, *, overwrite: bool = False) -> None: if item.native_id is None: raise ValueError("delete(): the item was never stored") if item.is_occurrence: - # EXDATE or STATUS:CANCELLED is undecided (API_DESIGN.md Q12). - raise UnsupportedError(None, "deleting one occurrence is undecided (API_DESIGN.md Q12)") + self._delete_occurrence(item, overwrite=overwrite) + return self._delete(item.native_id, etag=None if overwrite else item.etag) + def _delete_occurrence(self, item: Item, *, overwrite: bool) -> None: + """An ``EXDATE`` in the master's own form, and the override gone (API_DESIGN.md Q12). + + Siblings share the stored object's etag, so a stale etag is not a + conflict by itself: only a change to this occurrence on the server is. + It is found by comparing the stored occurrence with the item as the + caller holds it, so once the etag moved, a local edit before + ``delete()`` also counts as one; with the etag unchanged nothing + changed on the server, and a local edit is deleted with the rest. + An occurrence already gone is ``NotFoundError``. + """ + assert item.native_id is not None + stored = self._fetch(item.native_id) + if stored.etag != item.etag: + now = _occurrence_now(stored.calendar, item) + if now is None: + raise NotFoundError(f"the occurrence {item.component.decoded('RECURRENCE-ID')} is gone") + if not overwrite and now != item.component: + raise ConflictError( + f"the occurrence {item.component.decoded('RECURRENCE-ID')} changed since it was read" + ) + calendar = without_occurrence(stored.calendar, item.uid, item.component.decoded("RECURRENCE-ID")) + if not _items_of(calendar): + self._delete(item.native_id, etag=stored.etag) + return + self._update(item.native_id, calendar, etag=stored.etag, wire=None) + def complete( self, task: Task, diff --git a/src/calendaring/_sync/conformance.py b/src/calendaring/_sync/conformance.py index d66c8ce..a751da7 100644 --- a/src/calendaring/_sync/conformance.py +++ b/src/calendaring/_sync/conformance.py @@ -33,6 +33,7 @@ async def open_backend(self) -> AsyncBackend: from collections.abc import Iterator, Callable, Sequence from datetime import UTC, datetime, timedelta from typing import Any +from zoneinfo import ZoneInfo import icalendar import pytest @@ -953,6 +954,72 @@ def test_save_one_occurrence(self, collection: Collection) -> None: again = stored_by(collection.search(event=True, start=at(2), end=at(4), expand=True), seeded) assert sorted(str(i.component["SUMMARY"]) for i in again) == ["R: daily", "R: moved"] + def recurring_series(self, collection: Collection) -> SyncEvent: + """A stored daily series in Europe/Oslo time, with day 3 moved (an override), or the test ends here.""" + if not writable(collection): + self.assert_read_only(collection) + pytest.skip("read-only collection: the write paths were checked to raise") + if not holds(collection, Component.EVENT): + pytest.skip("the collection holds no events") + if level(collection, Feature.RECURRENCE) is Support.UNSUPPORTED: + pytest.skip("nothing recurs: test_expanded_search_returns_occurrences checks the refusal") + oslo = ZoneInfo("Europe/Oslo") + start = at(1, 9).astimezone(oslo) + series = Event.new(summary="R: daily", start=start, end=start + timedelta(hours=1)) + series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) + moved = Event.new(summary="R: moved", start=start + timedelta(days=2, hours=5), uid=series.uid) + moved.component["DTEND"] = icalendar.vDDDTypes(start + timedelta(days=2, hours=6)) + moved.component.add("RECURRENCE-ID", start + timedelta(days=2)) + series.icalendar.add_component(moved.component) + series.icalendar.add_missing_timezones() + [stored] = self.seed(collection, [series]) + assert isinstance(stored, SyncEvent) + return stored + + def occurrences(self, collection: Collection, series: SyncItem) -> dict[int, SyncItem]: + """The series' occurrences, by day of the month of their RECURRENCE-ID.""" + found = collection.search(event=True, start=at(1, 0), end=at(9, 0), expand=True) + return {i.component.decoded("RECURRENCE-ID").day: i for i in found if i.uid == series.uid} + + def test_delete_one_occurrence(self, collection: Collection) -> None: + series = self.recurring_series(collection) + before = self.occurrences(collection, series) + assert sorted(before) == [1, 2, 3, 4, 5] + collection.delete(before[2]) + with pytest.raises(NotFoundError): + collection.delete(before[2]) # already gone + collection.delete(before[3]) # the moved one + after = self.occurrences(collection, series) + assert sorted(after) == [1, 4, 5] + stored = collection.get(series.uid) + assert not stored.is_occurrence, "deleting an occurrence deleted the series" + dtstart = stored.component["DTSTART"] + exdates = stored.component.get("EXDATE") + exdates = exdates if isinstance(exdates, list) else [exdates] + assert sorted(d.dt for e in exdates for d in e.dts) == [ + before[2].component.decoded("RECURRENCE-ID"), + before[3].component.decoded("RECURRENCE-ID"), + ] + assert all(e.params.get("TZID") == dtstart.params.get("TZID") for e in exdates), "EXDATE not in DTSTART's form" + assert not [c for c in stored.icalendar.walk("VEVENT") if "RECURRENCE-ID" in c], "the override was kept" + + def test_delete_an_occurrence_read_before_a_sibling_was_deleted(self, collection: Collection) -> None: + series = self.recurring_series(collection) + read = self.occurrences(collection, series) + collection.delete(read[4]) + collection.delete(read[5]) # its etag is stale, but the occurrence itself is unchanged (Q12) + assert sorted(self.occurrences(collection, series)) == [1, 2, 3] + + def test_delete_an_occurrence_changed_since_it_was_read(self, collection: Collection) -> None: + series = self.recurring_series(collection) + read = self.occurrences(collection, series) + series.component["SUMMARY"] = "R: renamed" + collection.save(series) + with pytest.raises(ConflictError): + collection.delete(read[2]) + collection.delete(read[2], overwrite=True) + assert sorted(self.occurrences(collection, series)) == [1, 3, 4, 5] + # === relatives ==================================================== def test_relatives(self, collection: Collection) -> None: diff --git a/src/calendaring/items.py b/src/calendaring/items.py index ec134b2..b2e0726 100644 --- a/src/calendaring/items.py +++ b/src/calendaring/items.py @@ -83,6 +83,46 @@ def split_stored(calendar: icalendar.Calendar) -> list[icalendar.Calendar]: return out +def without_occurrence(calendar: icalendar.Calendar, uid: str, rid: date | datetime) -> icalendar.Calendar: + """A copy with one occurrence gone: its override removed, and an ``EXDATE`` on the master. + + The ``EXDATE`` takes the master's ``DTSTART`` form (a date, a date-time + in its ``TZID``, UTC or floating): clients that compare the two in + different forms have ignored it (Exchange 2010, Lightning, khal). A + master without ``DTSTART`` (invalid with an ``RRULE``, but seen) gets the + ``RECURRENCE-ID``'s own form. + """ + out = copy.deepcopy(calendar) + for component in list(out.subcomponents): + if str(component.get("UID")) == uid and component.get("RECURRENCE-ID") is not None: + if _same_instant(component.decoded("RECURRENCE-ID"), rid): + out.subcomponents.remove(component) + for master in _items_of(out): + if str(master.get("UID")) == uid and "RECURRENCE-ID" not in master: + master.add("EXDATE", _in_form_of(master.decoded("DTSTART"), rid) if "DTSTART" in master else rid) + return out + + +def _same_instant(a: date | datetime, b: date | datetime) -> bool: + """Equal as RECURRENCE-IDs: both dates, both floating, or both aware and the same instant.""" + if isinstance(a, datetime) != isinstance(b, datetime): + return False + if isinstance(a, datetime) and isinstance(b, datetime) and (a.tzinfo is None) != (b.tzinfo is None): + return False + return a == b + + +def _in_form_of(dtstart: date | datetime, rid: date | datetime) -> date | datetime: + """``rid`` written the way ``dtstart`` is: a date, the same zone, or floating.""" + if not isinstance(dtstart, datetime): + return rid.date() if isinstance(rid, datetime) else rid + if not isinstance(rid, datetime): + rid = datetime(rid.year, rid.month, rid.day, dtstart.hour, dtstart.minute, dtstart.second) + if dtstart.tzinfo is None: + return rid.replace(tzinfo=None) + return rid.astimezone(dtstart.tzinfo) if rid.tzinfo is not None else rid.replace(tzinfo=dtstart.tzinfo) + + class Item: """One event, task or journal: one ``VCALENDAR`` with the components of one ``UID``. diff --git a/tests/_async/test_generic.py b/tests/_async/test_generic.py index 07354d7..39ca0d7 100644 --- a/tests/_async/test_generic.py +++ b/tests/_async/test_generic.py @@ -173,15 +173,12 @@ async def test_completed_is_utc() -> None: assert b"COMPLETED:20261101T120000Z" in task.to_ical() -async def test_deleting_an_occurrence_is_a_library_limitation() -> None: +async def test_an_occurrence_is_not_a_whole_object_for_wire() -> None: collection = await AsyncMemoryBackend().collection("default") series = Event.new(summary="daily", start=at(1, 9), end=at(1, 10)) series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) await collection.add(series) first = (await collection.search(event=True, start=at(2, 0), end=at(2, 23), expand=True))[0] - with pytest.raises(UnsupportedError) as info: - await collection.delete(first) - assert info.value.feature is None with pytest.raises(ValueError, match="wire"): await collection.save(first, wire=b"") diff --git a/tests/_sync/test_generic.py b/tests/_sync/test_generic.py index a3210ac..7ccefce 100644 --- a/tests/_sync/test_generic.py +++ b/tests/_sync/test_generic.py @@ -173,15 +173,12 @@ def test_completed_is_utc() -> None: assert b"COMPLETED:20261101T120000Z" in task.to_ical() -def test_deleting_an_occurrence_is_a_library_limitation() -> None: +def test_an_occurrence_is_not_a_whole_object_for_wire() -> None: collection = MemoryBackend().collection("default") series = Event.new(summary="daily", start=at(1, 9), end=at(1, 10)) series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) collection.add(series) first = (collection.search(event=True, start=at(2, 0), end=at(2, 23), expand=True))[0] - with pytest.raises(UnsupportedError) as info: - collection.delete(first) - assert info.value.feature is None with pytest.raises(ValueError, match="wire"): collection.save(first, wire=b"") diff --git a/tests/test_items.py b/tests/test_items.py index 2fd10ba..c65913d 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -7,7 +7,7 @@ import pytest from calendaring import Event, InvalidDataError, Item, Journal, Task, TaskStatus -from calendaring.items import split_stored +from calendaring.items import split_stored, without_occurrence UTC_NOON = datetime(2026, 11, 1, 12, tzinfo=UTC) @@ -165,3 +165,36 @@ def test_new_has_one_uid_and_its_timezones() -> None: task = Task.new(summary="a", end=datetime(2026, 11, 1, 12, tzinfo=ZoneInfo("Europe/Oslo"))) assert "UID" not in task.icalendar, "the VCALENDAR must not carry a UID of its own" assert [c.name for c in task.icalendar.subcomponents] == ["VTIMEZONE", "VTODO"] + + +@pytest.mark.parametrize( + ("dtstart", "rid", "exdate"), + [ + ("DTSTART;VALUE=DATE:20261101", "RECURRENCE-ID;VALUE=DATE:20261102", "EXDATE;VALUE=DATE:20261102"), + ("DTSTART:20261101T120000", "RECURRENCE-ID:20261102T120000", "EXDATE:20261102T120000"), + ("DTSTART:20261101T120000Z", "RECURRENCE-ID:20261102T120000Z", "EXDATE:20261102T120000Z"), + ( + "DTSTART;TZID=Europe/Oslo:20261101T120000", + "RECURRENCE-ID:20261102T110000Z", + "EXDATE;TZID=Europe/Oslo:20261102T120000", + ), + ], + ids=["date", "floating", "utc", "zoned"], +) +def test_an_exdate_takes_the_form_of_dtstart(dtstart: str, rid: str, exdate: str) -> None: + master = todo("r", f"{dtstart}\r\nRRULE:FREQ=DAILY;COUNT=3\r\n") + override = todo("r", f"{rid}\r\n") + data = Task.from_ical(calendar(master, override)).icalendar + rid_value = Task.from_ical(calendar(override)).component.decoded("RECURRENCE-ID") + out = without_occurrence(data, "r", rid_value) + [left] = out.walk("VTODO") + assert "RECURRENCE-ID" not in left + assert exdate.encode() in left.to_ical() + + +def test_an_exdate_without_dtstart_takes_the_form_of_the_recurrence_id() -> None: + master = todo("r", "DUE:20261101T120000Z\r\nRRULE:FREQ=DAILY;COUNT=3\r\n") + override = todo("r", "RECURRENCE-ID:20261102T120000Z\r\n") + rid = Task.from_ical(calendar(override)).component.decoded("RECURRENCE-ID") + [left] = without_occurrence(Task.from_ical(calendar(master, override)).icalendar, "r", rid).walk("VTODO") + assert b"EXDATE:20261102T120000Z" in left.to_ical() From bdbfcbaa3a3a2de96aba91838103bf4170dc28ac Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 14:03:09 +0200 Subject: [PATCH 11/17] docs: record the CalDAV backend (2.1) News fragments for the CalDAV backend, deleting one occurrence and the reserved test_ keys. CONTRIBUTING says that make dev installs the embedded test servers, and how to wrap a dual-mode library such as caldav. SYNC_ASYNC_ARCHITECTURE marks the caldav bridge settled, and ROADMAP 2.1 is ticked, linking the caldav issues it filed. Prompt: read the tmp-handover* document and the ROADMAP and continue working on this project [tmp-handover*: an untracked session-handover note] Followup-Prompt: (...) then proceed with 2.1. [roadmap item 2.1, the CalDAV backend] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- CONTRIBUTING.md | 14 +++++++++++++- docs/design/ROADMAP.md | 19 +++++++++++-------- docs/design/SYNC_ASYNC_ARCHITECTURE.md | 8 ++++++++ news/+caldav-backend.feature.rst | 1 + news/+delete-occurrence.feature.rst | 1 + news/+test-keys.feature.rst | 1 + 6 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 news/+caldav-backend.feature.rst create mode 100644 news/+delete-occurrence.feature.rst create mode 100644 news/+test-keys.feature.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 838613a..bd1ee75 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,11 @@ pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commi make test ``` +`make dev` also installs caldav and the embedded Xandikos and Radicale +servers, which the CalDAV backend's conformance tests run against (they are +skipped when missing). Radicale listens on its default port, 5232, whatever +caldav's test helper is told, so that port must be free. + ## Sync and async code The async code in the `_async/` directories is the source; the `_sync/` @@ -50,7 +55,14 @@ copy is stale, and it also rejects what a token rewrite cannot translate: equivalent go in `src/calendaring/_concurrency.py`, written once per mode, with an entry in the replacement map (`agather` -> `gather`); - a third-party `Async*` name, such as `httpx.AsyncClient`, would become a - `SyncClient` that does not exist: wrap it in the per-mode layer too; + `SyncClient` that does not exist: import it through a mode-free module and + map it in the replacement map, as `_caldav_io.py` does for caldav's + `AsyncDAVClient`; +- a library whose methods return `X | Coroutine` (caldav's dual-mode + objects) hides a missing `await` from the type checker: call it only + through a thin, honestly typed wrapper in `_async/`, as `_async/dav.py` + does, where one helper resolves the union per mode (`_awaited`, which the + generator turns into `_returned`); - the token right after `await` is dropped, meant to be the space: write `await x`, never `await(x)`; - a name the generator renames must not sit inside an f-string, since diff --git a/docs/design/ROADMAP.md b/docs/design/ROADMAP.md index 918ee1e..3e6dbbe 100644 --- a/docs/design/ROADMAP.md +++ b/docs/design/ROADMAP.md @@ -492,27 +492,30 @@ and this is the backend that will expose every place the unified API and CalDAV disagree. **Tasks:** -- [ ] Map the unified API onto `caldav` 3.x -- [ ] Map caldav's compatibility-hint/quirk system onto the capability declaration -- [ ] Handle the sync/async impedance between caldav's dual-mode design and +- [x] Map the unified API onto `caldav` 3.x +- [x] Map caldav's compatibility-hint/quirk system onto the capability declaration +- [x] Handle the sync/async impedance between caldav's dual-mode design and p4 as 1.3 builds it — this is the interesting part, and the part most likely to overrun. caldav's `Async*` names are aliases of its dual-mode classes, so a thin, honestly typed shim is needed for type checking to work in this layer at all (see 0.2's comparison, *What is not settled*), and every caldav name needs a replacement-map entry so the generated sync layer imports names that exist (`SyncDAVClient` does not) -- [ ] Settle [API_DESIGN Q12](API_DESIGN.md#10-open-questions) (`delete()` on an +- [x] Settle [API_DESIGN Q12](API_DESIGN.md#10-open-questions) (`delete()` on an occurrence, and conflicts between sibling occurrences) before mapping `delete()` and `save()` for occurrences -- [ ] Tune `verify=True`'s comparison against what real servers rewrite +- [x] Tune `verify=True`'s comparison against what real servers rewrite (parameter quoting, split `CATEGORIES`, TZID to UTC), and compare subcomponents such as `VALARM` too; it compares raw unfolded lines today (roadmap 1.2 review) -- [ ] Decide what `get(uid)` returns for a stored object holding several +- [x] Decide what `get(uid)` returns for a stored object holding several overrides and no master; it returns the first today (roadmap 1.2 review) -- [ ] Pass the conformance suite -- [ ] Record, do not silently fix, bugs found in `caldav` itself: those fixes are +- [x] Pass the conformance suite +- [x] Record, do not silently fix, bugs found in `caldav` itself: those fixes are caldav's budget, not this project's + ([caldav #797](https://github.com/python-caldav/caldav/issues/797), its test + servers; the occurrence-deletion plan is on + [#598](https://github.com/python-caldav/caldav/issues/598)) --- diff --git a/docs/design/SYNC_ASYNC_ARCHITECTURE.md b/docs/design/SYNC_ASYNC_ARCHITECTURE.md index 13fd4ef..40f90ab 100644 --- a/docs/design/SYNC_ASYNC_ARCHITECTURE.md +++ b/docs/design/SYNC_ASYNC_ARCHITECTURE.md @@ -722,6 +722,14 @@ did not write the enforcement. has a thin, honestly typed shim over caldav — separate sync and async wrappers, or a `cast` at every call. With the shim, p4's static check would reach it too. That work is the same whichever design 1.3 builds. + + *Settled in roadmap 2.1:* the shim is `_async/dav.py`, written once, + its sync twin generated like the rest. Each call resolves caldav's + `X | Coroutine` through `_awaited` (generated as `_returned`), which + also fails if the client is of the wrong mode. caldav's `AsyncDAVClient` + is imported through the mode-free `_caldav_io.py` and mapped to + `DAVClient`. mypy covers the backend; the shim's casts are the only + unchecked spots. 6. **`check_ast.py` is a prototype, not the gate 1.3 needs (p2b).** The suite runs it (`test_static_check_passes_the_real_modules_and_catches_the_specimen`), but a review found three holes, each confirmed with a probe: it only knows diff --git a/news/+caldav-backend.feature.rst b/news/+caldav-backend.feature.rst new file mode 100644 index 0000000..2681051 --- /dev/null +++ b/news/+caldav-backend.feature.rst @@ -0,0 +1 @@ +A CalDAV backend, over the caldav library: ``pip install 'calendaring[caldav]'``, then ``Backend.connect("https://...", username=..., password=...)`` or a ``kind: caldav`` section in the configuration file. It declares its capabilities from caldav's knowledge of the server, uses the server's sync-token for ``changes()`` where it is reliable, and passes the conformance suite against Xandikos and Radicale. diff --git a/news/+delete-occurrence.feature.rst b/news/+delete-occurrence.feature.rst new file mode 100644 index 0000000..49fdb9f --- /dev/null +++ b/news/+delete-occurrence.feature.rst @@ -0,0 +1 @@ +``delete()`` on one occurrence of a recurring event or task removes just that occurrence (an ``EXDATE`` on the series), instead of raising ``UnsupportedError``. diff --git a/news/+test-keys.feature.rst b/news/+test-keys.feature.rst new file mode 100644 index 0000000..2c87aad --- /dev/null +++ b/news/+test-keys.feature.rst @@ -0,0 +1 @@ +Keys starting with ``test_`` in a configuration section are reserved for test tooling, and are ignored when connecting. From 8c024dacb1837ec1223fface9f2a918759163870 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 16:36:58 +0200 Subject: [PATCH 12/17] refactor: rename Backend.connect() to open() For CalDAV, files and feeds the factory does no network I/O: caldav's DAVClient() only builds an object, and opening a file connects to nothing, so "connect" over-promised. It stays a classmethod and a coroutine in async mode, since JMAP fetches a session resource and a tracker may check its token. The private hook _connect becomes _open. Nothing is released, so there is no deprecated alias. Prompts (comments made inline in the document while reviewing it): > (on the review finding that connect(token=...) raised a bare TypeError) Should we even have a `.connect`-method in the calendaring library? The `get_davclient()` and `DAVClient`-constructor in the caldav library does not (unless it has been changed recently) connect to any server, it just builds up an object that can be used for communicating. When opening a file there is also no "connect" going on. Prompt: Another subagent can work on renaming connect to open. [= rename Backend.connect() in calendaring to Backend.open()] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- CONTRIBUTING.md | 6 +++--- docs/design/API_DESIGN.md | 21 +++++++++++++-------- docs/design/CONFIGURATION_PROPOSAL.md | 10 +++++----- docs/design/ROADMAP.md | 8 ++++---- news/+caldav-backend.feature.rst | 2 +- news/+test-keys.feature.rst | 2 +- src/calendaring/_async/backend.py | 10 +++++----- src/calendaring/_async/caldav.py | 4 ++-- src/calendaring/_async/memory.py | 4 ++-- src/calendaring/_async/workspace.py | 6 +++--- src/calendaring/_sync/backend.py | 10 +++++----- src/calendaring/_sync/caldav.py | 4 ++-- src/calendaring/_sync/memory.py | 4 ++-- src/calendaring/_sync/workspace.py | 6 +++--- tests/_async/test_caldav_backend.py | 6 +++--- tests/_async/test_caldav_conformance.py | 2 +- tests/_async/test_generic.py | 12 ++++++------ tests/_sync/test_caldav_backend.py | 6 +++--- tests/_sync/test_caldav_conformance.py | 2 +- tests/_sync/test_generic.py | 12 ++++++------ tests/specimens/async_mode.py | 2 +- tests/specimens/sync_mode.py | 4 ++-- 22 files changed, 74 insertions(+), 69 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd1ee75..3538a1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,8 +79,8 @@ annotated, and do not silence an `unused-coroutine` error. 1. Write it in `src/calendaring/_async/.py` as `AsyncBackend` and `AsyncCollection`, subclassing `AsyncBackend` and `AsyncCollection`. Set `kind = ""` on the backend class, so - `Backend.connect(kind="")` finds it. The backend implements - `_connect` and `collections`; the collection implements the primitives + `Backend.open(kind="")` finds it. The backend implements + `_open` and `collections`; the collection implements the primitives (`_query`, `_fetch`, `_create`, `_update`, `_delete`). Capability checks, loss policies, client-side filtering and the rest are written once in the base classes. `_async/memory.py`, the reference backend, @@ -94,7 +94,7 @@ annotated, and do not silence an `unused-coroutine` error. 4. Run the generator, and import both modes from a public module (as `calendaring/memory.py` does). Import that module in `calendaring/__init__.py`: a backend registers its `kind` when it is - imported, so until then `connect()` cannot find it. + imported, so until then `open()` cannot find it. 5. Add `tests/_async/test__conformance.py`, with a class that subclasses `calendaring.testing.AsyncConformance` and returns the backend from `open_backend()`. The generator writes the sync tests. diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 12a5da2..b699e19 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -248,7 +248,7 @@ class AsyncWorkspace: whose `docs/format.md` specifies it ([roadmap 1.4](ROADMAP.md#14-configuration-and-credentials)). It makes one backend per section the name stands for, with the section's name as the backend's id. It is a coroutine in async mode, since -connecting may need the network; any problem with the file is a +opening a backend may need the network; any problem with the file is a `ConfigurationError`. **Fan-out returns partial results.** `Workspace.collections` asks every @@ -267,9 +267,9 @@ them in turn. ```python class AsyncBackend: @classmethod - async def connect(cls, url: str, *, username: str | None = None, - password: str | None = None, token: str | None = None, - kind: str | None = None, session: Any = None, **options: Any) -> AsyncBackend: ... + async def open(cls, url: str, *, username: str | None = None, + password: str | None = None, token: str | None = None, + kind: str | None = None, session: Any = None, **options: Any) -> AsyncBackend: ... id: str # stable, from config or derived from the URL kind: str # "caldav", "files", "feed", "jmap", "gitea", ... @@ -282,23 +282,28 @@ class AsyncBackend: async def close(self) -> None: ... ``` -- `connect` returns the subclass for the kind it picks from the URL when +- `open` returns the subclass for the kind it picks from the URL when `kind` is not given: `file:` or a path → files; `webcal:` → feed (fetched as `https:`, [prior art Part 2](PRIOR_ART_AND_DECISIONS.md#part-2-standards)); `http(s):` → probe, which is [roadmap 2.3](ROADMAP.md#23-icalendar-feed-backend-read-only-http)'s "feed or CalDAV" detection (until then, `http(s):` means CalDAV, so an `https:` feed needs `kind="feed"`); RFC 6764 discovery for a bare domain comes from `caldav`. Trackers cannot be detected and need `kind=`. +- *Renamed 2026-10-19 from `connect`:* for CalDAV, files and feeds + the factory does no network I/O (caldav's `DAVClient()` only builds an + object, and opening a file connects to nothing), so "connect" + over-promised. It stays a coroutine in async mode, since JMAP fetches + its session resource and a tracker may check its token. - `session` is the caller-owned HTTP session that Home Assistant's `inject-websession` rule asks for ([prior art §1.4, Home Assistant](PRIOR_ART_AND_DECISIONS.md#14-home-assistant)). A backend that cannot use it raises `ConfigurationError` instead of silently making its own. - Credentials are never required in the URL or in plain-text configuration - ([roadmap 1.4](ROADMAP.md#14-configuration-and-credentials)'s keyring work); `connect`'s keyword arguments are for programmatic + ([roadmap 1.4](ROADMAP.md#14-configuration-and-credentials)'s keyring work); `open`'s keyword arguments are for programmatic use. - *Decided 2026-10-19, with the CalDAV backend ([roadmap 2.1](ROADMAP.md#21-caldav-backend)):* which configuration keys a backend ignores. Keys starting with `test_` are reserved for test tooling, and `Workspace.from_config()` drops them before - `connect()`. Every other key reaches the backend, and one it does not + `open()`. Every other key reaches the backend, and one it does not take is a `ConfigurationError` naming the section, so a typo is not silently ignored. The only such keys found are caldav's test-server selection, `testing_allowed` and `priority`; caldav is to read @@ -1392,7 +1397,7 @@ caldav's public API as of 2026-10-08 (caldav 3.4.0). | caldav | calendaring | | |---|---|---| -| `get_davclient()`, `get_calendar(s)()`, config file | `Backend.connect()`, `Workspace.from_config()` | changed; same config file ([config proposal](CONFIGURATION_PROPOSAL.md)) | +| `get_davclient()`, `get_calendar(s)()`, config file | `Backend.open()`, `Workspace.from_config()` | changed; same config file ([config proposal](CONFIGURATION_PROPOSAL.md)) | | `principal()`, `calendars()`, `make_calendar(supported_calendar_component_set=…)` | `backend.collections()`, `create_collection(components=…)` | same | | `get_supported_components()` | `collection.components` | same | | `search(…)`, `CalDAVSearcher` | `collection.search(…)` with a `Searcher` | same; caldav already post-filters client-side | diff --git a/docs/design/CONFIGURATION_PROPOSAL.md b/docs/design/CONFIGURATION_PROPOSAL.md index b358584..451bec1 100644 --- a/docs/design/CONFIGURATION_PROPOSAL.md +++ b/docs/design/CONFIGURATION_PROPOSAL.md @@ -36,11 +36,11 @@ Requirements: [issue #49](https://github.com/pycalendar/plann/issues/49) leaves credential storage to this library). -## The format (F2): a `kind` key, with the keys `Backend.connect()` takes +## The format (F2): a `kind` key, with the keys `Backend.open()` takes The structure is caldav's: named sections, `inherits`, meta-sections with `contains`, globs, `disable` and `${VAR}` expansion. A section is the -keyword arguments of [`Backend.connect()`](API_DESIGN.md#22-backend): +keyword arguments of [`Backend.open()`](API_DESIGN.md#22-backend): `url`, `username`, `password`, `token`, `kind` and backend-specific options. @@ -76,12 +76,12 @@ everything: contains: [work, fastmail, notes, holidays, issues, legacy] ``` -- `kind` follows the rules `connect()` already has: a path or `file:` +- `kind` follows the rules `open()` already has: a path or `file:` means files, `webcal:` means a feed, `http(s):` is probed for feed or CalDAV, and JMAP and the trackers must say `kind:`. Once the secret keys are resolved and the structural keys (`contains`, `inherits`, `disable`) - are removed, `Workspace.from_config()` calls `connect(**section)` for - each section, so new-style key names need no mapping. One gap: `connect()` + are removed, `Workspace.from_config()` calls `open(**section)` for + each section, so new-style key names need no mapping. One gap: `open()` takes `url` as a required argument, and a GitHub section needs none (`api.github.com` is the default). API_DESIGN §2.2 has to make `url` optional for such kinds. diff --git a/docs/design/ROADMAP.md b/docs/design/ROADMAP.md index 3e6dbbe..c9d6dc5 100644 --- a/docs/design/ROADMAP.md +++ b/docs/design/ROADMAP.md @@ -344,7 +344,7 @@ Notes from doing it: project's own configuration, over the planted mistakes in `tests/specimens/` and asserts that exactly the marked lines are flagged, in both modes. The correct calls in the same files are 0.2's - case 1, for the main entry points (`connect`, `collection`, `save`, + case 1, for the main entry points (`open`, `collection`, `save`, `complete`, `tasks`, `iter_search`, `search`). The shapes the checkers miss, such as a missing `await` in a comprehension, are planted unmarked, so a checker that starts or stops catching one fails the test. The plain `mypy` and `ruff check .` runs @@ -404,8 +404,8 @@ therefore has a committed downstream consumer and should not be trimmed. - [x] Leave room for backends beyond the funded ones without format changes: GitHub first, then any plugin backend. A section's `kind` names any registered backend; a GitHub section still needs `url` until - `connect()` makes it optional, a gap the - [configuration proposal](CONFIGURATION_PROPOSAL.md#the-format-f2-a-kind-key-with-the-keys-backendconnect-takes) + `open()` makes it optional, a gap the + [configuration proposal](CONFIGURATION_PROPOSAL.md#the-format-f2-a-kind-key-with-the-keys-backendopen-takes) notes. - [x] Document the format: calendaring-config's `docs/format.md` @@ -703,7 +703,7 @@ effort is budgeted inside Phases 1 and 2. - **Dependencies:** 4.1 **Tasks:** -- [ ] Getting started: connect, list, read +- [ ] Getting started: open, list, read - [ ] CRUD for events and for tasks - [ ] Searching and filtering - [ ] The same program run against three different backends — the project's whole diff --git a/news/+caldav-backend.feature.rst b/news/+caldav-backend.feature.rst index 2681051..806f1b7 100644 --- a/news/+caldav-backend.feature.rst +++ b/news/+caldav-backend.feature.rst @@ -1 +1 @@ -A CalDAV backend, over the caldav library: ``pip install 'calendaring[caldav]'``, then ``Backend.connect("https://...", username=..., password=...)`` or a ``kind: caldav`` section in the configuration file. It declares its capabilities from caldav's knowledge of the server, uses the server's sync-token for ``changes()`` where it is reliable, and passes the conformance suite against Xandikos and Radicale. +A CalDAV backend, over the caldav library: ``pip install 'calendaring[caldav]'``, then ``Backend.open("https://...", username=..., password=...)`` or a ``kind: caldav`` section in the configuration file. It declares its capabilities from caldav's knowledge of the server, uses the server's sync-token for ``changes()`` where it is reliable, and passes the conformance suite against Xandikos and Radicale. diff --git a/news/+test-keys.feature.rst b/news/+test-keys.feature.rst index 2c87aad..9a583d3 100644 --- a/news/+test-keys.feature.rst +++ b/news/+test-keys.feature.rst @@ -1 +1 @@ -Keys starting with ``test_`` in a configuration section are reserved for test tooling, and are ignored when connecting. +Keys starting with ``test_`` in a configuration section are reserved for test tooling, and are ignored when opening a backend. diff --git a/src/calendaring/_async/backend.py b/src/calendaring/_async/backend.py index ec5d6cf..dbdba3b 100644 --- a/src/calendaring/_async/backend.py +++ b/src/calendaring/_async/backend.py @@ -19,7 +19,7 @@ __all__ = ["AsyncBackend"] -# URL scheme -> backend kind, for connect() without kind= (§2.2). http(s) +# URL scheme -> backend kind, for open() without kind= (§2.2). http(s) # is CalDAV until the "feed or CalDAV" probe of roadmap 2.3, and trackers need kind=. SCHEMES = {"": "files", "file": "files", "webcal": "feed", "memory": "memory", "http": "caldav", "https": "caldav"} @@ -64,7 +64,7 @@ def __repr__(self) -> str: return f"<{type(self).__name__} {self.id}>" @classmethod - async def connect( + async def open( cls, url: str, *, @@ -89,12 +89,12 @@ async def connect( extra = EXTRAS[kind] raise ConfigurationError(f"the {kind} backend needs: pip install 'calendaring[{extra}]'") from None raise ConfigurationError(f"no backend of kind {kind!r} (known: {sorted(kinds)})") from None - return await klass._connect(url, username=username, password=password, token=token, session=session, **options) + return await klass._open(url, username=username, password=password, token=token, session=session, **options) @classmethod @abstractmethod - async def _connect(cls, url: str, **options: Any) -> AsyncBackend: - """Make the backend ``connect()`` picked; credentials and ``session`` come in ``options``.""" + async def _open(cls, url: str, **options: Any) -> AsyncBackend: + """Make the backend ``open()`` picked; credentials and ``session`` come in ``options``.""" # --- collections -------------------------------------------------- diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index 86ca55b..f57aff2 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -189,7 +189,7 @@ async def _delete_collection(self) -> None: class AsyncCalDAVBackend(AsyncBackend): - """``AsyncBackend.connect("https://...", kind="caldav", username=..., password=...)``.""" + """``AsyncBackend.open("https://...", kind="caldav", username=..., password=...)``.""" kind = "caldav" collection_class: ClassVar[type[AsyncCalDAVCollection]] = AsyncCalDAVCollection @@ -201,7 +201,7 @@ def __init__(self, dav: AsyncDAV, *, id: str, loss: LossPolicy | None = None, ve ) @classmethod - async def _connect(cls, url: str, **options: Any) -> AsyncCalDAVBackend: + async def _open(cls, url: str, **options: Any) -> AsyncCalDAVBackend: if options.pop("session", None) is not None: raise ConfigurationError("the caldav backend cannot use a caller's session yet") token = options.pop("token", None) diff --git a/src/calendaring/_async/memory.py b/src/calendaring/_async/memory.py index 49749ea..0347791 100644 --- a/src/calendaring/_async/memory.py +++ b/src/calendaring/_async/memory.py @@ -107,7 +107,7 @@ def _wrap_native(self, native_item: object) -> StoredObject: class AsyncMemoryBackend(AsyncBackend): - """``AsyncMemoryBackend(profile=TRACKER)``, or ``AsyncBackend.connect("memory:")``.""" + """``AsyncMemoryBackend(profile=TRACKER)``, or ``AsyncBackend.open("memory:")``.""" kind = "memory" collection_class: ClassVar[type[AsyncMemoryCollection]] = AsyncMemoryCollection @@ -128,7 +128,7 @@ def __init__( ) @classmethod - async def _connect(cls, url: str, **options: Any) -> AsyncMemoryBackend: + async def _open(cls, url: str, **options: Any) -> AsyncMemoryBackend: for unused in ("username", "password", "token", "session"): options.pop(unused, None) return cls(**options) diff --git a/src/calendaring/_async/workspace.py b/src/calendaring/_async/workspace.py index b5c2986..44600e4 100644 --- a/src/calendaring/_async/workspace.py +++ b/src/calendaring/_async/workspace.py @@ -42,7 +42,7 @@ def _without_credentials(url: str) -> str: def _section_params(conn: calendaring_config.Connection) -> tuple[str, str, dict[str, Any]]: - """The section name, its url and the rest of ``connect()``'s keywords, the id included. + """The section name, its url and the rest of ``open()``'s keywords, the id included. Keys starting with ``test_`` are for test tooling (caldav's test-server selection, say) and never reach a backend. @@ -66,7 +66,7 @@ async def from_config(cls, path: str | Path | None = None, *, section: str | Non """One backend per section ``section`` stands for, each with the section's name as its id. The file, its discovery and its secrets are calendaring-config's - (its ``docs/format.md``); every problem with them, or with connecting + (its ``docs/format.md``); every problem with them, or with opening a section, is a ``ConfigurationError`` naming the section. Connections from the environment get the id ``environment-``. """ @@ -88,7 +88,7 @@ async def from_config(cls, path: str | Path | None = None, *, section: str | Non try: for (name, url, params), conn in zip(sections, connections, strict=True): try: - backends.append(await AsyncBackend.connect(url, kind=conn.kind, **params)) + backends.append(await AsyncBackend.open(url, kind=conn.kind, **params)) except Exception as e: message = str(e).replace(url, _without_credentials(url)) raise ConfigurationError(f"section {name}: {message}") from None diff --git a/src/calendaring/_sync/backend.py b/src/calendaring/_sync/backend.py index 3b0d573..56ca211 100644 --- a/src/calendaring/_sync/backend.py +++ b/src/calendaring/_sync/backend.py @@ -19,7 +19,7 @@ __all__ = ["Backend"] -# URL scheme -> backend kind, for connect() without kind= (§2.2). http(s) +# URL scheme -> backend kind, for open() without kind= (§2.2). http(s) # is CalDAV until the "feed or CalDAV" probe of roadmap 2.3, and trackers need kind=. SCHEMES = {"": "files", "file": "files", "webcal": "feed", "memory": "memory", "http": "caldav", "https": "caldav"} @@ -64,7 +64,7 @@ def __repr__(self) -> str: return f"<{type(self).__name__} {self.id}>" @classmethod - def connect( + def open( cls, url: str, *, @@ -89,12 +89,12 @@ def connect( extra = EXTRAS[kind] raise ConfigurationError(f"the {kind} backend needs: pip install 'calendaring[{extra}]'") from None raise ConfigurationError(f"no backend of kind {kind!r} (known: {sorted(kinds)})") from None - return klass._connect(url, username=username, password=password, token=token, session=session, **options) + return klass._open(url, username=username, password=password, token=token, session=session, **options) @classmethod @abstractmethod - def _connect(cls, url: str, **options: Any) -> Backend: - """Make the backend ``connect()`` picked; credentials and ``session`` come in ``options``.""" + def _open(cls, url: str, **options: Any) -> Backend: + """Make the backend ``open()`` picked; credentials and ``session`` come in ``options``.""" # --- collections -------------------------------------------------- diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index 23fbddd..ec31c72 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -189,7 +189,7 @@ def _delete_collection(self) -> None: class CalDAVBackend(Backend): - """``AsyncBackend.connect("https://...", kind="caldav", username=..., password=...)``.""" + """``AsyncBackend.open("https://...", kind="caldav", username=..., password=...)``.""" kind = "caldav" collection_class: ClassVar[type[CalDAVCollection]] = CalDAVCollection @@ -201,7 +201,7 @@ def __init__(self, dav: DAV, *, id: str, loss: LossPolicy | None = None, verify: ) @classmethod - def _connect(cls, url: str, **options: Any) -> CalDAVBackend: + def _open(cls, url: str, **options: Any) -> CalDAVBackend: if options.pop("session", None) is not None: raise ConfigurationError("the caldav backend cannot use a caller's session yet") token = options.pop("token", None) diff --git a/src/calendaring/_sync/memory.py b/src/calendaring/_sync/memory.py index 7129b67..21961f0 100644 --- a/src/calendaring/_sync/memory.py +++ b/src/calendaring/_sync/memory.py @@ -107,7 +107,7 @@ def _wrap_native(self, native_item: object) -> StoredObject: class MemoryBackend(Backend): - """``AsyncMemoryBackend(profile=TRACKER)``, or ``AsyncBackend.connect("memory:")``.""" + """``AsyncMemoryBackend(profile=TRACKER)``, or ``AsyncBackend.open("memory:")``.""" kind = "memory" collection_class: ClassVar[type[MemoryCollection]] = MemoryCollection @@ -128,7 +128,7 @@ def __init__( ) @classmethod - def _connect(cls, url: str, **options: Any) -> MemoryBackend: + def _open(cls, url: str, **options: Any) -> MemoryBackend: for unused in ("username", "password", "token", "session"): options.pop(unused, None) return cls(**options) diff --git a/src/calendaring/_sync/workspace.py b/src/calendaring/_sync/workspace.py index 8e56e9c..5a1939a 100644 --- a/src/calendaring/_sync/workspace.py +++ b/src/calendaring/_sync/workspace.py @@ -42,7 +42,7 @@ def _without_credentials(url: str) -> str: def _section_params(conn: calendaring_config.Connection) -> tuple[str, str, dict[str, Any]]: - """The section name, its url and the rest of ``connect()``'s keywords, the id included. + """The section name, its url and the rest of ``open()``'s keywords, the id included. Keys starting with ``test_`` are for test tooling (caldav's test-server selection, say) and never reach a backend. @@ -66,7 +66,7 @@ def from_config(cls, path: str | Path | None = None, *, section: str | None = No """One backend per section ``section`` stands for, each with the section's name as its id. The file, its discovery and its secrets are calendaring-config's - (its ``docs/format.md``); every problem with them, or with connecting + (its ``docs/format.md``); every problem with them, or with opening a section, is a ``ConfigurationError`` naming the section. Connections from the environment get the id ``environment-``. """ @@ -88,7 +88,7 @@ def from_config(cls, path: str | Path | None = None, *, section: str | None = No try: for (name, url, params), conn in zip(sections, connections, strict=True): try: - backends.append(Backend.connect(url, kind=conn.kind, **params)) + backends.append(Backend.open(url, kind=conn.kind, **params)) except Exception as e: message = str(e).replace(url, _without_credentials(url)) raise ConfigurationError(f"section {name}: {message}") from None diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py index 7365e51..b83d013 100644 --- a/tests/_async/test_caldav_backend.py +++ b/tests/_async/test_caldav_backend.py @@ -15,7 +15,7 @@ @pytest.fixture async def collection(xandikos_server: Any) -> AsyncIterator[AsyncCollection]: - backend = await AsyncBackend.connect( + backend = await AsyncBackend.open( xandikos_server.url, kind="caldav", username=xandikos_server.username, @@ -54,7 +54,7 @@ async def test_an_incomplete_incremental_sync_asks_to_start_again( async def test_a_token_is_bearer_auth() -> None: - backend = await AsyncBackend.connect("https://dav.example.com/", kind="caldav", token="s3cret") + backend = await AsyncBackend.open("https://dav.example.com/", kind="caldav", token="s3cret") async with backend: assert backend.native.auth_type == "bearer" # type: ignore[attr-defined] assert backend.native.password in ("s3cret", b"s3cret") # type: ignore[attr-defined] # bytes in sync mode @@ -62,4 +62,4 @@ async def test_a_token_is_bearer_auth() -> None: async def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None: with pytest.raises(ConfigurationError, match="no_such_option"): - await AsyncBackend.connect("https://dav.example.com/", kind="caldav", no_such_option=1) + await AsyncBackend.open("https://dav.example.com/", kind="caldav", no_such_option=1) diff --git a/tests/_async/test_caldav_conformance.py b/tests/_async/test_caldav_conformance.py index f0c2659..0872f7d 100644 --- a/tests/_async/test_caldav_conformance.py +++ b/tests/_async/test_caldav_conformance.py @@ -17,7 +17,7 @@ class CalDAVConformance(AsyncConformance): server: Any async def open_backend(self) -> AsyncBackend: - return await AsyncBackend.connect( + return await AsyncBackend.open( self.server.url, kind="caldav", username=self.server.username, diff --git a/tests/_async/test_generic.py b/tests/_async/test_generic.py index 39ca0d7..030fb9a 100644 --- a/tests/_async/test_generic.py +++ b/tests/_async/test_generic.py @@ -40,14 +40,14 @@ def at(day: int, hour: int = 12) -> datetime: return datetime(2026, 11, day, hour, tzinfo=UTC) -async def test_connect_picks_the_kind_from_the_url() -> None: - backend = await AsyncBackend.connect("memory:", id="from-url") +async def test_open_picks_the_kind_from_the_url() -> None: + backend = await AsyncBackend.open("memory:", id="from-url") assert isinstance(backend, AsyncMemoryBackend) assert backend.id == "from-url" with pytest.raises(ConfigurationError): - await AsyncBackend.connect("gopher://calendar.example.com/") + await AsyncBackend.open("gopher://calendar.example.com/") with pytest.raises(ConfigurationError): - await AsyncBackend.connect("memory:", kind="no-such-kind") + await AsyncBackend.open("memory:", kind="no-such-kind") async def test_verify_catches_a_silent_drop() -> None: @@ -376,12 +376,12 @@ class Backend(AsyncMemoryBackend): async def test_a_backend_from_a_missing_extra_says_what_to_install(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(AsyncBackend, "_kinds", {k: v for k, v in AsyncBackend._kinds.items() if k != "caldav"}) with pytest.raises(ConfigurationError, match=r"pip install 'calendaring\[caldav\]'"): - await AsyncBackend.connect("https://caldav.example.com/", kind="caldav") + await AsyncBackend.open("https://caldav.example.com/", kind="caldav") async def test_http_urls_are_caldav_until_the_feed_probe_exists() -> None: pytest.importorskip("caldav") - backend = await AsyncBackend.connect("https://caldav.example.com/dav/", username="u", password="p") + backend = await AsyncBackend.open("https://caldav.example.com/dav/", username="u", password="p") async with backend: assert backend.kind == "caldav" diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py index 7f29b61..3a16330 100644 --- a/tests/_sync/test_caldav_backend.py +++ b/tests/_sync/test_caldav_backend.py @@ -15,7 +15,7 @@ @pytest.fixture def collection(xandikos_server: Any) -> Iterator[Collection]: - backend = Backend.connect( + backend = Backend.open( xandikos_server.url, kind="caldav", username=xandikos_server.username, @@ -54,7 +54,7 @@ def test_an_incomplete_incremental_sync_asks_to_start_again( def test_a_token_is_bearer_auth() -> None: - backend = Backend.connect("https://dav.example.com/", kind="caldav", token="s3cret") + backend = Backend.open("https://dav.example.com/", kind="caldav", token="s3cret") with backend: assert backend.native.auth_type == "bearer" # type: ignore[attr-defined] assert backend.native.password in ("s3cret", b"s3cret") # type: ignore[attr-defined] # bytes in sync mode @@ -62,4 +62,4 @@ def test_a_token_is_bearer_auth() -> None: def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None: with pytest.raises(ConfigurationError, match="no_such_option"): - Backend.connect("https://dav.example.com/", kind="caldav", no_such_option=1) + Backend.open("https://dav.example.com/", kind="caldav", no_such_option=1) diff --git a/tests/_sync/test_caldav_conformance.py b/tests/_sync/test_caldav_conformance.py index b1eaa5f..b3d380c 100644 --- a/tests/_sync/test_caldav_conformance.py +++ b/tests/_sync/test_caldav_conformance.py @@ -17,7 +17,7 @@ class CalDAVConformance(SyncConformance): server: Any def open_backend(self) -> Backend: - return Backend.connect( + return Backend.open( self.server.url, kind="caldav", username=self.server.username, diff --git a/tests/_sync/test_generic.py b/tests/_sync/test_generic.py index 7ccefce..791cab4 100644 --- a/tests/_sync/test_generic.py +++ b/tests/_sync/test_generic.py @@ -40,14 +40,14 @@ def at(day: int, hour: int = 12) -> datetime: return datetime(2026, 11, day, hour, tzinfo=UTC) -def test_connect_picks_the_kind_from_the_url() -> None: - backend = Backend.connect("memory:", id="from-url") +def test_open_picks_the_kind_from_the_url() -> None: + backend = Backend.open("memory:", id="from-url") assert isinstance(backend, MemoryBackend) assert backend.id == "from-url" with pytest.raises(ConfigurationError): - Backend.connect("gopher://calendar.example.com/") + Backend.open("gopher://calendar.example.com/") with pytest.raises(ConfigurationError): - Backend.connect("memory:", kind="no-such-kind") + Backend.open("memory:", kind="no-such-kind") def test_verify_catches_a_silent_drop() -> None: @@ -376,12 +376,12 @@ class Backend(MemoryBackend): def test_a_backend_from_a_missing_extra_says_what_to_install(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(Backend, "_kinds", {k: v for k, v in Backend._kinds.items() if k != "caldav"}) with pytest.raises(ConfigurationError, match=r"pip install 'calendaring\[caldav\]'"): - Backend.connect("https://caldav.example.com/", kind="caldav") + Backend.open("https://caldav.example.com/", kind="caldav") def test_http_urls_are_caldav_until_the_feed_probe_exists() -> None: pytest.importorskip("caldav") - backend = Backend.connect("https://caldav.example.com/dav/", username="u", password="p") + backend = Backend.open("https://caldav.example.com/dav/", username="u", password="p") with backend: assert backend.kind == "caldav" diff --git a/tests/specimens/async_mode.py b/tests/specimens/async_mode.py index 91758d9..ce10f87 100644 --- a/tests/specimens/async_mode.py +++ b/tests/specimens/async_mode.py @@ -17,7 +17,7 @@ async def correct(backend: AsyncBackend, workspace: AsyncWorkspace, task: Task) -> str: - backend = await AsyncBackend.connect("memory:") + backend = await AsyncBackend.open("memory:") coll = await backend.collection("default") saved = await coll.save(task) await saved.save() diff --git a/tests/specimens/sync_mode.py b/tests/specimens/sync_mode.py index 4145c5d..bfd66ca 100644 --- a/tests/specimens/sync_mode.py +++ b/tests/specimens/sync_mode.py @@ -11,7 +11,7 @@ def correct(workspace: Workspace, task: Task) -> str: - backend = Backend.connect("memory:") + backend = Backend.open("memory:") coll = backend.collection("default") saved = coll.save(task) saved.save() @@ -33,6 +33,6 @@ async def bound_spurious(task: SyncTask) -> None: async def more_spurious(workspace: Workspace) -> None: - await Backend.connect("memory:") # expect: misc + await Backend.open("memory:") # expect: misc await workspace.search() # expect: misc await Workspace.from_config() # expect: misc From 1cfd18aa146f82debbc90aade148543bc81dcc04 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 19 Oct 2026 17:43:05 +0200 Subject: [PATCH 13/17] refactor: write CalDAV objects via caldav save() The backend wrote with raw PUT and DELETE, because caldav's save() sent no If-None-Match, and read caldav's private response body to recognise a UID conflict. caldav 3.6 sends If-None-Match: * for save(no_overwrite=True), so creates, updates and deletes now go through caldav's own objects and exceptions, and the raw-write helpers are gone. save() re-serialises what it sends, so write.wire is UNSUPPORTED on CalDAV until https://github.com/python-caldav/caldav/issues/763 is fixed. A weak ETag, which If-Match cannot match, is compared with a fresh read instead, and deleting a missing object is NotFoundError on every backend. The caldav extra now needs caldav 3.6, not released yet. Prompt: I will close the tmp-push-review-gate.md file - because when this fix is in caldav, we should get rid of the _body-logic in calendaring_client. The rename from connect to open should also go in through the same pull request. Followup-Prompt: (answer to a clarifying question: how the 2.1 PR gets the unreleased caldav) Wait for caldav 3.6 Followup-Prompt: (answer to a clarifying question: keep a raw PUT for wire= only?) Drop raw PUT entirely, and caldav #763 will land soon [https://github.com/python-caldav/caldav/issues/763] Followup-Prompt: meanwhile, rewrite the calendaring writes against the caldav branch Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- docs/design/API_DESIGN.md | 6 +- pyproject.toml | 5 +- src/calendaring/_async/caldav.py | 29 ++++++-- src/calendaring/_async/collection.py | 2 +- src/calendaring/_async/conformance.py | 4 + src/calendaring/_async/dav.py | 35 +++++---- src/calendaring/_caldav_features.py | 2 +- src/calendaring/_caldav_io.py | 103 +++++++++++++------------- src/calendaring/_sync/caldav.py | 29 ++++++-- src/calendaring/_sync/collection.py | 2 +- src/calendaring/_sync/conformance.py | 4 + src/calendaring/_sync/dav.py | 35 +++++---- tests/_async/test_caldav_backend.py | 62 +++++++++++++++- tests/_sync/test_caldav_backend.py | 62 +++++++++++++++- tests/test_caldav_io.py | 44 +++++++++++ 15 files changed, 322 insertions(+), 102 deletions(-) diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index b699e19..6d1afeb 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -350,7 +350,7 @@ class AsyncCollection: wire: bytes | None = None, loss: LossPolicy | None = None, verify: bool | None = None) -> AsyncItem: ... # scope: occurrences only, see "Recurrence"; verify: see "What happens when ... unsupported" - async def delete(self, item: Item | str, *, overwrite: bool = False) -> None: ... + async def delete(self, item: Item | str, *, overwrite: bool = False) -> None: ... # NotFoundError if gone async def complete(self, task: Task, at: datetime | None = None, mode: Literal["safe", "this_and_future"] = "safe", *, loss: LossPolicy | None = None, verify: bool | None = None) -> AsyncTask: ... @@ -1058,7 +1058,7 @@ initial set: | `search.server-side`, `search.time-range`, `search.text` | how much the server filters; never affects results ([§4 Search](#4-search)) | | `changes` | `changes()`: `FULL` with a native token, `EMULATED` by listing | | `changes.known` | `changes(known=...)` fetches only what changed: `FULL` where etags can be listed without content (CalDAV, vdir), `EMULATED` where everything is read and compared (JMAP, single `.ics` file, feed, Gitea) | -| `write.wire` | `add(wire=...)`, `save(wire=...)`: given bytes are sent as they are; `FULL` on CalDAV and vdir, `UNSUPPORTED` elsewhere | +| `write.wire` | `add(wire=...)`, `save(wire=...)`: given bytes are sent as they are; `FULL` on vdir, and on CalDAV once caldav sends given bytes unchanged ([caldav #763](https://github.com/python-caldav/caldav/issues/763)); `UNSUPPORTED` elsewhere | | `write.conditional` | `FULL` with an atomic precondition (ETag, `content_version`); `EMULATED` read-compare-write | | `identity.client-uid`, `identity.foreign-id` | [§3.2 Identity](#32-identity) | | `identity.client-native-id` | `add(native_id=...)`: the caller may name the stored object (CalDAV href, vdir file name) | @@ -1257,7 +1257,7 @@ Adopts vdirsyncer's contract ([prior art §1.2, vdirsyncer](PRIOR_ART_AND_DECISI | Backend | `etag` | `write.conditional` | |---|---|---| -| CalDAV | the server's ETag | `FULL` (`If-Match`) | +| CalDAV | the server's ETag | `FULL` (`If-Match`); for a weak ETag, which `If-Match` cannot match (GMX), a read-compare-write that can race | | files, vdir | `f"{st_mtime_ns};{st_ino}"` | `EMULATED`: compare, then atomic rename | | files, single `.ics` | hash of the item's serialisation | `EMULATED` | | feed | hash of the item's serialisation | n/a (read-only) | diff --git a/pyproject.toml b/pyproject.toml index 5a06eb7..7f7cba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,9 +30,10 @@ dependencies = [ ] [project.optional-dependencies] -caldav = ["caldav>=3.5"] +# 3.6: save(no_overwrite=True) sends If-None-Match: * (caldav branch feat/if-none-match-create). +caldav = ["caldav>=3.6"] # Embedded CalDAV servers the caldav backend's conformance tests run against. -test-servers = ["caldav>=3.5", "xandikos>=0.2", "radicale>=3.2"] +test-servers = ["caldav>=3.6", "xandikos>=0.2", "radicale>=3.2"] # ruff and mypy are pinned exactly; keep ruff in step with .pre-commit-config.yaml. # So is unasync: its output is committed, and the freshness test compares bytes. dev = [ diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index f57aff2..2f1fb1d 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -24,7 +24,7 @@ from .._caldav_features import backend_capabilities, collection_capabilities from .._caldav_io import Calendar from ..capabilities import Feature -from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError, SyncTokenError +from ..errors import AlreadyExistsError, ConfigurationError, ConflictError, NotFoundError, SyncTokenError from ..items import _items_of, _parse from ..loss import LossPolicy from ..types import Component, StoredObject, SyncToken @@ -156,11 +156,12 @@ async def _find_uid(self, uid: str) -> StoredObject: return self._stored(await self._backend._dav.by_uid(self.calendar, uid)) async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: - data = wire if wire is not None else calendar.to_ical() - uid = str(_items_of(calendar)[0].get("UID", "")) + data = calendar.to_ical() # wire= is refused before this: write.wire is UNSUPPORTED + first = _items_of(calendar)[0] + uid = str(first.get("UID", "")) name = native_id or (f"{uid}.ics" if SAFE_NAME.match(uid) else f"{uuid.uuid4()}.ics") try: - _, etag = await self._backend._dav.put(self._href(name), data, create=True, etag=None) + etag = await self._backend._dav.create(self.calendar, self._href(name), data, str(first.name)) except AlreadyExistsError as exc: if native_id is None: exc.taken = "uid" # the name was derived from the UID @@ -169,17 +170,31 @@ async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, return await self._fetch(name) return StoredObject(name, etag, _parse(data)) + async def _unless_changed(self, native_id: str, etag: str | None) -> str | None: + """The ETag to send as ``If-Match``. A weak one cannot match (RFC 9110 §13.1.1), and caldav + would send the write unconditionally, so it is compared with a fresh read instead: a + window between the read and the write, where a strong ETag has none.""" + if etag is None or not etag.startswith("W/"): + return etag + if (await self._fetch(native_id)).etag != etag: + raise ConflictError(f"{native_id!r} changed since it was read") + return None + async def _update( self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None ) -> StoredObject: - data = wire if wire is not None else calendar.to_ical() - _, new_etag = await self._backend._dav.put(self._href(native_id), data, create=False, etag=etag) + etag = await self._unless_changed(native_id, etag) + data = calendar.to_ical() + kind = str(_items_of(calendar)[0].name) + new_etag = await self._backend._dav.update(self.calendar, self._href(native_id), data, kind, etag) if new_etag is None: return await self._fetch(native_id) return StoredObject(native_id, new_etag, _parse(data)) async def _delete(self, native_id: str, *, etag: str | None) -> None: - await self._backend._dav.delete(self._href(native_id), etag=etag) + await self._backend._dav.delete( + self.calendar, self._href(native_id), await self._unless_changed(native_id, etag) + ) async def _set_name(self, name: str) -> None: await self._backend._dav.set_display_name(self.calendar, name) diff --git a/src/calendaring/_async/collection.py b/src/calendaring/_async/collection.py index b706c82..8ad57a6 100644 --- a/src/calendaring/_async/collection.py +++ b/src/calendaring/_async/collection.py @@ -720,7 +720,7 @@ async def save( return result async def delete(self, item: Item | str, *, overwrite: bool = False) -> None: - """Remove a stored item. Given an item, unless it changed since it was read.""" + """Remove a stored item: ``NotFoundError`` if it is gone. Given an item, unless it changed since it was read.""" with self._context(): self._require(Feature.WRITE) if isinstance(item, str): diff --git a/src/calendaring/_async/conformance.py b/src/calendaring/_async/conformance.py index d31af59..7a0df50 100644 --- a/src/calendaring/_async/conformance.py +++ b/src/calendaring/_async/conformance.py @@ -645,6 +645,10 @@ async def test_delete(self, collection: AsyncCollection) -> None: await collection.delete(nid(other)) with pytest.raises(NotFoundError): await collection.get(other.uid) + with pytest.raises(NotFoundError): + await collection.delete(nid(other)) + with pytest.raises(NotFoundError): + await collection.delete(stale, overwrite=True) async def test_read_only_rejects_every_write(self, backend: AsyncBackend, collection: AsyncCollection) -> None: if writable(collection): diff --git a/src/calendaring/_async/dav.py b/src/calendaring/_async/dav.py index 6b404de..66ab915 100644 --- a/src/calendaring/_async/dav.py +++ b/src/calendaring/_async/dav.py @@ -13,14 +13,13 @@ CalendarObjectResource, DisplayName, SyncResult, - WriteResult, _awaited, - _body, - _etag, _etags, - _status, + _resource, + _strong, _sync_result, - _write_headers, + creating, + missing_if, translated, ) from ..errors import ConfigurationError @@ -92,13 +91,23 @@ async def sync(self, calendar: Calendar, token: str | None) -> SyncResult: found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) return _sync_result(await _awaited(found)) - async def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: - with translated(): - response = await self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) - _status(response.status, f"PUT {url}", creating=create, body=_body(response)) - return response.status, _etag(response.headers) + async def create(self, calendar: Calendar, url: str, data: bytes, kind: str) -> str | None: + """Store a new object at ``url``: ``AlreadyExistsError`` if the name or the UID is taken. Its ETag, if strong.""" + obj = _resource(self.client, calendar, url, data, kind) + with translated(), creating(): + await _awaited(obj.save(no_overwrite=True, increase_seqno=False, only_this_recurrence=False)) + return _strong(obj.etag) - async def delete(self, url: str, *, etag: str | None) -> None: + async def update(self, calendar: Calendar, url: str, data: bytes, kind: str, etag: str | None) -> str | None: + """Replace the object at ``url``, if it still has ``etag`` (any version without one). Its new ETag, if strong.""" + obj = _resource(self.client, calendar, url, data, kind, etag) with translated(): - response = await self.client.delete(url, None if etag is None else {"If-Match": etag}) - _status(response.status, f"DELETE {url}") + await _awaited(obj.save(increase_seqno=False, only_this_recurrence=False, precondition="etag")) + return _strong(obj.etag) + + async def delete(self, calendar: Calendar, url: str, etag: str | None) -> None: + """Remove the object at ``url``: ``NotFoundError`` if it is gone. With ``etag``, ``ConflictError`` if it changed.""" + obj = _resource(self.client, calendar, url) + with translated(), missing_if(etag is None): + # caldav counts a 404 as success only without a precondition; If-Match: * keeps it an error. + await _awaited(obj.delete(if_match=etag if etag is not None else "*")) diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index 7bf95dc..75d1486 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -58,6 +58,7 @@ Feature.SEARCH_TEXT: E, Feature.CHANGES: {"level": E, "method": "etag-listing"}, Feature.MOVE: {"level": E, "note": "added to the target, then deleted here"}, + Feature.WRITE_WIRE: {"level": U, "note": "caldav re-serialises what it saves (caldav #763)"}, Feature.IDENTITY_FOREIGN_ID: {"level": U, "note": "the UID is native"}, Feature.RECURRENCE_EDIT_OCCURRENCE: {"level": U, "note": _PENDING}, Feature.RECURRENCE_EDIT_THIS_AND_FUTURE: U, @@ -80,7 +81,6 @@ def _level(features: Any, name: str) -> Support: STORED = frozenset( { Feature.WRITE, - Feature.WRITE_WIRE, Feature.WRITE_CONDITIONAL, Feature.IDENTITY_CLIENT_UID, Feature.IDENTITY_CLIENT_NATIVE_ID, diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py index f18565d..0106401 100644 --- a/src/calendaring/_caldav_io.py +++ b/src/calendaring/_caldav_io.py @@ -9,22 +9,20 @@ ``DAVClient``. Each wrapper resolves the union once, and fails loudly if the client is of the wrong mode. What both modes share lives here. -Writes go through the client's ``put()`` / ``delete()`` with the -preconditions this library needs (``If-None-Match: *`` to create, -``If-Match`` to update or delete), since ``CalendarObjectResource.save()`` -sends no ``If-None-Match`` and bumps ``SEQUENCE``. Those return the status -instead of raising, and ``_status()`` maps it. +Writes go through caldav's own ``save()`` and ``delete()``: ``save(no_overwrite=True)`` +sends ``If-None-Match: *`` where the server honours it (caldav 3.6), and an +update carries the ETag it was read with. ``SEQUENCE`` is left as given. """ from __future__ import annotations import inspect -from collections.abc import Coroutine, Iterator, Mapping +from collections.abc import Coroutine, Iterator from contextlib import contextmanager from typing import Any, TypeVar, cast from caldav.aio import AsyncDAVClient -from caldav.calendarobjectresource import CalendarObjectResource +from caldav.calendarobjectresource import CalendarObjectResource, Event, Journal, Todo from caldav.collection import Calendar from caldav.davclient import DAVClient from caldav.elements.dav import DisplayName, GetEtag @@ -53,13 +51,13 @@ "DAVClient", "DisplayName", "SyncResult", - "WriteResult", + "creating", + "missing_if", "translated", ] T = TypeVar("T") -CONTENT_TYPE = {"Content-Type": "text/calendar; charset=utf-8"} # A Depth: 1 PROPFIND for the ETag of every object in a calendar (RFC 4791 §8.2.1.1 lists hrefs this way). ETAG_PROPFIND = ( @@ -72,8 +70,6 @@ # none), and whether it is incomplete (truncated beyond what caldav itself follows, RFC 6578 §3.6). SyncResult = tuple[list[str], list[str], str | None, bool] -WriteResult = tuple[int, str | None] # HTTP status, and the ETag the server returned, if any - async def _awaited(value: T | Coroutine[Any, Any, T]) -> T: if not inspect.iscoroutine(value): @@ -113,42 +109,53 @@ def _translate(exc: dav_error.DAVError) -> CalendaringError: status = getattr(exc, "status", None) or getattr(getattr(exc, "response", None), "status", None) return (AuthenticationError if status == 401 else AuthorizationError)(message) if isinstance(exc, dav_error.RateLimitError): - return RateLimitError(message, retry_after=getattr(exc, "retry_after", None)) + return RateLimitError(message, retry_after=getattr(exc, "retry_after_seconds", None)) return BackendError(message) -def _status(status: int, what: str, *, creating: bool = False, body: bytes | str = b"") -> None: - """Raise for a write's status; 2xx passes.""" - if 200 <= status < 300: - return - if creating and status == 412: - raise AlreadyExistsError(f"{what}: the name is taken (412)", taken="native_id") - if creating and status in (403, 409) and "no-uid-conflict" in (body if isinstance(body, str) else body.decode()): - raise AlreadyExistsError(f"{what}: another object holds the UID (RFC 4791 §5.3.2.1)", taken="uid") - if status == 412: - raise ConflictError(f"{what}: changed since it was read (412)") - if status == 404: - raise NotFoundError(f"{what}: not found (404)") - if status == 401: - raise AuthenticationError(f"{what}: 401") - if status == 403: - raise AuthorizationError(f"{what}: 403") - if status == 429: - raise RateLimitError(f"{what}: 429") - raise BackendError(f"{what}: HTTP {status}") - - -def _write_headers(*, create: bool, etag: str | None) -> dict[str, str]: - headers = dict(CONTENT_TYPE) - if create: - headers["If-None-Match"] = "*" - elif etag is not None: - headers["If-Match"] = etag - return headers - - -def _body(response: Any) -> bytes | str: - return getattr(response, "_raw", b"") or b"" # DAVResponse keeps the body only privately +# The caldav class for each component a calendar object holds. +RESOURCE: dict[str, type[CalendarObjectResource]] = {"VEVENT": Event, "VTODO": Todo, "VJOURNAL": Journal} + + +def _resource( + client: Any, + calendar: Calendar, + url: str, + data: bytes | None = None, + kind: str | None = None, + etag: str | None = None, +) -> CalendarObjectResource: + """A caldav object for ``url``, holding ``data`` and the ETag it was read with, ready to save or delete.""" + props = {} if etag is None else {GetEtag.tag: etag} + cls = RESOURCE.get(kind or "", CalendarObjectResource) + return cls(client=client, url=url, data=None if data is None else data.decode(), parent=calendar, props=props) + + +def _strong(etag: str | None) -> str | None: + """The ETag a write left cached, if it is strong (RFC 4791 §5.3.4: a changed object gets none).""" + return None if etag is None or etag.startswith("W/") else etag + + +@contextmanager +def creating() -> Iterator[None]: + """caldav's refusals of a create-only save, as ``AlreadyExistsError``.""" + try: + yield + except dav_error.UidConflictError as exc: # another object holds the UID (RFC 4791 §5.3.2.1) + raise AlreadyExistsError(str(exc), taken="uid") from exc + except dav_error.ConsistencyError as exc: # If-None-Match failed, or caldav's search found it + raise AlreadyExistsError(str(exc), taken="native_id") from exc + + +@contextmanager +def missing_if(any_version: bool) -> Iterator[None]: + """With ``If-Match: *``, a failed precondition means there was nothing there (RFC 9110 §13.1.1).""" + try: + yield + except dav_error.ETagMismatchError as exc: + if not any_version: + raise + raise NotFoundError(str(exc)) from exc def _etags(response: Any, url: str) -> dict[str, str]: @@ -166,11 +173,3 @@ def _sync_result(found: Any) -> SyncResult: changed = [str(o.url) for o in found.objects if str(o.url) not in deleted] token = None if found.sync_token is None else str(found.sync_token) return changed, sorted(deleted), token, bool(found.truncated) - - -def _etag(headers: Mapping[str, str]) -> str | None: - """The ETag of a write's response, if the server sent a strong one (RFC 4791 §5.3.4).""" - for name, value in headers.items(): - if name.lower() == "etag": - return None if value.startswith("W/") else value - return None diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index ec31c72..588d73a 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -24,7 +24,7 @@ from .._caldav_features import backend_capabilities, collection_capabilities from .._caldav_io import Calendar from ..capabilities import Feature -from ..errors import AlreadyExistsError, ConfigurationError, NotFoundError, SyncTokenError +from ..errors import AlreadyExistsError, ConfigurationError, ConflictError, NotFoundError, SyncTokenError from ..items import _items_of, _parse from ..loss import LossPolicy from ..types import Component, StoredObject, SyncToken @@ -156,11 +156,12 @@ def _find_uid(self, uid: str) -> StoredObject: return self._stored(self._backend._dav.by_uid(self.calendar, uid)) def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: - data = wire if wire is not None else calendar.to_ical() - uid = str(_items_of(calendar)[0].get("UID", "")) + data = calendar.to_ical() # wire= is refused before this: write.wire is UNSUPPORTED + first = _items_of(calendar)[0] + uid = str(first.get("UID", "")) name = native_id or (f"{uid}.ics" if SAFE_NAME.match(uid) else f"{uuid.uuid4()}.ics") try: - _, etag = self._backend._dav.put(self._href(name), data, create=True, etag=None) + etag = self._backend._dav.create(self.calendar, self._href(name), data, str(first.name)) except AlreadyExistsError as exc: if native_id is None: exc.taken = "uid" # the name was derived from the UID @@ -169,17 +170,31 @@ def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: return self._fetch(name) return StoredObject(name, etag, _parse(data)) + def _unless_changed(self, native_id: str, etag: str | None) -> str | None: + """The ETag to send as ``If-Match``. A weak one cannot match (RFC 9110 §13.1.1), and caldav + would send the write unconditionally, so it is compared with a fresh read instead: a + window between the read and the write, where a strong ETag has none.""" + if etag is None or not etag.startswith("W/"): + return etag + if (self._fetch(native_id)).etag != etag: + raise ConflictError(f"{native_id!r} changed since it was read") + return None + def _update( self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None ) -> StoredObject: - data = wire if wire is not None else calendar.to_ical() - _, new_etag = self._backend._dav.put(self._href(native_id), data, create=False, etag=etag) + etag = self._unless_changed(native_id, etag) + data = calendar.to_ical() + kind = str(_items_of(calendar)[0].name) + new_etag = self._backend._dav.update(self.calendar, self._href(native_id), data, kind, etag) if new_etag is None: return self._fetch(native_id) return StoredObject(native_id, new_etag, _parse(data)) def _delete(self, native_id: str, *, etag: str | None) -> None: - self._backend._dav.delete(self._href(native_id), etag=etag) + self._backend._dav.delete( + self.calendar, self._href(native_id), self._unless_changed(native_id, etag) + ) def _set_name(self, name: str) -> None: self._backend._dav.set_display_name(self.calendar, name) diff --git a/src/calendaring/_sync/collection.py b/src/calendaring/_sync/collection.py index 288ba10..88f3178 100644 --- a/src/calendaring/_sync/collection.py +++ b/src/calendaring/_sync/collection.py @@ -720,7 +720,7 @@ def save( return result def delete(self, item: Item | str, *, overwrite: bool = False) -> None: - """Remove a stored item. Given an item, unless it changed since it was read.""" + """Remove a stored item: ``NotFoundError`` if it is gone. Given an item, unless it changed since it was read.""" with self._context(): self._require(Feature.WRITE) if isinstance(item, str): diff --git a/src/calendaring/_sync/conformance.py b/src/calendaring/_sync/conformance.py index a751da7..693bbbc 100644 --- a/src/calendaring/_sync/conformance.py +++ b/src/calendaring/_sync/conformance.py @@ -645,6 +645,10 @@ def test_delete(self, collection: Collection) -> None: collection.delete(nid(other)) with pytest.raises(NotFoundError): collection.get(other.uid) + with pytest.raises(NotFoundError): + collection.delete(nid(other)) + with pytest.raises(NotFoundError): + collection.delete(stale, overwrite=True) def test_read_only_rejects_every_write(self, backend: Backend, collection: Collection) -> None: if writable(collection): diff --git a/src/calendaring/_sync/dav.py b/src/calendaring/_sync/dav.py index 5fbba6f..767aa9c 100644 --- a/src/calendaring/_sync/dav.py +++ b/src/calendaring/_sync/dav.py @@ -13,14 +13,13 @@ CalendarObjectResource, DisplayName, SyncResult, - WriteResult, _returned, - _body, - _etag, _etags, - _status, + _resource, + _strong, _sync_result, - _write_headers, + creating, + missing_if, translated, ) from ..errors import ConfigurationError @@ -92,13 +91,23 @@ def sync(self, calendar: Calendar, token: str | None) -> SyncResult: found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) return _sync_result(_returned(found)) - def put(self, url: str, data: bytes, *, create: bool, etag: str | None) -> WriteResult: - with translated(): - response = self.client.put(url, data.decode(), _write_headers(create=create, etag=etag)) - _status(response.status, f"PUT {url}", creating=create, body=_body(response)) - return response.status, _etag(response.headers) + def create(self, calendar: Calendar, url: str, data: bytes, kind: str) -> str | None: + """Store a new object at ``url``: ``AlreadyExistsError`` if the name or the UID is taken. Its ETag, if strong.""" + obj = _resource(self.client, calendar, url, data, kind) + with translated(), creating(): + _returned(obj.save(no_overwrite=True, increase_seqno=False, only_this_recurrence=False)) + return _strong(obj.etag) - def delete(self, url: str, *, etag: str | None) -> None: + def update(self, calendar: Calendar, url: str, data: bytes, kind: str, etag: str | None) -> str | None: + """Replace the object at ``url``, if it still has ``etag`` (any version without one). Its new ETag, if strong.""" + obj = _resource(self.client, calendar, url, data, kind, etag) with translated(): - response = self.client.delete(url, None if etag is None else {"If-Match": etag}) - _status(response.status, f"DELETE {url}") + _returned(obj.save(increase_seqno=False, only_this_recurrence=False, precondition="etag")) + return _strong(obj.etag) + + def delete(self, calendar: Calendar, url: str, etag: str | None) -> None: + """Remove the object at ``url``: ``NotFoundError`` if it is gone. With ``etag``, ``ConflictError`` if it changed.""" + obj = _resource(self.client, calendar, url) + with translated(), missing_if(etag is None): + # caldav counts a 404 as success only without a precondition; If-Match: * keeps it an error. + _returned(obj.delete(if_match=etag if etag is not None else "*")) diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py index b83d013..a46d604 100644 --- a/tests/_async/test_caldav_backend.py +++ b/tests/_async/test_caldav_backend.py @@ -3,15 +3,27 @@ from __future__ import annotations +import base64 +import dataclasses from collections.abc import AsyncIterator from typing import Any import pytest -from calendaring import AsyncBackend, AsyncCollection, Component, ConfigurationError, SyncTokenError +from calendaring import ( + AsyncBackend, + AsyncCollection, + Component, + ConfigurationError, + ConflictError, + NotFoundError, + SyncTokenError, +) pytest.importorskip("caldav") +from calendaring._async.caldav import _name_of # noqa: E402 + @pytest.fixture async def collection(xandikos_server: Any) -> AsyncIterator[AsyncCollection]: @@ -63,3 +75,51 @@ async def test_a_token_is_bearer_auth() -> None: async def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None: with pytest.raises(ConfigurationError, match="no_such_option"): await AsyncBackend.open("https://dav.example.com/", kind="caldav", no_such_option=1) + + +def weakened(collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch) -> None: + """Make every ETag read from the server weak, as GMX hands them out.""" + stored = collection._stored # type: ignore[attr-defined] + monkeypatch.setattr( + collection, "_stored", lambda obj: dataclasses.replace(stored(obj), etag="W/" + stored(obj).etag) + ) + + +async def test_a_weak_etag_still_refuses_a_stale_write( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = await collection.add_task(summary="weak") + weakened(collection, monkeypatch) + mine = await collection.get_by_native_id(stored.native_id or "") + theirs = await collection.get_by_native_id(stored.native_id or "") + assert (mine.etag or "").startswith("W/") + theirs.component["SUMMARY"] = "theirs" + await theirs.save() + mine.component["SUMMARY"] = "mine" + with pytest.raises(ConflictError): + await mine.save() + with pytest.raises(ConflictError): + await collection.delete(mine) + fresh = await collection.get_by_native_id(stored.native_id or "") + fresh.component["SUMMARY"] = "fresh" + await fresh.save() + await collection.delete(await collection.get_by_native_id(stored.native_id or "")) + with pytest.raises(NotFoundError): + await collection.get_by_native_id(stored.native_id or "") + + +def test_a_native_id_with_a_slash_round_trips(collection: AsyncCollection) -> None: + # Xandikos refuses such a name, so this checks the mapping, not a server. + caldav_collection: Any = collection + href = caldav_collection._href("a/b.ics") + assert href == caldav_collection.url + "a%2Fb.ics" + assert _name_of(href) == "a/b.ics" + + +async def test_a_token_from_elsewhere_is_refused(collection: AsyncCollection) -> None: + caldav_collection: Any = collection + foreign = base64.urlsafe_b64encode(b'{"collection": "other", "sync-token": "t1"}').decode() + for token in (foreign, "not base64!", base64.urlsafe_b64encode(b"[]").decode()): + with pytest.raises(SyncTokenError): + caldav_collection._decode_server_token(token) + assert caldav_collection._decode_server_token(caldav_collection._encode_server_token("t1")) == "t1" diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py index 3a16330..5e16c3c 100644 --- a/tests/_sync/test_caldav_backend.py +++ b/tests/_sync/test_caldav_backend.py @@ -3,15 +3,27 @@ from __future__ import annotations +import base64 +import dataclasses from collections.abc import Iterator from typing import Any import pytest -from calendaring import Backend, Collection, Component, ConfigurationError, SyncTokenError +from calendaring import ( + Backend, + Collection, + Component, + ConfigurationError, + ConflictError, + NotFoundError, + SyncTokenError, +) pytest.importorskip("caldav") +from calendaring._sync.caldav import _name_of # noqa: E402 + @pytest.fixture def collection(xandikos_server: Any) -> Iterator[Collection]: @@ -63,3 +75,51 @@ def test_a_token_is_bearer_auth() -> None: def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None: with pytest.raises(ConfigurationError, match="no_such_option"): Backend.open("https://dav.example.com/", kind="caldav", no_such_option=1) + + +def weakened(collection: Collection, monkeypatch: pytest.MonkeyPatch) -> None: + """Make every ETag read from the server weak, as GMX hands them out.""" + stored = collection._stored # type: ignore[attr-defined] + monkeypatch.setattr( + collection, "_stored", lambda obj: dataclasses.replace(stored(obj), etag="W/" + stored(obj).etag) + ) + + +def test_a_weak_etag_still_refuses_a_stale_write( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + stored = collection.add_task(summary="weak") + weakened(collection, monkeypatch) + mine = collection.get_by_native_id(stored.native_id or "") + theirs = collection.get_by_native_id(stored.native_id or "") + assert (mine.etag or "").startswith("W/") + theirs.component["SUMMARY"] = "theirs" + theirs.save() + mine.component["SUMMARY"] = "mine" + with pytest.raises(ConflictError): + mine.save() + with pytest.raises(ConflictError): + collection.delete(mine) + fresh = collection.get_by_native_id(stored.native_id or "") + fresh.component["SUMMARY"] = "fresh" + fresh.save() + collection.delete(collection.get_by_native_id(stored.native_id or "")) + with pytest.raises(NotFoundError): + collection.get_by_native_id(stored.native_id or "") + + +def test_a_native_id_with_a_slash_round_trips(collection: Collection) -> None: + # Xandikos refuses such a name, so this checks the mapping, not a server. + caldav_collection: Any = collection + href = caldav_collection._href("a/b.ics") + assert href == caldav_collection.url + "a%2Fb.ics" + assert _name_of(href) == "a/b.ics" + + +def test_a_token_from_elsewhere_is_refused(collection: Collection) -> None: + caldav_collection: Any = collection + foreign = base64.urlsafe_b64encode(b'{"collection": "other", "sync-token": "t1"}').decode() + for token in (foreign, "not base64!", base64.urlsafe_b64encode(b"[]").decode()): + with pytest.raises(SyncTokenError): + caldav_collection._decode_server_token(token) + assert caldav_collection._decode_server_token(caldav_collection._encode_server_token("t1")) == "t1" diff --git a/tests/test_caldav_io.py b/tests/test_caldav_io.py index bf36b55..160acd3 100644 --- a/tests/test_caldav_io.py +++ b/tests/test_caldav_io.py @@ -28,3 +28,47 @@ def test_both_modes_have_the_same_methods_and_signatures() -> None: def test_only_the_async_mode_is_async(name: str) -> None: assert inspect.iscoroutinefunction(getattr(AsyncDAV, name)) assert not inspect.iscoroutinefunction(getattr(DAV, name)) + + +def test_caldav_errors_translate() -> None: + from caldav.lib import error as dav_error + + from calendaring._caldav_io import _translate + from calendaring.errors import BackendError, ConflictError, NotFoundError, RateLimitError, SyncTokenError + + assert isinstance(_translate(dav_error.NotFoundError(url="x")), NotFoundError) + assert isinstance(_translate(dav_error.ETagMismatchError(url="x")), ConflictError) + gone = dav_error.AuthorizationError(url="x", preconditions=["{DAV:}valid-sync-token"]) + assert isinstance(_translate(gone), SyncTokenError) + limited = _translate(dav_error.RateLimitError(url="x", retry_after="120", retry_after_seconds=120.0)) + assert isinstance(limited, RateLimitError) + assert limited.retry_after == 120.0 + assert type(_translate(dav_error.DAVError(url="x"))) is BackendError + + +def test_a_refused_create_says_what_was_taken() -> None: + from caldav.lib import error as dav_error + + from calendaring._caldav_io import creating + from calendaring.errors import AlreadyExistsError + + with pytest.raises(AlreadyExistsError) as uid, creating(): + raise dav_error.UidConflictError(url="x") + assert uid.value.taken == "uid" + with pytest.raises(AlreadyExistsError) as name, creating(): + raise dav_error.ConsistencyError("If-None-Match failed") + assert name.value.taken == "native_id" + + +def test_the_etag_listing_leaves_out_the_calendar_itself() -> None: + from calendaring._caldav_io import _etags + + class Response: + def expand_simple_props(self, props: object) -> dict[str, dict[str, str]]: + return { + "/cal/": {"{DAV:}getetag": '"c"'}, + "/cal/a.ics": {"{DAV:}getetag": '"a"'}, + "/cal/b.ics": {}, + } + + assert _etags(Response(), "https://dav.example.com/cal") == {"/cal/a.ics": '"a"'} From 1493f3f3e0fb61a90da03940e209a07f8a4e9638 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Tue, 20 Oct 2026 02:19:05 +0200 Subject: [PATCH 14/17] feat: raw bytes for wire and wire= on CalDAV (2.1) caldav 3.6 keeps the server's bytes as raw_data and sends given bytes unchanged with save(raw_data=...) (https://github.com/python-caldav/caldav/issues/763). item.wire is now raw_data rather than caldav's normalised copy, wire= goes out byte for byte, and write.wire is FULL on CalDAV. Xandikos refolds what it stores, so the new test checks the PUT body. Prompt: Continue working here. Check the handover-document. caldav 3.6 may not be released today or tomorrow, but do testing locally with the local caldav version [the handover's next item: use caldav's raw-bytes API for wire, as promised on https://github.com/pycalendar/calendaring-sync/pull/19] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- docs/design/API_DESIGN.md | 17 +++++++++-------- news/+caldav-backend.feature.rst | 2 +- pyproject.toml | 2 +- src/calendaring/_async/caldav.py | 17 +++++++++++------ src/calendaring/_async/dav.py | 20 +++++++++++++++----- src/calendaring/_caldav_features.py | 1 - src/calendaring/_caldav_io.py | 7 +++++++ src/calendaring/_sync/caldav.py | 17 +++++++++++------ src/calendaring/_sync/dav.py | 20 +++++++++++++++----- tests/_async/test_caldav_backend.py | 23 +++++++++++++++++++++++ tests/_sync/test_caldav_backend.py | 23 +++++++++++++++++++++++ 11 files changed, 116 insertions(+), 33 deletions(-) diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 6d1afeb..83624cf 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -541,11 +541,12 @@ untouched (a conformance test proposed in [prior art Part 2](PRIOR_ART_AND_DECIS `wire` holds the stored object exactly as the backend returned it when the item was read: the body of a CalDAV `GET`, the `calendar-data` of a `calendar-query` or `calendar-multiget` response, the content of a vdir -file. On CalDAV that means the raw response body, which `caldav` does not -keep today: it runs incoming data through `vcal.fix()`, and its -`wire_data` re-encodes that normalised copy. The CalDAV backend has to -capture the body itself, or wait for `caldav` to keep it -([caldav issue 763](https://github.com/python-caldav/caldav/issues/763)). It belongs to `etag`, and like `etag` it is a snapshot: an edit to +file. On CalDAV it is `caldav`'s `raw_data` (caldav 3.6, +[caldav issue 763](https://github.com/python-caldav/caldav/issues/763)), +taken before `vcal.fix()` normalises the data: a `GET` body byte for +byte, and from a `REPORT` the text of the `calendar-data` element, which +the XML parser has already decoded (entities, and CRLF turned into LF). +It belongs to `etag`, and like `etag` it is a snapshot: an edit to `component` does not change it, and a write sets it to `None`, because what the server stores may differ from what was sent (see "The content after a write" in [§2.3](#23-collection)); `reload()` reads it again. @@ -1058,7 +1059,7 @@ initial set: | `search.server-side`, `search.time-range`, `search.text` | how much the server filters; never affects results ([§4 Search](#4-search)) | | `changes` | `changes()`: `FULL` with a native token, `EMULATED` by listing | | `changes.known` | `changes(known=...)` fetches only what changed: `FULL` where etags can be listed without content (CalDAV, vdir), `EMULATED` where everything is read and compared (JMAP, single `.ics` file, feed, Gitea) | -| `write.wire` | `add(wire=...)`, `save(wire=...)`: given bytes are sent as they are; `FULL` on vdir, and on CalDAV once caldav sends given bytes unchanged ([caldav #763](https://github.com/python-caldav/caldav/issues/763)); `UNSUPPORTED` elsewhere | +| `write.wire` | `add(wire=...)`, `save(wire=...)`: given bytes are sent as they are; `FULL` on vdir, and on CalDAV through caldav's `save(raw_data=...)` (caldav 3.6, [caldav #763](https://github.com/python-caldav/caldav/issues/763)); `UNSUPPORTED` elsewhere | | `write.conditional` | `FULL` with an atomic precondition (ETag, `content_version`); `EMULATED` read-compare-write | | `identity.client-uid`, `identity.foreign-id` | [§3.2 Identity](#32-identity) | | `identity.client-native-id` | `add(native_id=...)`: the caller may name the stored object (CalDAV href, vdir file name) | @@ -1381,8 +1382,8 @@ not a live handle. The other direction also works: `collection.wrap(native_item)` turns a backend object (a `caldav.Todo`, say) into this library's item, bound to -that collection, without I/O; `wire` stays `None` for a `caldav` object, -whose `wire_data` is its normalised copy rather than the server's bytes +that collection, without I/O; for a `caldav` object `wire` is its +`raw_data`, `None` if it was not read from the server (see "The stored form" in [§3.1](#31-the-base-a-typed-view-over-icalendar)). Together with `.native` that lets code use both libraries side by side, which is what a migration needs ([§9 Migrating from caldav](#9-migrating-from-caldav)). diff --git a/news/+caldav-backend.feature.rst b/news/+caldav-backend.feature.rst index 806f1b7..5ea8ad5 100644 --- a/news/+caldav-backend.feature.rst +++ b/news/+caldav-backend.feature.rst @@ -1 +1 @@ -A CalDAV backend, over the caldav library: ``pip install 'calendaring[caldav]'``, then ``Backend.open("https://...", username=..., password=...)`` or a ``kind: caldav`` section in the configuration file. It declares its capabilities from caldav's knowledge of the server, uses the server's sync-token for ``changes()`` where it is reliable, and passes the conformance suite against Xandikos and Radicale. +A CalDAV backend, over the caldav library: ``pip install 'calendaring[caldav]'``, then ``Backend.open("https://...", username=..., password=...)`` or a ``kind: caldav`` section in the configuration file. It declares its capabilities from caldav's knowledge of the server, uses the server's sync-token for ``changes()`` where it is reliable, keeps the bytes the server sent as ``item.wire`` and sends ``wire=`` bytes unchanged, and passes the conformance suite against Xandikos and Radicale. diff --git a/pyproject.toml b/pyproject.toml index 7f7cba9..1bfb07e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ ] [project.optional-dependencies] -# 3.6: save(no_overwrite=True) sends If-None-Match: * (caldav branch feat/if-none-match-create). +# 3.6: save(no_overwrite=True) sends If-None-Match: *, and save(raw_data=...) sends given bytes. caldav = ["caldav>=3.6"] # Embedded CalDAV servers the caldav backend's conformance tests run against. test-servers = ["caldav>=3.6", "xandikos>=0.2", "radicale>=3.2"] diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index 2f1fb1d..bd638d7 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -67,9 +67,10 @@ def _href(self, native_id: str) -> str: return self.url + quote(native_id, safe="") def _stored(self, obj: Any) -> StoredObject: + # raw_data is what the server sent; data is what caldav made of it, which is what gets parsed. data = obj.data - wire = data.encode() if isinstance(data, str) else bytes(data) - return StoredObject(_name_of(obj.url), obj.etag, _parse(wire), wire, obj) + parsed = _parse(data.encode() if isinstance(data, str) else bytes(data)) + return StoredObject(_name_of(obj.url), obj.etag, parsed, obj.raw_data, obj) async def _query(self, searcher: Searcher | None) -> list[StoredObject]: """Each kind the search can match, narrowed on the server only where that cannot under-match (A3).""" @@ -156,12 +157,14 @@ async def _find_uid(self, uid: str) -> StoredObject: return self._stored(await self._backend._dav.by_uid(self.calendar, uid)) async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: - data = calendar.to_ical() # wire= is refused before this: write.wire is UNSUPPORTED + data = wire if wire is not None else calendar.to_ical() first = _items_of(calendar)[0] uid = str(first.get("UID", "")) name = native_id or (f"{uid}.ics" if SAFE_NAME.match(uid) else f"{uuid.uuid4()}.ics") try: - etag = await self._backend._dav.create(self.calendar, self._href(name), data, str(first.name)) + etag = await self._backend._dav.create( + self.calendar, self._href(name), data, str(first.name), raw=wire is not None + ) except AlreadyExistsError as exc: if native_id is None: exc.taken = "uid" # the name was derived from the UID @@ -184,9 +187,11 @@ async def _update( self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None ) -> StoredObject: etag = await self._unless_changed(native_id, etag) - data = calendar.to_ical() + data = wire if wire is not None else calendar.to_ical() kind = str(_items_of(calendar)[0].name) - new_etag = await self._backend._dav.update(self.calendar, self._href(native_id), data, kind, etag) + new_etag = await self._backend._dav.update( + self.calendar, self._href(native_id), data, kind, etag, raw=wire is not None + ) if new_etag is None: return await self._fetch(native_id) return StoredObject(native_id, new_etag, _parse(data)) diff --git a/src/calendaring/_async/dav.py b/src/calendaring/_async/dav.py index 66ab915..a7e6794 100644 --- a/src/calendaring/_async/dav.py +++ b/src/calendaring/_async/dav.py @@ -15,6 +15,7 @@ SyncResult, _awaited, _etags, + _raw, _resource, _strong, _sync_result, @@ -91,18 +92,27 @@ async def sync(self, calendar: Calendar, token: str | None) -> SyncResult: found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) return _sync_result(await _awaited(found)) - async def create(self, calendar: Calendar, url: str, data: bytes, kind: str) -> str | None: - """Store a new object at ``url``: ``AlreadyExistsError`` if the name or the UID is taken. Its ETag, if strong.""" + async def create(self, calendar: Calendar, url: str, data: bytes, kind: str, *, raw: bool = False) -> str | None: + """Store a new object at ``url``: ``AlreadyExistsError`` if the name or the UID is taken. Its ETag, if strong. + + With ``raw``, ``data`` is sent byte for byte, not re-serialised by caldav. + """ obj = _resource(self.client, calendar, url, data, kind) with translated(), creating(): - await _awaited(obj.save(no_overwrite=True, increase_seqno=False, only_this_recurrence=False)) + await _awaited( + obj.save(no_overwrite=True, increase_seqno=False, only_this_recurrence=False, **_raw(data, raw)) + ) return _strong(obj.etag) - async def update(self, calendar: Calendar, url: str, data: bytes, kind: str, etag: str | None) -> str | None: + async def update( + self, calendar: Calendar, url: str, data: bytes, kind: str, etag: str | None, *, raw: bool = False + ) -> str | None: """Replace the object at ``url``, if it still has ``etag`` (any version without one). Its new ETag, if strong.""" obj = _resource(self.client, calendar, url, data, kind, etag) with translated(): - await _awaited(obj.save(increase_seqno=False, only_this_recurrence=False, precondition="etag")) + await _awaited( + obj.save(increase_seqno=False, only_this_recurrence=False, precondition="etag", **_raw(data, raw)) + ) return _strong(obj.etag) async def delete(self, calendar: Calendar, url: str, etag: str | None) -> None: diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index 75d1486..bed8958 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -58,7 +58,6 @@ Feature.SEARCH_TEXT: E, Feature.CHANGES: {"level": E, "method": "etag-listing"}, Feature.MOVE: {"level": E, "note": "added to the target, then deleted here"}, - Feature.WRITE_WIRE: {"level": U, "note": "caldav re-serialises what it saves (caldav #763)"}, Feature.IDENTITY_FOREIGN_ID: {"level": U, "note": "the UID is native"}, Feature.RECURRENCE_EDIT_OCCURRENCE: {"level": U, "note": _PENDING}, Feature.RECURRENCE_EDIT_THIS_AND_FUTURE: U, diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py index 0106401..b89217c 100644 --- a/src/calendaring/_caldav_io.py +++ b/src/calendaring/_caldav_io.py @@ -12,6 +12,8 @@ Writes go through caldav's own ``save()`` and ``delete()``: ``save(no_overwrite=True)`` sends ``If-None-Match: *`` where the server honours it (caldav 3.6), and an update carries the ETag it was read with. ``SEQUENCE`` is left as given. +``wire=`` bytes go as ``save(raw_data=...)``, which caldav sends unchanged; +``wire`` on a read is caldav's ``raw_data``, what the server sent. """ from __future__ import annotations @@ -131,6 +133,11 @@ def _resource( return cls(client=client, url=url, data=None if data is None else data.decode(), parent=calendar, props=props) +def _raw(data: bytes, raw: bool) -> dict[str, Any]: + """``save()``'s argument for sending ``data`` as it is (caldav 3.6, caldav #763); left out otherwise.""" + return {"raw_data": data} if raw else {} + + def _strong(etag: str | None) -> str | None: """The ETag a write left cached, if it is strong (RFC 4791 §5.3.4: a changed object gets none).""" return None if etag is None or etag.startswith("W/") else etag diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index 588d73a..6ed24c2 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -67,9 +67,10 @@ def _href(self, native_id: str) -> str: return self.url + quote(native_id, safe="") def _stored(self, obj: Any) -> StoredObject: + # raw_data is what the server sent; data is what caldav made of it, which is what gets parsed. data = obj.data - wire = data.encode() if isinstance(data, str) else bytes(data) - return StoredObject(_name_of(obj.url), obj.etag, _parse(wire), wire, obj) + parsed = _parse(data.encode() if isinstance(data, str) else bytes(data)) + return StoredObject(_name_of(obj.url), obj.etag, parsed, obj.raw_data, obj) def _query(self, searcher: Searcher | None) -> list[StoredObject]: """Each kind the search can match, narrowed on the server only where that cannot under-match (A3).""" @@ -156,12 +157,14 @@ def _find_uid(self, uid: str) -> StoredObject: return self._stored(self._backend._dav.by_uid(self.calendar, uid)) def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: - data = calendar.to_ical() # wire= is refused before this: write.wire is UNSUPPORTED + data = wire if wire is not None else calendar.to_ical() first = _items_of(calendar)[0] uid = str(first.get("UID", "")) name = native_id or (f"{uid}.ics" if SAFE_NAME.match(uid) else f"{uuid.uuid4()}.ics") try: - etag = self._backend._dav.create(self.calendar, self._href(name), data, str(first.name)) + etag = self._backend._dav.create( + self.calendar, self._href(name), data, str(first.name), raw=wire is not None + ) except AlreadyExistsError as exc: if native_id is None: exc.taken = "uid" # the name was derived from the UID @@ -184,9 +187,11 @@ def _update( self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None ) -> StoredObject: etag = self._unless_changed(native_id, etag) - data = calendar.to_ical() + data = wire if wire is not None else calendar.to_ical() kind = str(_items_of(calendar)[0].name) - new_etag = self._backend._dav.update(self.calendar, self._href(native_id), data, kind, etag) + new_etag = self._backend._dav.update( + self.calendar, self._href(native_id), data, kind, etag, raw=wire is not None + ) if new_etag is None: return self._fetch(native_id) return StoredObject(native_id, new_etag, _parse(data)) diff --git a/src/calendaring/_sync/dav.py b/src/calendaring/_sync/dav.py index 767aa9c..a83d29e 100644 --- a/src/calendaring/_sync/dav.py +++ b/src/calendaring/_sync/dav.py @@ -15,6 +15,7 @@ SyncResult, _returned, _etags, + _raw, _resource, _strong, _sync_result, @@ -91,18 +92,27 @@ def sync(self, calendar: Calendar, token: str | None) -> SyncResult: found = calendar.get_objects_by_sync_token(token, load_objects=False, disable_fallback=True) return _sync_result(_returned(found)) - def create(self, calendar: Calendar, url: str, data: bytes, kind: str) -> str | None: - """Store a new object at ``url``: ``AlreadyExistsError`` if the name or the UID is taken. Its ETag, if strong.""" + def create(self, calendar: Calendar, url: str, data: bytes, kind: str, *, raw: bool = False) -> str | None: + """Store a new object at ``url``: ``AlreadyExistsError`` if the name or the UID is taken. Its ETag, if strong. + + With ``raw``, ``data`` is sent byte for byte, not re-serialised by caldav. + """ obj = _resource(self.client, calendar, url, data, kind) with translated(), creating(): - _returned(obj.save(no_overwrite=True, increase_seqno=False, only_this_recurrence=False)) + _returned( + obj.save(no_overwrite=True, increase_seqno=False, only_this_recurrence=False, **_raw(data, raw)) + ) return _strong(obj.etag) - def update(self, calendar: Calendar, url: str, data: bytes, kind: str, etag: str | None) -> str | None: + def update( + self, calendar: Calendar, url: str, data: bytes, kind: str, etag: str | None, *, raw: bool = False + ) -> str | None: """Replace the object at ``url``, if it still has ``etag`` (any version without one). Its new ETag, if strong.""" obj = _resource(self.client, calendar, url, data, kind, etag) with translated(): - _returned(obj.save(increase_seqno=False, only_this_recurrence=False, precondition="etag")) + _returned( + obj.save(increase_seqno=False, only_this_recurrence=False, precondition="etag", **_raw(data, raw)) + ) return _strong(obj.etag) def delete(self, calendar: Calendar, url: str, etag: str | None) -> None: diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py index a46d604..395a05f 100644 --- a/tests/_async/test_caldav_backend.py +++ b/tests/_async/test_caldav_backend.py @@ -18,6 +18,7 @@ ConflictError, NotFoundError, SyncTokenError, + Task, ) pytest.importorskip("caldav") @@ -77,6 +78,28 @@ async def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None await AsyncBackend.open("https://dav.example.com/", kind="caldav", no_such_option=1) +async def test_wire_bytes_are_sent_as_given(collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch) -> None: + # icalendar would fold this line. Xandikos refolds what it stores, so what is checked is the PUT. + client = collection._backend._dav.client # type: ignore[attr-defined] + put = client.put + sent: list[bytes] = [] + + async def spy(url: str, body: bytes, headers: Any = None) -> Any: + sent.append(bytes(body)) + return await put(url, body, headers) + + monkeypatch.setattr(client, "put", spy) + data = Task.new(summary="placeholder") + payload = data.to_ical().replace(b"SUMMARY:placeholder", b"SUMMARY:" + b"x" * 100) + written = await collection.add(data, wire=payload) + written = await collection.save(written, wire=payload.replace(b"SUMMARY:", b"SUMMARY;X-SAVED=1:")) + assert sent == [payload, payload.replace(b"SUMMARY:", b"SUMMARY;X-SAVED=1:")] + assert written.native_id is not None + reread = await collection.get_by_native_id(written.native_id) + assert reread.wire is not None + assert Task.from_ical(reread.wire) == reread + + def weakened(collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch) -> None: """Make every ETag read from the server weak, as GMX hands them out.""" stored = collection._stored # type: ignore[attr-defined] diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py index 5e16c3c..96b5658 100644 --- a/tests/_sync/test_caldav_backend.py +++ b/tests/_sync/test_caldav_backend.py @@ -18,6 +18,7 @@ ConflictError, NotFoundError, SyncTokenError, + Task, ) pytest.importorskip("caldav") @@ -77,6 +78,28 @@ def test_an_option_caldav_does_not_take_is_a_configuration_error() -> None: Backend.open("https://dav.example.com/", kind="caldav", no_such_option=1) +def test_wire_bytes_are_sent_as_given(collection: Collection, monkeypatch: pytest.MonkeyPatch) -> None: + # icalendar would fold this line. Xandikos refolds what it stores, so what is checked is the PUT. + client = collection._backend._dav.client # type: ignore[attr-defined] + put = client.put + sent: list[bytes] = [] + + def spy(url: str, body: bytes, headers: Any = None) -> Any: + sent.append(bytes(body)) + return put(url, body, headers) + + monkeypatch.setattr(client, "put", spy) + data = Task.new(summary="placeholder") + payload = data.to_ical().replace(b"SUMMARY:placeholder", b"SUMMARY:" + b"x" * 100) + written = collection.add(data, wire=payload) + written = collection.save(written, wire=payload.replace(b"SUMMARY:", b"SUMMARY;X-SAVED=1:")) + assert sent == [payload, payload.replace(b"SUMMARY:", b"SUMMARY;X-SAVED=1:")] + assert written.native_id is not None + reread = collection.get_by_native_id(written.native_id) + assert reread.wire is not None + assert Task.from_ical(reread.wire) == reread + + def weakened(collection: Collection, monkeypatch: pytest.MonkeyPatch) -> None: """Make every ETag read from the server weak, as GMX hands them out.""" stored = collection._stored # type: ignore[attr-defined] From 7f12bf085e28ec3831a43205860a946c7415bf15 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Tue, 20 Oct 2026 03:19:40 +0200 Subject: [PATCH 15/17] docs: plann review is about its impact (1.6) Roadmap 1.6 asked for a review from plann's author before the aggregation design freezes. What matters is how the design affects plann, whoever reviews it, so the item says that. Prompt: fix those two too [two leftover items: the test_ prefix in calendaring-config, already done, and this stashed ROADMAP 1.6 edit] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- docs/design/ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design/ROADMAP.md b/docs/design/ROADMAP.md index c9d6dc5..4919702 100644 --- a/docs/design/ROADMAP.md +++ b/docs/design/ROADMAP.md @@ -471,8 +471,8 @@ therefore must not be trimmed, and it must land before `plann` reaches its Phase - [ ] Aggregation primitives sufficient for `plann`'s `sum_hours` — grouping a selection into budget lines with per-line subtotals, keeping committed and spent time separate -- [ ] Coordinate the design with `plann` before freezing it; a review from its - author is cheap now and expensive after 1.0 +- [ ] Coordinate the design with `plann` before freezing it; a review of how + the design affects `plann` is cheap now and expensive after 1.0 --- From 310379fa110236e396a6fe2636c2039054a2c35d Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 23 Oct 2026 00:31:26 +0200 Subject: [PATCH 16/17] refactor: use caldav 3.6 error attributes (2.1) caldav 3.6.0 gives every DAVError a status and preconditions, and a RateLimitError its retry_after_seconds, so _translate reads them directly instead of probing with getattr. A UID that caldav's own search finds now raises UidConflictConsistencyError, which creating() already reports as taken="uid". Tests pin both: 401 vs 403 (https://github.com/python-caldav/caldav/issues/805) and the UID clash (https://github.com/python-caldav/caldav/issues/806). Prompt: IIRC, we had some PRs here that was waiting for the caldav 3.6.0-release? It has been released now Followup-Prompt: followup PR13 [pick up the caldav 3.6 fixes for issues 805 and 806 in PR 13's error translation] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/_caldav_io.py | 9 ++++----- tests/test_caldav_io.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py index b89217c..b530e25 100644 --- a/src/calendaring/_caldav_io.py +++ b/src/calendaring/_caldav_io.py @@ -101,17 +101,16 @@ def translated() -> Iterator[None]: def _translate(exc: dav_error.DAVError) -> CalendaringError: message = str(exc) - if VALID_SYNC_TOKEN in getattr(exc, "preconditions", ()): + if VALID_SYNC_TOKEN in exc.preconditions: return SyncTokenError(f"the server no longer honours the sync token: {message}") if isinstance(exc, dav_error.NotFoundError): return NotFoundError(message) if isinstance(exc, (dav_error.ETagMismatchError, dav_error.ScheduleTagMismatchError)): return ConflictError(message) if isinstance(exc, dav_error.AuthorizationError): - status = getattr(exc, "status", None) or getattr(getattr(exc, "response", None), "status", None) - return (AuthenticationError if status == 401 else AuthorizationError)(message) + return (AuthenticationError if exc.status == 401 else AuthorizationError)(message) if isinstance(exc, dav_error.RateLimitError): - return RateLimitError(message, retry_after=getattr(exc, "retry_after_seconds", None)) + return RateLimitError(message, retry_after=exc.retry_after_seconds) return BackendError(message) @@ -150,7 +149,7 @@ def creating() -> Iterator[None]: yield except dav_error.UidConflictError as exc: # another object holds the UID (RFC 4791 §5.3.2.1) raise AlreadyExistsError(str(exc), taken="uid") from exc - except dav_error.ConsistencyError as exc: # If-None-Match failed, or caldav's search found it + except dav_error.ConsistencyError as exc: # If-None-Match failed: something is at that URL raise AlreadyExistsError(str(exc), taken="native_id") from exc diff --git a/tests/test_caldav_io.py b/tests/test_caldav_io.py index 160acd3..4ac131b 100644 --- a/tests/test_caldav_io.py +++ b/tests/test_caldav_io.py @@ -46,6 +46,16 @@ def test_caldav_errors_translate() -> None: assert type(_translate(dav_error.DAVError(url="x"))) is BackendError +def test_a_401_is_authentication_and_a_403_authorization() -> None: + from caldav.lib import error as dav_error + + from calendaring._caldav_io import _translate + from calendaring.errors import AuthenticationError, AuthorizationError + + assert type(_translate(dav_error.AuthorizationError(url="x", status=401))) is AuthenticationError + assert type(_translate(dav_error.AuthorizationError(url="x", status=403))) is AuthorizationError + + def test_a_refused_create_says_what_was_taken() -> None: from caldav.lib import error as dav_error @@ -55,6 +65,9 @@ def test_a_refused_create_says_what_was_taken() -> None: with pytest.raises(AlreadyExistsError) as uid, creating(): raise dav_error.UidConflictError(url="x") assert uid.value.taken == "uid" + with pytest.raises(AlreadyExistsError) as found, creating(): # caldav's own UID search found it + raise dav_error.UidConflictConsistencyError(url="x") + assert found.value.taken == "uid" with pytest.raises(AlreadyExistsError) as name, creating(): raise dav_error.ConsistencyError("If-None-Match failed") assert name.value.taken == "native_id" From 13528598e292035206bf68b02d4e284abf8bbce0 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 23 Oct 2026 01:02:09 +0200 Subject: [PATCH 17/17] fix: CalDAV backend on the BrokenObject primitives Rebased onto https://github.com/pycalendar/calendaring/pull/24, which changed what the listing primitives return and added three features. An object that does not parse now comes back from _query, _fetch_many and the sync report as a BrokenObject carrying the server's bytes. caldav still parses every object when it filters the results itself, and fails on the first that does not parse (https://github.com/python-caldav/caldav/issues/815). It filters every search with a time range, every search on a server without case-sensitive text search (Radicale, CCS, Zimbra, SOGo, GMX, OX), and every UID lookup, so there one bad object still fails the call. The tests for that are marked as expected failures until caldav is fixed. timezones follows caldav's save-load.event.timezone hint; alarms and dates are FULL on a server caldav has hints for and UNKNOWN on another, like everything else the server has to store. Prompt: [merge the PRs] Followup-Prompt: "calendaring has worked around this for now by passing `post_filter=False` and filtering on its own side" - don't do that. just accept that there is a bug until it has been fixed upstream. [quoting the draft of https://github.com/python-caldav/caldav/issues/815] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- src/calendaring/_async/caldav.py | 32 ++++++--- src/calendaring/_caldav_features.py | 3 + src/calendaring/_sync/caldav.py | 32 ++++++--- tests/_async/test_caldav_backend.py | 105 ++++++++++++++++++++++++++++ tests/_sync/test_caldav_backend.py | 105 ++++++++++++++++++++++++++++ tests/test_caldav_features.py | 15 +++- 6 files changed, 273 insertions(+), 19 deletions(-) diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index bd638d7..091c2c0 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -14,7 +14,7 @@ import json import re import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, ClassVar from urllib.parse import quote, unquote @@ -24,10 +24,17 @@ from .._caldav_features import backend_capabilities, collection_capabilities from .._caldav_io import Calendar from ..capabilities import Feature -from ..errors import AlreadyExistsError, ConfigurationError, ConflictError, NotFoundError, SyncTokenError +from ..errors import ( + AlreadyExistsError, + ConfigurationError, + ConflictError, + InvalidDataError, + NotFoundError, + SyncTokenError, +) from ..items import _items_of, _parse from ..loss import LossPolicy -from ..types import Component, StoredObject, SyncToken +from ..types import BrokenObject, Component, StoredObject, SyncToken from .backend import AsyncBackend from .collection import AsyncCollection from .dav import AsyncDAV @@ -72,9 +79,16 @@ def _stored(self, obj: Any) -> StoredObject: parsed = _parse(data.encode() if isinstance(data, str) else bytes(data)) return StoredObject(_name_of(obj.url), obj.etag, parsed, obj.raw_data, obj) - async def _query(self, searcher: Searcher | None) -> list[StoredObject]: + def _listed(self, obj: Any) -> StoredObject | BrokenObject: + """As ``_stored``, but an object that does not parse is a ``BrokenObject`` with the server's bytes.""" + try: + return self._stored(obj) + except InvalidDataError as exc: + return BrokenObject(_name_of(obj.url), obj.etag, obj.raw_data, exc) + + async def _query(self, searcher: Searcher | None) -> list[StoredObject | BrokenObject]: """Each kind the search can match, narrowed on the server only where that cannot under-match (A3).""" - found: dict[str, StoredObject] = {} + found: dict[str, StoredObject | BrokenObject] = {} for kind in sorted(self._kinds(searcher), key=lambda c: c.value): flags: dict[str, Any] = {SEARCH_FLAG[kind]: True} if kind is Component.TASK: @@ -88,7 +102,7 @@ async def _query(self, searcher: Searcher | None) -> list[StoredObject]: ): flags.update({k: v for k, v in (("start", searcher.start), ("end", searcher.end)) if v is not None}) for obj in await self._backend._dav.search(self.calendar, **flags): - stored = self._stored(obj) + stored = self._listed(obj) found.setdefault(stored.native_id, stored) return list(found.values()) @@ -102,18 +116,18 @@ async def _fetch(self, native_id: str) -> StoredObject: return self._stored(obj) raise NotFoundError(f"no object {native_id!r}") - async def _fetch_many(self, native_ids: Any) -> list[StoredObject]: + async def _fetch_many(self, native_ids: Any) -> list[StoredObject | BrokenObject]: hrefs = [self._href(n) for n in native_ids] if not hrefs: return [] - return [self._stored(o) for o in await self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] + return [self._listed(o) for o in await self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] async def _list_etags(self) -> dict[str, str]: return {_name_of(href): etag for href, etag in (await self._backend._dav.etags(self.url)).items()} async def _changes( self, token: SyncToken | None, known: Mapping[str, str] | None - ) -> tuple[list[StoredObject], list[str], SyncToken]: + ) -> tuple[Sequence[StoredObject | BrokenObject], list[str], SyncToken]: """The server's sync-collection report (RFC 6578) where it has one; else the inherited etag diff. A token from the etag diff (``changes(known=...)``, or a server that diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index bed8958..3d1610d 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -40,6 +40,7 @@ Feature.RECURRENCE: "save-load.event.recurrences", Feature.TASK_RELATIONS_PARENT: "save-load.icalendar.related-to", Feature.TASK_RELATIONS_DEPENDS_ON: "save-load.icalendar.related-to", + Feature.TIMEZONES: "save-load.event.timezone", } COMPONENT_FROM_SERVER = { @@ -94,6 +95,8 @@ def _level(features: Any, name: str) -> Support: Feature.TASK_COMPLETED, Feature.TASK_PLANNED, Feature.TASK_ESTIMATE, + Feature.ALARMS, + Feature.DATES, } ) diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index 6ed24c2..b070816 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -14,7 +14,7 @@ import json import re import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, ClassVar from urllib.parse import quote, unquote @@ -24,10 +24,17 @@ from .._caldav_features import backend_capabilities, collection_capabilities from .._caldav_io import Calendar from ..capabilities import Feature -from ..errors import AlreadyExistsError, ConfigurationError, ConflictError, NotFoundError, SyncTokenError +from ..errors import ( + AlreadyExistsError, + ConfigurationError, + ConflictError, + InvalidDataError, + NotFoundError, + SyncTokenError, +) from ..items import _items_of, _parse from ..loss import LossPolicy -from ..types import Component, StoredObject, SyncToken +from ..types import BrokenObject, Component, StoredObject, SyncToken from .backend import Backend from .collection import Collection from .dav import DAV @@ -72,9 +79,16 @@ def _stored(self, obj: Any) -> StoredObject: parsed = _parse(data.encode() if isinstance(data, str) else bytes(data)) return StoredObject(_name_of(obj.url), obj.etag, parsed, obj.raw_data, obj) - def _query(self, searcher: Searcher | None) -> list[StoredObject]: + def _listed(self, obj: Any) -> StoredObject | BrokenObject: + """As ``_stored``, but an object that does not parse is a ``BrokenObject`` with the server's bytes.""" + try: + return self._stored(obj) + except InvalidDataError as exc: + return BrokenObject(_name_of(obj.url), obj.etag, obj.raw_data, exc) + + def _query(self, searcher: Searcher | None) -> list[StoredObject | BrokenObject]: """Each kind the search can match, narrowed on the server only where that cannot under-match (A3).""" - found: dict[str, StoredObject] = {} + found: dict[str, StoredObject | BrokenObject] = {} for kind in sorted(self._kinds(searcher), key=lambda c: c.value): flags: dict[str, Any] = {SEARCH_FLAG[kind]: True} if kind is Component.TASK: @@ -88,7 +102,7 @@ def _query(self, searcher: Searcher | None) -> list[StoredObject]: ): flags.update({k: v for k, v in (("start", searcher.start), ("end", searcher.end)) if v is not None}) for obj in self._backend._dav.search(self.calendar, **flags): - stored = self._stored(obj) + stored = self._listed(obj) found.setdefault(stored.native_id, stored) return list(found.values()) @@ -102,18 +116,18 @@ def _fetch(self, native_id: str) -> StoredObject: return self._stored(obj) raise NotFoundError(f"no object {native_id!r}") - def _fetch_many(self, native_ids: Any) -> list[StoredObject]: + def _fetch_many(self, native_ids: Any) -> list[StoredObject | BrokenObject]: hrefs = [self._href(n) for n in native_ids] if not hrefs: return [] - return [self._stored(o) for o in self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] + return [self._listed(o) for o in self._backend._dav.multiget(self.calendar, hrefs) if o.data is not None] def _list_etags(self) -> dict[str, str]: return {_name_of(href): etag for href, etag in (self._backend._dav.etags(self.url)).items()} def _changes( self, token: SyncToken | None, known: Mapping[str, str] | None - ) -> tuple[list[StoredObject], list[str], SyncToken]: + ) -> tuple[Sequence[StoredObject | BrokenObject], list[str], SyncToken]: """The server's sync-collection report (RFC 6578) where it has one; else the inherited etag diff. A token from the etag diff (``changes(known=...)``, or a server that diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py index 395a05f..b582472 100644 --- a/tests/_async/test_caldav_backend.py +++ b/tests/_async/test_caldav_backend.py @@ -13,10 +13,12 @@ from calendaring import ( AsyncBackend, AsyncCollection, + CalendaringError, Component, ConfigurationError, ConflictError, NotFoundError, + PartialResultError, SyncTokenError, Task, ) @@ -24,6 +26,7 @@ pytest.importorskip("caldav") from calendaring._async.caldav import _name_of # noqa: E402 +from calendaring._caldav_io import _resource # noqa: E402 @pytest.fixture @@ -146,3 +149,105 @@ async def test_a_token_from_elsewhere_is_refused(collection: AsyncCollection) -> with pytest.raises(SyncTokenError): caldav_collection._decode_server_token(token) assert caldav_collection._decode_server_token(caldav_collection._encode_server_token("t1")) == "t1" + + +BAD_TODO = ( # DUE:2026-10-20 is not a valid DATE; some client wrote it + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//some client//EN\r\nBEGIN:VTODO\r\nUID:from-phone\r\n" + b"DTSTAMP:20261001T000000Z\r\nSUMMARY:buy milk\r\nDUE:2026-10-20\r\nEND:VTODO\r\nEND:VCALENDAR\r\n" +) + + +@pytest.fixture +async def radicale_collection(radicale_server: Any) -> AsyncIterator[AsyncCollection]: + """On Radicale, caldav filters every search result itself: it has no case-sensitive text search.""" + backend = await AsyncBackend.open( + radicale_server.url, + kind="caldav", + username=radicale_server.username, + password=radicale_server.password, + features=radicale_server.config["features"], + ) + async with backend: + made = await backend.create_collection("backend-tests", components=(Component.TASK,)) + yield made + await made.delete_collection() + + +def serve_broken(collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch) -> None: + """Serve BAD_TODO as if the server stored it: in every search, below caldav's own filtering, and in sync. + + Servers check what they are given (Xandikos, Radicale), so it cannot be stored for real. + """ + caldav_collection: Any = collection + calendar = caldav_collection.calendar + url = caldav_collection.url + "from-phone.ics" + build = calendar._request_report_build_resultlist + + def broken(calendar: Any) -> Any: + return _resource(calendar.client, calendar, url, BAD_TODO, "VTODO", '"bad"')._keep_raw_data(BAD_TODO.decode()) + + async def with_broken(*args: Any, **kwargs: Any) -> Any: + response, objects = await build(*args, **kwargs) + return response, [*objects, broken(calendar)] + + monkeypatch.setattr(calendar, "_request_report_build_resultlist", with_broken) + dav = caldav_collection._backend._dav + sync = dav.sync + + async def sync_too(*args: Any) -> tuple[list[str], list[str], str | None, bool]: + hrefs, gone, token, incomplete = await sync(*args) + return [*hrefs, url], gone, token, incomplete + + monkeypatch.setattr(dav, "sync", sync_too) + multiget = dav.multiget + + async def multiget_too(calendar: Any, urls: list[str]) -> list[Any]: + # caldav's multiget does not parse what it fetches, so serving it here skips nothing of caldav's. + found = await multiget(calendar, [u for u in urls if u != url]) + return [*found, broken(calendar)] if url in urls else found + + monkeypatch.setattr(dav, "multiget", multiget_too) + + +async def check_partial_results(collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch) -> None: + good = await collection.add_task(summary="good task") + serve_broken(collection, monkeypatch) + with pytest.raises(PartialResultError) as info: + await collection.search() + assert [i.uid for i in info.value.results] == [good.uid] + assert [(b.native_id, b.wire) for b in info.value.broken] == [("from-phone.ics", BAD_TODO)] + with pytest.raises(PartialResultError) as changes: + await collection.changes() + assert [i.uid for i in changes.value.results.changed] == [good.uid] + assert [b.wire for b in changes.value.broken] == [BAD_TODO] + + +async def test_an_object_that_does_not_parse_gives_partial_results( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + await check_partial_results(collection, monkeypatch) + + +CALDAV_815 = pytest.mark.xfail( + raises=ValueError, + strict=True, + reason="caldav's own result filter fails on an object that does not parse: " + "https://github.com/python-caldav/caldav/issues/815", +) + + +@CALDAV_815 +async def test_partial_results_where_caldav_filters_every_search( + radicale_collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + await check_partial_results(radicale_collection, monkeypatch) + + +@CALDAV_815 +async def test_get_of_an_object_that_does_not_parse_says_so( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + """A server returns the object for its own UID; caldav's UID lookup then filters it, and fails.""" + serve_broken(collection, monkeypatch) + with pytest.raises(CalendaringError): + await collection.get("from-phone") diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py index 96b5658..3c4a4f7 100644 --- a/tests/_sync/test_caldav_backend.py +++ b/tests/_sync/test_caldav_backend.py @@ -13,10 +13,12 @@ from calendaring import ( Backend, Collection, + CalendaringError, Component, ConfigurationError, ConflictError, NotFoundError, + PartialResultError, SyncTokenError, Task, ) @@ -24,6 +26,7 @@ pytest.importorskip("caldav") from calendaring._sync.caldav import _name_of # noqa: E402 +from calendaring._caldav_io import _resource # noqa: E402 @pytest.fixture @@ -146,3 +149,105 @@ def test_a_token_from_elsewhere_is_refused(collection: Collection) -> None: with pytest.raises(SyncTokenError): caldav_collection._decode_server_token(token) assert caldav_collection._decode_server_token(caldav_collection._encode_server_token("t1")) == "t1" + + +BAD_TODO = ( # DUE:2026-10-20 is not a valid DATE; some client wrote it + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//some client//EN\r\nBEGIN:VTODO\r\nUID:from-phone\r\n" + b"DTSTAMP:20261001T000000Z\r\nSUMMARY:buy milk\r\nDUE:2026-10-20\r\nEND:VTODO\r\nEND:VCALENDAR\r\n" +) + + +@pytest.fixture +def radicale_collection(radicale_server: Any) -> Iterator[Collection]: + """On Radicale, caldav filters every search result itself: it has no case-sensitive text search.""" + backend = Backend.open( + radicale_server.url, + kind="caldav", + username=radicale_server.username, + password=radicale_server.password, + features=radicale_server.config["features"], + ) + with backend: + made = backend.create_collection("backend-tests", components=(Component.TASK,)) + yield made + made.delete_collection() + + +def serve_broken(collection: Collection, monkeypatch: pytest.MonkeyPatch) -> None: + """Serve BAD_TODO as if the server stored it: in every search, below caldav's own filtering, and in sync. + + Servers check what they are given (Xandikos, Radicale), so it cannot be stored for real. + """ + caldav_collection: Any = collection + calendar = caldav_collection.calendar + url = caldav_collection.url + "from-phone.ics" + build = calendar._request_report_build_resultlist + + def broken(calendar: Any) -> Any: + return _resource(calendar.client, calendar, url, BAD_TODO, "VTODO", '"bad"')._keep_raw_data(BAD_TODO.decode()) + + def with_broken(*args: Any, **kwargs: Any) -> Any: + response, objects = build(*args, **kwargs) + return response, [*objects, broken(calendar)] + + monkeypatch.setattr(calendar, "_request_report_build_resultlist", with_broken) + dav = caldav_collection._backend._dav + sync = dav.sync + + def sync_too(*args: Any) -> tuple[list[str], list[str], str | None, bool]: + hrefs, gone, token, incomplete = sync(*args) + return [*hrefs, url], gone, token, incomplete + + monkeypatch.setattr(dav, "sync", sync_too) + multiget = dav.multiget + + def multiget_too(calendar: Any, urls: list[str]) -> list[Any]: + # caldav's multiget does not parse what it fetches, so serving it here skips nothing of caldav's. + found = multiget(calendar, [u for u in urls if u != url]) + return [*found, broken(calendar)] if url in urls else found + + monkeypatch.setattr(dav, "multiget", multiget_too) + + +def check_partial_results(collection: Collection, monkeypatch: pytest.MonkeyPatch) -> None: + good = collection.add_task(summary="good task") + serve_broken(collection, monkeypatch) + with pytest.raises(PartialResultError) as info: + collection.search() + assert [i.uid for i in info.value.results] == [good.uid] + assert [(b.native_id, b.wire) for b in info.value.broken] == [("from-phone.ics", BAD_TODO)] + with pytest.raises(PartialResultError) as changes: + collection.changes() + assert [i.uid for i in changes.value.results.changed] == [good.uid] + assert [b.wire for b in changes.value.broken] == [BAD_TODO] + + +def test_an_object_that_does_not_parse_gives_partial_results( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + check_partial_results(collection, monkeypatch) + + +CALDAV_815 = pytest.mark.xfail( + raises=ValueError, + strict=True, + reason="caldav's own result filter fails on an object that does not parse: " + "https://github.com/python-caldav/caldav/issues/815", +) + + +@CALDAV_815 +def test_partial_results_where_caldav_filters_every_search( + radicale_collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + check_partial_results(radicale_collection, monkeypatch) + + +@CALDAV_815 +def test_get_of_an_object_that_does_not_parse_says_so( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + """A server returns the object for its own UID; caldav's UID lookup then filters it, and fails.""" + serve_broken(collection, monkeypatch) + with pytest.raises(CalendaringError): + collection.get("from-phone") diff --git a/tests/test_caldav_features.py b/tests/test_caldav_features.py index 337a314..4f9194b 100644 --- a/tests/test_caldav_features.py +++ b/tests/test_caldav_features.py @@ -36,7 +36,15 @@ def test_an_unhinted_server_is_unknown_not_unsupported() -> None: def test_what_the_server_stores_is_unknown_without_hints() -> None: unhinted = collection_capabilities(features(), ALL) known = collection_capabilities(hinted(save_load="full"), ALL) - for feature in (Feature.PROPERTIES_PASSTHROUGH, Feature.CATEGORIES, Feature.WRITE_CONDITIONAL, Feature.TASK_DUE): + for feature in ( + Feature.PROPERTIES_PASSTHROUGH, + Feature.CATEGORIES, + Feature.WRITE_CONDITIONAL, + Feature.TASK_DUE, + Feature.ALARMS, + Feature.DATES, + Feature.TIMEZONES, + ): assert unhinted.level(feature) is Support.UNKNOWN, feature assert known.level(feature) is Support.FULL, feature @@ -65,6 +73,11 @@ def test_levels_map_as_section_5_1_says(level: str, expected: Support) -> None: assert backend_capabilities(features).level(Feature.CREATE_COLLECTION) is expected +def test_time_zones_follow_the_servers_hint() -> None: + server = features({"save-load": {"support": "full"}, "save-load.event.timezone": {"support": "unsupported"}}) + assert collection_capabilities(server, ALL).level(Feature.TIMEZONES) is Support.UNSUPPORTED + + def test_a_calendar_without_a_component_cannot_hold_it() -> None: table = collection_capabilities(hinted(save_load="full"), frozenset({Component.TASK})) assert table.level(Feature.COMPONENT_TASK) is Support.FULL