diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 44c472c..ddc1902 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,3 +66,29 @@ jobs: - name: Run tests run: pytest + + suite-coverage: + name: Branch coverage of the conformance suite + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + + - name: Install dependencies + run: pip install -e ".[dev]" + + # A branch of a capability-aware test that no memory profile reaches + # tests nothing; the profiles in tests/_async/test_memory_conformance.py + # are there to reach them. + - name: Measure + run: | + coverage run --branch --source=calendaring -m pytest -q tests/_async/test_memory_conformance.py + coverage report -m --include='*/_async/conformance.py' --fail-under=95 diff --git a/.gitignore b/.gitignore index 03f9ad9..c12399a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ build/ *.egg-info/ .ruff_cache/ .lycheecache +.coverage diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 6091a17..4d0f53b 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -1049,7 +1049,7 @@ initial set: | `identity.client-uid`, `identity.foreign-id` | [§3.2 Identity](#32-identity) | | `identity.client-native-id` | `add(native_id=...)`: the caller may name the stored object (CalDAV href, vdir file name) | | `properties.passthrough` | unknown properties and components survive a round trip | -| `recurrence`, `recurrence.edit-this-and-future` | stores `RRULE`; [§3.7 Recurrence](#37-recurrence) | +| `recurrence`, `recurrence.edit-this-and-future` | stores a recurrence set: `RRULE`, `RDATE`, `EXDATE` and the overrides (`RECURRENCE-ID`); `UNSUPPORTED` drops all four, overrides included; [§3.7 Recurrence](#37-recurrence) | | `recurrence.edit-occurrence` | `save(occurrence)` merges it into the stored object ([§3.7](#37-recurrence)); `UNSUPPORTED` where the merge is not available yet (Q3) | | `task.complete-recurring` | `complete()` on a recurring task or one occurrence of it ([§3.7](#37-recurrence)) | | `move` | [§2.3 Collection](#23-collection) | @@ -1057,6 +1057,14 @@ initial set: | `task.start`, `task.due`, `task.completed`, `task.planned`, `task.estimate` | | | `task.relations.parent`, `task.relations.depends-on` | | | `categories`, `attendees` | for every component; on a task, attendees are its assignees | +| `alarms` | `VALARM` components survive; `UNSUPPORTED` drops them | +| `timezones` | a time with a `TZID` keeps it (with its `VTIMEZONE`), a floating time stays floating; `LOSSY` stores both in UTC, a floating time read as local time ([§4](#4-search)) | +| `dates` | a `DATE` value stays a date; `LOSSY` stores midnight UTC | + +Text is not a feature: non-ASCII, escaped and multi-line text, and every +value of a property with several, round-trip on every backend. The +conformance suite stores a corpus of real-world objects and lets `verify` +compare what comes back; only what the declaration says may differ. Feature names are identifiers, not attribute paths: `task.due` means "stores a task's `DUE`", which is read as `task.component.DUE`, and `task.estimate` @@ -1165,6 +1173,7 @@ CalendaringError ├── UnsupportedError # carries .feature │ └── LossyWriteError # carries .losses; raised before anything is sent ├── InvalidDataError # the backend rejected the data, or it does not parse; also a ValueError +│ └── PartialResultError # a read left out stored objects it could not use; .results, .broken ├── RateLimitError # carries .retry_after: float | None ├── VerificationError # verify=True: written, but stored differently; carries .differences └── BackendError # anything else from the backend or transport @@ -1197,7 +1206,23 @@ CalendaringWarning reload the item. It is still a `ConflictError`, as `AlreadyExistsError` is, because all three are a failed precondition; a handler that reloads on `ConflictError` should catch the subclasses first. -- Fan-out ([§2.1 Workspace](#21-workspace)) does not raise; `MultiResult.raise_for_errors()` raises an `ExceptionGroup` of these. +- **One bad stored object does not hide the rest.** A read that lists + (`search()`, `iter_search()` once exhausted, the listing shorthands, + `changes()`, `relatives()`, and `get(uid)` where the UID is not found) + leaves out an + object that does not parse, or that the searcher cannot match (two + masters for one UID), and then raises `PartialResultError`. `.results` + is what the call would have returned from the rest (for `changes()` the + `ChangeSet`, whose token is good; for `iter_search()` nothing, since it + yielded them already), and `.broken` lists the objects left + out (`BrokenObject`: native id, etag, error, and the bytes as read where + the backend has them, to repair the object from). A broken object is + never reported as deleted. `get_by_native_id()` of one raises + `InvalidDataError`. Raising rather than warning means no caller misses + it; a caller that can live with the gap takes `.results`. The listing + primitives return a `BrokenObject` in place of a `StoredObject` rather + than raising, so the backend decides per object. +- Fan-out ([§2.1 Workspace](#21-workspace)) does not raise; `MultiResult.raise_for_errors()` raises an `ExceptionGroup` of these. `Workspace.search()` keeps a `PartialResultError`'s `.results` among the items, and the error among the errors. caldav's errors map one to one where they overlap: `NotFoundError` → `NotFoundError`, `ETagMismatchError` and diff --git a/docs/design/ROADMAP.md b/docs/design/ROADMAP.md index b380b22..918ee1e 100644 --- a/docs/design/ROADMAP.md +++ b/docs/design/ROADMAP.md @@ -278,11 +278,15 @@ Notes from doing it: the suite: `FULL`, `TRACKER` (tasks only, mapped status and priority, assigned UIDs), `MINIMAL` (writable, most optional features left out, no collections of its own, so the suite borrows one that already holds an - item) and `FEED` (read-only). Between them the capability-aware tests run - both branches, except the supported branches of - `recurrence.edit-occurrence` and `task.complete-recurring` (below). Every - skip says why: a write test on the read-only profile after the write was - checked to raise, or a recurrence test where nothing recurs. + item) and `FEED` (read-only). Three more in the test module (events + only, tasks without a status, and a feed of tasks) reach the branches + these four left out, which a review found + ([issue 21](https://github.com/pycalendar/calendaring/issues/21)); CI + fails below 95 % branch coverage of the suite. The supported branches of + `recurrence.edit-occurrence`, `recurrence.edit-this-and-future` and + `task.complete-recurring` (below) are still not run. Every skip says why: + a write test on the read-only profile after the write was checked to + raise, or a recurrence test where nothing recurs. - A backend implements a handful of primitives (`_query`, `_fetch`, `_create`, `_update`, `_delete`); capability checks, loss policies, client-side filtering, change emulation, `move` emulation, `relatives` and diff --git a/news/15.feature.rst b/news/15.feature.rst new file mode 100644 index 0000000..7a2aecf --- /dev/null +++ b/news/15.feature.rst @@ -0,0 +1 @@ +Three new capability features, ``alarms``, ``timezones`` and ``dates``, let a backend say honestly that it cannot keep alarms, time zones or all-day dates; what it would lose is then reported before writing, as for any other loss. The conformance suite now checks that real-world objects (time zones, alarms, recurrence exceptions, non-ASCII and multi-line text) come back as they were stored. diff --git a/news/22.feature.rst b/news/22.feature.rst new file mode 100644 index 0000000..a3025e1 --- /dev/null +++ b/news/22.feature.rst @@ -0,0 +1 @@ +One stored item that cannot be read no longer makes a whole collection unreadable. A search, ``get()``, ``relatives()`` or ``changes()`` that has to leave items out raises ``PartialResultError``, which carries what could be read and the raw data of what could not, so it can be repaired. diff --git a/pyproject.toml b/pyproject.toml index 80ca3c2..1681bbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dev = [ "mypy==2.4.0", "pre-commit>=4.0", "towncrier>=25.8", + "coverage>=7.6", ] [project.urls] diff --git a/src/calendaring/__init__.py b/src/calendaring/__init__.py index 87e2621..ed526a8 100644 --- a/src/calendaring/__init__.py +++ b/src/calendaring/__init__.py @@ -39,6 +39,7 @@ LossyWriteError, LossyWriteWarning, NotFoundError, + PartialResultError, RateLimitError, SyncTokenError, TransportError, @@ -47,7 +48,7 @@ ) from .items import Event, Item, Journal, Task, TaskStatus from .loss import Loss, LossPolicy -from .types import ChangeSet, Component, MultiResult, Scope, StoredObject, SyncToken +from .types import BrokenObject, ChangeSet, Component, MultiResult, Scope, StoredObject, SyncToken # Register the reference backend's kind ("memory") in both modes. from . import memory as _memory # noqa: E402, F401 isort: skip @@ -86,8 +87,10 @@ "LossyWriteWarning", "MultiResult", "NotFoundError", + "PartialResultError", "RateLimitError", "Scope", + "BrokenObject", "StoredObject", "Support", "SyncEvent", diff --git a/src/calendaring/_async/collection.py b/src/calendaring/_async/collection.py index 6c0396a..3ce2f8c 100644 --- a/src/calendaring/_async/collection.py +++ b/src/calendaring/_async/collection.py @@ -14,8 +14,8 @@ import copy import json from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Generator, Iterable, Mapping -from contextlib import contextmanager +from collections.abc import AsyncIterator, Generator, Iterable, Mapping, Sequence +from contextlib import contextmanager, suppress from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, Literal, overload @@ -23,12 +23,21 @@ import recurring_ical_events from icalendar_searcher import Searcher -from .._mapping import Change, differences, plan_losses +from .._mapping import Change, differences, plan_calendar_losses, plan_losses from ..capabilities import BACKEND_FEATURES, Capabilities, Feature, Support -from ..errors import CalendaringError, NotFoundError, SyncTokenError, UnsupportedError, VerificationError +from ..errors import ( + CalendaringError, + ConflictError, + InvalidDataError, + NotFoundError, + PartialResultError, + SyncTokenError, + UnsupportedError, + VerificationError, +) from ..items import Event, Item, Journal, Task, _items_of, _parse, split_stored from ..loss import RAISE, LossPolicy -from ..types import ChangeSet, Component, Scope, StoredObject, SyncToken +from ..types import BrokenObject, ChangeSet, Component, Scope, StoredObject, SyncToken from .items import BOUND, AsyncEvent, AsyncItem, AsyncJournal, AsyncTask if TYPE_CHECKING: @@ -80,8 +89,13 @@ def __repr__(self) -> str: # === primitives a backend implements =============================== @abstractmethod - async def _query(self, searcher: Searcher | None) -> list[StoredObject]: - """The stored objects that may match: over-matching is fine, under-matching is a bug (A3).""" + async def _query(self, searcher: Searcher | None) -> Sequence[StoredObject | BrokenObject]: + """The stored objects that may match: over-matching is fine, under-matching is a bug (A3). + + An object that does not parse is returned as a ``BrokenObject``, so + one bad object leaves the rest readable. The same goes for every + primitive that lists: ``_iter_query``, ``_fetch_many``, ``_changes``. + """ @abstractmethod async def _fetch(self, native_id: str) -> StoredObject: @@ -106,35 +120,50 @@ async def _update( async def _delete(self, native_id: str, *, etag: str | None) -> None: """Remove a stored object; ``ConflictError`` if ``etag`` is given and no longer matches.""" - async def _iter_query(self, searcher: Searcher | None) -> AsyncIterator[StoredObject]: + async def _iter_query(self, searcher: Searcher | None) -> AsyncIterator[StoredObject | BrokenObject]: """Override to stream pages; the default yields what ``_query`` returns.""" for stored in await self._query(searcher): yield stored async def _find_uid(self, uid: str) -> StoredObject: - """Override where the backend can look a UID up; the default reads everything.""" + """Override where the backend can look a UID up; the default reads everything. + + ``PartialResultError`` if the UID is not found but some object could not be read. + """ + broken = [] for stored in await self._query(None): - if any(str(c.get("UID")) == uid for c in _items_of(stored.calendar)): + if isinstance(stored, BrokenObject): + broken.append(stored) + elif any(str(c.get("UID")) == uid for c in _items_of(stored.calendar)): return stored + if broken: + raise PartialResultError([], broken, f"no readable item with UID {uid!r}; {len(broken)} unreadable") raise NotFoundError(f"no item with UID {uid!r}") async def _list_etags(self) -> dict[str, str]: """``{native_id: etag}`` for every stored object; override where it can be listed without content.""" - return {s.native_id: s.etag for s in await self._query(None)} + # A broken object listed without an etag is still there: never report it deleted. + return {s.native_id: s.etag or "" for s in await self._query(None)} - async def _fetch_many(self, native_ids: Iterable[str]) -> list[StoredObject]: - """Override with a bulk fetch (CalDAV ``calendar-multiget``); objects gone meanwhile are left out.""" - out = [] + async def _fetch_many(self, native_ids: Iterable[str]) -> Sequence[StoredObject | BrokenObject]: + """Override with a bulk fetch (CalDAV ``calendar-multiget``); objects gone meanwhile are left out. + + The default cannot keep the bytes of an object that does not parse; + a backend that has them overrides this to return them. + """ + out: list[StoredObject | BrokenObject] = [] for native_id in native_ids: try: out.append(await self._fetch(native_id)) except NotFoundError: pass + except InvalidDataError as exc: + out.append(BrokenObject(native_id, None, None, exc)) return out async def _changes( self, token: SyncToken | None, known: Mapping[str, str] | None - ) -> tuple[list[StoredObject], list[str], SyncToken]: + ) -> tuple[Sequence[StoredObject | BrokenObject], list[str], SyncToken]: """Override with a native change feed; the default diffs an etag listing (``EMULATED``).""" if token is not None: base = self._decode_token(token) @@ -143,7 +172,11 @@ async def _changes( current = await self._list_etags() changed = [n for n, etag in current.items() if base.get(n) != etag] deleted = sorted(n for n in base if n not in current) - return await self._fetch_many(changed), deleted, self._encode_token(current) + fetched = await self._fetch_many(changed) + for stored in fetched: + if isinstance(stored, BrokenObject) and stored.etag is None: + stored.etag = current[stored.native_id] or None + return fetched, deleted, self._encode_token(current) async def _move(self, item: Item, target: AsyncCollection) -> StoredObject | None: """Override with the backend's own move; ``None`` means add to the target, then delete here.""" @@ -216,8 +249,18 @@ def _verifying(self, verify: bool | None) -> bool: return bool(setting) return False + def _token_scope(self) -> str: + """What an emulated token is bound to: by default the backend's kind and id, and the collection's id. + + The backend id is the caller's label for it, so two configurations + that give different stores the same id accept each other's tokens, + and renaming a backend invalidates its tokens. A backend that has a + locator (a directory, a URL) overrides this with it. + """ + return f"{self._backend.kind}:{self.backend_id}:{self.id}" + def _encode_token(self, etags: Mapping[str, str]) -> SyncToken: - raw = json.dumps({"collection": self.id, "etags": etags}, sort_keys=True).encode() + raw = json.dumps({"collection": self._token_scope(), "etags": etags}, sort_keys=True).encode() return SyncToken(base64.urlsafe_b64encode(raw).decode()) def _decode_token(self, token: SyncToken) -> dict[str, str]: @@ -227,7 +270,7 @@ def _decode_token(self, token: SyncToken) -> dict[str, str]: issued_by = data["collection"] except (binascii.Error, ValueError, TypeError, KeyError) as exc: raise SyncTokenError(f"not a token this collection issued: {token!r}") from exc - if issued_by != self.id or not isinstance(etags, dict): + if issued_by != self._token_scope() or not isinstance(etags, dict): raise SyncTokenError(f"a token from another collection: {issued_by!r}") return etags @@ -270,6 +313,7 @@ def _prepare( else: calendar = copy.deepcopy(item.icalendar) changes = [change for c in _items_of(calendar) for change in self._plan_losses(c)] + changes += plan_calendar_losses(calendar, self.capabilities) losses = [loss_ for loss_, _ in changes] if losses and wire is not None: raise UnsupportedError(Feature.WRITE_WIRE, "wire= bytes the backend cannot store as they are") @@ -291,12 +335,26 @@ def _searcher(self, searcher: Searcher | None, filters: Mapping[str, Any]) -> Se raise TypeError("give a Searcher or keyword filters, not both") return searcher if searcher is not None else Searcher(**filters) - def _matches(self, searcher: Searcher, stored: StoredObject) -> list[AsyncItem]: - """Run the client-side filter over one stored object (A3); expand into occurrences if asked.""" + def _matches( + self, searcher: Searcher, stored: StoredObject | BrokenObject, broken: list[BrokenObject] + ) -> list[AsyncItem]: + """Run the client-side filter over one stored object (A3); expand into occurrences if asked. + + An object that is broken, or that the searcher cannot match, goes to ``broken``. + """ + if isinstance(stored, BrokenObject): + broken.append(stored) + return [] out: list[AsyncItem] = [] parts = split_stored(stored.calendar) for part in parts: - matched = searcher.check_component(part) + try: + matched = searcher.check_component(part) + except ValueError as exc: # icalendar-searcher on what is no valid recurrence set + error = InvalidDataError(f"the searcher cannot match it: {exc}") + error.__cause__ = exc + broken.append(BrokenObject(stored.native_id, stored.etag, stored.wire, error)) + return [] if not matched: continue components = [c for c in matched if c.name != "VTIMEZONE"] @@ -349,20 +407,31 @@ def _sorted(searcher: Searcher, items: list[AsyncItem]) -> list[AsyncItem]: async def search(self, searcher: Searcher | None = None, **filters: Any) -> list[AsyncItem]: with self._context(): searcher = self._searcher(searcher, filters) - items = [item for stored in await self._query(searcher) for item in self._matches(searcher, stored)] - return self._sorted(searcher, items) + broken: list[BrokenObject] = [] + found = [i for stored in await self._query(searcher) for i in self._matches(searcher, stored, broken)] + items = self._sorted(searcher, found) + if broken: + raise PartialResultError(items, broken) + return items async def iter_search(self, searcher: Searcher | None = None, **filters: Any) -> AsyncIterator[AsyncItem]: - """``search()`` one item at a time; with sort keys, everything is read before the first.""" + """``search()`` one item at a time; with sort keys, everything is read before the first. + + Objects left out raise ``PartialResultError`` at the end, with empty + ``results``: they were yielded. + """ with self._context(): searcher = self._searcher(searcher, filters) if getattr(searcher, "_sort_keys", None): for item in await self.search(searcher): yield item return + broken: list[BrokenObject] = [] async for stored in self._iter_query(searcher): - for item in self._matches(searcher, stored): + for item in self._matches(searcher, stored, broken): yield item + if broken: + raise PartialResultError([], broken) # the caller has the results already async def tasks(self, **filters: Any) -> list[AsyncTask]: if Component.TASK not in self.components: @@ -421,19 +490,27 @@ async def relatives(self, item: Item, reltype: str | None = None) -> list[AsyncI related = item.component.get("RELATED-TO", []) related = related if isinstance(related, list) else [related] out: dict[str, AsyncItem] = {} + broken: list[BrokenObject] = [] for value in related: kind = str(value.params.get("RELTYPE", "PARENT")).upper() if wanted is None or kind == wanted: - found = await self._lookup_relative(str(value)) + found = await self._lookup_relative(str(value), broken) if found is not None: out.setdefault(found.uid, found) back = {"PARENT": "CHILD", "CHILD": "PARENT"} if wanted is None or wanted in back: - for other in await self.search(): + try: + listed = await self.search() + except PartialResultError as exc: + broken.extend(exc.broken) + listed = list(exc.results) + for other in listed: for value in self._related_to(other): kind = str(value.params.get("RELTYPE", "PARENT")).upper() if str(value) == item.uid and kind in back and (wanted is None or back[kind] == wanted): out.setdefault(other.uid, other) + if broken: + raise PartialResultError(list(out.values()), broken) return list(out.values()) @staticmethod @@ -441,18 +518,16 @@ def _related_to(item: Item) -> list[Any]: related = item.component.get("RELATED-TO", []) return related if isinstance(related, list) else [related] - async def _lookup_relative(self, uid: str) -> AsyncItem | None: - try: - return await self.get(uid) - except NotFoundError: - pass - for other in await self._backend.collections(): - if other.id == self.id: - continue + async def _lookup_relative(self, uid: str, broken: list[BrokenObject]) -> AsyncItem | None: + """Here first, then in the backend's other collections; past objects that cannot be read.""" + others = [c for c in await self._backend.collections() if c.id != self.id] + for collection in [self, *others]: try: - return await other.get(uid) + return await collection.get(uid) + except PartialResultError as exc: # not found among what could be read + broken.extend(exc.broken) except NotFoundError: - continue + pass return None # a dangling relation # === writing ======================================================= @@ -701,8 +776,15 @@ async def move( stored = await self._move(item, target) if stored is not None: return target._one(stored, written=True) + if item.etag is not None and (await self._fetch(item.native_id)).etag != item.etag: + raise ConflictError(f"{item.native_id!r} changed since it was read") moved = await target.add(item.as_data(), loss=loss, verify=verify) - await self.delete(item) + try: + await self.delete(item) + except ConflictError: + with suppress(CalendaringError): # best effort: the conflict is what to report + await target.delete(moved, overwrite=True) + raise return moved # === binding, no I/O =============================================== @@ -742,7 +824,12 @@ async def changes( raise ValueError("changes(): known= with a token; an incremental report has its own baseline") self._require(Feature.CHANGES_KNOWN) changed, deleted, new_token = await self._changes(token, known) - return ChangeSet([i for s in changed for i in self._bound(s)], deleted, new_token) + broken = [s for s in changed if isinstance(s, BrokenObject)] + items = [i for s in changed if not isinstance(s, BrokenObject) for i in self._bound(s)] + result = ChangeSet(items, deleted, new_token) + if broken: + raise PartialResultError(result, broken) + return result # === the collection itself ========================================= diff --git a/src/calendaring/_async/conformance.py b/src/calendaring/_async/conformance.py index 43060d9..350b9c8 100644 --- a/src/calendaring/_async/conformance.py +++ b/src/calendaring/_async/conformance.py @@ -21,12 +21,15 @@ async def open_backend(self) -> AsyncBackend: The suite makes its own collection where the backend can. Where it cannot, it uses the first one listed, compares only what the test itself stored, and -deletes what it stored afterwards. +deletes what it stored afterwards, and puts back a name it changed. Where it +can make collections but not delete them, it reuses the ones an earlier run +left, so they hold nothing of the suite's after a run. """ from __future__ import annotations import pickle +import uuid from collections.abc import AsyncIterator, Callable, Sequence from datetime import UTC, datetime, timedelta from typing import Any @@ -35,21 +38,24 @@ async def open_backend(self) -> AsyncBackend: import pytest from icalendar_searcher import Searcher -from .._mapping import property_lines +from .._corpus import CORPUS +from .._mapping import differences, property_lines from ..capabilities import BACKEND_FEATURES, Feature, Support from ..errors import ( AlreadyExistsError, + BackendError, CalendaringError, ConflictError, LossyWriteError, LossyWriteWarning, NotFoundError, + PartialResultError, SyncTokenError, UnsupportedError, ) from ..items import Event, Item, Journal, Task, TaskStatus from ..loss import ALLOW, WARN, Loss -from ..types import Component, Scope, SyncToken +from ..types import Component, Scope, StoredObject, SyncToken from .backend import AsyncBackend from .collection import AsyncCollection from .items import AsyncEvent, AsyncItem, AsyncJournal, AsyncTask @@ -95,9 +101,17 @@ def level(collection: AsyncCollection, feature: Feature) -> Support: return collection.capabilities.level(feature) +async def readable(collection: AsyncCollection) -> tuple[list[AsyncItem], set[str]]: + """What a borrowed collection holds that can be read, and the native ids of what cannot.""" + try: + return await collection.search(), set() + except PartialResultError as exc: + return list(exc.results), {b.native_id for b in exc.broken} + + async def snapshot(collection: AsyncCollection) -> list[tuple[str | None, str | None]]: - """What is stored, as (native_id, etag) pairs: unchanged means no write happened.""" - return sorted((i.native_id, i.etag) for i in await collection.search()) + """What is stored and readable, as (native_id, etag) pairs: unchanged means no write happened.""" + return sorted((i.native_id, i.etag) for i in (await readable(collection))[0]) # One item type per component: the data class, the add_ shorthand, the @@ -142,11 +156,30 @@ def categories() -> Searcher: s.add_property_filter("CATEGORIES", "work") return s + def text(value: str, operator: str = "contains", case_sensitive: bool = True) -> Callable[[], Searcher]: + def make() -> Searcher: + s = Searcher() + s.add_property_filter("SUMMARY", value, operator=operator, case_sensitive=case_sensitive) + return s + + return make + + def no_due() -> Searcher: + s = Searcher(todo=True) + s.add_property_filter("DUE", None, operator="undef") + return s + return { "everything": lambda: Searcher(), + "summary-contains": text("rain"), + "summary-any-case": text("REPORT", case_sensitive=False), + "summary-not-containing": text("report", operator="!contains"), + "tasks-without-due": no_due, + "alarms-in-range": lambda: Searcher(alarm_start=at(3, 9), alarm_end=at(3, 10)), "tasks": lambda: Searcher(todo=True), "open-tasks": lambda: Searcher(todo=True, include_completed=False), "events-in-range": lambda: Searcher(event=True, start=at(2), end=at(4)), + "tasks-in-range": lambda: Searcher(todo=True, start=at(2), end=at(4)), "events-expanded": lambda: Searcher(event=True, start=at(2), end=at(4), expand=True), "journals": lambda: Searcher(journal=True), "categories": categories, @@ -159,11 +192,20 @@ def dataset() -> list[Item]: b = Task.new(summary="B: done", end=at(5), status=TaskStatus.COMPLETED) c = Task.new(summary="C: open") e1 = Event.new(summary="E1: meeting", start=at(3, 10), end=at(3, 11)) + alarm = icalendar.Alarm() + alarm.add("ACTION", "DISPLAY") + alarm.add("DESCRIPTION", "E1 starts soon") + alarm.add("TRIGGER", timedelta(minutes=-30)) + e1.component.add_component(alarm) + t = Event.new(summary="T: Training session", start=at(10, 18), end=at(10, 19)) + s = Event.new(summary="S: overnight", start=at(1, 20), end=at(2, 14)) # began before the range + ts = Task.new(summary="TS: start only", start=at(3)) + tb = Task.new(summary="TB: spans the range", start=at(1), end=at(5)) e2 = Event.new(summary="E2: later", start=at(20, 10), end=at(20, 11)) r = Event.new(summary="R: daily", start=at(1, 9), end=at(1, 10)) r.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) j = Journal.new(summary="J: notes", start=at(2)) - return [a, b, c, e1, e2, r, j] + return [a, b, c, ts, tb, e1, e2, t, s, r, j] class AsyncConformance: @@ -172,14 +214,26 @@ class AsyncConformance: # Non-CalDAV backends are code, not servers, and must not declare UNKNOWN (§5.1). allow_unknown: bool = False + # How many objects the paging test stores: more than a page of the APIs + # backends list through (a JMAP /query limit, an issue tracker's list). + page_size_hint: int = 120 + # --- hooks a backend's test module may override -------------------- async def open_backend(self) -> AsyncBackend: raise NotImplementedError("a conformance test class must implement open_backend()") async def open_collection(self, backend: AsyncBackend, name: str = "conformance") -> AsyncCollection: - """A collection to test against: made if the backend may be able to, else the first one listed.""" + """A collection to test against: made if the backend may be able to, else the first one listed. + + Where collections can be made but not deleted, the one a previous run + left is used again, so a run leaves one collection per name behind. + """ if backend.capabilities.level(Feature.CREATE_COLLECTION) is not Support.UNSUPPORTED: + deletable = backend.capabilities.level(Feature.DELETE_COLLECTION) is not Support.UNSUPPORTED + left = None if deletable else await self.left_behind(backend, name) + if left is not None: + return await self.borrow(left) try: made = await backend.create_collection( name, components=(Component.EVENT, Component.TASK, Component.JOURNAL) @@ -188,40 +242,80 @@ async def open_collection(self, backend: AsyncBackend, name: str = "conformance" if backend.capabilities.level(Feature.CREATE_COLLECTION) is not Support.UNKNOWN: raise else: + if not deletable: + return await self.borrow(made) self._made.append(made) return made existing = await backend.collections() if not existing: pytest.fail("the backend can neither create a collection nor list one") - collection = existing[0] - self._borrowed.append((collection, {i.native_id for i in await collection.search()})) + return await self.borrow(existing[0]) + + async def left_behind(self, backend: AsyncBackend, name: str) -> AsyncCollection | None: + named = [c for c in await backend.collections() if c.name == name] + return named[0] if named else None + + async def borrow(self, collection: AsyncCollection) -> AsyncCollection: + """Use a collection the suite may not delete, and note what it writes there, to delete only that.""" + items, broken = await readable(collection) + before: set[str | None] = {i.native_id for i in items} | set(broken) + written: set[str] = set() # native ids and UIDs of what the suite created + create = collection._create + + async def noting(calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + stored = await create(calendar, native_id=native_id, wire=wire) + written.add(stored.native_id) + written.update(str(c.get("UID")) for c in stored.calendar.subcomponents if "UID" in c) + return stored + + collection._create = noting # type: ignore[method-assign] + self._borrowed.append((collection, before, written)) return collection async def seed(self, collection: AsyncCollection, items: Sequence[Item]) -> list[AsyncItem]: """Store items the way a test needs them. A read-only backend overrides this to seed natively.""" return [await collection.add(item, loss=ALLOW) for item in items] + async def unseed(self, collection: AsyncCollection, item: AsyncItem) -> None: + """Remove a seeded item. A read-only backend overrides this to remove it natively.""" + await collection.delete(item) + # --- fixtures ----------------------------------------------------- @pytest.fixture async def backend(self) -> AsyncIterator[AsyncBackend]: self._made: list[AsyncCollection] = [] - self._borrowed: list[tuple[AsyncCollection, set[str | None]]] = [] + self._borrowed: list[tuple[AsyncCollection, set[str | None], set[str]]] = [] backend = await self.open_backend() async with backend: yield backend await self.tidy_up(backend) async def tidy_up(self, backend: AsyncBackend) -> None: - """Delete the collections the test made, and what it stored in one it borrowed.""" + """Delete the collections the test made, and what it stored in one it borrowed. + + In a borrowed collection, an item is the suite's if it created it, or + if it carries or relates to a UID the suite created (a completed + occurrence the backend stored as a new object): anything else arrived + from another client while the test ran. + """ for made in self._made: - if backend.capabilities.level(Feature.DELETE_COLLECTION) is not Support.UNSUPPORTED: - await made.delete_collection() - for borrowed, before in self._borrowed: - if writable(borrowed): - for item in await borrowed.search(): - if item.native_id not in before and item.native_id is not None: - await borrowed.delete(item.native_id) + await made.delete_collection() + for borrowed, before, written in self._borrowed: + if not writable(borrowed): + continue + items, _ = await readable(borrowed) + for native_id in {i.native_id for i in items if self.wrote(i, before, written)}: + assert native_id is not None + await borrowed.delete(native_id) + + @staticmethod + def wrote(item: Item, before: set[str | None], written: set[str]) -> bool: + if item.native_id in before: + return False + related = item.component.get("RELATED-TO", []) + related = related if isinstance(related, list) else [related] + return item.native_id in written or item.uid in written or any(str(r) in written for r in related) @pytest.fixture async def collection(self, backend: AsyncBackend) -> AsyncCollection: @@ -351,7 +445,7 @@ async def test_adding_a_uid_twice(self, collection: AsyncCollection) -> None: async def test_add_with_native_id(self, collection: AsyncCollection) -> None: await self.add_task(collection, summary="setup") - name = "conformance-native-id.ics" + name = f"conformance-{uuid.uuid4().hex[:12]}.ics" # a crashed run may leave one behind if level(collection, Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.UNSUPPORTED: await self.assert_unsupported( collection, @@ -390,6 +484,88 @@ async def test_reads_are_bound_and_consistent(self, collection: AsyncCollection) assert str(read.component["SUMMARY"]) == "read me" assert reloaded is not stored + async def test_get_finds_the_exact_uid(self, backend: AsyncBackend, collection: AsyncCollection) -> None: + stored = [] + for kind, (data_class, _, _, bound_class, _) in KINDS.items(): + if holds(collection, kind) and writable(collection): + item = data_class.new(summary=f"get {kind.value}") + added = await collection.add(item) + if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: + assert added.uid == item.uid, f"identity.client-uid: the {kind.value} UID was replaced" + stored.append((added, bound_class)) + if not stored: + pytest.skip("nothing can be stored here") + for added, bound_class in stored: + found = await collection.get(added.uid) + assert isinstance(found, bound_class) + assert found.native_id == added.native_id + if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: + kind_class = stored[0][0].as_data().__class__ + suffix = f"{stored[0][0].uid}@conformance.invalid" + longer = await collection.add(kind_class.new(summary="longer", uid=f"11-{suffix}")) + shorter = await collection.add(kind_class.new(summary="shorter", uid=f"1-{suffix}")) + assert (await collection.get(shorter.uid)).native_id == shorter.native_id + assert (await collection.get(longer.uid)).native_id == longer.native_id + with pytest.raises(NotFoundError): + await collection.get(f"-{suffix}") + if backend.capabilities.supports(Feature.CREATE_COLLECTION): + other = await self.open_collection(backend, "conformance-other") + if writable(other) and holds(other, Component.TASK): + elsewhere = await other.add_task(summary="in another collection") + with pytest.raises(NotFoundError): + await collection.get(elsewhere.uid) + + async def test_add_with_native_id_keeps_the_uid(self, collection: AsyncCollection) -> None: + if level(collection, Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.UNSUPPORTED: + return # test_add_with_native_id checked that it raises + if level(collection, Feature.IDENTITY_CLIENT_UID) is Support.UNSUPPORTED: + pytest.skip("the backend assigns UIDs") + if not writable(collection): + return # test_read_only_rejects_every_write checked the writes + kind = next(k for k in KINDS if holds(collection, k)) + item = KINDS[kind][0].new(summary="named and identified") + name = f"conformance-{uuid.uuid4().hex[:12]}.ics" + stored = await collection.add(item, native_id=name) + assert stored.uid == item.uid + assert (await collection.get(item.uid)).native_id == name + + async def test_a_listing_shows_a_save_made_elsewhere(self, collection: AsyncCollection) -> None: + stored = await self.add_task(collection, summary="original", description="kept") + other = await collection.get(stored.uid) + other.component["SUMMARY"] = "saved elsewhere" + await other.save() + listed = [i for i in await collection.search() if i.native_id == stored.native_id] + streamed = [i async for i in collection.iter_search() if i.native_id == stored.native_id] + for [read] in (listed, streamed): + assert read.etag != stored.etag, "a listing gave the old etag after a save" + assert str(read.component["SUMMARY"]) == "saved elsewhere", "a listing gave old content" + assert str(read.component["DESCRIPTION"]) == "kept" + stored.component["SUMMARY"] = "stale" + with pytest.raises(ConflictError): + await stored.save() + + async def test_read_paths_on_seeded_items(self, collection: AsyncCollection) -> None: + """What a read-only backend must pass too: everything here is stored through ``seed()``.""" + kind = next(k for k in KINDS if holds(collection, k)) + new = KINDS[kind][0].new + [first] = await self.seed(collection, [new(summary="seeded first")]) + assert (await collection.get(first.uid)).native_id == first.native_id + assert (await collection.get_by_native_id(nid(first))).uid == first.uid + assert (await collection.reload(first)).etag == first.etag + if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: + return + start = await collection.changes() + [second] = await self.seed(collection, [new(summary="seeded second")]) + since = await collection.changes(start.token) + assert nid(second) in {nid(i) for i in since.changed} + assert nid(first) not in {nid(i) for i in since.changed} + assert since.deleted == [] + await self.unseed(collection, first) + gone = await collection.changes(since.token) + assert gone.deleted == [nid(first)] + with pytest.raises(NotFoundError): + await collection.get(first.uid) + async def test_wire(self, collection: AsyncCollection) -> None: stored = await self.add_task(collection, summary="bytes") read = await collection.get_by_native_id(nid(stored)) @@ -571,6 +747,26 @@ def policy(losses: Sequence[Loss], item: Item) -> None: assert [[loss.property for loss in losses] for losses in seen] == [[name]] assert seen[0][0].sent == outside[0] + @pytest.mark.parametrize("name", list(CORPUS)) + async def test_real_world_objects_round_trip(self, collection: AsyncCollection, name: str) -> None: + """Only what the declaration says may be lost: verify compares the stored object with what was sent.""" + run = f"-{uuid.uuid4().hex[:12]}@conformance.invalid".encode() # a crashed run may leave one behind + item = Item.from_ical(CORPUS[name].replace(b"@conformance.invalid", run)) + kind = Component(item.component.name) + if not holds(collection, kind): + pytest.skip(f"the collection holds no {kind.value}") + if writable(collection): + read = await collection.reload(await collection.add(item, loss=ALLOW, verify=True)) + else: + [seeded] = await self.seed(collection, [item]) + read = await collection.reload(seeded) + assert differences(item.icalendar, read.icalendar) == [] + zones = {tz["TZID"] for tz in read.icalendar.walk("VTIMEZONE")} + for component in read.icalendar.walk(): + for prop in component.property_items(recursive=False): + tzid = getattr(prop[1], "params", {}).get("TZID") + assert tzid is None or tzid in zones, f"{prop[0]} uses TZID {tzid}, which has no VTIMEZONE" + async def test_verify_passes_on_an_honest_write(self, collection: AsyncCollection) -> None: stored = await self.add_task(collection, summary="verified", description="checked after writing") stored.component["SUMMARY"] = "verified again" @@ -628,7 +824,35 @@ async def test_iter_search_yields_what_search_returns(self, collection: AsyncCol await self.seed(collection, wanted) listed = await collection.search() iterated = [i async for i in collection.iter_search()] - assert sorted(key(i) for i in iterated) == sorted(key(i) for i in listed) + + def fields(items: Sequence[AsyncItem]) -> list[tuple[Any, ...]]: + return sorted((key(i), i.native_id, i.etag, i.component.to_ical()) for i in items) + + assert fields(iterated) == fields(listed), "iter_search() must give what search() gives, field by field" + + async def test_listings_are_not_cut_at_a_page(self, collection: AsyncCollection) -> None: + kind = next(k for k in KINDS if holds(collection, k)) + new = KINDS[kind][0].new + seeded = await self.seed(collection, [new(summary=f"paged {n}") for n in range(self.page_size_hint)]) + mine = {nid(i) for i in seeded} + assert {nid(i) for i in stored_by(await collection.search(), seeded)} == mine + assert {nid(i) async for i in collection.iter_search() if i.native_id in mine} == mine + last = seeded[-1] + assert (await collection.get(last.uid)).native_id == last.native_id + if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: + return + full = await collection.changes() + assert mine <= {nid(i) for i in full.changed} + if level(collection, Feature.CHANGES_KNOWN) is not Support.UNSUPPORTED: + known = await collection.changes(known={nid(i): str(i.etag) for i in seeded}) + assert known.deleted == [] + assert not mine & {nid(i) for i in known.changed} + if writable(collection): + last.component["SUMMARY"] = "paged, changed" + await last.save() + since = await collection.changes(full.token) + assert nid(last) in {nid(i) for i in since.changed} + assert since.deleted == [] async def test_search_sorts(self, collection: AsyncCollection) -> None: searcher = Searcher(todo=True) @@ -652,6 +876,12 @@ async def test_listing_shorthands_return_one_type(self, collection: AsyncCollect async def test_complete_and_uncomplete(self, collection: AsyncCollection) -> None: task = await self.add_task(collection, summary="to do") + if level(collection, Feature.TASK_STATUS) is Support.UNSUPPORTED: + before = await snapshot(collection) + with pytest.raises(LossyWriteError): + await task.complete() # a backend without a status cannot store "completed" + assert await snapshot(collection) == before + return await task.complete() assert str(task.component["STATUS"]) == "COMPLETED" read = await collection.reload(task) @@ -675,9 +905,15 @@ async def test_complete_a_recurring_task(self, collection: AsyncCollection) -> N if level(collection, Feature.TASK_COMPLETE_RECURRING) is Support.UNSUPPORTED: await self.assert_unsupported(collection, Feature.TASK_COMPLETE_RECURRING, lambda: task.complete()) return + etag = task.etag await task.complete() read = await collection.reload(task) assert str(read.component.get("STATUS", "")) != "COMPLETED", "completing one occurrence ended the series" + # "safe" mode (§2.3): a completed copy of the occurrence, and the master's DTSTART moved on. + assert read.etag != etag, "complete() on a recurring task wrote nothing" + assert read.component.decoded("DTSTART") > at(1), "the master's DTSTART did not move" + done = [t for t in await collection.tasks() if str(t.component.get("STATUS", "")) == "COMPLETED"] + assert any(str(t.component["SUMMARY"]) == "weekly" for t in done), "no completed copy of the occurrence" # === recurrence =================================================== @@ -732,7 +968,7 @@ async def test_changes_full_listing(self, collection: AsyncCollection) -> None: if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: await self.assert_unsupported(collection, Feature.CHANGES, lambda: collection.changes()) return - kind = next(iter(collection.components)) + kind = next(k for k in KINDS if holds(collection, k)) seeded = await self.seed(collection, [KINDS[kind][0].new(summary=s) for s in ("one", "two")]) changes = await collection.changes() assert {i.native_id for i in changes.changed} >= {i.native_id for i in seeded} @@ -788,6 +1024,20 @@ async def test_a_bad_token_raises(self, collection: AsyncCollection) -> None: with pytest.raises(SyncTokenError): await collection.changes(SyncToken("conformance-not-a-token")) + async def test_a_token_from_another_collection_raises( + self, backend: AsyncBackend, collection: AsyncCollection + ) -> None: + if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: + return # test_a_bad_token_raises checked that changes() raises + if not backend.capabilities.supports(Feature.CREATE_COLLECTION): + pytest.skip("a second collection is needed, and the backend cannot make one") + other = await self.open_collection(backend, "conformance-other") + if level(other, Feature.CHANGES) is Support.UNSUPPORTED: + pytest.skip("the second collection has no change feed") + token = (await other.changes()).token + with pytest.raises(SyncTokenError): + await collection.changes(token) + # === moving ======================================================= async def test_move(self, backend: AsyncBackend, collection: AsyncCollection) -> None: @@ -805,6 +1055,25 @@ async def test_move(self, backend: AsyncBackend, collection: AsyncCollection) -> await collection.get_by_native_id(nid(task)) assert (await target.get_by_native_id(nid(moved))).uid == moved.uid + async def test_changes_follow_a_move(self, backend: AsyncBackend, collection: AsyncCollection) -> None: + if Support.UNSUPPORTED in (level(collection, Feature.CHANGES), level(collection, Feature.MOVE)): + return # the tests of each feature checked that it raises + task = await self.add_task(collection, summary="moving") + if not backend.capabilities.supports(Feature.CREATE_COLLECTION): + pytest.skip("a second collection is needed, and the backend cannot make one") + target = await self.open_collection(backend, "conformance-target") + if level(target, Feature.CHANGES) is Support.UNSUPPORTED: + pytest.skip("the second collection has no change feed") + here, there = await collection.changes(), await target.changes() + elsewhere = await target.add_task(summary="added there") + moved = await collection.move(task, target) + since_here = await collection.changes(here.token) + since_there = await target.changes(there.token) + assert since_here.deleted == [nid(task)], "a moved item is deleted where it was" + assert not {nid(elsewhere), nid(moved)} & {nid(i) for i in since_here.changed} + assert {nid(moved), nid(elsewhere)} <= {nid(i) for i in since_there.changed} + assert since_there.deleted == [] + # === collections ================================================== async def test_set_name(self, backend: AsyncBackend, collection: AsyncCollection) -> None: @@ -813,27 +1082,51 @@ async def test_set_name(self, backend: AsyncBackend, collection: AsyncCollection await collection.set_name("renamed") assert info.value.feature == Feature.COLLECTION_SET_NAME return + name = collection.name await collection.set_name("conformance renamed") - assert collection.name == "conformance renamed" - assert (await backend.collection(collection.id)).name == "conformance renamed" + try: + assert collection.name == "conformance renamed" + assert (await backend.collection(collection.id)).name == "conformance renamed" + finally: + if name is not None: # a borrowed or reused collection is put back as it was + await collection.set_name(name) async def test_create_and_delete_collection(self, backend: AsyncBackend) -> None: before = [c.id for c in await backend.collections()] + if backend.capabilities.level(Feature.CREATE_COLLECTION) is Support.UNKNOWN: + deletable = backend.capabilities.level(Feature.DELETE_COLLECTION) is not Support.UNSUPPORTED + attempted = None if deletable else await self.left_behind(backend, "conformance-new") + if attempted is None: + try: + attempted = await backend.create_collection("conformance-new", components=(Component.TASK,)) + except (UnsupportedError, BackendError): + return # unprobed, and it does not work here: attempted, as UNKNOWN promises + if deletable: + self._made.append(attempted) + assert attempted.components == frozenset({Component.TASK}) + assert attempted.id in [c.id for c in await backend.collections()] + return if not backend.capabilities.supports(Feature.CREATE_COLLECTION): with pytest.raises(UnsupportedError) as info: await backend.create_collection("conformance-new") assert info.value.feature == Feature.CREATE_COLLECTION assert [c.id for c in await backend.collections()] == before return - made = await backend.create_collection("conformance-new", components=(Component.TASK,)) + deletable = backend.capabilities.supports(Feature.DELETE_COLLECTION) + # Without delete-collection, only the first run on a backend makes it: later ones find it left there. + made = None if deletable else await self.left_behind(backend, "conformance-new") + if made is None: + made = await backend.create_collection("conformance-new", components=(Component.TASK,)) assert made.components == frozenset({Component.TASK}) assert made.id in [c.id for c in await backend.collections()] - if not backend.capabilities.supports(Feature.DELETE_COLLECTION): + if not deletable: with pytest.raises(UnsupportedError) as info: await made.delete_collection() assert info.value.feature == Feature.DELETE_COLLECTION return + self._made.append(made) await made.delete_collection() + self._made.remove(made) assert made.id not in [c.id for c in await backend.collections()] # === binding ====================================================== diff --git a/src/calendaring/_async/memory.py b/src/calendaring/_async/memory.py index 46dfb43..49749ea 100644 --- a/src/calendaring/_async/memory.py +++ b/src/calendaring/_async/memory.py @@ -10,7 +10,7 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from typing import Any, ClassVar import icalendar @@ -18,10 +18,10 @@ from .._memory_store import FULL, MemoryCalendar, MemoryObject, MemoryStore, Profile from ..capabilities import Capabilities, Feature, Support -from ..errors import ConflictError +from ..errors import ConflictError, InvalidDataError from ..items import Item from ..loss import LossPolicy -from ..types import Component, StoredObject +from ..types import BrokenObject, Component, StoredObject from .backend import AsyncBackend from .collection import AsyncCollection @@ -32,6 +32,13 @@ def _stored(obj: MemoryObject, *, written: bool = False) -> StoredObject: return StoredObject(obj.native_id, obj.etag, obj.calendar, None if written else obj.data, obj) +def _listed(obj: MemoryObject) -> StoredObject | BrokenObject: + try: + return _stored(obj) + except InvalidDataError as exc: + return BrokenObject(obj.native_id, obj.etag, obj.data, exc) + + class AsyncMemoryCollection(AsyncCollection): _backend: AsyncMemoryBackend @@ -47,8 +54,11 @@ def __init__(self, backend: AsyncMemoryBackend, store: MemoryCalendar) -> None: native=store, ) - async def _query(self, searcher: Searcher | None) -> list[StoredObject]: - return [_stored(o) for o in self.store.objects.values()] + async def _query(self, searcher: Searcher | None) -> Sequence[StoredObject | BrokenObject]: + return [_listed(o) for o in self.store.objects.values()] + + async def _fetch_many(self, native_ids: Iterable[str]) -> list[StoredObject | BrokenObject]: + return [_listed(self.store.objects[n]) for n in native_ids if n in self.store.objects] async def _fetch(self, native_id: str) -> StoredObject: return _stored(self.store.get(native_id)) @@ -56,6 +66,9 @@ async def _fetch(self, native_id: str) -> StoredObject: async def _list_etags(self) -> dict[str, str]: return {o.native_id: o.etag for o in self.store.objects.values()} + def _token_scope(self) -> str: + return f"memory:{self._backend.store.instance}:{self.id}" + async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: data = self._backend.store.drop(wire if wire is not None else calendar.to_ical()) obj = self.store.create(data, native_id=native_id, assign_uid=self._backend.store.assign_uid) diff --git a/src/calendaring/_async/workspace.py b/src/calendaring/_async/workspace.py index 58aa045..5941463 100644 --- a/src/calendaring/_async/workspace.py +++ b/src/calendaring/_async/workspace.py @@ -19,7 +19,7 @@ from icalendar_searcher import Searcher from .._concurrency import agather -from ..errors import AmbiguousError, CalendaringError, ConfigurationError, NotFoundError +from ..errors import AmbiguousError, CalendaringError, ConfigurationError, NotFoundError, PartialResultError from ..loss import LossPolicy from ..types import MultiResult from .backend import AsyncBackend @@ -137,6 +137,8 @@ async def search(self, searcher: Searcher | None = None, **filters: Any) -> Mult out: MultiResult[AsyncItem] = MultiResult() errors: dict[str, CalendaringError] = dict(listed.errors) for collection, result in zip(listed.items, results, strict=True): + if isinstance(result, PartialResultError): + out.items.extend(result.results) if isinstance(result, CalendaringError): errors[collection.id] = result else: diff --git a/src/calendaring/_corpus.py b/src/calendaring/_corpus.py new file mode 100644 index 0000000..7fea948 --- /dev/null +++ b/src/calendaring/_corpus.py @@ -0,0 +1,139 @@ +"""Objects as real calendars hold them, for the conformance suite's round trip (mode-free). + +Each one carries what a backend is tempted to lose: time zones and their +VTIMEZONE, floating times, dates, alarms, a recurrence set with EXDATE, +RDATE and an override, escaped, multi-line and non-ASCII text, properties +with several values, and X- properties with parameters on every component +type. A backend may lose only what its declaration says it does. +""" + +from __future__ import annotations + +__all__ = ["CORPUS"] + + +def _crlf(text: str) -> bytes: + return text.replace("{OSLO}", OSLO.strip()).strip().replace("\n", "\r\n").encode() + b"\r\n" + + +OSLO = """ +BEGIN:VTIMEZONE +TZID:Europe/Oslo +BEGIN:STANDARD +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +END:DAYLIGHT +END:VTIMEZONE +""" + +CORPUS: dict[str, bytes] = { + "event-series-in-a-time-zone": _crlf(r""" +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//calendaring//conformance corpus//EN +{OSLO} +BEGIN:VEVENT +UID:corpus-series@conformance.invalid +DTSTAMP:20261001T000000Z +DTSTART;TZID=Europe/Oslo:20261102T090000 +DTEND;TZID=Europe/Oslo:20261102T100000 +RRULE:FREQ=WEEKLY;COUNT=4 +EXDATE;TZID=Europe/Oslo:20261109T090000 +SUMMARY:Møte på kontoret +DESCRIPTION:Agenda:\n1. Budsjett\; tall\, og «sitater»\n2. Kaffe ☕ +LOCATION:Rom 3\, 2. etasje +URL:https://meet.example.com/weekly +CATEGORIES:work,planning +ATTENDEE;CN=Åse;PARTSTAT=ACCEPTED:mailto:ase@example.com +ATTENDEE;CN=Bob:mailto:bob@example.com +X-CONFORMANCE-COLOUR;X-SHADE=dark:teal +BEGIN:VALARM +ACTION:DISPLAY +DESCRIPTION:Møtet begynner snart +TRIGGER:-PT15M +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:corpus-series@conformance.invalid +DTSTAMP:20261001T000000Z +RECURRENCE-ID;TZID=Europe/Oslo:20261116T090000 +DTSTART;TZID=Europe/Oslo:20261116T130000 +DTEND;TZID=Europe/Oslo:20261116T140000 +SUMMARY:Møte på kontoret (ettermiddag) +END:VEVENT +END:VCALENDAR +"""), + "all-day-event-with-rdate": _crlf(r""" +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//calendaring//conformance corpus//EN +BEGIN:VEVENT +UID:corpus-all-day@conformance.invalid +DTSTAMP:20261001T000000Z +DTSTART;VALUE=DATE:20261105 +DTEND;VALUE=DATE:20261106 +RDATE;VALUE=DATE:20261112,20261119 +SUMMARY:Heldagsseminar +END:VEVENT +END:VCALENDAR +"""), + "floating-event": _crlf(r""" +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//calendaring//conformance corpus//EN +BEGIN:VEVENT +UID:corpus-floating@conformance.invalid +DTSTAMP:20261001T000000Z +DTSTART:20261103T070000 +DTEND:20261103T073000 +SUMMARY:Morning run, wherever I am +END:VEVENT +END:VCALENDAR +"""), + "task-due-on-a-date": _crlf(r""" +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//calendaring//conformance corpus//EN +BEGIN:VTODO +UID:corpus-task@conformance.invalid +DTSTAMP:20261001T000000Z +DUE;VALUE=DATE:20261120 +SUMMARY:Søknad\, del 2 +DESCRIPTION:Første linje\nAndre linje\n\nEtter en tom linje +CATEGORIES:grant,writing +CATEGORIES:urgent +RELATED-TO;RELTYPE=PARENT:corpus-parent@conformance.invalid +X-CONFORMANCE-TASK;X-ORIGIN=corpus:kept +BEGIN:VALARM +ACTION:DISPLAY +DESCRIPTION:Fristen er i morgen +TRIGGER;RELATED=END:-P1D +END:VALARM +END:VTODO +END:VCALENDAR +"""), + "journal-entry": _crlf(r""" +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//calendaring//conformance corpus//EN +BEGIN:VJOURNAL +UID:corpus-journal@conformance.invalid +DTSTAMP:20261001T000000Z +DTSTART;VALUE=DATE:20261102 +SUMMARY:Dagbok +DESCRIPTION:Det regnet.\nVi drakk kakao – og spiste vafler. +X-CONFORMANCE-JOURNAL;X-MOOD=calm:kept +END:VJOURNAL +END:VCALENDAR +"""), +} diff --git a/src/calendaring/_mapping.py b/src/calendaring/_mapping.py index 88e0b6c..e887cbe 100644 --- a/src/calendaring/_mapping.py +++ b/src/calendaring/_mapping.py @@ -9,7 +9,9 @@ from __future__ import annotations +import copy from collections.abc import Callable +from datetime import UTC, date, datetime, time from typing import Any import icalendar @@ -18,7 +20,7 @@ from .items import X_PLANNED_END, X_PLANNED_START from .loss import Loss -__all__ = ["Change", "differences", "plan_losses", "property_lines"] +__all__ = ["Change", "differences", "plan_calendar_losses", "plan_losses", "property_lines"] # A loss and the edit that makes the component storable. Change = tuple[Loss, Callable[[], None]] @@ -28,6 +30,20 @@ "ATTENDEE": Feature.ATTENDEES, "RRULE": Feature.RECURRENCE, "RDATE": Feature.RECURRENCE, + "EXDATE": Feature.RECURRENCE, +} + +# The properties whose values are dates or times, for timezones and dates. +TIME_PROPERTIES = ("DTSTART", "DTEND", "DUE", "RECURRENCE-ID", "RDATE", "EXDATE") + +# What a backend without the feature stores instead (§5.1). +CONVERSIONS: dict[Feature, Callable[[Any], Any]] = { + # A time with a TZID, or a floating one (read as local time, as the searcher does), becomes UTC. + Feature.TIMEZONES: lambda t: t.astimezone(UTC) if isinstance(t, datetime) and t.tzname() != "UTC" else t, + # A date becomes midnight UTC. + Feature.DATES: lambda t: ( + t if isinstance(t, datetime) or not isinstance(t, date) else datetime.combine(t, time(), UTC) + ), } TASK_ONLY: dict[str, Feature] = { @@ -83,9 +99,54 @@ def apply() -> None: return apply +def _times(component: icalendar.Component, name: str) -> list[Any]: + """One property's dates and times; ``RDATE`` and ``EXDATE`` may hold several, on several lines.""" + out: list[Any] = [] + for value in _values(component, name): + if hasattr(value, "dts"): + out.extend(d.dt for d in value.dts) + else: + out.append(value.dt) + return out + + +def _convert(component: icalendar.Component, name: str, new: list[Any]) -> Callable[[], None]: + def apply() -> None: + component.pop(name, None) + component.add(name, new if name in ("RDATE", "EXDATE") else new[0]) + + return apply + + +def _time_losses(component: icalendar.Component, capabilities: Capabilities) -> list[Change]: + changes: list[Change] = [] + for feature, convert in CONVERSIONS.items(): + if capabilities.level(feature) in (Support.FULL, Support.EMULATED, Support.UNKNOWN): + continue + for name in TIME_PROPERTIES: + if name not in component: + continue + sent = _times(component, name) + stored = [convert(t) for t in sent] + if any(new is not old for new, old in zip(stored, sent, strict=True)): # a conversion returns a new value + changes.append((Loss(name, sent[0], stored[0]), _convert(component, name, stored))) + return changes + + +def _alarm_losses(component: icalendar.Component, capabilities: Capabilities) -> list[Change]: + alarms = [c for c in component.subcomponents if c.name == "VALARM"] + if not alarms or capabilities.level(Feature.ALARMS) is not Support.UNSUPPORTED: + return [] + + def apply() -> None: + component.subcomponents[:] = [c for c in component.subcomponents if c.name != "VALARM"] + + return [(Loss("VALARM", alarms[0].to_ical().decode(), None), apply)] + + def plan_losses(component: icalendar.Component, capabilities: Capabilities) -> list[Change]: """The losses writing ``component`` would cause, each with the edit that applies it.""" - changes: list[Change] = [] + changes: list[Change] = _alarm_losses(component, capabilities) is_task = component.name == "VTODO" table = ANY_COMPONENT | TASK_ONLY if is_task else ANY_COMPONENT for name in list(component): @@ -122,13 +183,37 @@ def plan_losses(component: icalendar.Component, capabilities: Capabilities) -> l changes.append( (Loss("RELATED-TO", sent if len(sent) > 1 else sent[0], None), _keep_related(component, dropped)) ) + dropped_names = {loss.property for loss, _ in changes} + changes.extend(c for c in _time_losses(component, capabilities) if c[0].property not in dropped_names) return changes +def _lines(component: icalendar.Component) -> list[bytes]: + """The component's own content lines, unfolded: not those of its subcomponents.""" + own = copy.copy(component) + own.subcomponents = [] + return own.to_ical().replace(b"\r\n ", b"").split(b"\r\n") # type: ignore[no-any-return] + + +def plan_calendar_losses(calendar: icalendar.Calendar, capabilities: Capabilities) -> list[Change]: + """The losses across a stored object's components: overrides go with the RRULE they belong to.""" + if capabilities.level(Feature.RECURRENCE) is not Support.UNSUPPORTED: + return [] + overrides = [c for c in calendar.subcomponents if "RECURRENCE-ID" in c] + if not overrides or len(overrides) == len([c for c in calendar.subcomponents if c.name != "VTIMEZONE"]): + return [] # occurrences without a master are items of their own + + def apply() -> None: + calendar.subcomponents[:] = [c for c in calendar.subcomponents if "RECURRENCE-ID" not in c] + + sent = [c.get("RECURRENCE-ID").to_ical().decode() for c in overrides] + return [(Loss("RECURRENCE-ID", sent if len(sent) > 1 else sent[0], None), apply)] + + def property_lines(component: icalendar.Component, name: str) -> list[bytes]: """One property's content lines, unfolded and sorted, for an order-insensitive comparison.""" out = [] - for line in component.to_ical().replace(b"\r\n ", b"").split(b"\r\n"): + for line in _lines(component): head = line.split(b":", 1)[0].split(b";", 1)[0] if head.upper() == name.encode(): out.append(line) @@ -161,4 +246,25 @@ def differences(sent: icalendar.Calendar, stored: icalendar.Calendar) -> list[Lo theirs = property_lines(other, name) if mine != theirs: out.append(Loss(name, b"\n".join(mine).decode(), b"\n".join(theirs).decode() or None)) + for kind in dict.fromkeys(str(c.name) for c in component.subcomponents): + candidates = [c for c in other.subcomponents if c.name == kind] + missing = [] + for sent_part in (c for c in component.subcomponents if c.name == kind): + match = next((c for c in candidates if _covers(c, sent_part)), None) + if match is None: + missing.append(sent_part) + else: + candidates.remove(match) + if missing: + sent_text = b"".join(c.to_ical() for c in missing).decode() + out.append(Loss(kind, sent_text, b"".join(c.to_ical() for c in candidates).decode() or None)) return out + + +def _covers(stored: icalendar.Component, sent: icalendar.Component) -> bool: + """Whether ``stored`` (an alarm) holds every property of ``sent`` as sent; servers may add their own.""" + return all( + property_lines(stored, name) == property_lines(sent, name) + for name in dict.fromkeys(sent) + if name not in NOT_VERIFIED + ) diff --git a/src/calendaring/_memory_store.py b/src/calendaring/_memory_store.py index 3943a21..fe0f9cd 100644 --- a/src/calendaring/_memory_store.py +++ b/src/calendaring/_memory_store.py @@ -8,14 +8,16 @@ from __future__ import annotations import itertools +import uuid from collections.abc import Iterable, Mapping from dataclasses import dataclass, field +from functools import cached_property from typing import Any import icalendar from .capabilities import BACKEND_FEATURES, Feature, Support -from .errors import AlreadyExistsError, ConflictError, NotFoundError +from .errors import AlreadyExistsError, ConflictError, InvalidDataError, NotFoundError from .items import _items_of, _parse from .types import Component @@ -156,6 +158,14 @@ def calendar(self) -> icalendar.Calendar: return _parse(self.data) @property + def parses(self) -> bool: + try: + _ = self.uid + except InvalidDataError: + return False + return True + + @cached_property # every add compares it with every stored object's def uid(self) -> str: return str(_items_of(self.calendar)[0]["UID"]) @@ -189,7 +199,7 @@ def create(self, data: bytes, *, native_id: str | None, assign_uid: bool) -> Mem if assign_uid: data = self._with_uid(data, f"{self.store.next()}@memory.invalid") uid = str(_items_of(_parse(data))[0]["UID"]) - if any(o.uid == uid for o in self.objects.values()): + if any(o.uid == uid for o in self.objects.values() if o.parses): raise AlreadyExistsError(f"UID {uid!r} is taken", taken="uid") return self.get(self.put(data, native_id)) @@ -223,6 +233,7 @@ def __init__(self, profile: Profile, *, silently_drop: Iterable[str] = ()) -> No self.profile = profile self.calendars: dict[str, MemoryCalendar] = {} self._counter = itertools.count(1) + self.instance = uuid.uuid4().hex # what tells two stores apart, as a URL tells servers apart # Properties to drop on every write without saying so, like a server # that declares UNKNOWN and loses them: for testing verify=True. self.silently_drop = frozenset(silently_drop) diff --git a/src/calendaring/_sync/collection.py b/src/calendaring/_sync/collection.py index f705809..3f1131a 100644 --- a/src/calendaring/_sync/collection.py +++ b/src/calendaring/_sync/collection.py @@ -14,8 +14,8 @@ import copy import json from abc import ABC, abstractmethod -from collections.abc import Iterator, Generator, Iterable, Mapping -from contextlib import contextmanager +from collections.abc import Iterator, Generator, Iterable, Mapping, Sequence +from contextlib import contextmanager, suppress from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, Literal, overload @@ -23,12 +23,21 @@ import recurring_ical_events from icalendar_searcher import Searcher -from .._mapping import Change, differences, plan_losses +from .._mapping import Change, differences, plan_calendar_losses, plan_losses from ..capabilities import BACKEND_FEATURES, Capabilities, Feature, Support -from ..errors import CalendaringError, NotFoundError, SyncTokenError, UnsupportedError, VerificationError +from ..errors import ( + CalendaringError, + ConflictError, + InvalidDataError, + NotFoundError, + PartialResultError, + SyncTokenError, + UnsupportedError, + VerificationError, +) from ..items import Event, Item, Journal, Task, _items_of, _parse, split_stored from ..loss import RAISE, LossPolicy -from ..types import ChangeSet, Component, Scope, StoredObject, SyncToken +from ..types import BrokenObject, ChangeSet, Component, Scope, StoredObject, SyncToken from .items import BOUND, SyncEvent, SyncItem, SyncJournal, SyncTask if TYPE_CHECKING: @@ -80,8 +89,13 @@ def __repr__(self) -> str: # === primitives a backend implements =============================== @abstractmethod - def _query(self, searcher: Searcher | None) -> list[StoredObject]: - """The stored objects that may match: over-matching is fine, under-matching is a bug (A3).""" + def _query(self, searcher: Searcher | None) -> Sequence[StoredObject | BrokenObject]: + """The stored objects that may match: over-matching is fine, under-matching is a bug (A3). + + An object that does not parse is returned as a ``BrokenObject``, so + one bad object leaves the rest readable. The same goes for every + primitive that lists: ``_iter_query``, ``_fetch_many``, ``_changes``. + """ @abstractmethod def _fetch(self, native_id: str) -> StoredObject: @@ -106,35 +120,50 @@ def _update( def _delete(self, native_id: str, *, etag: str | None) -> None: """Remove a stored object; ``ConflictError`` if ``etag`` is given and no longer matches.""" - def _iter_query(self, searcher: Searcher | None) -> Iterator[StoredObject]: + def _iter_query(self, searcher: Searcher | None) -> Iterator[StoredObject | BrokenObject]: """Override to stream pages; the default yields what ``_query`` returns.""" for stored in self._query(searcher): yield stored def _find_uid(self, uid: str) -> StoredObject: - """Override where the backend can look a UID up; the default reads everything.""" + """Override where the backend can look a UID up; the default reads everything. + + ``PartialResultError`` if the UID is not found but some object could not be read. + """ + broken = [] for stored in self._query(None): - if any(str(c.get("UID")) == uid for c in _items_of(stored.calendar)): + if isinstance(stored, BrokenObject): + broken.append(stored) + elif any(str(c.get("UID")) == uid for c in _items_of(stored.calendar)): return stored + if broken: + raise PartialResultError([], broken, f"no readable item with UID {uid!r}; {len(broken)} unreadable") raise NotFoundError(f"no item with UID {uid!r}") def _list_etags(self) -> dict[str, str]: """``{native_id: etag}`` for every stored object; override where it can be listed without content.""" - return {s.native_id: s.etag for s in self._query(None)} + # A broken object listed without an etag is still there: never report it deleted. + return {s.native_id: s.etag or "" for s in self._query(None)} - def _fetch_many(self, native_ids: Iterable[str]) -> list[StoredObject]: - """Override with a bulk fetch (CalDAV ``calendar-multiget``); objects gone meanwhile are left out.""" - out = [] + def _fetch_many(self, native_ids: Iterable[str]) -> Sequence[StoredObject | BrokenObject]: + """Override with a bulk fetch (CalDAV ``calendar-multiget``); objects gone meanwhile are left out. + + The default cannot keep the bytes of an object that does not parse; + a backend that has them overrides this to return them. + """ + out: list[StoredObject | BrokenObject] = [] for native_id in native_ids: try: out.append(self._fetch(native_id)) except NotFoundError: pass + except InvalidDataError as exc: + out.append(BrokenObject(native_id, None, None, exc)) return out def _changes( self, token: SyncToken | None, known: Mapping[str, str] | None - ) -> tuple[list[StoredObject], list[str], SyncToken]: + ) -> tuple[Sequence[StoredObject | BrokenObject], list[str], SyncToken]: """Override with a native change feed; the default diffs an etag listing (``EMULATED``).""" if token is not None: base = self._decode_token(token) @@ -143,7 +172,11 @@ def _changes( current = self._list_etags() changed = [n for n, etag in current.items() if base.get(n) != etag] deleted = sorted(n for n in base if n not in current) - return self._fetch_many(changed), deleted, self._encode_token(current) + fetched = self._fetch_many(changed) + for stored in fetched: + if isinstance(stored, BrokenObject) and stored.etag is None: + stored.etag = current[stored.native_id] or None + return fetched, deleted, self._encode_token(current) def _move(self, item: Item, target: Collection) -> StoredObject | None: """Override with the backend's own move; ``None`` means add to the target, then delete here.""" @@ -216,8 +249,18 @@ def _verifying(self, verify: bool | None) -> bool: return bool(setting) return False + def _token_scope(self) -> str: + """What an emulated token is bound to: by default the backend's kind and id, and the collection's id. + + The backend id is the caller's label for it, so two configurations + that give different stores the same id accept each other's tokens, + and renaming a backend invalidates its tokens. A backend that has a + locator (a directory, a URL) overrides this with it. + """ + return f"{self._backend.kind}:{self.backend_id}:{self.id}" + def _encode_token(self, etags: Mapping[str, str]) -> SyncToken: - raw = json.dumps({"collection": self.id, "etags": etags}, sort_keys=True).encode() + raw = json.dumps({"collection": self._token_scope(), "etags": etags}, sort_keys=True).encode() return SyncToken(base64.urlsafe_b64encode(raw).decode()) def _decode_token(self, token: SyncToken) -> dict[str, str]: @@ -227,7 +270,7 @@ def _decode_token(self, token: SyncToken) -> dict[str, str]: issued_by = data["collection"] except (binascii.Error, ValueError, TypeError, KeyError) as exc: raise SyncTokenError(f"not a token this collection issued: {token!r}") from exc - if issued_by != self.id or not isinstance(etags, dict): + if issued_by != self._token_scope() or not isinstance(etags, dict): raise SyncTokenError(f"a token from another collection: {issued_by!r}") return etags @@ -270,6 +313,7 @@ def _prepare( else: calendar = copy.deepcopy(item.icalendar) changes = [change for c in _items_of(calendar) for change in self._plan_losses(c)] + changes += plan_calendar_losses(calendar, self.capabilities) losses = [loss_ for loss_, _ in changes] if losses and wire is not None: raise UnsupportedError(Feature.WRITE_WIRE, "wire= bytes the backend cannot store as they are") @@ -291,12 +335,26 @@ def _searcher(self, searcher: Searcher | None, filters: Mapping[str, Any]) -> Se raise TypeError("give a Searcher or keyword filters, not both") return searcher if searcher is not None else Searcher(**filters) - def _matches(self, searcher: Searcher, stored: StoredObject) -> list[SyncItem]: - """Run the client-side filter over one stored object (A3); expand into occurrences if asked.""" + def _matches( + self, searcher: Searcher, stored: StoredObject | BrokenObject, broken: list[BrokenObject] + ) -> list[SyncItem]: + """Run the client-side filter over one stored object (A3); expand into occurrences if asked. + + An object that is broken, or that the searcher cannot match, goes to ``broken``. + """ + if isinstance(stored, BrokenObject): + broken.append(stored) + return [] out: list[SyncItem] = [] parts = split_stored(stored.calendar) for part in parts: - matched = searcher.check_component(part) + try: + matched = searcher.check_component(part) + except ValueError as exc: # icalendar-searcher on what is no valid recurrence set + error = InvalidDataError(f"the searcher cannot match it: {exc}") + error.__cause__ = exc + broken.append(BrokenObject(stored.native_id, stored.etag, stored.wire, error)) + return [] if not matched: continue components = [c for c in matched if c.name != "VTIMEZONE"] @@ -349,20 +407,31 @@ def _sorted(searcher: Searcher, items: list[SyncItem]) -> list[SyncItem]: def search(self, searcher: Searcher | None = None, **filters: Any) -> list[SyncItem]: with self._context(): searcher = self._searcher(searcher, filters) - items = [item for stored in self._query(searcher) for item in self._matches(searcher, stored)] - return self._sorted(searcher, items) + broken: list[BrokenObject] = [] + found = [i for stored in self._query(searcher) for i in self._matches(searcher, stored, broken)] + items = self._sorted(searcher, found) + if broken: + raise PartialResultError(items, broken) + return items def iter_search(self, searcher: Searcher | None = None, **filters: Any) -> Iterator[SyncItem]: - """``search()`` one item at a time; with sort keys, everything is read before the first.""" + """``search()`` one item at a time; with sort keys, everything is read before the first. + + Objects left out raise ``PartialResultError`` at the end, with empty + ``results``: they were yielded. + """ with self._context(): searcher = self._searcher(searcher, filters) if getattr(searcher, "_sort_keys", None): for item in self.search(searcher): yield item return + broken: list[BrokenObject] = [] for stored in self._iter_query(searcher): - for item in self._matches(searcher, stored): + for item in self._matches(searcher, stored, broken): yield item + if broken: + raise PartialResultError([], broken) # the caller has the results already def tasks(self, **filters: Any) -> list[SyncTask]: if Component.TASK not in self.components: @@ -421,19 +490,27 @@ def relatives(self, item: Item, reltype: str | None = None) -> list[SyncItem]: related = item.component.get("RELATED-TO", []) related = related if isinstance(related, list) else [related] out: dict[str, SyncItem] = {} + broken: list[BrokenObject] = [] for value in related: kind = str(value.params.get("RELTYPE", "PARENT")).upper() if wanted is None or kind == wanted: - found = self._lookup_relative(str(value)) + found = self._lookup_relative(str(value), broken) if found is not None: out.setdefault(found.uid, found) back = {"PARENT": "CHILD", "CHILD": "PARENT"} if wanted is None or wanted in back: - for other in self.search(): + try: + listed = self.search() + except PartialResultError as exc: + broken.extend(exc.broken) + listed = list(exc.results) + for other in listed: for value in self._related_to(other): kind = str(value.params.get("RELTYPE", "PARENT")).upper() if str(value) == item.uid and kind in back and (wanted is None or back[kind] == wanted): out.setdefault(other.uid, other) + if broken: + raise PartialResultError(list(out.values()), broken) return list(out.values()) @staticmethod @@ -441,18 +518,16 @@ def _related_to(item: Item) -> list[Any]: related = item.component.get("RELATED-TO", []) return related if isinstance(related, list) else [related] - def _lookup_relative(self, uid: str) -> SyncItem | None: - try: - return self.get(uid) - except NotFoundError: - pass - for other in self._backend.collections(): - if other.id == self.id: - continue + def _lookup_relative(self, uid: str, broken: list[BrokenObject]) -> SyncItem | None: + """Here first, then in the backend's other collections; past objects that cannot be read.""" + others = [c for c in self._backend.collections() if c.id != self.id] + for collection in [self, *others]: try: - return other.get(uid) + return collection.get(uid) + except PartialResultError as exc: # not found among what could be read + broken.extend(exc.broken) except NotFoundError: - continue + pass return None # a dangling relation # === writing ======================================================= @@ -701,8 +776,15 @@ def move( stored = self._move(item, target) if stored is not None: return target._one(stored, written=True) + if item.etag is not None and (self._fetch(item.native_id)).etag != item.etag: + raise ConflictError(f"{item.native_id!r} changed since it was read") moved = target.add(item.as_data(), loss=loss, verify=verify) - self.delete(item) + try: + self.delete(item) + except ConflictError: + with suppress(CalendaringError): # best effort: the conflict is what to report + target.delete(moved, overwrite=True) + raise return moved # === binding, no I/O =============================================== @@ -742,7 +824,12 @@ def changes( raise ValueError("changes(): known= with a token; an incremental report has its own baseline") self._require(Feature.CHANGES_KNOWN) changed, deleted, new_token = self._changes(token, known) - return ChangeSet([i for s in changed for i in self._bound(s)], deleted, new_token) + broken = [s for s in changed if isinstance(s, BrokenObject)] + items = [i for s in changed if not isinstance(s, BrokenObject) for i in self._bound(s)] + result = ChangeSet(items, deleted, new_token) + if broken: + raise PartialResultError(result, broken) + return result # === the collection itself ========================================= diff --git a/src/calendaring/_sync/conformance.py b/src/calendaring/_sync/conformance.py index 3c848cb..3586023 100644 --- a/src/calendaring/_sync/conformance.py +++ b/src/calendaring/_sync/conformance.py @@ -21,12 +21,15 @@ async def open_backend(self) -> AsyncBackend: The suite makes its own collection where the backend can. Where it cannot, it uses the first one listed, compares only what the test itself stored, and -deletes what it stored afterwards. +deletes what it stored afterwards, and puts back a name it changed. Where it +can make collections but not delete them, it reuses the ones an earlier run +left, so they hold nothing of the suite's after a run. """ from __future__ import annotations import pickle +import uuid from collections.abc import Iterator, Callable, Sequence from datetime import UTC, datetime, timedelta from typing import Any @@ -35,21 +38,24 @@ async def open_backend(self) -> AsyncBackend: import pytest from icalendar_searcher import Searcher -from .._mapping import property_lines +from .._corpus import CORPUS +from .._mapping import differences, property_lines from ..capabilities import BACKEND_FEATURES, Feature, Support from ..errors import ( AlreadyExistsError, + BackendError, CalendaringError, ConflictError, LossyWriteError, LossyWriteWarning, NotFoundError, + PartialResultError, SyncTokenError, UnsupportedError, ) from ..items import Event, Item, Journal, Task, TaskStatus from ..loss import ALLOW, WARN, Loss -from ..types import Component, Scope, SyncToken +from ..types import Component, Scope, StoredObject, SyncToken from .backend import Backend from .collection import Collection from .items import SyncEvent, SyncItem, SyncJournal, SyncTask @@ -95,9 +101,17 @@ def level(collection: Collection, feature: Feature) -> Support: return collection.capabilities.level(feature) +def readable(collection: Collection) -> tuple[list[SyncItem], set[str]]: + """What a borrowed collection holds that can be read, and the native ids of what cannot.""" + try: + return collection.search(), set() + except PartialResultError as exc: + return list(exc.results), {b.native_id for b in exc.broken} + + def snapshot(collection: Collection) -> list[tuple[str | None, str | None]]: - """What is stored, as (native_id, etag) pairs: unchanged means no write happened.""" - return sorted((i.native_id, i.etag) for i in collection.search()) + """What is stored and readable, as (native_id, etag) pairs: unchanged means no write happened.""" + return sorted((i.native_id, i.etag) for i in (readable(collection))[0]) # One item type per component: the data class, the add_ shorthand, the @@ -142,11 +156,30 @@ def categories() -> Searcher: s.add_property_filter("CATEGORIES", "work") return s + def text(value: str, operator: str = "contains", case_sensitive: bool = True) -> Callable[[], Searcher]: + def make() -> Searcher: + s = Searcher() + s.add_property_filter("SUMMARY", value, operator=operator, case_sensitive=case_sensitive) + return s + + return make + + def no_due() -> Searcher: + s = Searcher(todo=True) + s.add_property_filter("DUE", None, operator="undef") + return s + return { "everything": lambda: Searcher(), + "summary-contains": text("rain"), + "summary-any-case": text("REPORT", case_sensitive=False), + "summary-not-containing": text("report", operator="!contains"), + "tasks-without-due": no_due, + "alarms-in-range": lambda: Searcher(alarm_start=at(3, 9), alarm_end=at(3, 10)), "tasks": lambda: Searcher(todo=True), "open-tasks": lambda: Searcher(todo=True, include_completed=False), "events-in-range": lambda: Searcher(event=True, start=at(2), end=at(4)), + "tasks-in-range": lambda: Searcher(todo=True, start=at(2), end=at(4)), "events-expanded": lambda: Searcher(event=True, start=at(2), end=at(4), expand=True), "journals": lambda: Searcher(journal=True), "categories": categories, @@ -159,11 +192,20 @@ def dataset() -> list[Item]: b = Task.new(summary="B: done", end=at(5), status=TaskStatus.COMPLETED) c = Task.new(summary="C: open") e1 = Event.new(summary="E1: meeting", start=at(3, 10), end=at(3, 11)) + alarm = icalendar.Alarm() + alarm.add("ACTION", "DISPLAY") + alarm.add("DESCRIPTION", "E1 starts soon") + alarm.add("TRIGGER", timedelta(minutes=-30)) + e1.component.add_component(alarm) + t = Event.new(summary="T: Training session", start=at(10, 18), end=at(10, 19)) + s = Event.new(summary="S: overnight", start=at(1, 20), end=at(2, 14)) # began before the range + ts = Task.new(summary="TS: start only", start=at(3)) + tb = Task.new(summary="TB: spans the range", start=at(1), end=at(5)) e2 = Event.new(summary="E2: later", start=at(20, 10), end=at(20, 11)) r = Event.new(summary="R: daily", start=at(1, 9), end=at(1, 10)) r.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) j = Journal.new(summary="J: notes", start=at(2)) - return [a, b, c, e1, e2, r, j] + return [a, b, c, ts, tb, e1, e2, t, s, r, j] class SyncConformance: @@ -172,14 +214,26 @@ class SyncConformance: # Non-CalDAV backends are code, not servers, and must not declare UNKNOWN (§5.1). allow_unknown: bool = False + # How many objects the paging test stores: more than a page of the APIs + # backends list through (a JMAP /query limit, an issue tracker's list). + page_size_hint: int = 120 + # --- hooks a backend's test module may override -------------------- def open_backend(self) -> Backend: raise NotImplementedError("a conformance test class must implement open_backend()") def open_collection(self, backend: Backend, name: str = "conformance") -> Collection: - """A collection to test against: made if the backend may be able to, else the first one listed.""" + """A collection to test against: made if the backend may be able to, else the first one listed. + + Where collections can be made but not deleted, the one a previous run + left is used again, so a run leaves one collection per name behind. + """ if backend.capabilities.level(Feature.CREATE_COLLECTION) is not Support.UNSUPPORTED: + deletable = backend.capabilities.level(Feature.DELETE_COLLECTION) is not Support.UNSUPPORTED + left = None if deletable else self.left_behind(backend, name) + if left is not None: + return self.borrow(left) try: made = backend.create_collection( name, components=(Component.EVENT, Component.TASK, Component.JOURNAL) @@ -188,40 +242,80 @@ def open_collection(self, backend: Backend, name: str = "conformance") -> Collec if backend.capabilities.level(Feature.CREATE_COLLECTION) is not Support.UNKNOWN: raise else: + if not deletable: + return self.borrow(made) self._made.append(made) return made existing = backend.collections() if not existing: pytest.fail("the backend can neither create a collection nor list one") - collection = existing[0] - self._borrowed.append((collection, {i.native_id for i in collection.search()})) + return self.borrow(existing[0]) + + def left_behind(self, backend: Backend, name: str) -> Collection | None: + named = [c for c in backend.collections() if c.name == name] + return named[0] if named else None + + def borrow(self, collection: Collection) -> Collection: + """Use a collection the suite may not delete, and note what it writes there, to delete only that.""" + items, broken = readable(collection) + before: set[str | None] = {i.native_id for i in items} | set(broken) + written: set[str] = set() # native ids and UIDs of what the suite created + create = collection._create + + def noting(calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + stored = create(calendar, native_id=native_id, wire=wire) + written.add(stored.native_id) + written.update(str(c.get("UID")) for c in stored.calendar.subcomponents if "UID" in c) + return stored + + collection._create = noting # type: ignore[method-assign] + self._borrowed.append((collection, before, written)) return collection def seed(self, collection: Collection, items: Sequence[Item]) -> list[SyncItem]: """Store items the way a test needs them. A read-only backend overrides this to seed natively.""" return [collection.add(item, loss=ALLOW) for item in items] + def unseed(self, collection: Collection, item: SyncItem) -> None: + """Remove a seeded item. A read-only backend overrides this to remove it natively.""" + collection.delete(item) + # --- fixtures ----------------------------------------------------- @pytest.fixture def backend(self) -> Iterator[Backend]: self._made: list[Collection] = [] - self._borrowed: list[tuple[Collection, set[str | None]]] = [] + self._borrowed: list[tuple[Collection, set[str | None], set[str]]] = [] backend = self.open_backend() with backend: yield backend self.tidy_up(backend) def tidy_up(self, backend: Backend) -> None: - """Delete the collections the test made, and what it stored in one it borrowed.""" + """Delete the collections the test made, and what it stored in one it borrowed. + + In a borrowed collection, an item is the suite's if it created it, or + if it carries or relates to a UID the suite created (a completed + occurrence the backend stored as a new object): anything else arrived + from another client while the test ran. + """ for made in self._made: - if backend.capabilities.level(Feature.DELETE_COLLECTION) is not Support.UNSUPPORTED: - made.delete_collection() - for borrowed, before in self._borrowed: - if writable(borrowed): - for item in borrowed.search(): - if item.native_id not in before and item.native_id is not None: - borrowed.delete(item.native_id) + made.delete_collection() + for borrowed, before, written in self._borrowed: + if not writable(borrowed): + continue + items, _ = readable(borrowed) + for native_id in {i.native_id for i in items if self.wrote(i, before, written)}: + assert native_id is not None + borrowed.delete(native_id) + + @staticmethod + def wrote(item: Item, before: set[str | None], written: set[str]) -> bool: + if item.native_id in before: + return False + related = item.component.get("RELATED-TO", []) + related = related if isinstance(related, list) else [related] + return item.native_id in written or item.uid in written or any(str(r) in written for r in related) @pytest.fixture def collection(self, backend: Backend) -> Collection: @@ -351,7 +445,7 @@ def test_adding_a_uid_twice(self, collection: Collection) -> None: def test_add_with_native_id(self, collection: Collection) -> None: self.add_task(collection, summary="setup") - name = "conformance-native-id.ics" + name = f"conformance-{uuid.uuid4().hex[:12]}.ics" # a crashed run may leave one behind if level(collection, Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.UNSUPPORTED: self.assert_unsupported( collection, @@ -390,6 +484,88 @@ def test_reads_are_bound_and_consistent(self, collection: Collection) -> None: assert str(read.component["SUMMARY"]) == "read me" assert reloaded is not stored + def test_get_finds_the_exact_uid(self, backend: Backend, collection: Collection) -> None: + stored = [] + for kind, (data_class, _, _, bound_class, _) in KINDS.items(): + if holds(collection, kind) and writable(collection): + item = data_class.new(summary=f"get {kind.value}") + added = collection.add(item) + if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: + assert added.uid == item.uid, f"identity.client-uid: the {kind.value} UID was replaced" + stored.append((added, bound_class)) + if not stored: + pytest.skip("nothing can be stored here") + for added, bound_class in stored: + found = collection.get(added.uid) + assert isinstance(found, bound_class) + assert found.native_id == added.native_id + if level(collection, Feature.IDENTITY_CLIENT_UID) is not Support.UNSUPPORTED: + kind_class = stored[0][0].as_data().__class__ + suffix = f"{stored[0][0].uid}@conformance.invalid" + longer = collection.add(kind_class.new(summary="longer", uid=f"11-{suffix}")) + shorter = collection.add(kind_class.new(summary="shorter", uid=f"1-{suffix}")) + assert (collection.get(shorter.uid)).native_id == shorter.native_id + assert (collection.get(longer.uid)).native_id == longer.native_id + with pytest.raises(NotFoundError): + collection.get(f"-{suffix}") + if backend.capabilities.supports(Feature.CREATE_COLLECTION): + other = self.open_collection(backend, "conformance-other") + if writable(other) and holds(other, Component.TASK): + elsewhere = other.add_task(summary="in another collection") + with pytest.raises(NotFoundError): + collection.get(elsewhere.uid) + + def test_add_with_native_id_keeps_the_uid(self, collection: Collection) -> None: + if level(collection, Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.UNSUPPORTED: + return # test_add_with_native_id checked that it raises + if level(collection, Feature.IDENTITY_CLIENT_UID) is Support.UNSUPPORTED: + pytest.skip("the backend assigns UIDs") + if not writable(collection): + return # test_read_only_rejects_every_write checked the writes + kind = next(k for k in KINDS if holds(collection, k)) + item = KINDS[kind][0].new(summary="named and identified") + name = f"conformance-{uuid.uuid4().hex[:12]}.ics" + stored = collection.add(item, native_id=name) + assert stored.uid == item.uid + assert (collection.get(item.uid)).native_id == name + + def test_a_listing_shows_a_save_made_elsewhere(self, collection: Collection) -> None: + stored = self.add_task(collection, summary="original", description="kept") + other = collection.get(stored.uid) + other.component["SUMMARY"] = "saved elsewhere" + other.save() + listed = [i for i in collection.search() if i.native_id == stored.native_id] + streamed = [i for i in collection.iter_search() if i.native_id == stored.native_id] + for [read] in (listed, streamed): + assert read.etag != stored.etag, "a listing gave the old etag after a save" + assert str(read.component["SUMMARY"]) == "saved elsewhere", "a listing gave old content" + assert str(read.component["DESCRIPTION"]) == "kept" + stored.component["SUMMARY"] = "stale" + with pytest.raises(ConflictError): + stored.save() + + def test_read_paths_on_seeded_items(self, collection: Collection) -> None: + """What a read-only backend must pass too: everything here is stored through ``seed()``.""" + kind = next(k for k in KINDS if holds(collection, k)) + new = KINDS[kind][0].new + [first] = self.seed(collection, [new(summary="seeded first")]) + assert (collection.get(first.uid)).native_id == first.native_id + assert (collection.get_by_native_id(nid(first))).uid == first.uid + assert (collection.reload(first)).etag == first.etag + if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: + return + start = collection.changes() + [second] = self.seed(collection, [new(summary="seeded second")]) + since = collection.changes(start.token) + assert nid(second) in {nid(i) for i in since.changed} + assert nid(first) not in {nid(i) for i in since.changed} + assert since.deleted == [] + self.unseed(collection, first) + gone = collection.changes(since.token) + assert gone.deleted == [nid(first)] + with pytest.raises(NotFoundError): + collection.get(first.uid) + def test_wire(self, collection: Collection) -> None: stored = self.add_task(collection, summary="bytes") read = collection.get_by_native_id(nid(stored)) @@ -571,6 +747,26 @@ def policy(losses: Sequence[Loss], item: Item) -> None: assert [[loss.property for loss in losses] for losses in seen] == [[name]] assert seen[0][0].sent == outside[0] + @pytest.mark.parametrize("name", list(CORPUS)) + def test_real_world_objects_round_trip(self, collection: Collection, name: str) -> None: + """Only what the declaration says may be lost: verify compares the stored object with what was sent.""" + run = f"-{uuid.uuid4().hex[:12]}@conformance.invalid".encode() # a crashed run may leave one behind + item = Item.from_ical(CORPUS[name].replace(b"@conformance.invalid", run)) + kind = Component(item.component.name) + if not holds(collection, kind): + pytest.skip(f"the collection holds no {kind.value}") + if writable(collection): + read = collection.reload(collection.add(item, loss=ALLOW, verify=True)) + else: + [seeded] = self.seed(collection, [item]) + read = collection.reload(seeded) + assert differences(item.icalendar, read.icalendar) == [] + zones = {tz["TZID"] for tz in read.icalendar.walk("VTIMEZONE")} + for component in read.icalendar.walk(): + for prop in component.property_items(recursive=False): + tzid = getattr(prop[1], "params", {}).get("TZID") + assert tzid is None or tzid in zones, f"{prop[0]} uses TZID {tzid}, which has no VTIMEZONE" + def test_verify_passes_on_an_honest_write(self, collection: Collection) -> None: stored = self.add_task(collection, summary="verified", description="checked after writing") stored.component["SUMMARY"] = "verified again" @@ -628,7 +824,35 @@ def test_iter_search_yields_what_search_returns(self, collection: Collection) -> self.seed(collection, wanted) listed = collection.search() iterated = [i for i in collection.iter_search()] - assert sorted(key(i) for i in iterated) == sorted(key(i) for i in listed) + + def fields(items: Sequence[SyncItem]) -> list[tuple[Any, ...]]: + return sorted((key(i), i.native_id, i.etag, i.component.to_ical()) for i in items) + + assert fields(iterated) == fields(listed), "iter_search() must give what search() gives, field by field" + + def test_listings_are_not_cut_at_a_page(self, collection: Collection) -> None: + kind = next(k for k in KINDS if holds(collection, k)) + new = KINDS[kind][0].new + seeded = self.seed(collection, [new(summary=f"paged {n}") for n in range(self.page_size_hint)]) + mine = {nid(i) for i in seeded} + assert {nid(i) for i in stored_by(collection.search(), seeded)} == mine + assert {nid(i) for i in collection.iter_search() if i.native_id in mine} == mine + last = seeded[-1] + assert (collection.get(last.uid)).native_id == last.native_id + if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: + return + full = collection.changes() + assert mine <= {nid(i) for i in full.changed} + if level(collection, Feature.CHANGES_KNOWN) is not Support.UNSUPPORTED: + known = collection.changes(known={nid(i): str(i.etag) for i in seeded}) + assert known.deleted == [] + assert not mine & {nid(i) for i in known.changed} + if writable(collection): + last.component["SUMMARY"] = "paged, changed" + last.save() + since = collection.changes(full.token) + assert nid(last) in {nid(i) for i in since.changed} + assert since.deleted == [] def test_search_sorts(self, collection: Collection) -> None: searcher = Searcher(todo=True) @@ -652,6 +876,12 @@ def test_listing_shorthands_return_one_type(self, collection: Collection) -> Non def test_complete_and_uncomplete(self, collection: Collection) -> None: task = self.add_task(collection, summary="to do") + if level(collection, Feature.TASK_STATUS) is Support.UNSUPPORTED: + before = snapshot(collection) + with pytest.raises(LossyWriteError): + task.complete() # a backend without a status cannot store "completed" + assert snapshot(collection) == before + return task.complete() assert str(task.component["STATUS"]) == "COMPLETED" read = collection.reload(task) @@ -675,9 +905,15 @@ def test_complete_a_recurring_task(self, collection: Collection) -> None: if level(collection, Feature.TASK_COMPLETE_RECURRING) is Support.UNSUPPORTED: self.assert_unsupported(collection, Feature.TASK_COMPLETE_RECURRING, lambda: task.complete()) return + etag = task.etag task.complete() read = collection.reload(task) assert str(read.component.get("STATUS", "")) != "COMPLETED", "completing one occurrence ended the series" + # "safe" mode (§2.3): a completed copy of the occurrence, and the master's DTSTART moved on. + assert read.etag != etag, "complete() on a recurring task wrote nothing" + assert read.component.decoded("DTSTART") > at(1), "the master's DTSTART did not move" + done = [t for t in collection.tasks() if str(t.component.get("STATUS", "")) == "COMPLETED"] + assert any(str(t.component["SUMMARY"]) == "weekly" for t in done), "no completed copy of the occurrence" # === recurrence =================================================== @@ -732,7 +968,7 @@ def test_changes_full_listing(self, collection: Collection) -> None: if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: self.assert_unsupported(collection, Feature.CHANGES, lambda: collection.changes()) return - kind = next(iter(collection.components)) + kind = next(k for k in KINDS if holds(collection, k)) seeded = self.seed(collection, [KINDS[kind][0].new(summary=s) for s in ("one", "two")]) changes = collection.changes() assert {i.native_id for i in changes.changed} >= {i.native_id for i in seeded} @@ -788,6 +1024,20 @@ def test_a_bad_token_raises(self, collection: Collection) -> None: with pytest.raises(SyncTokenError): collection.changes(SyncToken("conformance-not-a-token")) + def test_a_token_from_another_collection_raises( + self, backend: Backend, collection: Collection + ) -> None: + if level(collection, Feature.CHANGES) is Support.UNSUPPORTED: + return # test_a_bad_token_raises checked that changes() raises + if not backend.capabilities.supports(Feature.CREATE_COLLECTION): + pytest.skip("a second collection is needed, and the backend cannot make one") + other = self.open_collection(backend, "conformance-other") + if level(other, Feature.CHANGES) is Support.UNSUPPORTED: + pytest.skip("the second collection has no change feed") + token = (other.changes()).token + with pytest.raises(SyncTokenError): + collection.changes(token) + # === moving ======================================================= def test_move(self, backend: Backend, collection: Collection) -> None: @@ -805,6 +1055,25 @@ def test_move(self, backend: Backend, collection: Collection) -> None: collection.get_by_native_id(nid(task)) assert (target.get_by_native_id(nid(moved))).uid == moved.uid + def test_changes_follow_a_move(self, backend: Backend, collection: Collection) -> None: + if Support.UNSUPPORTED in (level(collection, Feature.CHANGES), level(collection, Feature.MOVE)): + return # the tests of each feature checked that it raises + task = self.add_task(collection, summary="moving") + if not backend.capabilities.supports(Feature.CREATE_COLLECTION): + pytest.skip("a second collection is needed, and the backend cannot make one") + target = self.open_collection(backend, "conformance-target") + if level(target, Feature.CHANGES) is Support.UNSUPPORTED: + pytest.skip("the second collection has no change feed") + here, there = collection.changes(), target.changes() + elsewhere = target.add_task(summary="added there") + moved = collection.move(task, target) + since_here = collection.changes(here.token) + since_there = target.changes(there.token) + assert since_here.deleted == [nid(task)], "a moved item is deleted where it was" + assert not {nid(elsewhere), nid(moved)} & {nid(i) for i in since_here.changed} + assert {nid(moved), nid(elsewhere)} <= {nid(i) for i in since_there.changed} + assert since_there.deleted == [] + # === collections ================================================== def test_set_name(self, backend: Backend, collection: Collection) -> None: @@ -813,27 +1082,51 @@ def test_set_name(self, backend: Backend, collection: Collection) -> None: collection.set_name("renamed") assert info.value.feature == Feature.COLLECTION_SET_NAME return + name = collection.name collection.set_name("conformance renamed") - assert collection.name == "conformance renamed" - assert (backend.collection(collection.id)).name == "conformance renamed" + try: + assert collection.name == "conformance renamed" + assert (backend.collection(collection.id)).name == "conformance renamed" + finally: + if name is not None: # a borrowed or reused collection is put back as it was + collection.set_name(name) def test_create_and_delete_collection(self, backend: Backend) -> None: before = [c.id for c in backend.collections()] + if backend.capabilities.level(Feature.CREATE_COLLECTION) is Support.UNKNOWN: + deletable = backend.capabilities.level(Feature.DELETE_COLLECTION) is not Support.UNSUPPORTED + attempted = None if deletable else self.left_behind(backend, "conformance-new") + if attempted is None: + try: + attempted = backend.create_collection("conformance-new", components=(Component.TASK,)) + except (UnsupportedError, BackendError): + return # unprobed, and it does not work here: attempted, as UNKNOWN promises + if deletable: + self._made.append(attempted) + assert attempted.components == frozenset({Component.TASK}) + assert attempted.id in [c.id for c in backend.collections()] + return if not backend.capabilities.supports(Feature.CREATE_COLLECTION): with pytest.raises(UnsupportedError) as info: backend.create_collection("conformance-new") assert info.value.feature == Feature.CREATE_COLLECTION assert [c.id for c in backend.collections()] == before return - made = backend.create_collection("conformance-new", components=(Component.TASK,)) + deletable = backend.capabilities.supports(Feature.DELETE_COLLECTION) + # Without delete-collection, only the first run on a backend makes it: later ones find it left there. + made = None if deletable else self.left_behind(backend, "conformance-new") + if made is None: + made = backend.create_collection("conformance-new", components=(Component.TASK,)) assert made.components == frozenset({Component.TASK}) assert made.id in [c.id for c in backend.collections()] - if not backend.capabilities.supports(Feature.DELETE_COLLECTION): + if not deletable: with pytest.raises(UnsupportedError) as info: made.delete_collection() assert info.value.feature == Feature.DELETE_COLLECTION return + self._made.append(made) made.delete_collection() + self._made.remove(made) assert made.id not in [c.id for c in backend.collections()] # === binding ====================================================== diff --git a/src/calendaring/_sync/memory.py b/src/calendaring/_sync/memory.py index 3f11820..7129b67 100644 --- a/src/calendaring/_sync/memory.py +++ b/src/calendaring/_sync/memory.py @@ -10,7 +10,7 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from typing import Any, ClassVar import icalendar @@ -18,10 +18,10 @@ from .._memory_store import FULL, MemoryCalendar, MemoryObject, MemoryStore, Profile from ..capabilities import Capabilities, Feature, Support -from ..errors import ConflictError +from ..errors import ConflictError, InvalidDataError from ..items import Item from ..loss import LossPolicy -from ..types import Component, StoredObject +from ..types import BrokenObject, Component, StoredObject from .backend import Backend from .collection import Collection @@ -32,6 +32,13 @@ def _stored(obj: MemoryObject, *, written: bool = False) -> StoredObject: return StoredObject(obj.native_id, obj.etag, obj.calendar, None if written else obj.data, obj) +def _listed(obj: MemoryObject) -> StoredObject | BrokenObject: + try: + return _stored(obj) + except InvalidDataError as exc: + return BrokenObject(obj.native_id, obj.etag, obj.data, exc) + + class MemoryCollection(Collection): _backend: MemoryBackend @@ -47,8 +54,11 @@ def __init__(self, backend: MemoryBackend, store: MemoryCalendar) -> None: native=store, ) - def _query(self, searcher: Searcher | None) -> list[StoredObject]: - return [_stored(o) for o in self.store.objects.values()] + def _query(self, searcher: Searcher | None) -> Sequence[StoredObject | BrokenObject]: + return [_listed(o) for o in self.store.objects.values()] + + def _fetch_many(self, native_ids: Iterable[str]) -> list[StoredObject | BrokenObject]: + return [_listed(self.store.objects[n]) for n in native_ids if n in self.store.objects] def _fetch(self, native_id: str) -> StoredObject: return _stored(self.store.get(native_id)) @@ -56,6 +66,9 @@ def _fetch(self, native_id: str) -> StoredObject: def _list_etags(self) -> dict[str, str]: return {o.native_id: o.etag for o in self.store.objects.values()} + def _token_scope(self) -> str: + return f"memory:{self._backend.store.instance}:{self.id}" + def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: data = self._backend.store.drop(wire if wire is not None else calendar.to_ical()) obj = self.store.create(data, native_id=native_id, assign_uid=self._backend.store.assign_uid) diff --git a/src/calendaring/_sync/workspace.py b/src/calendaring/_sync/workspace.py index ed37156..878b25e 100644 --- a/src/calendaring/_sync/workspace.py +++ b/src/calendaring/_sync/workspace.py @@ -19,7 +19,7 @@ from icalendar_searcher import Searcher from .._concurrency import gather -from ..errors import AmbiguousError, CalendaringError, ConfigurationError, NotFoundError +from ..errors import AmbiguousError, CalendaringError, ConfigurationError, NotFoundError, PartialResultError from ..loss import LossPolicy from ..types import MultiResult from .backend import Backend @@ -137,6 +137,8 @@ def search(self, searcher: Searcher | None = None, **filters: Any) -> MultiResul out: MultiResult[SyncItem] = MultiResult() errors: dict[str, CalendaringError] = dict(listed.errors) for collection, result in zip(listed.items, results, strict=True): + if isinstance(result, PartialResultError): + out.items.extend(result.results) if isinstance(result, CalendaringError): errors[collection.id] = result else: diff --git a/src/calendaring/capabilities.py b/src/calendaring/capabilities.py index 93a6edc..72de38f 100644 --- a/src/calendaring/capabilities.py +++ b/src/calendaring/capabilities.py @@ -57,6 +57,9 @@ class Feature(StrEnum): TASK_RELATIONS_DEPENDS_ON = "task.relations.depends-on" CATEGORIES = "categories" ATTENDEES = "attendees" + ALARMS = "alarms" + TIMEZONES = "timezones" + DATES = "dates" # Declared on the backend rather than on a collection (§5.2). diff --git a/src/calendaring/errors.py b/src/calendaring/errors.py index a557dc9..0f0c770 100644 --- a/src/calendaring/errors.py +++ b/src/calendaring/errors.py @@ -8,11 +8,12 @@ from __future__ import annotations from collections.abc import Sequence -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal if TYPE_CHECKING: from .capabilities import Feature from .loss import Loss + from .types import BrokenObject __all__ = [ "AlreadyExistsError", @@ -28,6 +29,7 @@ "LossyWriteError", "LossyWriteWarning", "NotFoundError", + "PartialResultError", "RateLimitError", "SyncTokenError", "TransportError", @@ -131,6 +133,32 @@ class InvalidDataError(CalendaringError, ValueError): """The backend rejected the data, or it does not parse.""" +class PartialResultError(InvalidDataError): + """A read that skipped stored objects it could not use; ``results`` is what it got from the rest. + + ``results`` is what the call returns otherwise (a list of items, or the + ``ChangeSet`` for ``changes()``, whose token is good to pass on), and + ``broken`` the objects left out. A broken object is not reported as + deleted. + """ + + def __init__( + self, + results: Any, + broken: Sequence[BrokenObject], + message: str = "", + *, + backend: str | None = None, + collection_id: str | None = None, + ) -> None: + text = message or f"{len(broken)} stored object(s) left out: " + ", ".join( + f"{b.native_id!r} ({b.error})" for b in broken + ) + super().__init__(text, backend=backend, collection_id=collection_id) + self.results = results + self.broken = list(broken) + + class RateLimitError(CalendaringError): """The backend asked us to slow down. Raised, never retried here.""" diff --git a/src/calendaring/types.py b/src/calendaring/types.py index a2e2b9b..eb300d4 100644 --- a/src/calendaring/types.py +++ b/src/calendaring/types.py @@ -10,9 +10,9 @@ if TYPE_CHECKING: import icalendar - from .errors import CalendaringError + from .errors import CalendaringError, InvalidDataError -__all__ = ["ChangeSet", "Component", "MultiResult", "Scope", "StoredObject", "SyncToken"] +__all__ = ["BrokenObject", "ChangeSet", "Component", "MultiResult", "Scope", "StoredObject", "SyncToken"] T = TypeVar("T") @@ -74,3 +74,19 @@ class StoredObject: native: object | None = None native_status: str | None = None native_priority: str | None = None + + +@dataclass +class BrokenObject: + """A stored object a listing could not use: it does not parse, or the searcher cannot match it. + + Listing primitives return one in place of a ``StoredObject`` instead of + raising, so the rest of the collection stays readable; the generic layer + then raises ``PartialResultError``. ``wire`` is the bytes as read, where + the backend has them, to repair the object from. + """ + + native_id: str + etag: str | None + wire: bytes | None + error: InvalidDataError diff --git a/tests/_async/test_generic.py b/tests/_async/test_generic.py index a320f42..0d111dd 100644 --- a/tests/_async/test_generic.py +++ b/tests/_async/test_generic.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedelta, timezone import pytest +from icalendar_searcher import Searcher from calendaring import ( AsyncBackend, @@ -13,19 +14,26 @@ AsyncTask, AsyncWorkspace, BackendError, + BrokenObject, + Component, ConfigurationError, + ConflictError, Event, Feature, + InvalidDataError, LossyWriteWarning, MultiResult, NotFoundError, + PartialResultError, Support, + SyncTokenError, Task, UnsupportedError, VerificationError, ) from calendaring.loss import WARN from calendaring.memory import TRACKER, AsyncMemoryBackend, AsyncMemoryCollection, MemoryObject +from calendaring.types import StoredObject def at(day: int, hour: int = 12) -> datetime: @@ -193,3 +201,176 @@ async def test_workspace_defaults_stay_the_workspace_s() -> None: assert backend.verify is None, "the backend's own setting is untouched" with pytest.raises(VerificationError): await (await backend.collection("default")).add_task(summary="x", categories=["a"]) + + +async def test_move_between_backends_checks_the_etag_before_copying() -> None: + source = await AsyncMemoryBackend(id="source").collection("default") + target = await AsyncMemoryBackend(TRACKER, id="tracker").collection("default") + task = await source.add_task(summary="file taxes") + other = await source.get(task.uid) + other.component["SUMMARY"] = "file taxes (by 31 Oct)" + await other.save() + for _ in range(2): + with pytest.raises(ConflictError): + await source.move(task, target) + assert await target.search() == [], "a move refused for a stale etag left a copy behind" + + +class LosesTheRace(AsyncMemoryCollection): + """Another client edits the item between the copy and the delete.""" + + async def _delete(self, native_id: str, *, etag: str | None) -> None: + raise ConflictError(f"{native_id!r} changed since it was read") + + +async def test_a_move_that_loses_the_race_removes_its_copy() -> None: + class Backend(AsyncMemoryBackend): + collection_class = LosesTheRace + + source = await Backend(id="source").collection("default") + target = await AsyncMemoryBackend(id="target").collection("default") + task = await source.add_task(summary="x") + with pytest.raises(ConflictError): + await source.move(task, target) + assert await target.search() == [] + + +async def test_a_token_from_another_backend_is_refused() -> None: + work = await AsyncMemoryBackend(id="work").collection("default") + home = await AsyncMemoryBackend(id="home").collection("default") + for summary in ("w1", "w2"): + await work.add_task(summary=summary) + token = (await work.changes()).token + with pytest.raises(SyncTokenError): + await home.changes(token) + twin = await AsyncMemoryBackend(id="work").collection("default") + with pytest.raises(SyncTokenError): + await twin.changes(token) + + +BAD_TODO = ( # DUE:2026-10-20 is not a valid DATE; some client wrote it + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//some client//EN\r\nBEGIN:VTODO\r\nUID:from-phone\r\n" + b"DTSTAMP:20261001T000000Z\r\nSUMMARY:buy milk\r\nDUE:2026-10-20\r\nEND:VTODO\r\nEND:VCALENDAR\r\n" +) + +TWO_MASTERS = ( # parses, but is no recurrence set: the searcher cannot match it + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\n" + b"BEGIN:VEVENT\r\nUID:twice\r\nDTSTAMP:20261001T000000Z\r\nDTSTART:20261105T090000Z\r\n" + b"RRULE:FREQ=DAILY;COUNT=2\r\nEND:VEVENT\r\n" + b"BEGIN:VEVENT\r\nUID:twice\r\nDTSTAMP:20261001T000000Z\r\nDTSTART:20261106T090000Z\r\n" + b"RRULE:FREQ=DAILY;COUNT=2\r\nEND:VEVENT\r\n" + b"END:VCALENDAR\r\n" +) + + +async def test_an_object_that_does_not_parse_gives_partial_results() -> None: + collection = await AsyncMemoryBackend().collection("default") + assert isinstance(collection, AsyncMemoryCollection) + good = await collection.add_task(summary="good task") + bad = collection.store.put(BAD_TODO) + for call in (collection.search, collection.tasks): + with pytest.raises(PartialResultError) as info: + await call() + assert [i.uid for i in info.value.results] == [good.uid] + assert [(b.native_id, b.wire) for b in info.value.broken] == [(bad, BAD_TODO)] + assert info.value.collection_id == "default" + streamed = [] + + async def drain() -> None: + async for item in collection.iter_search(): + streamed.append(item) + + with pytest.raises(PartialResultError) as info: + await drain() + assert [i.uid for i in streamed] == [good.uid] + assert info.value.results == [], "iter_search() already gave them; it keeps none" + assert (await collection.get(good.uid)).native_id == good.native_id + with pytest.raises(PartialResultError): + await collection.get("not-there") + with pytest.raises(PartialResultError) as info: + await collection.changes() + assert [i.uid for i in info.value.results.changed] == [good.uid] + assert info.value.results.deleted == [] + [broken] = info.value.broken + assert (broken.native_id, broken.etag, broken.wire) == (bad, collection.store.objects[bad].etag, BAD_TODO) + quiet = await collection.changes(info.value.results.token) + assert (quiet.changed, quiet.deleted) == ([], []) + with pytest.raises(InvalidDataError): + await collection.get_by_native_id(bad) + await collection.add_task(summary="added after") # the store still checks UIDs past the broken one + + +class ListsWithoutEtags(AsyncMemoryCollection): + """Lists a broken object without its etag, as a backend that reads etags from the content would.""" + + async def _query(self, searcher: Searcher | None) -> list[StoredObject | BrokenObject]: + out = list(await super()._query(searcher)) + return [BrokenObject(s.native_id, None, s.wire, s.error) if isinstance(s, BrokenObject) else s for s in out] + + _list_etags = AsyncCollection._list_etags + + +async def test_a_broken_object_is_never_reported_deleted() -> None: + class Backend(AsyncMemoryBackend): + collection_class = ListsWithoutEtags + + collection = await Backend().collection("default") + assert isinstance(collection, AsyncMemoryCollection) + await collection.add_task(summary="good") + bad = collection.store.put(BAD_TODO) + with pytest.raises(PartialResultError) as info: + await collection.changes() + with pytest.raises(PartialResultError) as again: + await collection.changes(known={bad: "an etag from before"}) + assert again.value.results.deleted == [] + assert bad not in (await collection.changes(info.value.results.token)).deleted + + +async def test_relatives_get_past_a_broken_object() -> None: + backend = AsyncMemoryBackend() + collection = await backend.collection("default") + other = await backend.create_collection("other", components=[Component.TASK]) + assert isinstance(collection, AsyncMemoryCollection) + parent = await other.add_task(summary="parent") + child = await collection.add_task(summary="child", related_to=parent.uid) + collection.store.put(BAD_TODO) + with pytest.raises(PartialResultError) as info: + await child.relatives() + assert [r.uid for r in info.value.results] == [parent.uid] + + +async def test_an_object_the_searcher_cannot_match_gives_partial_results() -> None: + work = AsyncMemoryBackend(id="work") + home = AsyncMemoryBackend(id="home") + collection = await work.collection("default") + assert isinstance(collection, AsyncMemoryCollection) + await collection.add_event(summary="unrelated", start=at(6)) + [home_default] = await home.collections() + await home_default.delete_collection() # so that only work's collection is "default" + await (await home.create_collection("personal")).add_event(summary="dentist", start=at(7)) + odd = collection.store.put(TWO_MASTERS) + with pytest.raises(PartialResultError) as info: + await collection.events() + assert [str(i.component["SUMMARY"]) for i in info.value.results] == ["unrelated"] + assert [b.native_id for b in info.value.broken] == [odd] + found = await AsyncWorkspace([work, home]).search() + assert sorted(str(i.component["SUMMARY"]) for i in found.items) == ["dentist", "unrelated"] + assert isinstance(found.errors["default"], PartialResultError) + + +class DefaultTokenScope(AsyncMemoryCollection): + def _token_scope(self) -> str: + return AsyncCollection._token_scope(self) + + +async def test_the_default_token_scope_is_kind_backend_and_collection() -> None: + class Backend(AsyncMemoryBackend): + collection_class = DefaultTokenScope + + work = await Backend(id="work").collection("default") + assert work._token_scope() == "memory:work:default" + token = (await work.changes()).token + with pytest.raises(SyncTokenError): + await (await Backend(id="home").collection("default")).changes(token) + # Another store under the same label: the default cannot tell them apart. + await (await Backend(id="work").collection("default")).changes(token) diff --git a/tests/_async/test_memory_conformance.py b/tests/_async/test_memory_conformance.py index 3dfcd69..46dbedc 100644 --- a/tests/_async/test_memory_conformance.py +++ b/tests/_async/test_memory_conformance.py @@ -1,21 +1,63 @@ # Hand-written source; tools/generate_sync.py derives ../_sync/ from this file. -"""The in-memory reference backend passes the conformance suite, in four profiles. - -A backend that can do everything, one shaped like an issue tracker (tasks -only, mapped statuses and priorities, assigned identity), a writable one -without most optional features, and a read-only feed. Between them they run -both branches of the capability-aware tests, except the supported branch of -``recurrence.edit-occurrence`` and ``task.complete-recurring``, which wait for -the merge in recurring-ical-events (API_DESIGN.md Q3). +"""The in-memory reference backend passes the conformance suite, in seven profiles. + +The library's four: a backend that can do everything, one shaped like an +issue tracker (tasks only, mapped statuses and priorities, assigned +identity), a writable one without most optional features, and a read-only +feed. Three more here reach the branches those leave out; CI checks the +branch coverage of the suite. The supported branches of +``recurrence.edit-occurrence``, ``recurrence.edit-this-and-future`` and +``task.complete-recurring`` wait for the merge in recurring-ical-events +(API_DESIGN.md Q3). + +The second half is backends that each break one rule, and the suite test +that must catch it: a test that cannot fail tests nothing. """ from __future__ import annotations -from collections.abc import Sequence +import copy +import inspect +from collections.abc import AsyncIterator, Mapping, Sequence +from datetime import UTC, date, datetime, time +from typing import Any, ClassVar -from calendaring import AsyncBackend, AsyncCollection, AsyncItem, Item, Task -from calendaring.memory import FEED, FULL, MINIMAL, TRACKER, AsyncMemoryBackend +import icalendar +import pytest +from icalendar_searcher import Searcher + +from calendaring import ( + AsyncBackend, + AsyncCollection, + AsyncItem, + AsyncTask, + BackendError, + BrokenObject, + Component, + Feature, + Item, + NotFoundError, + Support, + SyncToken, + Task, + VerificationError, +) +from calendaring._async.conformance import snapshot +from calendaring.loss import LossPolicy +from calendaring.memory import ( + FEED, + FULL, + MINIMAL, + TRACKER, + AsyncMemoryBackend, + AsyncMemoryCollection, + MemoryCalendar, + MemoryObject, + MemoryStore, + Profile, +) from calendaring.testing import AsyncConformance +from calendaring.types import StoredObject class TestFullProfile(AsyncConformance): @@ -48,9 +90,440 @@ async def test_the_borrowed_collection_is_left_as_it_was(self) -> None: class TestFeedProfile(AsyncConformance): async def open_backend(self) -> AsyncBackend: - return AsyncMemoryBackend(profile=FEED) + backend = AsyncMemoryBackend(profile=FEED) + # A feed is read-only as a whole, which a backend may say as well (§5.2). + backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.UNSUPPORTED}) + return backend async def seed(self, collection: AsyncCollection, items: Sequence[Item]) -> list[AsyncItem]: assert isinstance(collection, AsyncMemoryBackend.collection_class) native_ids = [collection.store.put(item.to_ical()) for item in items] return [await collection.get_by_native_id(n) for n in native_ids] + + async def unseed(self, collection: AsyncCollection, item: AsyncItem) -> None: + assert isinstance(collection, AsyncMemoryBackend.collection_class) + assert item.native_id is not None + del collection.store.objects[item.native_id] + + +# Three more profiles, for the branches the four above leave out. +EVENTS_ONLY = Profile( + "events-only", + { + **FULL.capabilities, + Feature.COMPONENT_TASK: Support.UNSUPPORTED, + Feature.COMPONENT_JOURNAL: Support.UNSUPPORTED, + Feature.CREATE_COLLECTION: Support.UNSUPPORTED, + Feature.DELETE_COLLECTION: Support.UNSUPPORTED, + Feature.CHANGES_KNOWN: Support.UNSUPPORTED, + }, + frozenset({Component.EVENT}), +) + +PLAIN_TASKS = Profile( + "plain-tasks", + { + **TRACKER.capabilities, + Feature.TASK_STATUS: Support.UNSUPPORTED, + Feature.TASK_PRIORITY: {"level": Support.LOSSY}, # mapped by rules of its own + Feature.CATEGORIES: {"level": Support.LOSSY}, + Feature.IDENTITY_CLIENT_NATIVE_ID: Support.FULL, + Feature.CREATE_COLLECTION: Support.UNKNOWN, + }, + frozenset({Component.TASK}), +) + +TASK_FEED = Profile( + "task-feed", + { + **FEED.capabilities, + Feature.COMPONENT_EVENT: Support.UNSUPPORTED, + Feature.COMPONENT_JOURNAL: Support.UNSUPPORTED, + Feature.RECURRENCE: Support.UNSUPPORTED, + }, + frozenset({Component.TASK}), +) + + +class TestEventsOnlyProfile(AsyncConformance): + """Writable without tasks; a change feed without known=; move, but no second collection.""" + + async def open_backend(self) -> AsyncBackend: + return AsyncMemoryBackend(profile=EVENTS_ONLY) + + +class CannotCreate(AsyncMemoryBackend): + async def _create_collection(self, name: str, components: frozenset[Component]) -> AsyncCollection: + raise BackendError("this server turned out not to make collections") + + +class TestPlainTasksProfile(AsyncConformance): + """No status, priority and categories mapped by its own rules, and an unprobed create-collection.""" + + allow_unknown = True + + async def open_backend(self) -> AsyncBackend: + return CannotCreate(profile=PLAIN_TASKS) + + +class TestTaskFeedProfile(TestFeedProfile): + """A read-only collection of tasks, none recurring.""" + + async def open_backend(self) -> AsyncBackend: + return AsyncMemoryBackend(profile=TASK_FEED) + + +def sharing(profile: Profile, store: MemoryStore | None = None) -> AsyncMemoryBackend: + """A backend on ``store``, like a second connection to a server that keeps its data.""" + backend = AsyncMemoryBackend(profile=profile) + if store is not None: + backend.store = backend.native = store + return backend + + +class TestTidyingUp: + """What the suite leaves behind on a backend that keeps data between tests.""" + + def suite(self) -> AsyncConformance: + suite = AsyncConformance() + suite._made, suite._borrowed = [], [] + return suite + + async def test_a_concurrent_write_to_a_borrowed_collection_survives(self) -> None: + backend = sharing(MINIMAL) + suite = self.suite() + collection = await suite.open_collection(backend) + await collection.add_task(summary="the suite's") + [shared] = await sharing(MINIMAL, backend.store).collections() + await shared.add_task(summary="the user's, from their phone") + await suite.tidy_up(backend) + assert [str(i.component["SUMMARY"]) for i in await shared.search()] == ["the user's, from their phone"] + + async def test_a_broken_object_in_a_borrowed_collection_is_left_alone(self) -> None: + backend = sharing(MINIMAL) + bad = backend.store.calendars["default"].put( + b"BEGIN:VCALENDAR\r\nBEGIN:VTODO\r\nDUE:2026-10-20\r\nEND:VTODO\r\n" + ) + suite = self.suite() + collection = await suite.open_collection(backend) + assert await snapshot(collection) == [] + await collection.add_task(summary="the suite's") + await suite.tidy_up(backend) + assert list(backend.store.calendars["default"].objects) == [bad] + + async def test_a_borrowed_collection_keeps_its_name(self) -> None: + backend = sharing(MINIMAL) + suite = self.suite() + await suite.test_set_name(backend, await suite.open_collection(backend)) + await suite.tidy_up(backend) + assert (await backend.collection("default")).name == "Default" + + async def test_without_delete_collection_one_collection_is_reused(self) -> None: + store = sharing(TRACKER).store + for _ in range(3): + backend = sharing(TRACKER, store) + suite = self.suite() + collection = await suite.open_collection(backend) + await collection.add_task(summary="the suite's") + await suite.test_create_and_delete_collection(backend) + await suite.tidy_up(backend) + assert sorted(store.calendars) == ["conformance", "conformance-new", "default"] + assert [len(c.objects) for c in store.calendars.values()] == [0, 0, 0] + + +# Backends that each break one rule, and the suite test that must catch it. + + +def _listed(o: MemoryObject, calendar: icalendar.Calendar | None = None) -> StoredObject: + return StoredObject(o.native_id, o.etag, calendar if calendar is not None else o.calendar, o.data, o) + + +class FirstPageOnly(AsyncMemoryCollection): + """Lists through a paged API and never asks for the second page.""" + + def _page(self) -> list[MemoryObject]: + return list(self.store.objects.values())[:50] + + async def _query(self, searcher: Searcher | None) -> list[StoredObject]: + return [_listed(o) for o in self._page()] + + async def _list_etags(self) -> dict[str, str]: + return {o.native_id: o.etag for o in self._page()} + + +class MatchesUidSubstrings(AsyncMemoryCollection): + """Looks a UID up with a server text-match, newest first.""" + + async def _find_uid(self, uid: str) -> StoredObject: + for o in reversed(list(self.store.objects.values())): + if uid in o.uid: + return _listed(o) + raise NotFoundError(f"no item with UID {uid!r}") + + +class SearchesTheAccount(AsyncMemoryCollection): + """Looks a UID up in every collection of the account.""" + + async def _find_uid(self, uid: str) -> StoredObject: + for calendar in self._backend.store.calendars.values(): + for o in calendar.objects.values(): + if o.uid == uid: + return _listed(o) + raise NotFoundError(f"no item with UID {uid!r}") + + +class ReplacesEventUids(AsyncMemoryCollection): + """Gives every event a UID of its own, while declaring identity.client-uid.""" + + async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + for component in calendar.subcomponents: + if component.name == "VEVENT": + component["UID"] = f"server-{self._backend.store.next()}" + return await super()._create(calendar, native_id=native_id, wire=None) + + +class StaleListing(AsyncMemoryCollection): + """Lists the content it first saw under each object's current etag.""" + + seen: ClassVar[dict[str, icalendar.Calendar]] = {} + + async def _query(self, searcher: Searcher | None) -> list[StoredObject]: + cached = [self.seen.setdefault(o.native_id, o.calendar) for o in self.store.objects.values()] + return [_listed(o, copy.deepcopy(c)) for o, c in zip(self.store.objects.values(), cached, strict=True)] + + +class ListViewStream(AsyncMemoryCollection): + """Streams the fields of a list view: no DESCRIPTION, no CATEGORIES.""" + + async def _iter_query(self, searcher: Searcher | None) -> AsyncIterator[StoredObject]: + for o in self.store.objects.values(): + calendar = o.calendar + for component in calendar.subcomponents: + for name in ("DESCRIPTION", "CATEGORIES"): + component.pop(name, None) + yield _listed(o, calendar) + + +class WordIndex(AsyncMemoryCollection): + """Narrows a text search with a word index, so "rain" misses "Training".""" + + async def _query(self, searcher: Searcher | None) -> list[StoredObject]: + wanted = {str(v) for v in getattr(searcher, "_property_filters", {}).values()} + out = [] + for o in self.store.objects.values(): + words = {w for c in o.calendar.subcomponents for w in str(c.get("SUMMARY", "")).split()} + if not wanted or wanted & words: + out.append(_listed(o)) + return out + + +class NoDeletionsFeed(AsyncMemoryCollection): + """A feed whose change listing never drops an id it has seen.""" + + seen: ClassVar[dict[str, str]] = {} + + async def _list_etags(self) -> dict[str, str]: + self.seen.update(await super()._list_etags()) + return dict(self.seen) + + +class FrozenFeed(AsyncMemoryCollection): + """Serves its first download forever.""" + + first: list[StoredObject] | None = None + + async def _query(self, searcher: Searcher | None) -> list[StoredObject]: + if FrozenFeed.first is None and self.store.objects: # the first download that has something in it + FrozenFeed.first = [_listed(o) for o in self.store.objects.values()] + return FrozenFeed.first or [] + + async def _list_etags(self) -> dict[str, str]: + return {s.native_id: s.etag for s in await self._query(None)} + + +class LeakyChangeFeed(AsyncMemoryCollection): + """Reports changes in every collection of the account.""" + + async def _changes( + self, token: SyncToken | None, known: Mapping[str, str] | None + ) -> tuple[Sequence[StoredObject | BrokenObject], list[str], SyncToken]: + changed, deleted, new_token = await super()._changes(token, known) + others = [ + _listed(o) + for c in self._backend.store.calendars.values() + if c is not self.store + for o in c.objects.values() + ] + return [*changed, *others], deleted, new_token + + +class Mangling(AsyncMemoryCollection): + """Stores a mangled copy of whatever it is given, declaring that it loses nothing.""" + + def mangle(self, component: icalendar.Component) -> None: + raise NotImplementedError + + def _mangled(self, calendar: icalendar.Calendar, wire: bytes | None) -> icalendar.Calendar: + out = icalendar.Calendar.from_ical(wire) if wire is not None else copy.deepcopy(calendar) + for component in out.subcomponents: + if component.name in ("VEVENT", "VTODO", "VJOURNAL"): + self.mangle(component) + return out + + async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + return await super()._create(self._mangled(calendar, wire), native_id=native_id, wire=None) + + +class DropsAlarms(Mangling): + def mangle(self, component: icalendar.Component) -> None: + component.subcomponents[:] = [c for c in component.subcomponents if c.name != "VALARM"] + + +class AsciiOnly(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("SUMMARY", "DESCRIPTION", "LOCATION"): + if name in component: + text = "".join(ch if ord(ch) < 128 else "?" for ch in str(component.pop(name))) + component.add(name, text) + + +class FlattensNewlines(Mangling): + def mangle(self, component: icalendar.Component) -> None: + if "DESCRIPTION" in component: + component["DESCRIPTION"] = icalendar.vText(str(component["DESCRIPTION"]).replace("\n", " ")) + + +class StoresUtc(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("DTSTART", "DTEND"): + value = component.decoded(name, None) + if isinstance(value, datetime): + component.pop(name) + component.add(name, value.astimezone(UTC)) + + +class DatesAsDateTimes(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("DTSTART", "DTEND", "DUE"): + value = component.decoded(name, None) + if isinstance(value, date) and not isinstance(value, datetime): + component.pop(name) + component.add(name, datetime.combine(value, time(), UTC)) + + +class DropsEventDetails(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("LOCATION", "URL", "EXDATE", "X-CONFORMANCE-COLOUR"): + component.pop(name, None) + + +class DropsOverrides(AsyncMemoryCollection): + async def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + calendar = copy.deepcopy(calendar) + calendar.subcomponents[:] = [c for c in calendar.subcomponents if "RECURRENCE-ID" not in c] + return await super()._create(calendar, native_id=native_id, wire=None) + + +class FirstValueOnly(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("CATEGORIES", "ATTENDEE"): + values = component.get(name) + if isinstance(values, list): + component[name] = values[0] + + +class StartsInRange(AsyncMemoryCollection): + """Narrows a time range by DTSTART alone, and leaves tasks out of it.""" + + async def _query(self, searcher: Searcher | None) -> list[StoredObject]: + start, end = getattr(searcher, "start", None), getattr(searcher, "end", None) + out = [] + for o in self.store.objects.values(): + components = [c for c in o.calendar.subcomponents if c.name in ("VEVENT", "VTODO")] + if start is None or any( + c.name == "VEVENT" and start <= c.decoded("DTSTART") < end for c in components if "DTSTART" in c + ): + out.append(_listed(o)) + return out + + +class CompletesNothing(AsyncMemoryCollection): + """Declares task.complete-recurring and writes nothing.""" + + def __init__(self, backend: AsyncMemoryBackend, store: MemoryCalendar) -> None: + super().__init__(backend, store) + self.capabilities = self.capabilities.updated({Feature.TASK_COMPLETE_RECURRING: Support.FULL}) + + async def _complete_recurring( + self, task: Task, at: datetime | None, mode: str, *, loss: LossPolicy | None, verify: bool | None + ) -> AsyncTask: + return self.bind(task) + + +# What the suite raises when it catches a defect: its own assertion, or a +# pytest.raises that did not raise. The round trip catches through verify=True. +ASSERTS = (AssertionError, pytest.fail.Exception) + +CAUGHT: list[tuple[type[AsyncMemoryCollection], str, dict[str, Any], Any]] = [ + (FirstPageOnly, "test_listings_are_not_cut_at_a_page", {}, ASSERTS), + (MatchesUidSubstrings, "test_get_finds_the_exact_uid", {}, ASSERTS), + (SearchesTheAccount, "test_get_finds_the_exact_uid", {}, ASSERTS), + (ReplacesEventUids, "test_get_finds_the_exact_uid", {}, ASSERTS), + (StaleListing, "test_a_listing_shows_a_save_made_elsewhere", {}, ASSERTS), + (ListViewStream, "test_iter_search_yields_what_search_returns", {}, ASSERTS), + (WordIndex, "test_search_gives_the_client_side_answer", {"name": "summary-contains"}, ASSERTS), + (FrozenFeed, "test_read_paths_on_seeded_items", {}, ASSERTS), + (NoDeletionsFeed, "test_read_paths_on_seeded_items", {}, ASSERTS), + (LeakyChangeFeed, "test_changes_follow_a_move", {}, ASSERTS), + (DropsAlarms, "test_real_world_objects_round_trip", {"name": "event-series-in-a-time-zone"}, VerificationError), + (DropsAlarms, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + (AsciiOnly, "test_real_world_objects_round_trip", {"name": "journal-entry"}, VerificationError), + (FlattensNewlines, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + (StoresUtc, "test_real_world_objects_round_trip", {"name": "event-series-in-a-time-zone"}, VerificationError), + (StoresUtc, "test_real_world_objects_round_trip", {"name": "floating-event"}, VerificationError), + (DatesAsDateTimes, "test_real_world_objects_round_trip", {"name": "all-day-event-with-rdate"}, VerificationError), + (DatesAsDateTimes, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + ( + DropsEventDetails, + "test_real_world_objects_round_trip", + {"name": "event-series-in-a-time-zone"}, + VerificationError, + ), + (DropsOverrides, "test_real_world_objects_round_trip", {"name": "event-series-in-a-time-zone"}, VerificationError), + (FirstValueOnly, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + (StartsInRange, "test_search_gives_the_client_side_answer", {"name": "events-in-range"}, ASSERTS), + (StartsInRange, "test_search_gives_the_client_side_answer", {"name": "tasks-in-range"}, ASSERTS), + (CompletesNothing, "test_complete_a_recurring_task", {}, ASSERTS), +] + + +@pytest.mark.parametrize( + ("broken", "test", "params", "expected"), + CAUGHT, + ids=[f"{c[0].__name__}-{c[2].get('name', '')}".rstrip("-") for c in CAUGHT], +) +async def test_the_suite_catches( + broken: type[AsyncMemoryCollection], test: str, params: dict[str, Any], expected: Any +) -> None: + class Backend(AsyncMemoryBackend): + collection_class = broken + + StaleListing.seen.clear() + NoDeletionsFeed.seen.clear() + FrozenFeed.first = None + profile, suite = (FEED, TestFeedProfile()) if broken in (FrozenFeed, NoDeletionsFeed) else (FULL, TestFullProfile()) + suite._made, suite._borrowed = [], [] + backend = Backend(profile=profile) + collection = await suite.open_collection(backend) + method = getattr(suite, test) + wanted = inspect.signature(method).parameters + fixtures = {"backend": backend, "collection": collection} + with pytest.raises(expected): + await method(**{k: v for k, v in fixtures.items() if k in wanted}, **params) + + +async def test_the_suite_catches_a_collection_wider_than_its_backend() -> None: + backend = AsyncMemoryBackend(profile=FULL) + backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.UNSUPPORTED}) + with pytest.raises(AssertionError): + await TestFullProfile().test_collection_never_widens_backend(backend, await backend.collection("default")) diff --git a/tests/_sync/test_generic.py b/tests/_sync/test_generic.py index 9e19f6a..821235d 100644 --- a/tests/_sync/test_generic.py +++ b/tests/_sync/test_generic.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedelta, timezone import pytest +from icalendar_searcher import Searcher from calendaring import ( Backend, @@ -13,19 +14,26 @@ SyncTask, Workspace, BackendError, + BrokenObject, + Component, ConfigurationError, + ConflictError, Event, Feature, + InvalidDataError, LossyWriteWarning, MultiResult, NotFoundError, + PartialResultError, Support, + SyncTokenError, Task, UnsupportedError, VerificationError, ) from calendaring.loss import WARN from calendaring.memory import TRACKER, MemoryBackend, MemoryCollection, MemoryObject +from calendaring.types import StoredObject def at(day: int, hour: int = 12) -> datetime: @@ -193,3 +201,176 @@ def test_workspace_defaults_stay_the_workspace_s() -> None: assert backend.verify is None, "the backend's own setting is untouched" with pytest.raises(VerificationError): (backend.collection("default")).add_task(summary="x", categories=["a"]) + + +def test_move_between_backends_checks_the_etag_before_copying() -> None: + source = MemoryBackend(id="source").collection("default") + target = MemoryBackend(TRACKER, id="tracker").collection("default") + task = source.add_task(summary="file taxes") + other = source.get(task.uid) + other.component["SUMMARY"] = "file taxes (by 31 Oct)" + other.save() + for _ in range(2): + with pytest.raises(ConflictError): + source.move(task, target) + assert target.search() == [], "a move refused for a stale etag left a copy behind" + + +class LosesTheRace(MemoryCollection): + """Another client edits the item between the copy and the delete.""" + + def _delete(self, native_id: str, *, etag: str | None) -> None: + raise ConflictError(f"{native_id!r} changed since it was read") + + +def test_a_move_that_loses_the_race_removes_its_copy() -> None: + class Backend(MemoryBackend): + collection_class = LosesTheRace + + source = Backend(id="source").collection("default") + target = MemoryBackend(id="target").collection("default") + task = source.add_task(summary="x") + with pytest.raises(ConflictError): + source.move(task, target) + assert target.search() == [] + + +def test_a_token_from_another_backend_is_refused() -> None: + work = MemoryBackend(id="work").collection("default") + home = MemoryBackend(id="home").collection("default") + for summary in ("w1", "w2"): + work.add_task(summary=summary) + token = (work.changes()).token + with pytest.raises(SyncTokenError): + home.changes(token) + twin = MemoryBackend(id="work").collection("default") + with pytest.raises(SyncTokenError): + twin.changes(token) + + +BAD_TODO = ( # DUE:2026-10-20 is not a valid DATE; some client wrote it + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//some client//EN\r\nBEGIN:VTODO\r\nUID:from-phone\r\n" + b"DTSTAMP:20261001T000000Z\r\nSUMMARY:buy milk\r\nDUE:2026-10-20\r\nEND:VTODO\r\nEND:VCALENDAR\r\n" +) + +TWO_MASTERS = ( # parses, but is no recurrence set: the searcher cannot match it + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\n" + b"BEGIN:VEVENT\r\nUID:twice\r\nDTSTAMP:20261001T000000Z\r\nDTSTART:20261105T090000Z\r\n" + b"RRULE:FREQ=DAILY;COUNT=2\r\nEND:VEVENT\r\n" + b"BEGIN:VEVENT\r\nUID:twice\r\nDTSTAMP:20261001T000000Z\r\nDTSTART:20261106T090000Z\r\n" + b"RRULE:FREQ=DAILY;COUNT=2\r\nEND:VEVENT\r\n" + b"END:VCALENDAR\r\n" +) + + +def test_an_object_that_does_not_parse_gives_partial_results() -> None: + collection = MemoryBackend().collection("default") + assert isinstance(collection, MemoryCollection) + good = collection.add_task(summary="good task") + bad = collection.store.put(BAD_TODO) + for call in (collection.search, collection.tasks): + with pytest.raises(PartialResultError) as info: + call() + assert [i.uid for i in info.value.results] == [good.uid] + assert [(b.native_id, b.wire) for b in info.value.broken] == [(bad, BAD_TODO)] + assert info.value.collection_id == "default" + streamed = [] + + def drain() -> None: + for item in collection.iter_search(): + streamed.append(item) + + with pytest.raises(PartialResultError) as info: + drain() + assert [i.uid for i in streamed] == [good.uid] + assert info.value.results == [], "iter_search() already gave them; it keeps none" + assert (collection.get(good.uid)).native_id == good.native_id + with pytest.raises(PartialResultError): + collection.get("not-there") + with pytest.raises(PartialResultError) as info: + collection.changes() + assert [i.uid for i in info.value.results.changed] == [good.uid] + assert info.value.results.deleted == [] + [broken] = info.value.broken + assert (broken.native_id, broken.etag, broken.wire) == (bad, collection.store.objects[bad].etag, BAD_TODO) + quiet = collection.changes(info.value.results.token) + assert (quiet.changed, quiet.deleted) == ([], []) + with pytest.raises(InvalidDataError): + collection.get_by_native_id(bad) + collection.add_task(summary="added after") # the store still checks UIDs past the broken one + + +class ListsWithoutEtags(MemoryCollection): + """Lists a broken object without its etag, as a backend that reads etags from the content would.""" + + def _query(self, searcher: Searcher | None) -> list[StoredObject | BrokenObject]: + out = list(super()._query(searcher)) + return [BrokenObject(s.native_id, None, s.wire, s.error) if isinstance(s, BrokenObject) else s for s in out] + + _list_etags = Collection._list_etags + + +def test_a_broken_object_is_never_reported_deleted() -> None: + class Backend(MemoryBackend): + collection_class = ListsWithoutEtags + + collection = Backend().collection("default") + assert isinstance(collection, MemoryCollection) + collection.add_task(summary="good") + bad = collection.store.put(BAD_TODO) + with pytest.raises(PartialResultError) as info: + collection.changes() + with pytest.raises(PartialResultError) as again: + collection.changes(known={bad: "an etag from before"}) + assert again.value.results.deleted == [] + assert bad not in (collection.changes(info.value.results.token)).deleted + + +def test_relatives_get_past_a_broken_object() -> None: + backend = MemoryBackend() + collection = backend.collection("default") + other = backend.create_collection("other", components=[Component.TASK]) + assert isinstance(collection, MemoryCollection) + parent = other.add_task(summary="parent") + child = collection.add_task(summary="child", related_to=parent.uid) + collection.store.put(BAD_TODO) + with pytest.raises(PartialResultError) as info: + child.relatives() + assert [r.uid for r in info.value.results] == [parent.uid] + + +def test_an_object_the_searcher_cannot_match_gives_partial_results() -> None: + work = MemoryBackend(id="work") + home = MemoryBackend(id="home") + collection = work.collection("default") + assert isinstance(collection, MemoryCollection) + collection.add_event(summary="unrelated", start=at(6)) + [home_default] = home.collections() + home_default.delete_collection() # so that only work's collection is "default" + (home.create_collection("personal")).add_event(summary="dentist", start=at(7)) + odd = collection.store.put(TWO_MASTERS) + with pytest.raises(PartialResultError) as info: + collection.events() + assert [str(i.component["SUMMARY"]) for i in info.value.results] == ["unrelated"] + assert [b.native_id for b in info.value.broken] == [odd] + found = Workspace([work, home]).search() + assert sorted(str(i.component["SUMMARY"]) for i in found.items) == ["dentist", "unrelated"] + assert isinstance(found.errors["default"], PartialResultError) + + +class DefaultTokenScope(MemoryCollection): + def _token_scope(self) -> str: + return Collection._token_scope(self) + + +def test_the_default_token_scope_is_kind_backend_and_collection() -> None: + class Backend(MemoryBackend): + collection_class = DefaultTokenScope + + work = Backend(id="work").collection("default") + assert work._token_scope() == "memory:work:default" + token = (work.changes()).token + with pytest.raises(SyncTokenError): + (Backend(id="home").collection("default")).changes(token) + # Another store under the same label: the default cannot tell them apart. + (Backend(id="work").collection("default")).changes(token) diff --git a/tests/_sync/test_memory_conformance.py b/tests/_sync/test_memory_conformance.py index 30abd78..373c4c5 100644 --- a/tests/_sync/test_memory_conformance.py +++ b/tests/_sync/test_memory_conformance.py @@ -1,21 +1,63 @@ # Code generated by tools/generate_sync.py from _async/; DO NOT EDIT. -"""The in-memory reference backend passes the conformance suite, in four profiles. - -A backend that can do everything, one shaped like an issue tracker (tasks -only, mapped statuses and priorities, assigned identity), a writable one -without most optional features, and a read-only feed. Between them they run -both branches of the capability-aware tests, except the supported branch of -``recurrence.edit-occurrence`` and ``task.complete-recurring``, which wait for -the merge in recurring-ical-events (API_DESIGN.md Q3). +"""The in-memory reference backend passes the conformance suite, in seven profiles. + +The library's four: a backend that can do everything, one shaped like an +issue tracker (tasks only, mapped statuses and priorities, assigned +identity), a writable one without most optional features, and a read-only +feed. Three more here reach the branches those leave out; CI checks the +branch coverage of the suite. The supported branches of +``recurrence.edit-occurrence``, ``recurrence.edit-this-and-future`` and +``task.complete-recurring`` wait for the merge in recurring-ical-events +(API_DESIGN.md Q3). + +The second half is backends that each break one rule, and the suite test +that must catch it: a test that cannot fail tests nothing. """ from __future__ import annotations -from collections.abc import Sequence +import copy +import inspect +from collections.abc import Iterator, Mapping, Sequence +from datetime import UTC, date, datetime, time +from typing import Any, ClassVar -from calendaring import Backend, Collection, SyncItem, Item, Task -from calendaring.memory import FEED, FULL, MINIMAL, TRACKER, MemoryBackend +import icalendar +import pytest +from icalendar_searcher import Searcher + +from calendaring import ( + Backend, + Collection, + SyncItem, + SyncTask, + BackendError, + BrokenObject, + Component, + Feature, + Item, + NotFoundError, + Support, + SyncToken, + Task, + VerificationError, +) +from calendaring._sync.conformance import snapshot +from calendaring.loss import LossPolicy +from calendaring.memory import ( + FEED, + FULL, + MINIMAL, + TRACKER, + MemoryBackend, + MemoryCollection, + MemoryCalendar, + MemoryObject, + MemoryStore, + Profile, +) from calendaring.testing import SyncConformance +from calendaring.types import StoredObject class TestFullProfile(SyncConformance): @@ -48,9 +90,440 @@ def test_the_borrowed_collection_is_left_as_it_was(self) -> None: class TestFeedProfile(SyncConformance): def open_backend(self) -> Backend: - return MemoryBackend(profile=FEED) + backend = MemoryBackend(profile=FEED) + # A feed is read-only as a whole, which a backend may say as well (§5.2). + backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.UNSUPPORTED}) + return backend def seed(self, collection: Collection, items: Sequence[Item]) -> list[SyncItem]: assert isinstance(collection, MemoryBackend.collection_class) native_ids = [collection.store.put(item.to_ical()) for item in items] return [collection.get_by_native_id(n) for n in native_ids] + + def unseed(self, collection: Collection, item: SyncItem) -> None: + assert isinstance(collection, MemoryBackend.collection_class) + assert item.native_id is not None + del collection.store.objects[item.native_id] + + +# Three more profiles, for the branches the four above leave out. +EVENTS_ONLY = Profile( + "events-only", + { + **FULL.capabilities, + Feature.COMPONENT_TASK: Support.UNSUPPORTED, + Feature.COMPONENT_JOURNAL: Support.UNSUPPORTED, + Feature.CREATE_COLLECTION: Support.UNSUPPORTED, + Feature.DELETE_COLLECTION: Support.UNSUPPORTED, + Feature.CHANGES_KNOWN: Support.UNSUPPORTED, + }, + frozenset({Component.EVENT}), +) + +PLAIN_TASKS = Profile( + "plain-tasks", + { + **TRACKER.capabilities, + Feature.TASK_STATUS: Support.UNSUPPORTED, + Feature.TASK_PRIORITY: {"level": Support.LOSSY}, # mapped by rules of its own + Feature.CATEGORIES: {"level": Support.LOSSY}, + Feature.IDENTITY_CLIENT_NATIVE_ID: Support.FULL, + Feature.CREATE_COLLECTION: Support.UNKNOWN, + }, + frozenset({Component.TASK}), +) + +TASK_FEED = Profile( + "task-feed", + { + **FEED.capabilities, + Feature.COMPONENT_EVENT: Support.UNSUPPORTED, + Feature.COMPONENT_JOURNAL: Support.UNSUPPORTED, + Feature.RECURRENCE: Support.UNSUPPORTED, + }, + frozenset({Component.TASK}), +) + + +class TestEventsOnlyProfile(SyncConformance): + """Writable without tasks; a change feed without known=; move, but no second collection.""" + + def open_backend(self) -> Backend: + return MemoryBackend(profile=EVENTS_ONLY) + + +class CannotCreate(MemoryBackend): + def _create_collection(self, name: str, components: frozenset[Component]) -> Collection: + raise BackendError("this server turned out not to make collections") + + +class TestPlainTasksProfile(SyncConformance): + """No status, priority and categories mapped by its own rules, and an unprobed create-collection.""" + + allow_unknown = True + + def open_backend(self) -> Backend: + return CannotCreate(profile=PLAIN_TASKS) + + +class TestTaskFeedProfile(TestFeedProfile): + """A read-only collection of tasks, none recurring.""" + + def open_backend(self) -> Backend: + return MemoryBackend(profile=TASK_FEED) + + +def sharing(profile: Profile, store: MemoryStore | None = None) -> MemoryBackend: + """A backend on ``store``, like a second connection to a server that keeps its data.""" + backend = MemoryBackend(profile=profile) + if store is not None: + backend.store = backend.native = store + return backend + + +class TestTidyingUp: + """What the suite leaves behind on a backend that keeps data between tests.""" + + def suite(self) -> SyncConformance: + suite = SyncConformance() + suite._made, suite._borrowed = [], [] + return suite + + def test_a_concurrent_write_to_a_borrowed_collection_survives(self) -> None: + backend = sharing(MINIMAL) + suite = self.suite() + collection = suite.open_collection(backend) + collection.add_task(summary="the suite's") + [shared] = sharing(MINIMAL, backend.store).collections() + shared.add_task(summary="the user's, from their phone") + suite.tidy_up(backend) + assert [str(i.component["SUMMARY"]) for i in shared.search()] == ["the user's, from their phone"] + + def test_a_broken_object_in_a_borrowed_collection_is_left_alone(self) -> None: + backend = sharing(MINIMAL) + bad = backend.store.calendars["default"].put( + b"BEGIN:VCALENDAR\r\nBEGIN:VTODO\r\nDUE:2026-10-20\r\nEND:VTODO\r\n" + ) + suite = self.suite() + collection = suite.open_collection(backend) + assert snapshot(collection) == [] + collection.add_task(summary="the suite's") + suite.tidy_up(backend) + assert list(backend.store.calendars["default"].objects) == [bad] + + def test_a_borrowed_collection_keeps_its_name(self) -> None: + backend = sharing(MINIMAL) + suite = self.suite() + suite.test_set_name(backend, suite.open_collection(backend)) + suite.tidy_up(backend) + assert (backend.collection("default")).name == "Default" + + def test_without_delete_collection_one_collection_is_reused(self) -> None: + store = sharing(TRACKER).store + for _ in range(3): + backend = sharing(TRACKER, store) + suite = self.suite() + collection = suite.open_collection(backend) + collection.add_task(summary="the suite's") + suite.test_create_and_delete_collection(backend) + suite.tidy_up(backend) + assert sorted(store.calendars) == ["conformance", "conformance-new", "default"] + assert [len(c.objects) for c in store.calendars.values()] == [0, 0, 0] + + +# Backends that each break one rule, and the suite test that must catch it. + + +def _listed(o: MemoryObject, calendar: icalendar.Calendar | None = None) -> StoredObject: + return StoredObject(o.native_id, o.etag, calendar if calendar is not None else o.calendar, o.data, o) + + +class FirstPageOnly(MemoryCollection): + """Lists through a paged API and never asks for the second page.""" + + def _page(self) -> list[MemoryObject]: + return list(self.store.objects.values())[:50] + + def _query(self, searcher: Searcher | None) -> list[StoredObject]: + return [_listed(o) for o in self._page()] + + def _list_etags(self) -> dict[str, str]: + return {o.native_id: o.etag for o in self._page()} + + +class MatchesUidSubstrings(MemoryCollection): + """Looks a UID up with a server text-match, newest first.""" + + def _find_uid(self, uid: str) -> StoredObject: + for o in reversed(list(self.store.objects.values())): + if uid in o.uid: + return _listed(o) + raise NotFoundError(f"no item with UID {uid!r}") + + +class SearchesTheAccount(MemoryCollection): + """Looks a UID up in every collection of the account.""" + + def _find_uid(self, uid: str) -> StoredObject: + for calendar in self._backend.store.calendars.values(): + for o in calendar.objects.values(): + if o.uid == uid: + return _listed(o) + raise NotFoundError(f"no item with UID {uid!r}") + + +class ReplacesEventUids(MemoryCollection): + """Gives every event a UID of its own, while declaring identity.client-uid.""" + + def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + for component in calendar.subcomponents: + if component.name == "VEVENT": + component["UID"] = f"server-{self._backend.store.next()}" + return super()._create(calendar, native_id=native_id, wire=None) + + +class StaleListing(MemoryCollection): + """Lists the content it first saw under each object's current etag.""" + + seen: ClassVar[dict[str, icalendar.Calendar]] = {} + + def _query(self, searcher: Searcher | None) -> list[StoredObject]: + cached = [self.seen.setdefault(o.native_id, o.calendar) for o in self.store.objects.values()] + return [_listed(o, copy.deepcopy(c)) for o, c in zip(self.store.objects.values(), cached, strict=True)] + + +class ListViewStream(MemoryCollection): + """Streams the fields of a list view: no DESCRIPTION, no CATEGORIES.""" + + def _iter_query(self, searcher: Searcher | None) -> Iterator[StoredObject]: + for o in self.store.objects.values(): + calendar = o.calendar + for component in calendar.subcomponents: + for name in ("DESCRIPTION", "CATEGORIES"): + component.pop(name, None) + yield _listed(o, calendar) + + +class WordIndex(MemoryCollection): + """Narrows a text search with a word index, so "rain" misses "Training".""" + + def _query(self, searcher: Searcher | None) -> list[StoredObject]: + wanted = {str(v) for v in getattr(searcher, "_property_filters", {}).values()} + out = [] + for o in self.store.objects.values(): + words = {w for c in o.calendar.subcomponents for w in str(c.get("SUMMARY", "")).split()} + if not wanted or wanted & words: + out.append(_listed(o)) + return out + + +class NoDeletionsFeed(MemoryCollection): + """A feed whose change listing never drops an id it has seen.""" + + seen: ClassVar[dict[str, str]] = {} + + def _list_etags(self) -> dict[str, str]: + self.seen.update(super()._list_etags()) + return dict(self.seen) + + +class FrozenFeed(MemoryCollection): + """Serves its first download forever.""" + + first: list[StoredObject] | None = None + + def _query(self, searcher: Searcher | None) -> list[StoredObject]: + if FrozenFeed.first is None and self.store.objects: # the first download that has something in it + FrozenFeed.first = [_listed(o) for o in self.store.objects.values()] + return FrozenFeed.first or [] + + def _list_etags(self) -> dict[str, str]: + return {s.native_id: s.etag for s in self._query(None)} + + +class LeakyChangeFeed(MemoryCollection): + """Reports changes in every collection of the account.""" + + def _changes( + self, token: SyncToken | None, known: Mapping[str, str] | None + ) -> tuple[Sequence[StoredObject | BrokenObject], list[str], SyncToken]: + changed, deleted, new_token = super()._changes(token, known) + others = [ + _listed(o) + for c in self._backend.store.calendars.values() + if c is not self.store + for o in c.objects.values() + ] + return [*changed, *others], deleted, new_token + + +class Mangling(MemoryCollection): + """Stores a mangled copy of whatever it is given, declaring that it loses nothing.""" + + def mangle(self, component: icalendar.Component) -> None: + raise NotImplementedError + + def _mangled(self, calendar: icalendar.Calendar, wire: bytes | None) -> icalendar.Calendar: + out = icalendar.Calendar.from_ical(wire) if wire is not None else copy.deepcopy(calendar) + for component in out.subcomponents: + if component.name in ("VEVENT", "VTODO", "VJOURNAL"): + self.mangle(component) + return out + + def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + return super()._create(self._mangled(calendar, wire), native_id=native_id, wire=None) + + +class DropsAlarms(Mangling): + def mangle(self, component: icalendar.Component) -> None: + component.subcomponents[:] = [c for c in component.subcomponents if c.name != "VALARM"] + + +class AsciiOnly(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("SUMMARY", "DESCRIPTION", "LOCATION"): + if name in component: + text = "".join(ch if ord(ch) < 128 else "?" for ch in str(component.pop(name))) + component.add(name, text) + + +class FlattensNewlines(Mangling): + def mangle(self, component: icalendar.Component) -> None: + if "DESCRIPTION" in component: + component["DESCRIPTION"] = icalendar.vText(str(component["DESCRIPTION"]).replace("\n", " ")) + + +class StoresUtc(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("DTSTART", "DTEND"): + value = component.decoded(name, None) + if isinstance(value, datetime): + component.pop(name) + component.add(name, value.astimezone(UTC)) + + +class DatesAsDateTimes(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("DTSTART", "DTEND", "DUE"): + value = component.decoded(name, None) + if isinstance(value, date) and not isinstance(value, datetime): + component.pop(name) + component.add(name, datetime.combine(value, time(), UTC)) + + +class DropsEventDetails(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("LOCATION", "URL", "EXDATE", "X-CONFORMANCE-COLOUR"): + component.pop(name, None) + + +class DropsOverrides(MemoryCollection): + def _create(self, calendar: icalendar.Calendar, *, native_id: str | None, wire: bytes | None) -> StoredObject: + calendar = copy.deepcopy(calendar) + calendar.subcomponents[:] = [c for c in calendar.subcomponents if "RECURRENCE-ID" not in c] + return super()._create(calendar, native_id=native_id, wire=None) + + +class FirstValueOnly(Mangling): + def mangle(self, component: icalendar.Component) -> None: + for name in ("CATEGORIES", "ATTENDEE"): + values = component.get(name) + if isinstance(values, list): + component[name] = values[0] + + +class StartsInRange(MemoryCollection): + """Narrows a time range by DTSTART alone, and leaves tasks out of it.""" + + def _query(self, searcher: Searcher | None) -> list[StoredObject]: + start, end = getattr(searcher, "start", None), getattr(searcher, "end", None) + out = [] + for o in self.store.objects.values(): + components = [c for c in o.calendar.subcomponents if c.name in ("VEVENT", "VTODO")] + if start is None or any( + c.name == "VEVENT" and start <= c.decoded("DTSTART") < end for c in components if "DTSTART" in c + ): + out.append(_listed(o)) + return out + + +class CompletesNothing(MemoryCollection): + """Declares task.complete-recurring and writes nothing.""" + + def __init__(self, backend: MemoryBackend, store: MemoryCalendar) -> None: + super().__init__(backend, store) + self.capabilities = self.capabilities.updated({Feature.TASK_COMPLETE_RECURRING: Support.FULL}) + + def _complete_recurring( + self, task: Task, at: datetime | None, mode: str, *, loss: LossPolicy | None, verify: bool | None + ) -> SyncTask: + return self.bind(task) + + +# What the suite raises when it catches a defect: its own assertion, or a +# pytest.raises that did not raise. The round trip catches through verify=True. +ASSERTS = (AssertionError, pytest.fail.Exception) + +CAUGHT: list[tuple[type[MemoryCollection], str, dict[str, Any], Any]] = [ + (FirstPageOnly, "test_listings_are_not_cut_at_a_page", {}, ASSERTS), + (MatchesUidSubstrings, "test_get_finds_the_exact_uid", {}, ASSERTS), + (SearchesTheAccount, "test_get_finds_the_exact_uid", {}, ASSERTS), + (ReplacesEventUids, "test_get_finds_the_exact_uid", {}, ASSERTS), + (StaleListing, "test_a_listing_shows_a_save_made_elsewhere", {}, ASSERTS), + (ListViewStream, "test_iter_search_yields_what_search_returns", {}, ASSERTS), + (WordIndex, "test_search_gives_the_client_side_answer", {"name": "summary-contains"}, ASSERTS), + (FrozenFeed, "test_read_paths_on_seeded_items", {}, ASSERTS), + (NoDeletionsFeed, "test_read_paths_on_seeded_items", {}, ASSERTS), + (LeakyChangeFeed, "test_changes_follow_a_move", {}, ASSERTS), + (DropsAlarms, "test_real_world_objects_round_trip", {"name": "event-series-in-a-time-zone"}, VerificationError), + (DropsAlarms, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + (AsciiOnly, "test_real_world_objects_round_trip", {"name": "journal-entry"}, VerificationError), + (FlattensNewlines, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + (StoresUtc, "test_real_world_objects_round_trip", {"name": "event-series-in-a-time-zone"}, VerificationError), + (StoresUtc, "test_real_world_objects_round_trip", {"name": "floating-event"}, VerificationError), + (DatesAsDateTimes, "test_real_world_objects_round_trip", {"name": "all-day-event-with-rdate"}, VerificationError), + (DatesAsDateTimes, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + ( + DropsEventDetails, + "test_real_world_objects_round_trip", + {"name": "event-series-in-a-time-zone"}, + VerificationError, + ), + (DropsOverrides, "test_real_world_objects_round_trip", {"name": "event-series-in-a-time-zone"}, VerificationError), + (FirstValueOnly, "test_real_world_objects_round_trip", {"name": "task-due-on-a-date"}, VerificationError), + (StartsInRange, "test_search_gives_the_client_side_answer", {"name": "events-in-range"}, ASSERTS), + (StartsInRange, "test_search_gives_the_client_side_answer", {"name": "tasks-in-range"}, ASSERTS), + (CompletesNothing, "test_complete_a_recurring_task", {}, ASSERTS), +] + + +@pytest.mark.parametrize( + ("broken", "test", "params", "expected"), + CAUGHT, + ids=[f"{c[0].__name__}-{c[2].get('name', '')}".rstrip("-") for c in CAUGHT], +) +def test_the_suite_catches( + broken: type[MemoryCollection], test: str, params: dict[str, Any], expected: Any +) -> None: + class Backend(MemoryBackend): + collection_class = broken + + StaleListing.seen.clear() + NoDeletionsFeed.seen.clear() + FrozenFeed.first = None + profile, suite = (FEED, TestFeedProfile()) if broken in (FrozenFeed, NoDeletionsFeed) else (FULL, TestFullProfile()) + suite._made, suite._borrowed = [], [] + backend = Backend(profile=profile) + collection = suite.open_collection(backend) + method = getattr(suite, test) + wanted = inspect.signature(method).parameters + fixtures = {"backend": backend, "collection": collection} + with pytest.raises(expected): + method(**{k: v for k, v in fixtures.items() if k in wanted}, **params) + + +def test_the_suite_catches_a_collection_wider_than_its_backend() -> None: + backend = MemoryBackend(profile=FULL) + backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.UNSUPPORTED}) + with pytest.raises(AssertionError): + TestFullProfile().test_collection_never_widens_backend(backend, backend.collection("default")) diff --git a/tests/test_mapping.py b/tests/test_mapping.py new file mode 100644 index 0000000..3b1445c --- /dev/null +++ b/tests/test_mapping.py @@ -0,0 +1,103 @@ +"""Write planning: the losses derived from a declaration, and what verify compares.""" + +from __future__ import annotations + +import copy +from datetime import UTC, date, datetime, timedelta +from zoneinfo import ZoneInfo + +import icalendar + +from calendaring import Capabilities, Event, Feature, Support +from calendaring._mapping import differences, plan_calendar_losses, plan_losses + +OSLO = ZoneInfo("Europe/Oslo") + + +def with_alarm(event: Event) -> Event: + alarm = icalendar.Alarm() + alarm.add("ACTION", "DISPLAY") + alarm.add("DESCRIPTION", "soon") + alarm.add("TRIGGER", timedelta(minutes=-15)) + event.component.add_component(alarm) + return event + + +def planned(event: Event, table: dict[Feature, Support]) -> tuple[list[str], icalendar.Component]: + component = copy.deepcopy(event.component) + changes = plan_losses(component, Capabilities({**dict.fromkeys(Feature, Support.FULL), **table})) + for _, apply in changes: + apply() + return [loss.property for loss, _ in changes], component + + +def test_a_dropped_alarm_is_a_difference() -> None: + sent = with_alarm(Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC), description="d")) + stored = copy.deepcopy(sent.icalendar) + for component in stored.walk("VEVENT"): + component.subcomponents[:] = [] + assert [d.property for d in differences(sent.icalendar, stored)] == ["VALARM"] + assert differences(sent.icalendar, copy.deepcopy(sent.icalendar)) == [] + + +def test_alarms_unsupported_drops_them() -> None: + event = with_alarm(Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC))) + losses, component = planned(event, {Feature.ALARMS: Support.UNSUPPORTED}) + assert losses == ["VALARM"] + assert component.subcomponents == [] + + +def test_timezones_lossy_stores_utc() -> None: + event = Event.new( + summary="x", start=datetime(2026, 11, 3, 9, tzinfo=OSLO), end=datetime(2026, 11, 3, 10, tzinfo=OSLO) + ) + floating = Event.new(summary="y", start=datetime(2026, 11, 3, 9)) + losses, component = planned(event, {Feature.TIMEZONES: Support.LOSSY}) + assert sorted(losses) == ["DTEND", "DTSTART"] + assert component.decoded("DTSTART") == datetime(2026, 11, 3, 8, tzinfo=UTC) + losses, component = planned(floating, {Feature.TIMEZONES: Support.LOSSY}) + assert losses == ["DTSTART"] + assert component.decoded("DTSTART").tzinfo == UTC + assert planned(event, {Feature.TIMEZONES: Support.FULL})[0] == [] + + +def test_dates_lossy_stores_midnight_utc() -> None: + event = Event.new(summary="x", start=date(2026, 11, 3), end=date(2026, 11, 4)) + event.component.add("EXDATE", [date(2026, 11, 10)]) + losses, component = planned(event, {Feature.DATES: Support.LOSSY, Feature.RECURRENCE: Support.FULL}) + assert sorted(losses) == ["DTEND", "DTSTART", "EXDATE"] + assert component.decoded("DTSTART") == datetime(2026, 11, 3, tzinfo=UTC) + assert b"EXDATE:20261110T000000Z" in component.to_ical() + + +def test_recurrence_unsupported_drops_exdate() -> None: + event = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + event.component.add("RRULE", {"FREQ": "WEEKLY"}) + event.component.add("EXDATE", [datetime(2026, 11, 10, 9, tzinfo=UTC)]) + assert sorted(planned(event, {Feature.RECURRENCE: Support.UNSUPPORTED})[0]) == ["EXDATE", "RRULE"] + + +def test_recurrence_unsupported_drops_the_overrides_too() -> None: + series = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 3}) + moved = Event.new(summary="moved", start=datetime(2026, 11, 4, 15, tzinfo=UTC), uid=series.uid) + moved.component.add("RECURRENCE-ID", datetime(2026, 11, 4, 9, tzinfo=UTC)) + series.icalendar.add_component(moved.component) + calendar = copy.deepcopy(series.icalendar) + table = Capabilities({**dict.fromkeys(Feature, Support.FULL), Feature.RECURRENCE: Support.UNSUPPORTED}) + changes = plan_calendar_losses(calendar, table) + assert [loss.property for loss, _ in changes] == ["RECURRENCE-ID"] + for _, apply in changes: + apply() + assert [str(c["SUMMARY"]) for c in calendar.walk("VEVENT")] == ["x"] + + +def test_an_alarm_the_server_annotated_is_no_difference() -> None: + sent = with_alarm(Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC))) + stored = copy.deepcopy(sent.icalendar) + [alarm] = stored.walk("VALARM") + alarm.add("UID", "server-alarm-1") + alarm.add("X-WR-ALARMUID", "server-alarm-1") + assert differences(sent.icalendar, stored) == [] + alarm["TRIGGER"] = icalendar.vDDDTypes(timedelta(minutes=-5)) + assert [d.property for d in differences(sent.icalendar, stored)] == ["VALARM"]