diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 00000000..d2359507 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,15 @@ +{ + "name": "osw-python", + "description": "Skills for working with OpenSemanticLab wikis through the osw CLI and MCP server.", + "owner": { + "name": "OpenSemanticLab", + "url": "https://github.com/OpenSemanticLab" + }, + "plugins": [ + { + "name": "osl-tasks", + "source": "./", + "description": "Task and project management for OpenSemanticLab, driven by the osw CLI and MCP server." + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 00000000..fefb5e77 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,14 @@ +{ + "name": "osl-tasks", + "displayName": "OSL task management", + "description": "Task and project management for OpenSemanticLab, driven by the osw CLI and MCP server.", + "version": "2.7.1", + "author": { + "name": "OpenSemanticLab" + }, + "homepage": "https://github.com/OpenSemanticLab/osw-python", + "repository": "https://github.com/OpenSemanticLab/osw-python", + "license": "Apache-2.0", + "keywords": ["opensemanticlab", "osl", "tasks", "wiki", "semantic-mediawiki"], + "skills": ["./src/osw/skills/"] +} diff --git a/.github/workflows/on-release-main.yml b/.github/workflows/on-release-main.yml index 565fc7dd..58398961 100644 --- a/.github/workflows/on-release-main.yml +++ b/.github/workflows/on-release-main.yml @@ -1,9 +1,10 @@ # Automated release: python-semantic-release analyzes the conventional # commits on main, and when a release is due it bumps the static version in -# pyproject.toml and CITATION.cff, updates CHANGELOG.md, relocks uv.lock, -# commits, tags vX.Y.Z, pushes and creates the GitHub release. Publishing -# happens in this same run: pushes made with GITHUB_TOKEN do not retrigger -# workflows, so a tag-triggered publish would never fire. +# pyproject.toml, CITATION.cff, the osl-tasks skill and the Claude Code +# plugin manifest, updates CHANGELOG.md, relocks uv.lock, commits, tags +# vX.Y.Z, pushes and creates the GitHub release. Publishing happens in this +# same run: pushes made with GITHUB_TOKEN do not retrigger workflows, so a +# tag-triggered publish would never fire. # # Filename and environment name are load-bearing: the PyPI trusted # publisher (OIDC) references workflow "on-release-main.yml" and diff --git a/docs/tools/cli.md b/docs/tools/cli.md index c357787f..51f820b2 100644 --- a/docs/tools/cli.md +++ b/docs/tools/cli.md @@ -32,11 +32,12 @@ Commands are grouped by subject: | Group | Commands | | --- | --- | -| `entity` | `get`, `put`, `export`, `delete` | +| `entity` | `get`, `put`, `export`, `delete`, `validate` | | `file` | `info`, `cat`, `write`, `download`, `upload` | -| `search` | `ask`, `titles`, `content`, `entities`, `sparql` | +| `search` | `ask`, `titles`, `content`, `entities`, `sparql`, `label` | | `slot` | `list`, `get`, `set` | -| `schema` | `get` | +| `schema` | `get`, `props`, `usage` | +| `skill` | `install` | | `instances` | `list`, `status` | | `ledger` | `path` | | top level | `status` | @@ -79,3 +80,36 @@ form. help. Failures exit non-zero with a short message on stderr and no traceback. + +## Tasks and projects + +There is no dedicated command group for tasks. Reading local todos into an +OSL wiki as Task entities, and reading tasks, projects and persons back out, +is done with the generic `entity`, `schema` and `search` commands against the +three OSL core categories (Task, Person, Project). + +**Configuration.** Four environment variables affect this, and all are +optional: `OSW_PERSON_IRI`, `OSW_TASK_CATEGORY`, `OSW_PERSON_CATEGORY` and +`OSW_PROJECT_CATEGORY`. Reading always queries the shared OSL core category, +since MediaWiki category membership includes the whole subclass tree, so a +task kept in a local subclass is found without any configuration. The three +category overrides only change where a newly created task, person or project +is written. + +**Vocabularies.** `status` and `prio` store the page name of a wiki item, not +a word. Read the allowed values from the category schema with `osw schema get + --resolve`: `status` carries them in `enum`, and `prio` names +the category that holds them in `range`. A stock instance offers To do, In +work and Done for `status`, and High, Medium and Low for `prio`. A due date is +written to `end_date_time`, since the Task category has no due-date property. + +### The Claude Code skill + +The skill that drives this ships at `src/osw/skills/osl-tasks/SKILL.md`. +Install it one of two ways: + +1. `osw skill install`, which copies it to `~/.claude/skills/osl-tasks/`. +2. `/plugin marketplace add OpenSemanticLab/osw-python` then + `/plugin install osl-tasks`. + +A new Claude Code session picks it up with no further action. diff --git a/docs/tools/configuration.md b/docs/tools/configuration.md index fad7ca7b..2837ea03 100644 --- a/docs/tools/configuration.md +++ b/docs/tools/configuration.md @@ -111,6 +111,10 @@ set wins: | `OSW_MAX_RESULTS` | `OSW_MCP_MAX_RESULTS` | Default result cap (100) | | `OSW_MAX_CHARS` | `OSW_MCP_MAX_CHARS` | Result size cap in characters (100000) | | `OSW_VERBOSE` | `OSW_MCP_VERBOSE` | `true` prints the configuration source report | +| `OSW_PERSON_IRI` | | Page name of the operator's own Person entity, read by the osl-tasks skill to filter tasks by actionee | +| `OSW_TASK_CATEGORY` | | Category a newly created task is written to, read by the osl-tasks skill | +| `OSW_PERSON_CATEGORY` | | Category a newly created person is written to, read by the osl-tasks skill | +| `OSW_PROJECT_CATEGORY` | | Category a newly created project is written to, read by the osl-tasks skill | ## Windows paths in a `.env` file diff --git a/pyproject.toml b/pyproject.toml index ab07065b..e665b189 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ dependencies = [ "PyLD", "SPARQLWrapper", "jsonpath-ng", + # validates jsondata against a resolved JSON Schema in + # osw.service.ops.entities.validate_entity + "jsonschema>=4.0", "numpy", "pyyaml", "typing_extensions", @@ -300,7 +303,8 @@ preview = true [tool.semantic_release] # python-semantic-release owns the version: it reads the conventional commits -# on main, bumps the static version below and in CITATION.cff, updates the +# on main, bumps the static version below, in CITATION.cff, in the osl-tasks +# skill's frontmatter and in the Claude Code plugin manifest, updates the # changelog, relocks uv.lock, commits, tags (vX.Y.Z) and creates the GitHub # release. Publishing to PyPI happens in the same workflow run # (on-release-main.yml). @@ -312,7 +316,14 @@ commit_message = "chore(release): v{version} [skip ci]" allow_zero_version = true exclude_commit_patterns = ['''chore\(release\):.*'''] version_toml = ["pyproject.toml:project.version"] -version_variables = ["CITATION.cff:version"] +version_variables = [ + "CITATION.cff:version", + "src/osw/skills/osl-tasks/SKILL.md:version", + # Claude Code caches an installed plugin under its version string, so a + # version that never changes keeps plugin users on the skill they first + # installed. + ".claude-plugin/plugin.json:version", +] # relock so uv.lock's own pin of osw's version stays consistent, and include # the relocked file in the release commit build_command = "uv lock" diff --git a/src/osw/cli/main.py b/src/osw/cli/main.py index 7eee7a90..6a7b67a8 100644 --- a/src/osw/cli/main.py +++ b/src/osw/cli/main.py @@ -323,6 +323,7 @@ def command(**kwargs: Any) -> None: "schema": "Category JSON Schemas.", "search": "Find pages. OSW pages are titled by OSW-ID, so use 'ask' " "to search by name.", + "skill": "Install the Claude Code skills that ship with this package.", "slot": "Read and write individual page slots.", } diff --git a/src/osw/cli/ops.py b/src/osw/cli/ops.py index 0b7d502e..62cf9669 100644 --- a/src/osw/cli/ops.py +++ b/src/osw/cli/ops.py @@ -148,6 +148,45 @@ def ledger_path(ctx: Context) -> dict: return {"path": str(ctx.ledger.path)} +@operation( + group="skill", + cli_name="install", + surfaces=frozenset({"cli"}), + idempotent_hint=True, +) +def install_skill( + ctx: Context, + name: str = "osl-tasks", + target_dir: Optional[str] = None, + force: bool = False, +) -> dict: + """Install a Claude Code skill that ships with this package. + + Copies the packaged skill directory ``src/osw/skills/`` to + ``~/.claude/skills/`` (or under ``target_dir`` when given). A new + Claude Code session picks the installed skill up automatically, with no + further action needed. This is the alternative to installing the + osw-python plugin from its marketplace. + """ + src = Path(__file__).resolve().parent.parent / "skills" / name + if not src.is_dir(): + available = sorted(p.name for p in src.parent.iterdir() if p.is_dir()) + raise errors.NotFound( + f"No packaged skill named '{name}'. Available: {', '.join(available)}." + ) + + base = Path(target_dir) if target_dir else Path.home() / ".claude" / "skills" + dest = base / name + if dest.exists() and not force: + raise errors.OpError(f"'{dest}' already exists. Pass --force to overwrite it.") + + shutil.copytree(src, dest, dirs_exist_ok=True) + files = sorted( + p.relative_to(dest).as_posix() for p in dest.rglob("*") if p.is_file() + ) + return {"name": name, "source": str(src), "target": str(dest), "files": files} + + @operation( group="instances", cli_name="list", diff --git a/src/osw/mcp/server.py b/src/osw/mcp/server.py index f8f3812a..3fe9579c 100644 --- a/src/osw/mcp/server.py +++ b/src/osw/mcp/server.py @@ -35,8 +35,11 @@ Entity and page titles are full MediaWiki page names, e.g. "Item:OSW1234...", never a bare id or label. -Before creating or updating an entity, fetch its category's JSON Schema -(get_category_schema) so the written jsondata validates against it. +Before creating or updating an entity, get its category's JSON Schema with +get_category_schema(resolve=True) - the unresolved schema alone is usually +missing inherited properties - then check the payload with validate_entity +before writing it. A new entity does not need "uuid" or "type" in its +payload, since create_or_update_entity generates both itself. This server has no filesystem access: file content moves inline as text, not as a path. For anything path-based (uploading/downloading a local file, the diff --git a/src/osw/service/config.py b/src/osw/service/config.py index c68e6649..d8e4499f 100644 --- a/src/osw/service/config.py +++ b/src/osw/service/config.py @@ -40,6 +40,10 @@ ENV_MAX_CHARS = ("OSW_MAX_CHARS", "OSW_MCP_MAX_CHARS") ENV_FILE = ("OSW_ENV_FILE", "OSW_MCP_ENV_FILE") ENV_VERBOSE = ("OSW_VERBOSE", "OSW_MCP_VERBOSE") +ENV_TASK_CATEGORY = ("OSW_TASK_CATEGORY",) +ENV_PERSON_CATEGORY = ("OSW_PERSON_CATEGORY",) +ENV_PROJECT_CATEGORY = ("OSW_PROJECT_CATEGORY",) +ENV_PERSON_IRI = ("OSW_PERSON_IRI",) def _first_env(names: tuple[str, ...]) -> Optional[str]: @@ -64,6 +68,10 @@ def _first_env(names: tuple[str, ...]) -> Optional[str]: "max_results": ENV_MAX_RESULTS, "max_chars": ENV_MAX_CHARS, "verbose": ENV_VERBOSE, + "task_category": ENV_TASK_CATEGORY, + "person_category": ENV_PERSON_CATEGORY, + "project_category": ENV_PROJECT_CATEGORY, + "person_iri": ENV_PERSON_IRI, } @@ -105,6 +113,18 @@ class Settings(BaseModel): # Only controls the startup configuration report. No tool or command # reads it, and Settings.redacted() deliberately does not expose it. verbose: bool = False + # Category a newly created task, person or project is written to. No + # operation reads these three; the osl-tasks skill reads the env variables + # directly, and only when a new entity has to go into a local subclass + # instead of the shared OSL core category. + task_category: Optional[str] = None + person_category: Optional[str] = None + project_category: Optional[str] = None + # Full page name of the Person entity representing the operator, e.g. + # "Item:OSW...". No operation reads it; the osl-tasks skill reads the env + # variable directly to filter tasks by their actionee. An actionee is never + # assigned from it, so a task created without an explicit actionee has none. + person_iri: Optional[str] = None @field_validator("domain") @classmethod @@ -166,6 +186,17 @@ def _validate_state_dir(cls, value: Optional[str]) -> Optional[str]: ) return value + @field_validator( + "task_category", "person_category", "project_category", "person_iri" + ) + @classmethod + def _validate_non_blank(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return value + if not value.strip(): + raise ValueError("must not be empty or whitespace-only") + return value + @field_validator("cred_filepath") @classmethod def _validate_cred_filepath(cls, value: Optional[str]) -> Optional[str]: @@ -755,6 +786,10 @@ def load(strict: bool = True) -> Settings: cred_filepath=cred_filepath, sparql_endpoint=_first_env(ENV_SPARQL_ENDPOINT), state_dir=_first_env(ENV_STATE_DIR), + task_category=_first_env(ENV_TASK_CATEGORY), + person_category=_first_env(ENV_PERSON_CATEGORY), + project_category=_first_env(ENV_PROJECT_CATEGORY), + person_iri=_first_env(ENV_PERSON_IRI), ) # An unset or blank/whitespace-only variable falls back to the model # default; pass the raw string only when there is one to validate. Letting diff --git a/src/osw/service/context.py b/src/osw/service/context.py index 5026891d..eb405bf9 100644 --- a/src/osw/service/context.py +++ b/src/osw/service/context.py @@ -155,6 +155,26 @@ def page(self, title: str): raise errors.NotFound(f"Page '{title}' does not exist.") return page + def get_page_uncached(self, title: str): + """Return the page for ``title``, bypassing the site page cache. + + The page cache is off by default, but ``OSW.fetch_schema`` turns it + on and only restores the previous state itself when its own call + finishes; a caller that does not save and restore the cache state + around ``fetch_schema`` (e.g. ``create_or_update_entity``) can leave + it on for the rest of a long-running process. A later read through + the plain cached path could then return a page revision from before + a write made earlier in the same process. Does not raise for a + missing page; the caller branches on ``page.exists``. + """ + cache_state = self.osw.site.get_cache_enabled() + self.osw.site.disable_cache() + try: + return self.osw.site.get_page(WtSite.GetPageParam(titles=[title])).pages[0] + finally: + if cache_state: + self.osw.site.enable_cache() + def require_write(self, op_name: str) -> None: """Raise if this context's policy disallows writes.""" if not self.policy.allow_writes: diff --git a/src/osw/service/ops/entities.py b/src/osw/service/ops/entities.py index b93c78a7..5ec2aee7 100644 --- a/src/osw/service/ops/entities.py +++ b/src/osw/service/ops/entities.py @@ -2,16 +2,19 @@ from __future__ import annotations +import copy import logging from typing import Annotated, Optional import typer +from jsonschema.validators import validator_for import osw.model.entity as model_entity from osw.core import OSW, AddOverwriteClassOptions, OverwriteOptions from osw.service import config, errors from osw.service.context import Context from osw.service.ledger import LedgerRecord +from osw.service.ops.schema import _resolve_schema from osw.service.params import json_value from osw.service.registry import operation from osw.service.serialization import maybe_truncate, to_jsonable @@ -112,7 +115,7 @@ def export_entity_jsonld( LedgerRecord( title=t, op="create_or_update", change_id=r["change_id"], slots=["jsondata"] ) - for t in r["titles"] + for t in r["created"] + r["updated"] ], ) def create_or_update_entity( @@ -120,16 +123,30 @@ def create_or_update_entity( category: str, jsondata: Annotated[dict, typer.Option(parser=json_value)], namespace: Optional[str] = None, - overwrite: str = "keep existing", + overwrite: str = "true", comment: Optional[str] = None, ) -> dict: """Create or update an entity of ``category`` from a ``jsondata`` payload. ``category`` is a full category page name (e.g. ``Category:Item``); use - ``get_category_schema`` to learn the valid fields first. ``overwrite`` - controls update behavior: one of true | false | only empty | - replace remote | keep existing. Records the resulting page(s) in the - provenance ledger so they can be deleted without extra confirmation. + ``get_category_schema`` to learn the valid fields first. + + **To update an existing entity, put its ``uuid`` in ``jsondata``.** The + page name is derived from that uuid, so omitting it generates a new one + and writes a second entity instead of changing the intended one. Read the + current record with ``get_entity`` first and keep its uuid. + + ``overwrite`` controls update behavior: one of true | false | only empty | + replace remote | keep existing. The default, true, merges field by field + into the stored record: fields absent from ``jsondata`` keep their stored + value, so only what is passed changes. ``keep existing`` writes nothing at + all when the page already exists, and ``replace remote`` drops every + stored field that ``jsondata`` does not repeat. + + The result splits the written pages into ``created``, ``updated`` and + ``skipped`` so a caller can tell an update apart from an accidental + create. Only created and updated pages are recorded in the provenance + ledger, since a skipped page was never written. """ fetch = ctx.osw.fetch_schema( OSW.FetchSchemaParam(schema_title=category, mode="append") @@ -158,14 +175,201 @@ def create_or_update_entity( ) ) titles = list(store.pages.keys()) + # WtPage.exists is set once, when the page is loaded, and no write path + # refreshes it, so after the store it still reports the pre-write state. + # A page that did not exist then was created by this call. + existed = [t for t in titles if store.pages[t].exists] + # _apply_overwrite_policy returns an existing page untouched under this + # one option, logging a warning and never editing it. Mirrors the check + # at the kept_existing branch in OSW.store_entity_, which also tests + # 'offline is not True' and reads the per-class policy. Both are the same + # value here, since the StoreEntityParam above sets neither 'offline' nor + # 'overwrite_per_class'. Setting either would make the two disagree. + kept = _parse_overwrite(overwrite) == AddOverwriteClassOptions.keep_existing domain = config.get_active_domain() return { "titles": titles, + "created": [t for t in titles if t not in existed], + "updated": [] if kept else existed, + "skipped": existed if kept else [], "change_id": store.change_id, "urls": [f"https://{domain}/wiki/{t}" for t in titles], } +def _auto_filled_fields() -> set: + """Field names the osw base model actually supplies a value for. + + ``create_or_update_entity`` validates by constructing the category's + generated subclass, and every generated class inherits from ``Entity``. + A field counts here only when the base model gives it a value on + construction, through a default factory or a non-``None`` default - + ``uuid`` (a fresh ``uuid4``) and ``type`` (the category default) are the + two this matters for in practice. Most other fields are merely optional + on ``Entity`` and default to ``None``, which is not a supplied value: a + category subclass may still declare one of them required, in which case + omitting it from the payload does make ``create_or_update_entity`` fail, + so it must stay required here too. Reading the field set off the base + class rather than naming the fields keeps this correct if the base model + changes. + """ + return { + name + for name, field in model_entity.Entity.__fields__.items() + if field.required is False + and (field.default_factory is not None or field.default is not None) + } + + +def _strip_remote_refs(node, path: str, unchecked_refs: list) -> object: + """Return a copy of ``node`` with every remote ``$ref`` replaced by ``{}``. + + A ``$ref`` is remote when its value does not start with ``#`` (a local + JSON pointer); OSL schemas instead point ``$ref`` at a URL that reads + another wiki page's schema, which a JSON Schema validator would try to + fetch over the network. ``{}`` is the empty schema, which accepts + anything, so that part of the payload is simply left unchecked. Each + replacement's location is appended to ``unchecked_refs`` as a JSON path + such as ``$.properties.parent``. + """ + if isinstance(node, dict): + ref = node.get("$ref") + if isinstance(ref, str) and not ref.startswith("#"): + unchecked_refs.append(path) + return {} + return { + key: _strip_remote_refs(value, f"{path}.{key}", unchecked_refs) + for key, value in node.items() + } + if isinstance(node, list): + return [ + _strip_remote_refs(item, f"{path}[{index}]", unchecked_refs) + for index, item in enumerate(node) + ] + return node + + +@operation( + group="entity", cli_name="validate", read_only_hint=True, idempotent_hint=True +) +def validate_entity( + ctx: Context, + category: str, + jsondata: Annotated[dict, typer.Option(parser=json_value)], +) -> dict: + """Check whether ``jsondata`` validates against ``category``'s resolved + JSON Schema, without writing anything. + + Resolves ``category``'s effective JSON Schema across its parent chain - + the same walk ``get_category_schema(resolve=True)`` performs - and + validates ``jsondata`` against it with the ``jsonschema`` package. + Passing here does not guarantee ``create_or_update_entity`` will + succeed: that operation validates by constructing the generated + pydantic model instead, a different check with different rules, and it + also fetches the category's schema and regenerates the local + ``osw.model.entity`` module as a side effect, which this operation never + does. + + ``unknown_fields`` lists top-level keys of ``jsondata`` that are not + declared in the resolved schema's ``properties``. This is the check + that matters most in practice: most OSL schemas do not set + ``additionalProperties: false``, so the validator itself would silently + accept a plausible but wrong field name. ``unchecked_refs`` lists the + JSON paths of parts of the schema that were not checked, because they + were a remote ``$ref`` - a URL pointing at another wiki page rather than + a local JSON pointer; validating against it would require a network + call, so that part of the payload is left unchecked instead. + + Returns ``{category, valid, errors, unknown_fields, unchecked_refs, + sources, skipped, auto_filled}``, where ``sources`` are the page titles + successfully read while resolving the schema and ``skipped`` names every + page the walk visited and could not use, and why - the walk's depth + limit was reached, the page does not exist, it has no schema slot, it + could not be read, or its slot did not parse. A non-empty ``skipped`` + means part of the category's inherited schema could not be read, so the + check was made against less than the full schema and ``valid: true`` is + weaker than it looks. ``auto_filled`` lists fields the category's schema + marks required but the osw model supplies by itself, so they were not + required here; ``uuid`` and ``type`` are the usual ones - omitting them + from a payload is fine, since ``create_or_update_entity`` generates a + uuid and sets the type from the category. An invalid payload is this + operation's normal, successful answer: ``valid`` is False and ``errors`` + holds one message per validation error, each including its JSON path; + this never raises for a bad payload. Raises ``errors.NotFound`` if + ``category`` does not exist, and ``errors.SchemaError`` if the resolved + schema could not actually be read (its ``jsonschema`` slot is missing, + empty or unparsable, or ``category`` is not a category page at all) or + is not itself a valid JSON Schema - in both cases the payload was not + checked. + """ + page = ctx.osw.site.get_page(WtSite.GetPageParam(titles=[category])).pages[0] + if not page.exists: + raise errors.NotFound(f"Category '{category}' does not exist.") + + merged_schema, sources, skipped = _resolve_schema(ctx, category) + if not merged_schema or not merged_schema.get("properties"): + raise errors.SchemaError( + f"Could not read a JSON Schema for '{category}': its " + "'jsonschema' slot is missing, empty or unparsable, or " + "'category' is not a category page. The payload was not checked." + ) + + # Strip remote $refs on a deep copy, so this validation-only check never + # mutates the schema _resolve_schema built. + unchecked_refs: list = [] + schema = _strip_remote_refs(copy.deepcopy(merged_schema), "$", unchecked_refs) + + # Drop the required fields the model fills in itself, so a payload that + # create_or_update_entity would accept is not reported invalid here. A + # malformed 'required' (not a list, or holding an unhashable item) is + # left alone here and surfaces as a SchemaError below, from check_schema, + # rather than raising a raw TypeError from this step. + required = schema.get("required") + auto_filled = ( + sorted( + {name for name in required if isinstance(name, str)} & _auto_filled_fields() + ) + if isinstance(required, list) + else [] + ) + if auto_filled: + schema["required"] = [ + name for name in schema["required"] if name not in auto_filled + ] + + try: + validator_cls = validator_for(schema) + validator_cls.check_schema(schema) + validator = validator_cls(schema) + except Exception as exc: + raise errors.SchemaError( + f"The resolved schema for '{category}' is not a valid JSON Schema: {exc}" + ) + + try: + validation_errors = [ + f"{err.json_path}: {err.message}" for err in validator.iter_errors(jsondata) + ] + except Exception as exc: + # An unresolvable local $ref (one iter_errors actually tries to + # follow, unlike the remote ones stripped above) raises instead of + # yielding, so surface it the same way a malformed schema is. + raise errors.SchemaError(f"Could not validate '{category}': {exc}") + declared = set(schema.get("properties") or {}) + unknown_fields = [key for key in jsondata if key not in declared] + + return { + "category": category, + "valid": not validation_errors, + "errors": validation_errors, + "unknown_fields": unknown_fields, + "unchecked_refs": unchecked_refs, + "sources": sources, + "skipped": skipped, + "auto_filled": auto_filled, + } + + @operation( group="entity", cli_name="delete", diff --git a/src/osw/service/ops/schema.py b/src/osw/service/ops/schema.py index 363feb17..08a0e286 100644 --- a/src/osw/service/ops/schema.py +++ b/src/osw/service/ops/schema.py @@ -3,12 +3,142 @@ from __future__ import annotations +import json + +from osw.service import errors from osw.service.context import Context from osw.service.registry import operation from osw.service.serialization import maybe_truncate +from osw.wiki_tools import _ask_results_as_dict from osw.wtsite import WtSite +def _merge_schema( + ctx: Context, + title: str, + max_depth: int, + merged: dict, + sources: list[str], + seen: set, + skipped: list, +) -> None: + """Merge one page's own JSON Schema into ``merged``, then recurse into + its parents. + + Mirrors ``WtSite._merge_jsonld_context``'s parent-first walk. ``title`` + is a full page title (``Category:...`` or ``JsonSchema:...``); a + ``Category:`` page keeps its schema in the ``jsonschema`` slot, a + ``JsonSchema:`` page in the ``main`` slot. Parent references come from + string entries of the schema's ``@context`` list and from ``$ref`` + values inside ``allOf``. Recursion visits parents before this page + applies its own values, so a child's own value always wins on conflict. + A page that cannot be read, or whose slot does not parse, is skipped + rather than aborting the walk; each skip is appended to ``skipped`` as + ``{"title": ..., "reason": ...}``, unless the page was already ``seen``, + since reaching the same page twice through two parents is a normal + diamond in the category graph, not a failure. ``seen`` also stops the + same page being read twice within one call, which breaks a cycle. + """ + if title in seen: + return + if max_depth <= 0: + skipped.append({"title": title, "reason": "maximum depth reached"}) + return + seen.add(title) + + try: + page = ctx.get_page_uncached(title) + if not page.exists: + skipped.append({"title": title, "reason": "page does not exist"}) + return + if "JsonSchema:" in title: + schema = page.get_slot_content("main") + else: + schema = page.get_slot_content("jsonschema") + if isinstance(schema, str) and not schema.strip(): + schema = None + if schema is None: + skipped.append({"title": title, "reason": "no schema slot"}) + return + if isinstance(schema, str): + schema = json.loads(schema) + except Exception as exc: + skipped.append({"title": title, "reason": f"could not be read: {exc}"}) + return + if not isinstance(schema, dict): + skipped.append({"title": title, "reason": "schema slot is not a JSON object"}) + return + sources.append(title) + + parents: list[str] = [] + context = schema.get("@context") + entries = context if isinstance(context, list) else ([context] if context else []) + for entry in entries: + if isinstance(entry, str): + parents.append(entry) + for ref in schema.get("allOf", []) or []: + if isinstance(ref, dict) and ref.get("$ref"): + parents.append(ref["$ref"]) + + for ref in parents: + parent_title = ref.split("/wiki/")[-1].split("?")[0] + if parent_title.startswith("Category:") or parent_title.startswith( + "JsonSchema:" + ): + _merge_schema( + ctx, parent_title, max_depth - 1, merged, sources, seen, skipped + ) + + # Apply this page's own schema last, so it overrides its parents: + # properties and definitions merge per key (this page wins on conflict), + # required is a de-duplicated union in first-seen order, and every other + # top-level key is simply taken from this page. + properties = {**merged.get("properties", {}), **schema.get("properties", {})} + required = list( + dict.fromkeys([*merged.get("required", []), *schema.get("required", [])]) + ) + definitions = {**merged.get("definitions", {}), **schema.get("definitions", {})} + merged.update(schema) + merged["properties"] = properties + merged["required"] = required + merged["definitions"] = definitions + + +def _resolve_schema(ctx: Context, category: str, max_depth: int = 10) -> tuple: + """Resolve ``category``'s effective JSON Schema across its parent chain. + + A category's ``jsonschema`` slot only ever declares its own properties + and points at its parent, so this walks the same chain + ``WtSite.get_jsonld_context`` does and merges each level, parent first. + No result is cached: each call may read several pages, which keeps the + result fresh when a category page is edited. + + Returns ``(merged_schema, sources, skipped)``, where ``sources`` lists the + full titles of every page successfully read, in the order visited, and + ``skipped`` lists every page the walk could not use, each as + ``{"title", "reason"}``, in the order visited. + """ + merged: dict = {} + sources: list[str] = [] + skipped: list = [] + _merge_schema(ctx, category, max_depth, merged, sources, set(), skipped) + + # A page can be recorded as depth-exhausted on one branch and then read + # successfully on a shorter one, and the same page can exhaust the depth + # on two branches. Neither should reach the caller, so reconcile the two + # lists here rather than complicating the walk. + sources_seen = set(sources) + reconciled: list = [] + reconciled_titles: set = set() + for entry in skipped: + title = entry["title"] + if title in sources_seen or title in reconciled_titles: + continue + reconciled_titles.add(title) + reconciled.append(entry) + return merged, sources, reconciled + + @operation( group="schema", cli_name="get", @@ -16,7 +146,7 @@ idempotent_hint=True, max_result_size_chars=200_000, ) -def get_category_schema(ctx: Context, category: str) -> dict: +def get_category_schema(ctx: Context, category: str, resolve: bool = False) -> dict: """Return the JSON Schema of a category (its ``jsonschema`` slot). ``category`` is a full category page name, e.g. ``Category:Item``. The @@ -24,15 +154,263 @@ def get_category_schema(ctx: Context, category: str) -> dict: generating models - does not modify any local files. Use the returned schema to construct a valid ``jsondata`` payload for ``create_or_update_entity``. + + Without ``resolve`` (the default), the schema covers one level only: a + category's own ``jsonschema`` slot, not what it inherits from its + parents. Since an OSL category inherits most of its properties from its + parent chain, the one-level schema is usually not enough on its own to + build a valid payload, even though that is what it is for. Pass + ``resolve=True`` to walk the parent chain and merge it into one schema - + a child's own property always wins over a parent's - which is what + ``create_or_update_entity`` actually needs. The resolved result gains + ``resolved: true`` and a ``sources`` list of every page title + successfully read while resolving it, in the order visited. It also + gains a ``skipped`` list naming every page the walk visited and could + not use, and why - the walk's depth limit was reached, the page does + not exist, it has no schema slot, it could not be read, or its slot did + not parse. A non-empty ``skipped`` means the merged schema may be + missing properties those pages would have contributed. """ - page = ctx.osw.site.get_page(WtSite.GetPageParam(titles=[category])).pages[0] + page = ctx.get_page_uncached(category) if not page.exists: return {"category": category, "exists": False, "schema": None} - schema = page.get_slot_content("jsonschema") - content, truncated = maybe_truncate(schema, ctx.settings.max_chars) + if not resolve: + schema = page.get_slot_content("jsonschema") + content, truncated = maybe_truncate(schema, ctx.settings.max_chars) + return { + "category": category, + "exists": True, + "schema": content, + "truncated": truncated, + } + merged, sources, skipped = _resolve_schema(ctx, category) + content, truncated = maybe_truncate(merged, ctx.settings.max_chars) return { "category": category, "exists": True, "schema": content, "truncated": truncated, + "resolved": True, + "sources": sources, + "skipped": skipped, + } + + +@operation( + group="schema", + cli_name="props", + read_only_hint=True, + idempotent_hint=True, +) +def get_category_property_map(ctx: Context, category: str) -> dict: + """Return the Semantic MediaWiki property name for each of a category's fields. + + These are the names to use in an SMW ``ask`` query and in the + ``printouts`` parameter of ``search ask`` - for example, the JSON field + ``status`` typically maps to the property ``HasStatus``. The mapping is + one-way and cannot be inverted: several JSON fields may share one + property. Returned names carry no ``Property:`` prefix. + + Every OSL category chain reaches Entity, which declares at least a + label mapping, so an empty map is never a genuinely property-less + category; it means ``category``'s page is missing or unreadable, or + that ``category`` does not actually name a category. Rather than + returning that as a false-looking empty success, this raises + ``errors.SchemaError`` naming ``category``. + """ + properties = ctx.osw.site.get_smw_property_map(category) + if not properties: + raise errors.SchemaError( + f"'{category}' declares no Semantic MediaWiki properties: its " + "page is missing or unreadable, or 'category' does not name a " + "real category." + ) + return { + "category": category, + "properties": properties, + "count": len(properties), + } + + +MAX_FIELD_USAGE_SAMPLE = 100 + + +def _check_injection(value: str, field: str) -> None: + """Reject a value that could change an ask query's structure. + + A private copy: ``ops.search`` keeps its own. + """ + if "]]" in value or "[[" in value or "|" in value: + raise errors.ValidationError(f"{field} must not contain ']]', '[[' or '|'.") + + +def _is_populated(value) -> bool: + """Whether a stored ``jsondata`` value counts as filled in. + + Only an absent value is empty: ``None``, ``""``, ``[]`` and ``{}``. + ``False`` and ``0`` are values a field genuinely holds, so a plain + falsy test would undercount them. + """ + if value is None: + return False + if isinstance(value, (str, list, dict)): + return len(value) > 0 + return True + + +@operation( + group="schema", + cli_name="usage", + read_only_hint=True, + idempotent_hint=True, +) +def get_field_usage(ctx: Context, category: str, sample: int = 20) -> dict: + """Report which of a category's declared fields real instances populate. + + A category schema often declares several fields of the same shape for + one idea: ``Category:Task`` on one instance declares ``related_to``, + ``related_to_project``, ``belongs_to_project`` and ``projects``, all + arrays of strings, and only one of them is used in practice. The schema + alone cannot say which. Counting Semantic MediaWiki properties cannot + say it either, because a category's ``@context`` maps several JSON + fields onto one property name, so ``related_to`` and + ``related_to_project`` both resolve to ``IsRelatedTo``. This operation + therefore reads a sample of stored instances and counts, per top-level + ``jsondata`` field, how many of them give that field a value. + + It reads one wiki page per sampled instance, which makes it slower than + the other schema operations. ``sample`` is how many instances to read; + it defaults to 20 and is capped at 100. + + The result describes the sampled instances only and may not represent + every instance of the category. ``more_instances_exist`` tells the + caller the sample was not the whole set, so a count of 0 there means + unused among the instances read, not proven unused. + + The sampled instances are not drawn at random. The query returns them in + the wiki's own order, and the same first ``sample`` of them come back + every time, so raising ``sample`` extends the set rather than replacing + it. That order appears to follow page creation, so a field that only + recent instances use can show a count of 0. Raise ``sample`` before + concluding that a field is unused. + + Instances are found with ``[[HasType::Category:]]``, the same + query ``search entities`` runs, so only pages declaring this exact + category as their type are read. Instances of a subcategory are not. + A category whose instances all declare a subcategory therefore reports + ``sampled`` 0. That result supports no conclusion about field usage, so + ``fields``, ``declared_unused`` and ``undeclared_present`` are all empty + whenever ``sampled`` is 0. Query the subcategory instead. + + Returns ``{category, sampled, fields, declared_unused, + undeclared_present, more_instances_exist, unreadable, skipped}``. + ``sampled`` is how many instances were read and counted. ``fields`` + holds one ``{field, populated, declared}`` entry per declared property + and per undeclared key seen in an instance, ordered by ``populated`` + descending, then by field name ascending. ``unreadable`` holds one + ``{title, reason}`` entry per sampled instance whose page or ``jsondata`` + slot could not be read; those are not counted. Check it before trusting a + low ``sampled``, since a connection failure and a deleted page both lower + that number and only the reason tells them apart. ``skipped`` lists the + schema pages the parent-chain walk could not use, as + ``get_category_schema`` does. + """ + if sample < 1 or sample > MAX_FIELD_USAGE_SAMPLE: + raise errors.ValidationError( + f"Invalid sample '{sample}'. It must be between 1 and " + f"{MAX_FIELD_USAGE_SAMPLE}, since every sampled instance costs " + "one page read." + ) + _check_injection(category, "category") + + merged, _sources, skipped = _resolve_schema(ctx, category) + declared = merged.get("properties") or {} + if not declared: + raise errors.SchemaError( + f"'{category}' declares no properties: its page is missing or " + "unreadable, or 'category' does not name a real category." + ) + + # The same ask query 'search entities' runs through OSW.query_instances, + # here with return_json so the raw response carries SMW's truncation + # marker. Unlike tasks._ask_rows this keeps every row rather than only + # those whose 'exists' flag is "1": a title whose page really is gone is + # recorded as unreadable below anyway. + name = category.split(":")[-1] + raw = ctx.osw.site.semantic_search( + WtSite.SearchParam( + query=f"[[HasType::Category:{name}]]", + limit=sample, + return_json=True, + ) + ) + response = raw[0] if raw else {} + # SMW serialises a result set as an object keyed by title, but an empty + # one as an array. The shared normaliser also keys a non-empty array by + # 'fulltext', so no row is dropped when that form appears. + results = _ask_results_as_dict(response.get("query", {}).get("results", {})) + titles = [ + row["fulltext"] + for row in results.values() + if isinstance(row, dict) and row.get("fulltext") + ][:sample] + more_instances_exist = "query-continue-offset" in response + + counts: dict = dict.fromkeys(declared, 0) + unreadable: list = [] + sampled = 0 + for title in titles: + # One bad page must not abort the whole sample, so anything that + # does not yield a jsondata object is recorded and skipped. + try: + # Uncached, like the schema walk above: a write earlier in the + # same process can leave the page cache enabled, and the cached + # path would then count a revision from before that write. + page = ctx.get_page_uncached(title) + if not page.exists: + reason = "page does not exist" + jsondata = None + else: + jsondata = page.get_slot_content("jsondata") + reason = ( + "no jsondata slot" + if jsondata is None + else "jsondata slot is not a JSON object" + ) + except Exception as exc: + # The reason matters: an outage that hits every page produces the + # same 'sampled' 0 as a category whose instances really are gone. + jsondata, reason = None, f"could not be read: {exc}" + if not isinstance(jsondata, dict): + unreadable.append({"title": title, "reason": reason}) + continue + sampled += 1 + for field, value in jsondata.items(): + counts.setdefault(field, 0) + if _is_populated(value): + counts[field] += 1 + + fields = [ + {"field": field, "populated": count, "declared": field in declared} + for field, count in counts.items() + ] + fields.sort(key=lambda entry: (-entry["populated"], entry["field"])) + # With nothing read, every count would be 0, and a caller would take that + # as 'no field is in use'. Zero observations support no such claim, so the + # counted keys stay empty and 'sampled' carries the whole answer. + if not sampled: + fields = [] + return { + "category": category, + "sampled": sampled, + "fields": fields, + "declared_unused": ( + sorted(f for f in declared if counts[f] == 0) if sampled else [] + ), + "undeclared_present": ( + sorted(f for f in counts if f not in declared) if sampled else [] + ), + "more_instances_exist": more_instances_exist, + "unreadable": sorted(unreadable, key=lambda entry: entry["title"]), + "skipped": skipped, } diff --git a/src/osw/service/ops/search.py b/src/osw/service/ops/search.py index 7942c34f..e14b5556 100644 --- a/src/osw/service/ops/search.py +++ b/src/osw/service/ops/search.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Optional +from typing import List, Optional from osw.core import OSW from osw.service import config, errors @@ -10,7 +10,7 @@ from osw.service.registry import operation from osw.service.serialization import cap_list, to_jsonable from osw.sparql_client_smw import SmwSparqlClient -from osw.wiki_tools import get_query_limit +from osw.wiki_tools import _ask_results_as_dict, get_query_limit from osw.wtsite import WtSite @@ -25,13 +25,25 @@ def _hit_limit(total: int, limit: Optional[int]) -> bool: return bool(limit) and total >= limit +def _check_injection(value: str, field: str) -> None: + """Reject a value that could change an ask query's structure.""" + if "]]" in value or "[[" in value or "|" in value: + raise errors.ValidationError(f"{field} must not contain ']]', '[[' or '|'.") + + @operation( group="search", cli_name="ask", read_only_hint=True, idempotent_hint=True, + max_result_size_chars=200_000, ) -def search_ask(ctx: Context, ask_query: str, limit: Optional[int] = None) -> dict: +def search_ask( + ctx: Context, + ask_query: str, + limit: Optional[int] = None, + printouts: Optional[List[str]] = None, +) -> dict: """Run a Semantic MediaWiki 'ask' query and return matching page titles. This is the only search that can find an entity by a property value, such @@ -56,26 +68,71 @@ def search_ask(ctx: Context, ask_query: str, limit: Optional[int] = None) -> dic ``limit`` defaults to ``OSW_MAX_RESULTS`` (100 when that is unset). A ``limit=N`` written into the query itself wins over it. + + ``printouts`` requests SMW property values alongside each hit, so a + caller does not have to follow up with one ``entity get`` per title. + Give bare property names, such as ``HasStatus``, without the + ``Property:`` prefix; use ``osw schema props`` to discover a category's + property names. Requesting a property the category does not define does + not raise - the row simply carries that key set to ``null``. + ``Display_title_of`` does not work as a printout: its printout key comes + back translated into the wiki's content language regardless of the + alias, so it always resolves to ``null`` here - read an entity's label + from its title or with ``entity get`` instead. When ``printouts`` is + omitted or empty, the query and result are exactly as without it. + Returns ``{titles, count, truncated}``, where ``titles`` are full page names, ``count`` is how many came back once hits whose page does not exist were dropped, and ``truncated`` reports that further matches may - exist beyond them. + exist beyond them. When ``printouts`` is given, the result also carries + ``rows``: a list of ``{title, printouts}`` in the same order as + ``titles``, where ``printouts`` maps each requested name to the raw + value SMW returned for it - a page reference keeps its ``fulltext`` and + ``fullurl`` rather than being flattened to a string. """ lim = ctx.limit(limit) - titles = ctx.osw.site.semantic_search( - WtSite.SearchParam(query=ask_query, limit=lim) - ) # semantic_search lets a 'limit=' written into the query win over `lim`, # so the flag has to compare against the limit that reached the wiki. - # `titles` excludes hits whose page does not exist, so a result set - # thinned that way reads as not truncated. query_limit = get_query_limit(ask_query) effective_limit = lim if query_limit is None else query_limit - capped, total, truncated = cap_list(titles, lim) + + if not printouts: + titles = ctx.osw.site.semantic_search( + WtSite.SearchParam(query=ask_query, limit=lim) + ) + # `titles` excludes hits whose page does not exist, so a result set + # thinned that way reads as not truncated. + capped, total, truncated = cap_list(titles, lim) + return { + "titles": capped, + "count": total, + "truncated": truncated or _hit_limit(total, effective_limit), + } + + # The '=name' alias forces the printout key in the result to the property + # name; without it, SMW keys the result by the property's display label, + # which need not equal the property name, and the row values below are + # looked up by name. + full_query = ask_query + "".join(f"|?{p}={p}" for p in printouts) + raw = ctx.osw.site.semantic_search( + WtSite.SearchParam(query=full_query, limit=lim, return_json=True) + ) + response = raw[0] if raw else {} + payload = _ask_results_as_dict(response.get("query", {}).get("results", {})) + hits = [p for p in payload.values() if p.get("exists") == "1"] + capped_hits, total, truncated = cap_list(hits, lim) + rows = [ + { + "title": hit["fulltext"], + "printouts": {p: hit.get("printouts", {}).get(p) for p in printouts}, + } + for hit in capped_hits + ] return { - "titles": capped, + "titles": [row["title"] for row in rows], "count": total, "truncated": truncated or _hit_limit(total, effective_limit), + "rows": rows, } @@ -220,3 +277,46 @@ def sparql_query( "count": total, "truncated": truncated, } + + +@operation( + group="search", + cli_name="label", + read_only_hint=True, + idempotent_hint=True, +) +def search_by_label( + ctx: Context, + label: str, + category: Optional[str] = None, + limit: Optional[int] = None, +) -> dict: + """Find an entity by its exact display label. + + Runs the ask query ``[[Display_title_of::