From f917ad6a209c332bde979b5ec5903ffe885f1da7 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Mon, 12 Oct 2026 19:33:38 +1100 Subject: [PATCH 01/38] refactor(signage-ai): share the credential helpers and the job long poll PPT-2833. AI map editing reuses the signage AI provider rows and job table with a text/vision adapter, so the pieces both adapter families need move out of ImageGen::Adapter: credentials, credential and vendor_message become Utils::VendorCredentials, and the job long poll and same-domain job lookup become Utils::AIJobs so the map AI controller can include them rather than copy them. SignageAI's behaviour is unchanged. The spec helpers setup_signage_ai, signage_group and await_signage_ai_job move into spec/spec_helpers so a second controller spec can use them without requiring the signage spec file. Co-Authored-By: Claude Fable 5.1 --- spec/controllers/signage/signage_ai_spec.cr | 40 ------------------ spec/spec_helpers/signage_ai.cr | 41 +++++++++++++++++++ src/placeos-rest-api/utilities/ai_jobs.cr | 39 ++++++++++++++++++ .../utilities/image_gen/adapter.cr | 22 ++-------- .../utilities/vendor_credentials.cr | 28 +++++++++++++ 5 files changed, 112 insertions(+), 58 deletions(-) create mode 100644 spec/spec_helpers/signage_ai.cr create mode 100644 src/placeos-rest-api/utilities/ai_jobs.cr create mode 100644 src/placeos-rest-api/utilities/vendor_credentials.cr diff --git a/spec/controllers/signage/signage_ai_spec.cr b/spec/controllers/signage/signage_ai_spec.cr index 5e7c697a..7cd7750f 100644 --- a/spec/controllers/signage/signage_ai_spec.cr +++ b/spec/controllers/signage/signage_ai_spec.cr @@ -1,46 +1,6 @@ require "../../helper" module PlaceOS::Api - # Everything a signage AI request needs: somewhere to put the image, and a - # vendor to ask for one. Returns (authority, storage, provider). - def self.setup_signage_ai(quotas : Hash(String, JSON::Any) = {} of String => JSON::Any) - authority = Model::Authority.find_by_domain("localhost").not_nil! - storage = Model::Generator.storage(authority_id: authority.id.as(String)).save! - provider = Model::Generator.signage_ai_provider( - authority: authority, - name: "openai-#{random_name}", - is_default: true, - quotas: quotas, - ).save! - - {authority, storage, provider} - end - - # A signage group the caller may act in, hung off the authority root (an - # authority only ever has one root group). - def self.signage_group(authority, user, permissions : Model::Permissions) - root = Model::Generator.group(authority: authority).save! - group = Model::Generator.group(authority: authority, parent: root, subsystems: ["signage"]).save! - Model::Generator.group_user(user: user, group: group, permissions: permissions).save! - group - end - - # Drive the long poll at `wait=0` until the job stops moving. The runner is a - # fiber in this same process, so each poll gives it a turn. - def self.await_signage_ai_job(base : String, id : String, headers : HTTP::Headers) : JSON::Any - path = File.join(base, "jobs", id) - - 100.times do - response = client.get(path, headers: headers) - response.status_code.should eq 200 - body = JSON.parse(response.body) - return body if {"done", "failed", "cancelled"}.includes?(body["state"].as_s) - sleep 100.milliseconds - end - - raise "signage AI job #{id} never reached a final state" - end - describe SignageAI do base = SignageAI.base_route diff --git a/spec/spec_helpers/signage_ai.cr b/spec/spec_helpers/signage_ai.cr new file mode 100644 index 00000000..1ec4db2c --- /dev/null +++ b/spec/spec_helpers/signage_ai.cr @@ -0,0 +1,41 @@ +module PlaceOS::Api + # Everything a signage AI request needs: somewhere to put the image, and a + # vendor to ask for one. Returns (authority, storage, provider). + def self.setup_signage_ai(quotas : Hash(String, JSON::Any) = {} of String => JSON::Any) + authority = Model::Authority.find_by_domain("localhost").not_nil! + storage = Model::Generator.storage(authority_id: authority.id.as(String)).save! + provider = Model::Generator.signage_ai_provider( + authority: authority, + name: "openai-#{random_name}", + is_default: true, + quotas: quotas, + ).save! + + {authority, storage, provider} + end + + # A signage group the caller may act in, hung off the authority root (an + # authority only ever has one root group). + def self.signage_group(authority, user, permissions : Model::Permissions) + root = Model::Generator.group(authority: authority).save! + group = Model::Generator.group(authority: authority, parent: root, subsystems: ["signage"]).save! + Model::Generator.group_user(user: user, group: group, permissions: permissions).save! + group + end + + # Drive the long poll at `wait=0` until the job stops moving. The runner is a + # fiber in this same process, so each poll gives it a turn. + def self.await_signage_ai_job(base : String, id : String, headers : HTTP::Headers) : JSON::Any + path = File.join(base, "jobs", id) + + 100.times do + response = client.get(path, headers: headers) + response.status_code.should eq 200 + body = JSON.parse(response.body) + return body if {"done", "failed", "cancelled"}.includes?(body["state"].as_s) + sleep 100.milliseconds + end + + raise "signage AI job #{id} never reached a final state" + end +end diff --git a/src/placeos-rest-api/utilities/ai_jobs.cr b/src/placeos-rest-api/utilities/ai_jobs.cr new file mode 100644 index 00000000..f47be6c0 --- /dev/null +++ b/src/placeos-rest-api/utilities/ai_jobs.cr @@ -0,0 +1,39 @@ +require "placeos-models/signage_ai_job" + +module PlaceOS::Api + # The long poll shared by the signage AI and map AI job routes. + # + # The including controller is an `Application` (for `on_primary`). + module Utils::AIJobs + MAX_WAIT = 25 + POLL_INTERVAL = 500.milliseconds + + # Holds until the job's version moves past `since` (the version the caller + # already has, defaulting to the one just read), the job is final, or + # `wait` seconds pass. Re-reads go to the primary: a replica may lag the + # runner's writes. + def await_job_change(job : ::PlaceOS::Model::SignageAIJob, wait : Int32, since : Int32?) : ::PlaceOS::Model::SignageAIJob + return job if wait <= 0 || job.final? + + deadline = Time.utc + Math.min(wait, MAX_WAIT).seconds + known = since || job.version + + while Time.utc < deadline + sleep POLL_INTERVAL + fresh = on_primary { ::PlaceOS::Model::SignageAIJob.find?(job.id.as(UUID)) } + next if fresh.nil? + job = fresh + break if job.version > known || job.final? + end + + job + end + + # A job of this domain, 404 otherwise so ids from other domains read as unknown. + def authority_job(id : UUID, authority_id : String) : ::PlaceOS::Model::SignageAIJob + job = ::PlaceOS::Model::SignageAIJob.find!(id) + raise Error::NotFound.new("no such job") unless job.authority_id == authority_id + job + end + end +end diff --git a/src/placeos-rest-api/utilities/image_gen/adapter.cr b/src/placeos-rest-api/utilities/image_gen/adapter.cr index 174058e0..3302c706 100644 --- a/src/placeos-rest-api/utilities/image_gen/adapter.cr +++ b/src/placeos-rest-api/utilities/image_gen/adapter.cr @@ -1,5 +1,7 @@ require "placeos-models/signage_ai_provider" +require "../vendor_credentials" + module PlaceOS::Api::ImageGen # A vendor. Adapters take an `AdapterRequest` and return images, or raise: # @@ -9,6 +11,8 @@ module PlaceOS::Api::ImageGen # # They never see a job row, a model or the request. abstract class Adapter + include Utils::VendorCredentials + getter row : ::PlaceOS::Model::SignageAIProvider def initialize(@row : ::PlaceOS::Model::SignageAIProvider) @@ -53,23 +57,5 @@ module PlaceOS::Api::ImageGen return defaults if allow.empty? defaults.select { |model| allow.includes?(model.id) } end - - protected def credentials : Hash(String, JSON::Any) - row.credentials_json - rescue ex : ::PlaceOS::Model::Error - raise Error::ImageGen::NotConfigured.new(ex.message || "provider credentials could not be read") - end - - protected def credential(key : String) : String - value = credentials[key]?.try(&.as_s?) - raise Error::ImageGen::NotConfigured.new("provider #{row.name} is missing #{key}") if value.nil? || value.empty? - value - end - - # trimmed so a vendor body never lands whole in the logs or an API response - protected def vendor_message(body : String, limit : Int32 = 300) : String - cleaned = body.gsub(/\s+/, " ").strip - cleaned.size > limit ? "#{cleaned[0, limit]}..." : cleaned - end end end diff --git a/src/placeos-rest-api/utilities/vendor_credentials.cr b/src/placeos-rest-api/utilities/vendor_credentials.cr new file mode 100644 index 00000000..a6f59446 --- /dev/null +++ b/src/placeos-rest-api/utilities/vendor_credentials.cr @@ -0,0 +1,28 @@ +require "placeos-models/signage_ai_provider" + +module PlaceOS::Api + # Reads a provider row's encrypted credentials for a vendor adapter. + # + # The including class exposes the row as `row`. Errors read as + # `Error::ImageGen::NotConfigured` so a bad secret or a missing field is + # reported as configuration rather than as a vendor fault. + module Utils::VendorCredentials + protected def credentials : Hash(String, JSON::Any) + row.credentials_json + rescue ex : ::PlaceOS::Model::Error + raise Error::ImageGen::NotConfigured.new(ex.message || "provider credentials could not be read") + end + + protected def credential(key : String) : String + value = credentials[key]?.try(&.as_s?) + raise Error::ImageGen::NotConfigured.new("provider #{row.name} is missing #{key}") if value.nil? || value.empty? + value + end + + # trimmed so a vendor body never lands whole in the logs or an API response + protected def vendor_message(body : String, limit : Int32 = 300) : String + cleaned = body.gsub(/\s+/, " ").strip + cleaned.size > limit ? "#{cleaned[0, limit]}..." : cleaned + end + end +end From 0844ef0403a874bf3276a73700f2cfa661523f74 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Mon, 12 Oct 2026 19:33:38 +1100 Subject: [PATCH 02/38] feat(zones): zone_updatable? gate without the signage grant PPT-2833. The AI map editor spends vendor calls on behalf of whoever may save the level's map, so the zone update gate is extracted into Utils::GroupPermissions#zone_updatable?: support or admin, the legacy org-zone ACL, or a support-subsystem Update on the zone. The broad signage subsystem grant that Zones#update also honours is left out of it. Zones#check_update_permissions keeps its behaviour exactly; its private record-form legacy check now delegates to the shared zone_legacy_access?. Co-Authored-By: Claude Fable 5.1 --- src/placeos-rest-api/controllers/zones.cr | 10 +------- .../utilities/group_permissions.cr | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/placeos-rest-api/controllers/zones.cr b/src/placeos-rest-api/controllers/zones.cr index 0379678a..6a0f71a4 100644 --- a/src/placeos-rest-api/controllers/zones.cr +++ b/src/placeos-rest-api/controllers/zones.cr @@ -118,15 +118,7 @@ module PlaceOS::Api # True if the legacy zone-based permission scheme (org_zone + # current_user.groups) grants the requested level on `zone`. private def has_legacy_access?(zone : ::PlaceOS::Model::Zone, admin_required : Bool = false) : Bool - authority = current_authority.as(::PlaceOS::Model::Authority) - org_zone_id = authority.config["org_zone"]?.try(&.as_s?) - return false unless org_zone_id - return false unless zone.persisted? || zone.parent_id.presence - return false unless zone.root_zone_id == org_zone_id - - zones = [org_zone_id, zone.id].compact.uniq! - access = check_access(current_user.groups, zones) - admin_required ? access.admin? : access.can_manage? + zone_legacy_access?(zone, admin_required: admin_required) end # True if the user has a Manage or Update grant in *any* group that diff --git a/src/placeos-rest-api/utilities/group_permissions.cr b/src/placeos-rest-api/utilities/group_permissions.cr index d23cbb34..c947be13 100644 --- a/src/placeos-rest-api/utilities/group_permissions.cr +++ b/src/placeos-rest-api/utilities/group_permissions.cr @@ -339,6 +339,30 @@ module PlaceOS::Api admin_required ? access.admin? : access.can_manage? end + # The legacy org_zone path for a zone record: the zone must sit under the + # authority's org zone, and the org zone's or the zone's own `permissions` + # metadata must grant `current_user.groups` manage (or admin, when + # `admin_required`). + def zone_legacy_access?(zone : ::PlaceOS::Model::Zone, admin_required : Bool = false) : Bool + org_zone_id = support_org_zone_id + return false unless org_zone_id + return false unless zone.persisted? || zone.parent_id.presence + return false unless zone.root_zone_id == org_zone_id + + zones = [org_zone_id, zone.id].compact.uniq! + access = check_access(current_user.groups, zones) + admin_required ? access.admin? : access.can_manage? + end + + # Whether the caller may change this zone: support or admin, the legacy + # org-zone ACL, or a support-subsystem Update on the zone. The broad + # signage-subsystem grant `Zones#update` also honours is not part of this. + def zone_updatable?(zone : ::PlaceOS::Model::Zone) : Bool + return true if user_support? + return true if zone_legacy_access?(zone, admin_required: false) + support_subsystem_grants?([zone.id.as(String)], ::PlaceOS::Model::Permissions::Update) + end + # The combined support gate. Raises `Error::Forbidden` unless one of: # - the JWT role bypasses (admin when `admin_required`, else support), # - the legacy org_zone path grants access, or From 861d820939b0f953ecae16f411c07dbd937ca7af Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Mon, 12 Oct 2026 19:33:38 +1100 Subject: [PATCH 03/38] feat(map-ai): Llm namespace, OpenAI chat adapter and the op schema PPT-2833. A text/vision call against a signage AI provider row for map editing. The editor posts the context it built from the level's SVG (element index, shortlisted subtrees, annotations, a marked up snapshot by upload id) and the model answers with a closed vocabulary of edit operations that the editor applies itself; nothing here reads or writes the map. - Llm::Schema defines the vocabulary once and renders it in the OpenAI strict dialect; the TypeScript side in libs/map-editor mirrors it field for field. - Llm::Prompt orders the request so the stable front (rules, op reference, index, subtrees) is identical across the turns of one edit and the instruction comes last, with byte caps per section and the parent chain's instruction, ops and resolution for a refine. - Llm::Ops parses the answer (markdown fence tolerated) and checks each op against the posted index; failing ops are kept with server_issue so the editor can show them unchecked with the reason. - Adapters::OpenAIChat posts chat completions with response_format json_schema strict, max_completion_tokens and reasoning_effort (dropped and resent on a 400 naming it), for OpenAI and both Azure URL styles; refusal and content_filter read as Moderated, 429 as Busy, IO::TimeoutError as Error::ImageGen::Timeout. No temperature: the gpt-5 family rejects it. - Llm::Runner is the one-call version of the image runner: Running, markup fetch, prompt, one vendor call under one Llm slot (sized by MAP_AI_MAX_CALLS, apart from the image slots), result written with bump_version, cost_units = total tokens, invalid_response for an answer that is not JSON or was cut off. - Error::ImageGen::InvalidResponse (502) and the MAP_AI_* constants. - The markup snapshot tag ai-map-markup, reaped by the existing sweep. Co-Authored-By: Claude Fable 5.1 --- src/constants.cr | 30 ++ .../controllers/application.cr | 5 +- src/placeos-rest-api/error.cr | 29 +- .../utilities/image_gen/store.cr | 2 + .../utilities/image_gen/sweep.cr | 2 +- src/placeos-rest-api/utilities/llm.cr | 24 ++ src/placeos-rest-api/utilities/llm/adapter.cr | 56 +++ .../utilities/llm/adapters/openai_chat.cr | 193 +++++++++++ src/placeos-rest-api/utilities/llm/ops.cr | 326 ++++++++++++++++++ src/placeos-rest-api/utilities/llm/prompt.cr | 309 +++++++++++++++++ src/placeos-rest-api/utilities/llm/runner.cr | 149 ++++++++ src/placeos-rest-api/utilities/llm/schema.cr | 191 ++++++++++ src/placeos-rest-api/utilities/llm/types.cr | 61 ++++ 13 files changed, 1363 insertions(+), 14 deletions(-) create mode 100644 src/placeos-rest-api/utilities/llm.cr create mode 100644 src/placeos-rest-api/utilities/llm/adapter.cr create mode 100644 src/placeos-rest-api/utilities/llm/adapters/openai_chat.cr create mode 100644 src/placeos-rest-api/utilities/llm/ops.cr create mode 100644 src/placeos-rest-api/utilities/llm/prompt.cr create mode 100644 src/placeos-rest-api/utilities/llm/runner.cr create mode 100644 src/placeos-rest-api/utilities/llm/schema.cr create mode 100644 src/placeos-rest-api/utilities/llm/types.cr diff --git a/src/constants.cr b/src/constants.cr index d7416cdf..0dbfbaca 100644 --- a/src/constants.cr +++ b/src/constants.cr @@ -54,6 +54,36 @@ module PlaceOS::Api SIGNAGE_AI_USER_PER_DAY = (ENV["SIGNAGE_AI_USER_PER_DAY"]? || "60").to_i SIGNAGE_AI_DOMAIN_PER_MONTH = (ENV["SIGNAGE_AI_DOMAIN_PER_MONTH"]? || "2000").to_i + # Map AI editing (text/vision calls against the same provider rows) + #################################################################################################### + + # kill switch for map editing alone; SIGNAGE_AI_DISABLED turns both off + MAP_AI_DISABLED = SIGNAGE_AI_DISABLED || ENV["MAP_AI_DISABLED"]?.try(&.downcase) == "true" + + # concurrent vendor text calls per replica, separate from the image slots + MAP_AI_MAX_CALLS = (ENV["MAP_AI_MAX_CALLS"]? || "4").to_i + + MAP_AI_READ_TIMEOUT = (ENV["MAP_AI_READ_TIMEOUT"]? || "180").to_i.seconds + + # the context JSON posted inline with an edit request (nginx's default body limit is 1MB) + MAP_AI_MAX_CONTEXT_BYTES = (ENV["MAP_AI_MAX_CONTEXT_KB"]? || "512").to_i * 1024 + + MAP_AI_MAX_INSTRUCTION_CHARS = (ENV["MAP_AI_MAX_INSTRUCTION_CHARS"]? || "4000").to_i + + # operations accepted from one answer; the rest are kept but flagged + MAP_AI_MAX_OPS = (ENV["MAP_AI_MAX_OPS"]? || "80").to_i + + MAP_AI_MAX_OUTPUT_TOKENS = (ENV["MAP_AI_MAX_OUTPUT_TOKENS"]? || "12000").to_i + + # default quotas in calls, overridden per provider row by map_user_per_day + # and map_domain_per_month + MAP_AI_USER_PER_DAY = (ENV["MAP_AI_USER_PER_DAY"]? || "40").to_i + MAP_AI_DOMAIN_PER_MONTH = (ENV["MAP_AI_DOMAIN_PER_MONTH"]? || "1000").to_i + + # used when a provider row names no text_model + MAP_AI_DEFAULT_OPENAI_MODEL = ENV["MAP_AI_DEFAULT_OPENAI_MODEL"]? || "gpt-5.4-mini" + MAP_AI_DEFAULT_VERTEX_MODEL = ENV["MAP_AI_DEFAULT_VERTEX_MODEL"]? || "gemini-3.8-flash" + # server defaults in `./app.cr` TRIGGERS_URI = URI.parse(ENV["TRIGGERS_URI"]? || "http://triggers:3000") diff --git a/src/placeos-rest-api/controllers/application.cr b/src/placeos-rest-api/controllers/application.cr index f3af5fbd..7de9ad01 100644 --- a/src/placeos-rest-api/controllers/application.cr +++ b/src/placeos-rest-api/controllers/application.cr @@ -251,8 +251,8 @@ module PlaceOS::Api CommonError.new(error, false) end - # Signage AI image generation. `kind` lets a client branch without matching - # on message text. + # Signage AI image generation and map AI editing. `kind` lets a client + # branch without matching on message text. struct ImageGenError include JSON::Serializable include YAML::Serializable @@ -269,6 +269,7 @@ module PlaceOS::Api @[AC::Route::Exception(Error::ImageGen::Busy, status_code: HTTP::Status::SERVICE_UNAVAILABLE)] @[AC::Route::Exception(Error::ImageGen::NotConfigured, status_code: HTTP::Status::SERVICE_UNAVAILABLE)] @[AC::Route::Exception(Error::ImageGen::Vendor, status_code: HTTP::Status::BAD_GATEWAY)] + @[AC::Route::Exception(Error::ImageGen::InvalidResponse, status_code: HTTP::Status::BAD_GATEWAY)] @[AC::Route::Exception(Error::ImageGen::Timeout, status_code: HTTP::Status::GATEWAY_TIMEOUT)] @[AC::Route::Exception(Error::ImageGen::Permission, status_code: HTTP::Status::FORBIDDEN)] def image_generation_failed(error) : ImageGenError diff --git a/src/placeos-rest-api/error.cr b/src/placeos-rest-api/error.cr index 115fdb66..6dd9cde4 100644 --- a/src/placeos-rest-api/error.cr +++ b/src/placeos-rest-api/error.cr @@ -37,9 +37,9 @@ module PlaceOS::Api class GuestAccessDisabled < Error end - # Signage AI image generation. Each subclass maps to its own status code in - # `Application`, and renders `{"error": ..., "kind": ...}` so a client can - # branch on the kind without matching on message text. + # Signage AI image generation and map AI editing. Each subclass maps to its + # own status code in `Application`, and renders `{"error": ..., "kind": ...}` + # so a client can branch on the kind without matching on message text. class ImageGen < Error # the caller has used up a per user or per domain quota class Quota < ImageGen @@ -69,17 +69,24 @@ module PlaceOS::Api class NotConfigured < ImageGen end + # the vendor answered, but not with something that parses as the result + # asked for (a map edit that is not JSON, or an answer cut off by the + # output token limit) + class InvalidResponse < ImageGen + end + # kind reported to the client, e.g. "moderation", "quota" def kind : String case self - when Quota then "quota" - when Moderated then "moderation" - when Busy then "busy" - when Vendor then "vendor" - when Timeout then "timeout" - when Permission then "permission" - when NotConfigured then "not_configured" - else "error" + when Quota then "quota" + when Moderated then "moderation" + when Busy then "busy" + when Vendor then "vendor" + when Timeout then "timeout" + when Permission then "permission" + when NotConfigured then "not_configured" + when InvalidResponse then "invalid_response" + else "error" end end end diff --git a/src/placeos-rest-api/utilities/image_gen/store.cr b/src/placeos-rest-api/utilities/image_gen/store.cr index ad627c27..c0dc27ee 100644 --- a/src/placeos-rest-api/utilities/image_gen/store.cr +++ b/src/placeos-rest-api/utilities/image_gen/store.cr @@ -8,6 +8,8 @@ module PlaceOS::Api::ImageGen module Store CANDIDATE_TAG = "ai-candidate" REFERENCE_TAG = "ai-reference" + # the marked up map snapshot attached to a map AI edit + MARKUP_TAG = "ai-map-markup" record Stored, upload : ::PlaceOS::Model::Upload, width : Int32?, height : Int32? diff --git a/src/placeos-rest-api/utilities/image_gen/sweep.cr b/src/placeos-rest-api/utilities/image_gen/sweep.cr index 0882aea6..61822430 100644 --- a/src/placeos-rest-api/utilities/image_gen/sweep.cr +++ b/src/placeos-rest-api/utilities/image_gen/sweep.cr @@ -71,7 +71,7 @@ module PlaceOS::Api::ImageGen cutoff = Time.utc - SIGNAGE_AI_RETENTION # bind each tag individually: an array cannot go in as one parameter - tags = [Store::CANDIDATE_TAG, Store::REFERENCE_TAG] + tags = [Store::CANDIDATE_TAG, Store::REFERENCE_TAG, Store::MARKUP_TAG] placeholders = Array.new(tags.size, "?").join(", ") args = ([cutoff] of ::PgORM::Value) + tags.map(&.as(::PgORM::Value)) diff --git a/src/placeos-rest-api/utilities/llm.cr b/src/placeos-rest-api/utilities/llm.cr new file mode 100644 index 00000000..097897b2 --- /dev/null +++ b/src/placeos-rest-api/utilities/llm.cr @@ -0,0 +1,24 @@ +require "./image_gen" +require "./llm/types" +require "./llm/schema" +require "./llm/prompt" +require "./llm/ops" +require "./llm/adapter" +require "./llm/adapters/openai_chat" +require "./llm/runner" + +module PlaceOS::Api + # Text/vision model calls for AI map editing, against the signage AI + # provider rows. + # + # A request handler validates, reserves a slot and writes a job row, then + # hands off to `Runner` in a spawned fiber. The answer is a list of edit + # operations the editor applies itself; nothing here touches the map. + module Llm + Log = ::Log.for(self) + + # shared across the process, sized by MAP_AI_MAX_CALLS, apart from the + # image slots so a burst of map edits cannot starve poster generation + class_getter slots : ImageGen::Slots { ImageGen::Slots.new(MAP_AI_MAX_CALLS) } + end +end diff --git a/src/placeos-rest-api/utilities/llm/adapter.cr b/src/placeos-rest-api/utilities/llm/adapter.cr new file mode 100644 index 00000000..8bb3b7cb --- /dev/null +++ b/src/placeos-rest-api/utilities/llm/adapter.cr @@ -0,0 +1,56 @@ +require "placeos-models/signage_ai_provider" + +require "../vendor_credentials" + +module PlaceOS::Api::Llm + # A vendor's text/vision model. Adapters take a `Request` and return a + # `Result`, or raise: + # + # - `Error::ImageGen::Moderated` when the vendor refused the prompt + # - `Error::ImageGen::Busy` when the vendor is rate limiting + # - `Error::ImageGen::Timeout` when it did not answer in time + # - `Error::ImageGen::Vendor` for anything else the vendor said + # + # They never see a job row or a request. + abstract class Adapter + include Utils::VendorCredentials + + getter row : ::PlaceOS::Model::SignageAIProvider + + def initialize(@row : ::PlaceOS::Model::SignageAIProvider) + end + + abstract def complete(request : Request) : Result + + # the model used when the row names no text_model + abstract def default_text_model : String + + def text_model : String + row.text_model.presence || default_text_model + end + + def capabilities : ProviderCapabilities + ProviderCapabilities.new( + id: row.id.to_s, + name: row.name, + provider: row.provider.to_s, + text_model: text_model, + ) + end + + # whether a row's vendor has a text adapter here + def self.supports?(row : ::PlaceOS::Model::SignageAIProvider) : Bool + row.provider.openai_compatible? + end + + def self.for(row : ::PlaceOS::Model::SignageAIProvider) : Adapter + case row.provider + in ::PlaceOS::Model::SignageAIProvider::Provider::OPENAI, + ::PlaceOS::Model::SignageAIProvider::Provider::AZURE_OPENAI + Adapters::OpenAIChat.new(row) + in ::PlaceOS::Model::SignageAIProvider::Provider::GOOGLE_VERTEX + raise Error::ImageGen::NotConfigured.new("provider #{row.name} is a Google Vertex row, which cannot edit maps yet") + end + end + end +end diff --git a/src/placeos-rest-api/utilities/llm/adapters/openai_chat.cr b/src/placeos-rest-api/utilities/llm/adapters/openai_chat.cr new file mode 100644 index 00000000..dbd59b53 --- /dev/null +++ b/src/placeos-rest-api/utilities/llm/adapters/openai_chat.cr @@ -0,0 +1,193 @@ +require "base64" +require "json" + +module PlaceOS::Api::Llm::Adapters + # OpenAI Chat Completions, and Azure OpenAI, which speaks the same request and + # response shape at a different base URL with a different auth header. + # + # On Azure the model id is the deployment name. An endpoint ending in + # `/openai/v1` is the v1 API (no api-version); any other endpoint is the + # legacy per-deployment URL with `credentials.api_version`. + class OpenAIChat < Adapter + DEFAULT_BASE = "https://api.openai.com/v1" + + # the legacy Azure chat completions API version used when the row's + # credentials name none + DEFAULT_VERSION = "2024-10-21" + + def default_text_model : String + MAP_AI_DEFAULT_OPENAI_MODEL + end + + def complete(request : Request) : Result + response = post(request.model, body_for(request, request.reasoning_effort)) + + # a model that takes no reasoning_effort rejects the whole request naming + # the parameter; sent again without it + if response.status_code == 400 && request.reasoning_effort && response.body.includes?("reasoning_effort") + response = post(request.model, body_for(request, nil)) + end + + raise_for_status(response) + parse(response.body) + rescue IO::TimeoutError + raise Error::ImageGen::Timeout.new("the model did not answer within #{MAP_AI_READ_TIMEOUT.total_seconds.to_i} seconds") + end + + private def body_for(request : Request, reasoning_effort : String?) : String + content = request.parts.map do |part| + case part + in TextPart + {type: "text", text: part.text}.to_json + in ImagePart + { + type: "image_url", + image_url: { + url: "data:#{part.mime};base64,#{Base64.strict_encode(part.bytes)}", + detail: part.detail, + }, + }.to_json + end + end + + JSON.build do |json| + json.object do + json.field "model", request.model + json.field "messages" do + json.array do + json.object do + json.field "role", "system" + json.field "content", request.system + end + json.object do + json.field "role", "user" + json.field "content" do + json.array do + content.each { |part| json.raw part } + end + end + end + end + end + json.field "max_completion_tokens", request.max_output_tokens + if (schema = request.schema) + json.field "response_format" do + json.object do + json.field "type", "json_schema" + json.field "json_schema" do + json.object do + json.field "name", request.schema_name + json.field "strict", true + json.field "schema" { schema.to_json(json) } + end + end + end + end + end + json.field "reasoning_effort", reasoning_effort if reasoning_effort + end + end + end + + private def azure? : Bool + row.provider.azure_openai? + end + + private def base : String + if (endpoint = row.endpoint.presence) + endpoint.rstrip('/') + elsif azure? + raise Error::ImageGen::NotConfigured.new("Azure OpenAI provider #{row.name} needs an endpoint") + else + DEFAULT_BASE + end + end + + private def url_for(model : String) : URI + if azure? && !base.ends_with?("/openai/v1") + version = credentials["api_version"]?.try(&.as_s?).presence || DEFAULT_VERSION + URI.parse("#{base}/openai/deployments/#{model}/chat/completions?api-version=#{version}") + else + URI.parse("#{base}/chat/completions") + end + end + + private def headers : HTTP::Headers + headers = HTTP::Headers{"Content-Type" => "application/json"} + if azure? + headers["api-key"] = credential("api_key") + else + headers["Authorization"] = "Bearer #{credential("api_key")}" + if organisation = credentials["organisation"]?.try(&.as_s?).presence + headers["OpenAI-Organization"] = organisation + end + end + headers + end + + private def post(model : String, body : String) : HTTP::Client::Response + uri = url_for(model) + ImageGen::Http.client(uri, MAP_AI_READ_TIMEOUT) do |client| + client.post(uri.request_target, headers: headers, body: body) + end + end + + private def raise_for_status(response : HTTP::Client::Response) : Nil + return if response.success? + + code = nil + message = nil + begin + error = JSON.parse(response.body)["error"]? + code = error.try(&.["code"]?).try(&.as_s?) + message = error.try(&.["message"]?).try(&.as_s?) + rescue JSON::ParseException + end + + if code == "content_filter" || code == "moderation_blocked" || code == "content_policy_violation" + raise Error::ImageGen::Moderated.new(message || "the request was blocked by the vendor's safety system") + end + + if response.status_code == 429 + raise Error::ImageGen::Busy.new(message || "the vendor is rate limiting this account") + end + + raise Error::ImageGen::Vendor.new(message || vendor_message(response.body)) + end + + private def parse(body : String) : Result + payload = begin + JSON.parse(body) + rescue JSON::ParseException + raise Error::ImageGen::Vendor.new("the vendor response was not JSON") + end + + choice = payload["choices"]?.try(&.as_a?).try(&.first?) + raise Error::ImageGen::Vendor.new("no choices in the vendor response") if choice.nil? + + message = choice["message"]? + if (refusal = message.try(&.["refusal"]?).try(&.as_s?).presence) + raise Error::ImageGen::Moderated.new(refusal) + end + + finish = choice["finish_reason"]?.try(&.as_s?) || "stop" + raise Error::ImageGen::Moderated.new("the answer was blocked by the vendor's content filter") if finish == "content_filter" + + text = case content = message.try(&.["content"]?).try(&.raw) + when String then content + when Array then content.compact_map { |part| part["text"]?.try(&.as_s?) }.join + else "" + end + + usage = payload["usage"]? + Result.new( + text: text, + finish: finish, + input_tokens: usage.try(&.["prompt_tokens"]?).try(&.as_i64?), + output_tokens: usage.try(&.["completion_tokens"]?).try(&.as_i64?), + total_tokens: usage.try(&.["total_tokens"]?).try(&.as_i64?), + vendor_id: payload["id"]?.try(&.as_s?), + ) + end + end +end diff --git a/src/placeos-rest-api/utilities/llm/ops.cr b/src/placeos-rest-api/utilities/llm/ops.cr new file mode 100644 index 00000000..3ebb29b2 --- /dev/null +++ b/src/placeos-rest-api/utilities/llm/ops.cr @@ -0,0 +1,326 @@ +require "json" +require "xml" + +module PlaceOS::Api::Llm + # Reads the model's answer and checks each operation against the context the + # editor posted. Failing ops are kept with `server_issue` set so the client can + # show them unchecked with the reason; the client's own validator, which has + # the live document, remains the authority. + module Ops + record Envelope, summary : String, ops : Array(Hash(String, JSON::Any)), questions : Array(String) + + FENCE = /\A\s*```[a-zA-Z]*\s*(.*?)\s*```\s*\z/m + + MAX_MARKUP_BYTES = 16_384 + MAX_DUPLICATES = 40 + + DENIED_ELEMENTS = %w(script foreignObject iframe embed object animate set animateMotion animateTransform use image a) + + # The whole answer as JSON, with a markdown fence stripped when the model + # wrapped it in one. + def self.parse(text : String) : Envelope + body = text.strip + if (match = FENCE.match(body)) + body = match[1] + end + raise Error::ImageGen::InvalidResponse.new("the model returned an empty answer") if body.empty? + + parsed = begin + JSON.parse(body) + rescue JSON::ParseException + raise Error::ImageGen::InvalidResponse.new("the model did not answer with JSON") + end + + hash = parsed.as_h? + raise Error::ImageGen::InvalidResponse.new("the model's answer is not an object") if hash.nil? + + ops = hash["ops"]?.try(&.as_a?) + raise Error::ImageGen::InvalidResponse.new("the model's answer has no ops list") if ops.nil? + + listed = ops.map do |op| + op.as_h? || {"op" => JSON::Any.new(nil), "server_issue" => JSON::Any.new("not an object")} + end + questions = hash["questions"]?.try(&.as_a?).try(&.compact_map(&.as_s?)) || [] of String + Envelope.new(summary: hash["summary"]?.try(&.as_s?) || "", ops: listed, questions: questions) + end + + # Each op with an `id` ("op-1" ...) and, when it fails a check, a + # `server_issue` naming the reason. + def self.check(ops : Array(Hash(String, JSON::Any)), context : JSON::Any) : Array(JSON::Any) + index = Index.new(context) + + ops.map_with_index do |op, position| + checked = op.dup + checked["id"] = JSON::Any.new("op-#{position + 1}") + issue = if position >= MAP_AI_MAX_OPS + "over the limit of #{MAP_AI_MAX_OPS} operations in one answer" + else + checked["server_issue"]?.try(&.as_s?) || issue_for(checked, index) + end + checked["server_issue"] = JSON::Any.new(issue) if issue + JSON::Any.new(checked) + end + end + + # What the editor posted, keyed for lookups. + class Index + getter anchors : Hash(String, Hash(String, JSON::Any)) + getter ids : Hash(String, Hash(String, JSON::Any)) + getter protected_ids : Set(String) + + def initialize(context : JSON::Any) + @anchors = {} of String => Hash(String, JSON::Any) + @ids = {} of String => Hash(String, JSON::Any) + (context["index"]?.try(&.as_a?) || [] of JSON::Any).each do |entry| + hash = entry.as_h? + next unless hash + if (anchor = hash["a"]?.try(&.as_s?)) + @anchors[anchor] = hash + end + if (id = hash["id"]?.try(&.as_s?)) + @ids[id] ||= hash + end + end + @protected_ids = (context["protected_ids"]?.try(&.as_a?) || [] of JSON::Any).compact_map(&.as_s?).to_set + end + + # The entry a target names, or the reason it does not resolve. + def resolve(target : JSON::Any?) : Tuple(Hash(String, JSON::Any)?, String?) + hash = target.try(&.as_h?) + return {nil, "target is missing"} if hash.nil? + + anchor = hash["a"]?.try(&.as_s?).presence + id = hash["id"]?.try(&.as_s?).presence + return {nil, "target names neither an anchor nor an id"} if anchor.nil? && id.nil? + + by_anchor = anchor ? @anchors[anchor]? : nil + by_id = id ? @ids[id]? : nil + return {nil, "anchor #{anchor} is not in the index"} if anchor && by_anchor.nil? + return {nil, "id #{id} is not in the index"} if id && by_id.nil? + if by_anchor && by_id && !by_anchor.same?(by_id) + return {nil, "anchor #{anchor} and id #{id} are different elements"} + end + + entry = (by_anchor || by_id).not_nil! + if (hash_given = hash["h"]?.try(&.as_s?).presence) && (current = entry["h"]?.try(&.as_s?)) && hash_given != current + return {nil, "the element changed since the index was built (hash mismatch)"} + end + {entry, nil} + end + + def known_id?(id : String) : Bool + @ids.has_key?(id) + end + end + + private def self.issue_for(op : Hash(String, JSON::Any), index : Index) : String? + kind = op["op"]?.try(&.as_s?) + return "unknown operation #{kind.inspect}" unless kind && Schema::OPS.includes?(kind) + + case kind + when "set_attr" + with_target(op, index) do + name = op["name"]?.try(&.as_s?).presence + return "set_attr needs a name" unless name + return "attribute #{name} may not be set" unless Schema.allowed_attribute?(name) + value = op["value"]?.try(&.as_s?) + if name == "visibility" && value && !Schema::VISIBILITY_VALUES.includes?(value.strip) + return "visibility must be one of #{Schema::VISIBILITY_VALUES.join(", ")}" + end + nil + end + when "set_style_rule" + selector = op["selector"]?.try(&.as_s?).try(&.strip).presence + declarations = op["declarations"]?.try(&.as_s?).try(&.strip).presence + return "set_style_rule needs a selector" unless selector + return "the selector * is not allowed" if selector == "*" + return "set_style_rule needs declarations" unless declarations + css_issue(declarations) + when "set_text" + with_target(op, index) do |entry| + tag = entry["tag"]?.try(&.as_s?) + tag == "text" || tag == "tspan" ? nil : "set_text only applies to text and tspan elements" + end + when "transform" + unless op["translate"]? || op["scale"]? || op["rotate"]? + return "transform needs translate, scale or rotate" + end + return "translate must be [dx, dy]" unless pair?(op["translate"]?) + return "scale must be [sx, sy]" unless pair?(op["scale"]?) + return "origin must be [x, y]" unless pair?(op["origin"]?) + with_targets(op, index) { nil } + when "delete" + with_targets(op, index) { nil } + when "duplicate" + with_target(op, index) do + return "translate must be [dx, dy]" unless pair?(op["translate"]?) + count = op["count"]?.try(&.as_i?) + return "count must be between 1 and #{MAX_DUPLICATES}" if count && (count < 1 || count > MAX_DUPLICATES) + new_id_issue(op["new_id"]?, index) + end + when "insert_shape" + shape = op["kind"]?.try(&.as_s?) + return "insert_shape needs a kind" unless shape + return "#{shape} is not a shape that can be inserted" unless Schema::SHAPES.includes?(shape) + if op["parent"]? && !op["parent"].raw.nil? + _, issue = index.resolve(op["parent"]) + return "parent: #{issue}" if issue + end + if op["after"]? && !op["after"].raw.nil? + _, issue = index.resolve(op["after"]) + return "after: #{issue}" if issue + end + (op["attrs"]?.try(&.as_a?) || [] of JSON::Any).each do |attr| + name = attr["name"]?.try(&.as_s?) + return "an attribute has no name" unless name + unless Schema.allowed_attribute?(name) || Schema::GEOMETRY_ATTRIBUTES.includes?(name) + return "attribute #{name} may not be set on a new element" + end + end + new_id_issue(op["new_id"]?, index) + when "set_id" + with_target(op, index) do + new_id = op["new_id"]? + new_id.nil? || new_id.raw.nil? ? nil : new_id_issue(new_id, index) + end + when "reorder" + with_target(op, index) do + where = [op["before"]?, op["after"]?, op["position"]?].count { |value| value && !value.raw.nil? } + return "reorder needs exactly one of before, after or position" unless where == 1 + if (position = op["position"]?.try(&.as_s?)) && !Schema::POSITIONS.includes?(position) + return "position must be first or last" + end + {op["before"]?, op["after"]?}.each do |sibling| + next if sibling.nil? || sibling.raw.nil? + _, issue = index.resolve(sibling) + return issue if issue + end + nil + end + when "group" + targets = op["targets"]?.try(&.as_a?) + return "group needs targets" if targets.nil? || targets.empty? + targets.each do |target| + _, issue = index.resolve(target) + return issue if issue + end + new_id_issue(op["new_id"]?, index) + when "ungroup" + with_target(op, index) do |entry| + entry["tag"]?.try(&.as_s?) == "g" ? nil : "ungroup only applies to a group" + end + when "replace_element" + with_target(op, index) do + markup = op["markup"]?.try(&.as_s?).try(&.strip).presence + return "replace_element needs markup" unless markup + return "the markup is larger than #{MAX_MARKUP_BYTES // 1024}KB" if markup.bytesize > MAX_MARKUP_BYTES + markup_issue(markup) + end + when "replace_path" + with_target(op, index) do |entry| + return "replace_path only applies to a path" unless entry["tag"]?.try(&.as_s?) == "path" + data = op["d"]?.try(&.as_s?).try(&.strip).presence + return "replace_path needs d" unless data + /\A[MmZzLlHhVvCcSsQqTtAa0-9\s,.\-+eE]+\z/.match(data) ? nil : "d is not path data" + end + when "no_change" + op["reason"]?.try(&.as_s?).presence ? nil : "no_change needs a reason" + when "ask" + op["question"]?.try(&.as_s?).presence ? nil : "ask needs a question" + end + end + + private def self.with_target(op : Hash(String, JSON::Any), index : Index, & : Hash(String, JSON::Any) -> String?) : String? + entry, issue = index.resolve(op["target"]?) + return issue if issue + yield entry.not_nil! + end + + private def self.with_targets(op : Hash(String, JSON::Any), index : Index, & : Array(Hash(String, JSON::Any)) -> String?) : String? + listed = [] of JSON::Any + if (target = op["target"]?) && !target.raw.nil? + listed << target + end + if (targets = op["targets"]?.try(&.as_a?)) + listed.concat(targets) + end + return "no target given" if listed.empty? + + entries = listed.map do |item| + entry, issue = index.resolve(item) + return issue if issue + entry.not_nil! + end + yield entries + end + + private def self.pair?(value : JSON::Any?) : Bool + return true if value.nil? || value.raw.nil? + list = value.as_a? + !list.nil? && list.size == 2 && list.all? { |item| item.as_f? || item.as_i? } + end + + private def self.new_id_issue(value : JSON::Any?, index : Index) : String? + return if value.nil? || value.raw.nil? + id = value.as_s? + return "new_id must be a string" if id.nil? + return "#{id.inspect} is not a valid id (letter first, then letters, digits, _ . -)" unless Schema.valid_id?(id) + return "#{id} is already an id in the map" if index.known_id?(id) + return "#{id} is a protected id" if index.protected_ids.includes?(id) + nil + end + + private def self.css_issue(declarations : String) : String? + lowered = declarations.downcase + return "declarations may not import or reference URLs" if lowered.includes?("@import") || lowered.includes?("url(") || lowered.includes?("expression(") || lowered.includes?("behavior:") + + declarations.split(';').each do |declaration| + next if declaration.strip.empty? + name, _, value = declaration.partition(':') + name = name.strip + return "declaration #{declaration.strip.inspect} has no value" if value.strip.empty? + return "property #{name} may not be set" unless Schema.allowed_attribute?(name) + end + nil + end + + # A libxml2 walk of replacement markup: one root element, nothing from the + # deny list, no event handlers, no external references. + private def self.markup_issue(markup : String) : String? + document = begin + XML.parse("#{markup}", XML::ParserOptions::NONET) + rescue + return "the markup does not parse" + end + return "the markup does not parse: #{document.errors.try(&.first?).try(&.message)}" unless document.errors.nil? || document.errors.not_nil!.empty? + + wrapper = document.root + return "the markup does not parse" if wrapper.nil? + roots = wrapper.children.select(&.element?) + return "the markup must be exactly one element" unless roots.size == 1 + + elements = [] of XML::Node + collect_elements(roots.first, elements) + elements.each do |node| + name = node.name + return "<#{name}> is not allowed" if DENIED_ELEMENTS.includes?(name) + if name == "style" && (text = node.content) && css_issue(text.gsub(/[^{}]*\{/, "").gsub('}', ';')) + return "the style content is not allowed" + end + node.attributes.each do |attribute| + attr_name = attribute.name + value = attribute.content + return "event handler attributes are not allowed" if attr_name.downcase.starts_with?("on") + return "href must point inside the document" if attr_name == "href" && !value.starts_with?('#') + return "the style attribute content is not allowed" if attr_name == "style" && css_issue(value) + end + end + nil + end + + private def self.collect_elements(node : XML::Node, found : Array(XML::Node)) : Nil + found << node if node.element? + node.children.each { |child| collect_elements(child, found) } + end + end +end diff --git a/src/placeos-rest-api/utilities/llm/prompt.cr b/src/placeos-rest-api/utilities/llm/prompt.cr new file mode 100644 index 00000000..7175d463 --- /dev/null +++ b/src/placeos-rest-api/utilities/llm/prompt.cr @@ -0,0 +1,309 @@ +require "json" + +module PlaceOS::Api::Llm + # Builds the vendor request for a map edit from the context the editor posted. + # + # Parts go in this order: rules and the op reference (system), then the index, + # the subtrees, the snapshot image, the annotations, earlier turns, and the + # instruction last, so the stable front of the prompt is the same across the + # turns of one editing session. + module Prompt + CONTEXT_SCHEMA = "placeos.map-ai-context/1" + + # byte budgets per section; roughly four bytes per token + MAX_INDEX_BYTES = 100_000 + MAX_SUBTREE_BYTES = 40_000 + MAX_SUBTREE_EACH = 8_000 + MAX_STYLE_BYTES = 16_000 + MAX_MARKUP_BYTES = 16_000 + MAX_HISTORY_TURNS = 6 + MAX_HISTORY_BYTES = 40_000 + MAX_HISTORY_OPS = 40 + + SYSTEM = <<-TEXT + You edit SVG floor-plan maps for a workplace platform by returning a list of operations; the application applies them to the SVG document, never you. + + You are given an index of the map's elements (one line each), the markup of a few shortlisted elements, sometimes a snapshot image with the person's annotations drawn on it, and an instruction. Anchors are the first column of the index (element ids where they exist, e otherwise). Groups list how many anonymous children they contain by tag. + + Rules: + - Refer to elements only by anchor or id exactly as they appear in the index, as a target object {"a": anchor, "id": id, "h": hash}. Never invent anchors or ids. Never estimate the coordinates of an existing element. + - Prefer the smallest change that satisfies the instruction. Do not touch elements the instruction does not concern. + - To move, resize or rotate an element use transform (user units); never rewrite its geometry. + - To recolour one element use set_attr with name fill or stroke on that element. To restyle every element of a class use set_style_rule with a selector like .cls-12. Never touch the style rules unless asked. + - To add a copy of an existing element use duplicate with an offset. To draw a new shape use insert_shape with a parent anchor, the element kind and its attributes; derive new geometry from the annotation or from a named reference element. New ids must start with a letter, be unique, and follow the naming pattern of their neighbours; when nothing suggests a name, start the id with "ai-". + - Text is often outlined to paths in these maps; set_text only works on text or tspan elements. If the instruction needs a label that does not exist as text, return an ask. + - Ids marked protected are bound to bookings; delete, rename or hide them only when the instruction asks for exactly that. + - If the instruction is ambiguous, or the target cannot be identified from the index, return a single ask op with a precise question instead of guessing. If nothing needs to change, return no_change. + - Every op carries a one-sentence why that names the element's label. summary is one sentence for the person. questions is usually empty. + Coordinates: user units of the viewBox; x grows to the right, y grows downward. + + Fields each operation reads (everything else stays null): + TEXT + + # One earlier turn of the same edit, oldest first. + record Turn, + instruction : String, + ops : Array(JSON::Any), + summary : String? = nil, + resolution : JSON::Any? = nil + + record Options, + context : JSON::Any, + instruction : String, + model : String, + history : Array(Turn) = [] of Turn, + image : ImagePart? = nil, + max_output_tokens : Int32 = MAP_AI_MAX_OUTPUT_TOKENS, + schema : JSON::Any? = Schema.openai + + def self.build(options : Options) : Request + context = options.context + parts = [] of Part + + parts << TextPart.new(index_text(context)) + + subtrees = subtrees_text(context) + parts << TextPart.new(subtrees) unless subtrees.empty? + + if (image = options.image) + parts << image + end + + markup = markup_text(context, !options.image.nil?) + parts << TextPart.new(markup) unless markup.empty? + + history = history_text(options.history) + parts << TextPart.new(history) unless history.empty? + + parts << TextPart.new("INSTRUCTION\n#{options.instruction.strip}") + + Request.new( + system: SYSTEM + Schema.reference, + parts: parts, + model: options.model, + max_output_tokens: options.max_output_tokens, + schema: options.schema, + schema_name: Schema::OPENAI_NAME, + ) + end + + # The map's coordinate contract, its style rules, the protected ids and one + # line per indexed element, capped by bytes with the tail dropped. + def self.index_text(context : JSON::Any) : String + String.build do |text| + text << "MAP\n" + if (view_box = context["viewBox"]?.try(&.as_a?)) + text << "viewBox: " << view_box.map { |value| number(value) }.join(' ') + text << " (all coordinates in these user units, origin top-left)\n" + end + if (image_px = context["image_px"]?.try(&.as_a?)) && (units = context["units_per_px"]?) + text << "snapshot: " << image_px.map { |value| number(value) }.join('x') + text << " px, " << number(units) << " user units per px\n" + end + if (stats = context["stats"]?.try(&.as_h?)) + text << "elements: " << stats.map { |key, value| "#{key} #{value}" }.join(", ") << '\n' + end + + rules = context["style_rules"]?.try(&.as_a?) || [] of JSON::Any + unless rules.empty? + text << "style rules: " + text << capped(rules.compact_map(&.as_s?).join(" "), MAX_STYLE_BYTES) << '\n' + end + + protected_ids = context["protected_ids"]?.try(&.as_a?) || [] of JSON::Any + unless protected_ids.empty? + text << "protected ids (bound to bookings): " << protected_ids.compact_map(&.as_s?).join(", ") << '\n' + end + + duplicates = context["duplicate_ids"]?.try(&.as_a?) || [] of JSON::Any + unless duplicates.empty? + text << "duplicate ids already in the file (not your doing): " << duplicates.compact_map(&.as_s?).join(", ") << '\n' + end + + text << "\nINDEX\nanchor | id | tag | class | label | bbox(x y w h) | parent | kind | protected | hash\n" + entries = context["index"]?.try(&.as_a?) || [] of JSON::Any + used = 0 + shown = 0 + entries.each do |entry| + line = index_line(entry) + break if used + line.bytesize > MAX_INDEX_BYTES + text << line << '\n' + used += line.bytesize + 1 + shown += 1 + end + if shown < entries.size + text << "... #{entries.size - shown} more elements omitted; ask for a narrower selection if the target is not listed\n" + end + end + end + + private def self.index_line(entry : JSON::Any) : String + hash = entry.as_h? || {} of String => JSON::Any + bbox = hash["bbox"]?.try(&.as_a?).try(&.map { |value| number(value) }.join(' ')) || "-" + label = hash["label"]?.try(&.as_s?) + [ + hash["a"]?.try(&.as_s?) || "-", + hash["id"]?.try(&.as_s?) || "-", + hash["tag"]?.try(&.as_s?) || "-", + hash["cls"]?.try(&.as_s?).presence || "-", + label ? %("#{label.gsub('"', "'")}") : "-", + bbox, + hash["p"]?.try(&.as_s?) || "root", + hash["kind"]?.try(&.as_s?) || "-", + hash["prot"]?.try(&.as_bool?) ? "protected" : "-", + hash["h"]?.try(&.as_s?) || "-", + ].join(" | ") + end + + # The markup of shortlisted elements, each cut at MAX_SUBTREE_EACH and the + # section at MAX_SUBTREE_BYTES. + def self.subtrees_text(context : JSON::Any) : String + subtrees = context["subtrees"]?.try(&.as_h?) + return "" if subtrees.nil? || subtrees.empty? + + String.build do |text| + text << "SUBTREES (markup of the shortlisted elements; use these for replace_path, replace_element and as size references)\n" + used = 0 + subtrees.each do |anchor, markup| + body = capped(markup.as_s? || markup.to_json, MAX_SUBTREE_EACH) + block = "[#{anchor}]\n#{body}\n" + if used + block.bytesize > MAX_SUBTREE_BYTES + text << "... further subtrees omitted\n" + break + end + text << block + used += block.bytesize + end + end + end + + # The person's annotations, with the hit-tested candidates under each one, + # and the selected elements. + def self.markup_text(context : JSON::Any, with_image : Bool) : String + markup = context["markup"]?.try(&.as_h?) + shortlist = context["shortlist"]?.try(&.as_a?).try(&.compact_map(&.as_s?)) || [] of String + return "" if markup.nil? && shortlist.empty? + + String.build do |text| + if markup + annotations = markup["annotations"]?.try(&.as_a?) || [] of JSON::Any + unless annotations.empty? + text << "ANNOTATIONS drawn by the person, in viewBox units" + text << "; the numbers match the badges on the snapshot" if with_image + text << '\n' + used = 0 + annotations.each do |mark| + line = annotation_line(mark) + break if used + line.bytesize > MAX_MARKUP_BYTES + text << line << '\n' + used += line.bytesize + 1 + end + end + selected = markup["selected"]?.try(&.as_a?).try(&.compact_map(&.as_s?)) || [] of String + text << "SELECTED by the person: " << selected.join(", ") << '\n' unless selected.empty? + end + text << "SHORTLIST (elements under or near the annotations and selection): " << shortlist.join(", ") << '\n' unless shortlist.empty? + end + end + + private def self.annotation_line(mark : JSON::Any) : String + hash = mark.as_h? || {} of String => JSON::Any + parts = [] of String + parts << "#{hash["n"]?.try(&.as_i?) || "?"}." + parts << (hash["kind"]?.try(&.as_s?) || "mark") + geometry = %w(x y width height x2 y2).compact_map do |key| + value = hash[key]? + value && !value.raw.nil? ? "#{key}=#{number(value)}" : nil + end + parts.concat(geometry) + if (points = hash["points"]?.try(&.as_a?)) + parts << "points=" + points.first(40).map { |point| point.as_a?.try(&.map { |value| number(value) }.join(',')) || "?" }.join(' ') + end + if (note = hash["text"]?.try(&.as_s?).presence) + parts << %(text="#{note.gsub('"', "'")}") + end + if (hits = hash["hits"]?.try(&.as_a?).try(&.compact_map(&.as_s?))) && !hits.empty? + parts << "hits: #{hits.join(", ")}" + end + if (near = hash["near"]?.try(&.as_a?).try(&.compact_map(&.as_s?))) && !near.empty? + parts << "near: #{near.join(", ")}" + end + parts.join(' ') + end + + # Earlier turns of this edit: the instruction, the ops answered and what the + # person did with them. + def self.history_text(history : Array(Turn)) : String + return "" if history.empty? + + turns = history.last(MAX_HISTORY_TURNS) + String.build do |text| + text << "PREVIOUS TURNS of this edit, oldest first; the map below is its current state, which may already include the applied operations\n" + used = 0 + turns.each_with_index do |turn, index| + block = String.build do |entry| + entry << "Turn #{index + 1} instruction: " << turn.instruction.strip << '\n' + entry << "Turn #{index + 1} summary: " << turn.summary << '\n' if turn.summary.presence + entry << "Turn #{index + 1} ops: " << turn.ops.first(MAX_HISTORY_OPS).map { |op| compact_op(op) }.to_json << '\n' + if (resolution = turn.resolution) + entry << "Turn #{index + 1} outcome: " << resolution_text(resolution) << '\n' + else + entry << "Turn #{index + 1} outcome: not reviewed\n" + end + end + if used + block.bytesize > MAX_HISTORY_BYTES + text << "... earlier turns omitted\n" + break + end + text << block + used += block.bytesize + end + end + end + + private def self.resolution_text(resolution : JSON::Any) : String + hash = resolution.as_h? || {} of String => JSON::Any + parts = [] of String + parts << (hash["decision"]?.try(&.as_s?) || "unknown") + if (accepted = hash["accepted_op_ids"]?.try(&.as_a?)) && !accepted.empty? + parts << "accepted: #{accepted.compact_map(&.as_s?).join(", ")}" + end + if (rejected = hash["rejected_op_ids"]?.try(&.as_a?)) && !rejected.empty? + parts << "rejected: #{rejected.compact_map(&.as_s?).join(", ")}" + end + if (reasons = hash["reasons"]?) + case raw = reasons.raw + when Array + parts << "reasons: #{raw.map(&.to_s).join("; ")}" unless raw.empty? + when Hash + parts << "reasons: #{raw.map { |key, value| "#{key}: #{value}" }.join("; ")}" unless raw.empty? + when String + parts << "reasons: #{raw}" unless raw.empty? + end + end + parts.join("; ") + end + + # an op with its null fields and the review bookkeeping dropped + private def self.compact_op(op : JSON::Any) : Hash(String, JSON::Any) + hash = op.as_h? || {} of String => JSON::Any + hash.reject { |key, value| value.raw.nil? || key == "why" } + end + + private def self.number(value : JSON::Any) : String + case raw = value.raw + when Int then raw.to_s + when Float then raw == raw.floor ? raw.to_i64.to_s : raw.round(2).to_s + else raw.to_s + end + end + + private def self.capped(text : String, limit : Int32) : String + return text if text.bytesize <= limit + cut = text.byte_slice(0, limit) + # never split a multibyte character + cut = cut.scrub("") + "#{cut}..." + end + end +end diff --git a/src/placeos-rest-api/utilities/llm/runner.cr b/src/placeos-rest-api/utilities/llm/runner.cr new file mode 100644 index 00000000..80597057 --- /dev/null +++ b/src/placeos-rest-api/utilities/llm/runner.cr @@ -0,0 +1,149 @@ +require "placeos-models/signage_ai_job" +require "placeos-models/upload" + +module PlaceOS::Api::Llm + # The body of the spawned fiber for a map edit: one vendor call under the + # slot the request handler reserved. + # + # Everything it needs is captured before `spawn`: inside the fiber there is + # no request and `Log.context` is empty. The context the editor posted is + # held here in memory only; the job row records its size and hash. + module Runner + record Context, + job_id : UUID, + authority_id : String, + user_id : String?, + adapter : Adapter, + model : String, + context : JSON::Any, + instruction : String, + history : Array(Prompt::Turn) = [] of Prompt::Turn, + markup_upload_id : String? = nil + + def self.run(context : Context) : Nil + ::Log.context.set( + map_ai_job: context.job_id.to_s, + authority_id: context.authority_id, + user_id: context.user_id, + ) + + started = Time.utc + job = ::PlaceOS::Model::SignageAIJob.find?(context.job_id) + return if job.nil? + + if job.cancel_requested + finish(job, started, nil, nil, nil) + return + end + + job.state = ::PlaceOS::Model::SignageAIJob::State::Running + job.started_at = started + job.save + ::PlaceOS::Model::SignageAIJob.bump_version(job.id.as(UUID)) + + result : Result? = nil + written : Hash(String, JSON::Any)? = nil + failure : Exception? = nil + + begin + request = Prompt.build(Prompt::Options.new( + context: context.context, + instruction: context.instruction, + model: context.model, + history: context.history, + image: markup_image(context.markup_upload_id), + )) + + answer = context.adapter.complete(request) + result = answer + + if answer.finish == "length" + raise Error::ImageGen::InvalidResponse.new("the answer was cut off at the output limit, try a smaller change") + end + + envelope = Ops.parse(answer.text) + ops = Ops.check(envelope.ops, context.context) + + written = { + "schema_version" => JSON::Any.new(Schema::VERSION.to_i64), + "summary" => JSON::Any.new(envelope.summary), + "ops" => JSON::Any.new(ops), + "questions" => JSON::Any.new(envelope.questions.map { |question| JSON::Any.new(question) }), + "usage" => JSON::Any.new(answer.usage.merge({"model" => JSON::Any.new(context.model)})), + } + rescue ex + Log.warn(exception: ex) { "map AI vendor call failed" } + failure = ex + end + + fresh = ::PlaceOS::Model::SignageAIJob.find?(context.job_id) + finish(fresh, started, written, result, failure) if fresh + rescue ex + Log.error(exception: ex) { "map AI job failed outside the vendor call" } + if (job = ::PlaceOS::Model::SignageAIJob.find?(context.job_id)) + finish(job, started || Time.utc, nil, nil, ex) + end + ensure + Llm.slots.release + end + + # The marked up snapshot the person attached, read back from storage. + private def self.markup_image(upload_id : String?) : ImagePart? + return unless upload_id + upload = ::PlaceOS::Model::Upload.find?(upload_id) + raise Error::ImageGen::Vendor.new("the markup image is gone") if upload.nil? + reference = ImageGen::Store.fetch(upload) + ImagePart.new(bytes: reference.bytes, mime: reference.mime) + end + + private def self.finish( + job : ::PlaceOS::Model::SignageAIJob, + started : Time, + written : Hash(String, JSON::Any)?, + result : Result?, + failure : Exception?, + ) : Nil + job.finished_at = Time.utc + job.latency_ms = (Time.utc - started).total_milliseconds.to_i64 + if (tokens = result.try(&.total_tokens)) + job.cost_units = tokens.to_f + end + job.result = JSON::Any.new(written) if written + + if job.cancel_requested + job.state = ::PlaceOS::Model::SignageAIJob::State::Cancelled + elsif written && failure.nil? + job.state = ::PlaceOS::Model::SignageAIJob::State::Done + else + job.state = ::PlaceOS::Model::SignageAIJob::State::Failed + job.error_kind = failure ? kind_of(failure) : "vendor" + job.error_message = failure ? message_of(failure) : "no answer was produced" + end + + job.save + ::PlaceOS::Model::SignageAIJob.bump_version(job.id.as(UUID)) + + Log.info { { + message: "map AI job finished", + state: job.state.to_s, + ops: written.try(&.["ops"]?).try(&.as_a?).try(&.size), + tokens: result.try(&.total_tokens), + latency: job.latency_ms, + model: job.model, + } } + end + + private def self.kind_of(error : Exception) : String + case error + when Error::ImageGen then error.as(Error::ImageGen).kind + when IO::TimeoutError then "timeout" + else "vendor" + end + end + + # never the vendor's whole body, and never the prompt + private def self.message_of(error : Exception) : String + (error.message || error.class.name).lines.first?.to_s[0, 300] + end + end +end diff --git a/src/placeos-rest-api/utilities/llm/schema.cr b/src/placeos-rest-api/utilities/llm/schema.cr new file mode 100644 index 00000000..fa531e45 --- /dev/null +++ b/src/placeos-rest-api/utilities/llm/schema.cr @@ -0,0 +1,191 @@ +require "json" + +module PlaceOS::Api::Llm + # The map edit operation vocabulary, defined once and rendered per vendor. + # + # The TypeScript types and JSON schema in user-interfaces `libs/map-editor` + # are the other copy of this vocabulary; the two have to agree field for + # field, which `ops-schema.json` in both test suites pins. + module Schema + VERSION = 1 + + OPS = %w(set_attr set_style_rule set_text transform delete duplicate insert_shape + set_id reorder group ungroup replace_element replace_path no_change ask) + + SHAPES = %w(rect circle ellipse line polygon polyline path text g) + + POSITIONS = %w(first last) + + # attributes set_attr may touch, plus data-*; style rules take the same list + ATTRIBUTES = %w(fill stroke stroke-width stroke-dasharray fill-opacity stroke-opacity + opacity class visibility rx ry font-size font-family font-weight text-anchor) + + # attributes insert_shape may set on top of ATTRIBUTES + GEOMETRY_ATTRIBUTES = %w(x y width height cx cy r x1 y1 x2 y2 points d transform) + + VISIBILITY_VALUES = %w(visible hidden collapse) + + ID_PATTERN = /\A[A-Za-z][A-Za-z0-9_.-]*\z/ + + # the prefix for ids the model makes up when none is asked for + NEW_ID_PREFIX = "ai-" + + # the fields each op reads; everything else on the op is ignored + FIELDS_BY_OP = { + "set_attr" => %w(target name value), + "set_style_rule" => %w(selector declarations), + "set_text" => %w(target text), + "transform" => %w(target targets translate scale rotate origin), + "delete" => %w(target targets), + "duplicate" => %w(target new_id translate count id_pattern), + "insert_shape" => %w(parent after kind attrs new_id text), + "set_id" => %w(target new_id), + "reorder" => %w(target before after position), + "group" => %w(targets new_id), + "ungroup" => %w(target), + "replace_element" => %w(target markup), + "replace_path" => %w(target d), + "no_change" => %w(reason), + "ask" => %w(question), + } + + # One property of a strict object. Nullable unless `required`. + class Field + getter name : String + getter type : String + getter description : String + getter? required : Bool + getter values : Array(String)? + getter properties : Array(Field)? + getter items : Field? + getter min_items : Int32? + getter max_items : Int32? + + def initialize(@name, @type, @description, @required = false, @values = nil, + @properties = nil, @items = nil, @min_items = nil, @max_items = nil) + end + end + + TARGET_PROPERTIES = [ + Field.new("a", "string", "anchor from the index, e.g. e417"), + Field.new("id", "string", "element id from the index"), + Field.new("h", "string", "content hash of the element, from the index"), + ] + + NUMBER_PAIR = Field.new("n", "number", "", required: true) + + ATTR_PROPERTIES = [ + Field.new("name", "string", "attribute name", required: true), + Field.new("value", "string", "attribute value", required: true), + ] + + OP_FIELDS = [ + Field.new("op", "string", "the operation", required: true, values: OPS), + Field.new("why", "string", "one sentence for the person reviewing the change", required: true), + Field.new("target", "object", "the element the operation applies to", properties: TARGET_PROPERTIES), + Field.new("targets", "array", "several elements, for transform, delete and group", + items: Field.new("t", "object", "", required: true, properties: TARGET_PROPERTIES)), + Field.new("name", "string", "set_attr: attribute name from the allow-list, or data-*"), + Field.new("value", "string", "set_attr: the new value, null to remove the attribute"), + Field.new("selector", "string", "set_style_rule: CSS selector of the rule to add or change"), + Field.new("declarations", "string", "set_style_rule: declarations, e.g. fill: #eee; stroke: #333"), + Field.new("text", "string", "set_text and insert_shape text: the text content"), + Field.new("translate", "array", "transform and duplicate: [dx, dy] in user units", + items: NUMBER_PAIR, min_items: 2, max_items: 2), + Field.new("scale", "array", "transform: [sx, sy]", items: NUMBER_PAIR, min_items: 2, max_items: 2), + Field.new("rotate", "number", "transform: degrees clockwise"), + Field.new("origin", "array", "transform: [x, y] the point scale and rotate act about", + items: NUMBER_PAIR, min_items: 2, max_items: 2), + Field.new("new_id", "string", "duplicate, insert_shape, group: id for the new element; set_id: the new id, null to remove the id"), + Field.new("count", "integer", "duplicate: number of copies, each offset by translate again"), + Field.new("id_pattern", "string", "duplicate: id pattern for the copies, e.g. table-01.{n:03}"), + Field.new("parent", "object", "insert_shape: the group to insert into, the root when null", properties: TARGET_PROPERTIES), + Field.new("after", "object", "insert_shape: insert after this sibling; reorder: move after this sibling", properties: TARGET_PROPERTIES), + Field.new("before", "object", "reorder: move before this sibling", properties: TARGET_PROPERTIES), + Field.new("position", "string", "reorder: first or last among the siblings", values: POSITIONS), + Field.new("kind", "string", "insert_shape: the element to create", values: SHAPES), + Field.new("attrs", "array", "insert_shape: attributes of the new element", + items: Field.new("attr", "object", "", required: true, properties: ATTR_PROPERTIES)), + Field.new("markup", "string", "replace_element: the replacement element's SVG markup"), + Field.new("d", "string", "replace_path: the new path data"), + Field.new("reason", "string", "no_change: why nothing needs to change"), + Field.new("question", "string", "ask: the question for the person"), + ] + + ENVELOPE_FIELDS = [ + Field.new("summary", "string", "one sentence for the person", required: true), + Field.new("ops", "array", "the operations, in order", required: true, + items: Field.new("op", "object", "", required: true, properties: OP_FIELDS)), + Field.new("questions", "array", "questions for the person, usually empty", required: true, + items: Field.new("q", "string", "", required: true)), + ] + + OPENAI_NAME = "map_edit" + + # The envelope in the OpenAI strict dialect: every object closed with + # additionalProperties false and every property listed in required; + # optional fields are nullable through ["T", "null"]. + @@openai : JSON::Any? + + def self.openai : JSON::Any + @@openai ||= JSON::Any.new(render_object(ENVELOPE_FIELDS)) + end + + private def self.render_object(fields : Array(Field)) : Hash(String, JSON::Any) + properties = {} of String => JSON::Any + fields.each { |field| properties[field.name] = JSON::Any.new(render(field)) } + { + "type" => JSON::Any.new("object"), + "properties" => JSON::Any.new(properties), + "required" => JSON::Any.new(fields.map { |field| JSON::Any.new(field.name) }), + "additionalProperties" => JSON::Any.new(false), + } + end + + private def self.render(field : Field) : Hash(String, JSON::Any) + rendered = if (properties = field.properties) + render_object(properties) + else + {"type" => JSON::Any.new(field.type)} + end + + unless field.required? + rendered["type"] = JSON::Any.new([JSON::Any.new(field.type), JSON::Any.new("null")]) + end + rendered["description"] = JSON::Any.new(field.description) unless field.description.empty? + + if (values = field.values) + listed = values.map { |value| JSON::Any.new(value) } + listed << JSON::Any.new(nil) unless field.required? + rendered["enum"] = JSON::Any.new(listed) + end + if (items = field.items) + rendered["items"] = JSON::Any.new(render(items)) + end + if (min = field.min_items) + rendered["minItems"] = JSON::Any.new(min.to_i64) + end + if (max = field.max_items) + rendered["maxItems"] = JSON::Any.new(max.to_i64) + end + rendered + end + + # One line per op naming the fields it reads, for the system prompt. + def self.reference : String + String.build do |text| + FIELDS_BY_OP.each do |op, fields| + text << op << ": " << fields.join(", ") << '\n' + end + end + end + + def self.valid_id?(id : String) : Bool + !ID_PATTERN.match(id).nil? + end + + def self.allowed_attribute?(name : String) : Bool + ATTRIBUTES.includes?(name) || !(/\Adata-[A-Za-z0-9_.-]+\z/ =~ name).nil? + end + end +end diff --git a/src/placeos-rest-api/utilities/llm/types.cr b/src/placeos-rest-api/utilities/llm/types.cr new file mode 100644 index 00000000..1c88db09 --- /dev/null +++ b/src/placeos-rest-api/utilities/llm/types.cr @@ -0,0 +1,61 @@ +require "json" + +module PlaceOS::Api::Llm + # One piece of the user turn. Parts are sent in the order given. + record TextPart, text : String + record ImagePart, bytes : Bytes, mime : String, detail : String = "high" + alias Part = TextPart | ImagePart + + # What an adapter is asked to do. Adapters never see a job row or a request. + # + # With a `schema` the vendor is asked for JSON matching it; without one the + # answer is free text. + record Request, + system : String, + parts : Array(Part), + model : String, + max_output_tokens : Int32, + schema : JSON::Any? = nil, + schema_name : String = "map_edit", + reasoning_effort : String? = "low" do + # every text part joined, in order + def text : String + parts.compact_map { |part| part.as?(TextPart).try(&.text) }.join("\n\n") + end + + def images : Array(ImagePart) + parts.compact_map(&.as?(ImagePart)) + end + end + + # One answer back from a vendor. `finish` is the vendor's stop reason, + # normalised to stop, length or other. + record Result, + text : String, + finish : String, + input_tokens : Int64? = nil, + output_tokens : Int64? = nil, + total_tokens : Int64? = nil, + vendor_id : String? = nil do + def usage : Hash(String, JSON::Any) + { + "input_tokens" => JSON::Any.new(input_tokens), + "output_tokens" => JSON::Any.new(output_tokens), + "total_tokens" => JSON::Any.new(total_tokens), + "finish" => JSON::Any.new(finish), + } + end + end + + struct ProviderCapabilities + include JSON::Serializable + + getter id : String + getter name : String + getter provider : String + getter text_model : String + + def initialize(@id, @name, @provider, @text_model) + end + end +end From 1e825184ba303c01bbd2da57200ce5e4cfe6ef00 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Mon, 12 Oct 2026 19:33:38 +1100 Subject: [PATCH 04/38] feat(map-ai): maps/ai routes, provider text model and text probe PPT-2833. /api/engine/v2/maps/ai: capabilities (providers that can edit maps, the default, the map quota, the limits), edit (202, the job), jobs/:id long poll, jobs index (kind, mine, zone_id), cancel, resolve (what the person did with the answer, written into result.resolution), usage (support, map edits only). The edit gate is "may update this zone": support or admin, the legacy org-zone ACL, or a support-subsystem Update, without the broad signage grant. A non-support caller sees 404 for a zone outside the authority's org zone tree. Quotas are counted by kind against map_user_per_day and map_domain_per_month (defaults 40 and 1000). The context is posted inline under MAP_AI_MAX_CONTEXT_BYTES; the row keeps its size and sha256, never the body. The markup upload must be the caller's own png, jpeg or webp under the image size cap, and is tagged for the sweep when it was uploaded for this request. Provider rows gain text_model (create, update, clear) and POST /:id/test with kind=text runs a five-token chat completion under an Llm slot. The signage jobs index takes a kind filter defaulting to the image kinds, so map jobs stay out of "recent generations". Co-Authored-By: Claude Fable 5.1 --- src/placeos-rest-api/controllers/map_ai.cr | 507 ++++++++++++++++++ .../controllers/signage/ai.cr | 39 +- .../controllers/signage/ai_providers.cr | 51 +- 3 files changed, 572 insertions(+), 25 deletions(-) create mode 100644 src/placeos-rest-api/controllers/map_ai.cr diff --git a/src/placeos-rest-api/controllers/map_ai.cr b/src/placeos-rest-api/controllers/map_ai.cr new file mode 100644 index 00000000..08e77ce1 --- /dev/null +++ b/src/placeos-rest-api/controllers/map_ai.cr @@ -0,0 +1,507 @@ +require "digest/sha256" +require "placeos-models/signage_ai_job" +require "placeos-models/signage_ai_provider" +require "placeos-models/upload" +require "placeos-models/zone" + +require "./application" + +module PlaceOS::Api + # AI editing of a level's SVG map, with the signage AI provider rows. + # + # The editor posts the instruction and the context it built from the map (the + # element index, the shortlisted subtrees, the annotations and a marked up + # snapshot by upload id). The answer is a list of operations the editor + # applies, reviews and saves itself; nothing here reads or writes the map. + # One job is one vendor call, long polled through `jobs/:id` as signage is. + @[AC::MCP(hide: true)] + class MapAI < Application + include Utils::Permissions + include Utils::GroupPermissions + include Utils::AIJobs + + base "/api/engine/v2/maps/ai" + + # Scopes + ############################################################################################### + + before_action :can_read, only: [:capabilities, :show_job, :index_jobs, :usage] + before_action :can_write, only: [:edit, :cancel, :resolve] + + ############################################################################################### + + getter authority : ::PlaceOS::Model::Authority { current_authority.as(::PlaceOS::Model::Authority) } + + @[AC::Route::Filter(:before_action, only: [:show_job, :cancel, :resolve])] + def find_current_job(id : UUID) + Log.context.set(map_ai_job: id.to_s) + job = authority_job(id, authority.id.as(String)) + raise Error::NotFound.new("no such job") unless job.kind.map_edit? + @current_job = job + end + + getter! current_job : ::PlaceOS::Model::SignageAIJob + + # Requests + ############################################################################################### + + struct EditParams + include JSON::Serializable + + getter zone_id : String = "" + getter instruction : String = "" + # placeos.map-ai-context/1, built by the editor + getter context : JSON::Any? = nil + getter markup_upload_id : String? = nil + getter parent_job_id : UUID? = nil + getter provider_id : UUID? = nil + getter model : String? = nil + getter idempotency_key : String? = nil + end + + DECISIONS = %w(applied partial rejected refined abandoned) + + struct ResolveParams + include JSON::Serializable + + getter decision : String = "" + getter accepted_op_ids : Array(String) = [] of String + getter rejected_op_ids : Array(String) = [] of String + getter reasons : JSON::Any? = nil + getter saved_upload_id : String? = nil + end + + # Responses + ############################################################################################### + + struct Capabilities + include JSON::Serializable + + getter? enabled : Bool + getter reason : String? + getter providers : Array(Llm::ProviderCapabilities) + getter default_provider_id : String? + getter quota : NamedTuple(user_remaining_today: Int32?, domain_remaining_month: Int32?) + getter limits : NamedTuple(max_context_bytes: Int32, max_ops: Int32, max_instruction_chars: Int32) + + def initialize(@enabled, @providers, @default_provider_id, @quota, @reason = nil) + @limits = { + max_context_bytes: MAP_AI_MAX_CONTEXT_BYTES, + max_ops: MAP_AI_MAX_OPS, + max_instruction_chars: MAP_AI_MAX_INSTRUCTION_CHARS, + } + end + end + + struct JobResponse + include JSON::Serializable + + getter id : String + getter state : String + getter kind : String + getter zone_id : String? + getter provider : String? + getter model : String? + getter parent_job_id : String? + getter version : Int32 + getter instruction : String? + getter markup_upload_id : String? + getter schema_version : Int64? + getter summary : String? + getter ops : Array(JSON::Any) + getter questions : Array(JSON::Any) + getter usage : JSON::Any? + getter resolution : JSON::Any? + getter error_kind : String? + getter error_message : String? + getter cost_units : Float64? + getter latency_ms : Int64? + getter created_at : Int64? + getter finished_at : Int64? + + def initialize(job : ::PlaceOS::Model::SignageAIJob) + result = job.result + request = job.request + @id = job.id.to_s + @state = job.state.to_s + @kind = job.kind.to_s + @zone_id = request["zone_id"]?.try(&.as_s?) + @provider = job.provider_type + @model = job.model + @parent_job_id = job.parent_job_id.try(&.to_s) + @version = job.version + @instruction = job.prompt + @markup_upload_id = request["markup_upload_id"]?.try(&.as_s?) + @schema_version = result["schema_version"]?.try(&.as_i64?) + @summary = result["summary"]?.try(&.as_s?) + @ops = result["ops"]?.try(&.as_a?) || [] of JSON::Any + @questions = result["questions"]?.try(&.as_a?) || [] of JSON::Any + @usage = result["usage"]? + @resolution = result["resolution"]? + @error_kind = job.error_kind + @error_message = job.error_message + @cost_units = job.cost_units + @latency_ms = job.latency_ms + @created_at = job.created_at.try(&.to_unix) + @finished_at = job.finished_at.try(&.to_unix) + end + end + + # Routes + ############################################################################################### + + # what this domain can do, and what the caller has left of their quota + @[AC::Route::GET("/capabilities")] + def capabilities : Capabilities + return disabled("the feature is switched off") if MAP_AI_DISABLED + + rows = ::PlaceOS::Model::SignageAIProvider.available_for(authority.id) + return disabled("no AI provider is configured for this domain") if rows.empty? + + usable = rows.select { |row| Llm::Adapter.supports?(row) } + return disabled("the configured AI provider cannot edit maps yet") if usable.empty? + + default = default_row(usable) + + Capabilities.new( + enabled: true, + providers: usable.map { |row| Llm::Adapter.for(row).capabilities }, + default_provider_id: default.id.to_s, + quota: remaining_quota(default), + ) + end + + # ask for operations that carry out an instruction on a level's map + @[AC::Route::POST("/edit", body: :params, status_code: HTTP::Status::ACCEPTED)] + def edit(params : EditParams) : JobResponse + raise Error::ImageGen::NotConfigured.new("the feature is switched off") if MAP_AI_DISABLED + + instruction = params.instruction.strip + raise Error::ModelValidation.new([Error::Field.new(:instruction, "is required")]) if instruction.empty? + if instruction.size > MAP_AI_MAX_INSTRUCTION_CHARS + raise Error::ModelValidation.new([Error::Field.new(:instruction, "is longer than #{MAP_AI_MAX_INSTRUCTION_CHARS} characters")]) + end + raise Error::ModelValidation.new([Error::Field.new(:zone_id, "is required")]) if params.zone_id.blank? + + zone = ::PlaceOS::Model::Zone.find?(params.zone_id) + raise Error::NotFound.new("no such zone") if zone.nil? || !zone_visible?(zone) + raise Error::Forbidden.new("you cannot change this zone's map") unless zone_updatable?(zone) + + context = params.context + if context.nil? || context.as_h?.nil? + raise Error::ModelValidation.new([Error::Field.new(:context, "is required")]) + end + unless context["schema"]?.try(&.as_s?) == Llm::Prompt::CONTEXT_SCHEMA + raise Error::ModelValidation.new([Error::Field.new(:context, "must be #{Llm::Prompt::CONTEXT_SCHEMA}")]) + end + index = context["index"]?.try(&.as_a?) + raise Error::ModelValidation.new([Error::Field.new(:context, "needs an index")]) if index.nil? + context_body = context.to_json + if context_body.bytesize > MAP_AI_MAX_CONTEXT_BYTES + raise Error::ModelValidation.new([Error::Field.new(:context, "is larger than #{MAP_AI_MAX_CONTEXT_BYTES // 1024}KB")]) + end + + # a repeat of a submission already accepted returns the same job rather + # than spending again + if (key = params.idempotency_key.presence) + existing = on_primary do + ::PlaceOS::Model::SignageAIJob.where(user_id: current_user.id.as(String), idempotency_key: key).first? + end + return JobResponse.new(existing) if existing + end + + row = if (provider_id = params.provider_id) + found = ::PlaceOS::Model::SignageAIProvider.find?(provider_id) + raise Error::NotFound.new("no such provider") if found.nil? + unless found.authority_id.nil? || found.authority_id == authority.id + raise Error::Forbidden.new("that provider belongs to another domain") + end + found + else + usable = ::PlaceOS::Model::SignageAIProvider.available_for(authority.id).select { |entry| Llm::Adapter.supports?(entry) } + usable.empty? ? nil : default_row(usable) + end + raise Error::ImageGen::NotConfigured.new("no AI provider is configured for this domain") if row.nil? + raise Error::ImageGen::NotConfigured.new("that provider is switched off") unless row.enabled + + adapter = Llm::Adapter.for(row) + chosen_model = params.model.presence || adapter.text_model + + user_limit, domain_limit = quotas_for(row) + used_today = ::PlaceOS::Model::SignageAIJob.sum_candidates(current_user.id.as(String), 1.day.ago, ::PlaceOS::Model::SignageAIJob::Kind::MapEdit) + raise Error::ImageGen::Quota.new("you have used your map editing allowance for today") if used_today + 1 > user_limit + used_month = ::PlaceOS::Model::SignageAIJob.sum_candidates_for_authority(authority.id.as(String), 30.days.ago, ::PlaceOS::Model::SignageAIJob::Kind::MapEdit) + raise Error::ImageGen::Quota.new("this domain has used its map editing allowance for the month") if used_month + 1 > domain_limit + + markup = params.markup_upload_id.presence.try { |upload_id| markup_upload(upload_id) } + + parent = if (parent_id = params.parent_job_id) + found = ::PlaceOS::Model::SignageAIJob.find?(parent_id) + if found.nil? || found.authority_id != authority.id || !found.kind.map_edit? + raise Error::NotFound.new("no such parent job") + end + found + end + + # the slot is taken before anything is written, so a busy replica never + # leaves a job nobody is working on + unless Llm.slots.try_reserve(1) + raise Error::ImageGen::Busy.new("too many map edits are running right now, try again in a moment") + end + + job = ::PlaceOS::Model::SignageAIJob.new( + authority_id: authority.id.as(String), + provider_id: row.id.as(UUID), + provider_type: row.provider.to_s, + model: chosen_model, + user_id: current_user.id, + user_email: current_user.email.to_s, + user_name: current_user.name, + parent_job_id: parent.try(&.id.as(UUID)), + kind: ::PlaceOS::Model::SignageAIJob::Kind::MapEdit, + candidates: 1, + idempotency_key: params.idempotency_key.presence, + prompt: instruction, + ) + job.request = JSON::Any.new({ + "zone_id" => JSON::Any.new(zone.id.as(String)), + "markup_upload_id" => JSON::Any.new(markup.try(&.id.as(String))), + "provider_id" => JSON::Any.new(row.id.to_s), + "model" => JSON::Any.new(chosen_model), + "context_schema" => JSON::Any.new(Llm::Prompt::CONTEXT_SCHEMA), + "context_bytes" => JSON::Any.new(context_body.bytesize.to_i64), + "context_sha256" => JSON::Any.new(Digest::SHA256.hexdigest(context_body)), + "element_count" => JSON::Any.new(index.size.to_i64), + }) + job.upload_ids = markup ? [markup.id.as(String)] : [] of String + + unless job.save + Llm.slots.release + + # a concurrent submission of the same key got there first: the partial + # unique index is what enforces the key, so answer with the job that won + if (key = params.idempotency_key.presence) + existing = on_primary do + ::PlaceOS::Model::SignageAIJob.where(user_id: current_user.id.as(String), idempotency_key: key).first? + end + return JobResponse.new(existing) if existing + end + + raise Error::ModelValidation.new(job.errors) + end + + runner_context = Llm::Runner::Context.new( + job_id: job.id.as(UUID), + authority_id: authority.id.as(String), + user_id: current_user.id, + adapter: adapter, + model: chosen_model, + context: context, + instruction: instruction, + history: history_for(parent), + markup_upload_id: markup.try(&.id.as(String)), + ) + + spawn { Llm::Runner.run(runner_context) } + + JobResponse.new(job) + end + + # long poll: holds until the job changes past `since`, or `wait` runs out + @[AC::Route::GET("/jobs/:id")] + def show_job( + @[AC::Param::Info(description: "seconds to hold the request open, 0 to answer immediately", example: "25")] + wait : Int32 = 0, + @[AC::Param::Info(description: "the version the caller already has", example: "3")] + since : Int32? = nil, + ) : JobResponse + JobResponse.new(await_job_change(current_job, wait, since)) + end + + # recent map edit jobs + @[AC::Route::GET("/jobs")] + def index_jobs( + @[AC::Param::Info(description: "only the caller's own jobs", example: "true")] + mine : Bool = true, + @[AC::Param::Info(description: "only jobs against this zone", example: "zone-1234")] + zone_id : String? = nil, + @[AC::Param::Info(description: "map_edit; another kind lists that kind instead", example: "map_edit")] + kind : String = "map_edit", + limit : Int32 = 20, + ) : Array(JobResponse) + limit = limit.clamp(1, 100) + + parsed = ::PlaceOS::Model::SignageAIJob::Kind.parse?(kind) + raise Error::ModelValidation.new([Error::Field.new(:kind, "must be one of generate, edit, map_edit")]) if parsed.nil? + + query = ::PlaceOS::Model::SignageAIJob + .where(authority_id: authority.id.as(String)) + .where("kind = ?", ::PlaceOS::Model::SignageAIJob.kind_label(parsed)) + if mine + query = query.where(user_id: current_user.id.as(String)) + else + check_support + end + if (zone = zone_id.presence) + query = query.where("request->>'zone_id' = ?", zone) + end + + query + .where("created_at > ?", 7.days.ago) + .order(created_at: :desc) + .limit(limit) + .to_a + .map { |job| JobResponse.new(job) } + end + + # ask a running job to stop. A call already with the vendor runs to completion. + @[AC::Route::POST("/jobs/:id/cancel")] + def cancel : JobResponse + job = current_job + raise Error::Forbidden.new unless job.user_id == current_user.id || user_support? + + unless job.final? + job.cancel_requested = true + job.save + ::PlaceOS::Model::SignageAIJob.bump_version(job.id.as(UUID)) + job = ::PlaceOS::Model::SignageAIJob.find!(job.id.as(UUID)) + end + + JobResponse.new(job) + end + + # record what the person did with the answer: which ops were applied or + # rejected and why, and the upload the saved map became + @[AC::Route::POST("/jobs/:id/resolve", body: :params)] + def resolve(params : ResolveParams) : JobResponse + job = current_job + raise Error::Forbidden.new unless job.user_id == current_user.id || user_support? + unless DECISIONS.includes?(params.decision) + raise Error::ModelValidation.new([Error::Field.new(:decision, "must be one of #{DECISIONS.join(", ")}")]) + end + raise Error::ModelValidation.new([Error::Field.new(:id, "the job has not finished")]) unless job.final? + + result = job.result.as_h.dup + result["resolution"] = JSON::Any.new({ + "decision" => JSON::Any.new(params.decision), + "accepted_op_ids" => JSON::Any.new(params.accepted_op_ids.map { |id| JSON::Any.new(id) }), + "rejected_op_ids" => JSON::Any.new(params.rejected_op_ids.map { |id| JSON::Any.new(id) }), + "reasons" => params.reasons || JSON::Any.new(nil), + "saved_upload_id" => JSON::Any.new(params.saved_upload_id.presence), + "resolved_at" => JSON::Any.new(Time.utc.to_unix), + "resolved_by" => JSON::Any.new(current_user.id), + }) + job.result = JSON::Any.new(result) + job.save + ::PlaceOS::Model::SignageAIJob.bump_version(job.id.as(UUID)) + + JobResponse.new(::PlaceOS::Model::SignageAIJob.find!(job.id.as(UUID))) + end + + # map edit spend per provider and model + @[AC::Route::GET("/usage")] + def usage( + @[AC::Param::Info(description: "unix seconds, defaults to 30 days ago")] + from : Int64? = nil, + @[AC::Param::Info(description: "unix seconds, defaults to now")] + to : Int64? = nil, + ) : Array(::PlaceOS::Model::SignageAIJob::UsageRow) + check_support + + start = from ? Time.unix(from) : 30.days.ago + finish = to ? Time.unix(to) : Time.utc + ::PlaceOS::Model::SignageAIJob.usage(authority.id.as(String), start, finish, ::PlaceOS::Model::SignageAIJob::Kind::MapEdit) + end + + # Internals + ############################################################################################### + + # how recently an upload must have been made to count as the throwaway + # snapshot of this request rather than a file from the library + MARKUP_TAG_WINDOW = 15.minutes + + IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"} + + private def disabled(reason : String) : Capabilities + Capabilities.new( + enabled: false, + providers: [] of Llm::ProviderCapabilities, + default_provider_id: nil, + quota: {user_remaining_today: nil, domain_remaining_month: nil}, + reason: reason, + ) + end + + # the domain's default row when it can edit maps, else the first that can + private def default_row(usable : Array(::PlaceOS::Model::SignageAIProvider)) : ::PlaceOS::Model::SignageAIProvider + default = ::PlaceOS::Model::SignageAIProvider.default_for(authority.id) + if default && usable.any? { |row| row.id == default.id } + default + else + usable.first + end + end + + # A zone outside the caller's org tree reads as unknown to anyone but + # support, which can change any zone's map through `PATCH /zones/:id` as well. + private def zone_visible?(zone : ::PlaceOS::Model::Zone) : Bool + return true if user_support? + org_zone_id = support_org_zone_id + return true if org_zone_id.nil? + zone.root_zone_id == org_zone_id + end + + private def quotas_for(row : ::PlaceOS::Model::SignageAIProvider?) : Tuple(Int32, Int32) + user = row.try(&.quota("map_user_per_day")) || MAP_AI_USER_PER_DAY + domain = row.try(&.quota("map_domain_per_month")) || MAP_AI_DOMAIN_PER_MONTH + {user, domain} + end + + private def remaining_quota(row : ::PlaceOS::Model::SignageAIProvider?) + user_limit, domain_limit = quotas_for(row) + kind = ::PlaceOS::Model::SignageAIJob::Kind::MapEdit + used_today = ::PlaceOS::Model::SignageAIJob.sum_candidates(current_user.id.as(String), 1.day.ago, kind) + used_month = ::PlaceOS::Model::SignageAIJob.sum_candidates_for_authority(authority.id.as(String), 30.days.ago, kind) + + { + user_remaining_today: Math.max(0, user_limit - used_today), + domain_remaining_month: Math.max(0, domain_limit - used_month), + } + end + + # The marked up snapshot: the caller's own image, under the size cap, tagged + # for the sweep when it was uploaded for this request. + private def markup_upload(upload_id : String) : ::PlaceOS::Model::Upload + upload = ::PlaceOS::Model::Upload.find?(upload_id) + raise Error::NotFound.new("no such upload") if upload.nil? + raise Error::ImageGen::Permission.new("that image is not yours") unless upload.uploaded_by == current_user.id + if upload.file_size > SIGNAGE_AI_MAX_IMAGE_BYTES + raise Error::ImageGen::Permission.new("that image is larger than #{SIGNAGE_AI_MAX_IMAGE_BYTES // (1024 * 1024)}MB") + end + unless IMAGE_EXTENSIONS.includes?(File.extname(upload.file_name).downcase) + raise Error::ImageGen::Permission.new("that file is not a png, jpeg or webp image") + end + + if upload.tags.empty? && upload.created_at > MARKUP_TAG_WINDOW.ago + upload.tags = [ImageGen::Store::MARKUP_TAG] + upload.save + end + upload + end + + # The refine chain, oldest first, as the prompt builder takes it. + private def history_for(parent : ::PlaceOS::Model::SignageAIJob?) : Array(Llm::Prompt::Turn) + return [] of Llm::Prompt::Turn if parent.nil? + + (parent.chain + [parent]).select(&.kind.map_edit?).map do |job| + result = job.result + Llm::Prompt::Turn.new( + instruction: job.prompt || "", + ops: result["ops"]?.try(&.as_a?) || [] of JSON::Any, + summary: result["summary"]?.try(&.as_s?), + resolution: result["resolution"]?, + ) + end + end + end +end diff --git a/src/placeos-rest-api/controllers/signage/ai.cr b/src/placeos-rest-api/controllers/signage/ai.cr index 2d9d4b99..497e7380 100644 --- a/src/placeos-rest-api/controllers/signage/ai.cr +++ b/src/placeos-rest-api/controllers/signage/ai.cr @@ -18,6 +18,7 @@ module PlaceOS::Api @[AC::MCP(hide: true)] class SignageAI < Application include Utils::GroupPermissions + include Utils::AIJobs base "/api/engine/v2/signage/ai" @@ -34,9 +35,7 @@ module PlaceOS::Api @[AC::Route::Filter(:before_action, only: [:show_job, :cancel, :claim])] def find_current_job(id : UUID) Log.context.set(signage_ai_job: id.to_s) - job = ::PlaceOS::Model::SignageAIJob.find!(id) - raise Error::NotFound.new("no such job") unless job.authority_id == authority.id - @current_job = job + @current_job = authority_job(id, authority.id.as(String)) end getter! current_job : ::PlaceOS::Model::SignageAIJob @@ -254,21 +253,7 @@ module PlaceOS::Api @[AC::Param::Info(description: "the version the caller already has", example: "3")] since : Int32? = nil, ) : JobResponse - job = current_job - return JobResponse.new(job) if wait <= 0 || job.final? - - deadline = Time.utc + Math.min(wait, MAX_WAIT).seconds - known = since || job.version - - while Time.utc < deadline - sleep POLL_INTERVAL - fresh = on_primary { ::PlaceOS::Model::SignageAIJob.find?(job.id.as(UUID)) } - next if fresh.nil? - job = fresh - break if job.version > known || job.final? - end - - JobResponse.new(job) + JobResponse.new(await_job_change(current_job, wait, since)) end # the caller's recent jobs, for the recent generations list @@ -277,10 +262,23 @@ module PlaceOS::Api @[AC::Param::Info(description: "only the caller's own jobs", example: "true")] mine : Bool = true, limit : Int32 = 20, + @[AC::Param::Info(description: "generate, edit or map_edit; the image kinds when left out", example: "generate")] + kind : String? = nil, ) : Array(JobResponse) limit = limit.clamp(1, 100) - query = ::PlaceOS::Model::SignageAIJob.where(authority_id: authority.id.as(String)) + kinds = if kind + parsed = ::PlaceOS::Model::SignageAIJob::Kind.parse?(kind) + raise Error::ModelValidation.new([Error::Field.new(:kind, "must be one of generate, edit, map_edit")]) if parsed.nil? + [parsed] + else + ::PlaceOS::Model::SignageAIJob::Kind.image + end + labels = kinds.map { |entry| ::PlaceOS::Model::SignageAIJob.kind_label(entry) } + + query = ::PlaceOS::Model::SignageAIJob + .where(authority_id: authority.id.as(String)) + .where("kind IN (#{labels.map { "?" }.join(", ")})", args: labels.map(&.as(::PgORM::Value))) if mine query = query.where(user_id: current_user.id.as(String)) else @@ -371,9 +369,6 @@ module PlaceOS::Api # Internals ############################################################################################### - MAX_WAIT = 25 - POLL_INTERVAL = 500.milliseconds - private def disabled(reason : String) : Capabilities Capabilities.new( enabled: false, diff --git a/src/placeos-rest-api/controllers/signage/ai_providers.cr b/src/placeos-rest-api/controllers/signage/ai_providers.cr index e03271f7..cf9bb160 100644 --- a/src/placeos-rest-api/controllers/signage/ai_providers.cr +++ b/src/placeos-rest-api/controllers/signage/ai_providers.cr @@ -3,7 +3,7 @@ require "placeos-models/signage_ai_provider" require "../application" module PlaceOS::Api - # Vendor credentials for signage image generation. + # Vendor credentials for signage image generation and AI map editing. # # The Data Stores rule: sys_admin writes, support reads. Responses never carry # the credentials, so every route renders `as_json` rather than the row. @@ -57,6 +57,7 @@ module PlaceOS::Api getter endpoint : String? = nil getter location : String? = nil getter default_model : String? = nil + getter text_model : String? = nil getter allowed_models : Array(String)? = nil getter enabled : Bool? = nil getter is_default : Bool? = nil @@ -71,6 +72,7 @@ module PlaceOS::Api endpoint: String?, location: String?, default_model: String?, + text_model: String?, allowed_models: Array(String), enabled: Bool, is_default: Bool, @@ -169,10 +171,21 @@ module PlaceOS::Api include JSON::Serializable end + # the text probe: a few output tokens, discarded + TEXT_PROBE = "Reply with the single word OK" + TEXT_PROBE_TOKENS = 5 + # Prove a row's credentials work, without leaving anything behind: one small - # image, discarded. + # image, discarded, or with `kind=text` one tiny chat completion against the + # row's text model. @[AC::Route::POST("/:id/test")] - def test : TestResult + def test( + @[AC::Param::Info(description: "image (the default) or text", example: "text")] + kind : String = "image", + ) : TestResult + return test_text if kind == "text" + raise Error::ModelValidation.new([Error::Field.new(:kind, "must be image or text")]) unless kind == "image" + row = current_provider adapter = ImageGen::Adapter.for(row) model = row.default_model || adapter.capabilities.default_model @@ -210,6 +223,37 @@ module PlaceOS::Api end end + private def test_text : TestResult + row = current_provider + model = row.text_model.presence + started = Time.utc + + unless Llm.slots.try_reserve(1) + raise Error::ImageGen::Busy.new("too many map edits are running right now, try again in a moment") + end + + begin + adapter = Llm::Adapter.for(row) + model = adapter.text_model + adapter.complete(Llm::Request.new( + system: "You are a connectivity check.", + parts: [Llm::TextPart.new(TEXT_PROBE)] of Llm::Part, + model: model, + max_output_tokens: TEXT_PROBE_TOKENS, + schema: nil, + reasoning_effort: nil, + )) + + TestResult.new(ok: true, latency_ms: (Time.utc - started).total_milliseconds.to_i64, model: model) + rescue ex : Error::ImageGen + TestResult.new(false, (Time.utc - started).total_milliseconds.to_i64, model, ex.message, ex.kind) + rescue ex + TestResult.new(false, (Time.utc - started).total_milliseconds.to_i64, model, ex.message, "vendor") + ensure + Llm.slots.release(1) + end + end + private def parse_provider(value : String?) : ::PlaceOS::Model::SignageAIProvider::Provider return ::PlaceOS::Model::SignageAIProvider::Provider::OPENAI if value.nil? ::PlaceOS::Model::SignageAIProvider::Provider.parse(value) @@ -225,6 +269,7 @@ module PlaceOS::Api row.endpoint = params.endpoint.try(&.presence) if params.endpoint row.location = params.location.try(&.presence) if params.location row.default_model = params.default_model.try(&.presence) if params.default_model + row.text_model = params.text_model.try(&.presence) if params.text_model row.allowed_models = params.allowed_models.not_nil! if params.allowed_models row.enabled = params.enabled.not_nil! unless params.enabled.nil? row.is_default = params.is_default.not_nil! unless params.is_default.nil? From 3eec16bb11b21d84e21e8b27e1074c6794e73cc2 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Mon, 12 Oct 2026 19:33:38 +1100 Subject: [PATCH 05/38] test(map-ai): vendor stubs, controller, prompt, schema and ops specs PPT-2833. Chat completions stubs keyed on the /v1/chat/completions path so the image stub (which matches every api.openai.com POST, and WebMock uses the first registered match) cannot swallow them: a verbatim answer with the requests it received, a refusal, a timeout, prose, and a 400 naming reasoning_effort followed by the answer. map_ai_spec covers capabilities (no provider, disabled, Vertex only, on with the text model and the map quota), the edit path end to end with what reached the vendor, the markup snapshot as an image part in the right position, fenced JSON, prose and a cut-off answer as invalid_response, a flagged unknown anchor, refusal, timeout, the reasoning_effort retry, the map quota by kind, the read-only scope, the concierge org-zone ACL, a caller who cannot update the zone, the broad signage grant, a zone outside the organisation, the context caps, idempotency replay, the parent chain in the prompt, an image job as parent, the jobs index and its kind filter on both controllers, long poll and cancel, resolve, and usage by kind. llm_prompt_spec covers the message order, the caps, the parent chain, the strict rendering of the schema, and every check in Llm::Ops. The storage GET stub now answers with a body_io: WebMock hands a stubbed response to Http.get_bytes's streaming block as-is, so with a body string Store.fetch always failed under WebMock and no spec had proven the source, reference or markup fetch path. Co-Authored-By: Claude Fable 5.1 --- spec/controllers/map_ai_spec.cr | 673 ++++++++++++++++++ .../signage/signage_ai_providers_spec.cr | 49 ++ spec/fixtures/map_ai/context.json | 33 + spec/llm_prompt_spec.cr | 309 ++++++++ spec/spec_helpers/http_mocks.cr | 81 ++- 5 files changed, 1143 insertions(+), 2 deletions(-) create mode 100644 spec/controllers/map_ai_spec.cr create mode 100644 spec/fixtures/map_ai/context.json create mode 100644 spec/llm_prompt_spec.cr diff --git a/spec/controllers/map_ai_spec.cr b/spec/controllers/map_ai_spec.cr new file mode 100644 index 00000000..f50fe479 --- /dev/null +++ b/spec/controllers/map_ai_spec.cr @@ -0,0 +1,673 @@ +require "../helper" + +module PlaceOS::Api + # A level zone under the org zone the auth helper points every authority at, + # which is what the legacy ACL (concierge: manage) and the AI gate look at. + def self.map_ai_level : Model::Zone + zone = Model::Generator.zone + zone.parent_id = Spec::Authentication.org_zone.id.as(String) + zone.tags = Set{"level"} + zone.map_id = "/api/engine/v2/uploads/upload-spec/url" + zone.save! + end + + def self.map_ai_context : JSON::Any + JSON.parse(File.read(File.join(__DIR__, "../fixtures/map_ai/context.json"))) + end + + # the model's answer for the fixture context: one grounded recolour + MAP_AI_ANSWER = { + summary: "Recoloured Meeting 1.01 light green.", + ops: [ + { + op: "set_attr", + why: "Meeting 1.01 is the room the instruction names.", + target: {a: "e2", id: "area-1.01-status", h: "b2b2b2b2"}, + name: "fill", + value: "#90ee90", + }, + ], + questions: [] of String, + }.to_json + + describe MapAI do + base = MapAI.base_route + + before_each do + Model::SignageAIJob.clear + Model::SignageAIProvider.clear + Model::Upload.clear + Model::Storage.clear + clear_group_tables + + WebMock.allow_net_connect = false + end + + describe "capabilities" do + it "is off, with a reason, when the domain has no provider" do + result = client.get(File.join(base, "capabilities"), headers: Spec::Authentication.headers) + result.status_code.should eq 200 + + body = JSON.parse(result.body) + body["enabled"].as_bool.should be_false + body["reason"].as_s.should eq "no AI provider is configured for this domain" + body["providers"].as_a.should be_empty + body["limits"]["max_context_bytes"].as_i.should eq MAP_AI_MAX_CONTEXT_BYTES + end + + it "is off when the only provider is switched off" do + authority = Model::Authority.find_by_domain("localhost").not_nil! + Model::Generator.signage_ai_provider(authority: authority, name: "off-#{random_name}", enabled: false).save! + + body = JSON.parse(client.get(File.join(base, "capabilities"), headers: Spec::Authentication.headers).body) + body["enabled"].as_bool.should be_false + end + + it "is off when the only provider cannot edit maps yet" do + authority = Model::Authority.find_by_domain("localhost").not_nil! + Model::Generator.signage_ai_provider( + authority: authority, + name: "vertex-#{random_name}", + provider: Model::SignageAIProvider::Provider::GOOGLE_VERTEX, + credentials: %({"project_id":"p","client_email":"e","private_key":"k"}), + ).save! + + body = JSON.parse(client.get(File.join(base, "capabilities"), headers: Spec::Authentication.headers).body) + body["enabled"].as_bool.should be_false + body["reason"].as_s.should eq "the configured AI provider cannot edit maps yet" + end + + it "is on, with the text model and what is left of the map quota" do + _, _, provider = setup_signage_ai(quotas: {"map_user_per_day" => JSON::Any.new(5_i64), "user_per_day" => JSON::Any.new(1_i64)}) + provider.text_model = "gpt-5.4-mini-custom" + provider.save! + user, headers = Spec::Authentication.authentication + + # image jobs do not count against the map quota, map jobs do + Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 2).save! + Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit).save! + + result = client.get(File.join(base, "capabilities"), headers: headers) + result.status_code.should eq 200 + + body = JSON.parse(result.body) + body["enabled"].as_bool.should be_true + body["default_provider_id"].as_s.should eq provider.id.to_s + body["quota"]["user_remaining_today"].as_i.should eq 4 + body["limits"]["max_ops"].as_i.should eq MAP_AI_MAX_OPS + + providers = body["providers"].as_a + providers.size.should eq 1 + providers.first["provider"].as_s.should eq "OPENAI" + providers.first["text_model"].as_s.should eq "gpt-5.4-mini-custom" + end + + it "falls back to the default text model" do + setup_signage_ai + body = JSON.parse(client.get(File.join(base, "capabilities"), headers: Spec::Authentication.headers).body) + body["providers"].as_a.first["text_model"].as_s.should eq MAP_AI_DEFAULT_OPENAI_MODEL + end + end + + describe "edit" do + it "accepts the request, records the context, and finishes the job with the ops" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + level = map_ai_level + requests = [] of JSON::Any + + HttpMocks.map_ai_vendor(MAP_AI_ANSWER, requests: requests) + HttpMocks.signage_ai_vendor + + context = map_ai_context + result = client.post( + File.join(base, "edit"), + headers: headers, + body: {zone_id: level.id, instruction: "Make meeting room 1.01 light green.", context: context}.to_json, + ) + + result.status_code.should eq 202 + accepted = JSON.parse(result.body) + accepted["state"].as_s.should eq "queued" + accepted["kind"].as_s.should eq "map_edit" + accepted["zone_id"].as_s.should eq level.id + accepted["provider"].as_s.should eq "OPENAI" + accepted["model"].as_s.should eq MAP_AI_DEFAULT_OPENAI_MODEL + accepted["instruction"].as_s.should eq "Make meeting room 1.01 light green." + accepted["ops"].as_a.should be_empty + + job_id = accepted["id"].as_s + stored = Model::SignageAIJob.find!(UUID.new(job_id)) + stored.kind.should eq Model::SignageAIJob::Kind::MapEdit + stored.candidates.should eq 1 + stored.provider_id.should eq provider.id + stored.user_id.should eq user.id + stored.request["context_bytes"].as_i.should eq context.to_json.bytesize + stored.request["context_sha256"].as_s.size.should eq 64 + stored.request["element_count"].as_i.should eq 7 + # the context body itself is not on the row + stored.request.to_json.should_not contain "table-01.004" + + final = await_signage_ai_job(base, job_id, headers) + final["state"].as_s.should eq "done" + final["summary"].as_s.should eq "Recoloured Meeting 1.01 light green." + final["schema_version"].as_i.should eq 1 + final["questions"].as_a.should be_empty + final["usage"]["total_tokens"].as_i.should eq 1280 + final["usage"]["model"].as_s.should eq MAP_AI_DEFAULT_OPENAI_MODEL + final["cost_units"].as_f.should eq 1280.0 + final["error_kind"]?.should be_nil + + ops = final["ops"].as_a + ops.size.should eq 1 + ops.first["id"].as_s.should eq "op-1" + ops.first["op"].as_s.should eq "set_attr" + ops.first["target"]["a"].as_s.should eq "e2" + ops.first["value"].as_s.should eq "#90ee90" + ops.first["server_issue"]?.should be_nil + + # what reached the vendor + requests.size.should eq 1 + sent = requests.first + sent["model"].as_s.should eq MAP_AI_DEFAULT_OPENAI_MODEL + sent["max_completion_tokens"].as_i.should eq MAP_AI_MAX_OUTPUT_TOKENS + sent["reasoning_effort"].as_s.should eq "low" + sent["response_format"]["type"].as_s.should eq "json_schema" + sent["response_format"]["json_schema"]["strict"].as_bool.should be_true + sent["response_format"]["json_schema"]["schema"]["properties"]["ops"]["items"]["properties"]["op"]["enum"].as_a.size.should eq Llm::Schema::OPS.size + messages = sent["messages"].as_a + messages.first["role"].as_s.should eq "system" + messages.first["content"].as_s.should contain "You edit SVG floor-plan maps" + user_parts = messages.last["content"].as_a + user_parts.first["type"].as_s.should eq "text" + user_parts.first["text"].as_s.should contain "e2 | area-1.01-status" + user_parts.last["text"].as_s.should end_with "INSTRUCTION\nMake meeting room 1.01 light green." + sent["temperature"]?.should be_nil + + Model::SignageAIJob.sum_candidates(user.id.as(String), 1.day.ago, Model::SignageAIJob::Kind::MapEdit).should eq 1 + end + + it "sends the markup snapshot as an image part and tags it for the sweep" do + _, storage, _ = setup_signage_ai + user, headers = Spec::Authentication.authentication + level = map_ai_level + requests = [] of JSON::Any + + snapshot = Model::Generator.upload(uploader: user, storage_id: storage.id) + snapshot.file_name = "snapshot.png" + snapshot.save! + + HttpMocks.map_ai_vendor(MAP_AI_ANSWER, requests: requests) + HttpMocks.signage_ai_storage + + result = client.post( + File.join(base, "edit"), + headers: headers, + body: {zone_id: level.id, instruction: "Recolour the room under pin 2.", context: map_ai_context, markup_upload_id: snapshot.id}.to_json, + ) + result.status_code.should eq 202 + job_id = JSON.parse(result.body)["id"].as_s + JSON.parse(result.body)["markup_upload_id"].as_s.should eq snapshot.id + + Model::Upload.find!(snapshot.id.as(String)).tags.should eq [ImageGen::Store::MARKUP_TAG] + Model::SignageAIJob.find!(UUID.new(job_id)).upload_ids.should eq [snapshot.id] + + final = await_signage_ai_job(base, job_id, headers) + final["state"].as_s.should eq "done" + + parts = requests.first["messages"].as_a.last["content"].as_a + image = parts.find! { |part| part["type"].as_s == "image_url" } + image["image_url"]["url"].as_s.should start_with "data:image/jpeg;base64," + image["image_url"]["detail"].as_s.should eq "high" + # subtrees before the image, annotations after it + parts.index!(image).should be > parts.index! { |part| part["type"].as_s == "text" && part["text"].as_s.starts_with?("SUBTREES") } + parts.index!(image).should be < parts.index! { |part| part["type"].as_s == "text" && part["text"].as_s.starts_with?("ANNOTATIONS") } + end + + it "refuses a snapshot the caller does not own" do + authority, storage, _ = setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + owner = Model::Generator.user(authority: authority).save! + snapshot = Model::Generator.upload(uploader: owner, storage_id: storage.id) + snapshot.file_name = "snapshot.png" + snapshot.save! + + result = client.post( + File.join(base, "edit"), + headers: headers, + body: {zone_id: level.id, instruction: "x", context: map_ai_context, markup_upload_id: snapshot.id}.to_json, + ) + result.status_code.should eq 403 + JSON.parse(result.body)["kind"].as_s.should eq "permission" + end + + it "accepts fenced JSON" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + HttpMocks.map_ai_vendor("```json\n#{MAP_AI_ANSWER}\n```") + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 202 + + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "done" + final["ops"].as_a.size.should eq 1 + end + + it "fails the job as invalid_response when the model answers in prose" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + HttpMocks.map_ai_vendor_garbage + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 202 + + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "failed" + final["error_kind"].as_s.should eq "invalid_response" + final["error_message"].as_s.should eq "the model did not answer with JSON" + # a failed call still spent tokens + final["cost_units"].as_f.should eq 1280.0 + end + + it "fails the job as invalid_response when the answer was cut off" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + HttpMocks.map_ai_vendor(%({"summary":"partial","ops":[{"op":"set_attr"), finish: "length") + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "failed" + final["error_kind"].as_s.should eq "invalid_response" + final["error_message"].as_s.should contain "cut off" + end + + it "keeps an op with an unknown anchor, flagged rather than dropped" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + answer = { + summary: "s", + ops: [ + {op: "set_attr", why: "w", target: {a: "e404", id: nil, h: nil}, name: "fill", value: "#fff"}, + {op: "set_attr", why: "w", target: {a: "e5", id: nil, h: nil}, name: "fill", value: "#fff"}, + ], + questions: [] of String, + }.to_json + HttpMocks.map_ai_vendor(answer) + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "done" + + ops = final["ops"].as_a + ops.size.should eq 2 + ops[0]["server_issue"].as_s.should eq "anchor e404 is not in the index" + ops[1]["server_issue"]?.should be_nil + end + + it "records a refusal as a moderation failure" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + HttpMocks.map_ai_vendor_refused + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "failed" + final["error_kind"].as_s.should eq "moderation" + final["error_message"].as_s.should eq "I can't help with that." + end + + it "records a vendor that never answers as a timeout" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + HttpMocks.map_ai_vendor_timeout + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "failed" + final["error_kind"].as_s.should eq "timeout" + end + + it "drops reasoning_effort when the model rejects it" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + requests = [] of JSON::Any + + HttpMocks.map_ai_vendor_no_reasoning(MAP_AI_ANSWER, requests) + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "done" + + requests.size.should eq 2 + requests[0]["reasoning_effort"].as_s.should eq "low" + requests[1]["reasoning_effort"]?.should be_nil + end + + it "answers 429 by the map quota, not the image quota" do + _, _, provider = setup_signage_ai(quotas: {"map_user_per_day" => JSON::Any.new(1_i64), "user_per_day" => JSON::Any.new(60_i64)}) + user, headers = Spec::Authentication.authentication + level = map_ai_level + + Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit).save! + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 429 + JSON.parse(result.body)["kind"].as_s.should eq "quota" + end + + it "does not count image jobs against the map quota" do + _, _, provider = setup_signage_ai(quotas: {"map_user_per_day" => JSON::Any.new(1_i64), "user_per_day" => JSON::Any.new(1_i64)}) + user, headers = Spec::Authentication.authentication + level = map_ai_level + Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 4).save! + + HttpMocks.map_ai_vendor(MAP_AI_ANSWER) + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 202 + await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + end + + it "refuses a write with a read-only scope" do + setup_signage_ai + level = map_ai_level + _, scoped_headers = Spec::Authentication.authentication(scope: [Model::UserJWT::Scope.new("map_ai", :read)]) + + result = client.post(File.join(base, "edit"), headers: scoped_headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 403 + end + + it "lets a concierge user with manage on the org zone edit" do + setup_signage_ai + _, headers = Spec::Authentication.authentication(sys_admin: false, support: false, groups: ["concierge"]) + level = map_ai_level + + HttpMocks.map_ai_vendor(MAP_AI_ANSWER) + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 202 + final = await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + final["state"].as_s.should eq "done" + end + + it "refuses a caller who cannot update the zone" do + setup_signage_ai + _, headers = Spec::Authentication.authentication(sys_admin: false, support: false) + level = map_ai_level + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 403 + Model::SignageAIJob.count.should eq 0 + end + + # Any signage group with Update may PATCH any zone today; the AI gate + # does not inherit that grant. + it "refuses the broad signage grant" do + authority, _, _ = setup_signage_ai + user, headers = Spec::Authentication.authentication(sys_admin: false, support: false) + signage_group(authority, user, Model::Permissions::Read | Model::Permissions::Update) + level = map_ai_level + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 403 + end + + it "hides a zone outside the caller's organisation" do + setup_signage_ai + _, headers = Spec::Authentication.authentication(sys_admin: false, support: false, groups: ["concierge"]) + + other_org = Model::Generator.zone + other_org.tags = Set{"org"} + other_org.save! + level = Model::Generator.zone + level.parent_id = other_org.id + level.save! + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context}.to_json) + result.status_code.should eq 404 + + missing = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: "zone-does-not-exist", instruction: "x", context: map_ai_context}.to_json) + missing.status_code.should eq 404 + end + + it "refuses a context over the cap, or of the wrong shape" do + setup_signage_ai + _, headers = Spec::Authentication.authentication + level = map_ai_level + + huge = map_ai_context + huge.as_h["subtrees"] = JSON::Any.new({"e2" => JSON::Any.new("x" * (MAP_AI_MAX_CONTEXT_BYTES + 1))}) + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: huge}.to_json) + result.status_code.should eq 422 + + wrong = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: {schema: "something/else", index: [] of String}}.to_json) + wrong.status_code.should eq 422 + + none = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x"}.to_json) + none.status_code.should eq 422 + + blank = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: " ", context: map_ai_context}.to_json) + blank.status_code.should eq 422 + + Model::SignageAIJob.count.should eq 0 + end + + it "replays an accepted submission rather than spending again" do + setup_signage_ai + user, headers = Spec::Authentication.authentication + level = map_ai_level + + HttpMocks.map_ai_vendor(MAP_AI_ANSWER) + + body = {zone_id: level.id, instruction: "x", context: map_ai_context, idempotency_key: "map-#{random_name}"}.to_json + first = client.post(File.join(base, "edit"), headers: headers, body: body) + first.status_code.should eq 202 + job_id = JSON.parse(first.body)["id"].as_s + + second = client.post(File.join(base, "edit"), headers: headers, body: body) + second.status_code.should eq 202 + JSON.parse(second.body)["id"].as_s.should eq job_id + + Model::SignageAIJob.where(user_id: user.id.as(String)).to_a.size.should eq 1 + await_signage_ai_job(base, job_id, headers) + end + + it "feeds the parent chain into the prompt" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + level = map_ai_level + requests = [] of JSON::Any + + parent = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit) + parent.state = Model::SignageAIJob::State::Done + parent.result = JSON.parse(%({"summary":"Recoloured it.","ops":[{"id":"op-1","op":"set_attr","target":{"a":"e2","id":null,"h":null},"name":"fill","value":"#90ee90","why":"w"}],"questions":[],"resolution":{"decision":"rejected","accepted_op_ids":[],"rejected_op_ids":["op-1"],"reasons":["too bright"]}})) + parent.save! + + HttpMocks.map_ai_vendor(MAP_AI_ANSWER, requests: requests) + + result = client.post( + File.join(base, "edit"), + headers: headers, + body: {zone_id: level.id, instruction: "Paler.", context: map_ai_context, parent_job_id: parent.id}.to_json, + ) + result.status_code.should eq 202 + JSON.parse(result.body)["parent_job_id"].as_s.should eq parent.id.to_s + await_signage_ai_job(base, JSON.parse(result.body)["id"].as_s, headers) + + text = requests.first["messages"].as_a.last["content"].as_a.map(&.["text"].as_s).join("\n") + text.should contain "Turn 1 instruction: recolour the boardroom light blue" + text.should contain "Turn 1 outcome: rejected; rejected: op-1; reasons: too bright" + text.index!("Turn 1 instruction").should be < text.index!("INSTRUCTION\nPaler.") + end + + it "refuses an image job or another domain's job as a parent" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + level = map_ai_level + image_job = Model::Generator.signage_ai_job(user: user, provider: provider).save! + + result = client.post(File.join(base, "edit"), headers: headers, body: {zone_id: level.id, instruction: "x", context: map_ai_context, parent_job_id: image_job.id}.to_json) + result.status_code.should eq 404 + end + end + + describe "jobs" do + it "lists only map edit jobs, by zone, and hides them from the signage list" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + + image_job = Model::Generator.signage_ai_job(user: user, provider: provider).save! + here = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit) + here.request = JSON.parse(%({"zone_id":"zone-here"})) + here.save! + elsewhere = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit) + elsewhere.request = JSON.parse(%({"zone_id":"zone-elsewhere"})) + elsewhere.save! + + result = client.get(File.join(base, "jobs"), headers: headers) + result.status_code.should eq 200 + ids = JSON.parse(result.body).as_a.map(&.["id"].as_s) + ids.should contain here.id.to_s + ids.should contain elsewhere.id.to_s + ids.should_not contain image_job.id.to_s + + by_zone = JSON.parse(client.get(File.join(base, "jobs") + "?zone_id=zone-here", headers: headers).body).as_a + by_zone.map(&.["id"].as_s).should eq [here.id.to_s] + by_zone.first["zone_id"].as_s.should eq "zone-here" + + signage = JSON.parse(client.get(File.join(SignageAI.base_route, "jobs"), headers: headers).body).as_a.map(&.["id"].as_s) + signage.should contain image_job.id.to_s + signage.should_not contain here.id.to_s + + signage_maps = JSON.parse(client.get(File.join(SignageAI.base_route, "jobs") + "?kind=map_edit", headers: headers).body).as_a.map(&.["id"].as_s) + signage_maps.should contain here.id.to_s + signage_maps.should_not contain image_job.id.to_s + + client.get(File.join(base, "jobs") + "?kind=nope", headers: headers).status_code.should eq 422 + end + + it "answers 404 for an image job on the map routes" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + image_job = Model::Generator.signage_ai_job(user: user, provider: provider).save! + + client.get(File.join(base, "jobs", image_job.id.to_s), headers: headers).status_code.should eq 404 + end + + it "long polls and cancels like signage" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + job = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit).save! + job_id = job.id.as(UUID) + + spawn do + sleep 700.milliseconds + Model::SignageAIJob.bump_version(job_id) + end + + started = Time.utc + result = client.get(File.join(base, "jobs", job_id.to_s) + "?wait=20&since=#{job.version}", headers: headers) + (Time.utc - started).should be < 10.seconds + result.status_code.should eq 200 + JSON.parse(result.body)["version"].as_i.should eq job.version + 1 + + cancelled = client.post(File.join(base, "jobs", job_id.to_s, "cancel"), headers: headers) + cancelled.status_code.should eq 200 + Model::SignageAIJob.find!(job_id).cancel_requested.should be_true + end + end + + describe "resolve" do + it "records what the person did with the answer" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + job = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit) + job.state = Model::SignageAIJob::State::Done + job.result = JSON.parse(%({"schema_version":1,"summary":"s","ops":[{"id":"op-1","op":"no_change","reason":"r","why":"w"}],"questions":[]})) + job.save! + + result = client.post( + File.join(base, "jobs", job.id.to_s, "resolve"), + headers: headers, + body: {decision: "partial", accepted_op_ids: ["op-1"], rejected_op_ids: ["op-2"], reasons: {"op-2" => "wrong room"}, saved_upload_id: "upload-123"}.to_json, + ) + result.status_code.should eq 200 + + body = JSON.parse(result.body) + body["version"].as_i.should eq job.version + 1 + resolution = body["resolution"] + resolution["decision"].as_s.should eq "partial" + resolution["accepted_op_ids"].as_a.map(&.as_s).should eq ["op-1"] + resolution["reasons"]["op-2"].as_s.should eq "wrong room" + resolution["saved_upload_id"].as_s.should eq "upload-123" + resolution["resolved_by"].as_s.should eq user.id + # the ops are untouched + body["ops"].as_a.size.should eq 1 + + stored = Model::SignageAIJob.find!(job.id.as(UUID)) + stored.result["resolution"]["decision"].as_s.should eq "partial" + + bad = client.post(File.join(base, "jobs", job.id.to_s, "resolve"), headers: headers, body: {decision: "meh"}.to_json) + bad.status_code.should eq 422 + end + + it "refuses somebody else's job, and a job still running" do + authority, _, provider = setup_signage_ai + _, headers = Spec::Authentication.authentication(sys_admin: false, support: false) + + someone_else = Model::Generator.user(authority: authority).save! + theirs = Model::Generator.signage_ai_job(user: someone_else, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit, state: Model::SignageAIJob::State::Done).save! + client.post(File.join(base, "jobs", theirs.id.to_s, "resolve"), headers: headers, body: {decision: "applied"}.to_json).status_code.should eq 403 + + user, own_headers = Spec::Authentication.authentication + running = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit, state: Model::SignageAIJob::State::Running).save! + client.post(File.join(base, "jobs", running.id.to_s, "resolve"), headers: own_headers, body: {decision: "applied"}.to_json).status_code.should eq 422 + end + end + + describe "usage" do + it "sums map edit spend only, for support" do + _, _, provider = setup_signage_ai + user, headers = Spec::Authentication.authentication + + image_job = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 2) + image_job.cost_units = 10.0 + image_job.save! + map_job = Model::Generator.signage_ai_job(user: user, provider: provider, candidates: 1, kind: Model::SignageAIJob::Kind::MapEdit) + map_job.cost_units = 1280.0 + map_job.save! + + result = client.get(File.join(base, "usage"), headers: headers) + result.status_code.should eq 200 + rows = Array(Model::SignageAIJob::UsageRow).from_json(result.body) + rows.size.should eq 1 + rows.first.kind.should eq "map_edit" + rows.first.cost_units.should eq 1280.0 + rows.first.candidates.should eq 1 + + # the signage usage report now carries the kind as well + signage_rows = Array(Model::SignageAIJob::UsageRow).from_json(client.get(File.join(SignageAI.base_route, "usage"), headers: headers).body) + signage_rows.map(&.kind).sort!.should eq ["generate", "map_edit"] + + _, regular = Spec::Authentication.authentication(sys_admin: false, support: false) + client.get(File.join(base, "usage"), headers: regular).status_code.should eq 403 + end + end + end +end diff --git a/spec/controllers/signage/signage_ai_providers_spec.cr b/spec/controllers/signage/signage_ai_providers_spec.cr index 9dfbc2eb..0af6247c 100644 --- a/spec/controllers/signage/signage_ai_providers_spec.cr +++ b/spec/controllers/signage/signage_ai_providers_spec.cr @@ -27,6 +27,7 @@ module PlaceOS::Api authority_id: authority.id, credentials: {api_key: "sk-spec-secret"}, default_model: "gpt-image-2", + text_model: "gpt-5.4-mini", is_default: true, }.to_json @@ -39,6 +40,7 @@ module PlaceOS::Api rendered = JSON.parse(result.body) rendered["provider"].as_s.should eq "OPENAI" rendered["is_default"].as_bool.should be_true + rendered["text_model"].as_s.should eq "gpt-5.4-mini" row = Model::SignageAIProvider.find!(UUID.new(rendered["id"].as_s)) row.credentials_encrypted?.should be_true @@ -51,6 +53,7 @@ module PlaceOS::Api authority: authority, name: "openai-#{random_name}", credentials: %({"api_key":"keep-me"}), + text_model: "gpt-5.4-mini", ).save! result = client.patch( @@ -66,6 +69,12 @@ module PlaceOS::Api row.name.should eq "renamed" row.enabled.should be_false row.credentials_json["api_key"].as_s.should eq "keep-me" + # left out of the body, left alone + row.text_model.should eq "gpt-5.4-mini" + + # sent empty, cleared + client.patch(File.join(base, provider.id.to_s), headers: Spec::Authentication.headers, body: {text_model: ""}.to_json).status_code.should eq 200 + Model::SignageAIProvider.find!(provider.id.as(UUID)).text_model.should be_nil end it "lets support read, and nothing more" do @@ -211,6 +220,46 @@ module PlaceOS::Api JSON.parse(result.body)["model"].as_s.should eq "gpt-image-2.5-sunburst" end + it "probes the text model with kind=text" do + authority = Model::Authority.find_by_domain("localhost").not_nil! + provider = Model::Generator.signage_ai_provider( + authority: authority, + name: "openai-#{random_name}", + text_model: "gpt-5.4-mini-custom", + ).save! + requests = [] of JSON::Any + + HttpMocks.map_ai_vendor("OK", requests: requests) + HttpMocks.signage_ai_vendor_refused + + result = client.post(File.join(base, provider.id.to_s, "test?kind=text"), headers: Spec::Authentication.headers) + result.status_code.should eq 200 + + body = JSON.parse(result.body) + body["ok"].as_bool.should be_true + body["model"].as_s.should eq "gpt-5.4-mini-custom" + + requests.size.should eq 1 + requests.first["model"].as_s.should eq "gpt-5.4-mini-custom" + requests.first["max_completion_tokens"].as_i.should eq 5 + requests.first["response_format"]?.should be_nil + requests.first["reasoning_effort"]?.should be_nil + + client.post(File.join(base, provider.id.to_s, "test?kind=sound"), headers: Spec::Authentication.headers).status_code.should eq 422 + end + + it "reports a text probe that times out" do + authority = Model::Authority.find_by_domain("localhost").not_nil! + provider = Model::Generator.signage_ai_provider(authority: authority, name: "openai-#{random_name}").save! + + HttpMocks.map_ai_vendor_timeout + + body = JSON.parse(client.post(File.join(base, provider.id.to_s, "test?kind=text"), headers: Spec::Authentication.headers).body) + body["ok"].as_bool.should be_false + body["kind"].as_s.should eq "timeout" + body["model"].as_s.should eq MAP_AI_DEFAULT_OPENAI_MODEL + end + it "reports a refusal as a moderation failure rather than an error" do authority = Model::Authority.find_by_domain("localhost").not_nil! provider = Model::Generator.signage_ai_provider( diff --git a/spec/fixtures/map_ai/context.json b/spec/fixtures/map_ai/context.json new file mode 100644 index 00000000..abd81e46 --- /dev/null +++ b/spec/fixtures/map_ai/context.json @@ -0,0 +1,33 @@ +{ + "schema": "placeos.map-ai-context/1", + "viewBox": [0, 0, 1725.26, 619], + "image_px": [2048, 735], + "units_per_px": 0.8424, + "protected_ids": ["table-01.004", "area-1.02-status"], + "duplicate_ids": ["table-01.015"], + "index": [ + {"a": "e1", "id": null, "tag": "g", "cls": null, "label": "walls", "bbox": [0, 0, 1725.26, 619], "p": null, "z": 1, "kind": "layer-group", "prot": false, "h": "a1a1a1a1"}, + {"a": "e2", "id": "area-1.01-status", "tag": "path", "cls": "cls-3", "label": "Meeting 1.01", "bbox": [100, 80, 120, 90], "p": "e1", "z": 2, "kind": "zone", "prot": false, "h": "b2b2b2b2"}, + {"a": "e3", "id": "area-1.02-status", "tag": "path", "cls": "cls-3", "label": "Meeting 1.02", "bbox": [240, 80, 120, 90], "p": "e1", "z": 3, "kind": "zone", "prot": true, "h": "c3c3c3c3"}, + {"a": "e4", "id": "table-01.004", "tag": "rect", "cls": "cls-12", "label": "Desk 01.004", "bbox": [640, 420, 30, 20], "p": "e1", "z": 4, "kind": "bookable", "prot": true, "h": "d4d4d4d4"}, + {"a": "e5", "id": "table-01.005", "tag": "rect", "cls": "cls-12", "label": "Desk 01.005", "bbox": [680, 420, 30, 20], "p": "e1", "z": 5, "kind": "bookable", "prot": false, "h": "e5e5e5e5"}, + {"a": "e6", "id": "label-1.01", "tag": "text", "cls": null, "label": "Meeting 1.01", "bbox": [110, 60, 60, 12], "p": "e1", "z": 6, "kind": "art", "prot": false, "h": "f6f6f6f6"}, + {"a": "e7", "id": null, "tag": "path", "cls": "cls-1", "label": null, "bbox": [0, 0, 1725.26, 4], "p": "e1", "z": 7, "kind": "art", "prot": false, "h": "07070707"} + ], + "shortlist": ["e2", "e4"], + "subtrees": { + "e2": "", + "e4": "" + }, + "style_rules": [".cls-1{fill:none;stroke:#231f20}", ".cls-3{fill:#fff}", ".cls-12{fill:#d1d3d4}"], + "markup": { + "schema": "placeos.map-markup/1", + "viewBox": [0, 0, 1725.26, 619], + "annotations": [ + {"n": 1, "kind": "box", "x": 630, "y": 410, "width": 90, "height": 40, "hits": ["e4", "e5"], "near": ["e7"]}, + {"n": 2, "kind": "pin", "x": 160, "y": 120, "text": "this room", "hits": ["e2"]} + ], + "selected": ["area-1.01-status"] + }, + "stats": {"elements": 748, "ids": 84, "bytes": 134861} +} diff --git a/spec/llm_prompt_spec.cr b/spec/llm_prompt_spec.cr new file mode 100644 index 00000000..ae650130 --- /dev/null +++ b/spec/llm_prompt_spec.cr @@ -0,0 +1,309 @@ +require "./helper" + +module PlaceOS::Api + def self.assert_strict_objects(node : JSON::Any) : Nil + if (hash = node.as_h?) + if hash["properties"]? + hash["additionalProperties"].as_bool.should be_false + hash["required"].as_a.map(&.as_s).sort!.should eq hash["properties"].as_h.keys.sort! + end + hash.each_value { |value| assert_strict_objects(value) } + elsif (list = node.as_a?) + list.each { |value| assert_strict_objects(value) } + end + end + + def self.op(json : String) : Hash(String, JSON::Any) + JSON.parse(json).as_h + end + + describe Llm::Prompt do + context = JSON.parse(File.read(File.join(__DIR__, "fixtures/map_ai/context.json"))) + image = Llm::ImagePart.new(bytes: Bytes[0x89, 0x50, 0x4E, 0x47], mime: "image/png") + + it "sends the rules, the op reference and the schema first, the instruction last" do + request = Llm::Prompt.build(Llm::Prompt::Options.new( + context: context, + instruction: "Make meeting room 1.01 light green.", + model: "gpt-5.4-mini", + )) + + request.system.should contain "You edit SVG floor-plan maps" + request.system.should contain "set_attr: target, name, value" + request.schema.should_not be_nil + request.model.should eq "gpt-5.4-mini" + request.max_output_tokens.should eq MAP_AI_MAX_OUTPUT_TOKENS + + text = request.text + text.index!("MAP\n").should be < text.index!("INDEX\n") + text.index!("INDEX\n").should be < text.index!("SUBTREES") + text.index!("SUBTREES").should be < text.index!("ANNOTATIONS") + text.index!("ANNOTATIONS").should be < text.index!("INSTRUCTION\n") + text.should end_with "INSTRUCTION\nMake meeting room 1.01 light green." + + # the index carries what the model may ground on, nothing else + text.should contain "viewBox: 0 0 1725.26 619" + text.should contain "protected ids (bound to bookings): table-01.004, area-1.02-status" + text.should contain "e4 | table-01.004 | rect | cls-12 | \"Desk 01.004\" | 640 420 30 20 | e1 | bookable | protected | d4d4d4d4" + text.should contain "e7 | - | path | cls-1 | - | 0 0 1725.26 4 | e1 | art | - | 07070707" + text.should contain ".cls-12{fill:#d1d3d4}" + text.should contain "[e4]\n kinds.index!(&.starts_with?("SUBTREES")) + kinds.index!("image").should be < kinds.index!(&.starts_with?("ANNOTATIONS")) + request.images.size.should eq 1 + request.text.should contain "the numbers match the badges on the snapshot" + end + + it "includes the parent chain, oldest first, with what the person did" do + first = Llm::Prompt::Turn.new( + instruction: "Make meeting room 1.01 light green.", + ops: [JSON.parse(%({"id":"op-1","op":"set_attr","target":{"a":"e2","id":null,"h":null},"name":"fill","value":"#90ee90","why":"recolours Meeting 1.01","selector":null}))], + summary: "Recoloured Meeting 1.01.", + resolution: JSON.parse(%({"decision":"rejected","accepted_op_ids":[],"rejected_op_ids":["op-1"],"reasons":{"op-1":"wrong green"}})), + ) + second = Llm::Prompt::Turn.new(instruction: "Use a paler green.", ops: [] of JSON::Any) + + request = Llm::Prompt.build(Llm::Prompt::Options.new( + context: context, + instruction: "Paler still.", + model: "gpt-5.4-mini", + history: [first, second], + )) + + text = request.text + text.index!("PREVIOUS TURNS").should be > text.index!("INDEX\n") + text.index!("PREVIOUS TURNS").should be < text.index!("INSTRUCTION\n") + text.index!("Turn 1 instruction").should be < text.index!("Turn 2 instruction") + text.should contain "Turn 1 outcome: rejected; rejected: op-1; reasons: op-1: wrong green" + text.should contain "Turn 2 outcome: not reviewed" + # the null fields and the why are not worth the tokens + text.should contain %("op":"set_attr") + text.should_not contain %("selector":null) + text.should_not contain "recolours Meeting 1.01" + end + + it "caps the index, the subtrees and the history" do + big = JSON.parse(context.to_json) + index = big["index"].as_a + 3000.times do |n| + index << JSON.parse(%({"a":"x#{n}","id":null,"tag":"path","cls":"cls-1","label":null,"bbox":[#{n},#{n},10,10],"p":"e1","z":#{n},"kind":"art","prot":false,"h":"#{n.to_s.rjust(8, '0')}"})) + end + subtrees = big["subtrees"].as_h + 12.times { |n| subtrees["s#{n}"] = JSON::Any.new("") } + + history = Array.new(10) do |n| + Llm::Prompt::Turn.new(instruction: "turn #{n}", ops: [] of JSON::Any) + end + + request = Llm::Prompt.build(Llm::Prompt::Options.new( + context: big, + instruction: "x", + model: "gpt-5.4-mini", + history: history, + )) + + index_text = Llm::Prompt.index_text(big) + index_text.bytesize.should be < Llm::Prompt::MAX_INDEX_BYTES + 2_000 + index_text.should contain "more elements omitted" + # the front of the index survives, the tail goes + index_text.should contain "e4 | table-01.004" + + subtree_text = Llm::Prompt.subtrees_text(big) + subtree_text.bytesize.should be < Llm::Prompt::MAX_SUBTREE_BYTES + 2_000 + subtree_text.should contain "further subtrees omitted" + + text = request.text + text.should_not contain "Turn 7 instruction" + text.should contain "turn 9" + text.should_not contain "turn 0" + end + + it "leaves out sections the context does not carry" do + bare = JSON.parse(%({"schema":"placeos.map-ai-context/1","index":[]})) + request = Llm::Prompt.build(Llm::Prompt::Options.new(context: bare, instruction: "Nothing.", model: "m")) + request.parts.size.should eq 2 + request.text.should_not contain "SUBTREES" + request.text.should_not contain "ANNOTATIONS" + end + end + + describe Llm::Schema do + it "renders the vocabulary in the strict dialect" do + schema = Llm::Schema.openai + + schema["type"].as_s.should eq "object" + schema["additionalProperties"].as_bool.should be_false + schema["required"].as_a.map(&.as_s).should eq ["summary", "ops", "questions"] + + op = schema["properties"]["ops"]["items"] + op["additionalProperties"].as_bool.should be_false + op["required"].as_a.map(&.as_s).should eq op["properties"].as_h.keys + op["properties"]["op"]["enum"].as_a.map(&.as_s).should eq Llm::Schema::OPS + op["properties"]["why"]["type"].as_s.should eq "string" + + # optional fields are nullable, enums included + op["properties"]["name"]["type"].as_a.map(&.as_s).should eq ["string", "null"] + op["properties"]["kind"]["type"].as_a.map(&.as_s).should eq ["string", "null"] + op["properties"]["kind"]["enum"].as_a.last.raw.should be_nil + op["properties"]["kind"]["enum"].as_a.size.should eq Llm::Schema::SHAPES.size + 1 + + target = op["properties"]["target"] + target["type"].as_a.map(&.as_s).should eq ["object", "null"] + target["required"].as_a.map(&.as_s).should eq ["a", "id", "h"] + target["additionalProperties"].as_bool.should be_false + + op["properties"]["translate"]["minItems"].as_i.should eq 2 + op["properties"]["translate"]["items"]["type"].as_s.should eq "number" + op["properties"]["attrs"]["items"]["required"].as_a.map(&.as_s).should eq ["name", "value"] + + # every object anywhere in the tree is closed and fully required + PlaceOS::Api.assert_strict_objects(schema) + end + + it "lists the fields each op reads" do + Llm::Schema::FIELDS_BY_OP.keys.should eq Llm::Schema::OPS + Llm::Schema.reference.should contain "duplicate: target, new_id, translate, count, id_pattern" + Llm::Schema.valid_id?("table-01.018").should be_true + Llm::Schema.valid_id?("1abc").should be_false + Llm::Schema.valid_id?("ai desk").should be_false + Llm::Schema.allowed_attribute?("data-name").should be_true + Llm::Schema.allowed_attribute?("onclick").should be_false + Llm::Schema.allowed_attribute?("href").should be_false + end + end + + describe Llm::Ops do + context = JSON.parse(File.read(File.join(__DIR__, "fixtures/map_ai/context.json"))) + + it "reads fenced JSON and refuses prose" do + envelope = Llm::Ops.parse("```json\n{\"summary\":\"s\",\"ops\":[],\"questions\":[\"which?\"]}\n```") + envelope.summary.should eq "s" + envelope.ops.should be_empty + envelope.questions.should eq ["which?"] + + expect_raises(Error::ImageGen::InvalidResponse, "did not answer with JSON") do + Llm::Ops.parse("I would change the fill of the boardroom.") + end + expect_raises(Error::ImageGen::InvalidResponse, "no ops list") do + Llm::Ops.parse(%({"summary":"s"})) + end + expect_raises(Error::ImageGen::InvalidResponse, "empty") { Llm::Ops.parse("") } + end + + it "numbers the ops and flags the ones that fail a check, keeping them" do + ops = [ + op(%({"op":"set_attr","target":{"a":"e2","id":"area-1.01-status","h":"b2b2b2b2"},"name":"fill","value":"#90ee90","why":"w"})), + op(%({"op":"set_attr","target":{"a":"e99","id":null,"h":null},"name":"fill","value":"#90ee90","why":"w"})), + op(%({"op":"set_attr","target":{"a":"e2","id":"table-01.004","h":null},"name":"fill","value":"#000","why":"w"})), + op(%({"op":"set_attr","target":{"a":"e2","id":null,"h":"stale"},"name":"fill","value":"#000","why":"w"})), + op(%({"op":"set_attr","target":{"a":"e2","id":null,"h":null},"name":"onclick","value":"x","why":"w"})), + op(%({"op":"set_attr","target":{"a":"e2","id":null,"h":null},"name":"data-name","value":"Boardroom","why":"w"})), + op(%({"op":"set_attr","target":{"a":"e2","id":null,"h":null},"name":"visibility","value":"display:none","why":"w"})), + op(%({"op":"set_text","target":{"a":"e2","id":null,"h":null},"text":"x","why":"w"})), + op(%({"op":"set_text","target":{"a":"e6","id":null,"h":null},"text":"Boardroom","why":"w"})), + op(%({"op":"transform","target":{"a":"e5","id":null,"h":null},"translate":[20,0],"why":"w"})), + op(%({"op":"transform","targets":[{"a":"e4","id":null,"h":null},{"a":"e5","id":null,"h":null}],"translate":[20],"why":"w"})), + op(%({"op":"duplicate","target":{"a":"e5","id":null,"h":null},"new_id":"table-01.006","translate":[40,0],"count":2,"why":"w"})), + op(%({"op":"duplicate","target":{"a":"e5","id":null,"h":null},"new_id":"table-01.004","translate":[40,0],"why":"w"})), + op(%({"op":"duplicate","target":{"a":"e5","id":null,"h":null},"new_id":"1bad","translate":[40,0],"why":"w"})), + op(%({"op":"insert_shape","parent":{"a":"e1","id":null,"h":null},"kind":"rect","attrs":[{"name":"x","value":"1"},{"name":"fill","value":"#fff"}],"new_id":"ai-desk-1","why":"w"})), + op(%({"op":"insert_shape","parent":{"a":"e1","id":null,"h":null},"kind":"script","attrs":[],"why":"w"})), + op(%({"op":"insert_shape","parent":{"a":"e1","id":null,"h":null},"kind":"rect","attrs":[{"name":"onload","value":"x"}],"why":"w"})), + op(%({"op":"set_style_rule","selector":".cls-12","declarations":"fill: #eee; stroke: #333","why":"w"})), + op(%({"op":"set_style_rule","selector":"*","declarations":"fill: #eee","why":"w"})), + op(%({"op":"set_style_rule","selector":".cls-12","declarations":"background: url(http://x)","why":"w"})), + op(%({"op":"set_id","target":{"a":"e5","id":null,"h":null},"new_id":"table-01.005-renamed","why":"w"})), + op(%({"op":"reorder","target":{"a":"e5","id":null,"h":null},"position":"last","why":"w"})), + op(%({"op":"reorder","target":{"a":"e5","id":null,"h":null},"why":"w"})), + op(%({"op":"group","targets":[{"a":"e4","id":null,"h":null},{"a":"e5","id":null,"h":null}],"new_id":"ai-desks","why":"w"})), + op(%({"op":"ungroup","target":{"a":"e5","id":null,"h":null},"why":"w"})), + op(%({"op":"replace_element","target":{"a":"e2","id":null,"h":null},"markup":"","why":"w"})), + op(%({"op":"replace_element","target":{"a":"e2","id":null,"h":null},"markup":"","why":"w"})), + op(%({"op":"replace_element","target":{"a":"e2","id":null,"h":null},"markup":"","why":"w"})), + op(%({"op":"replace_element","target":{"a":"e2","id":null,"h":null},"markup":"","why":"w"})), + op(%({"op":"replace_path","target":{"a":"e2","id":null,"h":null},"d":"M0 0 L10 10 Z","why":"w"})), + op(%({"op":"replace_path","target":{"a":"e5","id":null,"h":null},"d":"M0 0","why":"w"})), + op(%({"op":"no_change","reason":"nothing to do","why":"w"})), + op(%({"op":"ask","question":"which room?","why":"w"})), + op(%({"op":"ask","why":"w"})), + op(%({"op":"explode","why":"w"})), + ] + + checked = Llm::Ops.check(ops, context) + checked.size.should eq ops.size + checked.map(&.["id"].as_s).first(3).should eq ["op-1", "op-2", "op-3"] + + issues = checked.map { |entry| entry["server_issue"]?.try(&.as_s?) } + issues[0].should be_nil + issues[1].should eq "anchor e99 is not in the index" + issues[2].should eq "anchor e2 and id table-01.004 are different elements" + issues[3].to_s.should contain "hash mismatch" + issues[4].should eq "attribute onclick may not be set" + issues[5].should be_nil + issues[6].to_s.should contain "visibility must be one of" + issues[7].should eq "set_text only applies to text and tspan elements" + issues[8].should be_nil + issues[9].should be_nil + issues[10].should eq "translate must be [dx, dy]" + issues[11].should be_nil + issues[12].should eq "table-01.004 is already an id in the map" + issues[13].to_s.should contain "is not a valid id" + issues[14].should be_nil + issues[15].should eq "script is not a shape that can be inserted" + issues[16].should eq "attribute onload may not be set on a new element" + issues[17].should be_nil + issues[18].should eq "the selector * is not allowed" + issues[19].should eq "declarations may not import or reference URLs" + issues[20].should be_nil + issues[21].should be_nil + issues[22].should eq "reorder needs exactly one of before, after or position" + issues[23].should be_nil + issues[24].should eq "ungroup only applies to a group" + issues[25].should be_nil + issues[26].should eq "event handler attributes are not allowed" + issues[27].should eq "the markup must be exactly one element" + issues[28].should eq " is not allowed" + issues[29].should be_nil + issues[30].should eq "replace_path only applies to a path" + issues[31].should be_nil + issues[32].should be_nil + issues[33].should eq "ask needs a question" + issues[34].should eq %(unknown operation "explode") + end + + it "flags everything past the op limit" do + ops = Array.new(MAP_AI_MAX_OPS + 2) do + op(%({"op":"no_change","reason":"r","why":"w"})) + end + checked = Llm::Ops.check(ops, context) + checked[MAP_AI_MAX_OPS - 1]["server_issue"]?.should be_nil + checked[MAP_AI_MAX_OPS]["server_issue"].as_s.should contain "over the limit" + checked.last["server_issue"].as_s.should contain "over the limit" + end + + it "accepts a protected target but never a protected new id" do + ops = [ + op(%({"op":"delete","target":{"a":"e4","id":null,"h":null},"why":"w"})), + op(%({"op":"set_id","target":{"a":"e5","id":null,"h":null},"new_id":"area-1.02-status","why":"w"})), + ] + checked = Llm::Ops.check(ops, context) + checked[0]["server_issue"]?.should be_nil + checked[1]["server_issue"].as_s.should eq "area-1.02-status is already an id in the map" + end + end +end diff --git a/spec/spec_helpers/http_mocks.cr b/spec/spec_helpers/http_mocks.cr index b11bf068..7f1820dc 100644 --- a/spec/spec_helpers/http_mocks.cr +++ b/spec/spec_helpers/http_mocks.cr @@ -81,8 +81,85 @@ module PlaceOS::Api::HttpMocks .to_return(body: body, status: 400, headers: HTTP::Headers{"Content-Type" => "application/json"}) end + # The chat completions path alone, so the image stub above (which matches + # every api.openai.com POST) cannot swallow a map AI call. Only the first + # matching stub is used, so register this before `signage_ai_vendor`. + CHAT_COMPLETIONS = /api\.openai\.com\/v1\/chat\/completions/ + + # Chat completions answering `content` verbatim. `requests` collects every + # body the stub received, for assertions on what was sent. + def self.map_ai_vendor(content : String, finish : String = "stop", requests : Array(JSON::Any)? = nil) + WebMock + .stub(:post, CHAT_COMPLETIONS) + .to_return do |request| + if requests + requests << JSON.parse(WebMock.body(request).to_s) + end + + body = { + id: "chatcmpl-spec", + choices: [{message: {role: "assistant", content: content, refusal: nil}, finish_reason: finish}], + usage: {prompt_tokens: 1200, completion_tokens: 80, total_tokens: 1280}, + }.to_json + + HTTP::Client::Response.new(200, body, HTTP::Headers{"Content-Type" => "application/json"}) + end + end + + # The model declining to answer, which the adapter maps to `Moderated`. + def self.map_ai_vendor_refused + body = { + id: "chatcmpl-spec", + choices: [{message: {role: "assistant", content: nil, refusal: "I can't help with that."}, finish_reason: "stop"}], + usage: {prompt_tokens: 1200, completion_tokens: 10, total_tokens: 1210}, + }.to_json + + WebMock + .stub(:post, CHAT_COMPLETIONS) + .to_return(body: body, status: 200, headers: HTTP::Headers{"Content-Type" => "application/json"}) + end + + # The vendor never answering: the read timeout fires inside the client. + def self.map_ai_vendor_timeout + WebMock + .stub(:post, CHAT_COMPLETIONS) + .to_return do |_request| + raise IO::TimeoutError.new("Read timed out") + end + end + + # Prose instead of the JSON asked for. + def self.map_ai_vendor_garbage + map_ai_vendor("I think the boardroom should be light blue, so I would change its fill.") + end + + # A 400 naming reasoning_effort on the first call, the answer on the second. + def self.map_ai_vendor_no_reasoning(content : String, requests : Array(JSON::Any)) + WebMock + .stub(:post, CHAT_COMPLETIONS) + .to_return do |request| + parsed = JSON.parse(WebMock.body(request).to_s) + requests << parsed + if parsed["reasoning_effort"]? + error = {error: {message: "Unsupported parameter: 'reasoning_effort' is not supported with this model.", type: "invalid_request_error", param: "reasoning_effort", code: "unsupported_parameter"}}.to_json + HTTP::Client::Response.new(400, error, HTTP::Headers{"Content-Type" => "application/json"}) + else + body = { + id: "chatcmpl-spec", + choices: [{message: {role: "assistant", content: content, refusal: nil}, finish_reason: "stop"}], + usage: {prompt_tokens: 1200, completion_tokens: 80, total_tokens: 1280}, + }.to_json + HTTP::Client::Response.new(200, body, HTTP::Headers{"Content-Type" => "application/json"}) + end + end + end + # The object store a generated candidate is written to and read back from. # Both verbs are signed URLs against the storage provider's host. + # + # The read is streamed (`Http.get_bytes` takes the response in a block), and + # WebMock hands a stubbed response to that block as-is, so it carries a + # `body_io` rather than a `body` string. def self.signage_ai_storage object_store = /amazonaws\.com|blob\.core\.windows\.net/ @@ -93,8 +170,8 @@ module PlaceOS::Api::HttpMocks .to_return do |_request| HTTP::Client::Response.new( 200, - String.new(::Base64.decode(TINY_JPEG)), - HTTP::Headers{"Content-Type" => "image/jpeg"}, + headers: HTTP::Headers{"Content-Type" => "image/jpeg"}, + body_io: IO::Memory.new(::Base64.decode(TINY_JPEG)), ) end end From 50042847f759981d72ef5be27d87c8d7350b0eb6 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Mon, 12 Oct 2026 19:34:17 +1100 Subject: [PATCH 06/38] chore(docs): regenerate OPENAPI_DOC.yml for the maps/ai routes PPT-2833. `crystal build ./src/app.cr && ./app -d > OPENAPI_DOC.yml`, run locally against the models branch. Co-Authored-By: Claude Fable 5.1 --- OPENAPI_DOC.yml | 1167 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 1125 insertions(+), 42 deletions(-) diff --git a/OPENAPI_DOC.yml b/OPENAPI_DOC.yml index 42f716cd..54b0f415 100644 --- a/OPENAPI_DOC.yml +++ b/OPENAPI_DOC.yml @@ -20686,6 +20686,838 @@ paths: application/json: schema: $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + /api/engine/v2/maps/ai/capabilities: + summary: AI editing of a level's SVG map, with the signage AI provider rows. + description: 'AI editing of a level''s SVG map, with the signage AI provider rows. + + + The editor posts the instruction and the context it built from the map (the + + element index, the shortlisted subtrees, the annotations and a marked up + + snapshot by upload id). The answer is a list of operations the editor + + applies, reviews and saves itself; nothing here reads or writes the map. + + One job is one vendor call, long polled through `jobs/:id` as signage is.' + get: + summary: what this domain can do, and what the caller has left of their quota + tags: + - MapAI + operationId: PlaceOS::Api::MapAI_capabilities + parameters: [] + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.Capabilities' + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "503": + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "403": + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "429": + description: Too Many Requests + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "422": + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "502": + description: Bad Gateway + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "504": + description: Gateway Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "406": + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + "408": + description: Request Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "415": + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + /api/engine/v2/maps/ai/edit: + summary: AI editing of a level's SVG map, with the signage AI provider rows. + description: 'AI editing of a level''s SVG map, with the signage AI provider rows. + + + The editor posts the instruction and the context it built from the map (the + + element index, the shortlisted subtrees, the annotations and a marked up + + snapshot by upload id). The answer is a list of operations the editor + + applies, reviews and saves itself; nothing here reads or writes the map. + + One job is one vendor call, long polled through `jobs/:id` as signage is.' + post: + summary: ask for operations that carry out an instruction on a level's map + tags: + - MapAI + operationId: PlaceOS::Api::MapAI_edit + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.EditParams' + required: true + parameters: [] + responses: + "202": + description: Accepted + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.JobResponse' + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "503": + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "403": + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "429": + description: Too Many Requests + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "422": + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "502": + description: Bad Gateway + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "504": + description: Gateway Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "406": + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + "408": + description: Request Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "415": + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + /api/engine/v2/maps/ai/jobs/{id}: + summary: AI editing of a level's SVG map, with the signage AI provider rows. + description: 'AI editing of a level''s SVG map, with the signage AI provider rows. + + + The editor posts the instruction and the context it built from the map (the + + element index, the shortlisted subtrees, the annotations and a marked up + + snapshot by upload id). The answer is a list of operations the editor + + applies, reviews and saves itself; nothing here reads or writes the map. + + One job is one vendor call, long polled through `jobs/:id` as signage is.' + get: + summary: 'long poll: holds until the job changes past `since`, or `wait` runs + out' + tags: + - MapAI + operationId: PlaceOS::Api::MapAI_show_job + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + - name: wait + in: query + description: seconds to hold the request open, 0 to answer immediately + example: 25 + schema: + type: integer + format: Int32 + - name: since + in: query + description: the version the caller already has + example: 3 + schema: + anyOf: + - type: integer + format: Int32 + - type: "null" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.JobResponse' + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "503": + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "403": + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "429": + description: Too Many Requests + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "422": + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "502": + description: Bad Gateway + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "504": + description: Gateway Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "406": + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + "408": + description: Request Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "415": + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + /api/engine/v2/maps/ai/jobs: + summary: AI editing of a level's SVG map, with the signage AI provider rows. + description: 'AI editing of a level''s SVG map, with the signage AI provider rows. + + + The editor posts the instruction and the context it built from the map (the + + element index, the shortlisted subtrees, the annotations and a marked up + + snapshot by upload id). The answer is a list of operations the editor + + applies, reviews and saves itself; nothing here reads or writes the map. + + One job is one vendor call, long polled through `jobs/:id` as signage is.' + get: + summary: recent map edit jobs + tags: + - MapAI + operationId: PlaceOS::Api::MapAI_index_jobs + parameters: + - name: mine + in: query + description: only the caller's own jobs + example: true + schema: + type: boolean + - name: zone_id + in: query + description: only jobs against this zone + example: zone-1234 + schema: + anyOf: + - type: "null" + - type: string + - name: kind + in: query + description: map_edit; another kind lists that kind instead + example: map_edit + schema: + type: string + - name: limit + in: query + schema: + type: integer + format: Int32 + responses: + "200": + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.JobResponse' + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "503": + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "403": + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "429": + description: Too Many Requests + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "422": + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "502": + description: Bad Gateway + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "504": + description: Gateway Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "406": + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + "408": + description: Request Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "415": + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + /api/engine/v2/maps/ai/jobs/{id}/cancel: + summary: AI editing of a level's SVG map, with the signage AI provider rows. + description: 'AI editing of a level''s SVG map, with the signage AI provider rows. + + + The editor posts the instruction and the context it built from the map (the + + element index, the shortlisted subtrees, the annotations and a marked up + + snapshot by upload id). The answer is a list of operations the editor + + applies, reviews and saves itself; nothing here reads or writes the map. + + One job is one vendor call, long polled through `jobs/:id` as signage is.' + post: + summary: ask a running job to stop. A call already with the vendor runs to completion. + tags: + - MapAI + operationId: PlaceOS::Api::MapAI_cancel + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.JobResponse' + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "503": + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "403": + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "429": + description: Too Many Requests + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "422": + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "502": + description: Bad Gateway + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "504": + description: Gateway Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "406": + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + "408": + description: Request Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "415": + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + /api/engine/v2/maps/ai/jobs/{id}/resolve: + summary: AI editing of a level's SVG map, with the signage AI provider rows. + description: 'AI editing of a level''s SVG map, with the signage AI provider rows. + + + The editor posts the instruction and the context it built from the map (the + + element index, the shortlisted subtrees, the annotations and a marked up + + snapshot by upload id). The answer is a list of operations the editor + + applies, reviews and saves itself; nothing here reads or writes the map. + + One job is one vendor call, long polled through `jobs/:id` as signage is.' + post: + summary: 'record what the person did with the answer: which ops were applied + or' + description: 'record what the person did with the answer: which ops were applied + or + + rejected and why, and the upload the saved map became' + tags: + - MapAI + operationId: PlaceOS::Api::MapAI_resolve + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.ResolveParams' + required: true + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.MapAI.JobResponse' + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "503": + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "403": + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "429": + description: Too Many Requests + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "422": + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "502": + description: Bad Gateway + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "504": + description: Gateway Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "406": + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + "408": + description: Request Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "415": + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + /api/engine/v2/maps/ai/usage: + summary: AI editing of a level's SVG map, with the signage AI provider rows. + description: 'AI editing of a level''s SVG map, with the signage AI provider rows. + + + The editor posts the instruction and the context it built from the map (the + + element index, the shortlisted subtrees, the annotations and a marked up + + snapshot by upload id). The answer is a list of operations the editor + + applies, reviews and saves itself; nothing here reads or writes the map. + + One job is one vendor call, long polled through `jobs/:id` as signage is.' + get: + summary: map edit spend per provider and model + tags: + - MapAI + operationId: PlaceOS::Api::MapAI_usage + parameters: + - name: from + in: query + description: unix seconds, defaults to 30 days ago + schema: + anyOf: + - type: integer + format: Int64 + - type: "null" + - name: to + in: query + description: unix seconds, defaults to now + schema: + anyOf: + - type: integer + format: Int64 + - type: "null" + responses: + "200": + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PlaceOS.Model.SignageAIJob.UsageRow' + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "503": + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "403": + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "404": + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "429": + description: Too Many Requests + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ImageGenError' + "422": + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "502": + description: Bad Gateway + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "504": + description: Gateway Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "406": + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' + "408": + description: Request Timeout + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.CommonError' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ParameterError' + "415": + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' /api/engine/v2/metadata/{name}/bulk: summary: Metadata, named JSON documents attached to zones, systems and users (e.g. floor maps, booking rules) @@ -28844,6 +29676,14 @@ paths: schema: type: integer format: Int32 + - name: kind + in: query + description: generate, edit or map_edit; the image kinds when left out + example: generate + schema: + anyOf: + - type: "null" + - type: string responses: "200": description: OK @@ -29284,8 +30124,8 @@ paths: schema: $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' /api/engine/v2/signage/ai/providers: - summary: Vendor credentials for signage image generation. - description: 'Vendor credentials for signage image generation. + summary: Vendor credentials for signage image generation and AI map editing. + description: 'Vendor credentials for signage image generation and AI map editing. The Data Stores rule: sys_admin writes, support reads. Responses never carry @@ -29358,7 +30198,7 @@ paths: schema: type: array items: - $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' + $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-ntext_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' "409": description: Conflict content: @@ -29454,7 +30294,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' + $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-ntext_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' "409": description: Conflict content: @@ -29534,8 +30374,8 @@ paths: schema: $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' /api/engine/v2/signage/ai/providers/{id}: - summary: Vendor credentials for signage image generation. - description: 'Vendor credentials for signage image generation. + summary: Vendor credentials for signage image generation and AI map editing. + description: 'Vendor credentials for signage image generation and AI map editing. The Data Stores rule: sys_admin writes, support reads. Responses never carry @@ -29558,7 +30398,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' + $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-ntext_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' "409": description: Conflict content: @@ -29660,7 +30500,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' + $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-ntext_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' "409": description: Conflict content: @@ -29854,7 +30694,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' + $ref: '#/components/schemas/NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-ntext_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c' "409": description: Conflict content: @@ -29934,8 +30774,8 @@ paths: schema: $ref: '#/components/schemas/PlaceOS.Api.Application.ContentError' /api/engine/v2/signage/ai/providers/{id}/test: - summary: Vendor credentials for signage image generation. - description: 'Vendor credentials for signage image generation. + summary: Vendor credentials for signage image generation and AI map editing. + description: 'Vendor credentials for signage image generation and AI map editing. The Data Stores rule: sys_admin writes, support reads. Responses never carry @@ -29947,7 +30787,9 @@ paths: description: 'Prove a row''s credentials work, without leaving anything behind: one small - image, discarded.' + image, discarded, or with `kind=text` one tiny chat completion against the + + row''s text model.' tags: - SignageAIProviders operationId: PlaceOS::Api::SignageAIProviders_test @@ -29958,6 +30800,12 @@ paths: schema: type: string format: uuid + - name: kind + in: query + description: image (the default) or text + example: text + schema: + type: string responses: "200": description: OK @@ -45637,6 +46485,243 @@ components: anyOf: - type: "null" - $ref: '#/components/schemas/PlaceOS.Model.Zone' + PlaceOS.Api.MapAI.Capabilities: + type: object + properties: + enabled: + type: boolean + reason: + anyOf: + - type: "null" + - type: string + providers: + type: array + items: + $ref: '#/components/schemas/PlaceOS.Api.Llm.ProviderCapabilities' + default_provider_id: + anyOf: + - type: "null" + - type: string + quota: + type: object + properties: + user_remaining_today: + anyOf: + - type: integer + format: Int32 + - type: "null" + domain_remaining_month: + anyOf: + - type: integer + format: Int32 + - type: "null" + limits: + type: object + properties: + max_context_bytes: + type: integer + format: Int32 + max_ops: + type: integer + format: Int32 + max_instruction_chars: + type: integer + format: Int32 + required: + - max_context_bytes + - max_ops + - max_instruction_chars + required: + - enabled + - providers + - quota + - limits + PlaceOS.Api.MapAI.EditParams: + type: object + properties: + zone_id: + type: string + instruction: + type: string + context: + anyOf: + - type: object + - type: "null" + markup_upload_id: + anyOf: + - type: "null" + - type: string + parent_job_id: + anyOf: + - type: string + format: uuid + - type: "null" + provider_id: + anyOf: + - type: string + format: uuid + - type: "null" + model: + anyOf: + - type: "null" + - type: string + idempotency_key: + anyOf: + - type: "null" + - type: string + required: + - zone_id + - instruction + PlaceOS.Api.MapAI.JobResponse: + type: object + properties: + id: + type: string + state: + type: string + kind: + type: string + zone_id: + anyOf: + - type: "null" + - type: string + provider: + anyOf: + - type: "null" + - type: string + model: + anyOf: + - type: "null" + - type: string + parent_job_id: + anyOf: + - type: "null" + - type: string + version: + type: integer + format: Int32 + instruction: + anyOf: + - type: "null" + - type: string + markup_upload_id: + anyOf: + - type: "null" + - type: string + schema_version: + anyOf: + - type: integer + format: Int64 + - type: "null" + summary: + anyOf: + - type: "null" + - type: string + ops: + type: array + items: + type: object + questions: + type: array + items: + type: object + usage: + anyOf: + - type: object + - type: "null" + resolution: + anyOf: + - type: object + - type: "null" + error_kind: + anyOf: + - type: "null" + - type: string + error_message: + anyOf: + - type: "null" + - type: string + cost_units: + anyOf: + - type: number + format: Float64 + - type: "null" + latency_ms: + anyOf: + - type: integer + format: Int64 + - type: "null" + created_at: + anyOf: + - type: integer + format: Int64 + - type: "null" + finished_at: + anyOf: + - type: integer + format: Int64 + - type: "null" + required: + - id + - state + - kind + - version + - ops + - questions + PlaceOS.Api.MapAI.ResolveParams: + type: object + properties: + decision: + type: string + accepted_op_ids: + type: array + items: + type: string + rejected_op_ids: + type: array + items: + type: string + reasons: + anyOf: + - type: object + - type: "null" + saved_upload_id: + anyOf: + - type: "null" + - type: string + required: + - decision + - accepted_op_ids + - rejected_op_ids + PlaceOS.Model.SignageAIJob.UsageRow: + type: object + properties: + kind: + type: string + provider: + type: string + model: + type: string + jobs: + type: integer + format: Int64 + candidates: + type: integer + format: Int64 + images_produced: + type: integer + format: Int64 + cost_units: + type: number + format: Float64 + required: + - kind + - provider + - model + - jobs + - candidates + - images_produced + - cost_units PlaceOS.Api.Metadata.Children: type: object properties: @@ -46351,32 +47436,6 @@ components: required: - upload_id - item_id - PlaceOS.Model.SignageAIJob.UsageRow: - type: object - properties: - provider: - type: string - model: - type: string - jobs: - type: integer - format: Int64 - candidates: - type: integer - format: Int64 - images_produced: - type: integer - format: Int64 - cost_units: - type: number - format: Float64 - required: - - provider - - model - - jobs - - candidates - - images_produced - - cost_units PlaceOS.Api.SignageAIProviders.ProviderParams: type: object properties: @@ -46408,6 +47467,10 @@ components: anyOf: - type: "null" - type: string + text_model: + anyOf: + - type: "null" + - type: string allowed_models: anyOf: - type: "null" @@ -48210,10 +49273,10 @@ components: required: - error - kind - description: 'Signage AI image generation. `kind` lets a client branch without - matching + description: 'Signage AI image generation and map AI editing. `kind` lets a + client - on message text.' + branch without matching on message text.' PlaceOS.Api.Application.ValidationError: type: object properties: @@ -48522,6 +49585,22 @@ components: - landscape - portrait - square + PlaceOS.Api.Llm.ProviderCapabilities: + type: object + properties: + id: + type: string + name: + type: string + provider: + type: string + text_model: + type: string + required: + - id + - name + - provider + - text_model PlaceOS.Api.MicrosoftNotification.ChangeNotification: type: object properties: @@ -49291,7 +50370,7 @@ components: type: string required: - commit_hash - ? NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c + ? NamedTuple-oid-u00003A-u000020String-nname-u00003A-u000020String-nprovider-u00003A-u000020String-nauthority_id-u00003A-u000020String-pNil-nendpoint-u00003A-u000020String-pNil-nlocation-u00003A-u000020String-pNil-ndefault_model-u00003A-u000020String-pNil-ntext_model-u00003A-u000020String-pNil-nallowed_models-u00003A-u000020Array-oString-c-nenabled-u00003A-u000020Bool-nis_default-u00003A-u000020Bool-nquotas-u00003A-u000020JSON.Any-ncreated_at-u00003A-u000020Int64-nupdated_at-u00003A-u000020Int64-c : type: object properties: id: @@ -49316,6 +50395,10 @@ components: anyOf: - type: "null" - type: string + text_model: + anyOf: + - type: "null" + - type: string allowed_models: type: array items: From 367e36bfb7d506787149aa9eedeba4d6d4d6fac3 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Mon, 12 Oct 2026 20:27:22 +1100 Subject: [PATCH 07/38] feat(map-ai): one op contract with the editor, retry a cut-off answer The vendor is now sent libs/map-editor's ops-schema.json as it is: one anyOf variant per op, set_style_rule.declarations and insert_shape.attrs as {name, value} arrays, schema_version in the envelope, $defs for target and name_value. The Crystal rendering that was sent before was a flat 26-field object with declarations as a string, so nothing the model returned could pass the editor's own validator. The file is embedded at compile time and the vocabulary (ops, fields, shapes, positions) is read from it, so the two copies cannot drift apart silently; the spec asserts the rendering against the file. Both copies carry sha256 ff41e284fc4e145587baa7227f60e01b16dfc08ca59db14ff5184385f0b2bbd5. Ops are stored exactly as the model returned them. The server's bookkeeping moves off the op: result.server_issues maps op-N (N the 1-based position) to the reason, because the editor's schemaIssues rejects any key outside the variant and the plan's inline server_issue and id would have locked every row. The checks follow the editor's rules: attribute, style and geometry values are syntax-checked (value-syntax.ts ported, named colours included), declarations are arrays, insert_shape attributes are per kind with transform refused, the deny-list for replacement markup is sanitise.ts's list plus ","why":"w"}), "