From cd5a9721819c7dc09e5989d837ed47c6c496b5ff Mon Sep 17 00:00:00 2001 From: "Nathan C." <149914029+Natuworkguy@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:33:12 -0700 Subject: [PATCH 1/8] Add Flash Web docs and fix bugs --- flash/system_prompt.txt | 2 +- flash/theme.py | 14 + flash/tools.py | 229 +++++++- flash/voice.py | 51 +- flash/web.py | 120 +++- flash/web/index.html | 1242 +++++++++++++++++++++++++++++++++++++-- flash/workspace.py | 63 +- tests/test_voice.py | 62 +- tests/test_web.py | 249 +++++++- 9 files changed, 1928 insertions(+), 104 deletions(-) diff --git a/flash/system_prompt.txt b/flash/system_prompt.txt index a541b5d..7f20172 100644 --- a/flash/system_prompt.txt +++ b/flash/system_prompt.txt @@ -42,7 +42,7 @@ A new file too long for one call is written in pieces: the first with no `append == Sending what you make == The user sees only what you send. `view_image` and `screenshot` show a file to you, never to them, and an image in a tool result is one you looked at, not one the user gave you. -When you make an image, a PDF, or a web page for the user, send it the moment it is finished, without being asked: `send_image` for a picture, `send_pdf` for a PDF, `send_html` for a page. Every revision is a new version: send it again each time, and never just view it and describe the change. +When you make an image, a PDF, a web page, or a document for the user, send it the moment it is finished, without being asked: `send_image` for a picture, `send_pdf` for a PDF, `send_html` for a page, `send_document` for Markdown or text. The user can edit a document you send and comment on it; when they do, read the file again before you change it, and answer each comment. When a document you send has unfinished parts, gaps, or open questions, say so in comments on those passages rather than leaving them to be found. Every revision is a new version: send it again each time, and never just view it and describe the change. Check before you send, and check for the exact problem the user reported (overlapping lines, a wrong arrow, cut-off text). If you can still see it, or cannot tell, say so plainly. Never tell the user a problem is fixed because you changed the code meant to fix it: it is fixed when you looked and it was gone. diff --git a/flash/theme.py b/flash/theme.py index f5d8a1b..7d9d0fe 100644 --- a/flash/theme.py +++ b/flash/theme.py @@ -329,6 +329,20 @@ def tool_file(path: str) -> bool: return True +def tool_document(path: str, comments: list[dict]) -> bool: + """Hand a document to whoever is drawing, with the agent's comments + on it: each a quote from the document and a note about it. + + True when something took it, as with `tool_file`. + """ + + sink = _sink() + if sink is None: + return False + sink("document", json.dumps({"path": path, "comments": comments}), "") + return True + + def tool_plan(steps: list[dict]) -> bool: """Hand the plan's checklist to whoever is drawing, as its steps. diff --git a/flash/tools.py b/flash/tools.py index f87eddb..90c7ecd 100644 --- a/flash/tools.py +++ b/flash/tools.py @@ -61,6 +61,7 @@ plural, remote_answer, tool_diff, + tool_document, tool_file, tool_line, tool_result, @@ -120,9 +121,15 @@ To hand the user a finished web page, use the send_html tool with its path. It opens in their browser, or beside the chat in the web UI. Screenshot it first and send it once it looks right. -When you make an image, PDF, or web page for the user, send it with the - matching tool as soon as it is finished, without being asked: that is - how they see it. +To hand the user a Markdown or text document (a report, plan, README, + notes), use the send_document tool with its path. In the web UI it + opens beside the chat, where they can edit it and comment on it; their + comments reach you as a message that quotes each passage. To flag + unfinished work, a gap, or a question in it, pass comments, each + quoting the words it is about. +When you make an image, PDF, web page, or document for the user, send it + with the matching tool as soon as it is finished, without being asked: + that is how they see it. To see how a web page actually renders, use the screenshot tool on the .html file you wrote or on a URL. It runs a headless browser and attaches the picture, so it is the only way to check a page you built; @@ -2407,6 +2414,163 @@ def send_html(path: str, caption: str = "") -> str: ) +DOCUMENT_SUFFIXES = (".md", ".markdown", ".txt") +MAX_DOCUMENT_BYTES = 2 * 1024 * 1024 +MAX_DOC_COMMENTS = 20 +MAX_QUOTE_CHARS = 300 +MAX_NOTE_CHARS = 1000 + +_MD_LINK = re.compile(r"!?\[([^\]]*)\]\([^)]*\)") +_MD_LINE_MARK = re.compile( + r"^[ \t]*(?:#{1,6}[ \t]+|>[ \t]?|[-*+][ \t]+|\d+[.)][ \t]+)", re.M +) +_MD_INLINE_MARK = re.compile(r"?u>|\*\*|__|~~|`|(?|.+-])") + + +def _plain(text: str) -> str: + """Text as the page shows it: Markdown's marks gone, spaces single. + + A comment quotes words as they read, and the model may copy them + from the file with their marks or without, so both are compared + this way.""" + + text = _MD_LINK.sub(r"\1", text) + text = _MD_LINE_MARK.sub("", text) + text = _MD_INLINE_MARK.sub("", text) + text = _MD_ESCAPE.sub(r"\1", text) + return " ".join(text.split()) + + +def _doc_comments( + raw: Any, text: str, +) -> tuple[list[dict], list[str]]: + """The comments the model left on a document, cleaned, and a line + for each one whose quote is not in the document. + + A quote that is not found still goes: the page shows it as a note + on the whole document.""" + + if isinstance(raw, str): + try: + raw = json.loads(raw) if raw.strip() else [] + except json.JSONDecodeError: + return [], ["comments was not a list; none were added"] + if isinstance(raw, dict): + raw = [raw] + if not isinstance(raw, list): + return [], ["comments was not a list; none were added"] + + plain_text = _plain(text) + flat_text = " ".join(text.split()) + comments: list[dict] = [] + missing: list[str] = [] + for item in raw[:MAX_DOC_COMMENTS]: + if not isinstance(item, dict): + continue + quote = str(item.get("quote") or "").strip()[:MAX_QUOTE_CHARS] + note = str( + item.get("note") or item.get("comment") or item.get("text") or "" + ).strip()[:MAX_NOTE_CHARS] + if not note: + continue + comments.append({"quote": quote, "note": note}) + if quote and " ".join(quote.split()) not in flat_text and ( + _plain(quote) not in plain_text + ): + missing.append( + f'comment {len(comments)} quotes "{quote[:60]}", which is ' + "not in the document word for word, so it shows without " + "a place" + ) + return comments, missing + + +def send_document( + path: str, caption: str = "", comments: Any = None, +) -> str: + """Put a Markdown or text document in front of the user, with any + comments the model left on its passages.""" + + tool_line(f"SendDocument({path})") + + doc = Path(path).expanduser() + problem = "" + size = 0 + if not doc.is_file(): + problem = f"no file at {doc}" + elif doc.suffix.lower() not in DOCUMENT_SUFFIXES: + problem = f"{doc.name} is not a .md or .txt file" + else: + try: + size = doc.stat().st_size + doc.read_bytes().decode("utf-8") + except OSError as exc: + problem = f"could not read {doc}: {exc}" + except UnicodeDecodeError: + problem = f"{doc.name} is not UTF-8 text" + else: + if size > MAX_DOCUMENT_BYTES: + problem = ( + f"{doc.name} is {size // 1024} KB; the limit is " + f"{MAX_DOCUMENT_BYTES // (1024 * 1024)} MB" + ) + + if problem: + result = f"Error: {problem}." + tool_result(result, style=ERROR) + return result + + kilobytes = max(1, round(size / 1024)) + note = caption.strip() + label = f"{doc.name} ({kilobytes} KB)" + (f": {note}" if note else "") + left, missing = _doc_comments( + comments, doc.read_text(encoding="utf-8") + ) + count = len(left) + if count: + label += f", {count} comment{'' if count == 1 else 's'}" + unplaced = f" To fix: {'; '.join(missing)}." if missing else "" + pinned = ( + f" Your {count} comment{'' if count == 1 else 's'} show" + f"{'s' if count == 1 else ''} on the passages quoted." + if count else "" + ) + + shown = tool_document(str(doc), left) if left else tool_file(str(doc)) + if shown: + tool_result(label) + return ( + f"Sent {doc.name} ({kilobytes} KB) to the user's screen, " + f"beside the chat.{pinned} They can edit it there, and " + "saving writes the file. Their comments come to you as a " + f"message quoting each passage.{unplaced}" + ) + + problem = _open_with_spinner(doc) + tool_result(label + (f" ({problem})" if problem else "")) + console.print( + Text(f"{' ' * RESULT_INDENT}{_display_path(doc)}", style=DIM) + ) + # The app it opens in has no place for them: they print under it. + for item in left: + where = f'"{item["quote"]}": ' if item["quote"] else "" + console.print( + Text(f"{' ' * RESULT_INDENT}{where}{item['note']}", style=DIM) + ) + + if problem: + return ( + f"Could not open {doc.name}: {problem}. Its path is on " + "screen; tell the user where the file is." + ) + return ( + f"Sent {doc.name} ({kilobytes} KB). It opened in the user's " + "default app for it, with its path on screen" + + (", and your comments printed under it." if left else ".") + ) + + DEFAULT_SCREENSHOT_WIDTH = 1280 DEFAULT_SCREENSHOT_HEIGHT = 800 MIN_SCREENSHOT_SIDE = 200 @@ -3089,6 +3253,64 @@ def interact( }, }, }, + { + "type": "function", + "function": { + "name": "send_document", + "description": ( + "Show a Markdown or text document (.md, .markdown, .txt) " + "to the user: a report, plan, README, or notes you wrote. " + "In the web UI it opens beside the chat, rendered, where " + "they can edit it and comment on passages; their comments " + "reach you as a message quoting each one. Write the file " + "first; this only shows it. To flag something for them in " + "it (unfinished work, a gap to fill, an open question, an " + "assumption to check), add comments pinned to passages." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Path to the document.", + }, + "caption": { + "type": "string", + "description": ( + "Optional single line shown with it." + ), + }, + "comments": { + "type": "array", + "description": ( + "Optional notes for the user, each pinned to " + "a passage: what is unfinished, missing, or " + "needs their decision there." + ), + "items": { + "type": "object", + "properties": { + "quote": { + "type": "string", + "description": ( + "A few words from the document, " + "exactly as they read, marking " + "the passage." + ), + }, + "note": { + "type": "string", + "description": "What to tell them.", + }, + }, + "required": ["quote", "note"], + }, + }, + }, + "required": ["path"], + }, + }, + }, { "type": "function", "function": { @@ -3693,6 +3915,7 @@ def interact( "send_image": send_image, "send_pdf": send_pdf, "send_html": send_html, + "send_document": send_document, "screenshot": screenshot, "open_page": open_page, "interact": interact, diff --git a/flash/voice.py b/flash/voice.py index 7842ff5..4bb678b 100644 --- a/flash/voice.py +++ b/flash/voice.py @@ -22,6 +22,7 @@ from collections.abc import Callable from dataclasses import dataclass from pathlib import Path +from typing import Optional from urllib.error import URLError from urllib.request import Request, urlopen @@ -258,8 +259,23 @@ def web_missing() -> list[str]: return [p for p in missing_packages() if p != "sounddevice"] -def _download(url: str, out: Path, label: str, on_progress: Progress) -> str: - """Stream `url` to `out`, reporting percent complete as it goes.""" +# What a download returns when it was called off: not a failure to +# report, and nothing of it is left on disk. +CANCELLED = "cancelled" + + +class _Stopped(Exception): + """The download was called off between chunks.""" + + +def _download( + url: str, out: Path, label: str, on_progress: Progress, + stop: Optional[threading.Event] = None, +) -> str: + """Stream `url` to `out`, reporting percent complete as it goes. + + Setting `stop` calls it off at the next chunk, a moment at most, and + it returns CANCELLED with the part it had deleted.""" out.parent.mkdir(parents=True, exist_ok=True) part = out.with_suffix(out.suffix + ".part") @@ -274,6 +290,8 @@ def _download(url: str, out: Path, label: str, on_progress: Progress) -> str: with part.open("wb") as handle: while True: + if stop is not None and stop.is_set(): + raise _Stopped chunk = response.read(DOWNLOAD_CHUNK) if not chunk: break @@ -284,6 +302,9 @@ def _download(url: str, out: Path, label: str, on_progress: Progress) -> str: on_progress(label, percent) part.replace(out) + except _Stopped: + part.unlink(missing_ok=True) + return CANCELLED except (URLError, OSError, ValueError) as exc: part.unlink(missing_ok=True) return f"could not download {label}: {exc}" @@ -310,7 +331,9 @@ def _unpack(archive: Path, into: Path) -> str: return "" -def ensure_models(on_progress: Progress) -> str: +def ensure_models( + on_progress: Progress, stop: Optional[threading.Event] = None, +) -> str: """Download whatever voice mode is missing. Returns "" when ready. Both models are large enough that the download is worth showing, so @@ -321,11 +344,13 @@ def ensure_models(on_progress: Progress) -> str: if models_present(): return "" - return (download_listening(vosk_model(), on_progress) - or download_voice(piper_voice(), on_progress)) + return (download_listening(vosk_model(), on_progress, stop) + or download_voice(piper_voice(), on_progress, stop)) -def download_listening(name: str, on_progress: Progress) -> str: +def download_listening( + name: str, on_progress: Progress, stop: Optional[threading.Event] = None, +) -> str: """Fetch and unpack the Vosk model NAME. Returns "" once it is in.""" if listening_installed(name): @@ -335,6 +360,7 @@ def download_listening(name: str, on_progress: Progress) -> str: archive = MODELS_DIR / f"{name}.zip" why = _download( f"{VOSK_BASE}/{name}.zip", archive, "listening model", on_progress, + stop, ) if why: return why @@ -351,7 +377,9 @@ def download_listening(name: str, on_progress: Progress) -> str: return "" -def download_voice(name: str, on_progress: Progress) -> str: +def download_voice( + name: str, on_progress: Progress, stop: Optional[threading.Event] = None, +) -> str: """Fetch the Piper voice NAME, network and settings. Returns "" once it is in.""" @@ -368,8 +396,13 @@ def download_voice(name: str, on_progress: Progress) -> str: onnx, config = piper_paths(name) MODELS_DIR.mkdir(parents=True, exist_ok=True) - return (_download(onnx_url, onnx, "voice", on_progress) - or _download(config_url, config, "voice settings", on_progress)) + why = (_download(onnx_url, onnx, "voice", on_progress, stop) + or _download(config_url, config, "voice settings", on_progress, + stop)) + # Called off between its two files: no voice without its settings. + if why == CANCELLED: + onnx.unlink(missing_ok=True) + return why def remove_listening(name: str) -> None: diff --git a/flash/web.py b/flash/web.py index 7aba5f6..f8c07f8 100644 --- a/flash/web.py +++ b/flash/web.py @@ -442,6 +442,13 @@ def sign_out_others(self, keep: str) -> list[str]: self.token = secrets.token_urlsafe(24) return keys + def new_token(self) -> str: + """End the link and make another. Signed-in browsers stay.""" + + with self._lock: + self.token = secrets.token_urlsafe(24) + return self.token + def listing(self, current: str, watching: set) -> list[dict]: """The signed-in browsers, as the page shows them, newest first.""" @@ -630,10 +637,16 @@ class Session: def __init__(self, hub: Optional[Hub] = None) -> None: self.hub = hub or Hub() self.lan = False + # Documents the user edited in the page since each chat's last + # turn: chat id -> the paths, told to the model with its next + # message so it reads them again rather than writing over them. + self.edited: dict[str, list[str]] = {} # The voice models are being downloaded for the page: the first # use's pair, or one model picked in Settings, (kind, name). self.voice_setup = False self.voice_job: Optional[tuple[str, str]] = None + # Set to call off whichever of those is running. + self.voice_stop = threading.Event() # The link's token and the browsers signed in with it. A server # restarted after an update takes over the old one's (Server). self.access = Access() @@ -647,7 +660,7 @@ def __init__(self, hub: Optional[Hub] = None) -> None: # Starts this same `flash --web` again, as whatever version is # installed now. Only a server that owns its process can: one # beside a terminal session would take the session down with it. - self.restart: Optional[Callable[[], None]] = None + self.restart: Optional[Callable[..., None]] = None # Opens the server again listening on the network, or not. Set # by whatever runs the server (see _attach). self.switch_lan: Optional[Callable[[bool], None]] = None @@ -755,6 +768,7 @@ def set_up_voice(self) -> None: if self.voice_setup or self.voice_job: return self.voice_setup = True + self.voice_stop.clear() def run() -> None: said: dict[str, int] = {} @@ -769,13 +783,15 @@ def progress(label: str, percent: int) -> None: why = "" try: - why = voice.ensure_models(progress) + why = voice.ensure_models(progress, self.voice_stop) finally: with self._lock: self.voice_setup = False - self.hub.publish( - {"type": "voice-setup", "done": True, "error": why} - ) + cancelled = why == voice.CANCELLED + self.hub.publish({ + "type": "voice-setup", "done": True, + "error": "" if cancelled else why, "cancelled": cancelled, + }) threading.Thread(target=run, daemon=True).start() @@ -790,6 +806,7 @@ def fetch_voice_model(self, kind: str, name: str) -> None: if self.voice_setup or self.voice_job: raise ValueError("A voice model is already downloading.") self.voice_job = (kind, name) + self.voice_stop.clear() def run() -> None: said: list[int] = [-1] @@ -806,7 +823,7 @@ def progress(label: str, percent: int) -> None: try: fetch = (voice.download_listening if kind == "listening" else voice.download_voice) - why = fetch(name, progress) + why = fetch(name, progress, self.voice_stop) if not why: ai.set_config_var(VOICE_SETTINGS[kind], name) except Exception as exc: # noqa: BLE001 @@ -814,13 +831,25 @@ def progress(label: str, percent: int) -> None: finally: with self._lock: self.voice_job = None + cancelled = why == voice.CANCELLED self.hub.publish({ "type": "voice-model", "kind": kind, "name": name, - "done": True, "error": why, + "done": True, "error": "" if cancelled else why, + "cancelled": cancelled, }) threading.Thread(target=run, daemon=True).start() + def cancel_voice_download(self) -> bool: + """Call off the voice download running, if one is. True when one + was: it stops at its next chunk and says so to every page.""" + + with self._lock: + running = self.voice_setup or self.voice_job is not None + if running: + self.voice_stop.set() + return running + # Sub-agents ---------------------------------------------------- def adopt(self, chat: Chat, agent_ids: set) -> None: @@ -1245,9 +1274,12 @@ def sink(kind: str, text: str, style: str) -> None: session.emit(chat, {"type": "diff", "text": text}) elif kind == "plan": session.emit(chat, {"type": "plan", "steps": json.loads(text)}) - elif kind == "file": + elif kind in ("file", "document"): + shown = json.loads(text) if kind == "document" else {"path": text} try: - kept = workspace.keep_file(text) + kept = workspace.keep_file(shown["path"]) + if shown.get("comments"): + kept["comments"] = shown["comments"] session.emit(chat, {"type": "file", **kept}) except (workspace.WorkspaceError, OSError) as exc: session.emit(chat, { @@ -1325,6 +1357,17 @@ def project_prompt(found: "workspace.Project") -> str: return "\n".join(lines) +def edited_note(paths: list[str]) -> str: + """What the model is told about documents the user edited by hand.""" + + names = ", ".join(paths) + return ( + f"[The user edited {names} in the side panel and saved it. Read " + "it again before you change it: their version is the one that " + "counts.]" + ) + + def attachments(files: Optional[list]) -> list[dict]: """The files a message carries, as the page shows them: those that are really there, and no more than MAX_ATTACHMENTS.""" @@ -1392,6 +1435,11 @@ def run_turn( # its own sub-agents and never another chat's. owned = session.owned(chat.id) news, delivered = subagents.notices(owned) if owned else ("", []) + edited = session.edited.pop(chat.id, []) + if edited: + news = "\n\n".join( + part for part in (news, edited_note(edited)) if part + ) said, images = outgoing(ai, text, files) if images and not model_sees_images(ai.Config.host, ai.Config.model): session.emit(chat, {"type": "note", "text": ( @@ -1918,6 +1966,23 @@ def command(session: Session, body: dict, browser: str = "") -> dict: threading.Timer(RESTART_DELAY, session.restart).start() return session.updates.snapshot() + if name == "server-restart": + # From the page's shortcut: Flash started again, with a new link. + # The page is handed the new token to come back in with. + if session.restart is None: + raise ValueError( + "This Flash runs beside a terminal session. Quit it there " + "and start it again." + ) + if any(c.busy or c.queued for c in session.chats.values()): + raise ValueError("Wait for the reply to finish first.") + token = session.access.new_token() + threading.Timer( + RESTART_DELAY, session.restart, + ("Restarting, as asked from the page. The new link follows.",), + ).start() + return {"token": token} + if name == "skills": return {"skills": [ { @@ -2013,6 +2078,9 @@ def command(session: Session, body: dict, browser: str = "") -> dict: if name == "voice-models": return voice_models(session) + if name == "voice-cancel": + return {"cancelling": session.cancel_voice_download()} + if name in ("voice-model", "voice-model-remove"): kind = str(body.get("kind") or "") if kind not in VOICE_KINDS: @@ -2138,6 +2206,16 @@ def command(session: Session, body: dict, browser: str = "") -> dict: if name == "dirs": return {"dirs": workspace.folder_suggestions(arg)} + if name == "document-save": + saved = workspace.save_document(arg, str(body.get("text") or "")) + # Told to the model with the chat's next message. + where = saved["path"] or arg + if chat_id in session.chats: + listed = session.edited.setdefault(chat_id, []) + if where not in listed: + listed.append(where) + return saved + if name == "undo": message = checkpoint.undo() if chat_id in session.chats: @@ -2575,8 +2653,10 @@ def __init__( lan: bool = False, ): # Set before binding: a port already in use makes the base class - # call server_close() from inside its own __init__. + # call server_close() from inside its own __init__, before any of + # the rest is set up. self.closing = threading.Event() + self.keep_session = False super().__init__((LAN_HOST if lan else HOST, port), Handler) # Cookies ignore the port, so two servers on one machine each # need a name of their own. @@ -2596,7 +2676,6 @@ def __init__( self.swapping = False self.swapped = threading.Event() self.replacement: Optional[Server] = None - self.keep_session = False @property def port(self) -> int: @@ -2637,9 +2716,14 @@ def network_url(self) -> Optional[str]: def server_close(self) -> None: self.closing.set() - self.session.hub.close() - if not self.keep_session: - self.session.close() + # No session yet when the port could not be had: there is nothing + # to close but the socket, and the OSError that says why has to + # get out. + session = getattr(self, "session", None) + if session is not None: + session.hub.close() + if not self.keep_session: + session.close() super().server_close() @@ -2743,7 +2827,9 @@ def _listen( RESTART_DELAY = 0.4 -def _restart(server: "Server") -> None: +def _restart( + server: "Server", why: str = "Restarting to finish the update.", +) -> None: """Start this `flash --web` again as the version now installed. exec replaces the process in place: the same port, the same @@ -2759,7 +2845,7 @@ def _restart(server: "Server") -> None: args.append("--lan") if "--no-open" not in args: args.append("--no-open") - console.print(Text("Restarting to finish the update.", style=DIM)) + console.print(Text(why, style=DIM)) os.execv( # nosec B606 -- this same interpreter, running Flash again sys.executable, [sys.executable, "-m", "flash", *args] ) @@ -2814,7 +2900,7 @@ def _attach(server: "Server", standalone: bool) -> None: # Windows cannot replace a running Flash at all: its update # finishes after this one quits, so there is nothing to # restart. - session.restart = lambda: _restart(server) + session.restart = lambda *why: _restart(server, *why) session.updates.can_restart = True else: _background = server diff --git a/flash/web/index.html b/flash/web/index.html index a7250b5..ea2f311 100644 --- a/flash/web/index.html +++ b/flash/web/index.html @@ -6,6 +6,9 @@