Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions backend/app/assistant/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,31 @@ def _placement_of(action: Dict[str, Any]) -> Dict[str, Optional[str]]:
}


# Long enough to say what the picture is, short enough to read as a name in the Assets
# list rather than as the whole prompt pasted in.
IMAGE_NAME_MAX_LEN = 60


def image_name(alt: Optional[str], prompt: str) -> str:
"""The name a generated image block carries: the plan's `alt`, else its prompt's lead.

That name is the only label the file ever gets — on disk it is a UUID, and
`sync_note_assets` registers the asset under the block's name or caption. `alt` is
optional, and the per-paragraph example in the planner prompt never showed it, so a
plan could leave it out and list every image in the Assets tab by its UUID. A text-to-image
prompt nearly always opens on its subject, so its first sentence stands in, cut at
a word boundary.
"""
if alt and alt.strip():
return alt.strip()
text = re.sub(r"\s+", " ", prompt or "").strip()
lead = re.split(r"(?<=[.!?;:])\s", text, maxsplit=1)[0].rstrip(".!?;:")
if len(lead) <= IMAGE_NAME_MAX_LEN:
return lead
cut = lead[:IMAGE_NAME_MAX_LEN - 1].rsplit(" ", 1)[0].rstrip(" ,;:-")
return f"{cut}…"


def collect_embeds(blocks: Sequence[Any]) -> List[Any]:
"""Child-note, reference and diagram blocks found anywhere in `blocks`.

Expand Down Expand Up @@ -1137,8 +1162,9 @@ def _do_generate_image(self, action: Dict[str, Any]) -> ActionResult:
# Generate first: it is a paid, fallible call, and a failure must leave the
# note untouched rather than half-edited.
self.check_cancelled()
prompt = action.get("prompt") or ""
try:
url = self._generate_image_url(action.get("prompt") or "")
url = self._generate_image_url(prompt)
except Exception as exc:
from app.jobs.runner import readable_error

Expand All @@ -1149,7 +1175,7 @@ def _do_generate_image(self, action: Dict[str, Any]) -> ActionResult:
block = {
"id": str(uuid.uuid4()),
"type": "image",
"props": {"url": url, "name": action.get("alt") or "", "caption": "", "showPreview": True},
"props": {"url": url, "name": image_name(action.get("alt"), prompt), "caption": "", "showPreview": True},
"children": [],
}
placement = resolve_placement(blocks, **_placement_of(action))
Expand Down
42 changes: 39 additions & 3 deletions backend/tests/test_assistant_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@
from sqlmodel import Session, SQLModel, create_engine, select

from app.assistant.executor import (
ActionResult, Cancelled, ExecContext, PlanExecutor,
IMAGE_NAME_MAX_LEN, ActionResult, Cancelled, ExecContext, PlanExecutor,
build_result_summary, collect_embeds, find_anchor_index, find_section_index,
normalize_heading, resolve_placement, section_heading,
image_name, normalize_heading, resolve_placement, section_heading,
)
from app.models import Annotation, Category, Folder, Note, NoteVersion, Recipe
from app.models import Annotation, Category, Folder, Note, NoteAsset, NoteVersion, Recipe

USER = "user-1"
OTHER = "user-2"
Expand Down Expand Up @@ -501,6 +501,42 @@ def test_a_section_alone_places_relative_to_its_heading(session, fake_images, po
assert order[at - 1:at + 2] == expected


def image_block(session) -> dict:
return next(b for b in blocks_of(session, "note-1") if b["type"] == "image")


def test_a_generated_image_is_named_by_its_alt(session, fake_images):
make_note(session)
executor(session).run({"actions": [image(alt="Lighthouse at dusk")]})
assert image_block(session)["props"]["name"] == "Lighthouse at dusk"


def test_a_generated_image_without_alt_is_named_from_its_prompt(session, fake_images):
# The reported bug: a per-paragraph plan left out the optional "alt", and every
# image landed in the Assets tab under its UUID filename.
make_note(session)
executor(session).run({"actions": [image(
prompt="A lone lighthouse on a storm-battered cliff at dusk. Oil painting, moody palette.",
)]})
assert image_block(session)["props"]["name"] == "A lone lighthouse on a storm-battered cliff at dusk"


def test_image_name_cuts_a_long_prompt_at_a_word():
name = image_name(None, "A sweeping aerial view of an ancient walled city at golden hour, "
"with terracotta rooftops, winding cobbled streets and distant hills")
assert name == "A sweeping aerial view of an ancient walled city at golden…"
assert len(name) <= IMAGE_NAME_MAX_LEN
assert image_name(" ", "Fox") == "Fox"


def test_a_generated_image_reaches_the_assets_tab_by_name(session, monkeypatch):
monkeypatch.setattr(PlanExecutor, "_generate_image_url", lambda self, prompt: f"/media/{USER}/abc.jpg")
make_note(session)
executor(session).run({"actions": [image(prompt="A red fox asleep in fresh snow, soft light")]})
asset = session.exec(select(NoteAsset).where(NoteAsset.note_id == "note-1")).one()
assert asset.original_name == "A red fox asleep in fresh snow, soft light"


def test_a_diagram_can_be_placed_after_a_paragraph(session):
make_note(session, blocks=chapters())
result = executor(session).run({"actions": [{
Expand Down
12 changes: 7 additions & 5 deletions frontend/src/services/aiPlan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,10 @@ export type PlanAction =
| { type: 'edit_diagram'; noteId: string; diagramId: string; source: string; description?: string }
// Generates an image via fal.ai from `prompt` and inserts it into the note, placed by
// `section` / `anchorText` / `position` (see Placement; else appended at the end).
// `alt` (optional): a short caption for the image block. The prompt is authored by the
// model from the article content — it is NOT the user's raw request.
// `alt`: a short title for the image — it becomes the block's name, which is what the
// Assets tab lists the file under. The parser keeps it optional (the executor falls
// back to the prompt's opening) but the instructions ask for it every time. The prompt
// is authored by the model from the article content — it is NOT the user's raw request.
| ({ type: 'generate_image'; noteId: string; prompt: string; section?: string; alt?: string; description?: string } & Placement)
// Recipes: saved, reusable prompts the user runs later from the composer's picker or by
// voice ("run the summary recipe") instead of retyping them — see the "Recipes" rule below.
Expand Down Expand Up @@ -138,7 +140,7 @@ Action types (every action MAY also include an optional "description": one short
- delete_annotation: { "type":"delete_annotation", "noteId":"<id>", "annotationId":"<id>" }
- create_diagram: { "type":"create_diagram", "noteId":"<id>", "source":"<complete Mermaid diagram source>", "section":"<optional heading>", "anchorText":"<optional>", "position":"<optional>" } (placement: see "Placing blocks")
- edit_diagram: { "type":"edit_diagram", "noteId":"<id>", "diagramId":"<id>", "source":"<complete replacement Mermaid diagram source>" }
- generate_image: { "type":"generate_image", "noteId":"<id>", "prompt":"<detailed text-to-image prompt>", "section":"<optional heading>", "anchorText":"<optional>", "position":"<optional>", "alt":"<optional caption>" } (placement: see "Placing blocks")
- generate_image: { "type":"generate_image", "noteId":"<id>", "prompt":"<detailed text-to-image prompt>", "section":"<optional heading>", "anchorText":"<optional>", "position":"<optional>", "alt":"<short title naming the image>" } (placement: see "Placing blocks")
- create_recipe: { "type":"create_recipe", "name":"<short name>", "prompt":"<self-contained prompt text>", "tags":["optional", "lowercase", "tags"] }
- update_recipe: { "type":"update_recipe", "recipeId":"<id>", "name":"<optional new name>", "prompt":"<optional new prompt>", "tags":["optional new tags"] }
- delete_recipe: { "type":"delete_recipe", "recipeId":"<id>" }
Expand Down Expand Up @@ -166,10 +168,10 @@ Rules:
- "anchorText": the opening words (about 6–12, copied verbatim) of the existing paragraph, list item, quote or other block to put it next to. Use it WHENEVER the user places something relative to a paragraph or any other non-heading block ("above every paragraph", "after the paragraph about X", "below the list", "before the quote") — a heading alone cannot express that, and every block would end up stacked directly beneath the heading. Quote text that already exists in the note when the action runs: the body you were given, or text an earlier action in this plan writes inline in "content" (never a deferred "spec" body, whose words are not known yet).
- "position": with "anchorText", "before" puts the block directly above that paragraph and "after" (the default) directly below it. With only "section", "after" (the default) puts it directly beneath the heading, "before" just above the heading, and "end" after the last block of that section.
- "section": a heading's text. With "anchorText" it narrows the search to that section — set it whenever the paragraph sits in a known section/chapter, so the same wording elsewhere in the note cannot capture the block. On its own it places the block relative to that heading (see "position").
With none of these the block is appended at the end of the note. For "one per paragraph" requests, emit one action PER PARAGRAPH, each with its own "anchorText" taken from that paragraph — e.g. "add an image above every paragraph in chapter 2" → one generate_image per paragraph of chapter 2, each with "section":"Chapter 2", "anchorText":"<that paragraph's opening words>", "position":"before".
With none of these the block is appended at the end of the note. For "one per paragraph" requests, emit one action PER PARAGRAPH, each with its own "anchorText" taken from that paragraph — e.g. "add an image above every paragraph in chapter 2" → one generate_image per paragraph of chapter 2, each with "section":"Chapter 2", "anchorText":"<that paragraph's opening words>", "position":"before" and its own "alt".
- Annotations: a note's existing annotations are listed under it as "Annotations on this note" with an "[annotation <id>]" and the snippet of the block they are anchored to. To edit/delete one, use its "<id>" as "annotationId". To add one, set "anchorText" to a short verbatim snippet of the block the annotation should attach to (it is matched against the note's block text). When asked to "read the annotations and revise the note", read these annotation texts and apply the implied edits with edit_section / edit_note / append_note actions.
- Diagrams: use create_diagram to ADD a new diagram to a note, and edit_diagram to change an existing one. A note's existing diagrams are listed under it as "Diagrams on this note" with a "[diagram <id>]" tag and their current Mermaid source — use that "<id>" as "diagramId" for edit_diagram (which REPLACES the whole diagram, so "source" must be the complete new diagram, not a fragment). "source" must be complete, valid Mermaid syntax starting with the right header keyword for the kind: "flowchart TD" (or LR/BT/RL) for flow charts, "mindmap" for mind maps, "sequenceDiagram" for sequence diagrams, "classDiagram" for class diagrams, "stateDiagram-v2" for state diagrams, "erDiagram" for entity-relationship diagrams, "gantt" for Gantt charts, "pie" for pie charts, "timeline" for timelines. Node linking: in flowchart, classDiagram and stateDiagram-v2 ONLY, a node can link to another note or a URL by adding a line "click <nodeId> href \"/notes/<id>\"" (linking to a note id from the lists below) or "click <nodeId> href \"<url>\" \"_blank\"" (linking to the web) — do NOT add click/href lines for mindmap, sequenceDiagram, erDiagram, gantt, pie or timeline diagrams, since Mermaid does not support node links on those kinds (mindmap link support is a currently open Mermaid limitation). Only create or edit a diagram when the user explicitly asks for one (e.g. "make a mind map of this note", "add a step to the flow chart").
- Images (generate_image): Use ONLY when the user explicitly asks to create/generate/add an image, picture, illustration or photo (e.g. "make an image for this article", "add a picture", "create an image for each chapter and put it under each title"). YOU author the "prompt": write a vivid, self-contained text-to-image prompt derived from the relevant article content (describe subject, setting, style, mood, composition) — do NOT just copy the user's request verbatim. Place it as described in "Placing blocks": "section" alone puts it directly beneath that heading, "anchorText" + "position" puts it above or below a specific paragraph, and neither appends it at the end of the note. For "an image for each chapter/section", emit ONE generate_image action per chapter — each with that chapter's heading text as "section" and its own prompt tailored to that chapter; for "an image for each paragraph", emit one per paragraph, each anchored to its paragraph. Optionally set "alt" to a short caption. Generating images costs money, so create only the images the user asked for and no more.
- Images (generate_image): Use ONLY when the user explicitly asks to create/generate/add an image, picture, illustration or photo (e.g. "make an image for this article", "add a picture", "create an image for each chapter and put it under each title"). YOU author the "prompt": write a vivid, self-contained text-to-image prompt derived from the relevant article content (describe subject, setting, style, mood, composition) — do NOT just copy the user's request verbatim. Place it as described in "Placing blocks": "section" alone puts it directly beneath that heading, "anchorText" + "position" puts it above or below a specific paragraph, and neither appends it at the end of the note. For "an image for each chapter/section", emit ONE generate_image action per chapter — each with that chapter's heading text as "section" and its own prompt tailored to that chapter; for "an image for each paragraph", emit one per paragraph, each anchored to its paragraph. ALWAYS set "alt" to a short title for that image (3–8 words, e.g. "Lighthouse on a cliff at dusk") — it is the name the image is listed under in the note's assets, so without it the user sees only a meaningless file id. Generating images costs money, so create only the images the user asked for and no more.
- Recipes (create_recipe / update_recipe / delete_recipe): a Recipe is a saved, reusable prompt the user can run later — from a picker in the AI composer, or by voice ("run the summary recipe") — instead of retyping it each time. The user's existing recipes are listed below under "Recipes". Only create, update or delete a recipe when the user EXPLICITLY asks you to (e.g. "make a recipe that…", "save this as a recipe called…", "create a recipe for X", "rename/update/delete the X recipe") — never as a side effect of an unrelated request, and never in place of simply explaining how recipes work (use a "respond" action for that). When authoring "prompt": it will be sent later as a brand-new message with NO memory of the current conversation, so write it fully self-contained — never reference "this", "what we just discussed", or anything specific to the current chat. Use the placeholders {{title}} (the note open when the recipe is later run), {{selected text}} (the user's text selection at that time) and {{date}} (that day's date) so the recipe adapts to whatever it's run against, instead of hard-coding today's specifics. Give it a short, descriptive "name" and 0+ short lowercase "tags" for grouping (e.g. ["summary"]); omit "tags" if none apply. For update_recipe/delete_recipe, "recipeId" MUST be an id from the "Recipes" list below — never invent one — and update_recipe should omit any field the user isn't changing.
- If the request targets a note that is not listed below, or you otherwise lack the context to fulfil it, return ONLY a single respond action that explains what the user needs to add to the context. Do not guess or fabricate.
- Output ONLY the JSON object. No explanations and no code fences around it.`
Expand Down
Loading