diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 699556bd..7911fc17 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -19,9 +19,10 @@ The app, its skills and the toolkit carry one version. Everything below is on `main` and reaches no install until a version is cut. -- **A mockup goes to the agent as its file.** Its **+** still shows a picture - of it in the chat, but the agent is handed the board's `.html` path rather - than that picture, since the file is what it reads and changes. +- **Only pictures go to the agent as pictures.** The **+** on a board, a + video or a note still shows a picture of it in the chat, but the agent is + handed the file behind it rather than that picture, since the file is what + it can read and change. ## v1.7.0 diff --git a/canvas/server/agent.ts b/canvas/server/agent.ts index f6789142..b643562b 100644 --- a/canvas/server/agent.ts +++ b/canvas/server/agent.ts @@ -17,7 +17,7 @@ import path from "node:path"; import { CANVASES } from "./boards.ts"; import { command, stop } from "./command.ts"; import { AGENT_SKILLS, installSkills } from "./skills.ts"; -import { folderOf, sameOrigin } from "./sp.ts"; +import { canvasFile, folderOf, sameOrigin } from "./sp.ts"; import { SAFE_NAME } from "../src/layoutEdit.ts"; import { attach, @@ -311,13 +311,15 @@ export function createAgentServer(options: { ) return send(400, "bad image data"); if ( - i.page !== undefined && - (typeof i.page !== "object" || + i.reference !== undefined && + (typeof i.reference !== "object" || !["project", "community"].every( - (k) => i.page[k] === undefined || typeof i.page[k] === "string", + (k) => + i.reference[k] === undefined || + typeof i.reference[k] === "string", )) ) - return send(400, "bad image page"); + return send(400, "bad image reference"); } // The body cap above is the panel's limit in base64; a client that is not the // panel meets the limit itself here, in the bytes the files come out as. @@ -507,20 +509,23 @@ export function createAgentServer(options: { name: string; type: string; data: string; - page?: { project?: string; community?: string }; + reference?: { project?: string; community?: string }; }) => { - // A mockup's picture is kept for the panel, and the agent is pointed at its - // file, `/.html` in the canvases of the project it was attached - // from, which need not be the one it is sent from. A community project's is not - // on this machine, so it keeps the name `sp fetch` finds it by. - const [slug, file, ...rest] = i.name.split("/"); - const from = i.page?.project && projects().get(i.page.project); - const local = - i.page?.community === undefined && - (i.page?.project === undefined || from) && - rest.length === 0 && - SAFE_NAME.test(slug) && - SAFE_NAME.test(file ?? ""); + // Anything but a picture (a board, a video, a note) keeps its picture for the + // panel, and the agent is pointed at its file, in the canvases of the project it + // was attached from, which need not be the one it is sent from. A community + // project's is not on this machine, so it keeps the name `sp fetch` finds it by. + const ref = i.reference; + const from = ref?.project && projects().get(ref.project); + const file = + ref?.community === undefined && + (ref?.project === undefined || from) + ? canvasFile( + from ? path.join(from, CANVASES) : examplesDir, + examplesDir, + i.name, + ) + : undefined; return { ...i, path: picture( @@ -529,20 +534,12 @@ export function createAgentServer(options: { i.type, Buffer.from(i.data, "base64"), ), - page: - i.page && - (local - ? path.join( - folderOf( - from ? path.join(from, CANVASES) : examplesDir, - examplesDir, - slug, - ), - file, - ) - : i.page.community - ? `${i.name} of the community project ${i.page.community}` - : i.name), + reference: + ref && + (file ?? + (ref.community + ? `${i.name} of the community project ${ref.community}` + : i.name)), }; }, ); diff --git a/canvas/server/sp.test.ts b/canvas/server/sp.test.ts index 25760032..0c76aa6d 100644 --- a/canvas/server/sp.test.ts +++ b/canvas/server/sp.test.ts @@ -5,7 +5,7 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { expect, it, vi } from "vitest"; -import { createSpServer } from "./sp.ts"; +import { canvasFile, createSpServer } from "./sp.ts"; // The examples directory is listed beside the project's canvases, shadowed by a folder of the // project's own, refused every write, and cloned into the project. @@ -522,3 +522,34 @@ async function serve(options: Parameters[0]) { }; return { ask, listen, close }; } + +// What a chat attachment names is a file in a canvas folder, and nothing a browser sends reaches +// outside one. +it("resolves an attachment's name to a file in its canvas folder", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "sp-attach-")); + const own = path.join(tmp, "project"); + const examples = path.join(tmp, "examples"); + fs.mkdirSync(path.join(examples, "demo"), { recursive: true }); + expect(canvasFile(own, examples, "shop/01-home.html")).toBe( + path.join(own, "shop", "01-home.html"), + ); + expect(canvasFile(own, examples, "shop/files/clip.mp4")).toBe( + path.join(own, "shop", "files", "clip.mp4"), + ); + expect(canvasFile(own, examples, "shop/canvas.json#shape:a1")).toBe( + path.join(own, "shop", "canvas.json#shape:a1"), + ); + expect(canvasFile(own, examples, "demo/01-a.html")).toBe( + path.join(examples, "demo", "01-a.html"), + ); + for (const name of [ + "shop", + "../etc/passwd", + "shop/../../etc", + "shop/files/..", + "shop//x", + "shop/.env", + "shop/a\\..\\..\\x", + ]) + expect(canvasFile(own, examples, name)).toBeUndefined(); +}); diff --git a/canvas/server/sp.ts b/canvas/server/sp.ts index a385a9fc..b1bc73af 100644 --- a/canvas/server/sp.ts +++ b/canvas/server/sp.ts @@ -80,6 +80,20 @@ export function folderOf( return !isCanvas && fs.existsSync(example) ? example : own; } +/** + * The file a chat attachment names as `/`: a board's HTML, a file under `files/`, or + * `canvas.json#`. Undefined for a name that is not one, so nothing a browser sends can + * point outside a canvas folder: no part may be empty, start with a dot, or hold a backslash. + */ +export function canvasFile(canvasesDir: string, examplesDir: string, name: string) { + const [slug, ...rest] = name.split("/"); + return SAFE_NAME.test(slug) && + rest.length > 0 && + rest.every((s) => s && !s.startsWith(".") && !s.includes("\\")) + ? path.join(folderOf(canvasesDir, examplesDir, slug), ...rest) + : undefined; +} + /** * The project's own settings, beside its canvases: which cover it chose, and the name it is shown * by when that is not its folder's. A project made without a name is an "Untitled" folder whose diff --git a/canvas/src/ChatPanel.tsx b/canvas/src/ChatPanel.tsx index 0c6456a5..b21e53a9 100644 --- a/canvas/src/ChatPanel.tsx +++ b/canvas/src/ChatPanel.tsx @@ -87,9 +87,9 @@ const OPEN_KEY = "sp-chat-open"; /** * What the canvas hands the chat panel when the button is pressed (canvasAttach.tsx): a picture - * to attach to the message, or the reason none was. A board comes over as a picture too, for its - * tile only: the agent is handed the board's file, which its name says, and the panel shows that - * name under the tile. And the start + * to attach to the message, or the reason none was. A board, a video or a note comes over as a + * picture too, for its tile only: the agent is handed the file its name says, and the panel shows + * that name under the tile. And the start * of a message, from the strip's "+" (CanvasStrip.tsx), because a canvas is only ever the * agent's work, and a folder with no boards in it is not one. And a whole message, sent as it is, * from the new-project dialog (AppShell.tsx), which starts the agent defining the product. @@ -105,8 +105,9 @@ const OPEN_KEY = "sp-chat-open"; export const CANVAS_ATTACH = "sp:canvas-attach"; export type CanvasAttachDetail = - /** `page` for a mockup, whose drawing is only the tile's: the agent is handed its file. */ - | { kind: "board"; name: string; src: string; page?: true } + /** `reference` for anything but a picture, whose drawing is only the tile's: the agent is + * pointed at its file instead. */ + | { kind: "board"; name: string; src: string; reference?: true } | { kind: "image"; file: File } | { kind: "error"; message: string } | { kind: "draft"; text: string } @@ -251,9 +252,9 @@ interface Attached { url: string; /** A board still being drawn, with no `url` yet, or one whose drawing failed. */ state?: "pending" | "failed"; - /** A mockup, and the project it was attached from, which the panel may since have left: the - * agent gets its file there rather than this picture of it (agents.ts). */ - page?: { project?: string; community?: string }; + /** Anything but a picture, and the project it was attached from, which the panel may since + * have left: the agent gets its file there rather than this picture of it (agents.ts). */ + reference?: { project?: string; community?: string }; } /** @@ -718,7 +719,7 @@ export function ChatPanel(props: { type: r.file.type, size: r.file.size, url: r.url, - page: t.page, + reference: t.reference, } : t; }); @@ -764,7 +765,7 @@ export function ChatPanel(props: { * drawing asked of the server, to land in that tile. Asked for again it keeps the tile it has — * one already there or on its way is only named again, and one that failed is drawn again. */ - const addBoard = (name: string, src: string, page?: true) => { + const addBoard = (name: string, src: string, reference?: true) => { let tile = tray.current.find((t) => t.name === name); if (!tile && tray.current.length >= MAX_IMAGES) return setSendError( @@ -785,7 +786,7 @@ export function ChatPanel(props: { size: 0, url: "", state: "pending", - page: page && { project, community }, + reference: reference && { project, community }, }; const next = tile; tray.current = [...tray.current.filter((t) => t.n !== next.n), next].sort( @@ -827,7 +828,7 @@ export function ChatPanel(props: { // that is still being read. if (detail.kind === "board") { adds.current = adds.current.then(() => - addBoard(detail.name, detail.src, detail.page), + addBoard(detail.name, detail.src, detail.reference), ); return; } @@ -945,12 +946,12 @@ export function ChatPanel(props: { agent, model, effort, - images: attached.map(({ n, name, type, url, page }) => ({ + images: attached.map(({ n, name, type, url, reference }) => ({ n, name, type, data: url.slice(url.indexOf(",") + 1), - page, + reference, })), }), }); diff --git a/canvas/src/agents.test.ts b/canvas/src/agents.test.ts index f2d4ac29..0be0be3c 100644 --- a/canvas/src/agents.test.ts +++ b/canvas/src/agents.test.ts @@ -94,7 +94,7 @@ describe("AGENTS", () => { type: "image/png", data: "CCC", path: "/tmp/sp-chat-r/2.png", - page: "/proj/canvases/shop/01-home.html", + reference: "/proj/canvases/shop/01-home.html", }; expect( JSON.parse(def("claude").stdin("tighten #2", "P", [board])).message diff --git a/canvas/src/agents.ts b/canvas/src/agents.ts index 0ad085a2..906b28ff 100644 --- a/canvas/src/agents.ts +++ b/canvas/src/agents.ts @@ -80,9 +80,10 @@ export interface AgentImage { /** Where the server wrote it, for an agent that takes files rather than bytes; gone once * that agent has exited. */ path: string; - /** A mockup's HTML file, when the picture is only the panel's drawing of one: the agent is - * pointed at the file and not handed the picture, since the file is what it reads and edits. */ - page?: string; + /** The file behind it (a board's HTML, a video, a canvas.json record) when the picture is + * only the panel's drawing of it: the agent is pointed at the file, which is what it can read + * and change, and not handed the picture. */ + reference?: string; } /** What the composer chose, handed to `args`. An empty string means the CLI decides. */ @@ -203,8 +204,8 @@ export const AGENTS: AgentDef[] = [ // terminal, so the number arrives as something already read rather than a local convention. stdin: (message, _preamble, images) => { const blocks = images.flatMap((i) => - i.page - ? [{ type: "text", text: `[Image #${i.n}] ${i.page}` }] + i.reference + ? [{ type: "text", text: `[Image #${i.n}] ${i.reference}` }] : [ { type: "text", text: `[Image #${i.n}] ${i.name}` }, { @@ -315,7 +316,7 @@ export const AGENTS: AgentDef[] = [ stdin: (message, preamble, images) => [ preamble, - images.map((i) => `[Image #${i.n}] ${i.page ?? i.path}`).join("\n"), + images.map((i) => `[Image #${i.n}] ${i.reference ?? i.path}`).join("\n"), message, ] .filter(Boolean) diff --git a/canvas/src/canvasAttach.tsx b/canvas/src/canvasAttach.tsx index 505eea9d..dca53122 100644 --- a/canvas/src/canvasAttach.tsx +++ b/canvas/src/canvasAttach.tsx @@ -13,7 +13,7 @@ import { type CanvasFileShape, } from "./CanvasFileShapeUtil"; import { CANVAS_ATTACH, type CanvasAttachDetail } from "./ChatPanel"; -import { personsShape, personsShapeName } from "./canvasContent"; +import { personsShape, personsShapeName, saveNow } from "./canvasContent"; import { canvasBoardRef } from "./canvasLibrary"; import { Plus } from "./geistIcons"; import { @@ -54,17 +54,23 @@ async function attach(editor: Editor, target: TLShape) { `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`, window.location.href, ).href; - return dispatchAttach({ kind: "board", name, src, page: true }); + return dispatchAttach({ kind: "board", name, src, reference: true }); } - // One of the person's own: whatever it is, the agent gets a picture of it, named by where it - // reads the thing itself (canvasContent.ts). + // One of the person's own, drawn for the tile and named by where the agent reads the thing + // itself (canvasContent.ts). A picture goes over as one. Anything else, such as a video, a + // note or a drawing, is a file the agent is pointed at, as a board is, so its canvas is + // saved first: a shape drawn a moment ago is otherwise still waiting out the pause before + // canvas.json is written. const slug = personsShape(editor, target); if (slug) { + const reference = target.type === "image" ? undefined : true; + if (reference) await saveNow(slug); const { blob } = await editor.toImage([target.id], { format: "png" }); return dispatchAttach({ kind: "board", name: personsShapeName(editor, target, slug), src: URL.createObjectURL(blob), + reference, }); } const shape = target as TLImageShape;