From dc9f9da5b0e53bb06260fa9d624b0ba077035150 Mon Sep 17 00:00:00 2001
From: Yilin Jing <yilin.jing.ai@outlook.com>
Date: Sun, 27 Sep 2026 06:33:29 -0400
Subject: [PATCH 1/4] Option turns the pointer into a pen that quotes a region
 of a board in the chat

Held over the canvas, Option makes the pointer a pen. A drag across a board
marks a region, and that part of the board goes to the chat as a picture,
captioned with the board, where on it, and the elements there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 canvas/src/ChatPanel.tsx    |  33 +++++-
 canvas/src/canvasAttach.tsx |  37 +++---
 canvas/src/canvasChrome.tsx |  37 ++++--
 canvas/src/index.css        |  16 +++
 canvas/src/magicPen.tsx     | 227 ++++++++++++++++++++++++++++++++++++
 5 files changed, 322 insertions(+), 28 deletions(-)
 create mode 100644 canvas/src/magicPen.tsx

diff --git a/canvas/src/ChatPanel.tsx b/canvas/src/ChatPanel.tsx
index 78238a9e..0b0d82fb 100644
--- a/canvas/src/ChatPanel.tsx
+++ b/canvas/src/ChatPanel.tsx
@@ -100,6 +100,8 @@ const OPEN_KEY = "sp-chat-open";
  * A board comes as `board`: its name and where the server draws it. Drawing takes seconds, so the
  * panel puts its tile and its number up at once and asks for the drawing itself — from here and
  * not from the canvas's frame, which a reload or a change of tab would take the answer away with.
+ * A region the magic pen marked (magicPen.tsx) comes with `crop`, the part of that drawing it is,
+ * as fractions of it: the server draws at the display's pixel ratio, which only the drawing knows.
  *
  * On `window`, because the panel is a sibling of `<Tldraw>` and the button renders inside it.
  * Here rather than beside the button, so the home page, which has the panel and no canvas, has
@@ -108,7 +110,12 @@ const OPEN_KEY = "sp-chat-open";
 export const CANVAS_ATTACH = "sp:canvas-attach";
 
 export type CanvasAttachDetail =
-  | { kind: "board"; name: string; src: string }
+  | {
+      kind: "board";
+      name: string;
+      src: string;
+      crop?: { x: number; y: number; w: number; h: number };
+    }
   | { kind: "image"; file: File }
   | { kind: "error"; message: string }
   | { kind: "draft"; text: string }
@@ -816,7 +823,11 @@ export function ChatPanel(props: {
    * drawing asked of the server, to land in that tile. Asked for again it keeps the tile it has —
    * one already there or on its way is only named again, and one that failed is drawn again.
    */
-  const addBoard = (name: string, src: string) => {
+  const addBoard = (
+    name: string,
+    src: string,
+    crop?: { x: number; y: number; w: number; h: number },
+  ) => {
     let tile = tray.current.find((t) => t.name === name);
     if (!tile && tray.current.length >= MAX_IMAGES)
       return setSendError(
@@ -851,7 +862,21 @@ export function ChatPanel(props: {
     fetch(src)
       .then(async (shot) => {
         if (!shot.ok) throw new Error(await shot.text());
-        const png = await shot.blob();
+        let png = await shot.blob();
+        if (crop) {
+          const whole = await createImageBitmap(png);
+          const [w, h] = [whole.width, whole.height];
+          const part = await createImageBitmap(
+            whole,
+            Math.round(crop.x * w),
+            Math.round(crop.y * h),
+            Math.round(crop.w * w),
+            Math.round(crop.h * h),
+          );
+          const canvas = new OffscreenCanvas(part.width, part.height);
+          canvas.getContext("2d")!.drawImage(part, 0, 0);
+          png = await canvas.convertToBlob({ type: "image/png" });
+        }
         void addImages([new File([png], name, { type: png.type })], true);
       })
       .catch((error) => {
@@ -877,7 +902,7 @@ export function ChatPanel(props: {
       // that is still being read.
       if (detail.kind === "board") {
         adds.current = adds.current.then(() =>
-          addBoard(detail.name, detail.src),
+          addBoard(detail.name, detail.src, detail.crop),
         );
         return;
       }
diff --git a/canvas/src/canvasAttach.tsx b/canvas/src/canvasAttach.tsx
index 641c91de..0f701712 100644
--- a/canvas/src/canvasAttach.tsx
+++ b/canvas/src/canvasAttach.tsx
@@ -16,10 +16,7 @@ import { CANVAS_ATTACH, type CanvasAttachDetail } from "./ChatPanel";
 import { personsShape, personsShapeName } from "./canvasContent";
 import { canvasBoardRef } from "./canvasLibrary";
 import { Plus } from "./geistIcons";
-import {
-  asCanvasTarget,
-  shapeUnderPointer,
-} from "./canvasClicks";
+import { asCanvasTarget, shapeUnderPointer } from "./canvasClicks";
 
 /** A board, a picture the library placed, or anything the person put on a project canvas: what
  *  the **+** answers for. The canvas answers only for the first two (asCanvasTarget). */
@@ -27,9 +24,27 @@ const addable = (editor: Editor, shape: TLShape | undefined): TLShape | null =>
   asCanvasTarget(shape) ?? (personsShape(editor, shape) ? shape! : null);
 
 /** To the agent's panel, which is the window's, outside the canvas's frame (AppShell.tsx). */
-const dispatchAttach = (detail: CanvasAttachDetail) =>
+// oxlint-disable-next-line react/only-export-components
+export const dispatchAttach = (detail: CanvasAttachDetail) =>
   window.parent.dispatchEvent(new CustomEvent(CANVAS_ATTACH, { detail }));
 
+/** A board by its `<slug>/<file>.html`, and where the server draws it. */
+// oxlint-disable-next-line react/only-export-components
+export function boardShot(board: CanvasFileShape) {
+  const { w, h, path } = board.props;
+  const ref = canvasBoardRef(path);
+  if (!ref) throw new Error("that board has no file behind it");
+  const name = `${ref.slug}/${ref.file}`;
+  // At most 4000 a side, as CanvasFileShapeUtil's toSvg draws one.
+  const scale = Math.min(1, 4000 / Math.max(w, h));
+  const src = new URL(
+    `${import.meta.env.BASE_URL}__sp/shoot?path=${encodeURIComponent(name)}` +
+      `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`,
+    window.location.href,
+  ).href;
+  return { name, src };
+}
+
 /**
  * A board or a picture, handed to the chat. A board is a page in an `<iframe>`, so the server
  * draws it first (`/__sp/shoot`, server/sp.ts) and it goes over under its own
@@ -43,17 +58,7 @@ async function attach(editor: Editor, target: TLShape) {
     // A board, whoever placed it, before the check below that an agent-placed one also passes:
     // `toImage` of a board comes back blank, so the server shoots it.
     if (target.type === CANVAS_FILE_SHAPE_TYPE) {
-      const { w, h, path } = (target as CanvasFileShape).props;
-      const ref = canvasBoardRef(path);
-      if (!ref) throw new Error("that board has no file behind it");
-      const name = `${ref.slug}/${ref.file}`;
-      // At most 4000 a side, as CanvasFileShapeUtil's toSvg draws one.
-      const scale = Math.min(1, 4000 / Math.max(w, h));
-      const src = new URL(
-        `${import.meta.env.BASE_URL}__sp/shoot?path=${encodeURIComponent(name)}` +
-          `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`,
-        window.location.href,
-      ).href;
+      const { name, src } = boardShot(target as CanvasFileShape);
       return dispatchAttach({ kind: "board", name, src });
     }
     // One of the person's own: whatever it is, the agent gets a picture of it, named by where it
diff --git a/canvas/src/canvasChrome.tsx b/canvas/src/canvasChrome.tsx
index 57d199a5..b8d6df2d 100644
--- a/canvas/src/canvasChrome.tsx
+++ b/canvas/src/canvasChrome.tsx
@@ -42,6 +42,7 @@ import {
   CanvasSelectionAttachButton,
 } from "./canvasAttach";
 import { CommentUserDialog } from "./CommentUserDialog";
+import { MagicPen } from "./magicPen";
 import { GROUNDS, groundEditable, groundOf, setGround } from "./canvasGround";
 import {
   linkedBoard,
@@ -217,7 +218,13 @@ function setFresh(fresh: ReadonlyMap<TLShapeId, Fresh>) {
 
 /** Rings `ids` until the pointer passes over each, so the reader sees what just arrived or changed. */
 export function markFresh(ids: TLShapeId[], kind: Fresh) {
-  if (ids.length) setFresh(new Map([...freshShapes().get(), ...ids.map((id) => [id, kind] as const)]));
+  if (ids.length)
+    setFresh(
+      new Map([
+        ...freshShapes().get(),
+        ...ids.map((id) => [id, kind] as const),
+      ]),
+    );
 }
 
 export const canvasChromeComponents: TLComponents = {
@@ -240,7 +247,11 @@ export const canvasChromeComponents: TLComponents = {
       useEffect(() => {
         if (!fresh) return;
         const seen = (info: TLEventInfo) => {
-          if (info.name !== "pointer_move" || shapeUnderPointer(editor)?.id !== id) return;
+          if (
+            info.name !== "pointer_move" ||
+            shapeUnderPointer(editor)?.id !== id
+          )
+            return;
           const rest = new Map(freshShapes().get());
           rest.delete(id);
           setFresh(rest);
@@ -252,7 +263,9 @@ export const canvasChromeComponents: TLComponents = {
         <DefaultShapeWrapper
           ref={ref}
           {...props}
-          className={fresh ? `${props.className ?? ""} sp-fresh` : props.className}
+          className={
+            fresh ? `${props.className ?? ""} sp-fresh` : props.className
+          }
           data-fresh={fresh}
         />
       );
@@ -332,9 +345,10 @@ export const canvasChromeComponents: TLComponents = {
     const cover =
       canvasIndex().served && canvasIndex().project
         ? (board ?? (over && canvasImageRef(over.id)))
-      : undefined;
+        : undefined;
     // The canvas's ground, the strip's swatch as presets. Custom opens that swatch's picker.
-    const page = chrome.activeTab.kind === "canvas" ? pageOf(chrome.activeTab) : undefined;
+    const page =
+      chrome.activeTab.kind === "canvas" ? pageOf(chrome.activeTab) : undefined;
     const ground = page ? groundOf(page) : undefined;
 
     return (
@@ -355,10 +369,14 @@ export const canvasChromeComponents: TLComponents = {
           {links.length > 0 && (
             <TldrawUiMenuItem
               id="copy-link"
-              label={links.length > 1 ? `Copy ${links.length} links` : "Copy link"}
+              label={
+                links.length > 1 ? `Copy ${links.length} links` : "Copy link"
+              }
               icon={<Copy />}
               kbd="cmd+c,ctrl+c"
-              onSelect={() => void navigator.clipboard.writeText(links.join("\n"))}
+              onSelect={() =>
+                void navigator.clipboard.writeText(links.join("\n"))
+              }
             />
           )}
           {cover && !isExample(cover.slug) && (
@@ -398,7 +416,9 @@ export const canvasChromeComponents: TLComponents = {
                 checked={!GROUNDS.some(([, color]) => color === ground)}
                 onSelect={() =>
                   document
-                    .querySelector<HTMLInputElement>(".sp-canvas-tabs-ground input")
+                    .querySelector<HTMLInputElement>(
+                      ".sp-canvas-tabs-ground input",
+                    )
                     ?.showPicker()
                 }
               />
@@ -484,6 +504,7 @@ export const canvasChromeComponents: TLComponents = {
             whose boards its server draws as it draws the project's own (server/projects.ts). */}
         {local() && <CanvasAttachButtons />}
         {local() && <CanvasSelectionAttachButton />}
+        {local() && <MagicPen />}
         {/* Out of the tool as well as the bubble. Escape closes only the bubble and leaves the
             next click placing another one, which is not what an accidental comment wants. The
             draft is kept either way, so a real comment interrupted here is there next time. */}
diff --git a/canvas/src/index.css b/canvas/src/index.css
index 6a96a98c..8a91659a 100644
--- a/canvas/src/index.css
+++ b/canvas/src/index.css
@@ -1062,6 +1062,22 @@ a.sp-menu-row {
    over it (canvasAttach.tsx). The layer this is drawn on takes no pointer events, so the buttons
    ask for them back. Inside the corner, not over it: a button that crossed the shape's edge would
    leave the shape as the pointer arrived, and take itself away with the hover. */
+/* The magic pen (magicPen.tsx): over the whole canvas while Option is held, and the region it
+   marks on a board. */
+.sp-pen {
+  position: absolute;
+  inset: 0;
+  z-index: 1;
+  pointer-events: auto;
+}
+
+.sp-pen-region {
+  position: absolute;
+  border: 1.5px dashed var(--ds-focus);
+  border-radius: 4px;
+  background: color-mix(in srgb, var(--ds-focus) 12%, transparent);
+}
+
 .sp-attach {
   position: absolute;
   display: flex;
diff --git a/canvas/src/magicPen.tsx b/canvas/src/magicPen.tsx
new file mode 100644
index 00000000..f2fb862c
--- /dev/null
+++ b/canvas/src/magicPen.tsx
@@ -0,0 +1,227 @@
+import { useEffect, useRef, useState } from "react";
+import { useEditor, usePassThroughWheelEvents, useValue } from "tldraw";
+import {
+  CANVAS_FILE_SHAPE_TYPE,
+  type CanvasFileShape,
+} from "./CanvasFileShapeUtil";
+import { boardShot, dispatchAttach } from "./canvasAttach";
+import { loadCanvasFileHtml } from "./canvasLibrary";
+
+/** A region of a board, in the board's own pixels. */
+interface Region {
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+}
+
+/** A pen with a spark, its point at the lower left, black on white so it reads on any board. */
+const PEN = `url("data:image/svg+xml,${encodeURIComponent(
+  '<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24">' +
+    '<g stroke-linecap="round" stroke-linejoin="round">' +
+    '<path d="M3 21l2.5-6.5L16 4a2.1 2.1 0 013 3L8.5 17.5z" fill="#fff" stroke="#fff" stroke-width="4"/>' +
+    '<path d="M3 21l2.5-6.5L16 4a2.1 2.1 0 013 3L8.5 17.5z" fill="#fff" stroke="#000" stroke-width="1.5"/>' +
+    '<path d="M20 12.5v3M18.5 14h3M13 18.5v2M12 19.5h2" stroke="#fff" stroke-width="4"/>' +
+    '<path d="M20 12.5v3M18.5 14h3M13 18.5v2M12 19.5h2" stroke="#000" stroke-width="1.5"/></g></svg>',
+)}") 3 21, crosshair`;
+
+/**
+ * What is in a region of a board, for the agent to find it by: the outermost elements wholly
+ * inside it, or else the innermost one that holds all of it. Laid out in a frame of this page's
+ * own at the board's size, since the board's frame on the canvas is sandboxed and cannot be read.
+ * Scripts stay off there as they do on the canvas, so the layout is the one the person drew over.
+ */
+async function describeRegion(html: string, w: number, h: number, r: Region) {
+  const frame = document.createElement("iframe");
+  frame.setAttribute("sandbox", "allow-same-origin");
+  frame.style.cssText =
+    `position:fixed;left:${-w - 100}px;top:0;width:${w}px;height:${h}px;` +
+    "border:0;visibility:hidden";
+  const loaded = new Promise((done) => (frame.onload = done));
+  frame.srcdoc = html;
+  document.body.append(frame);
+  try {
+    await loaded;
+    const doc = frame.contentDocument!;
+    const box = (el: Element) => el.getBoundingClientRect();
+    const inside = new Set(
+      [...doc.body.querySelectorAll("*")].filter((el) => {
+        const b = box(el);
+        return (
+          b.width > 0 &&
+          b.height > 0 &&
+          b.left >= r.x - 2 &&
+          b.top >= r.y - 2 &&
+          b.right <= r.x + r.w + 2 &&
+          b.bottom <= r.y + r.h + 2
+        );
+      }),
+    );
+    let picked = [...inside].filter((el) => !inside.has(el.parentElement!));
+    if (!picked.length) {
+      let el = doc.elementFromPoint(r.x + r.w / 2, r.y + r.h / 2);
+      while (el && el !== doc.body) {
+        const b = box(el);
+        if (
+          b.left <= r.x &&
+          b.top <= r.y &&
+          b.right >= r.x + r.w &&
+          b.bottom >= r.y + r.h
+        )
+          break;
+        el = el.parentElement;
+      }
+      picked = el ? [el] : [];
+    }
+    return picked.map((el) => {
+      const tag = el.tagName.toLowerCase();
+      const words =
+        tag === "img"
+          ? (el as HTMLImageElement).alt
+          : (el.textContent ?? "").replace(/\s+/g, " ").trim();
+      return words ? `${tag} "${words.slice(0, 24)}"` : tag;
+    });
+  } finally {
+    frame.remove();
+  }
+}
+
+/**
+ * The magic pen. While Option is held the pointer over the canvas is a pen, and a drag across a
+ * board marks a region of it, which goes to the chat as a quote: that part of the board as a
+ * picture, captioned with the board, where on it, and the elements there.
+ *
+ * A layer over the whole canvas while the key is down, so tldraw never sees the drag, which under
+ * Option would copy the board. The key is heard in both windows, since the focus may be in the
+ * chat, and a window losing focus lets go of it.
+ */
+export function MagicPen() {
+  const editor = useEditor();
+  const [held, setHeld] = useState(false);
+  const [drag, setDrag] = useState<{
+    board: CanvasFileShape;
+    from: { x: number; y: number };
+    to: { x: number; y: number };
+  }>();
+  const layer = useRef<HTMLDivElement>(null);
+  usePassThroughWheelEvents(layer);
+
+  useEffect(() => {
+    const down = (e: KeyboardEvent) => e.key === "Alt" && setHeld(true);
+    const up = (e: KeyboardEvent) => e.key === "Alt" && setHeld(false);
+    const away = () => setHeld(false);
+    const top = window.top!;
+    const windows = top === window ? [window] : [window, top];
+    for (const w of windows) {
+      w.addEventListener("keydown", down, true);
+      w.addEventListener("keyup", up, true);
+      w.addEventListener("blur", away);
+    }
+    return () => {
+      for (const w of windows) {
+        w.removeEventListener("keydown", down, true);
+        w.removeEventListener("keyup", up, true);
+        w.removeEventListener("blur", away);
+      }
+    };
+  }, []);
+
+  // In page points, so the region stays on the board if the camera moves during the drag.
+  const pageAt = (e: React.PointerEvent) =>
+    editor.screenToPage({ x: e.clientX, y: e.clientY });
+
+  const finish = async (d: NonNullable<typeof drag>) => {
+    const b = editor.getShapePageBounds(d.board.id)!;
+    // The board's pixels per page unit, 1 unless it was resized on the canvas.
+    const [sx, sy] = [d.board.props.w / b.w, d.board.props.h / b.h];
+    const r: Region = {
+      x: Math.round((Math.min(d.from.x, d.to.x) - b.minX) * sx),
+      y: Math.round((Math.min(d.from.y, d.to.y) - b.minY) * sy),
+      w: Math.round(Math.abs(d.to.x - d.from.x) * sx),
+      h: Math.round(Math.abs(d.to.y - d.from.y) * sy),
+    };
+    // A click, or a slip of the hand, marks nothing.
+    if (r.w < 4 || r.h < 4) return;
+    try {
+      const { name, src } = boardShot(d.board);
+      // The board, where on it, and what is there: the agent's only word on the picture.
+      const html = await loadCanvasFileHtml(d.board.props.path);
+      const what = html
+        ? await describeRegion(html, d.board.props.w, d.board.props.h, r)
+        : [];
+      const caption =
+        `${name} at ${r.x},${r.y} ${r.w}x${r.h}` +
+        (what.length ? `: ${what.join(", ")}` : "");
+      dispatchAttach({
+        kind: "board",
+        // The server takes a caption of 200 characters at most (agent.ts).
+        name: caption.length > 200 ? `${caption.slice(0, 199)}…` : caption,
+        src,
+        crop: {
+          x: r.x / d.board.props.w,
+          y: r.y / d.board.props.h,
+          w: r.w / d.board.props.w,
+          h: r.h / d.board.props.h,
+        },
+      });
+    } catch (error) {
+      dispatchAttach({ kind: "error", message: String(error) });
+    }
+  };
+
+  const rect = useValue(
+    "pen region",
+    () => {
+      if (!drag) return null;
+      const a = editor.pageToViewport(drag.from);
+      const b = editor.pageToViewport(drag.to);
+      return {
+        left: Math.min(a.x, b.x),
+        top: Math.min(a.y, b.y),
+        width: Math.abs(b.x - a.x),
+        height: Math.abs(b.y - a.y),
+      };
+    },
+    [editor, drag],
+  );
+
+  if (!held && !drag) return null;
+  return (
+    <div
+      ref={layer}
+      className="sp-pen"
+      style={{ cursor: PEN }}
+      onPointerDown={(e) => {
+        const at = pageAt(e);
+        const hit = editor.getShapeAtPoint(at, {
+          hitInside: true,
+          hitLocked: true,
+          renderingOnly: true,
+        });
+        if (hit?.type !== CANVAS_FILE_SHAPE_TYPE) return;
+        e.currentTarget.setPointerCapture(e.pointerId);
+        setDrag({ board: hit as CanvasFileShape, from: at, to: at });
+      }}
+      onPointerMove={(e) => {
+        if (!drag) return;
+        // Held to the board it started on.
+        const b = editor.getShapePageBounds(drag.board.id)!;
+        const p = pageAt(e);
+        setDrag({
+          ...drag,
+          to: {
+            x: Math.min(Math.max(p.x, b.minX), b.maxX),
+            y: Math.min(Math.max(p.y, b.minY), b.maxY),
+          },
+        });
+      }}
+      onPointerUp={() => {
+        if (!drag) return;
+        setDrag(undefined);
+        void finish(drag);
+      }}
+    >
+      {rect && <div className="sp-pen-region" style={rect} />}
+    </div>
+  );
+}

From 1627a85802517cff6171ffde2011fbad1eb18d6c Mon Sep 17 00:00:00 2001
From: Yilin Jing <yilin.jing.ai@outlook.com>
Date: Sun, 27 Sep 2026 06:41:23 -0400
Subject: [PATCH 2/4] A tap of Option picks the pen up for one region, instead
 of holding it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 canvas/src/magicPen.tsx | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/canvas/src/magicPen.tsx b/canvas/src/magicPen.tsx
index f2fb862c..ea3f42ca 100644
--- a/canvas/src/magicPen.tsx
+++ b/canvas/src/magicPen.tsx
@@ -87,17 +87,18 @@ async function describeRegion(html: string, w: number, h: number, r: Region) {
 }
 
 /**
- * The magic pen. While Option is held the pointer over the canvas is a pen, and a drag across a
+ * The magic pen. A tap of Option makes the pointer over the canvas a pen, and a drag across a
  * board marks a region of it, which goes to the chat as a quote: that part of the board as a
- * picture, captioned with the board, where on it, and the elements there.
+ * picture, captioned with the board, where on it, and the elements there. One region, and the
+ * pointer is itself again; Escape or another tap puts the pen down without one.
  *
- * A layer over the whole canvas while the key is down, so tldraw never sees the drag, which under
- * Option would copy the board. The key is heard in both windows, since the focus may be in the
- * chat, and a window losing focus lets go of it.
+ * A tap is Option pressed and let go with nothing between, so Option-drag still copies a board
+ * and Option with a key is still that key. The key is heard in both windows, since the focus may
+ * be in the chat. A layer over the whole canvas while the pen is up, so tldraw never sees the drag.
  */
 export function MagicPen() {
   const editor = useEditor();
-  const [held, setHeld] = useState(false);
+  const [armed, setArmed] = useState(false);
   const [drag, setDrag] = useState<{
     board: CanvasFileShape;
     from: { x: number; y: number };
@@ -107,21 +108,28 @@ export function MagicPen() {
   usePassThroughWheelEvents(layer);
 
   useEffect(() => {
-    const down = (e: KeyboardEvent) => e.key === "Alt" && setHeld(true);
-    const up = (e: KeyboardEvent) => e.key === "Alt" && setHeld(false);
-    const away = () => setHeld(false);
+    let tap = false;
+    const down = (e: KeyboardEvent) => {
+      if (e.key === "Escape") setArmed(false);
+      if (!e.repeat) tap = e.key === "Alt";
+    };
+    const up = (e: KeyboardEvent) => {
+      if (e.key === "Alt" && tap) setArmed((a) => !a);
+      tap = false;
+    };
+    const press = () => (tap = false);
     const top = window.top!;
     const windows = top === window ? [window] : [window, top];
     for (const w of windows) {
       w.addEventListener("keydown", down, true);
       w.addEventListener("keyup", up, true);
-      w.addEventListener("blur", away);
+      w.addEventListener("pointerdown", press, true);
     }
     return () => {
       for (const w of windows) {
         w.removeEventListener("keydown", down, true);
         w.removeEventListener("keyup", up, true);
-        w.removeEventListener("blur", away);
+        w.removeEventListener("pointerdown", press, true);
       }
     };
   }, []);
@@ -185,7 +193,7 @@ export function MagicPen() {
     [editor, drag],
   );
 
-  if (!held && !drag) return null;
+  if (!armed && !drag) return null;
   return (
     <div
       ref={layer}
@@ -198,7 +206,8 @@ export function MagicPen() {
           hitLocked: true,
           renderingOnly: true,
         });
-        if (hit?.type !== CANVAS_FILE_SHAPE_TYPE) return;
+        // Off a board is a click away from the pen.
+        if (hit?.type !== CANVAS_FILE_SHAPE_TYPE) return setArmed(false);
         e.currentTarget.setPointerCapture(e.pointerId);
         setDrag({ board: hit as CanvasFileShape, from: at, to: at });
       }}
@@ -218,6 +227,7 @@ export function MagicPen() {
       onPointerUp={() => {
         if (!drag) return;
         setDrag(undefined);
+        setArmed(false);
         void finish(drag);
       }}
     >

From 7a2731788623ad9bacf09c09dc0f7695a645b76e Mon Sep 17 00:00:00 2001
From: Yilin Jing <yilin.jing.ai@outlook.com>
Date: Sun, 27 Sep 2026 07:46:31 -0400
Subject: [PATCH 3/4] A second double-click on a zoomed board steps back to the
 whole page, not the last view

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 canvas/src/canvasClicks.ts | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/canvas/src/canvasClicks.ts b/canvas/src/canvasClicks.ts
index bb0efc89..4923736a 100644
--- a/canvas/src/canvasClicks.ts
+++ b/canvas/src/canvasClicks.ts
@@ -5,7 +5,6 @@ import type {
   TLPageId,
   TLShape,
   TLShapeId,
-  VecModel,
 } from "tldraw";
 import {
   CANVAS_FILE_SHAPE_TYPE,
@@ -63,13 +62,13 @@ export function zoomToFill(editor: Editor, id: TLShapeId, animate = true) {
  * spend on editing its text zooms it to fill the canvas. tldraw's own double-click would crop a
  * picture or, over a locked shape such as a board, drop a new text box on the canvas, so this takes the gesture over in the select tool's
  * idle state, where tldraw handles it, rather than watching for it alongside. A note, a text or a
- * label still edits its text. A second double-click on the shape it last zoomed to puts the camera
- * back where it was before, on the same page. Returns the uninstaller.
+ * label still edits its text. A second double-click on the shape it last zoomed to steps back to
+ * the whole page, every shape on it in view. Returns the uninstaller.
  */
 export function installDoubleClickZoom(editor: Editor) {
   const idle = editor.getStateDescendant<StateNode>("select.idle")!;
   const own = idle.onDoubleClick!;
-  let before: { id: TLShapeId; page: TLPageId; camera: VecModel } | undefined;
+  let zoomed: { id: TLShapeId; page: TLPageId } | undefined;
   idle.onDoubleClick = (info) => {
     const hit = info.phase === "down" && shapeUnderPointer(editor);
     if (
@@ -80,14 +79,14 @@ export function installDoubleClickZoom(editor: Editor) {
         !editor.canEditShape(hit))
     ) {
       const page = editor.getCurrentPageId();
-      if (before?.id === hit.id && before.page === page) {
-        editor.setCamera(before.camera, {
+      if (zoomed?.id === hit.id && zoomed.page === page) {
+        zoomed = undefined;
+        editor.selectNone();
+        return editor.zoomToFit({
           animation: { duration: editor.options.animationMediumMs },
         });
-        before = undefined;
-        return;
       }
-      before = { id: hit.id, page, camera: editor.getCamera() };
+      zoomed = { id: hit.id, page };
       return zoomToFill(editor, hit.id);
     }
     own.call(idle, info);

From b712489082921eccdfef04faa6bbf50418380441 Mon Sep 17 00:00:00 2001
From: Yilin Jing <yilin.jing.ai@outlook.com>
Date: Sun, 27 Sep 2026 09:02:48 -0400
Subject: [PATCH 4/4] Colour in the canvas's right-click menu, for an arrow, a
 line or a note

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 canvas/src/canvasChrome.tsx | 39 +++++++++++++++++++++++++++++++++++++
 canvas/src/index.css        |  8 ++++++++
 2 files changed, 47 insertions(+)

diff --git a/canvas/src/canvasChrome.tsx b/canvas/src/canvasChrome.tsx
index b8d6df2d..4e9de62a 100644
--- a/canvas/src/canvasChrome.tsx
+++ b/canvas/src/canvasChrome.tsx
@@ -10,7 +10,9 @@ import {
   atom,
   type Atom,
   ConversionsMenuGroup,
+  DefaultColorStyle,
   DefaultContextMenu,
+  getColorValue,
   DefaultShapeWrapper,
   SelectAllMenuItem,
   TldrawUiButton,
@@ -350,6 +352,12 @@ export const canvasChromeComponents: TLComponents = {
     const page =
       chrome.activeTab.kind === "canvas" ? pageOf(chrome.activeTab) : undefined;
     const ground = page ? groundOf(page) : undefined;
+    // The colour of what the right-click picked out that can take one: an arrow, a line, a note
+    // the agent or the person drew. A board or anything else the layout placed is locked.
+    const colourable = editor
+      .getSelectedShapes()
+      .filter((shape) => !shape.isLocked && "color" in shape.props);
+    const colours = editor.getCurrentTheme().colors[editor.getColorMode()];
 
     return (
       <DefaultContextMenu {...props}>
@@ -425,6 +433,37 @@ export const canvasChromeComponents: TLComponents = {
             </TldrawUiMenuSubmenu>
           </TldrawUiMenuGroup>
         )}
+        {colourable.length > 0 && (
+          <TldrawUiMenuGroup id="colour">
+            <TldrawUiMenuSubmenu id="colour" label="Colour">
+              {DefaultColorStyle.values.map((color) => (
+                <TldrawUiMenuItem
+                  key={color}
+                  id={`colour-${color}`}
+                  label={
+                    color[0].toUpperCase() + color.slice(1).replace("-", " ")
+                  }
+                  iconLeft={
+                    <span
+                      className="sp-menu-swatch"
+                      style={{
+                        background: getColorValue(colours, color, "solid"),
+                      }}
+                    />
+                  }
+                  isSelected={colourable.every(
+                    (shape) => (shape.props as { color: string }).color === color,
+                  )}
+                  // tldraw leaves a locked shape as it is, so a board in the selection too is
+                  // no matter.
+                  onSelect={() => {
+                    editor.setStyleForSelectedShapes(DefaultColorStyle, color);
+                  }}
+                />
+              ))}
+            </TldrawUiMenuSubmenu>
+          </TldrawUiMenuGroup>
+        )}
         {/* tldraw's items one at a time, not its groups: this canvas is read, and every shape on
             it is locked and rebuilt from layout.json, so only what works on a locked shape is
             here. A group would bring Cut, Delete and Duplicate, greyed out on every shape here,
diff --git a/canvas/src/index.css b/canvas/src/index.css
index 8a91659a..c8cbc03d 100644
--- a/canvas/src/index.css
+++ b/canvas/src/index.css
@@ -2128,3 +2128,11 @@ a.sp-menu-row {
 .sp-chat-made button[data-made="updated"] {
   --ds-focus: light-dark(#1a9338, #3dbe5c);
 }
+
+/* A colour's dot in the canvas menu's Colour row (canvasChrome.tsx). */
+.sp-menu-swatch {
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  box-shadow: inset 0 0 0 1px color-mix(in srgb, currentColor 25%, transparent);
+}