feat(browser): leave notes on page elements for the agent

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E (Cmd+Shift+E on macOS) turns the crosshair on and off, and Esc leaves it. A send the session refuses keeps the notes and says why in the tray.
+
 ## [0.80.0] - 2026-10-27
 
 > [!IMPORTANT]

api/browser.go

@@ -35,6 +35,11 @@ var (
 	// copies to the system clipboard: the headless browser's own clipboard
 	// is not the person's.
 	BrowserSelection = define[SessionIDOptions, string]("browser.Selection")
+	// BrowserElementAt is the element under a point of the page, for a note
+	// the person leaves on it; BrowserLocate is where noted elements stand
+	// now, after the page scrolled or changed.
+	BrowserElementAt = define[BrowserPointOptions, BrowserElement]("browser.ElementAt")
+	BrowserLocate    = define[BrowserLocateOptions, []BrowserRect]("browser.Locate")
 )
 
 var (
@@ -223,3 +228,33 @@ type BrowserPageInfo struct {
 	Interactive []BrowserInteractive `json:"interactive"`
 	Truncated   bool                 `json:"truncated"`
 }
+
+// BrowserPointOptions is a point of session ID's page, in CSS pixels.
+type BrowserPointOptions struct {
+	ID string  `json:"id"`
+	X  float64 `json:"x"`
+	Y  float64 `json:"y"`
+}
+
+// BrowserElement is the element under a point, as a note on it names it.
+// Label is its tag and text (button "Buy now"), empty when the point holds
+// none; Selector finds it again, in the form browser.Click takes. Component
+// and Source are the React component that drew it and where, when the page
+// is a React development build. HTML is its markup, shortened, with no
+// field's value. URL is the page's, without query or fragment.
+type BrowserElement struct {
+	Label     string      `json:"label"`
+	Selector  string      `json:"selector"`
+	Component string      `json:"component,omitempty"`
+	Source    string      `json:"source,omitempty"`
+	HTML      string      `json:"html"`
+	URL       string      `json:"url"`
+	Box       BrowserRect `json:"box"`
+}
+
+// BrowserLocateOptions asks where the elements Selectors find stand in
+// session ID's page; one gone from the page answers an empty box.
+type BrowserLocateOptions struct {
+	ID        string   `json:"id"`
+	Selectors []string `json:"selectors"`
+}

api/testdata/contract.golden

@@ -8,10 +8,12 @@ browser.Back(SessionIDOptions) None
 browser.Click(BrowserClickOptions) BrowserPageInfo
 browser.Close(SessionIDOptions) None
 browser.Detach(SessionIDOptions) None
+browser.ElementAt(BrowserPointOptions) BrowserElement
 browser.Forward(SessionIDOptions) None
 browser.Info(SessionIDOptions) BrowserPageInfo
 browser.Key(BrowserKeyOptions) None
 browser.List(SessionIDOptions) []BrowserHandle
+browser.Locate(BrowserLocateOptions) []BrowserRect
 browser.Mouse(BrowserMouseOptions) None
 browser.Nav(BrowserNavOptions) None
 browser.Navigate(BrowserOpenOptions) BrowserHandle
@@ -339,6 +341,22 @@ BrowserPageInfo {
 	truncated bool
 }
 
+BrowserPointOptions {
+	id string
+	x float64
+	y float64
+}
+
+BrowserElement {
+	label string
+	selector string
+	component,omitempty string
+	source,omitempty string
+	html string
+	url string
+	box BrowserRect
+}
+
 BrowserKeyOptions {
 	id string
 	type string
@@ -348,6 +366,18 @@ BrowserKeyOptions {
 	modifiers,omitempty int
 }
 
+BrowserLocateOptions {
+	id string
+	selectors []string
+}
+
+BrowserRect {
+	x int
+	y int
+	w int
+	h int
+}
+
 BrowserMouseOptions {
 	id string
 	type string
@@ -1897,13 +1927,6 @@ SessionCache {
 	uncached int
 }
 
-BrowserRect {
-	x int
-	y int
-	w int
-	h int
-}
-
 PRThreadComment {
 	id int64
 	author string

docs/browser-tab.md

@@ -46,6 +46,19 @@ Page info never includes form values. Screenshots are files. Click, type and
 press are real input events. `press` is named keys only. Arbitrary JavaScript
 in the page is not a tool.
 
+## Notes on the page
+
+The crosshair on the pane's toolbar stops the pointer reaching the page: the
+element under it is outlined, and a click opens a note on it. Notes wait in a
+tray floating over the bottom of the page, which never resizes for it, each
+element pinned where it stands, and Send types them into the session's prompt
+as one message: each element's label, page, a
+selector `browser_click` takes, its React component when the page is a
+development build, its markup without any field's value, and the note. The
+page only answers what is where (`browser.ElementAt`, `browser.Locate`); the
+outline, the pins and the note are the window's, so nothing is injected into
+the page.
+
 ## Providers
 
 Claude Code, Codex and opencode receive the tools at spawn. Antigravity,

docs/ceilings.md

@@ -970,15 +970,26 @@ work when nobody knows it and that the call site never shows. The mechanism and
   `docs/browser-tab.md`): a headless Chrome, Edge, Chromium, Brave, Vivaldi or Helium already installed,
   whose page reaches the session's browser tab as a JPEG screencast over `/events`, at the pace the window
   takes frames. The native window cannot hold a webview: on Linux it speaks Wayland without GTK. So the pane
-  has no DevTools, shows the arrow whatever cursor the page asks for, does not copy the page's selection to
-  the system clipboard (paste works), loads what the page opens in a new tab or window in the page itself, and leaves downloads and the page's own dialogs out; Open in
+  has no DevTools, shows the arrow whatever cursor the page asks for, loads what the page opens in a new tab or window in the page itself, and leaves downloads and the page's own dialogs out; Open in
   a window moves the page to a real Chromium window for those, reopening it on the same profile with every
   cookie, since chromedp cannot show a headless process in place. A half-filled form or sessionStorage is
   lost on that move. Closing the card, or the session's process ending, closes the browser; closing that
   window does not close the session. A headless browser no pane shows and no one used for five minutes is
   closed; one a pane shows, or a window, never is. It loads only http, https and about:blank. Page info never
   includes a form's value. Screenshots are files. There is no tool that runs JavaScript in the page. It is
-  not a sandbox: the page runs as the user, in an ephemeral profile.
+  not a sandbox: the page runs as the user, in an ephemeral profile. Chromium needs its own sandbox, and on
+  Ubuntu 23.10 and later AppArmor refuses it the user namespace unless the browser ships an AppArmor profile,
+  as the distribution's Chrome, Edge and Brave do: one unpacked from a tarball fails to start with "No usable
+  sandbox!", which the agent reads as the browser's error.
+- **A note left on the page names only what the page shows lich** (`internal/browser/notes.go`,
+  `native/cmd/lich-native/browser_notes.go`): the pick asks the page for the element under the pointer, so an
+  element inside an iframe is picked as the iframe and one inside a closed shadow root as its host, and a
+  selector picked inside an open shadow root finds it again only from that root, where `browser_click` does
+  not look. The React component comes from the page's fiber, there only in a development build, and its file
+  and line from `_debugSource`, which React 19 dropped. Pins follow their elements by looking them up again
+  every 400 ms while notes wait, and every 30 ms for a moment after the wheel turns, so an element the page
+  moves on its own is outlined late. Notes not sent yet
+  live in the window: closing it, or the browser, loses them.
 
 ## Languages
 

internal/browser/fake_test.go

@@ -41,6 +41,10 @@ type fakePage struct {
 	texts     []string
 	settles   int
 	selection string
+	element   api.BrowserElement
+	pointedAt [2]float64
+	boxes     map[string]api.BrowserRect
+	located   [][]string
 	info      PageInfo
 	shotErr   error
 	navErr    error
@@ -307,3 +311,21 @@ func (f *fakeLauncher) launch(dir string, headed bool) (Page, error) {
 	f.dirs = append(f.dirs, dir)
 	return p, nil
 }
+
+func (p *fakePage) ElementAt(_ context.Context, x, y float64) (api.BrowserElement, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.pointedAt = [2]float64{x, y}
+	return p.element, nil
+}
+
+func (p *fakePage) Locate(_ context.Context, selectors []string) ([]api.BrowserRect, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.located = append(p.located, selectors)
+	out := make([]api.BrowserRect, len(selectors))
+	for i, s := range selectors {
+		out[i] = p.boxes[s]
+	}
+	return out, nil
+}

internal/browser/notes.go

@@ -0,0 +1,133 @@
+package browser
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/chromedp/chromedp"
+	"github.com/lichdotdev/lich/api"
+)
+
+// The pane's notes: the person points at an element of the page and leaves a
+// note on it for the agent. The page only reports what is where; the pane
+// draws the outline and the pins, so nothing is injected into its DOM.
+
+// elementScript reads the element under a point: a selector that finds it
+// again (an id, or up to two classes that are not a CSS-in-JS hash, with
+// :nth-of-type only where a level is not yet unique), a label, the React
+// component that drew it when the page is a development build (its file cut
+// to the project's src/, or else app/, pages/, components/ or lib/), and its
+// markup with every field's value taken out. A component name of one or two
+// letters is a production build's minifier, which tells the agent nothing. A
+// point on the page's root answers null.
+const elementScript = `((x, y) => {
+	const cut = (s, n) => s.length > n ? s.slice(0, n - 1) + '…' : s;
+	let el = document.elementFromPoint(x, y);
+	while (el && el.shadowRoot) {
+		const inner = el.shadowRoot.elementFromPoint(x, y);
+		if (!inner || inner === el) break;
+		el = inner;
+	}
+	if (!el || el === document.documentElement || el === document.body) return null;
+	const word = /^[a-zA-Z][\w-]*$/;
+	const stable = c => word.test(c) && !/^(css|sc|jsx|emotion|svelte)-|[0-9a-f]{5,}|__/.test(c);
+	const part = e => {
+		const tag = e.tagName.toLowerCase();
+		if (e.id && word.test(e.id)) return tag + '#' + e.id;
+		return tag + [...e.classList].filter(stable).slice(0, 2).map(c => '.' + c).join('');
+	};
+	const root = el.getRootNode();
+	const unique = s => { try { return root.querySelectorAll(s).length === 1; } catch { return false; } };
+	let selector = '';
+	for (let e = el, depth = 0; e && e.nodeType === 1 && depth < 6; e = e.parentElement, depth++) {
+		const p = part(e);
+		const join = q => selector ? q + ' > ' + selector : q;
+		if (unique(join(p))) { selector = join(p); break; }
+		const same = e.parentElement ? [...e.parentElement.children].filter(c => c.tagName === e.tagName) : [];
+		const q = same.length > 1 && !p.includes('#') ? p + ':nth-of-type(' + (same.indexOf(e) + 1) + ')' : p;
+		selector = join(q);
+		if (unique(selector)) break;
+	}
+	const text = (el.getAttribute('aria-label') || el.innerText || el.getAttribute('placeholder') ||
+		el.getAttribute('alt') || el.getAttribute('title') || '').trim().split('\n')[0];
+	const label = part(el) + (text ? ' "' + cut(text, 32) + '"' : '');
+	let component = '', source = '';
+	const fiber = Object.keys(el).find(k => k.startsWith('__reactFiber$'));
+	for (let f = fiber ? el[fiber] : null; f; f = f.return) {
+		if (!source && f._debugSource) {
+			const file = f._debugSource.fileName;
+			const root = ['/src/', '/app/', '/pages/', '/components/', '/lib/'].map(r => file.indexOf(r)).find(i => i >= 0);
+			source = (root === undefined ? file : file.slice(root + 1)) + ':' + f._debugSource.lineNumber;
+		}
+		const t = f.type;
+		if (!t || typeof t === 'string') continue;
+		const inner = t.render || t.type || t;
+		const name = t.displayName || inner.displayName || inner.name || '';
+		if (name.length > 2 && !/Provider|Context|Router|Boundary|^Anonymous$/.test(name)) { component = name; break; }
+	}
+	const copy = el.cloneNode(true);
+	for (const n of [copy, ...copy.querySelectorAll('*')]) {
+		if (n.matches('input, textarea, select, option')) { n.removeAttribute('value'); n.textContent = ''; }
+		if (n !== copy && n.matches('script, style')) n.remove();
+	}
+	const r = el.getBoundingClientRect();
+	return {
+		label, selector, component, source,
+		html: cut(copy.outerHTML.replace(/\s+/g, ' '), 600),
+		url: location.href.split(/[?#]/)[0],
+		box: {x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height)},
+	};
+})`
+
+// locateScript is the viewport box of the element each selector finds, an
+// empty one where it finds none.
+const locateScript = `(selectors => selectors.map(s => {
+	let e = null;
+	try { e = document.querySelector(s); } catch {}
+	if (!e) return {x: 0, y: 0, w: 0, h: 0};
+	const r = e.getBoundingClientRect();
+	return {x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height)};
+}))`
+
+func (p *chromePage) ElementAt(ctx context.Context, x, y float64) (api.BrowserElement, error) {
+	var el *api.BrowserElement
+	if err := p.run(ctx, chromedp.Evaluate(fmt.Sprintf("%s(%g, %g)", elementScript, x, y), &el)); err != nil {
+		return api.BrowserElement{}, err
+	}
+	if el == nil {
+		return api.BrowserElement{}, nil
+	}
+	return *el, nil
+}
+
+func (p *chromePage) Locate(ctx context.Context, selectors []string) ([]api.BrowserRect, error) {
+	arg, err := json.Marshal(selectors)
+	if err != nil {
+		return nil, err
+	}
+	boxes := []api.BrowserRect{}
+	return boxes, p.run(ctx, chromedp.Evaluate(fmt.Sprintf("%s(%s)", locateScript, arg), &boxes))
+}
+
+// ElementAtPoint is the element under a point of this session's page.
+func (s *Service) ElementAtPoint(_ context.Context, opts api.BrowserPointOptions) (api.BrowserElement, error) {
+	var el api.BrowserElement
+	_, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error {
+		var err error
+		el, err = p.ElementAt(ctx, opts.X, opts.Y)
+		return err
+	})
+	return el, err
+}
+
+// LocateElements is where the elements opts.Selectors find stand now.
+func (s *Service) LocateElements(_ context.Context, opts api.BrowserLocateOptions) ([]api.BrowserRect, error) {
+	var boxes []api.BrowserRect
+	_, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error {
+		var err error
+		boxes, err = p.Locate(ctx, opts.Selectors)
+		return err
+	})
+	return boxes, err
+}