feat(browser): notes follow their element: reveal on open, clip to scrolling boxes, hover tween

CHANGELOG.md

@@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E (Cmd+Shift+E on macOS) turns the crosshair on and off, and Esc leaves it. A send the session refuses keeps the notes and says why in the tray.
+- The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, the outline sliding from one element to the next as the pointer moves, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, and a click on it or its pin scrolls the element into view and edits the note. An element a scrolling list, sidebar or dialog cuts is outlined only where it shows, and one it hides keeps no pin over what stands beside it. Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E (Cmd+Shift+E on macOS) turns the crosshair on and off, and Esc leaves it. A send the session refuses keeps the notes and says why in the tray.
 - A session's browser downloads files. A download the page starts, from a link, a button, a new tab or a script, lands in your Downloads folder under the name the page gives it, with ` (1)` added rather than overwriting a file already there. The agent can also open a file's URL directly. The agent's click, key press or open answers with the file's path and size, and the window shows a toast naming the file, with a button that opens its folder. Before, a download button in the session's browser did nothing.
 
 ### Fixed

api/browser.go

@@ -281,8 +281,11 @@ type BrowserElement struct {
 }
 
 // BrowserLocateOptions asks where the elements Selectors find stand in
-// session ID's page; one gone from the page answers an empty box.
+// session ID's page; one gone from the page, or hidden by what it scrolls
+// in, answers an empty box. Reveal, when set, is a selector whose element
+// the page first scrolls into view unless it already shows whole.
 type BrowserLocateOptions struct {
 	ID        string   `json:"id"`
 	Selectors []string `json:"selectors"`
+	Reveal    string   `json:"reveal,omitempty"`
 }

api/testdata/contract.golden

@@ -372,6 +372,7 @@ BrowserKeyOptions {
 BrowserLocateOptions {
 	id string
 	selectors []string
+	reveal,omitempty string
 }
 
 BrowserRect {

docs/browser-tab.md

@@ -49,12 +49,16 @@ in the page is not a tool.
 ## Notes on the page
 
 The crosshair on the pane's toolbar stops the pointer reaching the page: the
-element under it is outlined, and a click opens a note on it. Notes wait in a
+element under it is outlined, the outline sliding to the next element in 160
+ms as the pointer moves, and a click opens a note on it. Notes wait in a
 tray floating over the bottom of the page, which never resizes for it, each
 element pinned where it stands, and Send types them into the session's prompt
 as one message: each element's label, page, a
 selector `browser_click` takes, its React component when the page is a
-development build, its markup without any field's value, and the note. The
+development build, its markup without any field's value, and the note. A note
+opened from the tray or its pin scrolls its element into view first, unless it
+already shows whole. An element inside a list, sidebar or dialog that scrolls
+is outlined only where that box shows it, and one the box hides has no pin. The
 page only answers what is where (`browser.ElementAt`, `browser.Locate`); the
 outline, the pins and the note are the window's, so nothing is injected into
 the page.

docs/ceilings.md

@@ -988,7 +988,9 @@ work when nobody knows it and that the call site never shows. The mechanism and
   not look. The React component comes from the page's fiber, there only in a development build, and its file
   and line from `_debugSource`, which React 19 dropped. Pins follow their elements by looking them up again
   every 400 ms while notes wait, and every 30 ms for a moment after the wheel turns, so an element the page
-  moves on its own is outlined late. Notes not sent yet
+  moves on its own is outlined late. A pin is cut to every ancestor whose overflow is not visible, up to a
+  fixed one, so an absolutely positioned element that escapes such an ancestor is cut anyway, and one a
+  sibling covers keeps its pin. Notes not sent yet
   live in the window: closing it, or the browser, loses them.
 
 - **A session browser's downloads go straight to the user's Downloads folder** (`internal/browser/downloads.go`,

internal/browser/fake_test.go

@@ -335,7 +335,7 @@ func (p *fakePage) ElementAt(_ context.Context, x, y float64) (api.BrowserElemen
 	return p.element, nil
 }
 
-func (p *fakePage) Locate(_ context.Context, selectors []string) ([]api.BrowserRect, error) {
+func (p *fakePage) Locate(_ context.Context, selectors []string, _ string) ([]api.BrowserRect, error) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	p.located = append(p.located, selectors)

internal/browser/notes.go

@@ -22,6 +22,7 @@ import (
 // letters is a production build's minifier, which tells the agent nothing. A
 // point on the page's root answers null.
 const elementScript = `((x, y) => {
+	const visible = ` + visibleScript + `;
 	const cut = (s, n) => s.length > n ? s.slice(0, n - 1) + '…' : s;
 	let el = document.elementFromPoint(x, y);
 	while (el && el.shadowRoot) {
@@ -71,24 +72,56 @@ const elementScript = `((x, y) => {
 		if (n.matches('input, textarea, select, option')) { n.removeAttribute('value'); n.textContent = ''; }
 		if (n !== copy && n.matches('script, style')) n.remove();
 	}
-	const r = el.getBoundingClientRect();
 	return {
 		label, selector, component, source,
 		html: cut(copy.outerHTML.replace(/\s+/g, ' '), 600),
 		url: location.href.split(/[?#]/)[0],
-		box: {x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height)},
+		box: visible(el),
 	};
 })`
 
-// locateScript is the viewport box of the element each selector finds, an
-// empty one where it finds none.
-const locateScript = `(selectors => selectors.map(s => {
-	let e = null;
-	try { e = document.querySelector(s); } catch {}
-	if (!e) return {x: 0, y: 0, w: 0, h: 0};
+// visibleScript is the viewport box of the part of an element that shows: a
+// list, sidebar or dialog that scrolls cuts its rows to its own box, and a
+// row it hides whole answers an empty box, so a pin is never drawn over what
+// stands beside the list. A partly hidden one keeps the part that shows,
+// where the person still sees it. The walk stops at a fixed ancestor, which
+// no scrolling box above it cuts.
+const visibleScript = `(e => {
 	const r = e.getBoundingClientRect();
-	return {x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height)};
-}))`
+	let x0 = r.left, y0 = r.top, x1 = r.right, y1 = r.bottom;
+	for (let a = e; a && a !== document.body && a !== document.documentElement; a = a.parentElement) {
+		const s = getComputedStyle(a);
+		if (a !== e && (s.overflowX !== 'visible' || s.overflowY !== 'visible')) {
+			const c = a.getBoundingClientRect(), left = c.left + a.clientLeft, top = c.top + a.clientTop;
+			x0 = Math.max(x0, left); y0 = Math.max(y0, top);
+			x1 = Math.min(x1, left + a.clientWidth); y1 = Math.min(y1, top + a.clientHeight);
+		}
+		if (s.position === 'fixed') break;
+	}
+	if (x1 <= x0 || y1 <= y0) return {x: 0, y: 0, w: 0, h: 0};
+	return {x: Math.round(x0), y: Math.round(y0), w: Math.round(x1 - x0), h: Math.round(y1 - y0)};
+})`
+
+// locateScript is the visible box of the element each selector finds, an
+// empty one where it finds none. It first scrolls reveal's element, when
+// that does not show whole, to the middle of every box it scrolls in, or to
+// their top when it is taller than the viewport. The scroll is instant so
+// the boxes it answers are where the element stops.
+const locateScript = `((selectors, reveal) => {
+	const visible = ` + visibleScript + `;
+	const find = s => { try { return document.querySelector(s); } catch { return null; } };
+	const shown = reveal ? find(reveal) : null;
+	if (shown) {
+		const r = shown.getBoundingClientRect(), v = visible(shown);
+		const whole = v.w === Math.round(r.width) && v.h === Math.round(r.height) && v.w > 0 &&
+			r.top >= 0 && r.left >= 0 && r.bottom <= innerHeight && r.right <= innerWidth;
+		if (!whole) shown.scrollIntoView({block: r.height > innerHeight ? 'start' : 'center', inline: 'nearest', behavior: 'instant'});
+	}
+	return selectors.map(s => {
+		const e = find(s);
+		return e ? visible(e) : {x: 0, y: 0, w: 0, h: 0};
+	});
+})`
 
 func (p *chromePage) ElementAt(ctx context.Context, x, y float64) (api.BrowserElement, error) {
 	var el *api.BrowserElement
@@ -101,13 +134,17 @@ func (p *chromePage) ElementAt(ctx context.Context, x, y float64) (api.BrowserEl
 	return *el, nil
 }
 
-func (p *chromePage) Locate(ctx context.Context, selectors []string) ([]api.BrowserRect, error) {
+func (p *chromePage) Locate(ctx context.Context, selectors []string, reveal string) ([]api.BrowserRect, error) {
 	arg, err := json.Marshal(selectors)
 	if err != nil {
 		return nil, err
 	}
+	shown, err := json.Marshal(reveal)
+	if err != nil {
+		return nil, err
+	}
 	boxes := []api.BrowserRect{}
-	return boxes, p.run(ctx, chromedp.Evaluate(fmt.Sprintf("%s(%s)", locateScript, arg), &boxes))
+	return boxes, p.run(ctx, chromedp.Evaluate(fmt.Sprintf("%s(%s, %s)", locateScript, arg, shown), &boxes))
 }
 
 // ElementAtPoint is the element under a point of this session's page.
@@ -121,12 +158,13 @@ func (s *Service) ElementAtPoint(_ context.Context, opts api.BrowserPointOptions
 	return el, err
 }
 
-// LocateElements is where the elements opts.Selectors find stand now.
+// LocateElements is where the elements opts.Selectors find stand now, after
+// scrolling opts.Reveal's element into view when it is set.
 func (s *Service) LocateElements(_ context.Context, opts api.BrowserLocateOptions) ([]api.BrowserRect, error) {
 	var boxes []api.BrowserRect
 	_, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error {
 		var err error
-		boxes, err = p.Locate(ctx, opts.Selectors)
+		boxes, err = p.Locate(ctx, opts.Selectors, opts.Reveal)
 		return err
 	})
 	return boxes, err

internal/browser/notes_live_test.go

@@ -23,11 +23,16 @@ const notesProbe = `<body style="margin:0;font:16px sans-serif">
 </form></body>`
 
 func notesPage(t *testing.T) (*chromePage, context.Context) {
+	t.Helper()
+	return notesPageOf(t, notesProbe)
+}
+
+func notesPageOf(t *testing.T, html string) (*chromePage, context.Context) {
 	t.Helper()
 	p, _ := liveChrome(t)
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.Header().Set("Content-Type", "text/html")
-		fmt.Fprint(w, notesProbe)
+		fmt.Fprint(w, html)
 	}))
 	t.Cleanup(srv.Close)
 	ctx, cancel := context.WithTimeout(context.Background(), pageWait)
@@ -43,7 +48,7 @@ func notesPage(t *testing.T) (*chromePage, context.Context) {
 
 func boxOf(t *testing.T, p *chromePage, ctx context.Context, selector string) api.BrowserRect {
 	t.Helper()
-	boxes, err := p.Locate(ctx, []string{selector})
+	boxes, err := p.Locate(ctx, []string{selector}, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,6 +129,64 @@ func TestChromeLocateAnswersAnEmptyBoxForAnElementGone(t *testing.T) {
 	}
 }
 
+// scrollProbe is a list that scrolls inside a 100px box, rows of 40px, above
+// a page taller than the viewport.
+const scrollProbe = `<body style="margin:0;font:16px sans-serif">
+<ul id="list" style="margin:0;padding:0;height:100px;overflow:auto;list-style:none">
+ <li id="i1" style="height:40px">One</li><li id="i2" style="height:40px">Two</li>
+ <li id="i3" style="height:40px">Three</li><li id="i4" style="height:40px">Four</li>
+ <li id="i5" style="height:40px">Five</li><li id="i6" style="height:40px">Six</li>
+</ul>
+<div style="height:2000px"></div>
+<button id="far" style="display:block;width:150px;height:40px">Far</button>
+<div style="height:300px"></div></body>`
+
+func TestChromeLocateHidesAnElementItsScrollingBoxHides(t *testing.T) {
+	p, ctx := notesPageOf(t, scrollProbe)
+	if got := boxOf(t, p, ctx, "#i5"); got != (api.BrowserRect{}) {
+		t.Errorf("row scrolled out of its list answers %+v, want an empty box", got)
+	}
+	if got, want := boxOf(t, p, ctx, "#i3"), (api.BrowserRect{X: 0, Y: 80, W: 800, H: 20}); got != want {
+		t.Errorf("row half out of its list answers %+v, want its visible part %+v", got, want)
+	}
+}
+
+func TestChromeElementAtBoxesOnlyThePartItsScrollingBoxShows(t *testing.T) {
+	p, ctx := notesPageOf(t, scrollProbe)
+	el, err := p.ElementAt(ctx, 20, 90)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := (api.BrowserRect{X: 0, Y: 80, W: 800, H: 20}); el.Box != want {
+		t.Fatalf("picked %q at %+v, want its visible part %+v", el.Label, el.Box, want)
+	}
+}
+
+func TestChromeLocateRevealsAnElementOutOfView(t *testing.T) {
+	p, ctx := notesPageOf(t, scrollProbe)
+	boxes, err := p.Locate(ctx, []string{"#i5", "#far"}, "#i5")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := boxes[0]; got.H != 40 || got.Y < 0 || got.Y+got.H > 100 {
+		t.Errorf("revealed row stands at %+v, want whole inside its list", got)
+	}
+	far, err := p.Locate(ctx, []string{"#far"}, "#far")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := far[0]; got.H != 40 || got.Y < 250 || got.Y > 310 {
+		t.Errorf("revealed button stands at %+v, want centered in the 600px viewport", got)
+	}
+	again, err := p.Locate(ctx, []string{"#far"}, "#far")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again[0] != far[0] {
+		t.Errorf("revealing a button already in view moved it from %+v to %+v", far[0], again[0])
+	}
+}
+
 // reactProbe gives two elements a React fiber, as a development build and a
 // production build leave them: a named component with its source, and one
 // whose name the minifier shortened.

internal/browser/page.go

@@ -113,9 +113,10 @@ type Page interface {
 	// Selection is the text selected in the page, "" in a password field.
 	Selection(ctx context.Context) (string, error)
 	// ElementAt is the element under x,y, zero when the point holds none;
-	// Locate is the box of the element each selector finds, zero for none.
+	// Locate is the box of the element each selector finds, zero for none,
+	// after scrolling reveal's element into view when reveal is set.
 	ElementAt(ctx context.Context, x, y float64) (api.BrowserElement, error)
-	Locate(ctx context.Context, selectors []string) ([]api.BrowserRect, error)
+	Locate(ctx context.Context, selectors []string, reveal string) ([]api.BrowserRect, error)
 	Alive() bool
 	Close() error
 }

native/backend/browser.go

@@ -78,7 +78,8 @@ func (c *Client) BrowserElementAt(ctx context.Context, id string, x, y float64)
 }
 
 // BrowserLocate is where the elements selectors find stand in session id's
-// page, an empty box for one gone.
-func (c *Client) BrowserLocate(ctx context.Context, id string, selectors []string) ([]api.BrowserRect, error) {
-	return callMethod(ctx, c, api.BrowserLocate, api.BrowserLocateOptions{ID: id, Selectors: selectors})
+// page, an empty box for one gone, after scrolling reveal's element into
+// view when reveal is set.
+func (c *Client) BrowserLocate(ctx context.Context, id string, selectors []string, reveal string) ([]api.BrowserRect, error) {
+	return callMethod(ctx, c, api.BrowserLocate, api.BrowserLocateOptions{ID: id, Selectors: selectors, Reveal: reveal})
 }