diff --git a/CHANGELOG.md b/CHANGELOG.md index c30b1d92f..97a8d7126 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E (Cmd+Shift+E on macOS) turns the crosshair on and off, and Esc leaves it. A send the session refuses keeps the notes and says why in the tray. - A session's browser downloads files. A download the page starts, from a link, a button, a new tab or a script, lands in your Downloads folder under the name the page gives it, with ` (1)` added rather than overwriting a file already there. The agent can also open a file's URL directly. The agent's click, key press or open answers with the file's path and size, and the window shows a toast naming the file, with a button that opens its folder. Before, a download button in the session's browser did nothing. +### Fixed + +- A session browser's click, type or key press by index reaches the element the agent's last page read gave that index. After a scroll and a second read, an element the first read had listed under the same index, now out of view, could take the click instead, with no error. + ## [0.80.0] - 2026-10-27 > [!IMPORTANT] diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go index b0ff232db..f51f3c2cb 100644 --- a/internal/browser/chrome_live_test.go +++ b/internal/browser/chrome_live_test.go @@ -379,3 +379,47 @@ func TestChromeClickRefusesAPointOutsideTheViewport(t *testing.T) { t.Fatalf("click at 400,715 in an 800x600 viewport: %v, want an error naming the viewport", err) } } + +func TestChromeClickByIndexHitsTheLastReadNotAnEarlierOne(t *testing.T) { + p, _ := liveChrome(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, ` + +
+`) + })) + t.Cleanup(srv.Close) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + if err := p.Resize(ctx, Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + if _, err := p.Info(ctx); err != nil { + t.Fatal(err) + } + if err := p.run(ctx, chromedp.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil)); err != nil { + t.Fatal(err) + } + info, err := p.Info(ctx) + if err != nil { + t.Fatal(err) + } + if len(info.Interactive) != 1 || info.Interactive[0].Text != "Bottom" { + t.Fatalf("second read listed %+v, want only Bottom", info.Interactive) + } + index := info.Interactive[0].Index + if _, _, err := p.Click(ctx, Target{Index: &index}); err != nil { + t.Fatal(err) + } + var hit string + if err := p.run(ctx, chromedp.Evaluate(`document.body.dataset.hit || ""`, &hit)); err != nil { + t.Fatal(err) + } + if hit != "bottom" { + t.Fatalf("index %d from the last read clicked %q, want bottom", index, hit) + } +} diff --git a/internal/browser/pageinfo.go b/internal/browser/pageinfo.go index f38fe37d5..4914533e3 100644 --- a/internal/browser/pageinfo.go +++ b/internal/browser/pageinfo.go @@ -4,6 +4,11 @@ package browser // data-lich-idx so a later click/type can find the same node, and it never // reads a form control's value into the result. // +// Each read first wipes the stamps of the one before. An element scrolled out +// of view keeps no index, so the index the last read gave to another element +// cannot resolve to it; an index the last read did not hand out finds nothing +// and click/type report the page changed. +// // A password input has no innerText, so an innerText || value label chain // hands the live credential to the agent and into the session transcript. // Rather than classify which fields are sensitive — type === 'password' is @@ -51,6 +56,7 @@ const pageInfoScript = `(() => { attr(el, 'name') ).trim(); }; + document.querySelectorAll('[data-lich-idx]').forEach((el) => el.removeAttribute('data-lich-idx')); const els = [...document.querySelectorAll(sel)].filter(isVisible); const interactive = els.slice(0, 200).map((el, i) => { el.setAttribute('data-lich-idx', String(i));