feat(browser): browser_info waits for text to appear or go

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E (Cmd+Shift+E on macOS) turns the crosshair on and off, and Esc leaves it. A send the session refuses keeps the notes and says why in the tray.
 - A session's browser downloads files. A download the page starts, from a link, a button, a new tab or a script, lands in your Downloads folder under the name the page gives it, with ` (1)` added rather than overwriting a file already there. The agent can also open a file's URL directly. The agent's click, key press or open answers with the file's path and size, and the window shows a toast naming the file, with a button that opens its folder. Before, a download button in the session's browser did nothing.
+- The agent can wait for its browser page to change instead of reading it again in a loop. `browser_info` takes `wait_for` (text that should appear) and `wait_gone` (text that should disappear), matched anywhere on the page, and answers as soon as the page shows it, or after 20 seconds with the page as it stands and a line saying the wait timed out.
 
 ### Fixed
 

api/browser.go

@@ -3,7 +3,7 @@ package api
 var (
 	BrowserShow = define[SessionIDOptions, BrowserHandle]("browser.Show")
 	BrowserOpen = define[BrowserOpenOptions, BrowserHandle]("browser.Open")
-	BrowserInfo = define[SessionIDOptions, BrowserPageInfo]("browser.Info")
+	BrowserInfo = define[BrowserInfoOptions, BrowserPageInfo]("browser.Info")
 	// Click and Press answer with the page as it stands after them, so an
 	// agent reads what its action did without a browser.Info of its own. Type
 	// does not: typing into a field seldom changes the page, and a page per
@@ -176,6 +176,15 @@ type BrowserOpenOptions struct {
 	URL string `json:"url"`
 }
 
+// BrowserInfoOptions reads session ID's page. WaitFor and WaitGone, when set,
+// hold the read until that text is on the page's visible text, or off it
+// (case-sensitive substring, the whole page, not only the viewport).
+type BrowserInfoOptions struct {
+	ID       string `json:"id"`
+	WaitFor  string `json:"waitFor,omitempty"`
+	WaitGone string `json:"waitGone,omitempty"`
+}
+
 // BrowserTarget picks an element. Index is the one page-info stamped;
 // selector is a CSS query; x/y are viewport coordinates. Coordinates win when
 // set, then selector, then index.
@@ -255,6 +264,9 @@ type BrowserPageInfo struct {
 	// state: what its last action saved, or one it saw start and that ended
 	// since.
 	Downloads []BrowserDownload `json:"downloads,omitempty"`
+	// WaitTimedOut says the text a read waited for never came (or never
+	// went): the page is the one standing when the wait gave up.
+	WaitTimedOut bool `json:"waitTimedOut,omitempty"`
 }
 
 // BrowserPointOptions is a point of session ID's page, in CSS pixels.

api/testdata/contract.golden

@@ -10,7 +10,7 @@ browser.Close(SessionIDOptions) None
 browser.Detach(SessionIDOptions) None
 browser.ElementAt(BrowserPointOptions) BrowserElement
 browser.Forward(SessionIDOptions) None
-browser.Info(SessionIDOptions) BrowserPageInfo
+browser.Info(BrowserInfoOptions) BrowserPageInfo
 browser.Key(BrowserKeyOptions) None
 browser.List(SessionIDOptions) []BrowserHandle
 browser.Locate(BrowserLocateOptions) []BrowserRect
@@ -342,6 +342,7 @@ BrowserPageInfo {
 	interactive []BrowserInteractive
 	truncated bool
 	downloads,omitempty []BrowserDownload
+	waitTimedOut,omitempty bool
 }
 
 BrowserPointOptions {
@@ -360,6 +361,12 @@ BrowserElement {
 	box BrowserRect
 }
 
+BrowserInfoOptions {
+	id string
+	waitFor,omitempty string
+	waitGone,omitempty string
+}
+
 BrowserKeyOptions {
 	id string
 	type string

internal/browser/browser.go

@@ -59,6 +59,9 @@ type Service struct {
 	stopped bool
 	// emit pushes to the window's /events (events.Hub.Emit).
 	emit func(name string, data any)
+	// textWait is maxTextWait, which the suite shortens to test a wait that
+	// gives up.
+	textWait time.Duration
 }
 
 type session struct {
@@ -90,16 +93,17 @@ func New(emit func(name string, data any)) *Service {
 
 func newService(launch Launcher, mkTmp func(string, string) (string, error), now func() time.Time, idle time.Duration) *Service {
 	return &Service{
-		launch:  launch,
-		mkTmp:   mkTmp,
-		now:     now,
-		idle:    idle,
-		quiet:   downloadQuiet,
-		byOwner: make(map[string]*session),
-		byID:    make(map[string]*session),
-		owners:  make(map[string]*sync.Mutex),
-		stop:    make(chan struct{}),
-		emit:    func(string, any) {},
+		launch:   launch,
+		mkTmp:    mkTmp,
+		now:      now,
+		idle:     idle,
+		quiet:    downloadQuiet,
+		textWait: maxTextWait,
+		byOwner:  make(map[string]*session),
+		byID:     make(map[string]*session),
+		owners:   make(map[string]*sync.Mutex),
+		stop:     make(chan struct{}),
+		emit:     func(string, any) {},
 	}
 }
 

internal/browser/chrome.go

@@ -192,6 +192,16 @@ func (p *chromePage) Info(ctx context.Context) (PageInfo, error) {
 	return info, nil
 }
 
+func (p *chromePage) HasText(ctx context.Context, text string) (bool, error) {
+	literal, err := json.Marshal(text)
+	if err != nil {
+		return false, err
+	}
+	var has bool
+	script := fmt.Sprintf(hasTextScript, literal)
+	return has, p.run(ctx, chromedp.Evaluate(script, &has))
+}
+
 func (p *chromePage) Click(ctx context.Context, t Target) (x, y float64, err error) {
 	pt, err := p.resolve(ctx, t)
 	if err != nil {

internal/browser/downloads_live_test.go

@@ -74,7 +74,7 @@ func TestChromeDownloadsLandInDownloadsAndThePageStays(t *testing.T) {
 				// and a read names a download still running, then again once
 				// it ends.
 				time.Sleep(100 * time.Millisecond)
-				read, err := s.Read(t.Context(), api.SessionIDOptions{ID: "s"})
+				read, err := s.Read(t.Context(), api.BrowserInfoOptions{ID: "s"})
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -88,7 +88,7 @@ func TestChromeDownloadsLandInDownloadsAndThePageStays(t *testing.T) {
 			if b, err := os.ReadFile(want); err != nil || string(b) != id+".csv,1" {
 				t.Fatalf("%s holds %q (%v)", want, b, err)
 			}
-			read, err := s.Read(t.Context(), api.SessionIDOptions{ID: "s"})
+			read, err := s.Read(t.Context(), api.BrowserInfoOptions{ID: "s"})
 			if err != nil {
 				t.Fatal(err)
 			}

internal/browser/fake_test.go

@@ -3,6 +3,7 @@ package browser
 import (
 	"context"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/chromedp/cdproto/network"
@@ -54,6 +55,11 @@ type fakePage struct {
 	shotErr   error
 	navErr    error
 	infoErr   error
+	// pageText is what HasText searches; onTextRead runs before each search,
+	// as the page changing between two reads.
+	pageText   string
+	textReads  int
+	onTextRead func(reads int)
 }
 
 type typed struct {
@@ -120,6 +126,19 @@ func (p *fakePage) Info(context.Context) (PageInfo, error) {
 	return info, nil
 }
 
+func (p *fakePage) HasText(_ context.Context, text string) (bool, error) {
+	p.mu.Lock()
+	p.textReads++
+	reads, then := p.textReads, p.onTextRead
+	p.mu.Unlock()
+	if then != nil {
+		then(reads)
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return strings.Contains(p.pageText, text), nil
+}
+
 func (p *fakePage) Click(_ context.Context, t Target) (x, y float64, err error) {
 	p.mu.Lock()
 	if p.clickErr != nil {

internal/browser/page.go

@@ -81,6 +81,9 @@ type PageInfo struct {
 type Page interface {
 	Navigate(ctx context.Context, url string) error
 	Info(ctx context.Context) (PageInfo, error)
+	// HasText says whether text is in the page's visible text, the whole
+	// document's and not only the viewport's.
+	HasText(ctx context.Context, text string) (bool, error)
 	// Settle waits until a navigation the last action may have started has
 	// its document, bounded by domSettle.
 	Settle(ctx context.Context) error

internal/browser/rpc.go

@@ -26,10 +26,17 @@ func (s *Service) OpenURL(_ context.Context, opts api.BrowserOpenOptions) (api.B
 	return s.withDownloads(handleAPI(h)), nil
 }
 
-// Read returns the page the agent is looking at.
-func (s *Service) Read(_ context.Context, opts api.SessionIDOptions) (api.BrowserPageInfo, error) {
+// Read returns the page the agent is looking at, once the text it waits for
+// came or went. A wait that gives up still answers with the page.
+func (s *Service) Read(_ context.Context, opts api.BrowserInfoOptions) (api.BrowserPageInfo, error) {
+	timedOut, err := s.waitText(opts)
+	if err != nil {
+		return api.BrowserPageInfo{}, err
+	}
 	info, err := s.Info(opts.ID)
-	return infoAPI(info), err
+	out := infoAPI(info)
+	out.WaitTimedOut = timedOut
+	return out, err
 }
 
 // ClickAt clicks in this session's browser.

internal/browser/session_downloads_test.go

@@ -32,7 +32,7 @@ func TestClickReportsTheFileItDownloadedAndTellsTheWindowOnce(t *testing.T) {
 	if len(done) != 1 || done[0].(api.BrowserDownload).Owner != "s" || done[0].(api.BrowserDownload).Path != "/dl/report.csv" {
 		t.Fatalf("window heard %+v, want the one completed download, owned by s", done)
 	}
-	again, err := s.Read(t.Context(), api.SessionIDOptions{ID: "s"})
+	again, err := s.Read(t.Context(), api.BrowserInfoOptions{ID: "s"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -79,7 +79,7 @@ func TestADownloadStillRunningIsReportedAgainWhenItEnds(t *testing.T) {
 		t.Fatalf("got %+v after %v: a download already reported in progress must not hold the next action", got, time.Since(start))
 	}
 	p.onDownload("g1", api.BrowserDownload{Path: "/dl/big.iso", Bytes: 9, State: api.DownloadCompleted})
-	info, err := s.Read(t.Context(), api.SessionIDOptions{ID: "s"})
+	info, err := s.Read(t.Context(), api.BrowserInfoOptions{ID: "s"})
 	if err != nil {
 		t.Fatal(err)
 	}

internal/browser/wait.go

@@ -0,0 +1,64 @@
+package browser
+
+import (
+	"context"
+	"time"
+
+	"github.com/lichdotdev/lich/api"
+)
+
+// textPoll is how often a wait reads the page's text again: short enough that
+// the read follows the page by a frame or so, long enough not to keep the
+// renderer busy answering.
+const textPoll = 100 * time.Millisecond
+
+// maxTextWait bounds a wait for text, after which the page is read as it
+// stands. 20 seconds leaves that read its own pageWait under the client's
+// 60-second openCall.
+const maxTextWait = 20 * time.Second
+
+// hasTextScript takes the text as a JSON string literal.
+const hasTextScript = `(document.body ? document.body.innerText : '').includes(%s)`
+
+// waitText holds a read until opts.WaitFor is on the page and opts.WaitGone
+// is off it, and says whether it gave up first.
+func (s *Service) waitText(opts api.BrowserInfoOptions) (timedOut bool, err error) {
+	if opts.WaitFor == "" && opts.WaitGone == "" {
+		return false, nil
+	}
+	sess, err := s.live(opts.ID)
+	if err != nil {
+		return false, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), s.textWait)
+	defer cancel()
+	tick := time.NewTicker(textPoll)
+	defer tick.Stop()
+	// A failed read is the page between two documents, which a click that
+	// navigates leaves for a moment: the wait reads again. A Chrome that died
+	// stops it, and the page read after reports that.
+	for sess.page.Alive() {
+		if ok, err := textSettled(ctx, sess.page, opts); err == nil && ok {
+			return false, nil
+		}
+		select {
+		case <-ctx.Done():
+			return true, nil
+		case <-tick.C:
+		}
+	}
+	return false, nil
+}
+
+func textSettled(ctx context.Context, p Page, opts api.BrowserInfoOptions) (bool, error) {
+	if opts.WaitFor != "" {
+		if has, err := p.HasText(ctx, opts.WaitFor); err != nil || !has {
+			return false, err
+		}
+	}
+	if opts.WaitGone == "" {
+		return true, nil
+	}
+	has, err := p.HasText(ctx, opts.WaitGone)
+	return !has, err
+}

internal/browser/wait_live_test.go

@@ -0,0 +1,64 @@
+package browser
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/lichdotdev/lich/api"
+	"github.com/lichdotdev/lich/internal/chromium"
+)
+
+// lateTextPage swaps Loading for Saved a moment after it loads, below the
+// fold so the wait is seen to read the whole page and not the viewport.
+const lateTextPage = `<title>Late</title><div style="height:3000px"></div>
+<p id="status">Loading</p>
+<script>setTimeout(() => { document.getElementById('status').textContent = 'Saved' }, 300)</script>`
+
+func TestChromeReadWaitsForTextThePageWritesLater(t *testing.T) {
+	if _, err := chromium.FindBrowser(exec.LookPath); err != nil {
+		t.Skip("no Chromium-family browser on this machine")
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, lateTextPage)
+	}))
+	t.Cleanup(srv.Close)
+	// The profiles' root is made before Cleanup is registered, so it is
+	// removed after Chrome exits: Windows refuses to delete a file it holds.
+	root := t.TempDir()
+	s := newService(chromeLaunch, func(_, pattern string) (string, error) { return os.MkdirTemp(root, pattern) }, time.Now, idleAfter)
+	t.Cleanup(func() {
+		if err := s.Cleanup(); err != nil {
+			t.Error(err)
+		}
+	})
+	// One browser per case: each reads the page's text change from its load.
+	for _, opts := range []api.BrowserInfoOptions{
+		{ID: "for", WaitFor: "Saved"},
+		{ID: "gone", WaitGone: "Loading"},
+	} {
+		if _, err := s.Open(opts.ID, srv.URL); err != nil {
+			t.Fatal(err)
+		}
+		info, err := s.Read(t.Context(), opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if info.WaitTimedOut {
+			t.Fatalf("%+v timed out, want the page's late text seen", opts)
+		}
+	}
+	s.textWait = 3 * textPoll
+	info, err := s.Read(t.Context(), api.BrowserInfoOptions{ID: "for", WaitFor: "saved"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !info.WaitTimedOut || info.Title != "Late" {
+		t.Fatalf("got timed out %v on %q, want the page answered with the wait timed out", info.WaitTimedOut, info.Title)
+	}
+}