feat(browser): name the page's console errors in the agent's next page answer

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E (Cmd+Shift+E on macOS) turns the crosshair on and off, and Esc leaves it. A send the session refuses keeps the notes and says why in the tray.
 - A session's browser downloads files. A download the page starts, from a link, a button, a new tab or a script, lands in your Downloads folder under the name the page gives it, with ` (1)` added rather than overwriting a file already there. The agent can also open a file's URL directly. The agent's click, key press or open answers with the file's path and size, and the window shows a toast naming the file, with a button that opens its folder. Before, a download button in the session's browser did nothing.
 - The agent can wait for its browser page to change instead of reading it again in a loop. `browser_info` takes `wait_for` (text that should appear) and `wait_gone` (text that should disappear), matched anywhere on the page, and answers as soon as the page shows it, or after 20 seconds with the page as it stands and a line saying the wait timed out.
+- The agent hears when the page in its session's browser breaks. Each page it reads, clicks or presses a key on answers with the console errors since its last look, once each: `console.error` calls, uncaught exceptions and resources that failed to load, cut to one short line, the first five and how many more. Warnings and other messages stay out.
 
 ### Fixed
 

api/browser.go

@@ -267,6 +267,11 @@ type BrowserPageInfo struct {
 	// WaitTimedOut says the text a read waited for never came (or never
 	// went): the page is the one standing when the wait gave up.
 	WaitTimedOut bool `json:"waitTimedOut,omitempty"`
+	// ConsoleErrors are the errors the page's console showed since the
+	// agent's last page answer, each one line cut short; ConsoleErrorsMore
+	// counts those past the cap.
+	ConsoleErrors     []string `json:"consoleErrors,omitempty"`
+	ConsoleErrorsMore int      `json:"consoleErrorsMore,omitempty"`
 }
 
 // BrowserPointOptions is a point of session ID's page, in CSS pixels.

api/testdata/contract.golden

@@ -343,6 +343,8 @@ BrowserPageInfo {
 	truncated bool
 	downloads,omitempty []BrowserDownload
 	waitTimedOut,omitempty bool
+	consoleErrors,omitempty []string
+	consoleErrorsMore,omitempty int
 }
 
 BrowserPointOptions {

docs/browser-tab.md

@@ -59,6 +59,18 @@ page only answers what is where (`browser.ElementAt`, `browser.Locate`); the
 outline, the pins and the note are the window's, so nothing is injected into
 the page.
 
+## Console errors
+
+Each page answer (`browser_info`, `browser_click`, `browser_press`) names the
+errors the page's console showed since the agent's last one, once each, as
+`console error:` lines: a `console.error` call, an uncaught exception with
+where it was thrown, or an error Chrome logs itself, such as a resource that
+failed to load. Each is one line cut to 200 characters; past the first five, a
+`(N more console errors)` line counts the rest. Warnings, info and log
+messages are left out. The text is what the page logs, so a page that logs a
+form's value hands it to the agent; lich reads nothing else. These lines are
+English, outside the prompt-language catalogs, like the rest of the page answer.
+
 ## Downloads
 
 A file the page downloads lands in the user's Downloads folder

internal/browser/actions.go

@@ -76,6 +76,7 @@ func (s *Service) Info(id string) (PageInfo, error) {
 	s.remember(sess, info.URL, info.Title)
 	s.agentActed(sess, api.BrowserAgentAct{})
 	info.Downloads = s.untold(sess)
+	info.ConsoleErrors, info.ConsoleErrorsMore = s.consoleUntold(sess)
 	return info, nil
 }
 

internal/browser/browser.go

@@ -77,6 +77,10 @@ type session struct {
 	// downloadBegun is when the last one began.
 	downloads     []*download
 	downloadBegun time.Time
+	// consoleErrors are the page's since the agent's last page answer, the
+	// first consoleErrorMax of them; consoleMore counts the rest.
+	consoleErrors []string
+	consoleMore   int
 	page          Page
 	lastUsed      time.Time
 }
@@ -269,7 +273,7 @@ func (s *Service) start(spec launchSpec) (Handle, error) {
 		id: newID(), owner: spec.owner, dir: spec.dir, url: spec.url, headed: spec.headed,
 		byAgent: spec.byAgent, page: page, lastUsed: s.now(),
 	}
-	page.Listen(s.frameSink(sess), s.infoSink(sess), s.downloadSink(sess))
+	page.Listen(s.frameSink(sess), s.infoSink(sess), s.downloadSink(sess), s.consoleSink(sess))
 	if err := runPage(func(ctx context.Context) error {
 		if len(spec.cookies) > 0 {
 			if err := page.SetCookies(ctx, spec.cookies); err != nil {

internal/browser/chrome_live_test.go

@@ -148,7 +148,7 @@ func TestChromeScreencastSendsFramesAndPageInfo(t *testing.T) {
 		case titles <- title:
 		default:
 		}
-	}, func(string, api.BrowserDownload) {})
+	}, func(string, api.BrowserDownload) {}, func(string) {})
 	ctx, cancel := context.WithTimeout(context.Background(), pageWait)
 	defer cancel()
 	if err := p.Navigate(ctx, srv.URL); err != nil {
@@ -225,7 +225,7 @@ func TestChromeLinkToANewTabLoadsInThePageAndKeepsItResponding(t *testing.T) {
 	t.Cleanup(srv.Close)
 	p := pg.(*chromePage)
 	urls := make(chan string, 8)
-	p.Listen(func(string) {}, func(url, _ string) { urls <- url }, func(string, api.BrowserDownload) {})
+	p.Listen(func(string) {}, func(url, _ string) { urls <- url }, func(string, api.BrowserDownload) {}, func(string) {})
 	ctx, cancel := context.WithTimeout(context.Background(), pageWait)
 	defer cancel()
 	if err := p.Navigate(ctx, srv.URL+"/"); err != nil {

internal/browser/chrome_view.go

@@ -31,7 +31,7 @@ type listeners struct {
 	onInfo  func(url, title string)
 }
 
-func (p *chromePage) Listen(onFrame func(data string), onInfo func(url, title string), onDownload func(guid string, d api.BrowserDownload)) {
+func (p *chromePage) Listen(onFrame func(data string), onInfo func(url, title string), onDownload func(guid string, d api.BrowserDownload), onConsoleError func(text string)) {
 	p.listen.mu.Lock()
 	p.listen.onFrame, p.listen.onInfo = onFrame, onInfo
 	p.listen.mu.Unlock()
@@ -45,6 +45,10 @@ func (p *chromePage) Listen(onFrame func(data string), onInfo func(url, title st
 			}
 		case *page.EventNavigatedWithinDocument, *page.EventDomContentEventFired:
 			go p.reportInfo()
+		default:
+			if text, ok := consoleError(ev); ok {
+				onConsoleError(text)
+			}
 		}
 	})
 	p.keepOneTab()

internal/browser/console.go

@@ -0,0 +1,118 @@
+package browser
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/chromedp/cdproto/log"
+	"github.com/chromedp/cdproto/runtime"
+)
+
+// consoleErrorMax and consoleTextMax keep a page answer lean: a page that
+// throws in a loop names its first errors and how many more, not every one.
+const (
+	consoleErrorMax = 5
+	consoleTextMax  = 200
+)
+
+// consoleSink keeps each error the page's console shows until the agent's
+// next page answer.
+func (s *Service) consoleSink(sess *session) func(text string) {
+	return func(text string) {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		if len(sess.consoleErrors) < consoleErrorMax {
+			sess.consoleErrors = append(sess.consoleErrors, consoleLine(text))
+		} else {
+			sess.consoleMore++
+		}
+	}
+}
+
+// consoleUntold is the errors the agent has not heard, which it now has.
+func (s *Service) consoleUntold(sess *session) ([]string, int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	out, more := sess.consoleErrors, sess.consoleMore
+	sess.consoleErrors, sess.consoleMore = nil, 0
+	return out, more
+}
+
+// consoleLine is text on one line, cut to consoleTextMax runes.
+func consoleLine(text string) string {
+	text = strings.Join(strings.Fields(text), " ")
+	if r := []rune(text); len(r) > consoleTextMax {
+		return string(r[:consoleTextMax]) + "…"
+	}
+	return text
+}
+
+// consoleError is the text of a CDP event that is a console error: a
+// console.error call, an uncaught exception, or an error Chrome logs itself,
+// such as a resource that failed to load. Warnings and the rest are not.
+func consoleError(ev any) (string, bool) {
+	switch e := ev.(type) {
+	case *runtime.EventConsoleAPICalled:
+		return consoleCallText(e), e.Type == runtime.APITypeError
+	case *runtime.EventExceptionThrown:
+		return exceptionText(e.ExceptionDetails), e.ExceptionDetails != nil
+	case *log.EventEntryAdded:
+		return logEntryText(e.Entry), e.Entry != nil && e.Entry.Level == log.LevelError
+	}
+	return "", false
+}
+
+func consoleCallText(e *runtime.EventConsoleAPICalled) string {
+	parts := make([]string, len(e.Args))
+	for i, a := range e.Args {
+		parts[i] = objectText(a)
+	}
+	return strings.Join(parts, " ")
+}
+
+// objectText is how the console prints one argument, first line only: an
+// Error's description carries its stack after it.
+func objectText(o *runtime.RemoteObject) string {
+	var s string
+	switch {
+	case len(o.Value) > 0:
+		if json.Unmarshal(o.Value, &s) != nil {
+			s = string(o.Value)
+		}
+	case o.UnserializableValue != "":
+		s = string(o.UnserializableValue)
+	case o.Description != "":
+		s = o.Description
+	default:
+		s = string(o.Type)
+	}
+	first, _, _ := strings.Cut(s, "\n")
+	return first
+}
+
+// exceptionText is "Uncaught TypeError: … (url:line)", the way the console
+// shows it, the line 1-based.
+func exceptionText(d *runtime.ExceptionDetails) string {
+	if d == nil {
+		return ""
+	}
+	text := d.Text
+	if d.Exception != nil {
+		text += " " + objectText(d.Exception)
+	}
+	if d.URL != "" {
+		text += fmt.Sprintf(" (%s:%d)", d.URL, d.LineNumber+1)
+	}
+	return text
+}
+
+func logEntryText(e *log.Entry) string {
+	if e == nil {
+		return ""
+	}
+	if e.URL == "" {
+		return e.Text
+	}
+	return e.Text + " " + e.URL
+}

internal/browser/console_live_test.go

@@ -0,0 +1,99 @@
+package browser
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/lichdotdev/lich/api"
+	"github.com/lichdotdev/lich/internal/chromium"
+)
+
+// consolePage logs one error and one warning as it loads, asks for an image
+// that is not there, and throws when its button is clicked.
+const consolePage = `<title>Console</title>
+<img src="/missing.png">
+<button id="throw" style="font-size:30px" onclick="null.f()">throw</button>
+<script>console.error("load failed:", 42); console.warn("only a warning")</script>`
+
+func TestChromeNamesTheConsoleErrorsOnceInTheNextPageAnswer(t *testing.T) {
+	if _, err := chromium.FindBrowser(exec.LookPath); err != nil {
+		t.Skip("no Chromium-family browser on this machine")
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Chrome asks for a favicon on its own, at a moment of its choosing:
+		// its 404 would land in whichever answer comes next.
+		if r.URL.Path == "/favicon.ico" {
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, consolePage)
+	}))
+	t.Cleanup(srv.Close)
+	// The profile's folder is made before the service so that its removal,
+	// a cleanup, runs after Cleanup closed Chrome: Windows refuses to delete
+	// a file Chrome still holds.
+	profile := t.TempDir()
+	s := newService(chromeLaunch, func(string, string) (string, error) { return profile, nil }, time.Now, idleAfter)
+	t.Cleanup(func() {
+		if err := s.Cleanup(); err != nil {
+			t.Error(err)
+		}
+	})
+	if _, err := s.Open("s", srv.URL+"/"); err != nil {
+		t.Fatal(err)
+	}
+	loaded := readConsole(t, s, 2)
+	if !hasError(loaded, "load failed: 42") || !hasError(loaded, "/missing.png") || hasError(loaded, "warning") {
+		t.Fatalf("the page answer named %q, want the error and the missing image, no warning", loaded)
+	}
+	clicked, err := s.ClickAt(t.Context(), api.BrowserClickOptions{ID: "s", BrowserTarget: api.BrowserTarget{Selector: "#throw"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(clicked.ConsoleErrors) != 1 || !strings.Contains(clicked.ConsoleErrors[0], "Uncaught TypeError") {
+		t.Fatalf("the click answered %q, want the one uncaught exception", clicked.ConsoleErrors)
+	}
+	again, err := s.Read(t.Context(), api.BrowserInfoOptions{ID: "s"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(again.ConsoleErrors) != 0 {
+		t.Fatalf("a read after the click repeated %q", again.ConsoleErrors)
+	}
+}
+
+// readConsole reads the page until its answers have named want errors: the
+// image's failure arrives when Chrome gets the 404, not with the document.
+func readConsole(t *testing.T, s *Service, want int) []string {
+	t.Helper()
+	var got []string
+	deadline := time.Now().Add(pageWait)
+	for len(got) < want && time.Now().Before(deadline) {
+		info, err := s.Read(t.Context(), api.BrowserInfoOptions{ID: "s"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		got = append(got, info.ConsoleErrors...)
+		time.Sleep(downloadPoll)
+	}
+	return got
+}
+
+func hasError(errs []string, part string) bool {
+	for _, e := range errs {
+		if strings.Contains(e, part) {
+			return true
+		}
+	}
+	return false
+}