From 084ab48b917dd3579b9f775e0fd28a213c5d01fb Mon Sep 17 00:00:00 2001 From: omartelo Date: Tue, 27 Oct 2026 15:23:03 -0300 Subject: [PATCH 1/3] feat(browser): leave notes on page elements for the agent A crosshair on the session browser's toolbar stops the pointer reaching the page and outlines the element under it, named in a tag. A click opens a note beside it, in the review's comment box; each note keeps a numbered tab on a dashed ring around its element, following it as the page scrolls. Notes wait in a tray floating over the page's bottom, so the page never resizes, and Send types them into the session as one message: label, page, a selector browser_click takes, the React component in a development build, the markup without field values, and the note. A send that fails keeps the notes. The page only answers what is where (browser.ElementAt, browser.Locate); the outline, tabs, popover and tray are drawn by the window, so nothing is injected into the page's DOM. --- CHANGELOG.md | 4 + api/browser.go | 35 ++ api/testdata/contract.golden | 37 +- docs/browser-tab.md | 13 + docs/ceilings.md | 9 + internal/browser/fake_test.go | 22 + internal/browser/notes.go | 129 +++++ internal/browser/notes_live_test.go | 125 +++++ internal/browser/notes_test.go | 47 ++ internal/browser/page.go | 4 + internal/routes/browser.go | 2 + native/backend/browser.go | 11 + native/cmd/lich-native/browser_notes.go | 368 ++++++++++++++ .../cmd/lich-native/browser_notes_prompt.go | 33 ++ native/cmd/lich-native/browser_notes_ring.go | 137 +++++ native/cmd/lich-native/browser_notes_test.go | 135 +++++ native/cmd/lich-native/browser_notes_view.go | 481 ++++++++++++++++++ native/cmd/lich-native/browser_pane.go | 36 +- native/cmd/lich-native/browser_queue.go | 9 +- native/cmd/lich-native/browser_queue_test.go | 22 + native/i18n/locales/en/layouts.json | 17 +- native/i18n/locales/en/prompts.json | 9 + native/i18n/locales/es/layouts.json | 17 +- native/i18n/locales/es/prompts.json | 9 + native/i18n/locales/ja/layouts.json | 16 +- native/i18n/locales/ja/prompts.json | 8 + native/i18n/locales/ko/layouts.json | 16 +- native/i18n/locales/ko/prompts.json | 8 + native/i18n/locales/pt-br/layouts.json | 17 +- native/i18n/locales/pt-br/prompts.json | 9 + native/i18n/locales/ru/layouts.json | 19 +- native/i18n/locales/ru/prompts.json | 11 + native/i18n/locales/zh-cn/layouts.json | 16 +- native/i18n/locales/zh-cn/prompts.json | 8 + 34 files changed, 1821 insertions(+), 18 deletions(-) create mode 100644 internal/browser/notes.go create mode 100644 internal/browser/notes_live_test.go create mode 100644 internal/browser/notes_test.go create mode 100644 native/cmd/lich-native/browser_notes.go create mode 100644 native/cmd/lich-native/browser_notes_prompt.go create mode 100644 native/cmd/lich-native/browser_notes_ring.go create mode 100644 native/cmd/lich-native/browser_notes_test.go create mode 100644 native/cmd/lich-native/browser_notes_view.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc03ca51..e8d80b953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E turns the crosshair on and off, and Esc leaves it. + ## [0.80.0] - 2026-10-27 > [!IMPORTANT] diff --git a/api/browser.go b/api/browser.go index 109b351d8..8daea24d7 100644 --- a/api/browser.go +++ b/api/browser.go @@ -35,6 +35,11 @@ var ( // copies to the system clipboard: the headless browser's own clipboard // is not the person's. BrowserSelection = define[SessionIDOptions, string]("browser.Selection") + // BrowserElementAt is the element under a point of the page, for a note + // the person leaves on it; BrowserLocate is where noted elements stand + // now, after the page scrolled or changed. + BrowserElementAt = define[BrowserPointOptions, BrowserElement]("browser.ElementAt") + BrowserLocate = define[BrowserLocateOptions, []BrowserRect]("browser.Locate") ) var ( @@ -223,3 +228,33 @@ type BrowserPageInfo struct { Interactive []BrowserInteractive `json:"interactive"` Truncated bool `json:"truncated"` } + +// BrowserPointOptions is a point of session ID's page, in CSS pixels. +type BrowserPointOptions struct { + ID string `json:"id"` + X float64 `json:"x"` + Y float64 `json:"y"` +} + +// BrowserElement is the element under a point, as a note on it names it. +// Label is its tag and text (button "Buy now"), empty when the point holds +// none; Selector finds it again, in the form browser.Click takes. Component +// and Source are the React component that drew it and where, when the page +// is a React development build. HTML is its markup, shortened, with no +// field's value. URL is the page's, without query or fragment. +type BrowserElement struct { + Label string `json:"label"` + Selector string `json:"selector"` + Component string `json:"component,omitempty"` + Source string `json:"source,omitempty"` + HTML string `json:"html"` + URL string `json:"url"` + Box BrowserRect `json:"box"` +} + +// BrowserLocateOptions asks where the elements Selectors find stand in +// session ID's page; one gone from the page answers an empty box. +type BrowserLocateOptions struct { + ID string `json:"id"` + Selectors []string `json:"selectors"` +} diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index 0715941fe..722a37306 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -8,10 +8,12 @@ browser.Back(SessionIDOptions) None browser.Click(BrowserClickOptions) BrowserPageInfo browser.Close(SessionIDOptions) None browser.Detach(SessionIDOptions) None +browser.ElementAt(BrowserPointOptions) BrowserElement browser.Forward(SessionIDOptions) None browser.Info(SessionIDOptions) BrowserPageInfo browser.Key(BrowserKeyOptions) None browser.List(SessionIDOptions) []BrowserHandle +browser.Locate(BrowserLocateOptions) []BrowserRect browser.Mouse(BrowserMouseOptions) None browser.Nav(BrowserNavOptions) None browser.Navigate(BrowserOpenOptions) BrowserHandle @@ -339,6 +341,22 @@ BrowserPageInfo { truncated bool } +BrowserPointOptions { + id string + x float64 + y float64 +} + +BrowserElement { + label string + selector string + component,omitempty string + source,omitempty string + html string + url string + box BrowserRect +} + BrowserKeyOptions { id string type string @@ -348,6 +366,18 @@ BrowserKeyOptions { modifiers,omitempty int } +BrowserLocateOptions { + id string + selectors []string +} + +BrowserRect { + x int + y int + w int + h int +} + BrowserMouseOptions { id string type string @@ -1897,13 +1927,6 @@ SessionCache { uncached int } -BrowserRect { - x int - y int - w int - h int -} - PRThreadComment { id int64 author string diff --git a/docs/browser-tab.md b/docs/browser-tab.md index 2c88f152a..d57c3beb2 100644 --- a/docs/browser-tab.md +++ b/docs/browser-tab.md @@ -46,6 +46,19 @@ Page info never includes form values. Screenshots are files. Click, type and press are real input events. `press` is named keys only. Arbitrary JavaScript in the page is not a tool. +## Notes on the page + +The crosshair on the pane's toolbar stops the pointer reaching the page: the +element under it is outlined, and a click opens a note on it. Notes wait in a +tray floating over the bottom of the page, which never resizes for it, each +element pinned where it stands, and Send types them into the session's prompt +as one message: each element's label, page, a +selector `browser_click` takes, its React component when the page is a +development build, its markup without any field's value, and the note. The +page only answers what is where (`browser.ElementAt`, `browser.Locate`); the +outline, the pins and the note are the window's, so nothing is injected into +the page. + ## Providers Claude Code, Codex and opencode receive the tools at spawn. Antigravity, diff --git a/docs/ceilings.md b/docs/ceilings.md index f7dcbae1f..58bef42a9 100644 --- a/docs/ceilings.md +++ b/docs/ceilings.md @@ -979,6 +979,15 @@ work when nobody knows it and that the call site never shows. The mechanism and closed; one a pane shows, or a window, never is. It loads only http, https and about:blank. Page info never includes a form's value. Screenshots are files. There is no tool that runs JavaScript in the page. It is not a sandbox: the page runs as the user, in an ephemeral profile. +- **A note left on the page names only what the page shows lich** (`internal/browser/notes.go`, + `native/cmd/lich-native/browser_notes.go`): the pick asks the page for the element under the pointer, so an + element inside an iframe is picked as the iframe and one inside a closed shadow root as its host, and a + selector picked inside an open shadow root finds it again only from that root, where `browser_click` does + not look. The React component comes from the page's fiber, there only in a development build, and its file + and line from `_debugSource`, which React 19 dropped. Pins follow their elements by looking them up again + every 400 ms while notes wait, and every 30 ms for a moment after the wheel turns, so an element the page + moves on its own is outlined late. Notes not sent yet + live in the window: closing it, or the browser, loses them. ## Languages diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index 007d054a4..fe5c216c8 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -41,6 +41,10 @@ type fakePage struct { texts []string settles int selection string + element api.BrowserElement + pointedAt [2]float64 + boxes map[string]api.BrowserRect + located [][]string info PageInfo shotErr error navErr error @@ -307,3 +311,21 @@ func (f *fakeLauncher) launch(dir string, headed bool) (Page, error) { f.dirs = append(f.dirs, dir) return p, nil } + +func (p *fakePage) ElementAt(_ context.Context, x, y float64) (api.BrowserElement, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.pointedAt = [2]float64{x, y} + return p.element, nil +} + +func (p *fakePage) Locate(_ context.Context, selectors []string) ([]api.BrowserRect, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.located = append(p.located, selectors) + out := make([]api.BrowserRect, len(selectors)) + for i, s := range selectors { + out[i] = p.boxes[s] + } + return out, nil +} diff --git a/internal/browser/notes.go b/internal/browser/notes.go new file mode 100644 index 000000000..a4ff62eda --- /dev/null +++ b/internal/browser/notes.go @@ -0,0 +1,129 @@ +package browser + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/chromedp/chromedp" + "github.com/lichdotdev/lich/api" +) + +// The pane's notes: the person points at an element of the page and leaves a +// note on it for the agent. The page only reports what is where; the pane +// draws the outline and the pins, so nothing is injected into its DOM. + +// elementScript reads the element under a point: a selector that finds it +// again (an id, or up to two classes that are not a CSS-in-JS hash, with +// :nth-of-type only where a level is not yet unique), a label, the React +// component that drew it when the page is a development build, and its +// markup with every field's value taken out. A point on the page's root +// answers null. +const elementScript = `((x, y) => { + const cut = (s, n) => s.length > n ? s.slice(0, n - 1) + '…' : s; + let el = document.elementFromPoint(x, y); + while (el && el.shadowRoot) { + const inner = el.shadowRoot.elementFromPoint(x, y); + if (!inner || inner === el) break; + el = inner; + } + if (!el || el === document.documentElement || el === document.body) return null; + const word = /^[a-zA-Z][\w-]*$/; + const stable = c => word.test(c) && !/^(css|sc|jsx|emotion|svelte)-|[0-9a-f]{5,}|__/.test(c); + const part = e => { + const tag = e.tagName.toLowerCase(); + if (e.id && word.test(e.id)) return tag + '#' + e.id; + return tag + [...e.classList].filter(stable).slice(0, 2).map(c => '.' + c).join(''); + }; + const root = el.getRootNode(); + const unique = s => { try { return root.querySelectorAll(s).length === 1; } catch { return false; } }; + let selector = ''; + for (let e = el, depth = 0; e && e.nodeType === 1 && depth < 6; e = e.parentElement, depth++) { + const p = part(e); + const join = q => selector ? q + ' > ' + selector : q; + if (unique(join(p))) { selector = join(p); break; } + const same = e.parentElement ? [...e.parentElement.children].filter(c => c.tagName === e.tagName) : []; + const q = same.length > 1 && !p.includes('#') ? p + ':nth-of-type(' + (same.indexOf(e) + 1) + ')' : p; + selector = join(q); + if (unique(selector)) break; + } + const text = (el.getAttribute('aria-label') || el.innerText || el.getAttribute('placeholder') || + el.getAttribute('alt') || el.getAttribute('title') || '').trim().split('\n')[0]; + const label = part(el) + (text ? ' "' + cut(text, 32) + '"' : ''); + let component = '', source = ''; + const fiber = Object.keys(el).find(k => k.startsWith('__reactFiber$')); + for (let f = fiber ? el[fiber] : null; f; f = f.return) { + if (!source && f._debugSource) { + source = f._debugSource.fileName.replace(/^.*?\/((src|app|pages|components|lib)\/)/, '$1') + ':' + f._debugSource.lineNumber; + } + const t = f.type; + if (!t || typeof t === 'string') continue; + const inner = t.render || t.type || t; + const name = t.displayName || inner.displayName || inner.name || ''; + if (name && !/Provider|Context|Router|Boundary|^Anonymous$/.test(name)) { component = name; break; } + } + const copy = el.cloneNode(true); + for (const n of [copy, ...copy.querySelectorAll('*')]) { + if (n.matches('input, textarea, select, option')) { n.removeAttribute('value'); n.textContent = ''; } + if (n !== copy && n.matches('script, style')) n.remove(); + } + const r = el.getBoundingClientRect(); + return { + label, selector, component, source, + html: cut(copy.outerHTML.replace(/\s+/g, ' '), 600), + url: location.href.split(/[?#]/)[0], + box: {x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height)}, + }; +})` + +// locateScript is the viewport box of the element each selector finds, an +// empty one where it finds none. +const locateScript = `(selectors => selectors.map(s => { + let e = null; + try { e = document.querySelector(s); } catch {} + if (!e) return {x: 0, y: 0, w: 0, h: 0}; + const r = e.getBoundingClientRect(); + return {x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height)}; +}))` + +func (p *chromePage) ElementAt(ctx context.Context, x, y float64) (api.BrowserElement, error) { + var el *api.BrowserElement + if err := p.run(ctx, chromedp.Evaluate(fmt.Sprintf("%s(%g, %g)", elementScript, x, y), &el)); err != nil { + return api.BrowserElement{}, err + } + if el == nil { + return api.BrowserElement{}, nil + } + return *el, nil +} + +func (p *chromePage) Locate(ctx context.Context, selectors []string) ([]api.BrowserRect, error) { + arg, err := json.Marshal(selectors) + if err != nil { + return nil, err + } + boxes := []api.BrowserRect{} + return boxes, p.run(ctx, chromedp.Evaluate(fmt.Sprintf("%s(%s)", locateScript, arg), &boxes)) +} + +// ElementAtPoint is the element under a point of this session's page. +func (s *Service) ElementAtPoint(_ context.Context, opts api.BrowserPointOptions) (api.BrowserElement, error) { + var el api.BrowserElement + _, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error { + var err error + el, err = p.ElementAt(ctx, opts.X, opts.Y) + return err + }) + return el, err +} + +// LocateElements is where the elements opts.Selectors find stand now. +func (s *Service) LocateElements(_ context.Context, opts api.BrowserLocateOptions) ([]api.BrowserRect, error) { + var boxes []api.BrowserRect + _, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error { + var err error + boxes, err = p.Locate(ctx, opts.Selectors) + return err + }) + return boxes, err +} diff --git a/internal/browser/notes_live_test.go b/internal/browser/notes_live_test.go new file mode 100644 index 000000000..a327942b7 --- /dev/null +++ b/internal/browser/notes_live_test.go @@ -0,0 +1,125 @@ +package browser + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/lichdotdev/lich/api" +) + +const notesProbe = ` +
+
Hobby
+
Pro
+
Team
+
+
+ + +
` + +func notesPage(t *testing.T) (*chromePage, context.Context) { + t.Helper() + p, _ := liveChrome(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, notesProbe) + })) + t.Cleanup(srv.Close) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + t.Cleanup(cancel) + if err := p.Navigate(ctx, srv.URL+"/pricing?plan=pro#top"); err != nil { + t.Fatal(err) + } + if err := p.Resize(ctx, Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + return p, ctx +} + +func boxOf(t *testing.T, p *chromePage, ctx context.Context, selector string) api.BrowserRect { + t.Helper() + boxes, err := p.Locate(ctx, []string{selector}) + if err != nil { + t.Fatal(err) + } + return boxes[0] +} + +func TestChromeElementAtNamesTheElementAndASelectorThatFindsItAgain(t *testing.T) { + p, ctx := notesPage(t) + team := boxOf(t, p, ctx, ".plan:nth-of-type(3) .cta") + el, err := p.ElementAt(ctx, float64(team.X+team.W/2), float64(team.Y+team.H/2)) + if err != nil { + t.Fatal(err) + } + if el.Label != `button.cta "Contact sales"` { + t.Errorf("label %q", el.Label) + } + if got := boxOf(t, p, ctx, el.Selector); got != team || got.W == 0 { + t.Errorf("selector %q finds %+v, want the Team button at %+v", el.Selector, got, team) + } + if el.Box != team { + t.Errorf("box %+v, want %+v", el.Box, team) + } + if strings.ContainsAny(el.URL, "?#") || !strings.HasSuffix(el.URL, "/pricing") { + t.Errorf("url %q keeps its query or fragment", el.URL) + } +} + +func TestChromeElementAtSkipsAGeneratedClassName(t *testing.T) { + p, ctx := notesPage(t) + hobby := boxOf(t, p, ctx, ".plan b") + el, err := p.ElementAt(ctx, float64(hobby.X+1), float64(hobby.Y+1)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(el.Selector, "css-") { + t.Fatalf("selector %q names a CSS-in-JS class", el.Selector) + } +} + +func TestChromeElementAtKeepsFieldValuesOutOfTheMarkup(t *testing.T) { + p, ctx := notesPage(t) + if err := p.Type(ctx, "my secret query", false, Target{Selector: "#q"}); err != nil { + t.Fatal(err) + } + q := boxOf(t, p, ctx, "#q") + el, err := p.ElementAt(ctx, float64(q.X+q.W+4), float64(q.Y+q.H/2)) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(el.Selector, "form#login") { + t.Fatalf("picked %q, want the form", el.Selector) + } + for _, secret := range []string{"hunter2", "my secret query"} { + if strings.Contains(el.HTML, secret) { + t.Errorf("markup carries %q: %s", secret, el.HTML) + } + } +} + +func TestChromeElementAtThePageRootAnswersNothing(t *testing.T) { + p, ctx := notesPage(t) + el, err := p.ElementAt(ctx, 790, 590) + if err != nil { + t.Fatal(err) + } + if el.Label != "" { + t.Fatalf("empty page area picked %+v", el) + } +} + +func TestChromeLocateAnswersAnEmptyBoxForAnElementGone(t *testing.T) { + p, ctx := notesPage(t) + if got := boxOf(t, p, ctx, "#gone"); got != (api.BrowserRect{}) { + t.Fatalf("box of a missing element = %+v", got) + } + if got := boxOf(t, p, ctx, "::bad"); got != (api.BrowserRect{}) { + t.Fatalf("box of an invalid selector = %+v", got) + } +} diff --git a/internal/browser/notes_test.go b/internal/browser/notes_test.go new file mode 100644 index 000000000..b9d6f37bc --- /dev/null +++ b/internal/browser/notes_test.go @@ -0,0 +1,47 @@ +package browser + +import ( + "context" + "testing" + + "github.com/lichdotdev/lich/api" +) + +func TestElementAtAsksThePageAtThePoint(t *testing.T) { + s, launch, _ := viewService(t) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + page := launch.pages[0] + page.element = api.BrowserElement{Label: `button "Buy"`, Selector: "button.buy"} + el, err := s.ElementAtPoint(context.Background(), api.BrowserPointOptions{ID: "s", X: 12, Y: 34}) + if err != nil { + t.Fatal(err) + } + if el.Selector != "button.buy" || page.pointedAt != [2]float64{12, 34} { + t.Fatalf("element %+v asked at %v, want button.buy at 12,34", el, page.pointedAt) + } +} + +func TestLocateAnswersABoxPerSelectorAndEmptyForOneGone(t *testing.T) { + s, launch, _ := viewService(t) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + launch.pages[0].boxes = map[string]api.BrowserRect{"h1": {X: 1, Y: 2, W: 30, H: 40}} + boxes, err := s.LocateElements(context.Background(), api.BrowserLocateOptions{ID: "s", Selectors: []string{"h1", "#gone"}}) + if err != nil { + t.Fatal(err) + } + want := []api.BrowserRect{{X: 1, Y: 2, W: 30, H: 40}, {}} + if len(boxes) != 2 || boxes[0] != want[0] || boxes[1] != want[1] { + t.Fatalf("boxes = %+v, want %+v", boxes, want) + } +} + +func TestElementAtWithoutABrowserFails(t *testing.T) { + s, _, _ := viewService(t) + if _, err := s.ElementAtPoint(context.Background(), api.BrowserPointOptions{ID: "s"}); err == nil { + t.Fatal("ElementAt on a session with no browser answered") + } +} diff --git a/internal/browser/page.go b/internal/browser/page.go index ab1e6a3dc..949db2fd9 100644 --- a/internal/browser/page.go +++ b/internal/browser/page.go @@ -109,6 +109,10 @@ type Page interface { InsertText(ctx context.Context, text string) error // Selection is the text selected in the page, "" in a password field. Selection(ctx context.Context) (string, error) + // ElementAt is the element under x,y, zero when the point holds none; + // Locate is the box of the element each selector finds, zero for none. + ElementAt(ctx context.Context, x, y float64) (api.BrowserElement, error) + Locate(ctx context.Context, selectors []string) ([]api.BrowserRect, error) Alive() bool Close() error } diff --git a/internal/routes/browser.go b/internal/routes/browser.go index 7608c2d5a..560386177 100644 --- a/internal/routes/browser.go +++ b/internal/routes/browser.go @@ -33,4 +33,6 @@ func Browser(d *rpc.Handler, s *browser.Service) { rpc.Handle(d, api.BrowserText, s.TextInput) rpc.Handle(d, api.BrowserNav, s.NavView) rpc.Handle(d, api.BrowserSelection, s.SelectedText) + rpc.Handle(d, api.BrowserElementAt, s.ElementAtPoint) + rpc.Handle(d, api.BrowserLocate, s.LocateElements) } diff --git a/native/backend/browser.go b/native/backend/browser.go index 623689bc7..ab1e94eed 100644 --- a/native/backend/browser.go +++ b/native/backend/browser.go @@ -71,3 +71,14 @@ func (c *Client) BrowserNav(ctx context.Context, id, action, url string) error { _, err := callMethod(ctx, c, api.BrowserNav, api.BrowserNavOptions{ID: id, Action: action, URL: url}) return err } + +// BrowserElementAt is the element under x,y of session id's page. +func (c *Client) BrowserElementAt(ctx context.Context, id string, x, y float64) (api.BrowserElement, error) { + return callMethod(ctx, c, api.BrowserElementAt, api.BrowserPointOptions{ID: id, X: x, Y: y}) +} + +// BrowserLocate is where the elements selectors find stand in session id's +// page, an empty box for one gone. +func (c *Client) BrowserLocate(ctx context.Context, id string, selectors []string) ([]api.BrowserRect, error) { + return callMethod(ctx, c, api.BrowserLocate, api.BrowserLocateOptions{ID: id, Selectors: selectors}) +} diff --git a/native/cmd/lich-native/browser_notes.go b/native/cmd/lich-native/browser_notes.go new file mode 100644 index 000000000..631cbfaac --- /dev/null +++ b/native/cmd/lich-native/browser_notes.go @@ -0,0 +1,368 @@ +package main + +import ( + "context" + "fmt" + "image" + "slices" + "time" + + "gioui.org/io/key" + "gioui.org/io/pointer" + "gioui.org/layout" + "gioui.org/op" + "gioui.org/widget" + + "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/native/backend" + "github.com/lichdotdev/lich/native/i18n" + "github.com/lichdotdev/lich/native/ui" + "github.com/lichdotdev/lich/native/ui/icons" +) + +// Notes on the page: with the pane's pick on, the pointer stops reaching the +// page; the element under it is outlined and a click opens a note on it. +// Notes collect in a tray under the page, each element pinned where it now +// stands, and reach the session's agent as one message. The page only reports +// what is where: the outline, the pins and the note are the pane's. + +var ( + icBrowserPick = icons.Lucide("crosshair") + icNoteRemove = icons.Lucide("x") +) + +// noteRelocate is how often noted elements are looked up again while their +// pins show: a page scrolls, reflows and animates with no event the pane +// hears. After the person turns the wheel they are looked up every +// noteFollow for noteFollowFor, the length of the page's smooth scroll and +// then some: at the slow pace the outlines trailed the page by a step. +const ( + noteRelocate = 400 * time.Millisecond + noteFollow = 30 * time.Millisecond + noteFollowFor = 700 * time.Millisecond +) + +// browserNote is one note: the element as it was picked, what the person +// wrote, and where the element stands now (empty when it left the page). +type browserNote struct { + el api.BrowserElement + text string + box api.BrowserRect + row, remove widget.Clickable + pin widget.Clickable +} + +// browserNotes is one pane's pick and its notes. +type browserNotes struct { + picking bool + pick iconActionState + // hover is the element under the pointer while picking. + hover api.BrowserElement + // draft is the element the note popover is open on; editing is the note + // it edits, -1 for a new one. + draft api.BrowserElement + editing int + open bool + focusEditor bool + // openedAt is when the popover opened, for its animate-in. + openedAt time.Time + // trayShown is when the tray appeared, for its animate-in. + trayShown time.Time + editor widget.Editor + add, cancel, del widget.Clickable + clear, send widget.Clickable + list []*browserNote + rows layout.List + found chan foundElement + located chan locatedNotes + sent chan []*browserNote + nextLocate time.Time + // followUntil is when the fast lookups after a scroll end. + followUntil time.Time +} + +// foundElement is the element a probe found, and whether a click asked for +// it, which opens a note on it. +type foundElement struct { + el api.BrowserElement + open bool +} + +// locatedNotes is where the elements selectors find stand; with draft set, +// the last one is the element a new note is open on. +type locatedNotes struct { + selectors []string + boxes []api.BrowserRect + draft bool +} + +func newBrowserNotes() browserNotes { + n := browserNotes{found: make(chan foundElement, 4), located: make(chan locatedNotes, 1), sent: make(chan []*browserNote, 4)} + n.rows.Axis = layout.Vertical + return n +} + +// offer hands v to the frame, dropping the oldest value when the frame has not +// taken the last ones yet: only the newest matters. +func offer[T any](ch chan T, v T) { + for { + select { + case ch <- v: + return + default: + } + select { + case <-ch: + default: + } + } +} + +// setPicking turns the pick on or off; off drops the outline and an open +// note, keeping the notes. +func (n *browserNotes) setPicking(on bool) { + n.picking = on + n.hover = api.BrowserElement{} + if !on { + n.open = false + } +} + +// openNote opens the popover on el, for note i or a new one (-1). +func (n *browserNotes) openNote(el api.BrowserElement, i int) { + if !n.open { + n.openedAt = time.Now() + } + n.draft, n.editing, n.open, n.focusEditor = el, i, true, true + n.hover = api.BrowserElement{} + text := "" + if i >= 0 { + text = n.list[i].text + } + n.editor.SetText(text) + n.editor.SetCaret(len([]rune(text)), len([]rune(text))) +} + +// draftBox is where the element the popover is open on stands now. +func (n *browserNotes) draftBox() api.BrowserRect { + if n.editing >= 0 && n.editing < len(n.list) { + return n.list[n.editing].box + } + return n.draft.Box +} + +// commit keeps the popover's note, replacing the one it edits. +func (n *browserNotes) commit() { + text := n.editor.Text() + if n.editing >= 0 && n.editing < len(n.list) { + n.list[n.editing].text = text + } else { + n.list = append(n.list, &browserNote{el: n.draft, text: text, box: n.draft.Box}) + } + n.open = false +} + +// drop removes note i. +func (n *browserNotes) drop(i int) { + n.list = append(n.list[:i], n.list[i+1:]...) + n.open = false +} + +// probeElement asks for the element under x,y of the page; open asks for the +// note popover on it. Hover probes queued behind one in flight merge. +func (u *window) probeElement(p *browserPaneUI, session string, x, y float64, open bool) { + merge := "hover" + if open { + merge = "" + } + p.queue.push(browserCall{merge: merge, send: func(ctx context.Context) error { + el, err := u.client.BrowserElementAt(ctx, session, x, y) + if err != nil { + return fmt.Errorf("element at: %w", err) + } + offer(p.notes.found, foundElement{el: el, open: open}) + u.model.invalidate() + return nil + }}) +} + +// relocateNotes looks the noted elements up again, and the one a new note +// is open on. +func (u *window) relocateNotes(p *browserPaneUI, session string) { + selectors := make([]string, 0, len(p.notes.list)+1) + for _, n := range p.notes.list { + selectors = append(selectors, n.el.Selector) + } + draft := p.notes.open && p.notes.editing < 0 + if draft { + selectors = append(selectors, p.notes.draft.Selector) + } + p.queue.push(browserCall{merge: "locate", send: func(ctx context.Context) error { + boxes, err := u.client.BrowserLocate(ctx, session, selectors) + if err != nil { + return fmt.Errorf("locate notes: %w", err) + } + offer(p.notes.located, locatedNotes{selectors: selectors, boxes: boxes, draft: draft}) + u.model.invalidate() + return nil + }}) +} + +// takeFound applies what the probes and lookups answered since last frame, +// and looks the noted elements up again when it is time. +func (u *window) takeFound(gtx C, p *browserPaneUI, session string) { + n := &p.notes + for done := false; !done; { + select { + case f := <-n.found: + switch { + case !n.picking || n.open: + case f.open && f.el.Label != "": + n.openNote(f.el, -1) + case !f.open: + n.hover = f.el + } + case l := <-n.located: + n.applyBoxes(l) + case sent := <-n.sent: + n.delivered(sent) + default: + done = true + } + } + if len(n.list) == 0 && !n.open { + return + } + if !gtx.Now.Before(n.nextLocate) { + every := noteRelocate + if gtx.Now.Before(n.followUntil) { + every = noteFollow + } + n.nextLocate = gtx.Now.Add(every) + u.relocateNotes(p, session) + } + gtx.Execute(op.InvalidateCmd{At: n.nextLocate}) +} + +// applyBoxes moves each pin to where its element stands, unless the notes +// changed since the lookup left. +func (n *browserNotes) applyBoxes(l locatedNotes) { + if l.draft && len(l.selectors) > 0 && len(l.boxes) == len(l.selectors) { + last := len(l.selectors) - 1 + if n.open && n.editing < 0 && n.draft.Selector == l.selectors[last] { + n.draft.Box = l.boxes[last] + } + l.selectors, l.boxes = l.selectors[:last], l.boxes[:last] + } + if len(l.selectors) != len(n.list) || len(l.boxes) != len(n.list) { + return + } + for i, note := range n.list { + if note.el.Selector != l.selectors[i] { + return + } + } + for i, note := range n.list { + note.box = l.boxes[i] + } +} + +// notesPointer handles the pointer while picking: a move outlines the element +// under it, a press opens a note on it. Nothing reaches the page. +func (u *window) notesPointer(gtx C, p *browserPaneUI, session string, x, y float64, e pointer.Event) { + n := &p.notes + switch e.Kind { + case pointer.Press: + gtx.Execute(key.FocusCmd{Tag: &p.tag}) + u.layoutStage.focus = browserPaneOf(session) + if !n.open && e.Buttons.Contain(pointer.ButtonPrimary) { + u.probeElement(p, session, x, y, true) + } + case pointer.Move, pointer.Drag: + if !n.open { + u.probeElement(p, session, x, y, false) + } + case pointer.Leave: + n.hover = api.BrowserElement{} + } +} + +// notesKey takes the keys of the pick from the page: Ctrl+Shift+E turns it +// on and off, Esc closes the note or stops the pick, and Ctrl+Enter sends +// the notes. It reports whether it took e. +func (u *window) notesKey(p *browserPaneUI, session string, e key.Event) bool { + if e.State != key.Press { + return false + } + n := &p.notes + switch { + case e.Name == "E" && e.Modifiers == key.ModCtrl|key.ModShift: + n.setPicking(!n.picking) + case e.Name == key.NameEscape && n.picking: + if n.open { + n.open = false + } else { + n.setPicking(false) + } + case (e.Name == key.NameReturn || e.Name == key.NameEnter) && e.Modifiers == key.ModCtrl && len(n.list) > 0: + u.sendNotes(p, session) + default: + return false + } + return true +} + +// sendNotes types the notes into the session's prompt as one message. They +// leave the tray only once it is there: a send that fails keeps them, to send +// again. +func (u *window) sendNotes(p *browserPaneUI, session string) { + n := &p.notes + notes := n.sending() + message := notesMessage(notes) + n.setPicking(false) + label := u.model.sessionLabel(session) + go func() { + if err := u.client.Submit(u.ctx, session, message); err != nil { + u.toaster.Show(ui.Toast{Variant: ui.ToastError, Title: i18n.T("layouts.browser.notesFailed", i18n.Args{"label": label, "error": backend.ErrorText(err)})}) + } else { + offer(n.sent, notes) + } + u.model.invalidate() + }() +} + +// sending is the notes a send takes: those in the tray now. +func (n *browserNotes) sending() []*browserNote { + return append([]*browserNote(nil), n.list...) +} + +// delivered takes the notes of a sent message out of the tray, keeping any +// added while it was sent. +func (n *browserNotes) delivered(sent []*browserNote) { + n.list = slices.DeleteFunc(n.list, func(note *browserNote) bool { return slices.Contains(sent, note) }) + if n.open && n.editing >= len(n.list) { + n.open = false + } +} + +// noteVisible is whether note's element stands on the page the pane shows: +// a pin of a page navigated away from is not drawn. +func noteVisible(note *browserNote, pageURL string) bool { + return note.box.W > 0 && note.box.H > 0 && note.el.URL == pageURLOf(pageURL) +} + +// pageURLOf is url without its query and fragment, as a note records it. +func pageURLOf(url string) string { + for i, c := range url { + if c == '?' || c == '#' { + return url[:i] + } + } + return url +} + +// cssRect is box, in CSS pixels, as pane pixels at scale. +func cssRect(box api.BrowserRect, scale float32) image.Rectangle { + px := func(v int) int { return int(float32(v) * scale) } + return image.Rect(px(box.X), px(box.Y), px(box.X+box.W), px(box.Y+box.H)) +} diff --git a/native/cmd/lich-native/browser_notes_prompt.go b/native/cmd/lich-native/browser_notes_prompt.go new file mode 100644 index 000000000..cf835b223 --- /dev/null +++ b/native/cmd/lich-native/browser_notes_prompt.go @@ -0,0 +1,33 @@ +package main + +import ( + "strconv" + "strings" + + "github.com/lichdotdev/lich/native/i18n" +) + +// notesMessage is the notes as the agent reads them: each element's label and +// page, the selector browser_click takes, the React component that drew it, +// its markup, and the person's note. +func notesMessage(notes []*browserNote) string { + lines := []string{i18n.Prompt("prompts.browserNotes.head", i18n.Args{"count": len(notes)})} + for i, n := range notes { + el := n.el + lines = append(lines, i18n.Prompt("prompts.browserNotes.item", i18n.Args{"index": strconv.Itoa(i + 1), "label": el.Label, "url": el.URL}), + " "+el.Selector) + if el.Component != "" { + react := " React: " + el.Component + if el.Source != "" { + react += " (" + el.Source + ")" + } + lines = append(lines, react) + } + lines = append(lines, " "+el.HTML) + if text := strings.TrimSpace(n.text); text != "" { + lines = append(lines, " "+i18n.Prompt("prompts.browserNotes.note", i18n.Args{"note": strings.ReplaceAll(text, "\n", " ")})) + } + } + lines = append(lines, i18n.Prompt("prompts.browserNotes.selectors")) + return strings.Join(lines, "\n") +} diff --git a/native/cmd/lich-native/browser_notes_ring.go b/native/cmd/lich-native/browser_notes_ring.go new file mode 100644 index 000000000..e06785b17 --- /dev/null +++ b/native/cmd/lich-native/browser_notes_ring.go @@ -0,0 +1,137 @@ +package main + +import ( + "image" + "image/color" + "math" + "time" + + "gioui.org/f32" + "gioui.org/op" + "gioui.org/op/clip" + "gioui.org/op/paint" + "gioui.org/unit" +) + +// A pick's outline is a dashed ring a few dp outside the element, so it never +// merges with the element's own border, its dashes marching while the pointer +// is on the element or its note is in focus. + +var ( + noteRingGap = unit.Dp(5) + noteRingRadius = unit.Dp(8) + noteDash = unit.Dp(5) + // noteMarch is how long the dashes take to move one dash and gap along. + noteMarch = 600 * time.Millisecond +) + +// noteRing is the ring around the element at r. +func noteRing(gtx C, r image.Rectangle) image.Rectangle { + return r.Inset(-gtx.Dp(noteRingGap)) +} + +// ringFill veils the ring's inside. +func ringFill(gtx C, ring image.Rectangle, c color.NRGBA) { + paint.FillShape(gtx.Ops, c, clip.UniformRRect(ring, gtx.Dp(noteRingRadius)).Op(gtx.Ops)) +} + +// dashedRing draws ring's dashes in c, over a white halo so they read on a +// dark page too; marching moves them with the clock. +func dashedRing(gtx C, ring image.Rectangle, c color.NRGBA, width unit.Dp, marching bool) { + dash := float32(gtx.Dp(noteDash)) + phase := float32(0) + if marching { + cycle := gtx.Now.UnixNano() % int64(noteMarch) + phase = 2 * dash * float32(cycle) / float32(noteMarch) + gtx.Execute(op.InvalidateCmd{}) + } + rr := roundedRect{min: f32.Pt(float32(ring.Min.X), float32(ring.Min.Y)), max: f32.Pt(float32(ring.Max.X), float32(ring.Max.Y)), radius: float32(gtx.Dp(noteRingRadius))} + w := float32(gtx.Dp(width)) + // One stroke a dash: Gio's stroker bridges the subpaths of one path, which + // drew lines across the ring from dash to dash. + rr.eachDash(dash, phase, func(from, to float32) { + paint.FillShape(gtx.Ops, noteHalo, clip.Stroke{Path: rr.span(gtx.Ops, from, to), Width: w + float32(gtx.Dp(1))}.Op()) + }) + rr.eachDash(dash, phase, func(from, to float32) { + paint.FillShape(gtx.Ops, c, clip.Stroke{Path: rr.span(gtx.Ops, from, to), Width: w}.Op()) + }) +} + +// roundedRect is a rectangle with round corners, walked clockwise from the +// end of its top left corner. +type roundedRect struct { + min, max f32.Point + radius float32 +} + +func (r roundedRect) dx() float32 { return r.max.X - r.min.X } +func (r roundedRect) dy() float32 { return r.max.Y - r.min.Y } + +func (r roundedRect) clamped() float32 { + return max(min(r.radius, r.dx()/2, r.dy()/2), 0) +} + +// perimeter is the length of its outline. +func (r roundedRect) perimeter() float32 { + rad := r.clamped() + return 2*(r.dx()-2*rad) + 2*(r.dy()-2*rad) + 2*math.Pi*rad +} + +// at is the point t along the outline, wrapping past its perimeter. +func (r roundedRect) at(t float32) f32.Point { + rad := r.clamped() + w, h := r.dx()-2*rad, r.dy()-2*rad + arc := math.Pi / 2 * rad + t = float32(math.Mod(float64(t), float64(r.perimeter()))) + if t < 0 { + t += r.perimeter() + } + x0, y0, x1, y1 := r.min.X, r.min.Y, r.max.X, r.max.Y + corner := func(cx, cy, from float32, s float32) f32.Point { + a := float64(from) + float64(s/max(rad, 1e-3)) + return f32.Pt(cx+rad*float32(math.Cos(a)), cy+rad*float32(math.Sin(a))) + } + switch { + case t < w: + return f32.Pt(x0+rad+t, y0) + case t < w+arc: + return corner(x1-rad, y0+rad, -math.Pi/2, t-w) + case t < w+arc+h: + return f32.Pt(x1, y0+rad+t-w-arc) + case t < w+2*arc+h: + return corner(x1-rad, y1-rad, 0, t-w-arc-h) + case t < 2*w+2*arc+h: + return f32.Pt(x1-rad-(t-w-2*arc-h), y1) + case t < 2*w+3*arc+h: + return corner(x0+rad, y1-rad, math.Pi/2, t-2*w-2*arc-h) + case t < 2*w+3*arc+2*h: + return f32.Pt(x0, y1-rad-(t-2*w-3*arc-h)) + } + return corner(x0+rad, y0+rad, math.Pi, t-2*w-3*arc-2*h) +} + +// eachDash calls fn with the start and end of each dash of length dash, with +// gaps as long, shifted along the outline by phase. +func (r roundedRect) eachDash(dash, phase float32, fn func(from, to float32)) { + total := r.perimeter() + if total <= 0 || dash <= 0 { + return + } + for start := phase - 2*dash; start < total; start += 2 * dash { + if from, to := max(start, 0), min(start+dash, total); to > from { + fn(from, to) + } + } +} + +// span is the outline from from to to, as a path to stroke. +func (r roundedRect) span(ops *op.Ops, from, to float32) clip.PathSpec { + var p clip.Path + p.Begin(ops) + p.MoveTo(r.at(from)) + for s := from + 2; s < to; s += 2 { + p.LineTo(r.at(s)) + } + p.LineTo(r.at(to)) + return p.End() +} diff --git a/native/cmd/lich-native/browser_notes_test.go b/native/cmd/lich-native/browser_notes_test.go new file mode 100644 index 000000000..09cd67989 --- /dev/null +++ b/native/cmd/lich-native/browser_notes_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "math" + "testing" + + "gioui.org/f32" + + "github.com/lichdotdev/lich/api" +) + +func TestNotesMessageNamesEachElementItsSelectorAndTheNote(t *testing.T) { + notes := []*browserNote{ + {el: api.BrowserElement{Label: `h1 "Simple pricing"`, Selector: "section.hero > h1", Component: "Hero", Source: "src/Hero.tsx:14", HTML: "

Simple pricing

", URL: "http://localhost:5173/pricing"}, text: "tighter,\nit wraps on mobile"}, + {el: api.BrowserElement{Label: `button "Buy"`, Selector: "button.buy", HTML: "", URL: "http://localhost:5173/pricing"}}, + } + got := notesMessage(notes) + want := `[lich] The person left 2 notes on the page in your browser tab: +1. h1 "Simple pricing" on http://localhost:5173/pricing + section.hero > h1 + React: Hero (src/Hero.tsx:14) +

Simple pricing

+ Note: tighter, it wraps on mobile +2. button "Buy" on http://localhost:5173/pricing + button.buy + +Each selector above works as the selector of browser_click and browser_type.` + if got != want { + t.Fatalf("message:\n%s\nwant:\n%s", got, want) + } +} + +func TestPinsHideOnAnotherPageOrOnceTheElementLeft(t *testing.T) { + note := &browserNote{el: api.BrowserElement{URL: "http://localhost:5173/pricing"}, box: api.BrowserRect{X: 1, Y: 1, W: 10, H: 10}} + if !noteVisible(note, "http://localhost:5173/pricing?plan=pro#top") { + t.Fatal("pin hidden on its own page with a query") + } + if noteVisible(note, "http://localhost:5173/docs") { + t.Fatal("pin shown on another page") + } + note.box = api.BrowserRect{} + if noteVisible(note, "http://localhost:5173/pricing") { + t.Fatal("pin shown for an element gone from the page") + } +} + +func TestNotesKeepEditsAndDrops(t *testing.T) { + n := newBrowserNotes() + n.setPicking(true) + n.openNote(api.BrowserElement{Label: "a", Selector: "a"}, -1) + n.editor.SetText("first") + n.commit() + n.openNote(api.BrowserElement{Label: "b", Selector: "b"}, -1) + n.commit() + n.openNote(n.list[0].el, 0) + if n.editor.Text() != "first" { + t.Fatalf("editing note 1 opened with %q", n.editor.Text()) + } + n.editor.SetText("changed") + n.commit() + if len(n.list) != 2 || n.list[0].text != "changed" { + t.Fatalf("notes after an edit: %+v", n.list) + } + n.drop(0) + if len(n.list) != 1 || n.list[0].el.Selector != "b" || n.open { + t.Fatalf("notes after dropping the first: %d, open %v", len(n.list), n.open) + } + n.setPicking(false) + if len(n.list) != 1 { + t.Fatal("stopping the pick dropped the notes") + } +} + +func TestBoxesFromAStaleLookupAreIgnored(t *testing.T) { + n := newBrowserNotes() + n.list = []*browserNote{{el: api.BrowserElement{Selector: "a"}}, {el: api.BrowserElement{Selector: "b"}}} + n.applyBoxes(locatedNotes{selectors: []string{"a"}, boxes: []api.BrowserRect{{W: 5, H: 5}}}) + if n.list[0].box.W != 0 { + t.Fatal("a lookup made before a note was added moved the pins") + } + n.applyBoxes(locatedNotes{selectors: []string{"a", "b"}, boxes: []api.BrowserRect{{W: 5, H: 5}, {W: 7, H: 7}}}) + if n.list[1].box.W != 7 { + t.Fatalf("pins not moved: %+v", n.list[1].box) + } +} + +func TestNotesStayUntilTheirMessageWasSent(t *testing.T) { + n := newBrowserNotes() + a, b := &browserNote{el: api.BrowserElement{Selector: "a"}}, &browserNote{el: api.BrowserElement{Selector: "b"}} + n.list = []*browserNote{a, b} + sending := n.sending() + if len(n.list) != 2 { + t.Fatal("notes left the tray before their message was sent") + } + c := &browserNote{el: api.BrowserElement{Selector: "c"}} + n.list = append(n.list, c) + n.delivered(sending) + if len(n.list) != 1 || n.list[0] != c { + t.Fatalf("after the send the tray holds %d notes, want only the one added meanwhile", len(n.list)) + } +} + +func TestAnOpenNoteFollowsItsElementAsThePageScrolls(t *testing.T) { + n := newBrowserNotes() + n.openNote(api.BrowserElement{Selector: "section.chart", Box: api.BrowserRect{Y: 400, W: 600, H: 300}}, -1) + n.applyBoxes(locatedNotes{selectors: []string{"section.chart"}, boxes: []api.BrowserRect{{Y: 250, W: 600, H: 300}}, draft: true}) + if got := n.draftBox(); got.Y != 250 { + t.Fatalf("the open note stayed at y %d after the page scrolled its element to 250", got.Y) + } +} + +func TestTheRingsOutlineWalksItsSidesAndCorners(t *testing.T) { + r := roundedRect{max: f32.Pt(100, 50), radius: 10} + if got, want := r.perimeter(), float32(2*80+2*30+2*math.Pi*10); math.Abs(float64(got-want)) > 1e-3 { + t.Fatalf("perimeter %v, want %v", got, want) + } + near := func(p, q f32.Point) bool { return math.Hypot(float64(p.X-q.X), float64(p.Y-q.Y)) < 1e-2 } + arc := float32(math.Pi / 2 * 10) + cases := []struct { + t float32 + want f32.Point + }{ + {0, f32.Pt(10, 0)}, + {80, f32.Pt(90, 0)}, + {80 + arc, f32.Pt(100, 10)}, + {80 + arc + 30, f32.Pt(100, 40)}, + {80 + 2*arc + 30, f32.Pt(90, 50)}, + {2*80 + 4*arc + 2*30, f32.Pt(10, 0)}, + } + for _, c := range cases { + if got := r.at(c.t); !near(got, c.want) { + t.Errorf("at(%v) = %v, want %v", c.t, got, c.want) + } + } +} diff --git a/native/cmd/lich-native/browser_notes_view.go b/native/cmd/lich-native/browser_notes_view.go new file mode 100644 index 000000000..e910df670 --- /dev/null +++ b/native/cmd/lich-native/browser_notes_view.go @@ -0,0 +1,481 @@ +package main + +import ( + "fmt" + "image" + "image/color" + "math" + "strconv" + "time" + + "gioui.org/f32" + + "gioui.org/font" + "gioui.org/io/event" + "gioui.org/io/key" + "gioui.org/io/pointer" + "gioui.org/layout" + "gioui.org/op" + "gioui.org/op/clip" + "gioui.org/op/paint" + "gioui.org/unit" + "gioui.org/widget" + + "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/native/i18n" + "github.com/lichdotdev/lich/native/ui" +) + +// notePick is the color of the pick, its outlines and its pins: one hue of its +// own, so a note is never taken for the agent's cursor, which wears the +// card's color or the theme's primary. +var notePick = cardColors["blue"] + +// The tag and the halos sit on the website, not on lich, so they keep one +// look on every page whatever lich's theme: a dark tag, and white under each +// outline so it reads on a dark page too. +var ( + noteTagBg = ui.OKLCH(0.21, 0.006, 285.885, 1) + noteTagText = color.NRGBA{R: 0xfa, G: 0xfa, B: 0xfa, A: 0xff} + noteHalo = color.NRGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xb0} +) + +var ( + notePopoverWidth = unit.Dp(280) + noteRowHeight = ui.Space(6.5) + noteTrayHead = ui.Space(9) + noteTrayPad = ui.Space(2) + // noteRowsShown is how many notes the tray shows before it scrolls. + noteRowsShown = 3 +) + +// notesTrayHeight is the tray's height for its notes; none hides it. The +// tray floats over the page instead of taking room from it: a page that +// shrank under each new note relaid itself out and blinked. +func notesTrayHeight(gtx C, n int) int { + if n == 0 { + return 0 + } + return gtx.Dp(noteTrayHead) + gtx.Dp(noteTrayPad) + min(n, noteRowsShown)*gtx.Dp(noteRowHeight) + 2*gtx.Dp(noteTrayPad) +} + +func withAlpha(c color.NRGBA, a uint8) color.NRGBA { c.A = a; return c } + +// notesOverlay draws, over the page, the outline of the element under the +// pointer with its name, every noted element's outline and pin, and the +// open note. +func (u *window) notesOverlay(gtx C, p *browserPaneUI, session, pageURL string) { + n := &p.notes + u.takeFound(gtx, p, session) + if !n.picking && len(n.list) == 0 { + return + } + defer clip.Rect{Max: gtx.Constraints.Max}.Push(gtx.Ops).Pop() + scale := gtx.Metric.PxPerDp + lit := -1 + for i, note := range n.list { + if note.row.Hovered() || (n.open && n.editing == i) { + lit = i + } + } + for i, note := range n.list { + if !noteVisible(note, pageURL) { + continue + } + ring := noteRing(gtx, cssRect(note.box, scale)) + if i == lit { + ringFill(gtx, ring, withAlpha(notePick, 0x1a)) + dashedRing(gtx, ring, notePick, 1.5, true) + } else { + ringFill(gtx, ring, withAlpha(notePick, 0x0d)) + dashedRing(gtx, ring, notePick, 1.5, false) + } + } + if n.open && n.editing < 0 { + ring := noteRing(gtx, cssRect(n.draftBox(), scale)) + ringFill(gtx, ring, withAlpha(notePick, 0x1a)) + dashedRing(gtx, ring, notePick, 1.5, true) + } + if n.picking && !n.open && n.hover.Label != "" { + ring := noteRing(gtx, cssRect(n.hover.Box, scale)) + ringFill(gtx, ring, withAlpha(notePick, 0x1a)) + dashedRing(gtx, ring, notePick, 1.5, true) + u.hoverTag(gtx, ring, n.hover) + } + for i, note := range n.list { + if r := cssRect(note.box, scale); noteVisible(note, pageURL) && r.Overlaps(image.Rectangle{Max: gtx.Constraints.Max}) { + u.notePin(gtx, n, i, noteRing(gtx, r), i == lit) + } + } + if n.open { + u.notePopover(gtx, p, scale) + } +} + +// hoverTag names the element under the pointer above it, or below when it +// sits at the page's top: its label, component and size, in the inverted +// colors of a tooltip. +func (u *window) hoverTag(gtx C, r image.Rectangle, el api.BrowserElement) { + th := u.theme + macro := op.Record(gtx.Ops) + tag := gtx + tag.Constraints.Min = image.Point{} + d := layout.Inset{Top: ui.Space(1), Bottom: ui.Space(1), Left: ui.Space(1.5), Right: ui.Space(1.5)}.Layout(tag, func(gtx C) D { + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(th.Text(ui.TextXS, el.Label).Mono().In(noteTagText).Layout), + ui.Gap(1.5), + layout.Rigid(th.Text(ui.TextXS, el.Component).In(withAlpha(noteTagText, 0xb3)).Layout), + ui.Gap(1.5), + layout.Rigid(th.Text(ui.TextXS, fmt.Sprintf("%d×%d", el.Box.W, el.Box.H)).In(withAlpha(noteTagText, 0xb3)).Layout), + ) + }) + call := macro.Stop() + gap := gtx.Dp(ui.Space(1)) + at := image.Pt(r.Min.X, r.Min.Y-d.Size.Y-gap) + if at.Y < 0 { + at.Y = r.Max.Y + gap + } + at.X = min(max(at.X, 0), gtx.Constraints.Max.X-d.Size.X) + at.Y = min(max(at.Y, 0), gtx.Constraints.Max.Y-d.Size.Y) + defer op.Offset(at).Push(gtx.Ops).Pop() + paint.FillShape(gtx.Ops, noteTagBg, clip.UniformRRect(image.Rectangle{Max: d.Size}, gtx.Dp(ui.RadiusMD)).Op(gtx.Ops)) + call.Add(gtx.Ops) +} + +// notePin is note i's number as a tab on top of its element's outline, at +// its left corner, the way DevTools labels an element; a click opens its +// note. The tab stays whole inside the page. +func (u *window) notePin(gtx C, n *browserNotes, i int, r image.Rectangle, lit bool) { + note := n.list[i] + if note.pin.Clicked(gtx) { + el := note.el + el.Box = note.box + n.openNote(el, i) + } + macro := op.Record(gtx.Ops) + tab := gtx + tab.Constraints.Min = image.Point{} + d := note.pin.Layout(tab, func(gtx C) D { return u.noteTab(gtx, i+1, lit, true) }) + call := macro.Stop() + page := gtx.Constraints.Max + at := image.Pt(r.Min.X, r.Min.Y-d.Size.Y) + at.X = min(max(at.X, 0), page.X-d.Size.X) + at.Y = min(max(at.Y, 0), page.Y-d.Size.Y) + defer op.Offset(at).Push(gtx.Ops).Pop() + call.Add(gtx.Ops) +} + +// noteTab is a note's number on the pick's color: square at the bottom when it +// sits on an outline (onOutline), rounded all round in the popover and the +// tray; lit widens it. +func (u *window) noteTab(gtx C, num int, lit, onOutline bool) D { + pad := ui.Space(1.5) + if lit { + pad = ui.Space(2) + } + white := color.NRGBA{R: 255, G: 255, B: 255, A: 255} + macro := op.Record(gtx.Ops) + d := layout.Inset{Left: pad, Right: pad}.Layout(gtx, u.theme.Text(ui.TextXS, strconv.Itoa(num)).Mono().In(white).Weight(font.SemiBold).Layout) + call := macro.Stop() + radius := gtx.Dp(ui.Space(1)) + shape := clip.RRect{Rect: image.Rectangle{Max: d.Size}, NW: radius, NE: radius, SE: radius, SW: radius} + if onOutline { + shape.SE, shape.SW = 0, 0 + } + paint.FillShape(gtx.Ops, notePick, shape.Op(gtx.Ops)) + call.Add(gtx.Ops) + return d +} + +// notePopover is the open note beside its element: what it is, a line to +// check the pick by, the note, and its buttons. +func (u *window) notePopover(gtx C, p *browserPaneUI, scale float32) { + n := &p.notes + th := u.theme + n.editor.Submit = true + for { + ev, ok := n.editor.Update(gtx) + if !ok { + break + } + if _, ok := ev.(widget.SubmitEvent); ok { + n.commit() + } + } + for { + ev, ok := gtx.Event(key.Filter{Focus: &n.editor, Name: key.NameEscape}) + if !ok { + break + } + if e, ok := ev.(key.Event); ok && e.State == key.Press { + n.open = false + } + } + if n.add.Clicked(gtx) { + n.commit() + } + if n.cancel.Clicked(gtx) { + n.open = false + } + if n.del.Clicked(gtx) && n.editing >= 0 { + n.drop(n.editing) + } + if !n.open { + gtx.Execute(key.FocusCmd{Tag: &p.tag}) + return + } + if n.focusEditor { + n.focusEditor = false + gtx.Execute(key.FocusCmd{Tag: &n.editor}) + } + width := gtx.Dp(notePopoverWidth) + macro := op.Record(gtx.Ops) + body := gtx + body.Constraints = layout.Constraints{Min: image.Pt(width, 0), Max: image.Pt(width, gtx.Constraints.Max.Y)} + d := u.notePopoverBody(body, n) + call := macro.Stop() + room := gtx.Constraints.Max + if t := notesTrayHeight(gtx, len(n.list)); t > 0 { + room.Y -= t + gtx.Dp(gutter) + } + el := cssRect(n.draftBox(), scale) + at := notePopoverAt(el, d.Size, room, gtx.Dp(ui.Space(3))) + defer op.Offset(at).Push(gtx.Ops).Pop() + defer noteEnter(gtx, n.openedAt, el.Sub(at), d.Size)() + shape := clip.UniformRRect(image.Rectangle{Max: d.Size}, gtx.Dp(ui.RadiusMD)) + paint.FillShape(gtx.Ops, th.Sidebar, shape.Op(gtx.Ops)) + area := shape.Push(gtx.Ops) + // The popover takes the pointer from the page under it. + event.Op(gtx.Ops, &n.draft) + for { + if _, ok := gtx.Event(pointer.Filter{Target: &n.draft, Kinds: pointer.Press | pointer.Release | pointer.Move | pointer.Drag}); !ok { + break + } + } + call.Add(gtx.Ops) + area.Pop() + paint.FillShape(gtx.Ops, th.Over(th.Border), clip.Stroke{Path: shape.Path(gtx.Ops), Width: float32(gtx.Dp(1))}.Op()) +} + +// noteEnterDuration and noteZoomFrom are a lich menu's animate-in, slowed a +// little: the popover appears away from the pointer, beside the element. +const ( + noteEnterDuration = 160 * time.Millisecond + noteZoomFrom = 0.95 +) + +var noteEase = ui.CubicBezier(0.25, 0.1, 0.25, 1) + +// noteEnter fades and zooms a popover or the tray in from its edge nearest +// el (local to it), sliding from that side, for noteEnterDuration after +// opened. The returned func pops what it pushed. +func noteEnter(gtx C, opened time.Time, el image.Rectangle, size image.Point) func() { + elapsed := gtx.Now.Sub(opened) + if elapsed >= noteEnterDuration || elapsed < 0 { + return func() {} + } + gtx.Execute(op.InvalidateCmd{}) + p := noteEase(float32(elapsed) / float32(noteEnterDuration)) + mid := el.Min.Add(el.Max).Div(2) + origin := f32.Pt(float32(min(max(mid.X, 0), size.X)), float32(min(max(mid.Y, 0), size.Y))) + toward := f32.Pt(float32(mid.X)-origin.X, float32(mid.Y)-origin.Y) + if l := float32(math.Hypot(float64(toward.X), float64(toward.Y))); l > 0 { + toward = toward.Mul(1 / l) + } + slide := toward.Mul(float32(gtx.Dp(ui.Space(2))) * (1 - p)) + zoom := noteZoomFrom + (1-noteZoomFrom)*p + transform := op.Affine(f32.Affine2D{}.Scale(origin, f32.Pt(zoom, zoom)).Offset(slide)).Push(gtx.Ops) + opacity := paint.PushOpacity(gtx.Ops, p) + return func() { + opacity.Pop() + transform.Pop() + } +} + +// notePopoverAt places a popover of size beside the element at r: to its +// right, else under it, else above it, kept inside the page. +func notePopoverAt(r image.Rectangle, size, page image.Point, gap int) image.Point { + at := image.Pt(r.Max.X+gap, r.Min.Y) + if at.X+size.X > page.X-gap { + at = image.Pt(r.Min.X, r.Max.Y+gap) + if at.Y+size.Y > page.Y-gap { + at.Y = r.Min.Y - gap - size.Y + } + } + at.X = min(max(at.X, gap), page.X-size.X-gap) + at.Y = min(max(at.Y, gap), page.Y-size.Y-gap) + return at +} + +// notePopoverBody is the review's comment box (commentComposer) for a note: +// a muted line naming the element, the field, then Add and Cancel; a note +// being edited adds Delete. +func (u *window) notePopoverBody(gtx C, n *browserNotes) D { + on := u.theme.On(u.theme.Sidebar) + num := len(n.list) + 1 + if n.editing >= 0 { + num = n.editing + 1 + } + heading := n.draft.Label + if n.draft.Component != "" { + heading += " · " + n.draft.Component + } + return layout.Inset{Left: ui.Space(3), Right: ui.Space(3), Top: ui.Space(2.5), Bottom: ui.Space(2.5)}.Layout(gtx, func(gtx C) D { + return layout.Flex{Axis: layout.Vertical}.Layout(gtx, + layout.Rigid(func(gtx C) D { + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(func(gtx C) D { return u.noteTab(gtx, num, false, false) }), + ui.Gap(2), + layout.Flexed(1, on.Text(ui.TextXS, heading).Mono().Muted().Layout), + ) + }), + ui.VGap(3), + layout.Rigid(func(gtx C) D { + gtx.Constraints.Max.Y = min(gtx.Constraints.Max.Y, gtx.Dp(ui.Space(64))) + return on.Textarea(&n.editor, i18n.T("layouts.browser.notePlaceholder")).Layout(gtx) + }), + ui.VGap(2), + layout.Rigid(func(gtx C) D { return u.notePopoverButtons(gtx, on, n) }), + ) + }) +} + +func (u *window) notePopoverButtons(gtx C, th *ui.Theme, n *browserNotes) D { + label := i18n.T("layouts.browser.addNote") + if n.editing >= 0 { + label = i18n.T("layouts.browser.saveNote") + } + add := th.Button(&n.add, label) + add.Size = ui.SizeSM + cancel := th.Button(&n.cancel, i18n.T("common.action.cancel")) + cancel.Size, cancel.Variant = ui.SizeSM, ui.VariantGhost + buttons := []layout.FlexChild{layout.Rigid(add.Layout), ui.Gap(2), layout.Rigid(cancel.Layout)} + if n.editing >= 0 { + del := th.Button(&n.del, i18n.T("layouts.browser.deleteNote")) + del.Size, del.Variant = ui.SizeSM, ui.VariantGhost + buttons = append(buttons, + layout.Flexed(1, func(gtx C) D { return D{Size: image.Pt(gtx.Constraints.Min.X, 0)} }), + layout.Rigid(del.Layout)) + } + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, buttons...) +} + +// notesTray is the notes waiting, floating over the page's bottom, inset +// from the pane's edges by the gutter the island keeps around its panes: a +// row each, Clear and Send. +func (u *window) notesTray(gtx C, p *browserPaneUI, session string) { + n := &p.notes + th := u.theme + if n.clear.Clicked(gtx) { + n.list, n.open = nil, false + } + if n.send.Clicked(gtx) && len(n.list) > 0 { + u.sendNotes(p, session) + } + for i := len(n.list) - 1; i >= 0; i-- { + note := n.list[i] + if note.remove.Clicked(gtx) { + n.drop(i) + continue + } + if note.row.Clicked(gtx) { + el := note.el + el.Box = note.box + n.openNote(el, i) + } + } + height := notesTrayHeight(gtx, len(n.list)) + if height == 0 { + n.trayShown = time.Time{} + return + } + if n.trayShown.IsZero() { + n.trayShown = gtx.Now + } + edge := gtx.Dp(gutter) + page := gtx.Constraints.Max + size := image.Pt(max(page.X-2*edge, 0), height) + defer op.Offset(image.Pt(edge, page.Y-height-edge)).Push(gtx.Ops).Pop() + defer noteEnter(gtx, n.trayShown, image.Rect(size.X/2, size.Y, size.X/2, size.Y+1), size)() + gtx.Constraints = layout.Exact(size) + shape := clip.UniformRRect(image.Rectangle{Max: size}, gtx.Dp(paneRadius)) + paint.FillShape(gtx.Ops, th.Background, shape.Op(gtx.Ops)) + area := shape.Push(gtx.Ops) + // The tray takes the pointer from the page under it. + event.Op(gtx.Ops, &n.rows) + for { + if _, ok := gtx.Event(pointer.Filter{Target: &n.rows, Kinds: pointer.Press | pointer.Release | pointer.Move | pointer.Drag}); !ok { + break + } + } + layout.UniformInset(noteTrayPad).Layout(gtx, func(gtx C) D { + return layout.Flex{Axis: layout.Vertical}.Layout(gtx, + layout.Rigid(func(gtx C) D { return u.notesTrayHead(gtx, n, session) }), + layout.Rigid(layout.Spacer{Height: noteTrayPad}.Layout), + layout.Flexed(1, func(gtx C) D { + return n.rows.Layout(gtx, len(n.list), func(gtx C, i int) D { return u.noteRow(gtx, n, i) }) + }), + ) + }) + area.Pop() + paint.FillShape(gtx.Ops, th.Over(th.Border), clip.Stroke{Path: shape.Path(gtx.Ops), Width: float32(gtx.Dp(1))}.Op()) +} + +func (u *window) notesTrayHead(gtx C, n *browserNotes, session string) D { + th := u.theme + gtx.Constraints.Min.Y = gtx.Dp(noteTrayHead) + gtx.Constraints.Max.Y = gtx.Constraints.Min.Y + return layout.Inset{Left: ui.Space(1.5)}.Layout(gtx, func(gtx C) D { + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.notes", i18n.Args{"count": len(n.list)})).Weight(font.SemiBold).Layout), + ui.Gap(1.5), + layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.notesFor", i18n.Args{"label": u.model.sessionLabel(session)})).Muted().Layout), + layout.Flexed(1, func(gtx C) D { return D{Size: image.Pt(gtx.Constraints.Min.X, 0)} }), + layout.Rigid(func(gtx C) D { + clear := th.Button(&n.clear, i18n.T("layouts.browser.clearNotes")) + clear.Variant, clear.Size = ui.VariantGhost, ui.SizeSM + return clear.Layout(gtx) + }), + ui.Gap(1.5), + layout.Rigid(func(gtx C) D { + send := th.Button(&n.send, i18n.T("layouts.browser.sendNotes")) + send.Icon, send.Size = icBatchSend, ui.SizeSM + return send.Layout(gtx) + }), + ) + }) +} + +// noteRow is one note in the tray: its pin's number, the element and the +// note; hovering it lights the element on the page, a click opens the note. +func (u *window) noteRow(gtx C, n *browserNotes, i int) D { + th := u.theme + note := n.list[i] + height := gtx.Dp(noteRowHeight) + gtx.Constraints = layout.Exact(image.Pt(gtx.Constraints.Max.X, height)) + return note.row.Layout(gtx, func(gtx C) D { + if note.row.Hovered() { + ui.Fill(gtx, th.Accent, ui.RadiusMD) + } + text := note.text + muted := th.MutedForeground + if text == "" { + text = i18n.T("layouts.browser.noNote") + } + return layout.Inset{Left: ui.Space(1.5), Right: ui.Space(0.5)}.Layout(gtx, func(gtx C) D { + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(func(gtx C) D { return u.noteTab(gtx, i+1, false, false) }), + ui.Gap(2), + layout.Rigid(func(gtx C) D { + gtx.Constraints.Max.X = min(gtx.Constraints.Max.X, gtx.Dp(ui.Space(48))) + return th.Text(ui.TextXS, note.el.Label).Mono().Layout(gtx) + }), + ui.Gap(2.5), + layout.Flexed(1, th.Text(ui.TextXS, text).In(muted).Layout), + layout.Rigid(func(gtx C) D { + return cornerButton(gtx, th, ¬e.remove, icNoteRemove, false) + }), + ) + }) + }) +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index 199cb889d..b9e593c9f 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -79,6 +79,7 @@ type browserPaneUI struct { // typing is the address it types into the field, from when. flash agentPress typing agentPress + notes browserNotes } // agentPress is what the agent's cursor does on the toolbar, from at: the @@ -94,7 +95,7 @@ func (u *window) browserPane(session string) *browserPaneUI { } p, ok := u.browserPanes[session] if !ok { - p = &browserPaneUI{queue: newBrowserQueue(), copied: make(chan string, 1)} + p = &browserPaneUI{queue: newBrowserQueue(), copied: make(chan string, 1), notes: newBrowserNotes()} go p.queue.run(u.ctx, func(err error) { log.Printf("browser of %s: %v", session, err) }) u.browserPanes[session] = p } @@ -146,6 +147,8 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta // Unlike a terminal, the page is not dimmed out of focus: the person keeps // the session's prompt in focus nearly always, and watches the page. u.browserPage(page, p, session, frame, focused) + u.notesOverlay(page, p, session, st.URL) + u.notesTray(page, p, session) off.Pop() u.paintAgentCursor(gtx, p, session) u.paneFocusTarget(gtx, pane) @@ -206,6 +209,9 @@ func (u *window) browserToolbar(gtx C, p *browserPaneUI, session string, st api. if p.reload.click.Clicked(gtx) { nav("reload") } + if p.notes.pick.click.Clicked(gtx) { + p.notes.setPicking(!p.notes.picking) + } if p.window.click.Clicked(gtx) { u.sendBrowser(session, "show", func(ctx context.Context) error { return u.client.ShowBrowser(ctx, session) }) } @@ -244,6 +250,15 @@ func (u *window) browserToolbar(gtx C, p *browserPaneUI, session string, st api. ui.Gap(1.5), layout.Flexed(1, field.Layout), ui.Gap(1.5), + layout.Rigid(func(gtx C) D { + if !p.notes.picking { + return D{} + } + return layout.Inset{Right: ui.Space(1.5)}.Layout(gtx, th.Text(ui.TextXS, i18n.T("layouts.browser.pickHint")).In(notePick).Layout) + }), + layout.Rigid(func(gtx C) D { + return iconAction(gtx, th, &u.root, &p.notes.pick, icBrowserPick, i18n.T("layouts.browser.pick"), p.notes.picking) + }), layout.Rigid(func(gtx C) D { return iconAction(gtx, th, &u.root, &p.window, icBrowserWindow, i18n.T("layouts.browser.openInWindow"), false) }), @@ -285,13 +300,13 @@ func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame brow // browserInput passes the pane's pointer and keys to the page. func (u *window) browserInput(gtx C, p *browserPaneUI, session string, scale float32, focused bool) { filters := append(gioterm.KeyFilters(&p.tag), - pointer.Filter{Target: &p.tag, Kinds: pointer.Scroll | pointer.Press | pointer.Release | pointer.Drag | pointer.Move, + pointer.Filter{Target: &p.tag, Kinds: pointer.Scroll | pointer.Press | pointer.Release | pointer.Drag | pointer.Move | pointer.Leave, ScrollX: pointer.ScrollRange{Min: -1 << 20, Max: 1 << 20}, ScrollY: pointer.ScrollRange{Min: -1 << 20, Max: 1 << 20}}, key.Filter{Focus: &p.tag, Name: "V", Required: key.ModCommand}, key.Filter{Focus: &p.tag, Name: "C", Required: key.ModCommand}, key.Filter{Focus: &p.tag, Name: "X", Required: key.ModCommand}, transfer.TargetFilter{Target: &p.tag, Type: "application/text"}) - if focused && !gtx.Focused(&p.url) && !gtx.Focused(&p.tag) { + if focused && !gtx.Focused(&p.url) && !gtx.Focused(&p.tag) && !gtx.Focused(&p.notes.editor) { gtx.Execute(key.FocusCmd{Tag: &p.tag}) } writeCopied(gtx, p) @@ -306,6 +321,9 @@ func (u *window) browserInput(gtx C, p *browserPaneUI, session string, scale flo u.sendBrowser(session, "text", func(ctx context.Context) error { return u.client.BrowserText(ctx, session, text) }) } case key.Event: + if u.notesKey(p, session, e) { + continue + } switch browserClipboardChord(e, runtime.GOOS) { case "paste": gtx.Execute(clipboard.ReadCmd{Tag: &p.tag}) @@ -333,6 +351,18 @@ func (u *window) browserInput(gtx C, p *browserPaneUI, session string, scale flo func (u *window) browserPointer(gtx C, p *browserPaneUI, session string, scale float32, e pointer.Event) { m := api.BrowserMouseOptions{ID: session, X: float64(e.Position.X / scale), Y: float64(e.Position.Y / scale), Modifiers: cdpModifiers(e.Modifiers)} + if p.notes.picking && e.Kind != pointer.Scroll { + u.notesPointer(gtx, p, session, m.X, m.Y, e) + return + } + if e.Kind == pointer.Scroll && (len(p.notes.list) > 0 || p.notes.open) { + p.notes.nextLocate, p.notes.followUntil = time.Time{}, gtx.Now.Add(noteFollowFor) + } + if e.Kind == pointer.Scroll && p.notes.picking { + // What the pointer outlined moved with the page: look again under it. + p.notes.hover = api.BrowserElement{} + defer u.probeElement(p, session, m.X, m.Y, false) + } switch e.Kind { case pointer.Press: gtx.Execute(key.FocusCmd{Tag: &p.tag}) diff --git a/native/cmd/lich-native/browser_queue.go b/native/cmd/lich-native/browser_queue.go index 1cc21dd84..832da8785 100644 --- a/native/cmd/lich-native/browser_queue.go +++ b/native/cmd/lich-native/browser_queue.go @@ -20,10 +20,12 @@ type browserQueue struct { } // browserCall is one queued call: a pointer event to merge (mouse set), a -// resize to merge (resize set), or any other call. +// resize to merge (resize set), a lookup whose newest one is all that +// matters (merge names its kind), or any other call. type browserCall struct { mouse *api.BrowserMouseOptions resize bool + merge string send func(context.Context) error } @@ -38,6 +40,11 @@ func (q *browserQueue) push(c browserCall) { q.mu.Unlock() return } + if n := len(q.calls); n > 0 && c.merge != "" && q.calls[n-1].merge == c.merge { + q.calls[n-1] = c + q.mu.Unlock() + return + } if n := len(q.calls); n > 0 && c.mouse != nil && q.calls[n-1].mouse != nil { last := q.calls[n-1].mouse switch { diff --git a/native/cmd/lich-native/browser_queue_test.go b/native/cmd/lich-native/browser_queue_test.go index be4091e00..5ff9f485b 100644 --- a/native/cmd/lich-native/browser_queue_test.go +++ b/native/cmd/lich-native/browser_queue_test.go @@ -60,6 +60,28 @@ func TestBrowserQueueKeepsOnlyTheLastPendingResize(t *testing.T) { } } +func TestBrowserQueueKeepsTheLastLookupOfAKindAndEveryOpen(t *testing.T) { + q := newBrowserQueue() + var sent []string + push := func(merge, name string) { + q.push(browserCall{merge: merge, send: func(context.Context) error { sent = append(sent, name); return nil }}) + } + push("hover", "hover 1") + push("hover", "hover 2") + push("", "open 1") + push("", "open 2") + push("locate", "locate 1") + push("hover", "hover 3") + push("hover", "hover 4") + for c, ok := q.next(); ok; c, ok = q.next() { + _ = c.send(context.Background()) + } + want := []string{"hover 2", "open 1", "open 2", "locate 1", "hover 4"} + if !reflect.DeepEqual(sent, want) { + t.Fatalf("sent %v, want %v", sent, want) + } +} + func TestToolbarPointsFollowTheToolbarsLayout(t *testing.T) { gtx := layout.Context{Metric: unit.Metric{PxPerDp: 1, PxPerSp: 1}} back, fwd, reload, addr := toolbarPoint(gtx, "back", 30, 36), toolbarPoint(gtx, "forward", 30, 36), toolbarPoint(gtx, "reload", 30, 36), toolbarPoint(gtx, "navigate", 30, 36) diff --git a/native/i18n/locales/en/layouts.json b/native/i18n/locales/en/layouts.json index 7c74696e6..00ba8fbd4 100644 --- a/native/i18n/locales/en/layouts.json +++ b/native/i18n/locales/en/layouts.json @@ -27,7 +27,22 @@ "opened": "{label} opened a browser", "dismiss": "Dismiss", "open": "Open", - "moveBody": "It leaves \"{layout}\". A browser tab holds only {label} and its browser." + "moveBody": "It leaves \"{layout}\". A browser tab holds only {label} and its browser.", + "pick": "Leave notes on the page", + "pickHint": "Click an element to leave a note", + "notePlaceholder": "What should change here?", + "addNote": "Add note", + "saveNote": "Save", + "deleteNote": "Delete", + "notes": { + "one": "{count} note", + "other": "{count} notes" + }, + "notesFor": "for {label}", + "clearNotes": "Clear", + "sendNotes": "Send", + "noNote": "No note", + "notesFailed": "Could not send the notes to {label}: {error}" }, "split": { "noRoom": "No room for another pane here. Make the window wider or take a pane out first." diff --git a/native/i18n/locales/en/prompts.json b/native/i18n/locales/en/prompts.json index 56504691a..7e7de115f 100644 --- a/native/i18n/locales/en/prompts.json +++ b/native/i18n/locales/en/prompts.json @@ -24,5 +24,14 @@ }, "issue": { "head": "GitHub issue #{number} — {title}\n{url}" + }, + "browserNotes": { + "head": { + "one": "[lich] The person left a note on the page in your browser tab:", + "other": "[lich] The person left {count} notes on the page in your browser tab:" + }, + "item": "{index}. {label} on {url}", + "note": "Note: {note}", + "selectors": "Each selector above works as the selector of browser_click and browser_type." } } diff --git a/native/i18n/locales/es/layouts.json b/native/i18n/locales/es/layouts.json index 69a04a06a..150a30d7e 100644 --- a/native/i18n/locales/es/layouts.json +++ b/native/i18n/locales/es/layouts.json @@ -27,7 +27,22 @@ "opened": "{label} abrió un navegador", "dismiss": "Descartar", "open": "Abrir", - "moveBody": "Sale de \"{layout}\". La pestaña del navegador solo admite {label} y su navegador." + "moveBody": "Sale de \"{layout}\". La pestaña del navegador solo admite {label} y su navegador.", + "pick": "Dejar notas en la página", + "pickHint": "Haz clic en un elemento para dejar una nota", + "notePlaceholder": "¿Qué debería cambiar aquí?", + "addNote": "Añadir nota", + "saveNote": "Guardar", + "deleteNote": "Eliminar", + "notes": { + "one": "{count} nota", + "other": "{count} notas" + }, + "notesFor": "para {label}", + "clearNotes": "Borrar", + "sendNotes": "Enviar", + "noNote": "Sin nota", + "notesFailed": "No se pudieron enviar las notas a {label}: {error}" }, "split": { "noRoom": "No hay espacio para otro panel aquí. Ensancha la ventana o saca antes un panel." diff --git a/native/i18n/locales/es/prompts.json b/native/i18n/locales/es/prompts.json index 1151e142e..8ea598271 100644 --- a/native/i18n/locales/es/prompts.json +++ b/native/i18n/locales/es/prompts.json @@ -24,5 +24,14 @@ }, "issue": { "head": "Issue de GitHub #{number}: {title}\n{url}" + }, + "browserNotes": { + "head": { + "one": "[lich] La persona dejó una nota en la página de tu pestaña del navegador:", + "other": "[lich] La persona dejó {count} notas en la página de tu pestaña del navegador:" + }, + "item": "{index}. {label} en {url}", + "note": "Nota: {note}", + "selectors": "Cada selector de arriba sirve como el selector de browser_click y browser_type." } } diff --git a/native/i18n/locales/ja/layouts.json b/native/i18n/locales/ja/layouts.json index d9cb24ffe..feea61bfe 100644 --- a/native/i18n/locales/ja/layouts.json +++ b/native/i18n/locales/ja/layouts.json @@ -25,7 +25,21 @@ "opened": "{label} がブラウザを開きました", "dismiss": "閉じる", "open": "開く", - "moveBody": "「{layout}」から外れます。ブラウザのタブには {label} とそのブラウザしか置けません。" + "moveBody": "「{layout}」から外れます。ブラウザのタブには {label} とそのブラウザしか置けません。", + "pick": "ページにメモを残す", + "pickHint": "要素をクリックしてメモを残す", + "notePlaceholder": "ここをどう変えますか?", + "addNote": "メモを追加", + "saveNote": "保存", + "deleteNote": "削除", + "notes": { + "other": "メモ {count} 件" + }, + "notesFor": "{label} へ", + "clearNotes": "クリア", + "sendNotes": "送信", + "noNote": "メモなし", + "notesFailed": "{label} にメモを送れませんでした: {error}" }, "split": { "noRoom": "ここには別のペインを置く余裕がありません。ウィンドウを広げるか、先にペインを外してください。" diff --git a/native/i18n/locales/ja/prompts.json b/native/i18n/locales/ja/prompts.json index 3afb4b752..a889704a0 100644 --- a/native/i18n/locales/ja/prompts.json +++ b/native/i18n/locales/ja/prompts.json @@ -24,5 +24,13 @@ }, "issue": { "head": "GitHub issue #{number}: {title}\n{url}" + }, + "browserNotes": { + "head": { + "other": "[lich] ブラウザタブのページに {count} 件のメモが残されました:" + }, + "item": "{index}. {url} の {label}", + "note": "メモ: {note}", + "selectors": "上の各セレクタは browser_click と browser_type の selector としてそのまま使えます。" } } diff --git a/native/i18n/locales/ko/layouts.json b/native/i18n/locales/ko/layouts.json index 93ec15d37..4e542263a 100644 --- a/native/i18n/locales/ko/layouts.json +++ b/native/i18n/locales/ko/layouts.json @@ -25,7 +25,21 @@ "opened": "{label}이(가) 브라우저를 열었습니다", "dismiss": "닫기", "open": "열기", - "moveBody": "\"{layout}\"에서 빠집니다. 브라우저 탭에는 {label}와(과) 그 브라우저만 둘 수 있습니다." + "moveBody": "\"{layout}\"에서 빠집니다. 브라우저 탭에는 {label}와(과) 그 브라우저만 둘 수 있습니다.", + "pick": "페이지에 메모 남기기", + "pickHint": "요소를 클릭해 메모를 남기세요", + "notePlaceholder": "여기서 무엇을 바꿔야 하나요?", + "addNote": "메모 추가", + "saveNote": "저장", + "deleteNote": "삭제", + "notes": { + "other": "메모 {count}개" + }, + "notesFor": "{label}에게", + "clearNotes": "지우기", + "sendNotes": "보내기", + "noNote": "메모 없음", + "notesFailed": "{label}에 메모를 보내지 못했습니다: {error}" }, "split": { "noRoom": "여기에는 창을 하나 더 놓을 공간이 없습니다. 창을 넓히거나 먼저 창 하나를 빼세요." diff --git a/native/i18n/locales/ko/prompts.json b/native/i18n/locales/ko/prompts.json index 869c16352..4bfe6bfa1 100644 --- a/native/i18n/locales/ko/prompts.json +++ b/native/i18n/locales/ko/prompts.json @@ -22,5 +22,13 @@ }, "issue": { "head": "GitHub 이슈 #{number}: {title}\n{url}" + }, + "browserNotes": { + "head": { + "other": "[lich] 브라우저 탭의 페이지에 메모 {count}개가 남겨졌습니다:" + }, + "item": "{index}. {url}의 {label}", + "note": "메모: {note}", + "selectors": "위의 각 셀렉터는 browser_click과 browser_type의 selector로 그대로 쓸 수 있습니다." } } diff --git a/native/i18n/locales/pt-br/layouts.json b/native/i18n/locales/pt-br/layouts.json index 90669962d..9f8a7e112 100644 --- a/native/i18n/locales/pt-br/layouts.json +++ b/native/i18n/locales/pt-br/layouts.json @@ -27,7 +27,22 @@ "opened": "{label} abriu um navegador", "dismiss": "Dispensar", "open": "Abrir", - "moveBody": "Ela sai de \"{layout}\". A aba do navegador só comporta {label} e o navegador dela." + "moveBody": "Ela sai de \"{layout}\". A aba do navegador só comporta {label} e o navegador dela.", + "pick": "Deixar notas na página", + "pickHint": "Clique num elemento para deixar uma nota", + "notePlaceholder": "O que deve mudar aqui?", + "addNote": "Adicionar nota", + "saveNote": "Salvar", + "deleteNote": "Excluir", + "notes": { + "one": "{count} nota", + "other": "{count} notas" + }, + "notesFor": "para {label}", + "clearNotes": "Limpar", + "sendNotes": "Enviar", + "noNote": "Sem nota", + "notesFailed": "Não foi possível enviar as notas para {label}: {error}" }, "split": { "noRoom": "Não há espaço para outro painel aqui. Alargue a janela ou tire um painel antes." diff --git a/native/i18n/locales/pt-br/prompts.json b/native/i18n/locales/pt-br/prompts.json index 776e9e107..2e08bb5fe 100644 --- a/native/i18n/locales/pt-br/prompts.json +++ b/native/i18n/locales/pt-br/prompts.json @@ -24,5 +24,14 @@ }, "issue": { "head": "Issue do GitHub #{number}: {title}\n{url}" + }, + "browserNotes": { + "head": { + "one": "[lich] A pessoa deixou uma nota na página da sua aba de navegador:", + "other": "[lich] A pessoa deixou {count} notas na página da sua aba de navegador:" + }, + "item": "{index}. {label} em {url}", + "note": "Nota: {note}", + "selectors": "Cada seletor acima serve como o selector de browser_click e browser_type." } } diff --git a/native/i18n/locales/ru/layouts.json b/native/i18n/locales/ru/layouts.json index b7349d599..b43bd485a 100644 --- a/native/i18n/locales/ru/layouts.json +++ b/native/i18n/locales/ru/layouts.json @@ -31,7 +31,24 @@ "opened": "{label} открыла браузер", "dismiss": "Скрыть", "open": "Открыть", - "moveBody": "Она уйдёт из «{layout}». Во вкладке браузера помещаются только {label} и её браузер." + "moveBody": "Она уйдёт из «{layout}». Во вкладке браузера помещаются только {label} и её браузер.", + "pick": "Оставить заметки на странице", + "pickHint": "Нажмите на элемент, чтобы оставить заметку", + "notePlaceholder": "Что здесь изменить?", + "addNote": "Добавить заметку", + "saveNote": "Сохранить", + "deleteNote": "Удалить", + "notes": { + "one": "{count} заметка", + "few": "{count} заметки", + "many": "{count} заметок", + "other": "{count} заметки" + }, + "notesFor": "для {label}", + "clearNotes": "Очистить", + "sendNotes": "Отправить", + "noNote": "Без заметки", + "notesFailed": "Не удалось отправить заметки в {label}: {error}" }, "split": { "noRoom": "Здесь нет места для ещё одной панели. Сделайте окно шире или сначала уберите панель." diff --git a/native/i18n/locales/ru/prompts.json b/native/i18n/locales/ru/prompts.json index 5136bcf5f..6c25ea228 100644 --- a/native/i18n/locales/ru/prompts.json +++ b/native/i18n/locales/ru/prompts.json @@ -30,5 +30,16 @@ }, "issue": { "head": "GitHub issue #{number}: {title}\n{url}" + }, + "browserNotes": { + "head": { + "one": "[lich] Человек оставил {count} заметку на странице во вкладке браузера:", + "few": "[lich] Человек оставил {count} заметки на странице во вкладке браузера:", + "many": "[lich] Человек оставил {count} заметок на странице во вкладке браузера:", + "other": "[lich] Человек оставил {count} заметки на странице во вкладке браузера:" + }, + "item": "{index}. {label} на {url}", + "note": "Заметка: {note}", + "selectors": "Каждый селектор выше подходит как selector для browser_click и browser_type." } } diff --git a/native/i18n/locales/zh-cn/layouts.json b/native/i18n/locales/zh-cn/layouts.json index bc9a4cfda..ba2ec587b 100644 --- a/native/i18n/locales/zh-cn/layouts.json +++ b/native/i18n/locales/zh-cn/layouts.json @@ -25,7 +25,21 @@ "opened": "{label} 打开了浏览器", "dismiss": "忽略", "open": "打开", - "moveBody": "它将离开“{layout}”。浏览器标签页只容纳 {label} 及其浏览器。" + "moveBody": "它将离开“{layout}”。浏览器标签页只容纳 {label} 及其浏览器。", + "pick": "在页面上留下备注", + "pickHint": "点击一个元素来留下备注", + "notePlaceholder": "这里应该改成什么?", + "addNote": "添加备注", + "saveNote": "保存", + "deleteNote": "删除", + "notes": { + "other": "{count} 条备注" + }, + "notesFor": "发给 {label}", + "clearNotes": "清除", + "sendNotes": "发送", + "noNote": "无备注", + "notesFailed": "无法把备注发送给 {label}:{error}" }, "split": { "noRoom": "这里没有空间再放一个窗格。请加宽窗口,或先移出一个窗格。" diff --git a/native/i18n/locales/zh-cn/prompts.json b/native/i18n/locales/zh-cn/prompts.json index c4e7f54b9..923b59fc1 100644 --- a/native/i18n/locales/zh-cn/prompts.json +++ b/native/i18n/locales/zh-cn/prompts.json @@ -24,5 +24,13 @@ }, "issue": { "head": "GitHub issue #{number},{title}\n{url}" + }, + "browserNotes": { + "head": { + "other": "[lich] 对方在你浏览器标签页的页面上留下了 {count} 条备注:" + }, + "item": "{index}. {url} 上的 {label}", + "note": "备注: {note}", + "selectors": "上面每个选择器都可以直接作为 browser_click 和 browser_type 的 selector 使用。" } } From 3f2922bc89a065b11691da456f3a199c62c1bd0e Mon Sep 17 00:00:00 2001 From: omartelo Date: Tue, 27 Oct 2026 16:13:15 -0300 Subject: [PATCH 2/3] fix(browser): tidy the page notes' loose ends - A production build's one or two letter component names (the minifier's) no longer reach the tag or the agent; a development build's source path is cut at the project's src/ even under a folder named app/. - A refused send says why in the tray, not in a toast over the toolbar. - Ctrl+Shift+E and Ctrl+Enter are Cmd on macOS. - The wheel over the popover or the tray no longer scrolls the page under it. - The ring's number tab sits on its straight top edge, past the corner. - docs/ceilings.md: the pane copies the page's selection since #941, and a Chromium without an AppArmor profile does not start on Ubuntu 23.10+. --- CHANGELOG.md | 2 +- docs/ceilings.md | 8 ++-- internal/browser/notes.go | 14 ++++--- internal/browser/notes_live_test.go | 43 ++++++++++++++++++++ native/cmd/lich-native/browser_notes.go | 25 +++++++----- native/cmd/lich-native/browser_notes_view.go | 26 ++++++++---- native/cmd/lich-native/browser_pane.go | 3 ++ 7 files changed, 94 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8d80b953..e13280273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E turns the crosshair on and off, and Esc leaves it. +- The crosshair on a session browser's toolbar lets you leave notes on the page for the agent. The element under the pointer is outlined and named, a click opens a note beside it, and each note pins its element with a number, following it as the page scrolls. Notes wait in a tray floating over the bottom of the page, which never resizes under you. Hovering a note there lights its element, a click edits it, and Send (or Ctrl+Enter) types them into the session as one message: which element, a selector the agent's `browser_click` takes, the React component that drew it in a development build, and your note. Ctrl+Shift+E (Cmd+Shift+E on macOS) turns the crosshair on and off, and Esc leaves it. A send the session refuses keeps the notes and says why in the tray. ## [0.80.0] - 2026-10-27 diff --git a/docs/ceilings.md b/docs/ceilings.md index 58bef42a9..4f3175324 100644 --- a/docs/ceilings.md +++ b/docs/ceilings.md @@ -970,15 +970,17 @@ work when nobody knows it and that the call site never shows. The mechanism and `docs/browser-tab.md`): a headless Chrome, Edge, Chromium, Brave, Vivaldi or Helium already installed, whose page reaches the session's browser tab as a JPEG screencast over `/events`, at the pace the window takes frames. The native window cannot hold a webview: on Linux it speaks Wayland without GTK. So the pane - has no DevTools, shows the arrow whatever cursor the page asks for, does not copy the page's selection to - the system clipboard (paste works), loads what the page opens in a new tab or window in the page itself, and leaves downloads and the page's own dialogs out; Open in + has no DevTools, shows the arrow whatever cursor the page asks for, loads what the page opens in a new tab or window in the page itself, and leaves downloads and the page's own dialogs out; Open in a window moves the page to a real Chromium window for those, reopening it on the same profile with every cookie, since chromedp cannot show a headless process in place. A half-filled form or sessionStorage is lost on that move. Closing the card, or the session's process ending, closes the browser; closing that window does not close the session. A headless browser no pane shows and no one used for five minutes is closed; one a pane shows, or a window, never is. It loads only http, https and about:blank. Page info never includes a form's value. Screenshots are files. There is no tool that runs JavaScript in the page. It is - not a sandbox: the page runs as the user, in an ephemeral profile. + not a sandbox: the page runs as the user, in an ephemeral profile. Chromium needs its own sandbox, and on + Ubuntu 23.10 and later AppArmor refuses it the user namespace unless the browser ships an AppArmor profile, + as the distribution's Chrome, Edge and Brave do: one unpacked from a tarball fails to start with "No usable + sandbox!", which the agent reads as the browser's error. - **A note left on the page names only what the page shows lich** (`internal/browser/notes.go`, `native/cmd/lich-native/browser_notes.go`): the pick asks the page for the element under the pointer, so an element inside an iframe is picked as the iframe and one inside a closed shadow root as its host, and a diff --git a/internal/browser/notes.go b/internal/browser/notes.go index a4ff62eda..f76439aef 100644 --- a/internal/browser/notes.go +++ b/internal/browser/notes.go @@ -16,9 +16,11 @@ import ( // elementScript reads the element under a point: a selector that finds it // again (an id, or up to two classes that are not a CSS-in-JS hash, with // :nth-of-type only where a level is not yet unique), a label, the React -// component that drew it when the page is a development build, and its -// markup with every field's value taken out. A point on the page's root -// answers null. +// component that drew it when the page is a development build (its file cut +// to the project's src/, or else app/, pages/, components/ or lib/), and its +// markup with every field's value taken out. A component name of one or two +// letters is a production build's minifier, which tells the agent nothing. A +// point on the page's root answers null. const elementScript = `((x, y) => { const cut = (s, n) => s.length > n ? s.slice(0, n - 1) + '…' : s; let el = document.elementFromPoint(x, y); @@ -54,13 +56,15 @@ const elementScript = `((x, y) => { const fiber = Object.keys(el).find(k => k.startsWith('__reactFiber$')); for (let f = fiber ? el[fiber] : null; f; f = f.return) { if (!source && f._debugSource) { - source = f._debugSource.fileName.replace(/^.*?\/((src|app|pages|components|lib)\/)/, '$1') + ':' + f._debugSource.lineNumber; + const file = f._debugSource.fileName; + const root = ['/src/', '/app/', '/pages/', '/components/', '/lib/'].map(r => file.indexOf(r)).find(i => i >= 0); + source = (root === undefined ? file : file.slice(root + 1)) + ':' + f._debugSource.lineNumber; } const t = f.type; if (!t || typeof t === 'string') continue; const inner = t.render || t.type || t; const name = t.displayName || inner.displayName || inner.name || ''; - if (name && !/Provider|Context|Router|Boundary|^Anonymous$/.test(name)) { component = name; break; } + if (name.length > 2 && !/Provider|Context|Router|Boundary|^Anonymous$/.test(name)) { component = name; break; } } const copy = el.cloneNode(true); for (const n of [copy, ...copy.querySelectorAll('*')]) { diff --git a/internal/browser/notes_live_test.go b/internal/browser/notes_live_test.go index a327942b7..b465557a8 100644 --- a/internal/browser/notes_live_test.go +++ b/internal/browser/notes_live_test.go @@ -123,3 +123,46 @@ func TestChromeLocateAnswersAnEmptyBoxForAnElementGone(t *testing.T) { t.Fatalf("box of an invalid selector = %+v", got) } } + +// reactProbe gives two elements a React fiber, as a development build and a +// production build leave them: a named component with its source, and one +// whose name the minifier shortened. +const reactProbe = ` + + +` + +func TestChromeElementAtNamesTheReactComponentButNotAMinifiedOne(t *testing.T) { + p, _ := liveChrome(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, reactProbe) + })) + t.Cleanup(srv.Close) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + dev, err := p.ElementAt(ctx, 20, 20) + if err != nil { + t.Fatal(err) + } + if dev.Component != "PlanCard" || dev.Source != "src/PlanCard.tsx:31" { + t.Errorf("dev build: component %q source %q, want PlanCard at src/PlanCard.tsx:31", dev.Component, dev.Source) + } + prod, err := p.ElementAt(ctx, 20, 60) + if err != nil { + t.Fatal(err) + } + if prod.Component != "" { + t.Errorf("production build names component %q, a minifier's name", prod.Component) + } +} diff --git a/native/cmd/lich-native/browser_notes.go b/native/cmd/lich-native/browser_notes.go index 631cbfaac..b48de1914 100644 --- a/native/cmd/lich-native/browser_notes.go +++ b/native/cmd/lich-native/browser_notes.go @@ -16,7 +16,6 @@ import ( "github.com/lichdotdev/lich/api" "github.com/lichdotdev/lich/native/backend" "github.com/lichdotdev/lich/native/i18n" - "github.com/lichdotdev/lich/native/ui" "github.com/lichdotdev/lich/native/ui/icons" ) @@ -76,7 +75,10 @@ type browserNotes struct { found chan foundElement located chan locatedNotes sent chan []*browserNote - nextLocate time.Time + failed chan string + // failure is why the last send failed, shown in the tray until the next. + failure string + nextLocate time.Time // followUntil is when the fast lookups after a scroll end. followUntil time.Time } @@ -97,7 +99,7 @@ type locatedNotes struct { } func newBrowserNotes() browserNotes { - n := browserNotes{found: make(chan foundElement, 4), located: make(chan locatedNotes, 1), sent: make(chan []*browserNote, 4)} + n := browserNotes{found: make(chan foundElement, 4), located: make(chan locatedNotes, 1), sent: make(chan []*browserNote, 4), failed: make(chan string, 1)} n.rows.Axis = layout.Vertical return n } @@ -226,6 +228,8 @@ func (u *window) takeFound(gtx C, p *browserPaneUI, session string) { n.applyBoxes(l) case sent := <-n.sent: n.delivered(sent) + case why := <-n.failed: + n.failure = why default: done = true } @@ -287,16 +291,16 @@ func (u *window) notesPointer(gtx C, p *browserPaneUI, session string, x, y floa } } -// notesKey takes the keys of the pick from the page: Ctrl+Shift+E turns it -// on and off, Esc closes the note or stops the pick, and Ctrl+Enter sends -// the notes. It reports whether it took e. +// notesKey takes the keys of the pick from the page: Ctrl+Shift+E (Cmd on +// macOS) turns it on and off, Esc closes the note or stops the pick, and +// Ctrl+Enter sends the notes. It reports whether it took e. func (u *window) notesKey(p *browserPaneUI, session string, e key.Event) bool { if e.State != key.Press { return false } n := &p.notes switch { - case e.Name == "E" && e.Modifiers == key.ModCtrl|key.ModShift: + case e.Name == "E" && e.Modifiers == key.ModShortcut|key.ModShift: n.setPicking(!n.picking) case e.Name == key.NameEscape && n.picking: if n.open { @@ -304,7 +308,7 @@ func (u *window) notesKey(p *browserPaneUI, session string, e key.Event) bool { } else { n.setPicking(false) } - case (e.Name == key.NameReturn || e.Name == key.NameEnter) && e.Modifiers == key.ModCtrl && len(n.list) > 0: + case (e.Name == key.NameReturn || e.Name == key.NameEnter) && e.Modifiers == key.ModShortcut && len(n.list) > 0: u.sendNotes(p, session) default: return false @@ -314,16 +318,17 @@ func (u *window) notesKey(p *browserPaneUI, session string, e key.Event) bool { // sendNotes types the notes into the session's prompt as one message. They // leave the tray only once it is there: a send that fails keeps them, to send -// again. +// again, and says why in the tray rather than in a toast over the page. func (u *window) sendNotes(p *browserPaneUI, session string) { n := &p.notes notes := n.sending() message := notesMessage(notes) n.setPicking(false) label := u.model.sessionLabel(session) + n.failure = "" go func() { if err := u.client.Submit(u.ctx, session, message); err != nil { - u.toaster.Show(ui.Toast{Variant: ui.ToastError, Title: i18n.T("layouts.browser.notesFailed", i18n.Args{"label": label, "error": backend.ErrorText(err)})}) + offer(n.failed, i18n.T("layouts.browser.notesFailed", i18n.Args{"label": label, "error": backend.ErrorText(err)})) } else { offer(n.sent, notes) } diff --git a/native/cmd/lich-native/browser_notes_view.go b/native/cmd/lich-native/browser_notes_view.go index e910df670..63f6ce153 100644 --- a/native/cmd/lich-native/browser_notes_view.go +++ b/native/cmd/lich-native/browser_notes_view.go @@ -49,6 +49,10 @@ var ( noteRowsShown = 3 ) +// noteScrollRange takes every wheel turn over the popover and the tray, so it +// does not scroll the page under them. +var noteScrollRange = pointer.ScrollRange{Min: -1 << 20, Max: 1 << 20} + // notesTrayHeight is the tray's height for its notes; none hides it. The // tray floats over the page instead of taking room from it: a page that // shrank under each new note relaid itself out and blinked. @@ -142,8 +146,8 @@ func (u *window) hoverTag(gtx C, r image.Rectangle, el api.BrowserElement) { call.Add(gtx.Ops) } -// notePin is note i's number as a tab on top of its element's outline, at -// its left corner, the way DevTools labels an element; a click opens its +// notePin is note i's number as a tab on top of its element's ring, at its +// left, the way DevTools labels an element; a click opens its // note. The tab stays whole inside the page. func (u *window) notePin(gtx C, n *browserNotes, i int, r image.Rectangle, lit bool) { note := n.list[i] @@ -158,7 +162,8 @@ func (u *window) notePin(gtx C, n *browserNotes, i int, r image.Rectangle, lit b d := note.pin.Layout(tab, func(gtx C) D { return u.noteTab(gtx, i+1, lit, true) }) call := macro.Stop() page := gtx.Constraints.Max - at := image.Pt(r.Min.X, r.Min.Y-d.Size.Y) + // On the ring's straight top edge, past its rounded corner. + at := image.Pt(r.Min.X+gtx.Dp(noteRingRadius), r.Min.Y-d.Size.Y) at.X = min(max(at.X, 0), page.X-d.Size.X) at.Y = min(max(at.Y, 0), page.Y-d.Size.Y) defer op.Offset(at).Push(gtx.Ops).Pop() @@ -248,7 +253,7 @@ func (u *window) notePopover(gtx C, p *browserPaneUI, scale float32) { // The popover takes the pointer from the page under it. event.Op(gtx.Ops, &n.draft) for { - if _, ok := gtx.Event(pointer.Filter{Target: &n.draft, Kinds: pointer.Press | pointer.Release | pointer.Move | pointer.Drag}); !ok { + if _, ok := gtx.Event(pointer.Filter{Target: &n.draft, Kinds: pointer.Press | pointer.Release | pointer.Move | pointer.Drag | pointer.Scroll, ScrollY: noteScrollRange}); !ok { break } } @@ -367,7 +372,7 @@ func (u *window) notesTray(gtx C, p *browserPaneUI, session string) { n := &p.notes th := u.theme if n.clear.Clicked(gtx) { - n.list, n.open = nil, false + n.list, n.open, n.failure = nil, false, "" } if n.send.Clicked(gtx) && len(n.list) > 0 { u.sendNotes(p, session) @@ -404,7 +409,7 @@ func (u *window) notesTray(gtx C, p *browserPaneUI, session string) { // The tray takes the pointer from the page under it. event.Op(gtx.Ops, &n.rows) for { - if _, ok := gtx.Event(pointer.Filter{Target: &n.rows, Kinds: pointer.Press | pointer.Release | pointer.Move | pointer.Drag}); !ok { + if _, ok := gtx.Event(pointer.Filter{Target: &n.rows, Kinds: pointer.Press | pointer.Release | pointer.Move | pointer.Drag | pointer.Scroll, ScrollY: noteScrollRange}); !ok { break } } @@ -429,8 +434,13 @@ func (u *window) notesTrayHead(gtx C, n *browserNotes, session string) D { return layout.Flex{Alignment: layout.Middle}.Layout(gtx, layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.notes", i18n.Args{"count": len(n.list)})).Weight(font.SemiBold).Layout), ui.Gap(1.5), - layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.notesFor", i18n.Args{"label": u.model.sessionLabel(session)})).Muted().Layout), - layout.Flexed(1, func(gtx C) D { return D{Size: image.Pt(gtx.Constraints.Min.X, 0)} }), + layout.Flexed(1, func(gtx C) D { + if n.failure != "" { + return th.Text(ui.TextXS, n.failure).In(th.Destructive).Layout(gtx) + } + return th.Text(ui.TextXS, i18n.T("layouts.browser.notesFor", i18n.Args{"label": u.model.sessionLabel(session)})).Muted().Layout(gtx) + }), + ui.Gap(2), layout.Rigid(func(gtx C) D { clear := th.Button(&n.clear, i18n.T("layouts.browser.clearNotes")) clear.Variant, clear.Size = ui.VariantGhost, ui.SizeSM diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index b9e593c9f..6356bd3f3 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -305,6 +305,9 @@ func (u *window) browserInput(gtx C, p *browserPaneUI, session string, scale flo key.Filter{Focus: &p.tag, Name: "V", Required: key.ModCommand}, key.Filter{Focus: &p.tag, Name: "C", Required: key.ModCommand}, key.Filter{Focus: &p.tag, Name: "X", Required: key.ModCommand}, + key.Filter{Focus: &p.tag, Name: "E", Required: key.ModShortcut | key.ModShift}, + key.Filter{Focus: &p.tag, Name: key.NameReturn, Required: key.ModShortcut}, + key.Filter{Focus: &p.tag, Name: key.NameEnter, Required: key.ModShortcut}, transfer.TargetFilter{Target: &p.tag, Type: "application/text"}) if focused && !gtx.Focused(&p.url) && !gtx.Focused(&p.tag) && !gtx.Focused(&p.notes.editor) { gtx.Execute(key.FocusCmd{Tag: &p.tag}) From 97316326e9628c7da6217c8b934d315dfd629c42 Mon Sep 17 00:00:00 2001 From: omartelo Date: Tue, 27 Oct 2026 17:38:06 -0300 Subject: [PATCH 3/3] perf(browser): march the note ring at 20 fps and keep it still under the popover Every frame of the marching dashes redraws the whole window: with a note open and idle that held 31 to 42% of a core, and a lit tray row 38 to 62% (measured 2026-10-27). The ring under an open note is now still, and the march elsewhere repaints 20 times a second at the same speed. --- native/cmd/lich-native/browser_notes_ring.go | 6 +++++- native/cmd/lich-native/browser_notes_view.go | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/native/cmd/lich-native/browser_notes_ring.go b/native/cmd/lich-native/browser_notes_ring.go index e06785b17..32c785012 100644 --- a/native/cmd/lich-native/browser_notes_ring.go +++ b/native/cmd/lich-native/browser_notes_ring.go @@ -23,6 +23,10 @@ var ( noteDash = unit.Dp(5) // noteMarch is how long the dashes take to move one dash and gap along. noteMarch = 600 * time.Millisecond + // noteMarchFrame paces the march at 20 frames a second: every frame of it + // redraws the whole window, which at the display's rate cost 30 to 60% of + // a core (measured 2026-10-27). + noteMarchFrame = 50 * time.Millisecond ) // noteRing is the ring around the element at r. @@ -43,7 +47,7 @@ func dashedRing(gtx C, ring image.Rectangle, c color.NRGBA, width unit.Dp, march if marching { cycle := gtx.Now.UnixNano() % int64(noteMarch) phase = 2 * dash * float32(cycle) / float32(noteMarch) - gtx.Execute(op.InvalidateCmd{}) + gtx.Execute(op.InvalidateCmd{At: gtx.Now.Add(noteMarchFrame)}) } rr := roundedRect{min: f32.Pt(float32(ring.Min.X), float32(ring.Min.Y)), max: f32.Pt(float32(ring.Max.X), float32(ring.Max.Y)), radius: float32(gtx.Dp(noteRingRadius))} w := float32(gtx.Dp(width)) diff --git a/native/cmd/lich-native/browser_notes_view.go b/native/cmd/lich-native/browser_notes_view.go index 63f6ce153..061169d9c 100644 --- a/native/cmd/lich-native/browser_notes_view.go +++ b/native/cmd/lich-native/browser_notes_view.go @@ -89,7 +89,7 @@ func (u *window) notesOverlay(gtx C, p *browserPaneUI, session, pageURL string) ring := noteRing(gtx, cssRect(note.box, scale)) if i == lit { ringFill(gtx, ring, withAlpha(notePick, 0x1a)) - dashedRing(gtx, ring, notePick, 1.5, true) + dashedRing(gtx, ring, notePick, 1.5, !n.open) } else { ringFill(gtx, ring, withAlpha(notePick, 0x0d)) dashedRing(gtx, ring, notePick, 1.5, false) @@ -98,7 +98,9 @@ func (u *window) notesOverlay(gtx C, p *browserPaneUI, session, pageURL string) if n.open && n.editing < 0 { ring := noteRing(gtx, cssRect(n.draftBox(), scale)) ringFill(gtx, ring, withAlpha(notePick, 0x1a)) - dashedRing(gtx, ring, notePick, 1.5, true) + // Still while the note is written: the pick is made, and redrawing the + // window each frame for it cost a third of a core (measured 2026-10-27). + dashedRing(gtx, ring, notePick, 1.5, false) } if n.picking && !n.open && n.hover.Label != "" { ring := noteRing(gtx, cssRect(n.hover.Box, scale))