From 616adc8997570f84b1ea82a69ba36114196fd0de Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:07:17 +0000 Subject: [PATCH 01/19] state: lower a clock floor that a better time source contradicts CapClockFloor lowers clockFloor and signedClockFloor to at most a given time, in one locked update. The clock keeper uses it when the clock is NTP-synchronised and the floor is ahead of it: such a floor was raised by something wrong and must not keep pushing the clock forward. --- internal/state/state.go | 29 +++++++++++++++++++++ internal/state/state_test.go | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/internal/state/state.go b/internal/state/state.go index ddc4ffa..187a086 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -109,6 +109,35 @@ func (f File) RaiseClockFloor(unix int64) error { return err } +// CapClockFloor lowers the floor, and the signed floor, to at most unix. It is +// for a time source that outranks the floor (an NTP-synchronised clock): a +// floor ahead of that time was raised by something wrong (an uploader's +// clock, a manual date, a bad RTC, a backend signing from the future) and is +// itself wrong. It returns the state as it was before, and whether anything +// was lowered. +func (f File) CapClockFloor(unix int64) (before State, lowered bool, err error) { + _, err = f.Update(func(s *State) error { + before = *s + if s.ClockFloor > unix { + s.ClockFloor, lowered = unix, true + } + if s.SignedClockFloor > unix { + s.SignedClockFloor, lowered = unix, true + } + if !lowered { + return errUnchanged + } + return nil + }) + if errors.Is(err, errUnchanged) { + err = nil + } + return before, lowered, err +} + +// errUnchanged aborts an Update that would write the same state back. +var errUnchanged = errors.New("unchanged") + // RaiseSignedClockFloor records a verified package's createdAt, which also // raises the floor. func (f File) RaiseSignedClockFloor(unix int64) error { diff --git a/internal/state/state_test.go b/internal/state/state_test.go index e497a9d..8b92bc2 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -1,6 +1,7 @@ package state import ( + "os" "path/filepath" "testing" ) @@ -25,3 +26,52 @@ func TestWallClockCannotRatchetFarPastSignedTime(t *testing.T) { t.Fatal("floor went down") } } + +func TestCapClockFloor(t *testing.T) { + const ntp = int64(1_795_000_000) + cases := []struct { + name string + floor, signed int64 + wantFloor, wantSigned int64 + wantLowered, wantFileExists bool + }{ + {"nothing recorded", 0, 0, 0, 0, false, false}, + {"both behind", ntp - 100, ntp - 1000, ntp - 100, ntp - 1000, false, true}, + {"floor ahead", ntp + 86400*300, ntp - 1000, ntp, ntp - 1000, true, true}, + {"signed ahead too", ntp + 86400*300, ntp + 86400*1500, ntp, ntp, true, true}, + {"exactly at", ntp, ntp, ntp, ntp, false, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := File{Path: filepath.Join(t.TempDir(), "state.json")} + if tc.floor > 0 || tc.signed > 0 { + if _, err := f.Update(func(s *State) error { + s.ClockFloor, s.SignedClockFloor, s.ContentSequence = tc.floor, tc.signed, 7 + return nil + }); err != nil { + t.Fatal(err) + } + } + before, lowered, err := f.CapClockFloor(ntp) + if err != nil { + t.Fatal(err) + } + if lowered != tc.wantLowered { + t.Fatalf("lowered = %v, want %v", lowered, tc.wantLowered) + } + if before.ClockFloor != tc.floor || before.SignedClockFloor != tc.signed { + t.Fatalf("before = %+v", before) + } + s, _ := f.Load() + if s.ClockFloor != tc.wantFloor || s.SignedClockFloor != tc.wantSigned { + t.Fatalf("floor, signed = %d, %d; want %d, %d", s.ClockFloor, s.SignedClockFloor, tc.wantFloor, tc.wantSigned) + } + if tc.wantFileExists && s.ContentSequence != 7 { + t.Fatalf("other fields lost: %+v", s) + } + if _, err := os.Stat(f.Path); (err == nil) != tc.wantFileExists { + t.Fatalf("state.json exists = %v, want %v", err == nil, tc.wantFileExists) + } + }) + } +} From 319cf6fbea2da3e523ee8b8ac439f4d95cbce657 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:07:25 +0000 Subject: [PATCH 02/19] Sign the refresh channel's upgrade with the device key The backend accepted any WebSocket on /api/refresh-musallahboard that named a known device id, and a newcomer replaced the board's own connection. Device ids are not secret, so anyone could keep evicting a board from its push channel and watch when pushes arrive. The upgrade is now a device-signed GET (section 9.2: the path without the query, an empty body), signed afresh for every dial with the same helper as content-bundle and weather. The deviceId query stays, so a backend that predates this still finds the board by it. The backend verifies the signature when present; unsigned upgrades stay accepted, without taking over from a signed one, until the owner sets musallahboard.board-stream.require-signature. --- docs/architecture.md | 8 ++- internal/boardsync/boardsync_test.go | 79 ++++++++++++++++++++++++++++ internal/boardsync/refresh.go | 45 ++++++++++++++-- 3 files changed, 127 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 57be101..ce19394 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -439,7 +439,13 @@ The daemon, when `content_sync` is on and the board is enrolled: - syncs at startup, every 30 minutes, and 10 s after any message on the backend's refresh channel (`wss:///api/refresh-musallahboard?deviceId=`, - debounced), + debounced). The upgrade request is device-signed (9.2: `GET`, the path + without the query, empty body), signed afresh for every dial. The backend + takes the device from `X-MB-Device-Id`, refuses a `deviceId` query that + disagrees, and refuses a timestamp it has already accepted on this channel. + Unsigned upgrades from older agents are accepted only while the backend's + `musallahboard.board-stream.require-signature` is `false`, and never take + over from a signed connection, - with `POST /api/agent/content-bundle`, device-authenticated (9.2), body `{"days": , "haveMedia": ["", ...]}`, - and imports the returned package with source `sync`. diff --git a/internal/boardsync/boardsync_test.go b/internal/boardsync/boardsync_test.go index cb2ca1e..45886a2 100644 --- a/internal/boardsync/boardsync_test.go +++ b/internal/boardsync/boardsync_test.go @@ -277,6 +277,9 @@ func TestRefreshTriggersDebouncedSync(t *testing.T) { if r.URL.Query().Get("deviceId") != testDevice { t.Errorf("refresh deviceId = %q", r.URL.Query().Get("deviceId")) } + if err := verifyDeviceAuth(r, nil); err != "" { + t.Errorf("refresh upgrade: %s", err) + } c, err := websocket.Accept(w, r, nil) if err != nil { return @@ -321,6 +324,82 @@ func TestRefreshTriggersDebouncedSync(t *testing.T) { } } +// verifyDeviceAuth checks r's device signature the way the backend does +// (DeviceRequestAuthenticator), and returns what is wrong, or "". +func verifyDeviceAuth(r *http.Request, body []byte) string { + if got := r.Header.Get(devauth.HeaderDeviceID); got != testDevice { + return "device header " + strconv.Quote(got) + } + ts, err := strconv.ParseInt(r.Header.Get(devauth.HeaderTimestamp), 10, 64) + if err != nil { + return "timestamp: " + err.Error() + } + if d := time.Since(time.UnixMilli(ts)); d < -5*time.Minute || d > 5*time.Minute { + return "timestamp " + strconv.FormatInt(ts, 10) + " is not now" + } + sig, err := base64.StdEncoding.DecodeString(r.Header.Get(devauth.HeaderSignature)) + if err != nil { + return "signature: " + err.Error() + } + msg := devauth.Message(r.Method, r.URL.Path, testDevice, ts, body) + if !ed25519.Verify(deviceKey.Public().(ed25519.PublicKey), msg, sig) { + return "signature does not verify" + } + return "" +} + +// The refresh upgrade is a device-signed GET over the path alone. Ed25519 is +// deterministic, so a fixed key, device and time give a fixed signature. The +// backend's BoardStreamHandshakeInterceptorTest verifies this exact value with +// the real DeviceRequestAuthenticator: if either side changes how it builds +// the signed message, one of the two tests fails. +func TestRefreshHeaderSignedGolden(t *testing.T) { + e := newEnv(t, "https://api.example/") + e.syncer.d.Key = ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, 32)) + e.syncer.now = func() time.Time { return time.UnixMilli(1790258531123) } + wsURL, err := refreshURL(e.cfg.BackendURL, testDevice) + if err != nil { + t.Fatal(err) + } + h, err := e.syncer.refreshHeader(wsURL) + if err != nil { + t.Fatal(err) + } + if got := h.Get(devauth.HeaderDeviceID); got != testDevice { + t.Errorf("device header = %q", got) + } + if got := h.Get(devauth.HeaderTimestamp); got != "1790258531123" { + t.Errorf("timestamp header = %q", got) + } + if got := h.Get("User-Agent"); got != "musallahboard-agent/0.3.0" { + t.Errorf("User-Agent = %q", got) + } + sig, _ := base64.StdEncoding.DecodeString(h.Get(devauth.HeaderSignature)) + msg := devauth.Message("GET", "/api/refresh-musallahboard", testDevice, 1790258531123, nil) + if !ed25519.Verify(e.syncer.d.Key.Public().(ed25519.PublicKey), msg, sig) { + t.Fatal("signature does not verify over GET, the path without its query, and an empty body") + } + const golden = "TGTiOMTSt8N/7ad06yMvmWo9a68Yz01LNGYnDOPGfedeiPTPjC9hqtKa/yLAOa7rXlDa5UfKTpmI28tdSME4Dg==" + if got := h.Get(devauth.HeaderSignature); got != golden { + t.Errorf("signature = %s, want %s", got, golden) + } +} + +// Each dial signs afresh: the backend refuses a timestamp it has already +// accepted on this channel, so a reconnect must never resend old headers. +func TestRefreshHeaderFreshPerDial(t *testing.T) { + e := newEnv(t, "https://api.example") + now := time.UnixMilli(1790258531000) + e.syncer.now = func() time.Time { now = now.Add(time.Second); return now } + wsURL, _ := refreshURL(e.cfg.BackendURL, testDevice) + a, _ := e.syncer.refreshHeader(wsURL) + b, _ := e.syncer.refreshHeader(wsURL) + if a.Get(devauth.HeaderTimestamp) == b.Get(devauth.HeaderTimestamp) || + a.Get(devauth.HeaderSignature) == b.Get(devauth.HeaderSignature) { + t.Fatal("two dials carried the same timestamp or signature") + } +} + func TestRefreshURL(t *testing.T) { for in, want := range map[string]string{ "https://api.example": "wss://api.example/api/refresh-musallahboard?deviceId=" + testDevice, diff --git a/internal/boardsync/refresh.go b/internal/boardsync/refresh.go index 43e34b5..d649ece 100644 --- a/internal/boardsync/refresh.go +++ b/internal/boardsync/refresh.go @@ -18,9 +18,10 @@ import ( // the socket would otherwise be redialled every second. const refreshStable = 30 * time.Second -// refreshLoop holds the backend's refresh channel open, the same socket the -// hosted board listens on. The backend sends a message whenever something on -// the board's content changes; each one schedules a content sync. +// refreshLoop holds the backend's refresh channel open. The backend sends a +// message whenever something on the board's content changes; each one +// schedules a content sync. The upgrade is device-signed (refreshHeader), so +// nobody who merely knows the device id can take the board's place on it. func (s *Syncer) refreshLoop(ctx context.Context) { wsURL, err := refreshURL(s.d.Cfg.BackendURL, s.d.Cfg.DeviceID) if err != nil { @@ -52,9 +53,13 @@ func (s *Syncer) refreshOnce(ctx context.Context, wsURL string) error { // meant to stay open for hours and is bounded by dialCtx instead. wsClient := *s.client wsClient.Timeout = 0 + header, err := s.refreshHeader(wsURL) + if err != nil { + return err + } conn, _, err := websocket.Dial(dialCtx, wsURL, &websocket.DialOptions{ HTTPClient: &wsClient, - HTTPHeader: http.Header{"User-Agent": []string{"musallahboard-agent/" + s.d.AgentVersion}}, + HTTPHeader: header, }) if err != nil { return fmt.Errorf("dial: %w", err) @@ -73,6 +78,38 @@ func (s *Syncer) refreshOnce(ctx context.Context, wsURL string) error { } } +// refreshHeader is the upgrade request's headers: the User-Agent and the +// device signature (docs/architecture.md, section 9.2) over GET, the path and +// an empty body. It is signed afresh for every dial, since the backend allows +// only a few minutes of clock difference and refuses a timestamp it has +// already seen on this channel. +// +// The deviceId query parameter stays: the signature does not cover the query, +// so the backend takes the device from X-MB-Device-Id and only checks that the +// two agree, and a backend that predates signed upgrades still finds the +// board by it. +func (s *Syncer) refreshHeader(wsURL string) (http.Header, error) { + u, err := url.Parse(wsURL) + if err != nil { + return nil, err + } + // The path is what is signed; the scheme only has to be one + // http.NewRequest accepts. websocket.Dial makes the same swap. + switch u.Scheme { + case "wss": + u.Scheme = "https" + case "ws": + u.Scheme = "http" + } + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "musallahboard-agent/"+s.d.AgentVersion) + signRequest(req, nil, s.d.Cfg.DeviceID, s.d.Key, s.now()) + return req.Header, nil +} + // refreshURL derives ws(s):///api/refresh-musallahboard?deviceId= // from backend_url. func refreshURL(backendURL, deviceID string) (string, error) { From c0ac84676e9304fc29fb3864317469c4b51758b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:07:30 +0000 Subject: [PATCH 03/19] Report a crashed or hung kiosk page as not alive checkKiosk fell through to `systemctl is-active` for any probe failure other than a missing getStatus() hook. A crashed tab ("Aw, Snap!") or hung JavaScript leaves the kiosk unit active, so a blank board showed as healthy in the admin console. - cdp: the page probe opens one connection, sends Inspector.enable (which Chromium answers with Inspector.targetCrashed on a tab whose renderer is gone) and then Runtime.evaluate. Failures are classified: ErrDebuggerUnreachable / ErrNoPageTarget (no browser to ask yet), ErrPageCrashed, ErrPageUnresponsive (connection accepted, no answer in time), ErrPageNavigating (context destroyed by a navigation), ErrPageThrew. CDP error replies are a typed RPCError. Error texts are unchanged. - telemetry: crashed or unresponsive is kioskAlive=false; splash, update screen, old board apps, a page mid-navigation or a throwing hook are alive; only an unreachable debugger falls back to the unit state. - First tests for internal/cdp, against a fake debugger endpoint. --- internal/cdp/cdp.go | 215 +++++++++++++++++++++++--- internal/cdp/cdp_test.go | 253 +++++++++++++++++++++++++++++++ internal/telemetry/kiosk_test.go | 47 ++++++ internal/telemetry/telemetry.go | 26 +++- 4 files changed, 513 insertions(+), 28 deletions(-) create mode 100644 internal/cdp/cdp_test.go diff --git a/internal/cdp/cdp.go b/internal/cdp/cdp.go index 5cc3a50..ceb8435 100644 --- a/internal/cdp/cdp.go +++ b/internal/cdp/cdp.go @@ -1,6 +1,7 @@ -// Package cdp is a minimal Chrome DevTools Protocol client for the three +// Package cdp is a minimal Chrome DevTools Protocol client for the few // methods the agent actually uses against the kiosk Chromium instance: -// Page.reload, Page.captureScreenshot, Runtime.evaluate. +// Page.reload, Page.navigate, Page.captureScreenshot, Runtime.evaluate, and +// Inspector.enable to tell a crashed renderer from a live one. // // We don't pull in chromedp because we don't drive a browser — we attach to // an already-running Chromium with --remote-debugging-port=9222 and only @@ -65,8 +66,63 @@ func New(debuggerURL string) *Client { } } +// Errors that say how far a probe of the kiosk got. They separate "the +// browser is not there (yet)", which only the kiosk unit's state can answer, +// from "the browser is there but its page is dead", which is a broken board. +var ( + // ErrDebuggerUnreachable: nothing answers on the debugging port, or it + // answers garbage. Chromium is starting, restarting, or was built without + // --remote-debugging-port. + ErrDebuggerUnreachable = errors.New("chrome not reachable") + // ErrNoPageTarget: the browser answers but has no page open, which is + // normal for a moment while it starts. + ErrNoPageTarget = errors.New("no page targets in chrome") + // ErrPageCrashed: the page's renderer is gone ("Aw, Snap!"). The browser + // keeps the tab, and its URL, so /json alone does not show it. + ErrPageCrashed = errors.New("cdp: page renderer crashed") + // ErrPageUnresponsive: the browser accepted a debugger connection to the + // page but the page did not answer in time (hung JavaScript, a wedged or + // crashed renderer). + ErrPageUnresponsive = errors.New("cdp: page is not answering") + // ErrPageNavigating: the page's JavaScript context went away under the + // call, because the page is navigating or reloading. Transient. + ErrPageNavigating = errors.New("cdp: page is navigating") + // ErrPageThrew: the expression threw. The page is alive and answered. + ErrPageThrew = errors.New("cdp: page threw") +) + +// RPCError is an error reply from Chromium to one CDP method call. +type RPCError struct { + Method string + Code int + Message string +} + +func (e *RPCError) Error() string { + return fmt.Sprintf("cdp %s: %s (code %d)", e.Method, e.Message, e.Code) +} + +// navigatingMessages are the error replies Chromium gives a Runtime call +// whose execution context was torn down by a navigation or reload. +var navigatingMessages = []string{ + "Execution context was destroyed", + "Cannot find default execution context", + "Cannot find context with specified id", + "Inspected target navigated or closed", +} + +func isNavigatingError(e *RPCError) bool { + for _, m := range navigatingMessages { + if strings.Contains(e.Message, m) { + return true + } + } + return false +} + // FirstPageTarget returns the WS URL of the first non-DevTools page tab. -// Errors if Chromium isn't reachable or has no pages open. +// Errors if Chromium isn't reachable (ErrDebuggerUnreachable) or has no pages +// open (ErrNoPageTarget). func (c *Client) FirstPageTarget(ctx context.Context) (Target, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.debuggerURL+"/json", nil) if err != nil { @@ -74,23 +130,23 @@ func (c *Client) FirstPageTarget(ctx context.Context) (Target, error) { } resp, err := c.http.Do(req) if err != nil { - return Target{}, fmt.Errorf("chrome not reachable at %s: %w", c.debuggerURL, err) + return Target{}, fmt.Errorf("%w at %s: %w", ErrDebuggerUnreachable, c.debuggerURL, err) } defer resp.Body.Close() if resp.StatusCode != 200 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) - return Target{}, fmt.Errorf("chrome /json returned %d: %s", resp.StatusCode, body) + return Target{}, fmt.Errorf("%w: /json returned %d: %s", ErrDebuggerUnreachable, resp.StatusCode, body) } var targets []Target if err := json.NewDecoder(resp.Body).Decode(&targets); err != nil { - return Target{}, fmt.Errorf("decode targets: %w", err) + return Target{}, fmt.Errorf("%w: decode targets: %w", ErrDebuggerUnreachable, err) } for _, t := range targets { if t.Type == "page" && t.WebSocketDebuggerURL != "" { return t, nil } } - return Target{}, fmt.Errorf("no page targets in chrome (%d total)", len(targets)) + return Target{}, fmt.Errorf("%w (%d total)", ErrNoPageTarget, len(targets)) } type rpcRequest struct { @@ -143,7 +199,7 @@ func (c *Client) Call(ctx context.Context, target Target, method string, params, continue } if resp.Error != nil { - return fmt.Errorf("cdp %s: %s (code %d)", method, resp.Error.Message, resp.Error.Code) + return &RPCError{Method: method, Code: resp.Error.Code, Message: resp.Error.Message} } if out != nil && len(resp.Result) > 0 { if err := json.Unmarshal(resp.Result, out); err != nil { @@ -192,20 +248,133 @@ type PageStatus struct { Error string `json:"error"` } -// PageStatus asks the board what it is currently showing. -// -// Three outcomes worth distinguishing: -// - nil error: the board is up and answered. -// - ErrEvalUndefined: a browser page exists but has no hook — the local -// enrollment splash, or a frontend build older than getStatus(). The -// browser is alive; it just isn't the board. -// - any other error: Chromium is unreachable. +// PageStatus asks the board what it is currently showing. It probes the page +// the way ProbeEvaluate does, so the error says what state the kiosk is in: +// - nil: the board is up and answered. +// - ErrEvalUndefined: a page answered but has no hook — the local +// enrollment splash, the update screen, or a frontend build older than +// getStatus(). The page is alive; it just isn't the board. +// - ErrPageThrew, ErrPageNavigating: the page is alive. +// - ErrPageCrashed, ErrPageUnresponsive: the browser is up and the page is +// dead. The board is not on screen. +// - ErrDebuggerUnreachable, ErrNoPageTarget: nothing to ask; the browser +// is not (yet) there. func (c *Client) PageStatus(ctx context.Context) (PageStatus, error) { var s PageStatus - err := c.EvaluateValue(ctx, "window.MusallahBoard?.getStatus?.() ?? null", &s) + raw, err := c.ProbeEvaluate(ctx, "window.MusallahBoard?.getStatus?.() ?? null") + if err != nil { + return s, err + } + err = decodeEvalReply(raw, &s) return s, err } +// PageHealth reports whether the kiosk page's renderer answers, with the +// same errors as PageStatus. A nil error means the page ran a trivial script. +func (c *Client) PageHealth(ctx context.Context) error { + raw, err := c.ProbeEvaluate(ctx, "true") + if err != nil { + return err + } + return decodeEvalReply(raw, nil) +} + +// ProbeEvaluate runs expression in the first page target like RuntimeEvaluate, +// but classifies how a failure happened instead of returning a bare network +// error: +// +// - the debugger cannot be reached or the WebSocket to the page cannot be +// opened: ErrDebuggerUnreachable (or ErrNoPageTarget); +// - Chromium reports the renderer crashed (Inspector.targetCrashed, which +// Inspector.enable sends at once on a tab whose renderer is gone): +// ErrPageCrashed; +// - the connection is open but no reply arrives before ctx's deadline: +// ErrPageUnresponsive. The Inspector domain is answered by the browser +// process, Runtime by the renderer, so a hung or dead renderer shows up +// here; +// - the call is cut off by a navigation: ErrPageNavigating. +// +// The caller's deadline is the evaluate timeout; it should leave the browser +// a few seconds (the board probes with 3-5 s on loopback). +func (c *Client) ProbeEvaluate(ctx context.Context, expression string) (json.RawMessage, error) { + target, err := c.FirstPageTarget(ctx) + if err != nil { + return nil, err + } + conn, _, err := websocket.Dial(ctx, target.WebSocketDebuggerURL, nil) + if err != nil { + // Not even a connection: the browser is going away or not there yet, + // which says nothing about the page. + return nil, fmt.Errorf("%w: ws dial cdp: %w", ErrDebuggerUnreachable, err) + } + defer conn.CloseNow() + conn.SetReadLimit(maxReplyBytes) + + enableID := c.id.Add(1) + evalID := c.id.Add(1) + for _, req := range []rpcRequest{ + {ID: enableID, Method: "Inspector.enable"}, + {ID: evalID, Method: "Runtime.evaluate", Params: map[string]any{ + "expression": expression, + "awaitPromise": true, + "returnByValue": true, + }}, + } { + data, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal: %w", err) + } + if err := conn.Write(ctx, websocket.MessageText, data); err != nil { + return nil, probeReadError(ctx, "write", err) + } + } + + for { + _, msg, err := conn.Read(ctx) + if err != nil { + return nil, probeReadError(ctx, "read", err) + } + var frame struct { + rpcResponse + Method string `json:"method"` + } + if err := json.Unmarshal(msg, &frame); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + switch { + case frame.Method == "Inspector.targetCrashed": + return nil, fmt.Errorf("%w (%s)", ErrPageCrashed, target.URL) + case frame.Method != "" || frame.ID != evalID: + // Other events, and the reply to Inspector.enable. + continue + case frame.Error != nil: + rerr := &RPCError{Method: "Runtime.evaluate", Code: frame.Error.Code, Message: frame.Error.Message} + if isNavigatingError(rerr) { + return nil, fmt.Errorf("%w: %w", ErrPageNavigating, rerr) + } + if strings.Contains(strings.ToLower(rerr.Message), "crashed") { + return nil, fmt.Errorf("%w: %w", ErrPageCrashed, rerr) + } + return nil, rerr + default: + return frame.Result, nil + } + } +} + +// probeReadError classifies a failure on an open debugger connection. Once +// the browser has accepted the connection, running out of time means the page +// did not answer; the connection dropping means the browser went away. +func probeReadError(ctx context.Context, op string, err error) error { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return fmt.Errorf("%w: %s: %w", ErrPageUnresponsive, op, err) + } + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("%w: %s: %w", ErrDebuggerUnreachable, op, err) +} + // ErrEvalUndefined means the expression resolved to `undefined`. Usually an // optional call short-circuiting because the page doesn't implement the hook // (an old frontend build, or the local enrollment splash rather than the @@ -237,6 +406,12 @@ func (c *Client) EvaluateValue(ctx context.Context, expression string, out any) if err != nil { return err } + return decodeEvalReply(raw, out) +} + +// decodeEvalReply turns a Runtime.evaluate result into out, ErrPageThrew or +// ErrEvalUndefined. +func decodeEvalReply(raw json.RawMessage, out any) error { var reply evalReply if err := json.Unmarshal(raw, &reply); err != nil { return fmt.Errorf("cdp: decode evaluate reply: %w", err) @@ -246,7 +421,7 @@ func (c *Client) EvaluateValue(ctx context.Context, expression string, out any) if reply.ExceptionDetails.Exception != nil && reply.ExceptionDetails.Exception.Description != "" { msg = reply.ExceptionDetails.Exception.Description } - return fmt.Errorf("cdp: page threw: %s", msg) + return fmt.Errorf("%w: %s", ErrPageThrew, msg) } if reply.Result.Type == "undefined" || len(reply.Result.Value) == 0 || string(reply.Result.Value) == "null" { return ErrEvalUndefined @@ -269,8 +444,8 @@ func (c *Client) RuntimeEvaluate(ctx context.Context, expression string) (json.R } var out json.RawMessage if err := c.Call(ctx, target, "Runtime.evaluate", map[string]any{ - "expression": expression, - "awaitPromise": true, + "expression": expression, + "awaitPromise": true, "returnByValue": true, }, &out); err != nil { return nil, err diff --git a/internal/cdp/cdp_test.go b/internal/cdp/cdp_test.go new file mode 100644 index 0000000..fca81b2 --- /dev/null +++ b/internal/cdp/cdp_test.go @@ -0,0 +1,253 @@ +package cdp + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/coder/websocket" +) + +// fakeChrome is a debugger endpoint with one page target. Each incoming CDP +// request is answered by reply; a nil frame list means "never answer" (a +// hung renderer). +type fakeChrome struct { + t *testing.T + srv *httptest.Server + url string // the page target's URL as /json lists it + noPages bool + reply func(req rpcRequest) []any + + mu sync.Mutex + methods []string +} + +func newFakeChrome(t *testing.T, reply func(req rpcRequest) []any) *fakeChrome { + t.Helper() + f := &fakeChrome{t: t, url: "http://127.0.0.1:8080/", reply: reply} + mux := http.NewServeMux() + mux.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) { + targets := []Target{{ID: "devtools", Type: "other", URL: "devtools://x"}} + if !f.noPages { + ws := "ws" + strings.TrimPrefix(f.srv.URL, "http") + "/devtools/page/P1" + targets = append(targets, Target{ID: "P1", Type: "page", URL: f.url, WebSocketDebuggerURL: ws}) + } + _ = json.NewEncoder(w).Encode(targets) + }) + mux.HandleFunc("/devtools/page/P1", func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer conn.CloseNow() + ctx := r.Context() + for { + _, msg, err := conn.Read(ctx) + if err != nil { + return + } + var req rpcRequest + if err := json.Unmarshal(msg, &req); err != nil { + return + } + f.mu.Lock() + f.methods = append(f.methods, req.Method) + f.mu.Unlock() + for _, frame := range f.reply(req) { + data, _ := json.Marshal(frame) + if err := conn.Write(ctx, websocket.MessageText, data); err != nil { + return + } + } + } + }) + f.srv = httptest.NewServer(mux) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeChrome) client() *Client { return New(f.srv.URL) } + +func (f *fakeChrome) seen() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.methods...) +} + +func result(id int64, v any) map[string]any { return map[string]any{"id": id, "result": v} } + +func rpcErr(id int64, code int, msg string) map[string]any { + return map[string]any{"id": id, "error": map[string]any{"code": code, "message": msg}} +} + +func evalValue(v any) map[string]any { + return map[string]any{"result": map[string]any{"type": "object", "value": v}} +} + +// healthy answers everything; Runtime.evaluate with value. +func healthy(value any) func(rpcRequest) []any { + return func(req rpcRequest) []any { + if req.Method == "Runtime.evaluate" { + if value == nil { + return []any{result(req.ID, map[string]any{"result": map[string]any{"type": "undefined"}})} + } + return []any{result(req.ID, evalValue(value))} + } + return []any{result(req.ID, map[string]any{})} + } +} + +func probeCtx(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + t.Cleanup(cancel) + return ctx +} + +func TestPageStatusBoardAnswers(t *testing.T) { + f := newFakeChrome(t, healthy(map[string]any{"paired": true, "slideKey": "poster-3"})) + + st, err := f.client().PageStatus(probeCtx(t)) + if err != nil { + t.Fatalf("PageStatus: %v", err) + } + if st.SlideKey != "poster-3" || !st.Paired { + t.Errorf("status = %+v", st) + } + if got := f.seen(); len(got) != 2 || got[0] != "Inspector.enable" || got[1] != "Runtime.evaluate" { + t.Errorf("methods = %v, want Inspector.enable then Runtime.evaluate", got) + } +} + +// The splash, the update screen and old board apps have no getStatus(). +func TestPageStatusNoHook(t *testing.T) { + f := newFakeChrome(t, healthy(nil)) + if _, err := f.client().PageStatus(probeCtx(t)); !errors.Is(err, ErrEvalUndefined) { + t.Fatalf("err = %v, want ErrEvalUndefined", err) + } +} + +// Chromium answers Inspector.enable on a tab whose renderer is gone with an +// Inspector.targetCrashed event, and never answers Runtime.evaluate. +func TestPageStatusCrashedRenderer(t *testing.T) { + f := newFakeChrome(t, func(req rpcRequest) []any { + if req.Method == "Inspector.enable" { + return []any{map[string]any{"method": "Inspector.targetCrashed", "params": map[string]any{}}, result(req.ID, map[string]any{})} + } + return nil + }) + start := time.Now() + _, err := f.client().PageStatus(probeCtx(t)) + if !errors.Is(err, ErrPageCrashed) { + t.Fatalf("err = %v, want ErrPageCrashed", err) + } + if time.Since(start) > 400*time.Millisecond { + t.Errorf("a crash should be reported at once, took %v", time.Since(start)) + } +} + +// Hung JavaScript: the browser takes the connection, the renderer never +// replies. +func TestPageStatusHungRenderer(t *testing.T) { + f := newFakeChrome(t, func(req rpcRequest) []any { + if req.Method == "Inspector.enable" { + return []any{result(req.ID, map[string]any{})} + } + return nil + }) + _, err := f.client().PageStatus(probeCtx(t)) + if !errors.Is(err, ErrPageUnresponsive) { + t.Fatalf("err = %v, want ErrPageUnresponsive", err) + } + if errors.Is(err, ErrDebuggerUnreachable) { + t.Errorf("a hung page must not read as an unreachable debugger: %v", err) + } +} + +func TestPageStatusNavigating(t *testing.T) { + f := newFakeChrome(t, func(req rpcRequest) []any { + if req.Method == "Runtime.evaluate" { + return []any{rpcErr(req.ID, -32000, "Execution context was destroyed.")} + } + return []any{result(req.ID, map[string]any{})} + }) + if _, err := f.client().PageStatus(probeCtx(t)); !errors.Is(err, ErrPageNavigating) { + t.Fatalf("err = %v, want ErrPageNavigating", err) + } +} + +func TestPageStatusThrew(t *testing.T) { + f := newFakeChrome(t, func(req rpcRequest) []any { + if req.Method == "Runtime.evaluate" { + return []any{result(req.ID, map[string]any{ + "result": map[string]any{"type": "object"}, + "exceptionDetails": map[string]any{"text": "Uncaught", "exception": map[string]any{"description": "TypeError: boom"}}, + })} + } + return []any{result(req.ID, map[string]any{})} + }) + _, err := f.client().PageStatus(probeCtx(t)) + if !errors.Is(err, ErrPageThrew) || !strings.Contains(err.Error(), "TypeError: boom") { + t.Fatalf("err = %v, want ErrPageThrew with the description", err) + } +} + +func TestPageStatusNoPageTarget(t *testing.T) { + f := newFakeChrome(t, healthy(true)) + f.noPages = true + if _, err := f.client().PageStatus(probeCtx(t)); !errors.Is(err, ErrNoPageTarget) { + t.Fatalf("err = %v, want ErrNoPageTarget", err) + } +} + +func TestPageStatusDebuggerUnreachable(t *testing.T) { + f := newFakeChrome(t, healthy(true)) + c := f.client() + f.srv.Close() + _, err := c.PageStatus(probeCtx(t)) + if !errors.Is(err, ErrDebuggerUnreachable) { + t.Fatalf("err = %v, want ErrDebuggerUnreachable", err) + } + if !strings.Contains(err.Error(), "chrome not reachable") { + t.Errorf("message changed: %v", err) + } +} + +func TestPageHealth(t *testing.T) { + f := newFakeChrome(t, healthy(true)) + if err := f.client().PageHealth(probeCtx(t)); err != nil { + t.Fatalf("PageHealth: %v", err) + } +} + +// Call keeps returning CDP error replies as errors, now typed. +func TestCallRPCError(t *testing.T) { + f := newFakeChrome(t, func(req rpcRequest) []any { + return []any{rpcErr(req.ID, -32601, "'Page.bogus' wasn't found")} + }) + c := f.client() + target, err := c.FirstPageTarget(probeCtx(t)) + if err != nil { + t.Fatal(err) + } + err = c.Call(probeCtx(t), target, "Page.bogus", nil, nil) + var rerr *RPCError + if !errors.As(err, &rerr) || rerr.Code != -32601 || rerr.Method != "Page.bogus" { + t.Fatalf("err = %v, want an RPCError for Page.bogus", err) + } +} + +func TestPageReloadSendsReload(t *testing.T) { + f := newFakeChrome(t, healthy(true)) + if err := f.client().PageReload(probeCtx(t)); err != nil { + t.Fatalf("PageReload: %v", err) + } + if got := f.seen(); len(got) != 1 || got[0] != "Page.reload" { + t.Errorf("methods = %v", got) + } +} diff --git a/internal/telemetry/kiosk_test.go b/internal/telemetry/kiosk_test.go index 4422f3b..60fca61 100644 --- a/internal/telemetry/kiosk_test.go +++ b/internal/telemetry/kiosk_test.go @@ -3,6 +3,7 @@ package telemetry import ( "context" "errors" + "fmt" "testing" "github.com/LensBridge/agent/internal/cdp" @@ -74,3 +75,49 @@ func TestCheckKioskNilProber(t *testing.T) { t.Errorf("frame = %q, want empty", frame) } } + +// A crashed or hung renderer leaves the kiosk unit active, so the unit state +// must not be consulted: the board is blank and has to be reported as such. +func TestCheckKioskDeadPageIsNotAlive(t *testing.T) { + for _, err := range []error{ + fmt.Errorf("%w (http://127.0.0.1:8080/)", cdp.ErrPageCrashed), + fmt.Errorf("%w: read: context deadline exceeded", cdp.ErrPageUnresponsive), + } { + p := &stubProber{err: err} + alive, frame := checkKiosk(context.Background(), p) + if alive { + t.Errorf("%v: kiosk reported alive", err) + } + if frame != "" { + t.Errorf("%v: frame = %q, want empty", err, frame) + } + } +} + +// A page between documents, or one whose hook threw, still answered. +func TestCheckKioskTransientPageStatesAreAlive(t *testing.T) { + for _, err := range []error{ + fmt.Errorf("%w: Execution context was destroyed.", cdp.ErrPageNavigating), + fmt.Errorf("%w: TypeError: x is undefined", cdp.ErrPageThrew), + } { + p := &stubProber{err: err} + if alive, _ := checkKiosk(context.Background(), p); !alive { + t.Errorf("%v: kiosk reported dead", err) + } + } +} + +// No browser to ask is not proof of death: the unit's state decides, and +// the probe errors that mean "no browser" must take that path. +func TestCheckKioskUnreachableUsesUnitState(t *testing.T) { + want := systemdKioskActive(context.Background()) + for _, err := range []error{ + fmt.Errorf("%w at http://localhost:9222: connection refused", cdp.ErrDebuggerUnreachable), + fmt.Errorf("%w (0 total)", cdp.ErrNoPageTarget), + } { + p := &stubProber{err: err} + if alive, _ := checkKiosk(context.Background(), p); alive != want { + t.Errorf("%v: alive = %v, want the unit state %v", err, alive, want) + } + } +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index c7b2abd..86e57cd 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -162,10 +162,14 @@ func readDiskUsedPct() (float64, error) { // // The systemd unit is Restart=always, so `is-active` says "active" almost // unconditionally — including while Chromium is crash-looping, parked on the -// enrollment splash, or showing a network error page. That made kioskAlive -// useless as an alert signal. Ask the browser instead, and fall back to the -// unit state only when the debugger is unreachable (no CDP port, browser still -// starting, or a build without --remote-debugging-port). +// enrollment splash, or showing a crashed tab ("Aw, Snap!"). Ask the browser +// instead, and fall back to the unit state only when there is no browser to +// ask (no CDP port, browser still starting, or a build without +// --remote-debugging-port). +// +// A browser that is reachable but whose page target crashed or does not +// answer evaluate in time is not alive, whatever systemd says: that is the +// blank board the admin console must see. func checkKiosk(ctx context.Context, prober PageProber) (alive bool, frameID string) { if prober != nil { pctx, cancel := context.WithTimeout(ctx, pageProbeTimeout) @@ -176,12 +180,18 @@ func checkKiosk(ctx context.Context, prober PageProber) (alive bool, frameID str case err == nil: // The board answered. It's alive by definition. return true, status.SlideKey - case errors.Is(err, cdp.ErrEvalUndefined): - // A page exists but isn't the board — the splash, or a frontend - // too old to expose getStatus(). The browser is up regardless. + case errors.Is(err, cdp.ErrEvalUndefined), + errors.Is(err, cdp.ErrPageThrew), + errors.Is(err, cdp.ErrPageNavigating): + // A page answered (or is between documents) but isn't the board: + // the splash, the update screen, a frontend too old to expose + // getStatus(), or a reload in flight. The browser is up. return true, "" + case errors.Is(err, cdp.ErrPageCrashed), errors.Is(err, cdp.ErrPageUnresponsive): + // The browser is there and its page is dead. + return false, "" } - // Anything else: Chromium unreachable. Fall through. + // Anything else: Chromium unreachable, or no page yet. Fall through. } return systemdKioskActive(ctx), "" } From 4dd7e9e36cf5e4126c75ebaa700771ca2b7a9651 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:09:09 +0000 Subject: [PATCH 04/19] Heal a crashed or hung kiosk page: reload, then restart the kiosk The kiosk watcher only navigated away from chrome-error: pages. A renderer crash or hung JavaScript left a blank board until someone noticed. Every 30 s the watcher now probes the page (Inspector.enable plus a trivial Runtime.evaluate, 5 s): - a crashed renderer after one probe, or a page that has not answered for three probes in a row (90 s; a Pi decoding a large poster can miss one), is reloaded; - if it is still dead at the next threshold, the kiosk unit is restarted through the same sudo path as the kiosk.restart command; - restarts are at least 5 min apart, doubling to 1 hour, and that backoff only starts over after an hour of a healthy page. A page that never comes back costs at most one reload and one restart an hour; one that recovers on reload is reloaded whenever it dies. The splash, the update screen (skipped entirely while it is up), old board apps without getStatus(), a page mid-navigation and a browser that is not up yet are never counted as failures. --- cmd/agent/board.go | 185 ++++++++++++++++++++++++++++-- cmd/agent/board_test.go | 245 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+), 9 deletions(-) create mode 100644 cmd/agent/board_test.go diff --git a/cmd/agent/board.go b/cmd/agent/board.go index ec3214c..d589ee3 100644 --- a/cmd/agent/board.go +++ b/cmd/agent/board.go @@ -148,7 +148,7 @@ func startBoard(ctx context.Context, logger *slog.Logger, cfg *config.Config, sa _, _ = daemon.SdNotify(false, "STATUS=serving the board on "+localserver.ListenAddr) logInstalled(logger, layout, cfg) - goRun(wg, func() { watchKiosk(ctx, logger, cdpClient) }) + goRun(wg, func() { watchKiosk(ctx, logger, cdpClient, screen.Active) }) goRun(wg, func() { // First say how an agent update that restarted us went: the rest of // its batch may be waiting in the inbox, and belongs on the same @@ -221,9 +221,28 @@ func startCommandChannel(ctx context.Context, logger *slog.Logger, cfg *config.C return wsClient } -// watchKiosk navigates Chromium back to the board if it lands on its own -// error page (the server restarting under it, for instance). -func watchKiosk(ctx context.Context, logger *slog.Logger, c *cdp.Client) { +// Kiosk self-healing (watchKiosk). The page is probed every +// kioskWatchInterval. A crashed renderer is certain after one probe; a page +// that does not answer may be busy (a large poster decoding on a Pi), so it +// gets kioskHungThreshold probes. Then the watcher reloads the page; if the +// page is still dead after that, it restarts the kiosk unit. Restarts back off +// from kioskRestartMinGap to kioskRestartMaxGap, and the backoff only starts +// over after the page has been healthy for kioskRestartResetAfter, so a page +// that never comes back costs at most one reload and one restart an hour. +const ( + kioskProbeTimeout = 5 * time.Second + kioskCrashThreshold = 1 + kioskHungThreshold = 3 + kioskRestartMinGap = 5 * time.Minute + kioskRestartMaxGap = time.Hour + kioskRestartResetAfter = time.Hour +) + +// watchKiosk keeps the kiosk on the board: it navigates Chromium back if it +// lands on its own error page (the server restarting under it, for instance) +// and heals a crashed or hung page (kioskHealer). +func watchKiosk(ctx context.Context, logger *slog.Logger, c *cdp.Client, updateActive func() bool) { + h := newKioskHealer(logger, c, updateActive) t := time.NewTicker(kioskWatchInterval) defer t.Stop() for { @@ -232,14 +251,162 @@ func watchKiosk(ctx context.Context, logger *slog.Logger, c *cdp.Client) { return case <-t.C: } - cctx, cancel := context.WithTimeout(ctx, 5*time.Second) - target, err := c.FirstPageTarget(cctx) - if err == nil && strings.HasPrefix(target.URL, "chrome-error:") { - logger.Info("kiosk is on an error page; returning it to the board") - _ = c.Call(cctx, target, "Page.navigate", map[string]any{"url": localserver.BaseURL}, nil) + h.tick(ctx) + } +} + +// kioskHealer is the watcher's state between ticks. Its browser and systemd +// operations are fields so the escalation can be tested without either. +type kioskHealer struct { + log *slog.Logger + now func() time.Time + updateActive func() bool + target func(context.Context) (cdp.Target, error) + navigate func(context.Context, cdp.Target, string) error + health func(context.Context) error + reload func(context.Context) error + restart func(context.Context) error + + fails int // consecutive dead-page probes + reloaded bool // reloaded since the last restart (or since healthy) + restarts int // restarts since the backoff last started over + nextRestart time.Time // no restart before this + healthySince time.Time // zero while the page is dead + waitLogged bool +} + +func newKioskHealer(logger *slog.Logger, c *cdp.Client, updateActive func() bool) *kioskHealer { + if updateActive == nil { + updateActive = func() bool { return false } + } + return &kioskHealer{ + log: logger, + now: time.Now, + updateActive: updateActive, + target: c.FirstPageTarget, + navigate: func(ctx context.Context, t cdp.Target, url string) error { + return c.Call(ctx, t, "Page.navigate", map[string]any{"url": url}, nil) + }, + health: c.PageHealth, + reload: c.PageReload, + restart: func(ctx context.Context) error { + // The same privileged path as the kiosk.restart remote command + // (sudo -n systemctl restart, allowed by the sudoers drop-in). + _, err := (&commands.KioskRestart{}).Execute(ctx, nil, func(string, string, *int) {}) + return err + }, + } +} + +func (h *kioskHealer) tick(ctx context.Context) { + // The update screen is driven over CDP by the importer and may be + // waiting on an agent restart; nothing here may navigate under it. + if h.updateActive() { + h.fails = 0 + return + } + + cctx, cancel := context.WithTimeout(ctx, kioskProbeTimeout) + target, err := h.target(cctx) + cancel() + if err == nil && strings.HasPrefix(target.URL, "chrome-error:") { + h.log.Info("kiosk is on an error page; returning it to the board") + cctx, cancel := context.WithTimeout(ctx, kioskProbeTimeout) + _ = h.navigate(cctx, target, localserver.BaseURL) + cancel() + return + } + if err != nil { + // No browser, or no page yet: starting, or restarting (perhaps + // because we asked). systemd's Restart=always owns that case. + return + } + + cctx, cancel = context.WithTimeout(ctx, kioskProbeTimeout) + err = h.health(cctx) + cancel() + if ctx.Err() != nil { + return + } + + threshold := 0 + switch { + case errors.Is(err, cdp.ErrPageCrashed): + threshold = kioskCrashThreshold + case errors.Is(err, cdp.ErrPageUnresponsive): + threshold = kioskHungThreshold + case errors.Is(err, cdp.ErrDebuggerUnreachable), errors.Is(err, cdp.ErrNoPageTarget): + // The browser went away between the two calls: not the page's fault, + // and not proof it recovered either. + return + default: + // Answered, or between documents, or the hook threw: the page is + // alive. A navigation in flight is not a failure. + h.healthy() + return + } + + h.healthySince = time.Time{} + h.fails++ + if h.fails < threshold { + h.log.Warn("kiosk page is not answering", "err", err, "failures", h.fails, "url", target.URL) + return + } + h.heal(ctx, err, target.URL) +} + +func (h *kioskHealer) healthy() { + now := h.now() + if h.fails > 0 || h.reloaded { + h.log.Info("kiosk page is answering again") + } + h.fails, h.reloaded, h.waitLogged = 0, false, false + if h.healthySince.IsZero() { + h.healthySince = now + } + if h.restarts > 0 && now.Sub(h.healthySince) >= kioskRestartResetAfter { + h.restarts, h.nextRestart = 0, time.Time{} + } +} + +// heal takes the next step for a dead page: a reload, at most once until the +// page answers again or the kiosk restarts; then a restart, at most once per +// backoff gap. A page that recovers each time it is reloaded is reloaded each +// time it dies, which is what it needs; a page that never recovers gets one +// reload and one restart per gap. +func (h *kioskHealer) heal(ctx context.Context, cause error, url string) { + if !h.reloaded { + h.fails, h.reloaded = 0, true + h.log.Warn("kiosk page is dead; reloading it", "err", cause, "url", url) + cctx, cancel := context.WithTimeout(ctx, kioskProbeTimeout) + if err := h.reload(cctx); err != nil { + h.log.Warn("could not reload the kiosk page", "err", err) } cancel() + return + } + + now := h.now() + if now.Before(h.nextRestart) { + if !h.waitLogged { + h.log.Error("kiosk page is still dead; waiting before restarting the kiosk again so a broken page cannot restart it in a loop", + "err", cause, "url", url, "nextAttempt", h.nextRestart.Format(time.RFC3339)) + h.waitLogged = true + } + return + } + + h.fails, h.reloaded, h.waitLogged = 0, false, false + gap := min(kioskRestartMinGap< 30 { + t.Errorf("%d restarts in a day: the backoff is not holding", r.k.restarts) + } + if r.k.restarts < 10 { + t.Errorf("only %d restarts in a day: the healer gave up", r.k.restarts) + } + if r.k.reloads > r.k.restarts+1 { + t.Errorf("%d reloads for %d restarts: reloads must be bounded too", r.k.reloads, r.k.restarts) + } +} + +func TestKioskHealerBackoffMinimumGap(t *testing.T) { + r := newRig(t) + r.k.health = errCrashed + r.ticks(2) + r.want(1, 1) + // After the restart the page gets one more reload, then nothing until + // kioskRestartMinGap has passed. + r.ticks(int(kioskRestartMinGap/kioskWatchInterval) - 1) + r.want(2, 1) + r.ticks(1) + r.want(2, 2) + r.ticks(1) + r.want(3, 2) +} + +// A page that recovers when reloaded is reloaded each time it dies, whatever +// the restart backoff says. +func TestKioskHealerReloadsAPageThatRecovers(t *testing.T) { + r := newRig(t) + for i := 0; i < 5; i++ { + r.k.health = errCrashed + r.ticks(1) + r.k.health = nil + r.ticks(2) + } + r.want(5, 0) +} + +// The backoff starts over only after a sustained healthy stretch; a page that +// comes up for a minute and dies again keeps its backoff. +func TestKioskHealerBackoffResetsAfterSustainedHealth(t *testing.T) { + r := newRig(t) + r.k.health = errCrashed + r.ticks(2) + r.want(1, 1) + + r.k.health = nil + r.ticks(2) + r.k.health = errCrashed + r.ticks(2) + r.want(2, 1) // reloaded; the restart is held by the backoff + + r.k.health = nil + r.ticks(int(kioskRestartResetAfter/kioskWatchInterval) + 1) + r.k.health = errCrashed + r.ticks(2) + r.want(3, 2) // reload, then a restart at once: the backoff started over + r.ticks(1) + r.want(4, 2) + r.ticks(1) + r.want(4, 2) // and the next restart waits kioskRestartMinGap again +} + +// Legitimate states are never healed. +func TestKioskHealerLeavesLegitimateStatesAlone(t *testing.T) { + for name, err := range map[string]error{ + "board": nil, + "splash or old": cdp.ErrEvalUndefined, + "navigating": fmt.Errorf("%w: Execution context was destroyed.", cdp.ErrPageNavigating), + "hook threw": fmt.Errorf("%w: TypeError", cdp.ErrPageThrew), + "browser gone": fmt.Errorf("%w: ws dial", cdp.ErrDebuggerUnreachable), + } { + t.Run(name, func(t *testing.T) { + r := newRig(t) + r.k.health = err + r.ticks(50) + r.want(0, 0) + }) + } +} + +func TestKioskHealerNoBrowser(t *testing.T) { + r := newRig(t) + r.k.targetErr = cdp.ErrDebuggerUnreachable + r.k.health = errCrashed // never consulted + r.ticks(50) + r.want(0, 0) +} + +// The update screen is driven over CDP by the importer; the healer must not +// reload or restart under it, even when it does not answer. +func TestKioskHealerSkipsUpdateScreen(t *testing.T) { + r := newRig(t) + r.updating = true + r.k.health = errHung + r.ticks(50) + r.want(0, 0) + + // Failures seen during the update do not carry over. + r.updating = false + r.ticks(kioskHungThreshold - 1) + r.want(0, 0) +} + +func TestKioskHealerChromeErrorPage(t *testing.T) { + r := newRig(t) + r.k.url = "chrome-error://chromewebdata/" + r.k.health = errCrashed + r.ticks(1) + if r.k.navigated != 1 { + t.Errorf("navigated = %d, want 1", r.k.navigated) + } + r.want(0, 0) +} From 129e9bf9d4785d3b8dcd8d9db4d40c3a6cdf06ab Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:10:10 +0000 Subject: [PATCH 05/19] clock: NTP is the authority; only trusted time raises the floor A laptop with a wrong clock uploading on the service port could pin a board in the future for good: the uploader's time became the clock floor, the keeper persisted the (now wrong) wall clock into it every ten minutes, and when timesyncd corrected the clock the keeper pushed it forward again, every ten minutes. Prayer times were wrong and, past five minutes of skew, the backend refused the board's signed requests. A manual date or a wrong RTC did the same. Only editing state.json helped. - With NTP synchronised, the keeper never moves the clock and ignores uploaders' clocks. A floor (or signed floor) ahead of NTP time is wrong: it is lowered to NTP time, with a warning in the journal. Boards already in the bad state recover by themselves once NTP has synchronised. - NTP time is written to the floor once two checks an interval apart agree with the monotonic clock, so a clock stepped by hand between timesyncd's polls is not taken for NTP time. - Without NTP the wall clock is never written to the floor: the floor advances from the last trusted time (the floor itself, a signed createdAt, NTP time) by the monotonic clock. A board that loses power still comes back no earlier than the time it last knew. - An uploader's time sets the clock (and the RTC) but no longer the floor, so the next uploader with a right clock, or NTP, undoes a wrong one. It still cannot move the clock below the floor or more than 90 days past it. The timesyncd check moves to internal/clock/timesync so the importer can ask it too. --- internal/clock/clock.go | 150 +++++++++--- internal/clock/clock_test.go | 347 +++++++++++++++++++++++++++- internal/clock/timesync/timesync.go | 26 +++ 3 files changed, 482 insertions(+), 41 deletions(-) create mode 100644 internal/clock/timesync/timesync.go diff --git a/internal/clock/clock.go b/internal/clock/clock.go index e1b0b1f..a46d8b6 100644 --- a/internal/clock/clock.go +++ b/internal/clock/clock.go @@ -4,14 +4,25 @@ // Which day's content is shown, and every prayer time on it, depends on the // clock. A Pi has no battery-backed clock unless someone fitted one, so after // a power cut on a board without internet it can come up years in the past. -// The defence is a floor that only moves forward: the newest signed createdAt -// of any verified package, and the wall clock itself, persisted every ten -// minutes. The clock is never allowed to sit more than a minute behind it. // -// The only other way the clock moves is an uploader's time, and only when a -// package in that same upload verified and the time is within 90 days of the -// floor, so an unauthenticated client cannot set the board to any date it -// likes. +// When systemd-timesyncd has synchronised the clock, NTP is the authority: +// the keeper never moves the clock, ignores uploaders' clocks, and lowers a +// remembered floor that is ahead of NTP time (it was raised by something +// wrong). The floor follows NTP time, once two checks an interval apart agree +// with the monotonic clock. +// +// Without NTP the defence is a floor that only moves forward: the newest +// signed createdAt of any verified package, and the last trusted time carried +// forward by the monotonic clock, persisted every ten minutes. The clock is +// never allowed to sit more than a minute behind it. The wall clock itself is +// never written into the floor without NTP: nothing has confirmed it, and a +// wrong one (an uploader's, a manual date, a bad RTC) would otherwise lock the +// board in the future. +// +// The only other way the clock moves is an uploader's time, and only without +// NTP, when a package in that same upload verified and the time is within 90 +// days past the floor. It sets the clock but not the floor, so a wrong +// uploader's time is undone by the next good one, or by NTP. // // The floor keeps the clock from going backwards; it cannot say the clock is // right. Info does: a board without NTP, an RTC, or a laptop's time since it @@ -26,8 +37,10 @@ import ( "os/exec" "strconv" "strings" + "sync" "time" + "github.com/LensBridge/agent/internal/clock/timesync" "github.com/LensBridge/agent/internal/importer" "github.com/LensBridge/agent/internal/state" "github.com/LensBridge/agent/internal/store" @@ -38,7 +51,10 @@ const ( Interval = 10 * time.Minute // BehindTolerance is how far behind the floor the clock may be before it // is corrected. Persisting happens every ten minutes, so a floor a few - // seconds ahead of a correct clock is normal and not worth a jump. + // seconds ahead of a correct clock is normal and not worth a jump. It is + // also how far a floor may be ahead of NTP time before it is lowered, and + // how far the wall clock may disagree with the monotonic clock between + // two checks before NTP time is believed again. BehindTolerance = 60 * time.Second // ClientTolerance is the drift an uploader's time must exceed to be // applied; below it, laptop clocks disagree with each other anyway. @@ -100,8 +116,28 @@ type Keeper struct { NTPSynced func() bool Uptime func() time.Duration BootID func() string + // Monotonic is a clock that only counts real elapsed time: setting the + // wall clock does not move it. Replaceable for tests. + Monotonic func() time.Duration + + mu sync.Mutex + // anchor is the last time the keeper trusted, and when by Monotonic. + // Without NTP the floor advances from it by elapsed monotonic time, not + // by whatever the wall clock says. + anchor *anchor +} + +// anchor is a trusted wall-clock time and the monotonic reading taken with it. +type anchor struct { + wall time.Time + mono time.Duration + // ntp: wall was NTP-synchronised time, not the floor. + ntp bool } +// at is the anchor carried forward to the monotonic reading mono. +func (a *anchor) at(mono time.Duration) time.Time { return a.wall.Add(mono - a.mono) } + // New returns a Keeper for the board's state in layout. func New(layout store.Layout, logger *slog.Logger) *Keeper { if logger == nil { @@ -114,9 +150,10 @@ func New(layout store.Layout, logger *slog.Logger) *Keeper { SetClock: setSystemClock, WriteRTC: writeRTC, RTCDevice: DefaultRTCDevice, - NTPSynced: timesyncdSynced, + NTPSynced: timesync.Synced, Uptime: uptime, BootID: bootID, + Monotonic: monotonic, } } @@ -149,13 +186,11 @@ func (k *Keeper) verifiedThisBoot() { } } -// timesyncdSynced reports whether systemd-timesyncd has synchronised the -// clock since boot; it creates this file when it does (systemd 239 and -// later). -func timesyncdSynced() bool { - _, err := os.Stat("/run/systemd/timesync/synchronized") - return err == nil -} +// processStart carries Go's monotonic reading; time.Since(processStart) is +// unaffected by changes to the wall clock. +var processStart = time.Now() + +func monotonic() time.Duration { return time.Since(processStart) } func uptime() time.Duration { raw, err := os.ReadFile("/proc/uptime") @@ -201,27 +236,79 @@ func (k *Keeper) Run(ctx context.Context) { } } -// Check corrects a clock that is behind the floor, then persists the wall -// clock into the floor. The order matters: persisting first would be a no-op -// for a clock behind the floor anyway, but checking first means the floor is -// then raised to the corrected time. +// Check runs at start and every Interval. +// +// With NTP it never moves the clock; it lowers a floor that is ahead of NTP +// time and persists NTP time into the floor (checkNTP). +// +// Without NTP it moves a clock that is more than BehindTolerance behind the +// floor up to the floor, then persists the last trusted time carried forward +// by the monotonic clock: never the wall clock, which nothing has confirmed. func (k *Keeper) Check() { + k.mu.Lock() + defer k.mu.Unlock() st, err := k.layout.State().Load() if err != nil { k.logger.Warn("clock: cannot read the clock floor", "err", err) return } - now := k.Now() + now, mono := k.Now(), k.Monotonic() + if k.NTPSynced() { + k.checkNTP(st, now, mono) + return + } floor := time.Unix(st.ClockFloor, 0) if st.ClockFloor > 0 && now.Before(floor.Add(-BehindTolerance)) { - k.logger.Warn("clock is behind the newest time this board has proof of; moving it forward", + k.logger.Warn("clock is behind the newest time this board has proof of, and NTP has not synchronised it; moving it forward", "now", now.UTC().Format(time.RFC3339), "floor", floor.UTC().Format(time.RFC3339), "behind", floor.Sub(now).Round(time.Second).String()) if err := k.set(floor); err != nil { k.logger.Error("clock: could not set the system clock", "err", err) return } - now = floor + } + if st.ClockFloor <= 0 { + // Nothing trusted to carry forward yet: no signed package, no NTP. + return + } + if k.anchor == nil || k.anchor.at(mono).Unix() < st.ClockFloor { + // First check, or a signed createdAt raised the floor since. + k.anchor = &anchor{wall: floor, mono: mono} + } + if err := k.layout.State().RaiseClockFloor(k.anchor.at(mono).Unix()); err != nil { + k.logger.Warn("clock: could not persist the clock floor", "err", err) + } +} + +// checkNTP is Check for an NTP-synchronised clock. NTP time is believed once +// two checks agree: since the previous check, the wall clock moved as far as +// the monotonic clock did. That keeps a clock stepped by hand between +// timesyncd's polls out of the floor. The first check after start, or after +// a step, only takes a new anchor. +func (k *Keeper) checkNTP(st state.State, now time.Time, mono time.Duration) { + prev := k.anchor + k.anchor = &anchor{wall: now, mono: mono, ntp: true} + if prev == nil || !prev.ntp { + return + } + if d := now.Sub(prev.at(mono)); d > BehindTolerance || d < -BehindTolerance { + k.logger.Info("clock: the clock was stepped since the last check; waiting for it to settle before trusting it", + "step", d.Round(time.Second).String()) + return + } + limit := now.Add(BehindTolerance).Unix() + if st.ClockFloor > limit || st.SignedClockFloor > limit { + before, lowered, err := k.layout.State().CapClockFloor(now.Unix()) + switch { + case err != nil: + k.logger.Warn("clock: could not lower the clock floor", "err", err) + case lowered: + k.logger.Warn("clock: the clock floor was ahead of NTP-synchronised time, so it was wrong; lowered it to NTP time", + "ntp", now.UTC().Format(time.RFC3339), + "floor", time.Unix(before.ClockFloor, 0).UTC().Format(time.RFC3339), + "signedFloor", time.Unix(before.SignedClockFloor, 0).UTC().Format(time.RFC3339), + "floorAhead", time.Unix(before.ClockFloor, 0).Sub(now).Round(time.Second).String()) + } } if err := k.layout.State().RaiseClockFloor(now.Unix()); err != nil { k.logger.Warn("clock: could not persist the clock floor", "err", err) @@ -243,9 +330,13 @@ func (k *Keeper) set(t time.Time) error { } // ApplyClientTime considers an uploader's clock (X-MB-Client-Time, Unix -// seconds) after the upload's batch b ran. It applies it only when a package -// in b verified, the difference is more than ClientTolerance, and the time is -// within [floor, floor + ClientWindow]. +// seconds) after the upload's batch b ran. It applies it only when the clock +// is not NTP-synchronised, a package in b verified, the difference is more +// than ClientTolerance, and the time is within [floor, floor + ClientWindow]. +// +// It sets the clock (and the RTC) but never the floor: the uploader is not +// authenticated, and its clock may be wrong. A wrong time it set is undone by +// the next uploader with a better clock, or by NTP. func (k *Keeper) ApplyClientTime(clientUnix int64, b importer.Batch) ClockReport { now := k.Now() drift := now.Unix() - clientUnix @@ -257,6 +348,10 @@ func (k *Keeper) ApplyClientTime(clientUnix int64, b importer.Batch) ClockReport } return rep } + if k.NTPSynced() { + rep.Note = fmt.Sprintf("the board's clock is %s, but it is set from the internet, so it was not changed; this device's clock may be wrong", describeDrift(drift)) + return rep + } if !b.Verified { rep.Note = fmt.Sprintf("the board's clock is %s; it was not changed because no package in this upload verified", describeDrift(drift)) return rep @@ -285,9 +380,6 @@ func (k *Keeper) ApplyClientTime(clientUnix int64, b importer.Batch) ClockReport k.verifiedThisBoot() rep.Note = fmt.Sprintf("the board's clock was %s and has been set to this device's time", describeDrift(drift)) k.logger.Info("clock set from uploader", "driftSeconds", drift, "to", target.UTC().Format(time.RFC3339)) - if err := k.layout.State().RaiseClockFloor(clientUnix); err != nil { - k.logger.Warn("clock: could not persist the clock floor", "err", err) - } return rep } diff --git a/internal/clock/clock_test.go b/internal/clock/clock_test.go index 12a4807..c0f60d1 100644 --- a/internal/clock/clock_test.go +++ b/internal/clock/clock_test.go @@ -14,11 +14,19 @@ import ( type fake struct { now time.Time + mono time.Duration + ntp bool sets []time.Time rtc int fail error } +// tick lets d of real time pass: the wall and the monotonic clock both move. +func (f *fake) tick(d time.Duration) { + f.now = f.now.Add(d) + f.mono += d +} + func newKeeper(t *testing.T, floor int64, now time.Time, withRTC bool) (*Keeper, *fake) { t.Helper() l := store.Layout{Root: t.TempDir()} @@ -27,6 +35,13 @@ func newKeeper(t *testing.T, floor int64, now time.Time, withRTC bool) (*Keeper, t.Fatal(err) } } + return keeperOn(t, l, now, withRTC) +} + +// keeperOn is a fresh agent process on an existing board: same state.json, +// new monotonic clock. +func keeperOn(t *testing.T, l store.Layout, now time.Time, withRTC bool) (*Keeper, *fake) { + t.Helper() f := &fake{now: now} k := New(l, nil) k.Now = func() time.Time { return f.now } @@ -39,7 +54,8 @@ func newKeeper(t *testing.T, floor int64, now time.Time, withRTC bool) (*Keeper, return nil } k.WriteRTC = func() error { f.rtc++; return nil } - k.NTPSynced = func() bool { return false } + k.NTPSynced = func() bool { return f.ntp } + k.Monotonic = func() time.Duration { return f.mono } k.Uptime = func() time.Duration { return time.Hour } k.BootID = func() string { return "boot-1" } k.RTCDevice = filepath.Join(t.TempDir(), "rtc0") @@ -50,23 +66,41 @@ func newKeeper(t *testing.T, floor int64, now time.Time, withRTC bool) (*Keeper, } func floorOf(t *testing.T, k *Keeper) int64 { + return stateOf(t, k).ClockFloor +} + +func stateOf(t *testing.T, k *Keeper) state.State { + t.Helper() st, err := k.layout.State().Load() if err != nil { t.Fatal(err) } - return st.ClockFloor + return st +} + +// setState writes the floors as an earlier agent left them. +func setState(t *testing.T, k *Keeper, floor, signed int64) { + t.Helper() + if _, err := k.layout.State().Update(func(s *state.State) error { + s.ClockFloor, s.SignedClockFloor = floor, signed + return nil + }); err != nil { + t.Fatal(err) + } } var base = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) -func TestCheckPersistsWallClock(t *testing.T) { +func TestCheckWithoutFloorOrNTPPersistsNothing(t *testing.T) { k, f := newKeeper(t, 0, base, false) k.Check() + f.tick(Interval) + k.Check() if len(f.sets) != 0 { t.Fatalf("clock set with no floor: %v", f.sets) } - if got := floorOf(t, k); got != base.Unix() { - t.Fatalf("floor = %d, want %d", got, base.Unix()) + if got := floorOf(t, k); got != 0 { + t.Fatalf("an unconfirmed wall clock became the floor: %d", got) } } @@ -107,22 +141,29 @@ func TestApplyClientTime(t *testing.T) { cases := []struct { name string now time.Time + ntp bool client int64 batch importer.Batch adjusted bool drift int64 }{ - {"within tolerance", base, floor + 4, verified, false, -4}, - {"not verified", base, floor + 3600, importer.Batch{}, false, -3600}, - {"behind floor", base.Add(time.Hour), floor - 10, verified, false, 3610}, - {"beyond 90 days", base, floor + int64(91*24*3600), verified, false, -int64(91 * 24 * 3600)}, - {"board behind", base.Add(-time.Hour), floor + 60, verified, true, -3660}, - {"board ahead, client at floor", base.Add(time.Hour), floor, verified, true, 3600}, - {"exactly 90 days", base, floor + int64(90*24*3600), verified, true, -int64(90 * 24 * 3600)}, + {"within tolerance", base, false, floor + 4, verified, false, -4}, + {"not verified", base, false, floor + 3600, importer.Batch{}, false, -3600}, + {"behind floor", base.Add(time.Hour), false, floor - 10, verified, false, 3610}, + {"beyond 90 days", base, false, floor + int64(91*24*3600), verified, false, -int64(91 * 24 * 3600)}, + {"board behind", base.Add(-time.Hour), false, floor + 60, verified, true, -3660}, + {"board ahead, client at floor", base.Add(time.Hour), false, floor, verified, true, 3600}, + {"exactly 90 days", base, false, floor + int64(90*24*3600), verified, true, -int64(90 * 24 * 3600)}, + // NTP is the authority: an unauthenticated uploader never moves it. + {"ntp, laptop 60 days ahead", base, true, floor + int64(60*24*3600), verified, false, -int64(60 * 24 * 3600)}, + {"ntp, laptop an hour behind", base, true, floor - 3600, verified, false, 3600}, + {"ntp, board an hour behind the laptop", base.Add(-time.Hour), true, floor, verified, false, -3600}, + {"ntp, within tolerance", base, true, floor + 3, verified, false, -3}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { k, f := newKeeper(t, floor, tc.now, false) + f.ntp = tc.ntp rep := k.ApplyClientTime(tc.client, tc.batch) if rep.Adjusted != tc.adjusted || rep.DriftSeconds != tc.drift { t.Fatalf("report = %+v, want adjusted=%v drift=%d", rep, tc.adjusted, tc.drift) @@ -136,6 +177,10 @@ func TestApplyClientTime(t *testing.T) { if !tc.adjusted && len(f.sets) != 0 { t.Fatalf("clock set: %v", f.sets) } + // An uploader's time never becomes the floor. + if got := floorOf(t, k); got != floor { + t.Fatalf("floor = %d, want %d unchanged", got, floor) + } }) } } @@ -186,3 +231,281 @@ func TestInfo(t *testing.T) { t.Fatalf("with NTP: %+v", i) } } + +// checks runs n Checks an Interval apart, starting now. +func checks(k *Keeper, f *fake, n int) { + for i := 0; i < n; i++ { + if i > 0 { + f.tick(Interval) + } + k.Check() + } +} + +// Without NTP nothing has confirmed the wall clock, so however it got ahead +// (a manual date, a bad RTC, an uploader) it must not become the floor. The +// floor advances by the time that really passed since the last trusted time. +func TestUnconfirmedClockAheadDoesNotRaiseFloor(t *testing.T) { + cases := []struct { + name string + withRTC bool + ahead func(k *Keeper, f *fake) + }{ + {"manual date", false, func(k *Keeper, f *fake) { f.now = f.now.Add(365 * 24 * time.Hour) }}, + {"wrong RTC", true, func(k *Keeper, f *fake) { f.now = f.now.Add(200 * 24 * time.Hour) }}, + {"uploader", false, func(k *Keeper, f *fake) { + if rep := k.ApplyClientTime(f.now.Add(60*24*time.Hour).Unix(), importer.Batch{Verified: true}); !rep.Adjusted { + panic("uploader time not applied: " + rep.Note) + } + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + k, f := newKeeper(t, base.Unix(), base, tc.withRTC) + k.Check() + tc.ahead(k, f) + sets := len(f.sets) + f.tick(Interval) + checks(k, f, 3) + if got, want := floorOf(t, k), base.Add(3*Interval).Unix(); got != want { + t.Fatalf("floor = %s, want %s (trusted time carried forward, not the wall clock)", + time.Unix(got, 0).UTC(), time.Unix(want, 0).UTC()) + } + if len(f.sets) != sets { + t.Fatalf("clock set again: %v", f.sets[sets:]) + } + }) + } +} + +// The floor's purpose without NTP: a board that loses power comes back no +// earlier than the time it last knew, carried forward while it ran. +func TestFloorSurvivesPowerCutWithoutNTP(t *testing.T) { + k, f := newKeeper(t, base.Unix(), base, false) + checks(k, f, 7) // an hour + if got, want := floorOf(t, k), base.Add(time.Hour).Unix(); got != want { + t.Fatalf("floor = %d, want %d", got, want) + } + // Power cut. No RTC, so the clock comes up wherever. + k2, f2 := keeperOn(t, k.layout, time.Unix(0, 0), false) + k2.Check() + if len(f2.sets) != 1 || f2.sets[0].Unix() != base.Add(time.Hour).Unix() { + t.Fatalf("sets = %v, want the floor", f2.sets) + } + checks(k2, f2, 2) + if got, want := floorOf(t, k2), base.Add(time.Hour+Interval).Unix(); got != want { + t.Fatalf("floor after restart = %d, want %d", got, want) + } +} + +// A verified package's createdAt raises the floor while the keeper runs; the +// keeper carries the floor forward from there, not from its older anchor. +func TestSignedTimeReanchors(t *testing.T) { + k, f := newKeeper(t, base.Unix(), base, false) + k.Check() + if err := k.layout.State().RaiseSignedClockFloor(base.Add(5 * time.Hour).Unix()); err != nil { + t.Fatal(err) + } + f.tick(Interval) + k.Check() // the clock is ~5 h behind the signed time: moved forward + if len(f.sets) != 1 || f.sets[0].Unix() != base.Add(5*time.Hour).Unix() { + t.Fatalf("sets = %v", f.sets) + } + f.tick(Interval) + k.Check() + if got, want := floorOf(t, k), base.Add(5*time.Hour+Interval).Unix(); got != want { + t.Fatalf("floor = %d, want %d", got, want) + } +} + +// With NTP the keeper never moves the clock, whatever the floor says, and the +// floor follows NTP time. +func TestNTPIsTheAuthority(t *testing.T) { + cases := []struct { + name string + floor, signed time.Time + wantFloorAtNTP bool + wantSignedLowered bool + }{ + {"floor behind", base.Add(-time.Hour), base.Add(-2 * time.Hour), false, false}, + {"floor a few seconds ahead", base.Add(30 * time.Second), base.Add(-time.Hour), false, false}, + {"floor ahead", base.Add(300 * 24 * time.Hour), base.Add(-time.Hour), true, false}, + {"signed floor ahead too", base.Add(400 * 24 * time.Hour), base.Add(4 * 365 * 24 * time.Hour), true, true}, + {"floor behind, signed ahead", base.Add(-time.Hour), base.Add(2 * time.Hour), true, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + k, f := newKeeper(t, 0, base, true) + setState(t, k, tc.floor.Unix(), tc.signed.Unix()) + f.ntp = true + k.Check() + if st := stateOf(t, k); st.ClockFloor != tc.floor.Unix() || st.SignedClockFloor != tc.signed.Unix() { + t.Fatalf("first NTP check changed the floor: %+v", st) + } + checks(k, f, 1+6) + if len(f.sets) != 0 || f.rtc != 0 { + t.Fatalf("the clock was moved against NTP: sets=%v rtc=%d", f.sets, f.rtc) + } + st := stateOf(t, k) + if want := f.now.Unix(); tc.wantFloorAtNTP && st.ClockFloor != want { + t.Fatalf("floor = %s, want lowered to NTP time %s", time.Unix(st.ClockFloor, 0).UTC(), f.now.UTC()) + } + if !tc.wantFloorAtNTP && st.ClockFloor < tc.floor.Unix() { + t.Fatalf("floor lowered: %d < %d", st.ClockFloor, tc.floor.Unix()) + } + if st.ClockFloor < f.now.Unix()-1 && st.ClockFloor < tc.floor.Unix() { + t.Fatalf("floor %d does not follow NTP time %d", st.ClockFloor, f.now.Unix()) + } + if tc.wantSignedLowered != (st.SignedClockFloor != tc.signed.Unix()) { + t.Fatalf("signed floor = %d (was %d), lowered want %v", st.SignedClockFloor, tc.signed.Unix(), tc.wantSignedLowered) + } + if st.SignedClockFloor > f.now.Unix() { + t.Fatalf("signed floor still ahead of NTP: %d > %d", st.SignedClockFloor, f.now.Unix()) + } + }) + } +} + +// A clock stepped by hand between two of timesyncd's polls is not NTP time, +// even though timesyncd synchronised it earlier this boot. +func TestNTPStepBetweenChecksIsNotPersisted(t *testing.T) { + for _, step := range []time.Duration{365 * 24 * time.Hour, -30 * 24 * time.Hour} { + t.Run(step.String(), func(t *testing.T) { + k, f := newKeeper(t, base.Unix(), base, false) + f.ntp = true + checks(k, f, 2) + want := floorOf(t, k) + f.tick(Interval) + f.now = f.now.Add(step) + k.Check() + if got := floorOf(t, k); got != want { + t.Fatalf("floor = %d, want %d: a stepped clock was trusted at once", got, want) + } + if len(f.sets) != 0 { + t.Fatalf("sets = %v", f.sets) + } + }) + } +} + +// F1: a laptop with a wrong clock uploads on the service port. Before, the +// board jumped ahead, the floor followed, and the keeper pushed the clock +// forward against timesyncd every ten minutes, for good. +func TestWrongLaptopClockDoesNotPinBoard(t *testing.T) { + laptopAhead := 60 * 24 * time.Hour + for _, ntpAtUpload := range []bool{false, true} { + t.Run(map[bool]string{false: "offline at upload", true: "online at upload"}[ntpAtUpload], func(t *testing.T) { + k, f := newKeeper(t, base.Unix(), base, false) + real := base + f.ntp = ntpAtUpload + k.Check() + + rep := k.ApplyClientTime(real.Add(laptopAhead).Unix(), importer.Batch{Verified: true}) + if rep.Adjusted == ntpAtUpload { + t.Fatalf("adjusted = %v with NTP %v: %s", rep.Adjusted, ntpAtUpload, rep.Note) + } + if got := floorOf(t, k); got != base.Unix() { + t.Fatalf("the laptop's time reached the floor: %d", got) + } + + // The board keeps running offline for a while... + for i := 0; i < 3; i++ { + f.tick(Interval) + real = real.Add(Interval) + k.Check() + } + // ...then the network comes back and timesyncd corrects it. + f.ntp, f.now = true, real + sets := len(f.sets) + for i := 0; i < 24*6; i++ { // a day + f.tick(Interval) + real = real.Add(Interval) + k.Check() + } + if len(f.sets) != sets { + t.Fatalf("the keeper moved an NTP-synchronised clock: %v", f.sets[sets:]) + } + if !f.now.Equal(real) { + t.Fatalf("clock = %s, want %s", f.now.UTC(), real.UTC()) + } + if got := floorOf(t, k); got != real.Unix() { + t.Fatalf("floor = %s, want NTP time %s", time.Unix(got, 0).UTC(), real.UTC()) + } + }) + } +} + +// A state.json left by an earlier agent with the floor (and possibly the +// signed floor) pushed into the future heals by itself once NTP has +// synchronised, and stays healed across a reboot without network. +func TestPoisonedStateRecoversWithNTP(t *testing.T) { + cases := []struct { + name string + floor, signed time.Duration // ahead of the real time + }{ + {"floor ahead (uploader, manual date, RTC)", 300 * 24 * time.Hour, -24 * time.Hour}, + {"floor at the 400 day cap", (400 + 30) * 24 * time.Hour, 30 * 24 * time.Hour}, + {"signed createdAt from a backend clock in 2030", 4 * 365 * 24 * time.Hour, 4 * 365 * 24 * time.Hour}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + k, f := newKeeper(t, 0, base, false) + setState(t, k, base.Add(tc.floor).Unix(), base.Add(tc.signed).Unix()) + real := base + + // Boot after the upgrade, before timesyncd has synchronised: the + // floor is all the keeper knows, so it is enforced. + k.Check() + if len(f.sets) != 1 { + t.Fatalf("sets = %v", f.sets) + } + // timesyncd synchronises a minute later and steps the clock back. + f.tick(time.Minute) + real = real.Add(time.Minute) + f.ntp, f.now = true, real + for i := 0; i < 3; i++ { + f.tick(Interval) + real = real.Add(Interval) + k.Check() + } + if len(f.sets) != 1 { + t.Fatalf("the keeper fought NTP: %v", f.sets) + } + st := stateOf(t, k) + if st.ClockFloor != real.Unix() || st.SignedClockFloor > real.Unix() { + t.Fatalf("not healed: floor %s, signed %s, NTP %s", time.Unix(st.ClockFloor, 0).UTC(), + time.Unix(st.SignedClockFloor, 0).UTC(), real.UTC()) + } + + // Power cut; the board comes back without network or RTC. + k2, f2 := keeperOn(t, k.layout, real.Add(-2*time.Hour), false) + k2.Check() + if len(f2.sets) != 1 || f2.sets[0].Unix() != real.Unix() { + t.Fatalf("after reboot: sets = %v, want the healed floor %s", f2.sets, real.UTC()) + } + }) + } +} + +// Without NTP, an uploader's time is not written to the floor, so a later +// uploader with a right clock can undo a wrong one. Nobody can move the clock +// below the floor. +func TestUploaderTimeIsCorrectableButNotRollback(t *testing.T) { + v := importer.Batch{Verified: true} + k, f := newKeeper(t, base.Unix(), base, true) + checks(k, f, 7) // an hour of trusted time + real := f.now + + if rep := k.ApplyClientTime(real.Add(80*24*time.Hour).Unix(), v); !rep.Adjusted { + t.Fatalf("wrong laptop: %+v", rep) + } + if rep := k.ApplyClientTime(real.Unix(), v); !rep.Adjusted || f.now.Unix() != real.Unix() { + t.Fatalf("a right laptop could not undo the wrong one: %+v, clock %s", rep, f.now.UTC()) + } + if rep := k.ApplyClientTime(real.Add(-30*time.Minute).Unix(), v); rep.Adjusted { + t.Fatalf("clock rolled back below the floor: %+v", rep) + } + if got, want := floorOf(t, k), base.Add(time.Hour).Unix(); got != want { + t.Fatalf("floor = %d, want %d", got, want) + } +} diff --git a/internal/clock/timesync/timesync.go b/internal/clock/timesync/timesync.go new file mode 100644 index 0000000..0cc0335 --- /dev/null +++ b/internal/clock/timesync/timesync.go @@ -0,0 +1,26 @@ +// Package timesync says whether the system clock is synchronised to network +// time. It is its own package so that both the clock keeper and the importer +// (which the keeper imports) can ask. +package timesync + +import ( + "os" + "time" +) + +// SyncedFile is created by systemd-timesyncd (systemd 239 and later) once it +// has synchronised the clock. It lives in /run, so it says "since this boot". +const SyncedFile = "/run/systemd/timesync/synchronized" + +// Synced reports whether systemd-timesyncd has synchronised the clock since +// boot. +func Synced() bool { + _, err := os.Stat(SyncedFile) + return err == nil +} + +// Now is the system clock and whether it is NTP-synchronised: the only time +// the board treats as an authority over signed or remembered times. +func Now() (time.Time, bool) { + return time.Now(), Synced() +} From f17dc3679f126c0c5ec4472f8a4733ef7c14feae Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:11:11 +0000 Subject: [PATCH 06/19] Enroll only over https, with a wss socket on the backend's host enroll took any --backend scheme and saved whatever websocketUrl the backend returned. Over http the enrollment reply, which pins the content signing keys and the WebSocket URL, can be rewritten by anyone on the LAN. - config: CheckBackendURL (https://, a host, no credentials, query or fragment) and CheckWebSocketURL (wss://, same host as the backend; the port may differ, since TLS binds the host name and not the port). No override for split-host deployments. A new explicit key, insecure_dev = true, allows http:// and ws:// for development boards; the host rule still applies. Config.CheckTransport applies both. - enroll: the backend URL is checked before the token is sent, and the returned WebSocket URL before anything is saved, so a refused reply leaves the board unenrolled and pins no keys. --insecure-dev (with a loud warning) writes insecure_dev = true. Tests run against TLS servers now. --- cmd/agent/main.go | 8 +++ internal/config/config.go | 103 ++++++++++++++++++++++++++- internal/config/config_test.go | 84 ++++++++++++++++++++++ internal/enroll/enroll.go | 30 +++++++- internal/enroll/enroll_test.go | 126 ++++++++++++++++++++++++++++++--- 5 files changed, 341 insertions(+), 10 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index d11314c..11d90b2 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -104,6 +104,9 @@ Enroll flags: --backend Backend base URL (e.g. https://backend.utmmsa.ca) --config Path to write agent config (default: ` + defaultConfigPath + `) --force Enroll a board that is already enrolled (it gets a new identity) + --insecure-dev + Development boards only: allow an http:// backend and ws:// socket + (saved in the config as insecure_dev = true; never on a deployed board) `) } @@ -113,6 +116,7 @@ func runEnroll(args []string) { backend := fs.String("backend", "", "backend base URL") configPath := fs.String("config", defaultConfigPath, "path to write config") force := fs.Bool("force", false, "enroll again, replacing this board's identity") + insecureDev := fs.Bool("insecure-dev", false, "development only: allow a plaintext http:// backend") if err := fs.Parse(args); err != nil { os.Exit(2) } @@ -121,6 +125,9 @@ func runEnroll(args []string) { fmt.Fprintln(os.Stderr, "Copy the whole command from the admin portal: Devices, Enroll a board.") os.Exit(2) } + if err := config.CheckBackendURL(*backend, *insecureDev); err != nil { + fail("%v", err) + } requireRoot("enroll") // Enrolling again gives the board a new identity; the old device stays @@ -138,6 +145,7 @@ func runEnroll(args []string) { BackendURL: *backend, ConfigPath: *configPath, AgentVersion: version.Version, + InsecureDev: *insecureDev, }); err != nil { fmt.Fprintf(os.Stderr, "Enrollment failed: %v\n", err) fmt.Fprintln(os.Stderr, "Tokens can be used once and expire; if this one was used or has expired, issue a new one in the portal.") diff --git a/internal/config/config.go b/internal/config/config.go index d188406..c3a3db7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,14 +9,19 @@ // // Optional keys (docs/architecture.md, section 11): service_port, usb_import, // content_sync, content_days, auto_update, app_channel_url, agent_channel_url, -// update_time. +// update_time, insecure_dev. +// +// backend_url must be https:// and websocket_url wss:// on the same host, +// unless insecure_dev = true (CheckTransport). // // Identity is proved by signing a server-issued challenge with the Ed25519 key // at KeyPath. package config import ( + "errors" "fmt" + "net/url" "os" "path/filepath" "strings" @@ -56,8 +61,17 @@ type Config struct { AppChannelSet *string `toml:"app_channel_url,omitempty"` AgentChannelSet *string `toml:"agent_channel_url,omitempty"` UpdateTimeSet *string `toml:"update_time,omitempty"` + + // InsecureDevSet is the explicit development opt-in to plaintext + // http:// and ws:// backends (enroll --insecure-dev). Written to + // agent.toml so that what the daemon accepts is visible there. + InsecureDevSet *bool `toml:"insecure_dev,omitempty"` } +// InsecureDev reports whether this board may talk to its backend without TLS. +// Development boards only. +func (c *Config) InsecureDev() bool { return boolOr(c.InsecureDevSet, false) } + func boolOr(p *bool, def bool) bool { if p == nil { return def @@ -115,6 +129,93 @@ func (c *Config) UpdateTime() (hour, minute int) { return DefaultUpdateHour, 0 } +// ErrInsecureTransport marks a backend or WebSocket URL that is refused +// because it is not TLS (or not where the backend is) and insecure_dev is off. +var ErrInsecureTransport = errors.New("insecure backend transport") + +// CheckBackendURL checks a backend base URL: https:// with a host, no +// credentials, query or fragment. http:// is accepted only with insecureDev. +// +// Over plain http the enrollment reply, which pins the content signing keys +// and the WebSocket URL, can be rewritten by anyone on the LAN; so can every +// later sync. TLS is what makes the pinned keys mean anything. +func CheckBackendURL(raw string, insecureDev bool) error { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return fmt.Errorf("backend URL %q: %w", raw, err) + } + switch { + case u.Scheme == "https": + case u.Scheme == "http" && insecureDev: + case u.Scheme == "http": + return fmt.Errorf("%w: backend URL %s is plain http; use https:// (development boards only: enroll with --insecure-dev, which sets insecure_dev = true)", ErrInsecureTransport, raw) + default: + return fmt.Errorf("backend URL %q must start with https://", raw) + } + if u.Hostname() == "" { + return fmt.Errorf("backend URL %q has no host", raw) + } + if u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("backend URL %q must be a plain base URL (no credentials, query or fragment)", raw) + } + return nil +} + +// CheckWebSocketURL checks the WebSocket URL the backend handed out at +// enrollment against the backend URL it was enrolled with: wss:// (ws:// only +// with insecureDev), and on the same host. +// +// Same host, not same origin: the port may differ. TLS binds the server's +// identity to the host name, not the port, and one host serving the API and +// the socket on different ports is the same operator. A different host is +// refused: the device signs auth challenges for whoever answers there and +// takes reboot, screenshot and log commands from it, so it has to be the +// backend the board enrolled with. There is no override for split-host +// deployments; put the socket behind the backend's host instead. +func CheckWebSocketURL(raw, backendURL string, insecureDev bool) error { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return fmt.Errorf("websocket URL %q: %w", raw, err) + } + switch { + case u.Scheme == "wss": + case u.Scheme == "ws" && insecureDev: + case u.Scheme == "ws": + return fmt.Errorf("%w: websocket URL %s is plain ws; the backend must hand out wss:// (development boards only: insecure_dev = true)", ErrInsecureTransport, raw) + default: + return fmt.Errorf("websocket URL %q must start with wss://", raw) + } + be, err := url.Parse(strings.TrimSpace(backendURL)) + if err != nil { + return fmt.Errorf("backend URL %q: %w", backendURL, err) + } + if u.Hostname() == "" || !sameHost(u.Hostname(), be.Hostname()) { + return fmt.Errorf("%w: websocket URL %s is not on the backend's host %s", ErrInsecureTransport, raw, be.Hostname()) + } + if u.User != nil { + return fmt.Errorf("websocket URL %q must not carry credentials", raw) + } + return nil +} + +func sameHost(a, b string) bool { + return strings.EqualFold(strings.TrimSuffix(a, "."), strings.TrimSuffix(b, ".")) +} + +// CheckTransport applies CheckBackendURL and CheckWebSocketURL to this +// config. The daemon calls it before it builds anything that talks to the +// backend (content sync, the refresh channel, release checks, the command +// WebSocket) and talks to none of them when it fails. +// +// Load does not call it: a board with an insecure config still shows the +// content it has (local first), and the local CLI tools still work on it. +func (c *Config) CheckTransport() error { + if err := CheckBackendURL(c.BackendURL, c.InsecureDev()); err != nil { + return err + } + return CheckWebSocketURL(c.WebSocketURL, c.BackendURL, c.InsecureDev()) +} + func Load(path string) (*Config, error) { var c Config if _, err := toml.DecodeFile(path, &c); err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index fb16494..292ee6b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "errors" "os" "path/filepath" "strings" @@ -67,3 +68,86 @@ func TestSetServicePortKeepsOtherFields(t *testing.T) { t.Fatal("expected an error for a missing config") } } + +func TestCheckTransport(t *testing.T) { + cases := []struct { + name, backend, ws string + insecure bool + ok, insecureErr bool + }{ + {"tls", "https://api.example.com", "wss://api.example.com/api/agent/ws", false, true, false}, + {"tls, ws port differs", "https://api.example.com", "wss://api.example.com:8443/ws", false, true, false}, + {"tls, case and trailing dot", "https://API.example.com.", "wss://api.example.com/ws", false, true, false}, + {"http backend", "http://api.example.com:8080", "wss://api.example.com/ws", false, false, true}, + {"ws socket", "https://api.example.com", "ws://api.example.com/ws", false, false, true}, + {"other ws host", "https://api.example.com", "wss://evil.example.net/ws", false, false, true}, + {"other ws host, insecure dev", "http://10.0.2.2:8080", "ws://localhost:8080/ws", true, false, true}, + {"insecure dev", "http://10.0.2.2:8080", "ws://10.0.2.2:8080/api/agent/ws", true, true, false}, + {"insecure dev with tls", "https://api.example.com", "wss://api.example.com/ws", true, true, false}, + {"ftp", "ftp://api.example.com", "wss://api.example.com/ws", true, false, false}, + {"no host", "https://", "wss://api.example.com/ws", false, false, false}, + {"credentials", "https://u:p@api.example.com", "wss://api.example.com/ws", false, false, false}, + {"query", "https://api.example.com/?x=1", "wss://api.example.com/ws", false, false, false}, + {"https socket", "https://api.example.com", "https://api.example.com/ws", false, false, false}, + } + for _, tc := range cases { + c := &Config{BackendURL: tc.backend, WebSocketURL: tc.ws} + if tc.insecure { + on := true + c.InsecureDevSet = &on + } + err := c.CheckTransport() + if (err == nil) != tc.ok { + t.Errorf("%s: err = %v, want ok=%v", tc.name, err, tc.ok) + } + if got := errors.Is(err, ErrInsecureTransport); got != tc.insecureErr { + t.Errorf("%s: errors.Is(ErrInsecureTransport) = %v, want %v (%v)", tc.name, got, tc.insecureErr, err) + } + } +} + +// An http board must still load: it keeps showing its content and the CLI +// tools keep working. The daemon refuses the backend via CheckTransport. +func TestLoadKeepsInsecureConfigReadable(t *testing.T) { + body := strings.NewReplacer("https://", "http://", "wss://", "ws://").Replace(baseTOML) + c, err := Load(writeConfig(t, body)) + if err != nil { + t.Fatal(err) + } + if c.InsecureDev() { + t.Error("insecure_dev defaults to false") + } + if err := c.CheckTransport(); !errors.Is(err, ErrInsecureTransport) { + t.Errorf("CheckTransport = %v, want ErrInsecureTransport", err) + } + + c, err = Load(writeConfig(t, body+"insecure_dev = true\n")) + if err != nil { + t.Fatal(err) + } + if !c.InsecureDev() || c.CheckTransport() != nil { + t.Errorf("insecure_dev = true: InsecureDev=%v CheckTransport=%v", c.InsecureDev(), c.CheckTransport()) + } +} + +// The opt-in is written as an explicit key, and only when on. +func TestSaveWritesInsecureDevOnlyWhenSet(t *testing.T) { + p := filepath.Join(t.TempDir(), "agent.toml") + c := &Config{DeviceID: "d", BackendURL: "https://a", WebSocketURL: "wss://a/ws", KeyPath: "/k"} + if err := Save(p, c); err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(p) + if strings.Contains(string(b), "insecure_dev") { + t.Errorf("insecure_dev written when unset:\n%s", b) + } + on := true + c.InsecureDevSet = &on + if err := Save(p, c); err != nil { + t.Fatal(err) + } + b, _ = os.ReadFile(p) + if !strings.Contains(string(b), "insecure_dev = true") { + t.Errorf("insecure_dev missing:\n%s", b) + } +} diff --git a/internal/enroll/enroll.go b/internal/enroll/enroll.go index 7e6de89..5ed762f 100644 --- a/internal/enroll/enroll.go +++ b/internal/enroll/enroll.go @@ -38,6 +38,11 @@ type Params struct { // in; optional, defaults to trust.DefaultPath. TrustPath string AgentVersion string + // InsecureDev allows a plain http:// backend and a ws:// WebSocket URL, + // for development boards only. It is saved as insecure_dev = true. + InsecureDev bool + // HTTPClient is optional; tests pass their TLS server's client. + HTTPClient *http.Client } // Run performs first-boot enrollment. Idempotent only if the same token is @@ -49,6 +54,16 @@ func Run(ctx context.Context, logger *slog.Logger, p Params) error { keyPath = config.DefaultKeyPath } + // Before anything goes on the wire: the reply pins the content keys and + // the WebSocket URL, so it must come over TLS. + if err := config.CheckBackendURL(p.BackendURL, p.InsecureDev); err != nil { + return err + } + if p.InsecureDev { + logger.Warn("INSECURE DEVELOPMENT ENROLLMENT: this board will accept a plaintext backend (insecure_dev = true in its config); anyone on the network can impersonate the backend. Never use this on a deployed board.", + "backend", p.BackendURL) + } + pub, priv, err := keystore.Generate() if err != nil { return fmt.Errorf("keygen: %w", err) @@ -60,9 +75,13 @@ func Run(ctx context.Context, logger *slog.Logger, p Params) error { // Client and request/response types are generated from the backend's // openapi.yaml (see internal/api/oapi-codegen.yaml). A field renamed on the // server now breaks this build instead of silently decoding to a zero value. + httpClient := p.HTTPClient + if httpClient == nil { + httpClient = &http.Client{Timeout: 30 * time.Second} + } client, err := api.NewClientWithResponses( strings.TrimRight(p.BackendURL, "/"), - api.WithHTTPClient(&http.Client{Timeout: 30 * time.Second}), + api.WithHTTPClient(httpClient), ) if err != nil { return fmt.Errorf("build client: %w", err) @@ -104,6 +123,11 @@ func Run(ctx context.Context, logger *slog.Logger, p Params) error { // client connects directly without a Host: port. Patch it back in if the // backend URL carried an explicit port that the returned WS URL is missing. websocketURL := restorePort(*enrolled.WebsocketUrl, p.BackendURL) + if err := config.CheckWebSocketURL(websocketURL, p.BackendURL, p.InsecureDev); err != nil { + // Nothing is saved: the board stays unenrolled. The backend has + // consumed the token, so a retry needs a new one once it is fixed. + return fmt.Errorf("the backend's reply cannot be used: %w", err) + } // Persist the private key BEFORE the config: if we crash between the two, // re-running enroll is harmless (token is consumed, but no orphan config @@ -119,6 +143,10 @@ func Run(ctx context.Context, logger *slog.Logger, p Params) error { WebSocketURL: websocketURL, KeyPath: keyPath, } + if p.InsecureDev { + on := true + cfg.InsecureDevSet = &on + } if err := config.Save(p.ConfigPath, cfg); err != nil { return fmt.Errorf("save config: %w", err) } diff --git a/internal/enroll/enroll_test.go b/internal/enroll/enroll_test.go index 9abcf1d..3663826 100644 --- a/internal/enroll/enroll_test.go +++ b/internal/enroll/enroll_test.go @@ -6,6 +6,7 @@ import ( "crypto/ed25519" "encoding/base64" "encoding/json" + "errors" "io" "log/slog" "net/http" @@ -42,11 +43,17 @@ func signingKeyJSON(pub ed25519.PublicKey) map[string]any { return map[string]any{"keyId": trust.KeyID(pub), "publicKey": base64.StdEncoding.EncodeToString(pub)} } +// wsURL is the WebSocket URL a backend at server would hand out. +func wsURL(server *httptest.Server, scheme string) string { + return scheme + "://" + strings.TrimPrefix(strings.TrimPrefix(server.URL, "https://"), "http://") + "/api/agent/ws" +} + func TestRunEnrollsAndPersistsConfig(t *testing.T) { const deviceID = "6f1e1d94-1f4a-4a1e-9a1e-2f3c4d5e6a7b" var gotBody map[string]any - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var server *httptest.Server + server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/agent/enroll" { t.Errorf("unexpected path %q", r.URL.Path) } @@ -61,7 +68,7 @@ func TestRunEnrollsAndPersistsConfig(t *testing.T) { // port-patching behaviour has its own test below. json.NewEncoder(w).Encode(map[string]any{ "deviceId": deviceID, - "websocketUrl": "ws://board.example:8080/api/agent/ws", + "websocketUrl": wsURL(server, "wss"), "contentSigningKeys": []any{signingKeyJSON(contentPub)}, }) })) @@ -75,6 +82,7 @@ func TestRunEnrollsAndPersistsConfig(t *testing.T) { KeyPath: keyPath, TrustPath: trustPathBeside(cfgPath), AgentVersion: "1.2.3", + HTTPClient: server.Client(), }) if err != nil { t.Fatalf("Run() = %v, want nil", err) @@ -101,9 +109,12 @@ func TestRunEnrollsAndPersistsConfig(t *testing.T) { if cfg.DeviceID != deviceID { t.Errorf("DeviceID = %q, want %q", cfg.DeviceID, deviceID) } - if cfg.WebSocketURL != "ws://board.example:8080/api/agent/ws" { + if cfg.WebSocketURL != wsURL(server, "wss") { t.Errorf("WebSocketURL = %q", cfg.WebSocketURL) } + if cfg.InsecureDev() || cfg.CheckTransport() != nil { + t.Errorf("a TLS enrollment must give a secure config: insecure_dev=%v, CheckTransport=%v", cfg.InsecureDev(), cfg.CheckTransport()) + } if _, err := os.Stat(keyPath); err != nil { t.Errorf("private key not written: %v", err) } @@ -121,11 +132,12 @@ func TestRunEnrollsAndPersistsConfig(t *testing.T) { // A backend with no content signing key configured sends no keys; the board // still enrolls, and gets them later from `trust fetch`. func TestRunWithoutContentKeys(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var server *httptest.Server + server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{ "deviceId": "6f1e1d94-1f4a-4a1e-9a1e-2f3c4d5e6a7b", - "websocketUrl": "ws://board.example:8080/api/agent/ws", + "websocketUrl": wsURL(server, "wss"), }) })) defer server.Close() @@ -133,7 +145,7 @@ func TestRunWithoutContentKeys(t *testing.T) { cfgPath, keyPath := paths(t) if err := Run(context.Background(), quietLogger(), Params{ Token: "t", BackendURL: server.URL, ConfigPath: cfgPath, KeyPath: keyPath, - TrustPath: trustPathBeside(cfgPath), + TrustPath: trustPathBeside(cfgPath), HTTPClient: server.Client(), }); err != nil { t.Fatalf("Run() = %v", err) } @@ -189,7 +201,7 @@ func TestFetchSigningKeys(t *testing.T) { // Before the client was generated this response body was untyped, so the reason // was not reliably available to report. func TestRunReportsRejectionMessage(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) json.NewEncoder(w).Encode(map[string]any{ @@ -205,6 +217,7 @@ func TestRunReportsRejectionMessage(t *testing.T) { ConfigPath: cfgPath, KeyPath: keyPath, TrustPath: trustPathBeside(cfgPath), + HTTPClient: server.Client(), }) if err == nil { t.Fatal("Run() = nil, want error") @@ -222,7 +235,7 @@ func TestRunReportsRejectionMessage(t *testing.T) { } func TestRunRejectsIncompleteResponse(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") // deviceId present, websocketUrl missing. json.NewEncoder(w).Encode(map[string]any{ @@ -238,12 +251,109 @@ func TestRunRejectsIncompleteResponse(t *testing.T) { ConfigPath: cfgPath, KeyPath: keyPath, TrustPath: trustPathBeside(cfgPath), + HTTPClient: server.Client(), }) if err == nil || !strings.Contains(err.Error(), "missing deviceId or websocketUrl") { t.Fatalf("Run() = %v, want missing-field error", err) } } +// A plain-http backend is refused before anything is sent: the reply would +// pin the content keys and the WebSocket URL, and anyone on the LAN could +// write it. +func TestRunRefusesPlainHTTPBackend(t *testing.T) { + hits := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits++ })) + defer server.Close() + + cfgPath, keyPath := paths(t) + err := Run(context.Background(), quietLogger(), Params{ + Token: "t", BackendURL: server.URL, ConfigPath: cfgPath, KeyPath: keyPath, + TrustPath: trustPathBeside(cfgPath), + }) + if !errors.Is(err, config.ErrInsecureTransport) || !strings.Contains(err.Error(), "--insecure-dev") { + t.Fatalf("Run() = %v, want ErrInsecureTransport naming --insecure-dev", err) + } + if hits != 0 { + t.Errorf("the token was sent over plain http (%d requests)", hits) + } + assertNothingSaved(t, cfgPath, keyPath) +} + +// --insecure-dev enrolls over http and records the opt-in in the config. +func TestRunInsecureDevEnrollsOverHTTP(t *testing.T) { + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "deviceId": "6f1e1d94-1f4a-4a1e-9a1e-2f3c4d5e6a7b", + "websocketUrl": wsURL(server, "ws"), + }) + })) + defer server.Close() + + cfgPath, keyPath := paths(t) + if err := Run(context.Background(), quietLogger(), Params{ + Token: "t", BackendURL: server.URL, ConfigPath: cfgPath, KeyPath: keyPath, + TrustPath: trustPathBeside(cfgPath), InsecureDev: true, + }); err != nil { + t.Fatalf("Run() = %v", err) + } + b, _ := os.ReadFile(cfgPath) + if !strings.Contains(string(b), "insecure_dev = true") { + t.Errorf("insecure_dev not written:\n%s", b) + } + cfg, err := config.Load(cfgPath) + if err != nil || cfg.CheckTransport() != nil { + t.Fatalf("Load = %v, CheckTransport = %v", err, cfg.CheckTransport()) + } +} + +// The backend's WebSocket URL must be wss:// on the backend's own host; +// otherwise nothing is saved and the board stays unenrolled. +func TestRunRefusesUnsafeWebSocketURL(t *testing.T) { + for name, ws := range map[string]func(*httptest.Server) string{ + "plain ws": func(s *httptest.Server) string { return wsURL(s, "ws") }, + "other host": func(*httptest.Server) string { return "wss://attacker.example/api/agent/ws" }, + } { + t.Run(name, func(t *testing.T) { + var server *httptest.Server + server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "deviceId": "6f1e1d94-1f4a-4a1e-9a1e-2f3c4d5e6a7b", + "websocketUrl": ws(server), + "contentSigningKeys": []any{signingKeyJSON(contentPub)}, + }) + })) + defer server.Close() + + cfgPath, keyPath := paths(t) + err := Run(context.Background(), quietLogger(), Params{ + Token: "t", BackendURL: server.URL, ConfigPath: cfgPath, KeyPath: keyPath, + TrustPath: trustPathBeside(cfgPath), HTTPClient: server.Client(), + }) + if !errors.Is(err, config.ErrInsecureTransport) { + t.Fatalf("Run() = %v, want ErrInsecureTransport", err) + } + assertNothingSaved(t, cfgPath, keyPath) + if _, err := os.Stat(trustPathBeside(cfgPath)); err == nil { + t.Error("content keys pinned from a refused reply") + } + }) + } +} + +func assertNothingSaved(t *testing.T, cfgPath, keyPath string) { + t.Helper() + if _, err := os.Stat(cfgPath); err == nil { + t.Error("config written") + } + if _, err := os.Stat(keyPath); err == nil { + t.Error("private key written") + } +} + func TestRestorePort(t *testing.T) { tests := []struct { name, wsURL, backendURL, want string From 8bf1b858a42fdc3a5bf2acc38b0b8390cacc2c63 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:11:22 +0000 Subject: [PATCH 07/19] Refuse packages signed in the future of an NTP-synchronised clock A backend (or build machine) whose clock was briefly far ahead signs a createdAt in, say, 2030. Every verified createdAt raises the board's clock floor, so that one package pinned every board that took it in 2030. With NTP synchronised, a package whose createdAt is more than a day past the board's clock is now refused ("An update is dated in the future: the computer that made it has a wrong clock"; the times are in the detail), and so is its Preflight. Nothing about it is recorded: not its time, and not its sequence, so the backend's next right package installs even if its sequence is lower. A createdAt less than a day ahead installs, but raises the floor only up to NTP time. Without NTP the board has nothing better than the signed time, and it counts as before. Content sequence is unchanged: still strictly increasing, with no upper bound. --- internal/importer/importer.go | 47 ++++++++++++++-- internal/importer/importer_test.go | 86 +++++++++++++++++++++++++++++- internal/mbu/manifest.go | 19 +++++++ internal/mbu/mbu_test.go | 28 ++++++++++ 4 files changed, 176 insertions(+), 4 deletions(-) diff --git a/internal/importer/importer.go b/internal/importer/importer.go index e9b3254..1b7cb1c 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/LensBridge/agent/internal/clock/timesync" "github.com/LensBridge/agent/internal/fsutil" "github.com/LensBridge/agent/internal/mbu" "github.com/LensBridge/agent/internal/notice" @@ -135,6 +136,10 @@ type Deps struct { AgentStaged func(n notice.Notice) Logger *slog.Logger Now func() time.Time + // TrustedNow is the system clock and whether it is NTP-synchronised. + // Only a synchronised clock may judge a signed createdAt (section 10). + // Nil means timesync.Now. + TrustedNow func() (time.Time, bool) } // Importer serialises imports. @@ -180,6 +185,9 @@ func New(d Deps) *Importer { if d.Logger == nil { d.Logger = slog.New(slog.DiscardHandler) } + if d.TrustedNow == nil { + d.TrustedNow = timesync.Now + } return &Importer{d: d} } @@ -360,12 +368,17 @@ func (im *Importer) one(ctx context.Context, src Source, path, name string, ring } defer pkg.Close() m := pkg.Manifest + created, err := im.signedTime(m) + if err != nil { + return Result{File: name, Type: m.Type, Version: m.Version, Sequence: m.Sequence, Action: ActionRejected, + Message: futureMessage, Detail: err.Error()} + } b.Verified = true - if t := m.CreatedTime(); t.After(b.MaxCreatedAt) { - b.MaxCreatedAt = t + if created.After(b.MaxCreatedAt) { + b.MaxCreatedAt = created } // A signed createdAt is a lower bound on the real time (section 10). - if err := l.State().RaiseSignedClockFloor(m.CreatedTime().Unix()); err != nil { + if err := l.State().RaiseSignedClockFloor(created.Unix()); err != nil { im.d.Logger.Warn("could not record clock floor", "err", err) } st, err := l.State().Load() @@ -450,6 +463,31 @@ func (im *Importer) one(ctx context.Context, src Source, path, name string, ring return res } +// futureMessage is what the board says about a package refused by +// signedTime. +const futureMessage = "An update is dated in the future: the computer that made it has a wrong clock" + +// signedTime is the time a verified package proves has passed: its createdAt. +// With an NTP-synchronised clock, NTP outranks it: a createdAt more than +// mbu.MaxCreatedAhead ahead is refused (the signer's clock was wrong, and the +// package is not installed, so neither its time nor its sequence is +// recorded), and one slightly ahead counts only up to NTP time. Without NTP, +// createdAt is the best evidence the board has and counts as it is. +func (im *Importer) signedTime(m *mbu.Manifest) (time.Time, error) { + created := m.CreatedTime() + now, synced := im.d.TrustedNow() + if !synced { + return created, nil + } + if err := m.CheckCreatedAt(now); err != nil { + return time.Time{}, err + } + if created.After(now) { + created = now + } + return created, nil +} + // describe names a package for the working screen's caption. func describe(m *mbu.Manifest) string { switch m.Type { @@ -525,6 +563,9 @@ func (im *Importer) Preflight(path string) (*mbu.Manifest, error) { return nil, err } defer pkg.Close() + if _, err := im.signedTime(pkg.Manifest); err != nil { + return nil, fmt.Errorf("%s: %w", futureMessage, err) + } st, err := im.d.Layout.State().Load() if err != nil { return nil, err diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index d19dcea..860aff4 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -87,8 +87,12 @@ func build(t *testing.T, m mbu.Manifest, src []mbu.Source, k ed25519.PrivateKey) } func content(t *testing.T, seq int64, device, poster string, omitMedia bool) string { + return contentAt(t, seq, device, poster, omitMedia, "2026-09-24T10:00:00Z") +} + +func contentAt(t *testing.T, seq int64, device, poster string, omitMedia bool, createdAt string) string { m := mbu.Manifest{Type: mbu.TypeContent, Sequence: seq, DeviceID: device, - CreatedAt: "2026-09-24T10:00:00Z", + CreatedAt: createdAt, Content: &mbu.ContentInfo{Timezone: "America/Toronto", FirstDay: "2026-09-24", LastDay: "2026-09-25", Media: []mbu.MediaInfo{{Path: imgPath(), ContentType: "image/png"}}}} src := []mbu.Source{ @@ -354,3 +358,83 @@ func otherArch() string { } return "arm64" } + +// F4: a backend whose clock is briefly far ahead signs createdAt in the +// future. Taken as a lower bound on the time it would pin every board there. +// With NTP the board refuses such a package outright, so neither its time nor +// its sequence is recorded; a createdAt only slightly ahead installs but +// counts only up to NTP time. Without NTP there is nothing better than the +// signed time, and it counts as it is. +func TestSignedCreatedAtAgainstNTP(t *testing.T) { + ntp := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) + cases := []struct { + name string + synced bool + createdAt time.Time + action string + wantSigned int64 // signedClockFloor afterwards + }{ + {"synced, backend in 2030", true, time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC), ActionRejected, 0}, + {"synced, two days ahead", true, ntp.Add(48 * time.Hour), ActionRejected, 0}, + {"synced, a minute past the limit", true, ntp.Add(mbu.MaxCreatedAhead + time.Minute), ActionRejected, 0}, + {"synced, at the limit", true, ntp.Add(mbu.MaxCreatedAhead), ActionInstalled, ntp.Unix()}, + {"synced, an hour ahead", true, ntp.Add(time.Hour), ActionInstalled, ntp.Unix()}, + {"synced, in the past", true, ntp.Add(-2 * time.Hour), ActionInstalled, ntp.Add(-2 * time.Hour).Unix()}, + {"not synced, backend in 2030", false, time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC), ActionInstalled, + time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC).Unix()}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + im, l, _, _ := setup(t) + im.d.TrustedNow = func() (time.Time, bool) { return ntp, tc.synced } + p := contentAt(t, 100, dev, "/"+imgPath(), false, tc.createdAt.Format(time.RFC3339)) + b := im.Import(context.Background(), SourceSync, []string{p}) + r := one(t, b) + if r.Action != tc.action { + t.Fatalf("action = %s (%s: %s), want %s", r.Action, r.Message, r.Detail, tc.action) + } + st, _ := l.State().Load() + if st.SignedClockFloor != tc.wantSigned || st.ClockFloor != tc.wantSigned { + t.Fatalf("signed floor %d, floor %d; want %d", st.SignedClockFloor, st.ClockFloor, tc.wantSigned) + } + if tc.action == ActionRejected { + if r.Message != futureMessage || !strings.Contains(r.Detail, "createdAt") { + t.Fatalf("rejection: %+v", r) + } + if st.ContentSequence != 0 { + t.Fatalf("a refused package's sequence was recorded: %d", st.ContentSequence) + } + if _, err := l.CurrentContent(); err == nil { + t.Fatal("a refused package was installed") + } + if b.Verified { + t.Fatal("a refused package counted as verified") + } + // The board is not wedged: the backend's next, right package + // installs even though its sequence is lower. + fixed := contentAt(t, 99, dev, "/"+imgPath(), false, ntp.Format(time.RFC3339)) + if r := one(t, im.Import(context.Background(), SourceSync, []string{fixed})); r.Action != ActionInstalled { + t.Fatalf("next package after a refused one: %+v", r) + } + } + }) + } +} + +func TestFutureSoftwareIsRefused(t *testing.T) { + im, _, _, _ := setup(t) + real := time.Now() + im.d.TrustedNow = func() (time.Time, bool) { return real.Add(-3 * mbu.MaxCreatedAhead), true } + // app() signs with the real time: three days "ahead" of this board. + p := app(t, "2.1.0", 2) + if r := one(t, im.Import(context.Background(), SourceUpload, []string{p})); r.Action != ActionRejected || r.Message != futureMessage { + t.Fatalf("future app: %+v", r) + } + if _, err := im.Preflight(p); err == nil || !strings.Contains(err.Error(), "createdAt") { + t.Fatalf("Preflight = %v", err) + } + im.d.TrustedNow = func() (time.Time, bool) { return real, true } + if _, err := im.Preflight(p); err != nil { + t.Fatalf("Preflight with a right clock: %v", err) + } +} diff --git a/internal/mbu/manifest.go b/internal/mbu/manifest.go index aa24bd7..3fd18c2 100644 --- a/internal/mbu/manifest.go +++ b/internal/mbu/manifest.go @@ -117,6 +117,25 @@ type AgentInfo struct { // CreatedTime is createdAt, parsed. func (m *Manifest) CreatedTime() time.Time { return m.createdAt } +// MaxCreatedAhead is how far past a trusted clock a package's createdAt may +// be. Build machines' and the backend's clocks disagree with the board's by +// seconds, not by days. +const MaxCreatedAhead = 24 * time.Hour + +// CheckCreatedAt refuses a createdAt more than MaxCreatedAhead past now, which +// must be a trusted (NTP-synchronised) time. Such a package was signed by a +// machine whose clock was wrong; taking its createdAt as a lower bound on the +// time would pin the board in the future (docs/architecture.md, section 10). +// Without a trusted time there is nothing to check against, and createdAt is +// the best evidence the board has. +func (m *Manifest) CheckCreatedAt(now time.Time) error { + if ahead := m.createdAt.Sub(now); ahead > MaxCreatedAhead { + return fmt.Errorf("createdAt %s is %s past the board's NTP-synchronised clock (%s); the machine that signed it has a wrong clock", + m.createdAt.UTC().Format(time.RFC3339), ahead.Round(time.Second), now.UTC().Format(time.RFC3339)) + } + return nil +} + // File returns the listed file at path. func (m *Manifest) File(path string) (File, bool) { f, ok := m.byPath[path] diff --git a/internal/mbu/mbu_test.go b/internal/mbu/mbu_test.go index 3ca5fb1..0a846b6 100644 --- a/internal/mbu/mbu_test.go +++ b/internal/mbu/mbu_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/LensBridge/agent/internal/trust" ) @@ -233,6 +234,33 @@ func TestManifestRules(t *testing.T) { } } +func TestCheckCreatedAt(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + cases := []struct { + createdAt string + ok bool + }{ + {"2026-11-17T12:00:00Z", true}, + {"2020-01-01T00:00:00Z", true}, // old is fine: an old USB stick + {"2026-11-17T12:05:00Z", true}, // a backend a few minutes fast + {"2026-11-18T12:00:00Z", true}, // exactly MaxCreatedAhead + {"2026-11-18T12:00:01Z", false}, // just past it + {"2030-01-01T00:00:00Z", false}, // a backend clock years ahead + {"2026-11-18T14:00:00+05:00", true}, // offsets count: 09:00Z + {"2026-11-19T00:00:00-01:00", false}, // 01:00Z on the 19th + } + for _, tc := range cases { + m, err := ParseManifest([]byte(`{"format":"mbu","formatVersion":2,"type":"app","version":"1.0.0","createdAt":"` + + tc.createdAt + `","files":[{"path":"index.html","sha256":"` + strings.Repeat("a", 64) + `","bytes":1}],"app":{"localApi":1}}`)) + if err != nil { + t.Fatalf("%s: %v", tc.createdAt, err) + } + if err := m.CheckCreatedAt(now); (err == nil) != tc.ok { + t.Errorf("%s: err = %v, want ok=%v", tc.createdAt, err, tc.ok) + } + } +} + func TestCompareVersions(t *testing.T) { cases := []struct { a, b string From 15c3c17662158616b601261c4d200bd2e8e70559 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:11:34 +0000 Subject: [PATCH 08/19] Upload page: say whose clock is wrong when the board has NTP The page told the uploader a drifting board clock would be "corrected when you send an update". A board set from the internet no longer takes an uploader's time, so the page now says the uploader's clock is probably the wrong one. --- internal/uploadserver/page.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/uploadserver/page.go b/internal/uploadserver/page.go index 611dc3b..415a29e 100644 --- a/internal/uploadserver/page.go +++ b/internal/uploadserver/page.go @@ -40,7 +40,7 @@ function loadStatus(){fetch('/api/status',{cache:'no-store'}).then(function(r){r row(dl,'Content',s.content?(s.content.firstDay+' to '+s.content.lastDay):'none installed'); if(s.content){row(dl,'Days left',s.staleDays>0?('ran out '+s.staleDays+' day(s) ago'):String(s.daysRemaining))} var drift=Math.round(s.clock.unix-Date.now()/1000); - row(dl,'Board clock',new Date(s.clock.unix*1000).toLocaleString()+(Math.abs(drift)>5?(' ('+drift+' s off this computer; corrected when you send an update)'):'')); + row(dl,'Board clock',new Date(s.clock.unix*1000).toLocaleString()+(Math.abs(drift)>5?(' ('+drift+' s off this computer; '+(s.clock.source==='ntp'?'the board is set from the internet, so this computer\'s clock is probably wrong':'corrected when you send an update')+')'):'')); row(dl,'RTC',s.rtc?'present':'not fitted'); var src={ntp:'set from the internet',rtc:'kept by the hardware clock',uploader:'set from a laptop or phone',starting:'checking',unverified:'NOT CONFIRMED: send an update from this page to set it'}; if(s.clock.source)row(dl,'Clock',src[s.clock.source]||s.clock.source); From 5836a202b5c221eeea70141ff2a00ff9815faf6b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:12:03 +0000 Subject: [PATCH 09/19] docs: the clock with NTP as the authority, and what the floor is now Section 10 says when the keeper moves the clock and what it writes to the floor, how a floor ahead of NTP time heals, that an uploader's time no longer reaches the floor, and what this trades: what rollback resistance remains without NTP, and what whoever controls NTP can do. Sections 4 and 8 and the threat table say that packages signed more than a day ahead of an NTP clock are refused. --- docs/architecture.md | 72 +++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 57be101..5b7bd52 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,7 +122,7 @@ Common fields, all required unless noted: | `format` | `"mbu"` | | `formatVersion` | `2`. Anything else is rejected. | | `type` | `"content"`, `"app"` or `"agent"` | -| `createdAt` | RFC 3339 UTC, second precision. Used as a signed lower bound for the clock (section 10). | +| `createdAt` | RFC 3339 UTC, second precision. Used as a signed lower bound for the clock (section 10). With an NTP-synchronised clock, a package whose `createdAt` is more than a day ahead of it is refused. | | `files` | Array as above. `sha256` lowercase hex of the file bytes; `bytes` its exact size. Paths unique and valid per the entry-name rules. | Per type: @@ -385,7 +385,10 @@ Per package: - agent: write the verified package to `agent/staged/package.mbu` and create `agent/staged/ready`. The root updater does the rest (section 12). 4. **Clock**: every verified package raises `clockFloor` to its `createdAt` - (section 10). + (section 10). With NTP synchronised, a `createdAt` more than a day ahead + of the clock is refused before anything is recorded ("An update is dated + in the future: the computer that made it has a wrong clock"), and one + less than a day ahead raises the floor only to NTP time. A batch processes `agent` packages first, then `app`, then `content`. When an agent package is staged, the rest of the batch stays in the inbox and is @@ -612,21 +615,44 @@ installs. ## 10. Clock -Correct time decides which day's content is shown and the prayer times. - -- `clockFloor` in `state.json` is the maximum of every verified package's - `createdAt` (also kept as `signedClockFloor`) and the wall clock the daemon - persists every 10 minutes. The persisted wall clock may not raise the floor - more than 400 days past `signedClockFloor`, so a clock once set far into the - future cannot lock the board there. -- At daemon start and every 10 minutes: if the clock is more than 60 s behind - `clockFloor`, set it to `clockFloor` (the agent has `CAP_SYS_TIME`) and write - the RTC when one is present. A signed `createdAt` in the future of the board - is proof its clock is behind. +Correct time decides which day's content is shown and the prayer times, and +the backend refuses a board's signed requests when its clock is more than 5 +minutes off. + +- **NTP is the authority.** When systemd-timesyncd has synchronised the + clock (`/run/systemd/timesync/synchronized`), the agent never moves the + clock, ignores uploaders' clocks, and refuses packages signed more than a + day in its future (section 8). +- `clockFloor` in `state.json` is a lower bound on the time: the maximum of + every verified package's `createdAt` (also kept as `signedClockFloor`) and + the trusted time the daemon persists every 10 minutes. The persisted time + may not raise the floor more than 400 days past `signedClockFloor`. +- **What is persisted.** With NTP: NTP time, once two checks 10 minutes + apart agree with the monotonic clock (a clock stepped by hand between + timesyncd's polls is not taken for NTP time). Without NTP: the last + trusted time (the floor at daemon start, a newer signed `createdAt`, or NTP + time earlier this boot) carried forward by the monotonic clock. The wall + clock itself is never written to the floor without NTP: a manual `date`, a + wrong RTC or an uploader's time cannot raise it. +- **A floor ahead of NTP time is wrong** (an earlier agent let an uploader, + a manual date or a wrong RTC raise it, or a backend signed from the + future). With NTP, once two checks agree, `clockFloor` and + `signedClockFloor` are lowered to NTP time and the journal says so + (`clock: the clock floor was ahead of NTP-synchronised time`). A board in + that state recovers by itself within about 10 minutes of NTP + synchronising. +- **Without NTP**, at daemon start and every 10 minutes: if the clock is + more than 60 s behind `clockFloor`, set it to `clockFloor` (the agent has + `CAP_SYS_TIME`) and write the RTC when one is present. This is what keeps + a board with no RTC and no network from coming up in the past after a + power cut. - An upload's `X-MB-Client-Time` (the uploader's clock when it started - sending; the board measures the offset when the request arrives) is applied only after at least one package in - that request verified, only when it differs by more than 5 s, and only within - `[clockFloor, clockFloor + 90 days]`. The response reports the drift. + sending; the board measures the offset when the request arrives) is + applied only when the clock is not NTP-synchronised, only after at least + one package in that request verified, only when it differs by more than + 5 s, and only within `[clockFloor, clockFloor + 90 days]`. It sets the + clock and the RTC but not the floor, so a later uploader with a right + clock, or NTP, undoes a wrong one. The response reports the drift. - **Trust.** The floor keeps the clock from going backwards; it cannot say the clock is right. `clock` in `/api/local/status` does: `source` is `ntp` (systemd-timesyncd has synchronised it), `rtc` (a hardware clock is @@ -636,6 +662,18 @@ Correct time decides which day's content is shown and the prayer times. `unverified`, and then the board shows "The clock may be wrong: connect the board to the internet, or send an update from a laptop", the upload page says the clock is not confirmed, and `status` suggests an RTC. +- **Trade-offs.** Without NTP, the floor resists rollback as before: nothing + moves the clock below it, and an uploader moves it at most 90 days ahead. + The floor now only counts time the board was running since its last + trusted time, so an uploader's correction and time spent powered off are + not in it, and a later uploader (with a verified package) can set the + clock back as far as the floor. Whoever can answer the board's NTP + queries (unauthenticated NTP: on the LAN, or on the path to the NTP + servers) decides the time, as they always could through timesyncd; the + floor no longer fights them, and they can lower it to their time. Set + back more than a day, the board also refuses new packages as dated in the + future until the spoofing stops. Signed content still cannot be replayed + (sequence and versions are monotonic and do not depend on the clock). - A DS3231 RTC is still recommended (`setup.sh --rtc`), and online boards keep `systemd-timesyncd`. @@ -768,7 +806,7 @@ per-day payloads from installed content, and weather comes through the agent. | Zip-slip, zip bombs, disk exhaustion | Strict entry names, declared sizes enforced while reading, global caps, one import at a time. | | Hostile USB stick | Allow-listed filesystems, read-only `nosuid,nodev,noexec` mount, only `*.mbu` regular files copied, no network in the helper, signature checks as for any source. | | Rogue DHCP on a building network | Service port only after a DHCP client attempt fails; off by default; `service-port off` before connecting. | -| Clock tampering | Clock only moves forward to signed times automatically; uploader time only with a verified package and within 90 days of the floor. | +| Clock tampering | With NTP, nothing but NTP moves the clock, and a floor ahead of NTP time is lowered. Without NTP, the clock only moves forward to the floor automatically; uploader time only with a verified package and within 90 days past the floor, and never into the floor. Packages signed more than a day ahead of an NTP clock are refused. | | DNS rebinding against the upload server | `Host` allow-list. | ## 17. Out of scope From 55cbc5de50eefa809d68ad9461fd032248fc9792 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:12:35 +0000 Subject: [PATCH 10/19] Refuse an insecure backend config at daemon start Nothing checked the backend and WebSocket URLs after enrollment, so a board enrolled over http kept syncing content, dialling ws:// and taking remote commands in the clear. - startBoard runs config.CheckTransport before it builds anything that talks to the backend. When it fails, no syncer (content sync, refresh channel, release checks) and no command WebSocket are started; the board keeps showing what it has. The reason and the fix go to the log at every start, to `systemctl status` (STATUS=), and to the local status, so `musallahboard-agent status` shows "LensBridge: NOT connected" and "Sync: failing" with it, as does the board app. - With insecure_dev = true a loud warning is logged at every start. - wsclient never dials a WebSocket URL that fails the check (backstop). - trust fetch refuses a plain-http backend unless insecure_dev is set in the config, or --insecure-dev is passed with --backend. --- cmd/agent/board.go | 51 ++++++++++++++++++++++++++--- cmd/agent/board_test.go | 34 +++++++++++++++++++ cmd/agent/trust_cli.go | 9 ++++- internal/wsclient/transport_test.go | 49 +++++++++++++++++++++++++++ internal/wsclient/wsclient.go | 9 +++++ 5 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 internal/wsclient/transport_test.go diff --git a/cmd/agent/board.go b/cmd/agent/board.go index d589ee3..f79aad3 100644 --- a/cmd/agent/board.go +++ b/cmd/agent/board.go @@ -77,6 +77,16 @@ func startBoard(ctx context.Context, logger *slog.Logger, cfg *config.Config, sa logger.Error("device key unreadable: serving installed content only (no sync, no remote commands)", "err", keyErr, "keyPath", cfg.KeyPath) } + // Nothing that talks to the backend is built from an insecure config: + // content sync, its refresh channel and the command WebSocket all take + // their addresses from it (docs/architecture.md, section 11). + transportErr := backendTransport(logger, cfg) + offline := func() string { + if transportErr != nil { + return "not talking to LensBridge: " + transportErr.Error() + } + return "device key unreadable" + } hour, minute := cfg.UpdateTime() var syncer *boardsync.Syncer sched := updates.New(updates.Deps{ @@ -84,12 +94,12 @@ func startBoard(ctx context.Context, logger *slog.Logger, cfg *config.Config, sa Changed: func(info updates.Info) { hub.UpdatesChanged(info) }, Check: func(ctx context.Context) error { if syncer == nil { - return errors.New("this board cannot reach the release channels (device key unreadable)") + return errors.New("this board cannot reach the release channels (" + offline() + ")") } return syncer.CheckChannels(ctx) }, }) - if keyErr == nil { + if keyErr == nil && transportErr == nil { syncer = boardsync.New(boardsync.Deps{ Cfg: cfg, Key: priv, Layout: layout, Importer: imp, Updates: sched, AgentVersion: version.Version, Logger: logger, @@ -103,7 +113,9 @@ func startBoard(ctx context.Context, logger *slog.Logger, cfg *config.Config, sa Layout: layout, DeviceID: cfg.DeviceID, AgentVersion: version.Version, Hub: hub, Sync: func() any { if syncer == nil { - return map[string]any{"enabled": false, "lastError": "device key unreadable"} + // Enabled as configured, so the board app and `status` show + // the reason as a sync error rather than "sync off". + return map[string]any{"enabled": transportErr != nil && cfg.ContentSync(), "lastError": offline()} } return syncer.Status() }, @@ -121,6 +133,9 @@ func startBoard(ctx context.Context, logger *slog.Logger, cfg *config.Config, sa s := ws.State() return &s } + if transportErr != nil { + return map[string]any{"connected": false, "lastError": offline()} + } return nil }, ServicePort: cfg.ServicePort(), @@ -145,7 +160,11 @@ func startBoard(ctx context.Context, logger *slog.Logger, cfg *config.Config, sa logger.Warn("could not write kiosk url", "err", err) } _, _ = daemon.SdNotify(false, daemon.SdNotifyReady) - _, _ = daemon.SdNotify(false, "STATUS=serving the board on "+localserver.ListenAddr) + if transportErr != nil { + _, _ = daemon.SdNotify(false, "STATUS=serving installed content only; refusing the backend: "+transportErr.Error()) + } else { + _, _ = daemon.SdNotify(false, "STATUS=serving the board on "+localserver.ListenAddr) + } logInstalled(logger, layout, cfg) goRun(wg, func() { watchKiosk(ctx, logger, cdpClient, screen.Active) }) @@ -172,6 +191,30 @@ func startBoard(ctx context.Context, logger *slog.Logger, cfg *config.Config, sa return nil } +// backendTransport checks that the config may be used to talk to the +// backend (config.CheckTransport) and says so loudly at every start when it +// may not, or when it may only because of the insecure_dev opt-in. A board +// enrolled over plain http before https was required lands here: it keeps +// showing what it has, and the log, `systemctl status`, `musallahboard-agent +// status` and the board app's sync error all carry the reason and the fix. +func backendTransport(logger *slog.Logger, cfg *config.Config) error { + if err := cfg.CheckTransport(); err != nil { + logger.Error("REFUSING TO TALK TO THE BACKEND: this board's config is not safe to use. "+ + "Serving installed content only: no content sync, no release channel checks, no remote commands. "+ + "Re-enroll it over https (sudo musallahboard-agent enroll --force --token=... --backend=https://...), "+ + "or, for a development board only, add insecure_dev = true to "+defaultConfigPath+" and restart the agent.", + "err", err, "backendUrl", cfg.BackendURL, "websocketUrl", cfg.WebSocketURL) + return err + } + if cfg.InsecureDev() { + logger.Warn("INSECURE DEVELOPMENT BOARD: insecure_dev = true, so this board accepts a plaintext backend. "+ + "Anyone on its network can impersonate the backend: send it content, reboot it, read its screen and logs. "+ + "Never deploy a board like this.", + "backendUrl", cfg.BackendURL, "websocketUrl", cfg.WebSocketURL) + } + return nil +} + func startUploadServer(ctx context.Context, logger *slog.Logger, cfg *config.Config, layout store.Layout, imp *importer.Importer, keeper *clock.Keeper, screen *updatescreen.Screen, wg *sync.WaitGroup) { ln, err := uploadserver.Listen() diff --git a/cmd/agent/board_test.go b/cmd/agent/board_test.go index 78127b2..8dcb4f3 100644 --- a/cmd/agent/board_test.go +++ b/cmd/agent/board_test.go @@ -1,13 +1,16 @@ package main import ( + "bytes" "context" "fmt" "log/slog" + "strings" "testing" "time" "github.com/LensBridge/agent/internal/cdp" + "github.com/LensBridge/agent/internal/config" ) // fakeKiosk scripts what the browser reports and records what the healer did. @@ -243,3 +246,34 @@ func TestKioskHealerChromeErrorPage(t *testing.T) { } r.want(0, 0) } + +func TestBackendTransport(t *testing.T) { + on := true + cases := []struct { + name string + cfg config.Config + wantErr bool + wantLog string + }{ + {"tls", config.Config{BackendURL: "https://api.example.com", WebSocketURL: "wss://api.example.com/ws"}, false, ""}, + {"http without opt-in", config.Config{BackendURL: "http://10.0.2.2:8080", WebSocketURL: "ws://10.0.2.2:8080/ws"}, true, "REFUSING TO TALK TO THE BACKEND"}, + {"socket elsewhere", config.Config{BackendURL: "https://api.example.com", WebSocketURL: "wss://other.example.com/ws"}, true, "REFUSING TO TALK TO THE BACKEND"}, + {"insecure dev", config.Config{BackendURL: "http://10.0.2.2:8080", WebSocketURL: "ws://10.0.2.2:8080/ws", InsecureDevSet: &on}, false, "INSECURE DEVELOPMENT BOARD"}, + {"insecure dev with tls still warns", config.Config{BackendURL: "https://api.example.com", WebSocketURL: "wss://api.example.com/ws", InsecureDevSet: &on}, false, "INSECURE DEVELOPMENT BOARD"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + err := backendTransport(slog.New(slog.NewTextHandler(&buf, nil)), &tc.cfg) + if (err != nil) != tc.wantErr { + t.Errorf("err = %v, want error %v", err, tc.wantErr) + } + if tc.wantLog == "" && buf.Len() != 0 { + t.Errorf("unexpected log: %s", buf.String()) + } + if tc.wantLog != "" && !strings.Contains(buf.String(), tc.wantLog) { + t.Errorf("log %q does not say %q", buf.String(), tc.wantLog) + } + }) + } +} diff --git a/cmd/agent/trust_cli.go b/cmd/agent/trust_cli.go index 0ee49fb..12e6d8c 100644 --- a/cmd/agent/trust_cli.go +++ b/cmd/agent/trust_cli.go @@ -152,17 +152,24 @@ func trustRemove(keyID string) { func trustFetch(args []string) { fs := flag.NewFlagSet("trust fetch", flag.ExitOnError) backend := fs.String("backend", "", "backend base URL, for a board that is not enrolled (default: backend_url from "+defaultConfigPath+")") + insecureDev := fs.Bool("insecure-dev", false, "development only: allow a plaintext http:// --backend") if err := fs.Parse(args); err != nil || fs.NArg() != 0 { trustUsage() } requireRoot("trust fetch") url := *backend + insecure := *insecureDev if url == "" { cfg, err := config.Load(defaultConfigPath) if err != nil { fail("cannot tell which backend to ask: %v\n Pass it explicitly: sudo musallahboard-agent trust fetch --backend https://", err) } - url = cfg.BackendURL + url, insecure = cfg.BackendURL, cfg.InsecureDev() + } + // The keys fetched here decide which content this board accepts; over + // plain http anyone on the network could choose them. + if err := config.CheckBackendURL(url, insecure); err != nil { + fail("%v\n Nothing has changed.", err) } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) diff --git a/internal/wsclient/transport_test.go b/internal/wsclient/transport_test.go new file mode 100644 index 0000000..d1cfc8b --- /dev/null +++ b/internal/wsclient/transport_test.go @@ -0,0 +1,49 @@ +package wsclient + +import ( + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/LensBridge/agent/internal/config" +) + +// A plaintext or foreign-host WebSocket URL is never dialled without the +// insecure_dev opt-in. +func TestRunRefusesInsecureWebSocketURL(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) })) + defer srv.Close() + host := strings.TrimPrefix(srv.URL, "http://") + + for name, cfg := range map[string]*config.Config{ + "ws": {DeviceID: "d", BackendURL: srv.URL, WebSocketURL: "ws://" + host + "/ws"}, + "other host": {DeviceID: "d", BackendURL: "https://api.example.com", WebSocketURL: "wss://" + host + "/ws"}, + } { + t.Run(name, func(t *testing.T) { + c := New(cfg, nil, slog.New(slog.DiscardHandler), "test", false) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c.Run(ctx) // must return on its own + if ctx.Err() != nil { + t.Fatal("Run kept going") + } + if hits.Load() != 0 { + t.Errorf("dialled %d times", hits.Load()) + } + st := c.State() + if st.Connected || st.LastError == "" { + t.Errorf("state = %+v, want the reason in lastError", st) + } + if !errors.Is(cfg.CheckTransport(), config.ErrInsecureTransport) { + t.Errorf("CheckTransport = %v", cfg.CheckTransport()) + } + }) + } +} diff --git a/internal/wsclient/wsclient.go b/internal/wsclient/wsclient.go index 36eb513..190b643 100644 --- a/internal/wsclient/wsclient.go +++ b/internal/wsclient/wsclient.go @@ -131,7 +131,16 @@ func (c *Client) SetBoardReport(f func() any) { c.boardReport = f } // Run blocks until ctx is cancelled, repeatedly attempting to maintain a live // authenticated session. Errors are logged and trigger a backoff-and-retry. +// +// A config whose WebSocket URL is not wss:// on the backend's host (and +// insecure_dev is off) is never dialled: Run records why and returns. The +// daemon does not start a client for such a config; this is the backstop. func (c *Client) Run(ctx context.Context) { + if err := c.cfg.CheckTransport(); err != nil { + c.setState(false, err) + c.logger.Error("not connecting to the backend WebSocket", "err", err, "url", c.cfg.WebSocketURL) + return + } backoff := minBackoff for { if ctx.Err() != nil { From ddcf77758647a5bd255fafea38634399186fb626 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:13:05 +0000 Subject: [PATCH 11/19] Docs: https-only enrollment, insecure_dev, kiosk self-healing - README and vm/README: enroll a local plain-http backend with --insecure-dev; what happens to a board already enrolled over http and how to fix it. - Makefile: install-remote examples use an https backend. - architecture.md: the transport rule and insecure_dev (section 11), the kiosk probe and its reload/restart backoff (section 14), and the security table. --- Makefile | 6 ++++-- README.md | 10 +++++++++- docs/architecture.md | 28 +++++++++++++++++++++++++++- vm/README.md | 13 ++++++++++++- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c5174f5..b5d7c25 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,9 @@ deploy: build-arm64 ssh -t $(PI) 'sudo bash /tmp/update.sh /tmp/musallahboard-agent-arm64' # First-time full install on a remote host via the release tarball. -# Usage: make install-remote PI=admin@pi.local TOKEN=xxx BACKEND=http://host:8080 +# Usage: make install-remote PI=admin@pi.local TOKEN=xxx BACKEND=https://backend.example.org +# (BACKEND must be https://; for a plain-http dev backend, enroll afterwards on +# the board with `musallahboard-agent enroll ... --insecure-dev`.) install-remote: package @if [ -z "$(PI)" ]; then echo "Usage: make install-remote PI=user@host [TOKEN=xxx BACKEND=url]"; exit 1; fi scp build/musallahboard-agent-$(VERSION)-arm64.tar.gz $(PI):/tmp/ @@ -104,7 +106,7 @@ install-remote: package $(if $(TOKEN),--token=$(TOKEN)) $(if $(BACKEND),--backend=$(BACKEND))' # First-time full install on a remote x86-64 host via the release tarball. -# Usage: make install-remote-x86 HOST=admin@server TOKEN=xxx BACKEND=http://host:8080 +# Usage: make install-remote-x86 HOST=admin@server TOKEN=xxx BACKEND=https://backend.example.org install-remote-x86: package-amd64 @if [ -z "$(HOST)" ]; then echo "Usage: make install-remote-x86 HOST=user@host [TOKEN=xxx BACKEND=url]"; exit 1; fi scp build/musallahboard-agent-$(VERSION)-amd64.tar.gz $(HOST):/tmp/ diff --git a/README.md b/README.md index 59666f6..97bd04a 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,16 @@ sudo musallahboard-agent enroll --token= --backend= # Easiest: copy the whole command from the admin portal when you issue the token. ``` +The backend URL must be `https://`, and the WebSocket address the backend hands back must be `wss://` on the same host; the agent refuses anything else, because the enrollment reply pins the keys the board trusts content from. (A development backend on plain http needs `--insecure-dev`; see [Development boards](#development-boards-on-plain-http).) + The device will then be enrolled and the board will load automatically once its first content and board app packages have synced. The device is now ready to be used as a kiosk! +### Development boards on plain http + +For a backend running on your own machine without TLS, add `--insecure-dev` to the enroll command. It is saved in `/etc/musallahboard/agent.toml` as `insecure_dev = true`, and the agent logs a warning at every start while it is on: anyone on that board's network can impersonate the backend. Never use it on a deployed board. + +A board enrolled over http before https was required, without `insecure_dev = true`, keeps showing the content it has but no longer talks to the backend (no sync, no update checks, no remote commands). `musallahboard-agent status`, `systemctl status musallahboard-agent` and the journal say why. Re-enroll it over https with `--force`, or, if it is a development board, add `insecure_dev = true` to its `agent.toml` and `sudo systemctl restart musallahboard-agent`. + ## Building A little unorthodox, but this go project uses a Makefile to build the agent binary. The Makefile has targets for both native and cross-compilation. @@ -81,7 +89,7 @@ agent release (or a build of this checkout): ```bash vm/mbvm create -vm/mbvm enroll --token= --backend=http://localhost:8080 +vm/mbvm enroll --token= --backend=http://localhost:8080 --insecure-dev vm/mbvm reset --agent=0.2.1 ``` diff --git a/docs/architecture.md b/docs/architecture.md index 57be101..b7e97ce 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -642,7 +642,16 @@ Correct time decides which day's content is shown and the prayer times. ## 11. Configuration (`/etc/musallahboard/agent.toml`) Existing keys (`device_id`, `backend_url`, `websocket_url`, `key_path`) are -unchanged. New keys, all optional: +unchanged, with one rule (`config.CheckTransport`): `backend_url` must be +`https://` and `websocket_url` `wss://` on the same host as `backend_url` (the +port may differ: TLS binds the host name, not the port; there is no override +for a socket on another host). `enroll` checks the backend URL before it sends +the token and the returned WebSocket URL before it saves anything, and the +daemon checks both at every start. A config that fails is still read, so the +board keeps showing what it has and the local CLI keeps working, but nothing +that talks to the backend is started (content sync, refresh channel, release +checks, the command WebSocket); the reason goes to the log, `systemctl status` +and the `sync`/backend errors in `/api/local/status`. New keys, all optional: | Key | Default | Meaning | |---|---|---| @@ -653,6 +662,7 @@ unchanged. New keys, all optional: | `auto_update` | `true` | Follow the release channels. | | `app_channel_url`, `agent_channel_url` | section 9.4 | Channel pointers. Empty disables that channel. | | `update_time` | `"23:00"` | When software from the channels installs, `HH:MM` in the board's time zone (section 9.4). | +| `insecure_dev` | `false` | Development boards only: allow `http://` and `ws://` (the same-host rule still applies). Set by `enroll --insecure-dev`; the daemon logs a warning at every start while it is on. | An offline board needs no configuration change: sync attempts fail fast with no route and cost nothing, and the moment it gets a network it starts syncing. @@ -730,6 +740,21 @@ connecting a board to a network permanently. `RestartMaxDelaySec`, systemd 254 and later). - The daemon still watches for Chromium's error page and navigates back to the board, as a backstop. +- Every 30 s it also probes the page over CDP: `Inspector.enable` (which + Chromium answers with `Inspector.targetCrashed` when the tab's renderer is + gone, "Aw, Snap!") and a trivial `Runtime.evaluate` with a 5 s limit. A + crashed page (one probe) or one that has not answered three probes in a row + is reloaded; if it is still dead at the next threshold the kiosk unit is + restarted (`sudo -n systemctl restart`, as the `kiosk.restart` command). + Restarts are at least 5 minutes apart, doubling to an hour, and the backoff + starts over only after an hour of a healthy page. A reload is tried at most + once until the page answers again or the kiosk restarts. Nothing is done + while the update screen is up, nor for a page that answers without the + board's `getStatus()` hook (splash, update screen, an old board app), a page + mid-navigation, or a browser that is not up yet. +- The heartbeat's `kioskAlive` comes from the same probe: false when the + browser is reachable but the page crashed or does not answer; the unit's + state is used only when the browser cannot be reached. ## 15. Frontend (MusallahBoard) @@ -770,6 +795,7 @@ per-day payloads from installed content, and weather comes through the agent. | Rogue DHCP on a building network | Service port only after a DHCP client attempt fails; off by default; `service-port off` before connecting. | | Clock tampering | Clock only moves forward to signed times automatically; uploader time only with a verified package and within 90 days of the floor. | | DNS rebinding against the upload server | `Host` allow-list. | +| Enrollment or backend impersonated on the LAN | `https://` backend and `wss://` socket on the backend's host, enforced at enrollment and at every daemon start; plaintext only with the explicit `insecure_dev` opt-in (section 11). | ## 17. Out of scope diff --git a/vm/README.md b/vm/README.md index 0767b59..045744b 100644 --- a/vm/README.md +++ b/vm/README.md @@ -26,13 +26,24 @@ architecture is emulated, which works but is very slow. ```sh vm/mbvm create # first time: downloads Debian, runs setup.sh (a few minutes) -vm/mbvm enroll --token= --backend=http://localhost:8080 +vm/mbvm enroll --token= --backend=http://localhost:8080 --insecure-dev ``` `localhost` in `--backend` means your computer: mbvm rewrites it to the address the VM reaches your computer at (10.0.2.2). A deployed backend URL is passed through unchanged. +The agent only enrolls over `https://` (with a `wss://` socket on the same +host). A local backend on plain http needs `--insecure-dev`, which the VM's +config keeps as `insecure_dev = true`; the backend must then hand out a +`ws://` URL on the host the board used, `10.0.2.2`, which it does when it +builds the URL from the request's `Host`. For a deployed backend leave the +flag off: + +```sh +vm/mbvm enroll --token= --backend=https://backend.example.org +``` + Once enrolled, the board installs the latest board app from its release channel, syncs content and shows the board, exactly like a Pi. From 81f82de0f3f7a287c662bb5e49e2f772833d9c91 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:15:56 +0000 Subject: [PATCH 12/19] Agent self-update: a crash-safe transaction and a real probation The updater judged a new agent only by "/api/local/status reports the new version within 120 s". A crash after that, a release that cannot reach LensBridge, or a power cut between the swap and the verdict all left the new agent installed and never judged again. - The swap is a transaction with a root-owned journal, /usr/lib/musallahboard/update-in-progress.json, written (temp, fsync, rename, fsync dir) before the binary is replaced and at every phase: swapping, probation, committing, rolling-back. The next run finishes whatever it finds, deciding by phase and by the SHA-256 of what is installed, so a power cut at any step resolves the same way. The `ready` marker is put back while the transaction is open, so boards whose path unit predates the journal also start the updater at boot. - Probation: the new agent must report its version within 2 minutes, then run 4 minutes with no restart (systemd's MainPID, NRestarts and ActiveState) and, if the board was connected to LensBridge before the update, connect again (authenticated WebSocket or a content sync) within 3 minutes. A board that was offline is not asked to be online. - A crash loop, a restart or not coming up rolls back and rejects the version. Not reconnecting rolls back but only counts a strike: the backend or network may be at fault, so the version is rejected the second time. - A probation cut short (shutdown, the unit's timeout) leaves the journal and is judged again from scratch, at most 3 times. A rollback that has begun always finishes; one that cannot restore agent.prev says so (rollback-failed) instead of claiming it went back. --- cmd/agent/selfupdate.go | 21 +- internal/selfupdate/probation.go | 250 ++++++++++ internal/selfupdate/selfupdate.go | 572 +++++++++++++++++------ internal/selfupdate/selfupdate_test.go | 613 ++++++++++++++++++++++--- internal/selfupdate/txn.go | 160 +++++++ 5 files changed, 1408 insertions(+), 208 deletions(-) create mode 100644 internal/selfupdate/probation.go create mode 100644 internal/selfupdate/txn.go diff --git a/cmd/agent/selfupdate.go b/cmd/agent/selfupdate.go index fe5b108..4970cac 100644 --- a/cmd/agent/selfupdate.go +++ b/cmd/agent/selfupdate.go @@ -14,21 +14,25 @@ import ( // runSelfUpdate is `selfupdate apply` (docs/architecture.md, section 12), // started as root by musallahboard-agent-update.service when the daemon has -// staged an agent package. Its output goes to the journal. It exits non-zero -// when the update was refused or rolled back, so `systemctl status` shows -// that something needs a look; the outcome is also in last-update.json. +// staged an agent package, or when an update a previous run left unfinished +// (a power cut, a reboot) has to be resumed. Its output goes to the journal. +// It exits non-zero when the update was refused, rolled back or interrupted, +// so `systemctl status` shows that something needs a look; the outcome is +// also in last-update.json. func runSelfUpdate(args []string) { if len(args) != 1 || args[0] != "apply" { fmt.Fprintln(os.Stderr, "usage: musallahboard-agent selfupdate apply") - fmt.Fprintln(os.Stderr, " Installs the agent update the daemon staged, and rolls it back if the new agent does not start.") + fmt.Fprintln(os.Stderr, " Installs the agent update the daemon staged, watches the new agent for a few minutes,") + fmt.Fprintln(os.Stderr, " and rolls it back if it does not stay up (or reconnect, if the board was connected).") fmt.Fprintln(os.Stderr, " It normally runs by itself; there is rarely a reason to run it by hand.") os.Exit(2) } requireRoot("selfupdate apply") - // Once the binary is swapped the rest must finish (health check, and - // rollback if needed), so a hangup of an interactive terminal is ignored - // and SIGTERM only cancels the waits. + // A hangup of an interactive terminal is ignored. SIGTERM (shutdown, or + // the unit's start timeout) stops a probation where it is: the update + // record stays, and the next run (at boot at the latest) judges the new + // agent again. A rollback that has begun finishes regardless. signal.Ignore(syscall.SIGHUP, syscall.SIGPIPE) ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer cancel() @@ -38,6 +42,9 @@ func runSelfUpdate(args []string) { switch { case errors.Is(err, selfupdate.ErrNothingStaged): fmt.Println("No agent update is staged; nothing to do.") + case errors.Is(err, selfupdate.ErrInterrupted): + fmt.Printf("Agent update %s -> %s: %v; it will be resumed.\n", out.From, out.To, err) + os.Exit(1) case err != nil: fmt.Printf("Agent update %s: %s\n", out.Status, out.Message) os.Exit(1) diff --git a/internal/selfupdate/probation.go b/internal/selfupdate/probation.go new file mode 100644 index 0000000..61d4642 --- /dev/null +++ b/internal/selfupdate/probation.go @@ -0,0 +1,250 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +// Probation timing. Every window but ComeUpTimeout starts when the new agent +// first reports its version. +// +// Boards installed before the update transaction existed run the updater +// under a unit with TimeoutStartSec=5min; a typical update (seconds to verify +// and start, then ProbationPeriod) fits inside it. If one does not, systemd +// stops the updater, the record stays, and the probation is started again +// (see Txn.Attempts). +const ( + // ComeUpTimeout is how long the restarted agent has to report its + // version on the loopback status endpoint. + ComeUpTimeout = 120 * time.Second + // ResumeComeUpTimeout replaces it when a probation is resumed at boot: + // the agent waits for the network before it starts. + ResumeComeUpTimeout = 5 * time.Minute + // BackendTimeout is how long a board that was connected to LensBridge + // before the update has to connect again. + BackendTimeout = 3 * time.Minute + // ProbationPeriod is how long the new agent must then run without being + // restarted. + ProbationPeriod = 4 * time.Minute + // FinalCheckGrace is how long, after ProbationPeriod, the updater keeps + // trying for the clean final reading the verdict needs. + FinalCheckGrace = 30 * time.Second + // PollInterval is how often the agent is looked at during probation. + PollInterval = 3 * time.Second + // MaxProbationAttempts bounds how often a probation cut short is + // started again before the update is rolled back. + MaxProbationAttempts = 3 +) + +// AgentStatus is the part of /api/local/status (docs/architecture.md, +// section 7) the updater judges by. Every field has been there since local +// API 2, so the previous agent's answer reads the same as the new one's. +type AgentStatus struct { + AgentVersion string `json:"agentVersion"` + Backend *struct { + Connected bool `json:"connected"` + } `json:"backend"` + Sync *struct { + LastSuccessAt *string `json:"lastSuccessAt"` + LastError *string `json:"lastError"` + } `json:"sync"` +} + +// BackendUp reports whether the agent is connected to LensBridge: its +// WebSocket is authenticated, or a content sync worked and none has failed +// since. Both live in the agent's memory, so for a freshly started agent +// they say what that process achieved. +func (s AgentStatus) BackendUp() bool { + if s.Backend != nil && s.Backend.Connected { + return true + } + return s.Sync != nil && s.Sync.LastSuccessAt != nil && s.Sync.LastError == nil +} + +// ServiceState is what systemd says about the agent's unit. +type ServiceState struct { + ActiveState string + MainPID int + NRestarts int +} + +// serviceState asks systemd about the agent unit. systemd, not the agent, +// is the authority on whether the agent was restarted: a crash between two +// looks shows as a new MainPID or a higher NRestarts. +func (u *Updater) serviceState(ctx context.Context) (ServiceState, error) { + out, err := u.Run(ctx, "systemctl", "show", AgentUnit, "--property=ActiveState,MainPID,NRestarts") + if err != nil { + return ServiceState{}, fmt.Errorf("systemctl show: %v %s", err, firstLine(out)) + } + var s ServiceState + seen := 0 + for _, line := range strings.Split(out, "\n") { + k, v, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + switch k { + case "ActiveState": + s.ActiveState, seen = v, seen+1 + case "MainPID": + s.MainPID, _ = strconv.Atoi(v) + seen++ + case "NRestarts": + s.NRestarts, _ = strconv.Atoi(v) + seen++ + } + } + if seen != 3 { + return ServiceState{}, fmt.Errorf("systemctl show said %q", firstLine(out)) + } + return s, nil +} + +// ErrInterrupted means the updater was told to stop mid-probation. The +// record stays, so the probation is resumed by the next run. +var ErrInterrupted = errors.New("the updater was stopped before the new agent could be judged") + +// verdict is a failed probation: why, and whether the version should be +// rejected outright (false: only a strike, see strikes). +type verdict struct { + reason string + reject bool +} + +// probation watches the restarted agent and returns nil if it passes: +// +// - it reports rec.To on the loopback status endpoint within the come-up +// timeout; +// - from then on systemd never restarts it (same MainPID, same NRestarts, +// still active) for ProbationPeriod; +// - if rec.BackendBefore, it connects to LensBridge within BackendTimeout. +// +// A board that was offline before the update is not asked to be online +// after it. +func (u *Updater) probation(ctx context.Context, rec *Txn, resumed bool) (*verdict, error) { + comeUp := u.ComeUpTimeout + if resumed { + comeUp = u.ResumeComeUpTimeout + } + start := u.Now() + var ( + healthy bool + healthyAt time.Time + base ServiceState + backendOK = !rec.BackendBefore + last = "no answer" + ) + need := "" + if rec.BackendBefore { + need = ", and connect to LensBridge within " + u.BackendTimeout.String() + } + u.Logf("judging agent %s (attempt %d): it must report its version within %s, then run %s without a restart%s", + rec.To, rec.Attempts, comeUp, u.ProbationPeriod, need) + for { + if ctx.Err() != nil { + return nil, ErrInterrupted + } + st, serr := u.Status(ctx) + svc, verr := u.serviceState(ctx) + if ctx.Err() != nil { + return nil, ErrInterrupted + } + now := u.Now() + switch { + case serr != nil: + last = serr.Error() + case st.AgentVersion != rec.To: + last = "it reports version " + st.AgentVersion + default: + last = "" + } + good := serr == nil && st.AgentVersion == rec.To && verr == nil && svc.ActiveState == "active" && svc.MainPID > 0 + + if !healthy { + if good { + healthy, healthyAt, base = true, now, svc + u.Logf("agent %s is up (pid %d)", rec.To, svc.MainPID) + } else if now.Sub(start) >= comeUp { + if last == "" && verr != nil { + last = verr.Error() + } + return &verdict{fmt.Sprintf("agent %s did not come up within %s: %s", rec.To, comeUp, last), true}, nil + } + } else { + if verr == nil && (svc.MainPID != base.MainPID || svc.NRestarts != base.NRestarts || + (svc.ActiveState != "active" && svc.ActiveState != "reloading")) { + return &verdict{fmt.Sprintf("agent %s was restarted %s after it came up (systemd: %s, pid %d, %d restarts)", + rec.To, now.Sub(healthyAt).Round(time.Second), svc.ActiveState, svc.MainPID, svc.NRestarts), true}, nil + } + if serr == nil && st.AgentVersion != rec.To { + return &verdict{fmt.Sprintf("agent %s was replaced by %s during its probation", rec.To, st.AgentVersion), true}, nil + } + } + if healthy && !backendOK { + if serr == nil && st.BackendUp() { + backendOK = true + u.Logf("agent %s is connected to LensBridge after %s", rec.To, now.Sub(healthyAt).Round(time.Second)) + } else if now.Sub(healthyAt) >= u.BackendTimeout { + return &verdict{fmt.Sprintf("agent %s did not connect to LensBridge within %s, although the board was connected before the update", + rec.To, u.BackendTimeout), false}, nil + } + } + if healthy && backendOK && now.Sub(healthyAt) >= u.ProbationPeriod { + if good && svc.MainPID == base.MainPID && svc.NRestarts == base.NRestarts { + return nil, nil + } + if now.Sub(healthyAt) >= u.ProbationPeriod+u.FinalCheckGrace { + if last == "" && verr != nil { + last = verr.Error() + } + return &verdict{fmt.Sprintf("agent %s stopped answering at the end of its probation: %s", rec.To, last), true}, nil + } + } + if err := u.Sleep(ctx, u.PollInterval); err != nil { + return nil, ErrInterrupted + } + } +} + +// FetchStatus reads the agent's /api/local/status. +func FetchStatus(ctx context.Context, statusURL string) (AgentStatus, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, statusURL, nil) + if err != nil { + return AgentStatus{}, err + } + // Loopback only: never through a proxy from the environment. + client := &http.Client{Transport: &http.Transport{Proxy: nil, DisableKeepAlives: true}} + resp, err := client.Do(req) + if err != nil { + return AgentStatus{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return AgentStatus{}, fmt.Errorf("status returned %d", resp.StatusCode) + } + var st AgentStatus + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&st); err != nil { + return AgentStatus{}, err + } + return st, nil +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/internal/selfupdate/selfupdate.go b/internal/selfupdate/selfupdate.go index 423a831..a9c504d 100644 --- a/internal/selfupdate/selfupdate.go +++ b/internal/selfupdate/selfupdate.go @@ -11,9 +11,12 @@ // and only then replaces /usr/bin/musallahboard-agent. A compromised daemon // can therefore at worst make the updater refuse something. // -// A new agent that does not come up and report its version within two -// minutes is rolled back, and its version is recorded so the board never -// tries it again. +// Replacing the binary is a transaction with a root-owned journal (txn.go), +// so a crash or power cut at any step is finished by the next run. The new +// agent then has a probation (probation.go): it must come up, stay up without +// a restart for several minutes and, if the board was connected to LensBridge +// before, connect again. If it does not, the previous agent is restored and +// the version is recorded so the board does not try it again. package selfupdate import ( @@ -21,9 +24,7 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/fs" - "net/http" "os" "os/exec" "path/filepath" @@ -44,9 +45,13 @@ const ( DefaultBinary = "/usr/bin/musallahboard-agent" DefaultNewBinary = "/usr/bin/.musallahboard-agent.new" DefaultPrev = "/usr/lib/musallahboard/agent.prev" - AgentUnit = "musallahboard-agent.service" - StatusURL = "http://127.0.0.1:8080/api/local/status" - HealthTimeout = 120 * time.Second + // DefaultTxn is the update record (txn.go). Root-owned, beside + // agent.prev, where the daemon cannot write. + DefaultTxn = "/usr/lib/musallahboard/update-in-progress.json" + // DefaultStrikes counts connection-only failures per version. + DefaultStrikes = "/usr/lib/musallahboard/update-strikes.json" + AgentUnit = "musallahboard-agent.service" + StatusURL = "http://127.0.0.1:8080/api/local/status" ) // Outcome statuses written to last-update.json. @@ -54,6 +59,9 @@ const ( StatusOK = "ok" StatusRolledBack = "rolled-back" StatusRefused = "refused" + // StatusRollbackFailed: the new agent failed its probation but the + // previous one could not be restored, so the new one still runs. + StatusRollbackFailed = "rollback-failed" ) // Outcome is agent/last-update.json. @@ -73,6 +81,8 @@ type Updater struct { Binary string // the installed agent NewBinary string // where the new binary is extracted before the swap Prev string // the previous agent, kept for rollback + Txn string // the update record (txn.go) + Strikes string // connection-only failures per version // TempRoot is where the private working directory is made ("" is the // system temp directory; under systemd's PrivateTmp that is private too). TempRoot string @@ -82,32 +92,82 @@ type Updater struct { // Run executes a command and returns its combined output. Run func(ctx context.Context, name string, args ...string) (string, error) - // Healthy waits until the restarted daemon reports version, or fails. - Healthy func(ctx context.Context, version string) error - Now func() time.Time + // Status reads the running agent's /api/local/status. + Status func(ctx context.Context) (AgentStatus, error) + Now func() time.Time + // Sleep waits d or until ctx ends. + Sleep func(ctx context.Context, d time.Duration) error // Logf reports progress (to the journal, via stdout). Logf func(format string, args ...any) + + // Probation timing (probation.go); zero means the default. + ComeUpTimeout, ResumeComeUpTimeout, BackendTimeout time.Duration + ProbationPeriod, FinalCheckGrace, PollInterval time.Duration + + // crash, in tests, is called at each step of the transaction; true + // simulates a power cut there: nothing more runs or is cleaned up. + crash func(step string) bool + poweredOff bool } // New returns an Updater with the production paths, for the agent version // running now. func New(currentVersion string) *Updater { - return &Updater{ + u := &Updater{ Layout: store.Default(), TrustPath: trust.DefaultPath, Binary: DefaultBinary, NewBinary: DefaultNewBinary, Prev: DefaultPrev, + Txn: DefaultTxn, + Strikes: DefaultStrikes, Arch: runtime.GOARCH, CurrentVersion: currentVersion, Run: runCommand, - Healthy: func(ctx context.Context, v string) error { return WaitHealthy(ctx, StatusURL, v, HealthTimeout) }, + Status: func(ctx context.Context) (AgentStatus, error) { return FetchStatus(ctx, StatusURL) }, Now: time.Now, + Sleep: sleepCtx, Logf: func(f string, a ...any) { fmt.Printf(f+"\n", a...) }, } + u.defaults() + return u +} + +func (u *Updater) defaults() { + set := func(d *time.Duration, v time.Duration) { + if *d == 0 { + *d = v + } + } + set(&u.ComeUpTimeout, ComeUpTimeout) + set(&u.ResumeComeUpTimeout, ResumeComeUpTimeout) + set(&u.BackendTimeout, BackendTimeout) + set(&u.ProbationPeriod, ProbationPeriod) + set(&u.FinalCheckGrace, FinalCheckGrace) + set(&u.PollInterval, PollInterval) + if u.Txn == "" { + u.Txn = filepath.Join(filepath.Dir(u.Prev), "update-in-progress.json") + } + if u.Strikes == "" { + u.Strikes = filepath.Join(filepath.Dir(u.Prev), "update-strikes.json") + } + if u.Sleep == nil { + u.Sleep = sleepCtx + } +} + +// PowerCut is what a simulated power cut panics with (tests only). +type PowerCut struct{ Step string } + +func (u *Updater) crashpoint(step string) { + if u.crash != nil && u.crash(step) { + u.poweredOff = true + panic(PowerCut{step}) + } } -// ErrNothingStaged means there was no staged package to apply. +// ErrNothingStaged means there was no staged package to apply and no +// unfinished update to resolve. var ErrNothingStaged = errors.New("no agent update is staged") // refusal is a reason not to install that leaves the running agent alone. @@ -117,153 +177,445 @@ func (r *refusal) Error() string { return r.msg } func refuse(format string, args ...any) error { return &refusal{fmt.Sprintf(format, args...)} } -// Apply installs the staged package, health-checks it and rolls back on -// failure. The returned Outcome has been written to last-update.json (unless -// the error is ErrNothingStaged). -func (u *Updater) Apply(ctx context.Context) (Outcome, error) { - out := Outcome{From: u.CurrentVersion} - // The marker goes first: whatever happens next, the path unit must not - // start this again for the same staging. +// Apply first finishes an update a previous run left unfinished, if there is +// one, and stops there. Otherwise it installs the staged package, puts the +// new agent on probation and commits it or rolls it back. The returned +// Outcome has been written to last-update.json, except with ErrNothingStaged +// and ErrInterrupted (stopped mid-probation: the record stays, and the next +// run resumes it). +func (u *Updater) Apply(ctx context.Context) (out Outcome, err error) { + u.defaults() + // The marker goes first: the path unit must not start this again for + // the same staging. settleReady puts it back if something is still + // waiting when we finish. if err := os.Remove(u.Layout.StagedReady()); err != nil && !errors.Is(err, fs.ErrNotExist) { u.Logf("could not remove %s: %v", u.Layout.StagedReady(), err) } + defer func() { + if u.poweredOff { + return + } + if errors.Is(err, ErrInterrupted) { + // Keep a trigger for the next run (older path units watch + // only `ready`; newer ones also watch the record). + u.markReady() + return + } + u.settleReady() + }() + + rec, lerr := loadTxn(u.Txn) + switch { + case lerr == nil: + u.Logf("resuming the unfinished update from agent %s to %s (%s)", rec.From, rec.To, rec.Phase) + return u.resume(ctx, rec) + case !errors.Is(lerr, fs.ErrNotExist): + return u.unreadableTxn(ctx, lerr) + } if _, err := os.Lstat(u.Layout.StagedPackage()); errors.Is(err, fs.ErrNotExist) { - return out, ErrNothingStaged + return Outcome{From: u.CurrentVersion}, ErrNothingStaged } - err := u.apply(ctx, &out) - var r *refusal - switch { - case err == nil: - out.Status = StatusOK - out.Message = fmt.Sprintf("Updated the agent from %s to %s", out.From, out.To) - case errors.As(err, &r): + out = Outcome{From: u.CurrentVersion} + rec, err = u.prepare(ctx, &out) + if err != nil { + var r *refusal out.Status = StatusRefused - out.Message = r.msg - default: - out.Status = StatusRolledBack - out.Message = err.Error() + if errors.As(err, &r) { + out.Message = r.msg + } else { + out.Message = err.Error() + } + err := u.finish(&out) + return out, err + } + return u.swap(ctx, rec) +} + +// finish writes a final outcome and returns it as an error unless it is a +// success. +func (u *Updater) finish(out *Outcome) error { + if out.At == "" { + out.At = u.Now().UTC().Format(time.RFC3339) } - out.At = u.Now().UTC().Format(time.RFC3339) u.Logf("%s: %s", out.Status, out.Message) - if werr := u.writeOutcome(out); werr != nil { + if werr := u.writeOutcome(*out); werr != nil { u.Logf("could not write %s: %v", u.Layout.AgentLastUpdate(), werr) } - // Only now start the previous agent again: it reads the outcome when it - // starts, to tell the people at the board what happened. - if out.Status == StatusRolledBack { - if o, rerr := u.Run(ctx, "systemctl", "restart", AgentUnit); rerr != nil { - u.Logf("restarting the previous agent failed: %v %s", rerr, firstLine(o)) - } + if out.Status == StatusOK { + return nil } - return out, err + return errors.New(out.Message) } -func (u *Updater) apply(ctx context.Context, out *Outcome) error { +// prepare verifies the staged package, extracts the new binary, keeps the +// running one in agent.prev and writes the update record. Every failure is +// a refusal: nothing installed has changed. +func (u *Updater) prepare(ctx context.Context, out *Outcome) (*Txn, error) { work, err := os.MkdirTemp(u.TempRoot, "musallahboard-selfupdate-") if err != nil { - return refuse("cannot make a private working directory: %v", err) + return nil, refuse("cannot make a private working directory: %v", err) } defer os.RemoveAll(work) if err := os.Chmod(work, 0o700); err != nil { - return refuse("cannot secure the working directory: %v", err) + return nil, refuse("cannot secure the working directory: %v", err) } // Copy first, verify the copy: the daemon owns the staged file and could // swap it between a check and a use. pkgPath := filepath.Join(work, "package.mbu") - if err := copyStaged(u.Layout.StagedPackage(), pkgPath); err != nil { - return refuse("cannot read the staged package: %v", err) - } - // Consumed; a leftover would only waste space. + err = copyStaged(u.Layout.StagedPackage(), pkgPath) + // Consumed, or unusable. Either way it must go: a package left staged + // gets its `ready` marker back (settleReady), which would start this + // again at once. _ = os.Remove(u.Layout.StagedPackage()) + if err != nil { + return nil, refuse("cannot read the staged package: %v", err) + } ring, err := trust.LoadRingRootOwned(u.TrustPath) if err != nil { - return refuse("cannot read the trust store: %v", err) + return nil, refuse("cannot read the trust store: %v", err) } pkg, err := mbu.Open(pkgPath, ring, mbu.OpenOptions{}) if err != nil { - return refuse("the staged package did not verify: %v", err) + return nil, refuse("the staged package did not verify: %v", err) } defer pkg.Close() m := pkg.Manifest out.To = m.Version if m.Type != mbu.TypeAgent { - return refuse("the staged package is a %s package, not an agent", m.Type) + return nil, refuse("the staged package is a %s package, not an agent", m.Type) } if m.Agent.Arch != u.Arch { - return refuse("agent %s is built for %s; this board is %s", m.Version, m.Agent.Arch, u.Arch) + return nil, refuse("agent %s is built for %s; this board is %s", m.Version, m.Agent.Arch, u.Arch) } if mbu.CompareVersions(m.Version, u.CurrentVersion) <= 0 { - return refuse("agent %s is not newer than the installed %s", m.Version, u.CurrentVersion) + return nil, refuse("agent %s is not newer than the installed %s", m.Version, u.CurrentVersion) } st, err := u.Layout.State().Load() if err != nil { - return refuse("cannot read %s: %v", u.Layout.State().Path, err) + return nil, refuse("cannot read %s: %v", u.Layout.State().Path, err) } if st.AgentRejected(m.Version) { - return refuse("agent %s failed to start on this board before; it will not be retried", m.Version) + return nil, refuse("agent %s failed to start on this board before; it will not be retried", m.Version) } u.Logf("verified agent %s (signed by %s); installing", m.Version, pkg.SignedBy) if err := u.extract(pkg, m.Agent.Binary); err != nil { os.Remove(u.NewBinary) - return refuse("cannot write the new agent: %v", err) + return nil, refuse("cannot write the new agent: %v", err) + } + fail := func(format string, args ...any) (*Txn, error) { + os.Remove(u.NewBinary) + return nil, refuse(format, args...) } vout, err := u.Run(ctx, u.NewBinary, "version") if err != nil || !strings.Contains(vout, m.Version) { - os.Remove(u.NewBinary) - return refuse("the new agent does not run on this board (`version` said %q, error %v)", firstLine(vout), err) + return fail("the new agent does not run on this board (`version` said %q, error %v)", firstLine(vout), err) } if err := os.MkdirAll(filepath.Dir(u.Prev), 0o755); err != nil { - os.Remove(u.NewBinary) - return refuse("cannot keep the current agent for rollback: %v", err) + return fail("cannot keep the current agent for rollback: %v", err) } if err := copyFileAtomic(u.Binary, u.Prev, 0o755); err != nil { - os.Remove(u.NewBinary) - return refuse("cannot keep the current agent for rollback: %v", err) + return fail("cannot keep the current agent for rollback: %v", err) + } + cur, err1 := fileSHA256(u.Binary) + prev, err2 := fileSHA256(u.Prev) + next, err3 := fileSHA256(u.NewBinary) + if err := errors.Join(err1, err2, err3); err != nil { + return fail("cannot fingerprint the agents: %v", err) + } + if cur != prev { + return fail("the copy of the current agent kept for rollback does not match it") } + + // Whether the board is connected now decides whether the new agent must + // connect to pass. The agent answering is the one about to be replaced. + backend := false + if s, err := u.Status(ctx); err != nil { + u.Logf("could not read the running agent's status (%v); the new agent will not be required to connect to LensBridge", err) + } else { + backend = s.BackendUp() + u.Logf("connected to LensBridge before the update: %v", backend) + } + + rec := &Txn{ + Phase: PhaseSwapping, From: u.CurrentVersion, To: m.Version, + PrevSHA256: prev, NewSHA256: next, BackendBefore: backend, + StartedAt: u.Now().UTC().Format(time.RFC3339Nano), + } + if err := saveTxn(u.Txn, rec); err != nil { + removeDurable(u.Txn) + return fail("cannot record the update before installing it: %v", err) + } + return rec, nil +} + +// swap puts the new binary in place and starts its probation. +func (u *Updater) swap(ctx context.Context, rec *Txn) (Outcome, error) { + u.crashpoint("recorded") + // From here a power cut must lead to a run that finishes the update. + // The record (watched by current path units) and `ready` (watched by + // every path unit since the first release) both start one at boot. + u.markReady() + u.crashpoint("marked") if err := os.Rename(u.NewBinary, u.Binary); err != nil { os.Remove(u.NewBinary) - return refuse("cannot put the new agent in place: %v", err) + if rerr := removeDurable(u.Txn); rerr != nil { + u.Logf("could not remove %s: %v", u.Txn, rerr) + } + out := Outcome{From: rec.From, To: rec.To, Status: StatusRefused, + Message: fmt.Sprintf("cannot put the new agent in place: %v", err)} + err := u.finish(&out) + return out, err } _ = fsutil.SyncDir(filepath.Dir(u.Binary)) + u.crashpoint("swapped") + return u.startProbation(ctx, rec, false) +} + +// startProbation records the probation, restarts the agent unless resuming, +// judges it and commits or rolls back. +func (u *Updater) startProbation(ctx context.Context, rec *Txn, resumed bool) (Outcome, error) { + rec.Phase = PhaseProbation + rec.Attempts++ + if rec.Attempts > MaxProbationAttempts { + return u.rollBack(ctx, rec, fmt.Sprintf("agent %s could not be judged: its probation was cut short %d times", + rec.To, rec.Attempts-1), true) + } + if err := saveTxn(u.Txn, rec); err != nil { + u.Logf("could not update %s: %v", u.Txn, err) + } + u.crashpoint("probation") - u.Logf("restarting %s", AgentUnit) - if o, err := u.Run(ctx, "systemctl", "restart", AgentUnit); err != nil { - return u.rollback(ctx, m.Version, fmt.Sprintf("restarting the agent failed: %v %s", err, firstLine(o))) + if !resumed { + u.Logf("restarting %s", AgentUnit) + if o, err := u.Run(ctx, "systemctl", "restart", AgentUnit); err != nil { + if ctx.Err() != nil { + return Outcome{From: rec.From, To: rec.To}, ErrInterrupted + } + return u.rollBack(ctx, rec, fmt.Sprintf("restarting the agent failed: %v %s", err, firstLine(o)), true) + } + } + v, err := u.probation(ctx, rec, resumed) + if err != nil { + u.Logf("%v; the next run resumes the update", err) + return Outcome{From: rec.From, To: rec.To, Message: err.Error()}, err } - if err := u.Healthy(ctx, m.Version); err != nil { - return u.rollback(ctx, m.Version, fmt.Sprintf("agent %s did not come up healthy: %v", m.Version, err)) + if v != nil { + return u.rollBack(ctx, rec, v.reason, v.reject) } + return u.commit(ctx, rec) +} + +// commit makes a passed update final. +func (u *Updater) commit(ctx context.Context, rec *Txn) (Outcome, error) { + if rec.Phase != PhaseCommitting { + rec.Phase = PhaseCommitting + rec.OutcomeAt = u.Now().UTC().Format(time.RFC3339) + if err := saveTxn(u.Txn, rec); err != nil { + u.Logf("could not update %s: %v", u.Txn, err) + } + } + u.crashpoint("committing") + out := Outcome{From: rec.From, To: rec.To, At: rec.OutcomeAt, Status: StatusOK, + Message: fmt.Sprintf("Updated the agent from %s to %s", rec.From, rec.To)} + err := u.finish(&out) + u.crashpoint("committed") // The enrollment splash ships inside the binary; bring it up to date with // it. Best effort: the splash is only seen before enrollment. - if o, err := u.Run(ctx, u.Binary, "splash", "install"); err != nil { + if o, err := u.Run(context.WithoutCancel(ctx), u.Binary, "splash", "install"); err != nil { u.Logf("could not refresh the enrollment splash: %v %s", err, firstLine(o)) } - return nil + if rerr := removeDurable(u.Txn); rerr != nil { + u.Logf("could not remove %s: %v", u.Txn, rerr) + } + return out, err } -// rollback restores the previous agent and records the version as rejected -// so the release channel or a USB stick cannot loop on it. Apply restarts -// the previous agent once the outcome is written. -func (u *Updater) rollback(ctx context.Context, version, reason string) error { - u.Logf("%s; rolling back to %s", reason, u.CurrentVersion) - msg := reason + "; rolled back to " + u.CurrentVersion - if err := copyFileAtomic(u.Prev, u.Binary, 0o755); err != nil { - msg += fmt.Sprintf(" (restoring the previous binary FAILED: %v)", err) - } - if _, err := u.Layout.State().Update(func(s *state.State) error { - if !slices.Contains(s.AgentVersionsRejected, version) { - s.AgentVersionsRejected = append(s.AgentVersionsRejected, version) +// rollBack restores the previous agent, records the version as rejected (or, +// for a connection-only failure, a strike against it) so the release channel +// or a USB stick cannot loop on it, writes the outcome and only then +// restarts the previous agent, which reads the outcome when it starts. Every +// step can be repeated, so a run that resumes it simply does it again. +func (u *Updater) rollBack(ctx context.Context, rec *Txn, reason string, reject bool) (Outcome, error) { + // Once begun, a rollback finishes: a stop request does not cut it short. + ctx = context.WithoutCancel(ctx) + if rec.Phase != PhaseRollingBack { + rec.Phase, rec.Reason, rec.Reject = PhaseRollingBack, reason, reject + rec.OutcomeAt = u.Now().UTC().Format(time.RFC3339) + if err := saveTxn(u.Txn, rec); err != nil { + u.Logf("could not update %s: %v", u.Txn, err) } - return nil - }); err != nil { - msg += fmt.Sprintf(" (could not record %s as rejected: %v)", version, err) } - return errors.New(msg) + u.crashpoint("rolling-back") + u.Logf("%s; rolling back to %s", rec.Reason, rec.From) + + out := Outcome{From: rec.From, To: rec.To, At: rec.OutcomeAt, Status: StatusRolledBack, + Message: rec.Reason + "; rolled back to " + rec.From} + if err := u.restorePrev(ctx, rec); err != nil { + out.Status = StatusRollbackFailed + out.Message = fmt.Sprintf("%s; restoring agent %s FAILED (%v), so %s still runs", rec.Reason, rec.From, err, rec.To) + } + u.crashpoint("restored") + + reject = rec.Reject + if !reject { + n, err := addStrike(u.Strikes, rec.To, rec.StartedAt) + switch { + case err != nil: + u.Logf("could not record the failure in %s: %v", u.Strikes, err) + case n >= StrikesToReject: + reject = true + out.Message += fmt.Sprintf(" (failed this way %d times; not retried)", n) + default: + out.Message += " (it will be tried again)" + } + } + if reject { + if _, err := u.Layout.State().Update(func(s *state.State) error { + if !slices.Contains(s.AgentVersionsRejected, rec.To) { + s.AgentVersionsRejected = append(s.AgentVersionsRejected, rec.To) + } + return nil + }); err != nil { + out.Message += fmt.Sprintf(" (could not record %s as rejected: %v)", rec.To, err) + } + } + err := u.finish(&out) + u.crashpoint("rolled-back") + if rerr := removeDurable(u.Txn); rerr != nil { + u.Logf("could not remove %s: %v", u.Txn, rerr) + } + u.crashpoint("closed") + if o, rerr := u.Run(ctx, "systemctl", "restart", AgentUnit); rerr != nil { + u.Logf("restarting the agent failed: %v %s", rerr, firstLine(o)) + } + return out, err +} + +// restorePrev puts agent.prev back in place, if it is not there already, and +// only if agent.prev is still the agent that ran before the update. +func (u *Updater) restorePrev(ctx context.Context, rec *Txn) error { + var err error + for try := 0; try < 3; try++ { + if try > 0 { + _ = u.Sleep(ctx, 2*time.Second) + } + if h, herr := fileSHA256(u.Binary); herr == nil && h == rec.PrevSHA256 { + return nil + } + h, herr := fileSHA256(u.Prev) + switch { + case herr != nil: + err = herr + case h != rec.PrevSHA256: + return fmt.Errorf("%s is not the agent that ran before the update", u.Prev) + default: + err = copyFileAtomic(u.Prev, u.Binary, 0o755) + if err == nil { + if h, herr := fileSHA256(u.Binary); herr == nil && h == rec.PrevSHA256 { + return nil + } + err = errors.New("the restored binary does not match agent.prev") + } + } + } + return err +} + +// resume finishes a transaction a previous run left behind. +func (u *Updater) resume(ctx context.Context, rec *Txn) (Outcome, error) { + switch rec.Phase { + case PhaseCommitting: + return u.commit(ctx, rec) + case PhaseRollingBack: + return u.rollBack(ctx, rec, rec.Reason, rec.Reject) + } + h, err := fileSHA256(u.Binary) + switch { + case err == nil && h == rec.NewSHA256 && (rec.Phase == PhaseSwapping || rec.Phase == PhaseProbation): + // The new agent is in place and was not judged: judge it now. + return u.startProbation(ctx, rec, true) + case err == nil && h == rec.NewSHA256: + return u.rollBack(ctx, rec, fmt.Sprintf("the update record is in an unknown state (%q)", rec.Phase), true) + case err == nil && h == rec.PrevSHA256: + // Interrupted before the swap: nothing changed. + os.Remove(u.NewBinary) + if rerr := removeDurable(u.Txn); rerr != nil { + u.Logf("could not remove %s: %v", u.Txn, rerr) + } + out := Outcome{From: rec.From, To: rec.To, Status: StatusRefused, + Message: fmt.Sprintf("the update to agent %s was interrupted before the new agent was put in place; agent %s still runs", rec.To, rec.From)} + err := u.finish(&out) + return out, err + default: + // Something else was installed since (by hand, or setup.sh). It + // is not ours to judge or undo. + if err == nil { + err = errors.New("it is neither the previous nor the new agent") + } + os.Remove(u.NewBinary) + if rerr := removeDurable(u.Txn); rerr != nil { + u.Logf("could not remove %s: %v", u.Txn, rerr) + } + out := Outcome{From: rec.From, To: rec.To, Status: StatusRefused, + Message: fmt.Sprintf("the update to agent %s was interrupted, and the installed agent was changed since (%v); it is left as it is", rec.To, err)} + err := u.finish(&out) + return out, err + } +} + +// unreadableTxn handles a record that cannot be parsed (it is written +// atomically, so only a damaged disk does this). An update was under way, so +// the agent that is known to have worked, agent.prev, is put back. +func (u *Updater) unreadableTxn(ctx context.Context, lerr error) (Outcome, error) { + u.Logf("%v", lerr) + out := Outcome{Status: StatusRolledBack, Message: "an unfinished agent update could not be read back; the previous agent was put back"} + prev, perr := fileSHA256(u.Prev) + cur, cerr := fileSHA256(u.Binary) + switch { + case perr != nil: + out.Status = StatusRollbackFailed + out.Message = fmt.Sprintf("an unfinished agent update could not be read back, and there is no previous agent to return to (%v)", perr) + case cerr == nil && cur == prev: + out.Message = "an unfinished agent update could not be read back; the previous agent is in place" + default: + if err := copyFileAtomic(u.Prev, u.Binary, 0o755); err != nil { + out.Status = StatusRollbackFailed + out.Message = fmt.Sprintf("an unfinished agent update could not be read back, and putting the previous agent back failed: %v", err) + } + } + err := u.finish(&out) + if rerr := removeDurable(u.Txn); rerr != nil { + u.Logf("could not remove %s: %v", u.Txn, rerr) + } + if o, rerr := u.Run(context.WithoutCancel(ctx), "systemctl", "restart", AgentUnit); rerr != nil { + u.Logf("restarting the agent failed: %v %s", rerr, firstLine(o)) + } + return out, err +} + +// markReady (re)creates the `ready` marker, so the path unit starts the +// updater again. The marker carries no trust; it only triggers a run. +func (u *Updater) markReady() { + if err := fsutil.WriteAtomic(u.Layout.StagedReady(), []byte("resume\n"), 0o640); err != nil { + u.Logf("could not write %s: %v", u.Layout.StagedReady(), err) + } +} + +// settleReady removes `ready`, then puts it back if a package is staged (the +// daemon may have staged another while this ran). Removing before looking +// means a staging that lands in between always leaves its own marker. +func (u *Updater) settleReady() { + if err := os.Remove(u.Layout.StagedReady()); err != nil && !errors.Is(err, fs.ErrNotExist) { + u.Logf("could not remove %s: %v", u.Layout.StagedReady(), err) + } + if _, err := os.Lstat(u.Layout.StagedPackage()); err == nil { + u.markReady() + } } func (u *Updater) extract(pkg *mbu.Package, binary string) error { @@ -367,49 +719,3 @@ func firstLine(s string) string { s, _, _ = strings.Cut(strings.TrimSpace(s), "\n") return s } - -// WaitHealthy polls statusURL until it reports agentVersion == version, or -// timeout passes. The daemon needs a few seconds to start its local server; -// connection errors until then are expected. -func WaitHealthy(ctx context.Context, statusURL, version string, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - client := &http.Client{Timeout: 5 * time.Second} - last := "no answer" - for { - if v, err := probe(ctx, client, statusURL); err != nil { - last = err.Error() - } else if v == version { - return nil - } else { - last = "it reports version " + v - } - select { - case <-ctx.Done(): - return fmt.Errorf("after %s: %s", timeout, last) - case <-time.After(2 * time.Second): - } - } -} - -func probe(ctx context.Context, client *http.Client, statusURL string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, statusURL, nil) - if err != nil { - return "", err - } - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("status returned %d", resp.StatusCode) - } - var st struct { - AgentVersion string `json:"agentVersion"` - } - if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&st); err != nil { - return "", err - } - return st.AgentVersion, nil -} diff --git a/internal/selfupdate/selfupdate_test.go b/internal/selfupdate/selfupdate_test.go index 198e6dc..51f9f05 100644 --- a/internal/selfupdate/selfupdate_test.go +++ b/internal/selfupdate/selfupdate_test.go @@ -11,8 +11,8 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" - "sync/atomic" "testing" "time" @@ -29,17 +29,71 @@ var ( const oldBinary = "old agent binary" +// fakeAgent is musallahboard-agent.service as systemd and the loopback +// status endpoint show it. It runs whatever binary is installed when it is +// (re)started. +type fakeAgent struct { + version string + pid int + restarts int + active bool + startedAt time.Time + // crashAfter makes the new version crash (and systemd restart it) once + // it has run this long; 0 never. + crashAfter time.Duration + // backendOld and backendNew say whether each version connects to + // LensBridge. + backendOld, backendNew bool + // downNew makes the new version never answer on the status endpoint. + downNew bool +} + type harness struct { + t *testing.T u *Updater cmds []string versionO string - healthy error + now time.Time + agent fakeAgent + // restartsSeen records, for each `systemctl restart`, whether + // last-update.json existed at that moment. + outcomeAtRestart []bool + // tick runs after every fake sleep. + tick func(h *harness) +} + +func versionOf(binary string) string { + if binary == oldBinary { + return "0.3.0" + } + return strings.TrimPrefix(binary, "new agent ") +} + +func (h *harness) start() { + h.agent.pid += 100 + h.agent.version = versionOf(read(h.t, h.u.Binary)) + h.agent.active = true + h.agent.startedAt = h.now +} + +func (h *harness) isNew() bool { return h.agent.version != "0.3.0" } + +// advance moves the fake clock, crashing the agent when it is due to. +func (h *harness) advance(d time.Duration) { + h.now = h.now.Add(d) + if h.isNew() && h.agent.crashAfter > 0 && h.now.Sub(h.agent.startedAt) >= h.agent.crashAfter { + h.agent.restarts++ + h.start() + } + if h.tick != nil { + h.tick(h) + } } func newHarness(t *testing.T, withContentKey bool) *harness { t.Helper() dir := t.TempDir() - h := &harness{} + h := &harness{t: t, now: time.Date(2026, 9, 24, 23, 0, 0, 0, time.UTC)} l := store.Layout{Root: filepath.Join(dir, "var")} os.MkdirAll(l.AgentStagedDir(), 0o750) bin := filepath.Join(dir, "usr", "bin", "musallahboard-agent") @@ -65,20 +119,60 @@ func newHarness(t *testing.T, withContentKey bool) *harness { TempRoot: dir, Arch: "arm64", CurrentVersion: "0.3.0", - Run: func(ctx context.Context, name string, args ...string) (string, error) { - h.cmds = append(h.cmds, strings.TrimSpace(filepath.Base(name)+" "+strings.Join(args, " "))) - if len(args) == 1 && args[0] == "version" { - return h.versionO, nil - } - return "", nil - }, - Healthy: func(ctx context.Context, v string) error { return h.healthy }, - Now: func() time.Time { return time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) }, - Logf: t.Logf, + Logf: t.Logf, } + h.wire(h.u) + h.u.defaults() + h.start() return h } +// wire connects an Updater to the harness's fakes. +func (h *harness) wire(u *Updater) { + u.Run = func(ctx context.Context, name string, args ...string) (string, error) { + cmd := strings.TrimSpace(filepath.Base(name) + " " + strings.Join(args, " ")) + h.cmds = append(h.cmds, cmd) + switch { + case len(args) == 1 && args[0] == "version": + return h.versionO, nil + case cmd == "systemctl restart "+AgentUnit: + _, err := os.Stat(u.Layout.AgentLastUpdate()) + h.outcomeAtRestart = append(h.outcomeAtRestart, err == nil) + h.start() + case strings.HasPrefix(cmd, "systemctl show "+AgentUnit): + state := "active" + if !h.agent.active { + state = "inactive" + } + return fmt.Sprintf("MainPID=%d\nNRestarts=%d\nActiveState=%s", h.agent.pid, h.agent.restarts, state), nil + } + return "", nil + } + u.Status = func(ctx context.Context) (AgentStatus, error) { + if h.isNew() && h.agent.downNew { + return AgentStatus{}, errors.New("connection refused") + } + var st AgentStatus + st.AgentVersion = h.agent.version + up := h.agent.backendOld + if h.isNew() { + up = h.agent.backendNew + } + st.Backend = &struct { + Connected bool `json:"connected"` + }{up} + return st, nil + } + u.Now = func() time.Time { return h.now } + u.Sleep = func(ctx context.Context, d time.Duration) error { + if err := ctx.Err(); err != nil { + return err + } + h.advance(d) + return nil + } +} + // stage writes a signed agent package and the ready marker. func (h *harness) stage(t *testing.T, version, arch string, key ed25519.PrivateKey) { t.Helper() @@ -101,17 +195,52 @@ func read(t *testing.T, p string) string { return string(b) } -func lastUpdate(t *testing.T, h *harness) Outcome { +func exists(p string) bool { _, err := os.Lstat(p); return err == nil } + +func lastUpdate(t *testing.T, u *Updater) Outcome { var o Outcome - if err := json.Unmarshal([]byte(read(t, h.u.Layout.AgentLastUpdate())), &o); err != nil { + if err := json.Unmarshal([]byte(read(t, u.Layout.AgentLastUpdate())), &o); err != nil { t.Fatal(err) } return o } +func rejected(t *testing.T, u *Updater, v string) bool { + st, err := u.Layout.State().Load() + if err != nil { + t.Fatal(err) + } + return st.AgentRejected(v) +} + +func (h *harness) count(cmd string) int { + n := 0 + for _, c := range h.cmds { + if c == cmd { + n++ + } + } + return n +} + +// settled checks that no transaction or trigger is left behind. +func (h *harness) settled(t *testing.T) { + t.Helper() + for _, p := range []string{h.u.Txn, h.u.Layout.StagedReady(), h.u.Layout.StagedPackage(), h.u.NewBinary} { + if exists(p) { + t.Errorf("%s left behind", p) + } + } + if m, _ := filepath.Glob(filepath.Join(h.u.TempRoot, "musallahboard-selfupdate-*")); len(m) != 0 { + t.Errorf("work dirs left: %v", m) + } +} + func TestApplyOK(t *testing.T) { h := newHarness(t, false) + h.agent.backendOld, h.agent.backendNew = true, true h.stage(t, "0.4.0", "arm64", releaseKey) + start := h.now out, err := h.u.Apply(context.Background()) if err != nil { t.Fatalf("Apply: %v", err) @@ -128,64 +257,159 @@ func TestApplyOK(t *testing.T) { if got := read(t, h.u.Prev); got != oldBinary { t.Errorf("prev = %q", got) } - for _, p := range []string{h.u.Layout.StagedReady(), h.u.Layout.StagedPackage(), h.u.NewBinary} { - if _, err := os.Stat(p); err == nil { - t.Errorf("%s left behind", p) - } + h.settled(t) + if took := h.now.Sub(start); took < ProbationPeriod { + t.Errorf("committed after %s, before the probation was over", took) } - want := []string{".musallahboard-agent.new version", "systemctl restart musallahboard-agent.service", - "musallahboard-agent splash install"} - if fmt.Sprint(h.cmds) != fmt.Sprint(want) { - t.Errorf("commands = %q, want %q", h.cmds, want) + if h.count("systemctl restart "+AgentUnit) != 1 || h.count("musallahboard-agent splash install") != 1 { + t.Errorf("commands = %q", h.cmds) } - if o := lastUpdate(t, h); o.Status != StatusOK || o.At == "" { - t.Errorf("last-update = %+v", o) + if h.cmds[0] != ".musallahboard-agent.new version" { + t.Errorf("first command = %q", h.cmds[0]) } - // No temp directories left in TempRoot. - if m, _ := filepath.Glob(filepath.Join(h.u.TempRoot, "musallahboard-selfupdate-*")); len(m) != 0 { - t.Errorf("work dirs left: %v", m) + if o := lastUpdate(t, h.u); o.Status != StatusOK || o.At == "" { + t.Errorf("last-update = %+v", o) } } -func TestApplyRollsBack(t *testing.T) { +func TestApplyNoComeUpRollsBack(t *testing.T) { h := newHarness(t, true) + h.agent.downNew = true h.stage(t, "0.4.0", "arm64", releaseKey) - h.healthy = errors.New("it reports version 0.3.0") + out, err := h.u.Apply(context.Background()) + if err == nil || out.Status != StatusRolledBack || !strings.Contains(out.Message, "did not come up") { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if got := read(t, h.u.Binary); got != oldBinary { + t.Errorf("binary after rollback = %q", got) + } + if !rejected(t, h.u, "0.4.0") { + t.Error("0.4.0 not recorded as rejected") + } // The previous agent reads the outcome when it starts, so it must be // written before the second restart. - run := h.u.Run - outcomeAtRestart := []bool{} - h.u.Run = func(ctx context.Context, name string, args ...string) (string, error) { - if name == "systemctl" { - _, err := os.Stat(h.u.Layout.AgentLastUpdate()) - outcomeAtRestart = append(outcomeAtRestart, err == nil) + if fmt.Sprint(h.outcomeAtRestart) != "[false true]" { + t.Errorf("outcome present at restarts: %v (%q)", h.outcomeAtRestart, h.cmds) + } + if h.agent.version != "0.3.0" { + t.Errorf("agent running %s after rollback", h.agent.version) + } + if o := lastUpdate(t, h.u); o.Status != StatusRolledBack || !strings.Contains(o.Message, "rolled back") { + t.Errorf("last-update = %+v", o) + } + h.settled(t) +} + +func TestApplyCrashLoopRollsBack(t *testing.T) { + h := newHarness(t, true) + // Up long enough to pass the old two-minute check, then crashes. + h.agent.crashAfter = 150 * time.Second + h.stage(t, "0.4.0", "arm64", releaseKey) + out, err := h.u.Apply(context.Background()) + if err == nil || out.Status != StatusRolledBack || !strings.Contains(out.Message, "was restarted") { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if read(t, h.u.Binary) != oldBinary || !rejected(t, h.u, "0.4.0") { + t.Error("not rolled back and rejected") + } + h.settled(t) +} + +func TestApplyStoppedAgentRollsBack(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + h.tick = func(h *harness) { + if h.isNew() && h.now.Sub(h.agent.startedAt) > time.Minute { + h.agent.active = false } - return run(ctx, name, args...) } out, err := h.u.Apply(context.Background()) if err == nil || out.Status != StatusRolledBack { t.Fatalf("outcome = %+v, err = %v", out, err) } - if got := read(t, h.u.Binary); got != oldBinary { - t.Errorf("binary after rollback = %q", got) +} + +func TestApplyBackendLostRollsBackWithStrike(t *testing.T) { + h := newHarness(t, true) + h.agent.backendOld, h.agent.backendNew = true, false + h.stage(t, "0.4.0", "arm64", releaseKey) + out, err := h.u.Apply(context.Background()) + if err == nil || out.Status != StatusRolledBack || !strings.Contains(out.Message, "did not connect to LensBridge") || + !strings.Contains(out.Message, "tried again") { + t.Fatalf("outcome = %+v, err = %v", out, err) } - st, _ := h.u.Layout.State().Load() - if !st.AgentRejected("0.4.0") { - t.Errorf("0.4.0 not recorded as rejected: %+v", st) + if read(t, h.u.Binary) != oldBinary { + t.Error("not rolled back") } - restarts := 0 - for _, c := range h.cmds { - if c == "systemctl restart musallahboard-agent.service" { - restarts++ + // Once might be the backend's fault: not rejected yet. + if rejected(t, h.u, "0.4.0") { + t.Error("rejected after one connection failure") + } + h.settled(t) + + // The second time it is. + h.now = h.now.Add(24 * time.Hour) + h.stage(t, "0.4.0", "arm64", releaseKey) + out, err = h.u.Apply(context.Background()) + if err == nil || out.Status != StatusRolledBack || !strings.Contains(out.Message, "not retried") { + t.Fatalf("second outcome = %+v, err = %v", out, err) + } + if !rejected(t, h.u, "0.4.0") { + t.Error("not rejected after two connection failures") + } +} + +func TestApplyOfflineBoardIsNotRolledBack(t *testing.T) { + h := newHarness(t, true) + h.agent.backendOld, h.agent.backendNew = false, false + h.stage(t, "0.4.0", "arm64", releaseKey) + out, err := h.u.Apply(context.Background()) + if err != nil || out.Status != StatusOK { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if read(t, h.u.Binary) != "new agent 0.4.0" { + t.Error("offline board rolled back") + } +} + +func TestApplyNoStatusBeforeIsNotRequiredOnline(t *testing.T) { + h := newHarness(t, true) + h.agent.backendNew = false + h.stage(t, "0.4.0", "arm64", releaseKey) + status := h.u.Status + calls := 0 + h.u.Status = func(ctx context.Context) (AgentStatus, error) { + calls++ + if calls == 1 { + return AgentStatus{}, errors.New("connection refused") } + return status(ctx) } - if restarts != 2 { - t.Errorf("restarts = %d, want 2 (%q)", restarts, h.cmds) + if out, err := h.u.Apply(context.Background()); err != nil || out.Status != StatusOK { + t.Fatalf("outcome = %+v, err = %v", out, err) } - if len(outcomeAtRestart) != 2 || !outcomeAtRestart[1] { - t.Errorf("outcome written before the previous agent restarted: %v", outcomeAtRestart) +} + +func TestApplyRollbackFails(t *testing.T) { + h := newHarness(t, true) + h.agent.downNew = true + h.stage(t, "0.4.0", "arm64", releaseKey) + h.u.crash = func(step string) bool { + if step == "probation" { + os.WriteFile(h.u.Prev, []byte("damaged"), 0o755) + } + return false } - if o := lastUpdate(t, h); o.Status != StatusRolledBack || !strings.Contains(o.Message, "rolled back") { + out, err := h.u.Apply(context.Background()) + if err == nil || out.Status != StatusRollbackFailed || !strings.Contains(out.Message, "FAILED") { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if read(t, h.u.Binary) != "new agent 0.4.0" { + t.Error("binary replaced by a damaged agent.prev") + } + // Finished, not left for a loop of retries. + h.settled(t) + if o := lastUpdate(t, h.u); o.Status != StatusRollbackFailed { t.Errorf("last-update = %+v", o) } } @@ -220,6 +444,9 @@ func TestApplyRefuses(t *testing.T) { h := newHarness(t, true) tc.setup(t, h) out, err := h.u.Apply(context.Background()) + if out.At == "" { + t.Error("outcome returned without its time") + } if err == nil || out.Status != StatusRefused || !strings.Contains(out.Message, tc.wantMsg) { t.Fatalf("outcome = %+v, err = %v; want refused with %q", out, err, tc.wantMsg) } @@ -228,13 +455,13 @@ func TestApplyRefuses(t *testing.T) { } for _, c := range h.cmds { if strings.HasPrefix(c, "systemctl") { - t.Errorf("service restarted on refusal: %q", h.cmds) + t.Errorf("service touched on refusal: %q", h.cmds) } } - if _, err := os.Stat(h.u.NewBinary); err == nil { - t.Error("new binary left behind") + if exists(h.u.NewBinary) || exists(h.u.Txn) || exists(h.u.Layout.StagedReady()) { + t.Error("new binary, record or ready left behind") } - if o := lastUpdate(t, h); o.Status != StatusRefused { + if o := lastUpdate(t, h.u); o.Status != StatusRefused { t.Errorf("last-update = %+v", o) } }) @@ -246,26 +473,276 @@ func TestApplyNothingStaged(t *testing.T) { if _, err := h.u.Apply(context.Background()); !errors.Is(err, ErrNothingStaged) { t.Fatalf("err = %v", err) } - if _, err := os.Stat(h.u.Layout.AgentLastUpdate()); err == nil { + if exists(h.u.Layout.AgentLastUpdate()) { t.Fatal("last-update written with nothing staged") } } -func TestWaitHealthy(t *testing.T) { - var calls atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - v := "0.3.0" - if calls.Add(1) >= 2 { - v = "0.4.0" +// A package staged by the new agent while its predecessor was on probation +// keeps its trigger, so the path unit runs the updater again for it. +func TestApplyKeepsReadyForANewStaging(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + staged := false + h.tick = func(h *harness) { + if !staged && h.isNew() { + staged = true + os.WriteFile(h.u.Layout.StagedPackage(), []byte("next"), 0o640) + os.WriteFile(h.u.Layout.StagedReady(), []byte("now\n"), 0o640) } - fmt.Fprintf(w, `{"localApi":2,"agentVersion":%q}`, v) - })) - defer srv.Close() - if err := WaitHealthy(context.Background(), srv.URL, "0.4.0", 10*time.Second); err != nil { + } + if _, err := h.u.Apply(context.Background()); err != nil { t.Fatal(err) } - err := WaitHealthy(context.Background(), srv.URL, "9.9.9", 100*time.Millisecond) - if err == nil || !strings.Contains(err.Error(), "reports version 0.4.0") { + if !exists(h.u.Layout.StagedReady()) { + t.Error("ready removed although a package is staged") + } +} + +// cutAt runs Apply on h until the transaction reaches step, as a power cut +// would leave it: nothing after that point runs, and nothing is cleaned up. +func cutAt(t *testing.T, h *harness, step string) { + t.Helper() + h.u.crash = func(s string) bool { return s == step } + defer func() { + h.u.crash, h.u.poweredOff = nil, false + r := recover() + if pc, ok := r.(PowerCut); !ok || pc.Step != step { + t.Fatalf("Apply did not reach %q (recovered %v)", step, r) + } + }() + h.u.Apply(context.Background()) +} + +// reboot is the board starting again: systemd starts whatever agent is +// installed, and the path unit starts the updater, which is that same binary +// (so its CurrentVersion is the installed one), if the record or the `ready` +// marker is there. +func (h *harness) reboot(t *testing.T) (*Updater, bool) { + t.Helper() + h.now = h.now.Add(time.Minute) + h.agent.restarts = 0 + h.start() + u := &Updater{Layout: h.u.Layout, TrustPath: h.u.TrustPath, Binary: h.u.Binary, NewBinary: h.u.NewBinary, + Prev: h.u.Prev, TempRoot: h.u.TempRoot, Arch: h.u.Arch, CurrentVersion: h.agent.version, Logf: t.Logf} + h.wire(u) + u.defaults() + triggered := exists(u.Txn) || exists(u.Layout.StagedReady()) + return u, triggered +} + +func TestPowerCutAtEveryStep(t *testing.T) { + type want struct { + binary, status string + rejected bool + } + installed := want{"new agent 0.4.0", StatusOK, false} + untouched := want{oldBinary, StatusRefused, false} + rolledBack := want{oldBinary, StatusRolledBack, true} + cases := []struct { + step string + bad bool // the new agent never comes up + want want + }{ + {"recorded", false, untouched}, + {"marked", false, untouched}, + {"swapped", false, installed}, + {"probation", false, installed}, + {"committing", false, installed}, + {"committed", false, installed}, + {"swapped", true, rolledBack}, + {"probation", true, rolledBack}, + {"rolling-back", true, rolledBack}, + {"restored", true, rolledBack}, + {"rolled-back", true, rolledBack}, + {"closed", true, rolledBack}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("%s/bad=%v", tc.step, tc.bad), func(t *testing.T) { + h := newHarness(t, true) + h.agent.downNew = tc.bad + h.stage(t, "0.4.0", "arm64", releaseKey) + cutAt(t, h, tc.step) + + u, triggered := h.reboot(t) + if tc.step != "closed" && !triggered { + t.Fatal("nothing would start the updater at boot") + } + if triggered { + out, err := u.Apply(context.Background()) + t.Logf("after reboot: %+v, %v", out, err) + if (errors.Is(err, ErrNothingStaged) && tc.step != "closed") || errors.Is(err, ErrInterrupted) { + t.Fatalf("not resolved: %v", err) + } + } + if got := read(t, u.Binary); got != tc.want.binary { + t.Errorf("binary = %q, want %q", got, tc.want.binary) + } + if o := lastUpdate(t, u); o.Status != tc.want.status || o.To != "0.4.0" || o.From != "0.3.0" { + t.Errorf("last-update = %+v, want status %s", o, tc.want.status) + } + if got := rejected(t, u, "0.4.0"); got != tc.want.rejected { + t.Errorf("rejected = %v, want %v", got, tc.want.rejected) + } + h.u = u + h.settled(t) + if h.agent.version != versionOf(tc.want.binary) { + t.Errorf("agent runs %s", h.agent.version) + } + // Resolved for good: another run finds nothing to do. + if _, err := u.Apply(context.Background()); !errors.Is(err, ErrNothingStaged) { + t.Errorf("second run: %v", err) + } + }) + } +} + +func TestOutcomeOfAResumedCommitIsTheSame(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + cutAt(t, h, "committed") + first := lastUpdate(t, h.u) + u, _ := h.reboot(t) + if _, err := u.Apply(context.Background()); err != nil { + t.Fatal(err) + } + // agentupdate shows an outcome once, by its timestamp. + if again := lastUpdate(t, u); again != first { + t.Errorf("outcome rewritten: %+v, was %+v", again, first) + } +} + +func TestProbationCutShortRepeatedlyRollsBack(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + cutAt(t, h, "swapped") + for i := 0; i < MaxProbationAttempts; i++ { + u, _ := h.reboot(t) + h.u = u + cutAt(t, h, "probation") + } + u, _ := h.reboot(t) + out, err := u.Apply(context.Background()) + if err == nil || out.Status != StatusRolledBack || !strings.Contains(out.Message, "cut short") { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if read(t, u.Binary) != oldBinary || !rejected(t, u, "0.4.0") { + t.Error("not rolled back and rejected") + } +} + +// SIGTERM (shutdown, or the unit's start timeout) mid-probation leaves the +// update to the next run, which judges it again. +func TestApplyInterruptedIsResumed(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + ctx, cancel := context.WithCancel(context.Background()) + h.tick = func(h *harness) { + if h.isNew() && h.now.Sub(h.agent.startedAt) > time.Minute { + cancel() + } + } + _, err := h.u.Apply(ctx) + if !errors.Is(err, ErrInterrupted) { t.Fatalf("err = %v", err) } + if !exists(h.u.Txn) || !exists(h.u.Layout.StagedReady()) { + t.Fatal("record or trigger missing after an interruption") + } + if exists(h.u.Layout.AgentLastUpdate()) { + t.Error("an outcome was written for an unjudged update") + } + h.tick = nil + // The unit's timeout, not a reboot: the agent kept running. + u := *h.u + u.CurrentVersion = "0.4.0" + out, err := u.Apply(context.Background()) + if err != nil || out.Status != StatusOK { + t.Fatalf("resumed: %+v, %v", out, err) + } + if h.count("systemctl restart "+AgentUnit) != 1 { + t.Errorf("a resumed probation restarted the agent: %q", h.cmds) + } + h.settled(t) +} + +func TestUnreadableRecordRestoresPrevious(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + cutAt(t, h, "probation") + os.WriteFile(h.u.Txn, []byte("{garbage"), 0o644) + u, _ := h.reboot(t) + out, err := u.Apply(context.Background()) + if err == nil || out.Status != StatusRolledBack { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if read(t, u.Binary) != oldBinary || exists(u.Txn) { + t.Error("previous agent not restored, or record left") + } +} + +func TestBinaryReplacedByHandIsLeftAlone(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + cutAt(t, h, "probation") + os.WriteFile(h.u.Binary, []byte("new agent 0.9.0"), 0o755) + u, _ := h.reboot(t) + out, err := u.Apply(context.Background()) + if err == nil || out.Status != StatusRefused || !strings.Contains(out.Message, "left as it is") { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if read(t, u.Binary) != "new agent 0.9.0" || exists(u.Txn) { + t.Error("hand-installed agent touched, or record left") + } +} + +func TestStrikesAreIdempotent(t *testing.T) { + p := filepath.Join(t.TempDir(), "s.json") + for _, id := range []string{"a", "a", "b"} { + addStrike(p, "1.0.0", id) + } + if n, _ := addStrike(p, "1.0.0", "b"); n != 2 { + t.Errorf("strikes = %d, want 2", n) + } + if s := loadStrikes(p); !slices.Equal(s["1.0.0"], []string{"a", "b"}) { + t.Errorf("strikes = %v", s) + } +} + +func TestServiceState(t *testing.T) { + u := &Updater{Run: func(ctx context.Context, name string, args ...string) (string, error) { + return "ActiveState=active\nMainPID=812\nNRestarts=2\n", nil + }} + s, err := u.serviceState(context.Background()) + if err != nil || s != (ServiceState{"active", 812, 2}) { + t.Fatalf("%+v %v", s, err) + } + u.Run = func(ctx context.Context, name string, args ...string) (string, error) { return "MainPID=1", nil } + if _, err := u.serviceState(context.Background()); err == nil { + t.Fatal("partial answer accepted") + } +} + +func TestFetchStatus(t *testing.T) { + body := `{"localApi":2,"agentVersion":"0.4.0","backend":{"connected":false},"sync":{"enabled":true,"lastSuccessAt":"2026-09-24T23:01:00Z","lastError":null}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + st, err := FetchStatus(context.Background(), srv.URL) + if err != nil || st.AgentVersion != "0.4.0" || !st.BackendUp() { + t.Fatalf("%+v %v", st, err) + } + for _, b := range []string{ + `{"agentVersion":"0.4.0"}`, + `{"agentVersion":"0.4.0","backend":{"connected":false},"sync":{"lastSuccessAt":"x","lastError":"timeout"}}`, + `{"agentVersion":"0.4.0","sync":{"enabled":false,"lastError":"device key unreadable"}}`, + } { + body = b + if st, err := FetchStatus(context.Background(), srv.URL); err != nil || st.BackendUp() { + t.Errorf("%s: up=%v err=%v", b, st.BackendUp(), err) + } + } + body = `{"agentVersion":"0.4.0","backend":{"connected":true}}` + if st, _ := FetchStatus(context.Background(), srv.URL); !st.BackendUp() { + t.Error("connected backend not up") + } } diff --git a/internal/selfupdate/txn.go b/internal/selfupdate/txn.go new file mode 100644 index 0000000..d39c542 --- /dev/null +++ b/internal/selfupdate/txn.go @@ -0,0 +1,160 @@ +package selfupdate + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + + "github.com/LensBridge/agent/internal/fsutil" +) + +// The update transaction (docs/architecture.md, section 12). +// +// Before the updater touches /usr/bin/musallahboard-agent it writes a record +// of what it is about to do to a root-owned file beside agent.prev. The +// record is the transaction's journal: it is rewritten (temp file, fsync, +// rename, fsync of the directory) at every change of phase, and removed only +// once the outcome has been written. Whoever finds a record (the updater that +// wrote it, or the next one to start, after a crash, a timeout or a power +// cut) finishes the transaction from it. Which way it goes is decided by the +// phase and by what is actually installed, compared by SHA-256 with the two +// binaries the record names, so the answer does not depend on how far the +// interrupted run got. +// +// The record lives in /usr/lib/musallahboard, not under /var/lib/musallahboard: +// the daemon owns the latter and must not be able to forge, edit or delete +// the journal of a root operation. + +// Phases of the transaction, in order. +const ( + // PhaseSwapping: agent.prev holds the running agent and the new binary + // is extracted; it may or may not have been renamed into place yet. + PhaseSwapping = "swapping" + // PhaseProbation: the new binary is in place and is being judged. + PhaseProbation = "probation" + // PhaseCommitting: the new agent passed; the outcome is being written. + PhaseCommitting = "committing" + // PhaseRollingBack: the new agent failed; agent.prev is being restored. + PhaseRollingBack = "rolling-back" +) + +// txnFormat is the record's format. A newer agent may resume a record an +// older one wrote (after a power cut, the binary that starts is the new one), +// so fields are only ever added. +const txnFormat = 1 + +// Txn is the on-disk update record. +type Txn struct { + Format int `json:"format"` + Phase string `json:"phase"` + From string `json:"from"` + To string `json:"to"` + // PrevSHA256 is the agent that ran before the update, as kept in + // agent.prev; NewSHA256 is the binary being installed. + PrevSHA256 string `json:"prevSha256"` + NewSHA256 string `json:"newSha256"` + // BackendBefore is whether the board was connected to LensBridge (an + // authenticated WebSocket, or a content sync that worked) just before + // the update. Only then must the new agent connect to pass. + BackendBefore bool `json:"backendBefore"` + // Attempts counts the probations started for this update. A probation + // cut short (power cut, reboot, the updater being stopped) is started + // again from scratch, at most MaxProbationAttempts times. + Attempts int `json:"attempts"` + StartedAt string `json:"startedAt"` + // Reason and Reject are set when rolling back: why, and whether the + // version is to be recorded as rejected. + Reason string `json:"reason,omitempty"` + Reject bool `json:"reject,omitempty"` + // OutcomeAt is the outcome's timestamp, fixed once committing or + // rolling back starts so that a resumed run writes the same outcome. + OutcomeAt string `json:"outcomeAt,omitempty"` +} + +func loadTxn(path string) (*Txn, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var t Txn + if err := json.Unmarshal(raw, &t); err != nil { + return nil, fmt.Errorf("%s is unreadable: %w", path, err) + } + return &t, nil +} + +func saveTxn(path string, t *Txn) error { + t.Format = txnFormat + raw, err := json.MarshalIndent(t, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return fsutil.WriteAtomic(path, append(raw, '\n'), 0o644) +} + +// removeDurable removes path and makes the removal durable. +func removeDurable(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return fsutil.SyncDir(filepath.Dir(path)) +} + +// fileSHA256 is the hex SHA-256 of the file at path. +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// strikes records, per agent version, the updates that were rolled back only +// because the new agent did not reconnect to LensBridge. One such failure may +// be the backend's or the network's fault, so it does not reject the version; +// a second one does (StrikesToReject). Keyed by the transaction's StartedAt so +// that resuming an interrupted rollback does not count it twice. +type strikes map[string][]string + +// StrikesToReject is how many connection-only failures reject a version. +const StrikesToReject = 2 + +func loadStrikes(path string) strikes { + s := strikes{} + if raw, err := os.ReadFile(path); err == nil { + _ = json.Unmarshal(raw, &s) + } + return s +} + +// addStrike records a connection-only failure of version and returns how +// many there have been. +func addStrike(path, version, id string) (int, error) { + s := loadStrikes(path) + if !slices.Contains(s[version], id) { + s[version] = append(s[version], id) + } + raw, err := json.MarshalIndent(s, "", " ") + if err != nil { + return len(s[version]), err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return len(s[version]), err + } + return len(s[version]), fsutil.WriteAtomic(path, append(raw, '\n'), 0o644) +} From 5c44ec342dc7668bc73f1f393caf3de6bb2804e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:16:37 +0000 Subject: [PATCH 13/19] Updater units: start on an unfinished update, time for the probation - The path unit also watches the update journal, so an update a power cut or reboot interrupted is finished at boot. - TimeoutStartSec 15 minutes: room for the probation and a rollback. - The update unit no longer gets write access to /etc/musallahboard: it only reads trust.json (the comment about `trust fetch` was stale). It gets /usr/share/musallahboard instead, which `splash install` after an update writes and which ProtectSystem=strict made read-only, and only loopback network access. - The agent unit's comments say what the watchdog and the probation do. setup.sh's inline copies match. --- packaging/musallahboard-agent-update.path | 10 ++++- packaging/musallahboard-agent-update.service | 42 ++++++++++++-------- packaging/musallahboard-agent.service | 17 +++++--- setup.sh | 12 ++++-- 4 files changed, 53 insertions(+), 28 deletions(-) diff --git a/packaging/musallahboard-agent-update.path b/packaging/musallahboard-agent-update.path index ece189a..c91506f 100644 --- a/packaging/musallahboard-agent-update.path +++ b/packaging/musallahboard-agent-update.path @@ -4,13 +4,19 @@ Documentation=https://github.com/LensBridge/agent/blob/main/docs/architecture.md # The daemon (unprivileged) verifies an agent package, writes it to # agent/staged/package.mbu and then creates agent/staged/ready. This starts the # root updater, which removes `ready` first thing, re-verifies the package from -# a private copy against root-trusted keys only, swaps the binary and -# health-checks the new agent (docs/architecture.md section 12). +# a private copy against root-trusted keys only, swaps the binary and keeps +# the new agent on probation (docs/architecture.md section 12). [Path] # PathExists, not PathModified: a `ready` left behind by a power cut between # staging and the updater starting still triggers the update on next boot. PathExists=/var/lib/musallahboard/agent/staged/ready +# The update journal, root-owned: an update a power cut, a reboot or a stopped +# updater left unfinished is finished at boot (or at once). The updater +# removes it whenever it ends other than by being stopped, so this cannot +# loop. It also puts `ready` back while an update is open, for boards whose +# path unit predates this line. +PathExists=/usr/lib/musallahboard/update-in-progress.json Unit=musallahboard-agent-update.service [Install] diff --git a/packaging/musallahboard-agent-update.service b/packaging/musallahboard-agent-update.service index ffda7ac..6b6d78e 100644 --- a/packaging/musallahboard-agent-update.service +++ b/packaging/musallahboard-agent-update.service @@ -1,31 +1,41 @@ [Unit] Description=Apply a staged MusallahBoard agent update Documentation=https://github.com/LensBridge/agent/blob/main/docs/architecture.md -# Started only by musallahboard-agent-update.path. Runs as root: it replaces -# /usr/bin/musallahboard-agent, keeps the previous binary in +# Started only by musallahboard-agent-update.path: when the daemon has staged +# a package, or when an update is unfinished (its journal, +# /usr/lib/musallahboard/update-in-progress.json, exists: at boot after a +# power cut, or after this unit was stopped mid-way). Runs as root: it +# replaces /usr/bin/musallahboard-agent, keeps the previous binary in # /usr/lib/musallahboard/agent.prev, restarts musallahboard-agent.service and -# rolls back if the new agent does not report its version on -# 127.0.0.1:8080 within 120 s. Nothing the daemon can write decides trust -# here: the updater re-verifies against /etc/musallahboard/trust.json -# (root-owned) and the keys compiled into this binary. +# keeps the new agent on probation: it must report its version on +# 127.0.0.1:8080 within 2 minutes, then run 4 minutes without a restart and, +# if the board was connected to LensBridge before, connect again. Otherwise it +# rolls back. Nothing the daemon can write decides trust here: the updater +# re-verifies against /etc/musallahboard/trust.json (root-owned, only read) +# and the keys compiled into this binary. After=musallahboard-agent.service [Service] Type=oneshot ExecStart=/usr/bin/musallahboard-agent selfupdate apply -# Verify + swap + restart + a 120 s health check, with room to spare. Killing -# it mid-way is safe (every step is a rename), but it would leave the version -# unjudged, so the limit is generous. -TimeoutStartSec=5min +# Verify + swap + restart + up to 2 minutes to come up + 4 minutes of +# probation + a rollback, with room to spare. Stopping it mid-way is safe: +# the journal stays, the path unit starts it again, and it judges the new +# agent from scratch (at most 3 times, then it rolls back). +TimeoutStartSec=15min # Hardening. It needs to write /usr/bin (the binary), /usr/lib/musallahboard -# (agent.prev), /var/lib/musallahboard (state.json, agent/last-update.json) -# and /etc/musallahboard (trust.json, when it runs `trust fetch` because no -# content key is pinned yet). It needs the network for that fetch and for -# the loopback health check, and it talks to PID 1 over its private socket -# to restart the agent, which a read-only /run does not prevent. +# (agent.prev, the journal, update-strikes.json), /var/lib/musallahboard +# (state.json, agent/last-update.json, agent/staged/ready) and +# /usr/share/musallahboard (the kiosk pages `splash install` refreshes after +# an update). It only reads /etc/musallahboard. Its only network use is the +# loopback status check, and it talks to PID 1 over its private socket to +# restart the agent and read its state, which a read-only /run does not +# prevent. ProtectSystem=strict -ReadWritePaths=/usr/bin -/usr/lib/musallahboard /var/lib/musallahboard /etc/musallahboard +ReadWritePaths=/usr/bin -/usr/lib/musallahboard /var/lib/musallahboard -/usr/share/musallahboard +IPAddressDeny=any +IPAddressAllow=localhost ProtectHome=yes PrivateTmp=yes NoNewPrivileges=yes diff --git a/packaging/musallahboard-agent.service b/packaging/musallahboard-agent.service index a1384d2..0defec3 100644 --- a/packaging/musallahboard-agent.service +++ b/packaging/musallahboard-agent.service @@ -5,9 +5,10 @@ After=network-online.target Wants=network-online.target # Never give up on the agent: without it the screen has nothing to show, and a # board nobody is standing at cannot be restarted by hand. A crash loop backs -# off instead (RestartSteps below) so it costs little while it lasts. A bad -# self-update is caught earlier by the updater's health check and rollback -# (musallahboard-agent-update.service). +# off instead (RestartSteps below) so it costs little while it lasts. A new +# agent that restarts during its first minutes after a self-update is rolled +# back by the updater's probation (musallahboard-agent-update.service); one +# that starts crashing later is not, and loops here. StartLimitIntervalSec=0 [Service] @@ -23,9 +24,13 @@ RestartSec=5 RestartSteps=6 RestartMaxDelaySec=120 -# sd_notify watchdog: agent must call WATCHDOG=1 every 30s or systemd -# kills it. Catches goroutine deadlocks where the process is alive but -# not making progress. +# sd_notify watchdog: the agent sends WATCHDOG=1 every 15 s, but only while +# it is live: while it waits for enrollment its poll loop must have run, and +# once enrolled its own local server must have answered /api/local/status +# (which takes the locks of the sync, backend, update and clock state) within +# the last 60 s. A wedged local server or a deadlock in that state stops the +# pings and systemd kills and restarts it about two minutes later. An idle or +# offline board answers the same as a busy one, so that is never a reason. WatchdogSec=60s # Capabilities (docs/architecture.md sections 9.5 and 10). The daemon runs as diff --git a/setup.sh b/setup.sh index 1a33fab..0a9d49b 100755 --- a/setup.sh +++ b/setup.sh @@ -702,8 +702,9 @@ ExecStart=/bin/systemctl --no-block try-restart musallahboard-kiosk.service EOF # Root self-updater: the daemon stages a verified agent package and - # creates staged/ready; this re-verifies it as root, swaps the binary and - # rolls back if the new agent does not come up. + # creates staged/ready; this re-verifies it as root, swaps the binary, + # keeps the new agent on probation and rolls back if it fails. The update + # journal in $LIB_DIR also starts it, to finish an interrupted update. sudo tee "$UNIT_DIR/musallahboard-agent-update.path" > /dev/null << 'EOF' [Unit] Description=Watch for a staged MusallahBoard agent update @@ -711,6 +712,7 @@ Documentation=https://github.com/LensBridge/agent/blob/main/docs/architecture.md [Path] PathExists=/var/lib/musallahboard/agent/staged/ready +PathExists=/usr/lib/musallahboard/update-in-progress.json Unit=musallahboard-agent-update.service [Install] @@ -726,9 +728,11 @@ After=musallahboard-agent.service [Service] Type=oneshot ExecStart=/usr/bin/musallahboard-agent selfupdate apply -TimeoutStartSec=5min +TimeoutStartSec=15min ProtectSystem=strict -ReadWritePaths=/usr/bin -/usr/lib/musallahboard /var/lib/musallahboard /etc/musallahboard +ReadWritePaths=/usr/bin -/usr/lib/musallahboard /var/lib/musallahboard -/usr/share/musallahboard +IPAddressDeny=any +IPAddressAllow=localhost ProtectHome=yes PrivateTmp=yes NoNewPrivileges=yes From c2cd919cb3dd85c327be202d584928dc93247f0b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:17:54 +0000 Subject: [PATCH 14/19] Watchdog pings only while the agent is live The WATCHDOG=1 pings came from a goroutine of their own, so a daemon with a wedged local server or a deadlock still pinged and was never restarted, contrary to the unit's comment. The pings now depend on a heartbeat: before enrollment the poll loop bumps it, and once enrolled a probe of the agent's own /api/local/status (the HTTP server plus the sync, backend, update and clock state behind it) every 10 seconds. A heartbeat older than WatchdogSec withholds the ping, so a stuck agent is restarted 1 to 2.25 minutes after its last sign of life. Nothing in the probe waits on the network, so an idle or offline board is never restarted for it; the heartbeat uses the monotonic clock, so the agent stepping the wall clock does not either. --- cmd/agent/main.go | 128 ++++++++++++++++++++++++++++++++++--- cmd/agent/watchdog_test.go | 87 +++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 cmd/agent/watchdog_test.go diff --git a/cmd/agent/main.go b/cmd/agent/main.go index d11314c..d5f7ffd 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -6,10 +6,12 @@ import ( "flag" "fmt" "log/slog" + "net/http" "os" "os/signal" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -18,6 +20,7 @@ import ( "github.com/LensBridge/agent/internal/cdp" "github.com/LensBridge/agent/internal/config" "github.com/LensBridge/agent/internal/enroll" + "github.com/LensBridge/agent/internal/localserver" "github.com/LensBridge/agent/internal/netinfo" "github.com/LensBridge/agent/internal/safemode" "github.com/LensBridge/agent/internal/splash" @@ -163,16 +166,18 @@ func runDaemon() { // Started before the config gate: awaitEnrollment below can hold the // process for days, and systemd's WatchdogSec does not pause for it. + // It pings only while hb is fresh (watchdogLoop). + hb := newHeartbeat() wg.Add(1) go func() { defer wg.Done() - watchdogLoop(ctx) + watchdogLoop(ctx, logger, hb) }() cfg, err := config.Load(defaultConfigPath) if err != nil { logger.Info("not enrolled yet — entering pre-enrollment mode", "reason", err) - cfg = awaitEnrollment(ctx, logger) + cfg = awaitEnrollment(ctx, logger, hb) if cfg == nil { // context cancelled while waiting wg.Wait() logger.Info("shutdown complete") @@ -187,10 +192,17 @@ func runDaemon() { logger.Warn("agent starting in safe mode (repeated crashes detected)") } + hb.beat() if err := startBoard(ctx, logger, cfg, safeMode, &wg); err != nil { logger.Error("could not start", "err", err) os.Exit(1) } + // From now on the heartbeat is the local server answering. + wg.Add(1) + go func() { + defer wg.Done() + probeLoop(ctx, hb, "http://"+localserver.ListenAddr+"/api/local/status", livenessProbeInterval) + }() wg.Add(1) go func() { @@ -225,7 +237,7 @@ func runDaemon() { // Waiting is a legitimate running state rather than a slow start, so we report // READY to systemd up front; otherwise Type=notify would hold the unit in // "activating" and kill it at TimeoutStartSec. -func awaitEnrollment(ctx context.Context, logger *slog.Logger) *config.Config { +func awaitEnrollment(ctx context.Context, logger *slog.Logger, hb *heartbeat) *config.Config { _, _ = daemon.SdNotify(false, daemon.SdNotifyReady) _, _ = daemon.SdNotify(false, "STATUS=waiting for enrollment") @@ -237,6 +249,9 @@ func awaitEnrollment(ctx context.Context, logger *slog.Logger) *config.Config { t := time.NewTicker(enrollPollInterval) defer t.Stop() for { + // Every step below is bounded, so a loop that stops coming round + // is stuck, and the watchdog should restart us. + hb.beat() info := netinfo.Collect(ctx) if !reported || !info.Equal(last) { logger.Info("device network address", @@ -268,24 +283,119 @@ func awaitEnrollment(ctx context.Context, logger *slog.Logger) *config.Config { } } -// watchdogLoop pings systemd's WATCHDOG=1 at half the configured interval. -// systemd kills (and restarts) the agent if it stops pinging — useful when -// a goroutine deadlocks but the process is technically still alive. -func watchdogLoop(ctx context.Context) { +// The systemd watchdog (WatchdogSec= in musallahboard-agent.service). +// +// A ping on a timer of its own would only prove that the Go runtime still +// schedules goroutines. So the agent pings only while a heartbeat is fresh, +// and the heartbeat is bumped by real work: before enrollment by the poll +// loop in awaitEnrollment (every 5 s), and once enrolled by probeLoop, which +// asks the agent's own local server for /api/local/status. Answering that +// takes the HTTP server and the locks of the sync, backend, update and clock +// state, i.e. what the board and the admin portal depend on. None of it +// waits on the network or on the disk beyond a few small files, so an idle or +// offline board stays live; only a wedged server or a deadlock goes stale. +// +// The loops that do wait on the network (backend WebSocket, content sync, +// release channels) are not covered: a stall there is indistinguishable here +// from a slow network, and restarting the agent for it would do more harm +// than good. +const ( + // livenessProbeInterval is how often probeLoop asks the local server. + livenessProbeInterval = 10 * time.Second + // livenessProbeTimeout bounds one probe. + livenessProbeTimeout = 15 * time.Second +) + +// heartbeat is the time of the last proof of life. It holds a time.Time +// with its monotonic reading, so stepping the wall clock (which this agent +// does, with CAP_SYS_TIME) never makes it look stale. +type heartbeat struct{ last atomic.Pointer[time.Time] } + +func newHeartbeat() *heartbeat { + h := &heartbeat{} + h.beat() + return h +} + +func (h *heartbeat) beat() { + t := time.Now() + h.last.Store(&t) +} + +func (h *heartbeat) age() time.Duration { return time.Since(*h.last.Load()) } + +// watchdogLoop pings systemd's WATCHDOG=1 while the heartbeat is fresh. +func watchdogLoop(ctx context.Context, logger *slog.Logger, hb *heartbeat) { interval, err := daemon.SdWatchdogEnabled(false) if err != nil || interval == 0 { return } - t := time.NewTicker(interval / 2) + runWatchdog(ctx, logger, hb, interval, func() { _, _ = daemon.SdNotify(false, daemon.SdNotifyWatchdog) }) +} + +// runWatchdog checks every interval/4 and pings when the heartbeat is at +// most interval old. A daemon that stops beating is therefore killed between +// interval and about 2.25 x interval after its last beat (2 to 2.5 minutes +// with WatchdogSec=60s): long enough that a busy moment is not mistaken for +// a hang. +func runWatchdog(ctx context.Context, logger *slog.Logger, hb *heartbeat, interval time.Duration, ping func()) { + t := time.NewTicker(interval / 4) defer t.Stop() + withheld := false for { select { case <-t.C: - _, _ = daemon.SdNotify(false, daemon.SdNotifyWatchdog) case <-ctx.Done(): return } + age := hb.age() + if age <= interval { + if withheld { + logger.Info("agent is live again; watchdog pings resumed") + withheld = false + } + ping() + continue + } + if !withheld { + logger.Error("agent is not live: no heartbeat; withholding the watchdog ping so systemd restarts it", + "lastHeartbeat", age.Round(time.Second).String()) + withheld = true + } + } +} + +// probeLoop bumps hb whenever the local server answers url with 200. +func probeLoop(ctx context.Context, hb *heartbeat, url string, every time.Duration) { + client := &http.Client{ + Timeout: livenessProbeTimeout, + Transport: &http.Transport{Proxy: nil, DisableKeepAlives: true}, + } + t := time.NewTicker(every) + defer t.Stop() + for { + if probe(ctx, client, url) { + hb.beat() + } + select { + case <-t.C: + case <-ctx.Done(): + return + } + } +} + +func probe(ctx context.Context, client *http.Client, url string) bool { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false + } + resp, err := client.Do(req) + if err != nil { + return false } + resp.Body.Close() + return resp.StatusCode == http.StatusOK } func newLogger() *slog.Logger { diff --git a/cmd/agent/watchdog_test.go b/cmd/agent/watchdog_test.go new file mode 100644 index 0000000..0f94f9d --- /dev/null +++ b/cmd/agent/watchdog_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestWatchdogPingsOnlyWhileLive(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hb := newHeartbeat() + var pings atomic.Int32 + interval := 80 * time.Millisecond + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + runWatchdog(ctx, slog.New(slog.DiscardHandler), hb, interval, func() { pings.Add(1) }) + }() + + // Beating: pings. + for range 10 { + hb.beat() + time.Sleep(interval / 4) + } + if pings.Load() == 0 { + t.Fatal("no pings while live") + } + // Stalled: once the heartbeat is older than interval, no more pings. + time.Sleep(interval + interval/2) + before := pings.Load() + time.Sleep(3 * interval) + if after := pings.Load(); after != before { + t.Errorf("pinged %d times with a stale heartbeat", after-before) + } + // Live again: pings resume. + deadline := time.Now().Add(2 * time.Second) + for pings.Load() == before && time.Now().Before(deadline) { + hb.beat() + time.Sleep(interval / 8) + } + if pings.Load() == before { + t.Error("pings did not resume") + } + cancel() + wg.Wait() +} + +func TestProbeLoopBeatsOnlyOnAnswer(t *testing.T) { + var ok atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !ok.Load() { + http.Error(w, "no", http.StatusInternalServerError) + return + } + w.Write([]byte(`{}`)) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hb := newHeartbeat() + start := *hb.last.Load() + done := make(chan struct{}) + go func() { probeLoop(ctx, hb, srv.URL, 10*time.Millisecond); close(done) }() + + time.Sleep(60 * time.Millisecond) + if !hb.last.Load().Equal(start) { + t.Fatal("a failing server counted as a heartbeat") + } + ok.Store(true) + deadline := time.Now().Add(2 * time.Second) + for hb.last.Load().Equal(start) && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if hb.last.Load().Equal(start) { + t.Fatal("an answering server did not count as a heartbeat") + } + cancel() + <-done +} From 7de3c08dd58d1b7baff9ac2759d3d4358bcd11cf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:18:11 +0000 Subject: [PATCH 15/19] Safe mode: say what it is, and write the counter durably Its comment claimed a board in safe mode refuses commands. It does not: safe mode is only the safeMode flag in heartbeats, so the portal shows a board that keeps restarting. The comment now says so, and where crash loops after a self-update are actually handled. The counter is written with fsync like every other state file. --- internal/safemode/safemode.go | 22 +++++++++++++--------- internal/safemode/safemode_test.go | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 internal/safemode/safemode_test.go diff --git a/internal/safemode/safemode.go b/internal/safemode/safemode.go index 76ad933..64bd6ea 100644 --- a/internal/safemode/safemode.go +++ b/internal/safemode/safemode.go @@ -1,3 +1,4 @@ +// Package safemode counts agent starts that did not lead to a stable run. package safemode import ( @@ -6,6 +7,8 @@ import ( "path/filepath" "strconv" "strings" + + "github.com/LensBridge/agent/internal/fsutil" ) const ( @@ -18,10 +21,15 @@ const ( // resets it after the agent has been up long enough to be considered // stable (typically 5 minutes — caller schedules that). // -// If the counter exceeds safeModeThreshold, the agent should boot in -// heartbeat-only mode: still report in (so admins see the device is -// degraded) but refuse to execute commands. This complements systemd's -// StartLimitBurst, which stops the death-loop entirely. +// If the counter exceeds safeModeThreshold, the agent is "in safe mode". +// That is a report, nothing more: the flag goes out with every backend +// heartbeat (telemetry safeMode) so the admin portal shows that the board is +// restarting over and over. The agent does not behave differently in safe +// mode, it does not refuse commands, and nothing stops the restarts: +// musallahboard-agent.service restarts the agent for ever +// (StartLimitIntervalSec=0), with a growing delay. A new agent that crashes +// in its first minutes after a self-update is rolled back by the root +// updater's probation (package selfupdate), independently of this counter. type Manager struct { path string } @@ -61,9 +69,5 @@ func (m *Manager) write(n int) { if err := os.MkdirAll(filepath.Dir(m.path), 0o755); err != nil { return } - tmp := m.path + ".tmp" - if err := os.WriteFile(tmp, []byte(fmt.Sprintf("%d\n", n)), 0o644); err != nil { - return - } - _ = os.Rename(tmp, m.path) + _ = fsutil.WriteAtomic(m.path, []byte(fmt.Sprintf("%d\n", n)), 0o644) } diff --git a/internal/safemode/safemode_test.go b/internal/safemode/safemode_test.go new file mode 100644 index 0000000..dfa54fc --- /dev/null +++ b/internal/safemode/safemode_test.go @@ -0,0 +1,19 @@ +package safemode + +import "testing" + +func TestSafeModeAfterRepeatedStarts(t *testing.T) { + m := New(t.TempDir()) + for i := 1; i <= safeModeThreshold; i++ { + if m.OnStartup() { + t.Fatalf("safe mode after %d starts", i) + } + } + if !m.OnStartup() { + t.Fatal("no safe mode after repeated starts") + } + m.OnSuccessfulRun() + if m.OnStartup() { + t.Fatal("safe mode after a stable run") + } +} From bf51cbe101731e8c114f9bd96b6209a67ede7013 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:18:45 +0000 Subject: [PATCH 16/19] Docs: the agent update journal, probation and watchdog as they are Section 12 describes the probation, rollback rules and strikes, and the journal with what the next run does from each state. The layout lists the two new root-owned files, section 14 says what the watchdog does and does not catch, and safe mode is described as the report it is. --- docs/architecture.md | 88 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 57be101..5749297 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -40,7 +40,9 @@ The v1 offline design was bolted onto an online-only board. The audit found: self-updater and the USB import helper. 6. **Atomic and recoverable.** Every install is stage, fsync, rename, symlink swap. A power cut at any point leaves the previous state serving. A new - agent that does not come up healthy is rolled back automatically. + agent is on probation for its first minutes and is rolled back + automatically if it fails it; an interrupted agent update is finished + from its journal at the next boot. ## 3. Components @@ -247,6 +249,8 @@ packer (`scripts/package-mbu.mjs`) producing the same format. |---|---|---| | `/usr/bin/musallahboard-agent` | root 0755 | the agent | | `/usr/lib/musallahboard/agent.prev` | root 0755 | previous agent, for rollback | +| `/usr/lib/musallahboard/update-in-progress.json` | root 0644 | journal of an agent update not yet finished (section 12) | +| `/usr/lib/musallahboard/update-strikes.json` | root 0644 | per agent version, updates rolled back only for not reconnecting (section 12) | | `/etc/musallahboard/agent.toml` | daemon 0600 | config (section 11) | | `/etc/musallahboard/agent.key` | daemon 0600 | device identity key (unchanged) | | `/etc/musallahboard/trust.json` | root 0644 | trust roots | @@ -664,27 +668,75 @@ route and cost nothing, and the moment it gets a network it starts syncing. 2. `musallahboard-agent-update.path` (`PathExists=` on `ready`) starts `musallahboard-agent-update.service` (root, oneshot): `musallahboard-agent selfupdate apply`. -3. As root, the updater: removes `ready`; copies `package.mbu` into a +3. As root, the updater: removes `ready`; finishes any unfinished update + first (below) and stops there; copies `package.mbu` into a root-only temporary directory (so the daemon cannot swap it mid-check); - verifies it from scratch against root-trusted keys only; checks arch and - that the version is newer than its own; extracts the binary to + verifies it from scratch against root-trusted keys only; checks arch, + that the version is newer than its own and that it is not in + `agentVersionsRejected`; extracts the binary to `/usr/bin/.musallahboard-agent.new`; runs `.new version` and requires the expected version; copies the current binary to - `/usr/lib/musallahboard/agent.prev`; renames the new binary into place; - restarts `musallahboard-agent.service`. -4. Health check: within 120 s, `GET http://127.0.0.1:8080/api/local/status` - must report the new `agentVersion`. Otherwise the updater restores - `agent.prev`, restarts the service, and adds the version to - `agentVersionsRejected`, so a broken release cannot loop. -5. The outcome is written to `agent/last-update.json`, and only then, after a - rollback, is the previous agent restarted, so it can read it. + `/usr/lib/musallahboard/agent.prev`; asks the running agent whether it is + connected to LensBridge; writes the journal; renames the new binary into + place; restarts `musallahboard-agent.service`. +4. Probation. The new agent must report its `agentVersion` on + `GET http://127.0.0.1:8080/api/local/status` within 2 minutes; then + systemd must not restart it (same `MainPID` and `NRestarts`, still + `active`) for 4 minutes; and if the board was connected before the update + (`backend.connected`, or `sync.lastSuccessAt` with no `lastError`), it + must connect again within 3 minutes. A board that was offline is not + asked to be online. If it passes, the outcome is written and the + enrollment splash refreshed (`splash install`). +5. If it fails, the updater restores `agent.prev`, records the outcome in + `agent/last-update.json`, and only then restarts the previous agent, so it + can read it. Not coming up, a restart or crash loop, or a probation cut + short 3 times adds the version to `agentVersionsRejected`, so a broken + release cannot loop. Not reconnecting only counts a strike + (`update-strikes.json`), because the backend or the network may be what + failed; the second strike rejects the version. If `agent.prev` cannot be + restored the outcome is `rollback-failed` and the new agent keeps running. + +**The update journal.** `/usr/lib/musallahboard/update-in-progress.json` is +root-owned, outside the daemon's reach, and written (temp file, fsync, +rename, fsync of the directory) before the binary is replaced and at every +change of phase. It names both versions and the SHA-256 of the previous and +the new binary. The path unit watches it as well as `ready`, and the updater +also puts `ready` back while an update is open (boards whose path unit +predates the journal watch only `ready`), so an update cut short by a power +cut, a reboot or the updater being stopped is finished by the next run, at +boot at the latest. That run is the binary installed then, so the record's +format only ever gains fields. What it does: + +| Journal phase | Installed binary | The next run | +|---|---|---| +| none | | nothing to finish; handles a staged package, if any | +| `swapping` | previous | the swap never happened: removes the extracted binary, outcome `refused` ("interrupted"), nothing else changes | +| `swapping` or `probation` | new | starts the probation again from scratch, without restarting the agent; after 3 cut-short probations, rolls back and rejects | +| `swapping` or `probation` | anything else | someone installed another agent: left alone, outcome `refused` | +| `committing` | | writes the `ok` outcome again (same timestamp), refreshes the splash, removes the journal | +| `rolling-back` | | repeats the rollback: restores `agent.prev` if it is not in place, rejects or strikes (idempotent), writes the outcome, removes the journal, restarts the agent | +| unreadable | | puts `agent.prev` back if it differs, outcome `rolled-back` | + +The journal is removed only after the outcome is written, and whenever the +updater ends other than by being stopped, so the path unit cannot loop on +it. SIGTERM (shutdown, or the unit's 15 minute start timeout) stops a +probation and leaves the journal; a rollback that has begun always finishes. +Boards set up before the journal existed run the updater under a 5 minute +timeout; a typical update (seconds, then the 4 minute probation) fits, and +one that does not is resumed as above. + +Not covered: a new agent that starts crashing after its probation is +restarted by systemd for ever (with a growing delay) and is not rolled +back. Safe mode (`internal/safemode`) only reports such a board, as +`safeMode` in its heartbeats; it does not change what the agent does. Telling the people at the board (`internal/agentupdate`): the agent that staged the package leaves the screen on "Restarting MusallahBoard" and its batch's notice in `agent/pending-notice.json`. Then: - **success**: the new agent shows that notice ("Update complete · Agent - 0.3.0 · Board app 2.1.0") at startup; + 0.3.0 · Board app 2.1.0") at startup, which is before its probation ends: + a rollback afterwards replaces it with the rollback notice below; - **rollback**: the previous agent, restarted, finds an unseen `rolled-back` outcome and shows "Update not installed · Agent 0.3.0 did not start correctly, so the board went back to 0.2.1", with the batch's other lines; @@ -728,6 +780,14 @@ connecting a board to a network permanently. - The agent's unit never gives up restarting (`StartLimitIntervalSec=0`); a crash loop backs off from 5 s to 2 minutes (`RestartSteps`, `RestartMaxDelaySec`, systemd 254 and later). +- systemd's watchdog (`WatchdogSec=60s`) restarts an agent that is alive but + stuck. The agent pings only while a heartbeat is fresh: before enrollment + its poll loop bumps it, once enrolled a probe of its own + `/api/local/status` every 10 s (the local server and the locks of the sync, + backend, update and clock state). A heartbeat older than 60 s withholds + the ping, so a stuck agent is restarted within about 2 minutes. The probe + never waits on the network, so an offline board is not restarted for it; + a stall in the network loops themselves is not detected. - The daemon still watches for Chromium's error page and navigates back to the board, as a backstop. @@ -763,7 +823,7 @@ per-day payloads from installed content, and weather comes through the agent. | Malicious code (SPA or agent) | Only the release key signs software; the backend never holds it. | | Backend compromise | Limited to content on screens; cannot push code. Content key rotation via `previous-public-keys` and `trust` CLI. | | Replay of old packages | Monotonic sequence/version in `state.json`. | -| Bad agent release | Health-checked self-update with automatic rollback and a rejected-version list. | +| Bad agent release | Self-update with a crash-safe journal and a probation (stays up; reconnects if it was connected), automatic rollback and a rejected-version list. | | Daemon compromise escalating to root | Daemon is unprivileged (only `CAP_SYS_TIME`, `CAP_NET_BIND_SERVICE`); the root updater re-verifies from a private copy with root-owned keys. | | Zip-slip, zip bombs, disk exhaustion | Strict entry names, declared sizes enforced while reading, global caps, one import at a time. | | Hostile USB stick | Allow-listed filesystems, read-only `nosuid,nodev,noexec` mount, only `*.mbu` regular files copied, no network in the helper, signature checks as for any source. | From c79ec973e2ac40156e209d388af5e335fe4c8420 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 06:21:09 +0000 Subject: [PATCH 17/19] install.sh: the backend URL must be https; point dev installs at --insecure-dev Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01YY5BHXmNUhs48eubkKCNuE --- packaging/install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/install.sh b/packaging/install.sh index 4d16504..d62fe12 100644 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -30,7 +30,9 @@ # Defaults to the directory containing this script. # --token One-time enrollment token. If omitted, enrollment # is skipped (run `musallahboard-agent enroll` later). -# --backend Backend base URL (e.g. http://localhost:8080). +# --backend Backend base URL. Must be https (e.g. https://lensbridge.example). +# For a plain-http development backend, skip --token here and +# enroll on the board with `musallahboard-agent enroll --insecure-dev`. # Required when --token is provided. # --kiosk Install the cage kiosk unit + launcher. The display account is # always `musallahkiosk` and is created here if absent — it is From badfa69f5cc274571c91c849884ab3ab75a83a95 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 04:52:24 +0000 Subject: [PATCH 18/19] Drop the signed refresh-channel upgrade The refresh stream is being removed in favour of a content-changed message on the agent WebSocket, so signing its upgrade is wasted work. This reverts merge 6ae00be. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YY5BHXmNUhs48eubkKCNuE --- docs/architecture.md | 8 +-- internal/boardsync/boardsync_test.go | 79 ---------------------------- internal/boardsync/refresh.go | 45 ++-------------- 3 files changed, 5 insertions(+), 127 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 33245e2..e3e1a63 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -446,13 +446,7 @@ The daemon, when `content_sync` is on and the board is enrolled: - syncs at startup, every 30 minutes, and 10 s after any message on the backend's refresh channel (`wss:///api/refresh-musallahboard?deviceId=`, - debounced). The upgrade request is device-signed (9.2: `GET`, the path - without the query, empty body), signed afresh for every dial. The backend - takes the device from `X-MB-Device-Id`, refuses a `deviceId` query that - disagrees, and refuses a timestamp it has already accepted on this channel. - Unsigned upgrades from older agents are accepted only while the backend's - `musallahboard.board-stream.require-signature` is `false`, and never take - over from a signed connection, + debounced), - with `POST /api/agent/content-bundle`, device-authenticated (9.2), body `{"days": , "haveMedia": ["", ...]}`, - and imports the returned package with source `sync`. diff --git a/internal/boardsync/boardsync_test.go b/internal/boardsync/boardsync_test.go index 45886a2..cb2ca1e 100644 --- a/internal/boardsync/boardsync_test.go +++ b/internal/boardsync/boardsync_test.go @@ -277,9 +277,6 @@ func TestRefreshTriggersDebouncedSync(t *testing.T) { if r.URL.Query().Get("deviceId") != testDevice { t.Errorf("refresh deviceId = %q", r.URL.Query().Get("deviceId")) } - if err := verifyDeviceAuth(r, nil); err != "" { - t.Errorf("refresh upgrade: %s", err) - } c, err := websocket.Accept(w, r, nil) if err != nil { return @@ -324,82 +321,6 @@ func TestRefreshTriggersDebouncedSync(t *testing.T) { } } -// verifyDeviceAuth checks r's device signature the way the backend does -// (DeviceRequestAuthenticator), and returns what is wrong, or "". -func verifyDeviceAuth(r *http.Request, body []byte) string { - if got := r.Header.Get(devauth.HeaderDeviceID); got != testDevice { - return "device header " + strconv.Quote(got) - } - ts, err := strconv.ParseInt(r.Header.Get(devauth.HeaderTimestamp), 10, 64) - if err != nil { - return "timestamp: " + err.Error() - } - if d := time.Since(time.UnixMilli(ts)); d < -5*time.Minute || d > 5*time.Minute { - return "timestamp " + strconv.FormatInt(ts, 10) + " is not now" - } - sig, err := base64.StdEncoding.DecodeString(r.Header.Get(devauth.HeaderSignature)) - if err != nil { - return "signature: " + err.Error() - } - msg := devauth.Message(r.Method, r.URL.Path, testDevice, ts, body) - if !ed25519.Verify(deviceKey.Public().(ed25519.PublicKey), msg, sig) { - return "signature does not verify" - } - return "" -} - -// The refresh upgrade is a device-signed GET over the path alone. Ed25519 is -// deterministic, so a fixed key, device and time give a fixed signature. The -// backend's BoardStreamHandshakeInterceptorTest verifies this exact value with -// the real DeviceRequestAuthenticator: if either side changes how it builds -// the signed message, one of the two tests fails. -func TestRefreshHeaderSignedGolden(t *testing.T) { - e := newEnv(t, "https://api.example/") - e.syncer.d.Key = ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, 32)) - e.syncer.now = func() time.Time { return time.UnixMilli(1790258531123) } - wsURL, err := refreshURL(e.cfg.BackendURL, testDevice) - if err != nil { - t.Fatal(err) - } - h, err := e.syncer.refreshHeader(wsURL) - if err != nil { - t.Fatal(err) - } - if got := h.Get(devauth.HeaderDeviceID); got != testDevice { - t.Errorf("device header = %q", got) - } - if got := h.Get(devauth.HeaderTimestamp); got != "1790258531123" { - t.Errorf("timestamp header = %q", got) - } - if got := h.Get("User-Agent"); got != "musallahboard-agent/0.3.0" { - t.Errorf("User-Agent = %q", got) - } - sig, _ := base64.StdEncoding.DecodeString(h.Get(devauth.HeaderSignature)) - msg := devauth.Message("GET", "/api/refresh-musallahboard", testDevice, 1790258531123, nil) - if !ed25519.Verify(e.syncer.d.Key.Public().(ed25519.PublicKey), msg, sig) { - t.Fatal("signature does not verify over GET, the path without its query, and an empty body") - } - const golden = "TGTiOMTSt8N/7ad06yMvmWo9a68Yz01LNGYnDOPGfedeiPTPjC9hqtKa/yLAOa7rXlDa5UfKTpmI28tdSME4Dg==" - if got := h.Get(devauth.HeaderSignature); got != golden { - t.Errorf("signature = %s, want %s", got, golden) - } -} - -// Each dial signs afresh: the backend refuses a timestamp it has already -// accepted on this channel, so a reconnect must never resend old headers. -func TestRefreshHeaderFreshPerDial(t *testing.T) { - e := newEnv(t, "https://api.example") - now := time.UnixMilli(1790258531000) - e.syncer.now = func() time.Time { now = now.Add(time.Second); return now } - wsURL, _ := refreshURL(e.cfg.BackendURL, testDevice) - a, _ := e.syncer.refreshHeader(wsURL) - b, _ := e.syncer.refreshHeader(wsURL) - if a.Get(devauth.HeaderTimestamp) == b.Get(devauth.HeaderTimestamp) || - a.Get(devauth.HeaderSignature) == b.Get(devauth.HeaderSignature) { - t.Fatal("two dials carried the same timestamp or signature") - } -} - func TestRefreshURL(t *testing.T) { for in, want := range map[string]string{ "https://api.example": "wss://api.example/api/refresh-musallahboard?deviceId=" + testDevice, diff --git a/internal/boardsync/refresh.go b/internal/boardsync/refresh.go index d649ece..43e34b5 100644 --- a/internal/boardsync/refresh.go +++ b/internal/boardsync/refresh.go @@ -18,10 +18,9 @@ import ( // the socket would otherwise be redialled every second. const refreshStable = 30 * time.Second -// refreshLoop holds the backend's refresh channel open. The backend sends a -// message whenever something on the board's content changes; each one -// schedules a content sync. The upgrade is device-signed (refreshHeader), so -// nobody who merely knows the device id can take the board's place on it. +// refreshLoop holds the backend's refresh channel open, the same socket the +// hosted board listens on. The backend sends a message whenever something on +// the board's content changes; each one schedules a content sync. func (s *Syncer) refreshLoop(ctx context.Context) { wsURL, err := refreshURL(s.d.Cfg.BackendURL, s.d.Cfg.DeviceID) if err != nil { @@ -53,13 +52,9 @@ func (s *Syncer) refreshOnce(ctx context.Context, wsURL string) error { // meant to stay open for hours and is bounded by dialCtx instead. wsClient := *s.client wsClient.Timeout = 0 - header, err := s.refreshHeader(wsURL) - if err != nil { - return err - } conn, _, err := websocket.Dial(dialCtx, wsURL, &websocket.DialOptions{ HTTPClient: &wsClient, - HTTPHeader: header, + HTTPHeader: http.Header{"User-Agent": []string{"musallahboard-agent/" + s.d.AgentVersion}}, }) if err != nil { return fmt.Errorf("dial: %w", err) @@ -78,38 +73,6 @@ func (s *Syncer) refreshOnce(ctx context.Context, wsURL string) error { } } -// refreshHeader is the upgrade request's headers: the User-Agent and the -// device signature (docs/architecture.md, section 9.2) over GET, the path and -// an empty body. It is signed afresh for every dial, since the backend allows -// only a few minutes of clock difference and refuses a timestamp it has -// already seen on this channel. -// -// The deviceId query parameter stays: the signature does not cover the query, -// so the backend takes the device from X-MB-Device-Id and only checks that the -// two agree, and a backend that predates signed upgrades still finds the -// board by it. -func (s *Syncer) refreshHeader(wsURL string) (http.Header, error) { - u, err := url.Parse(wsURL) - if err != nil { - return nil, err - } - // The path is what is signed; the scheme only has to be one - // http.NewRequest accepts. websocket.Dial makes the same swap. - switch u.Scheme { - case "wss": - u.Scheme = "https" - case "ws": - u.Scheme = "http" - } - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - req.Header.Set("User-Agent", "musallahboard-agent/"+s.d.AgentVersion) - signRequest(req, nil, s.d.Cfg.DeviceID, s.d.Key, s.now()) - return req.Header, nil -} - // refreshURL derives ws(s):///api/refresh-musallahboard?deviceId= // from backend_url. func refreshURL(backendURL, deviceID string) (string, error) { From b53f78b755a13b55b1429b6eeec1a847d799b3fb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 04:54:46 +0000 Subject: [PATCH 19/19] Start the new agent when resuming an interrupted swap without a reboot A run stopped after the swap but before the restart, then resumed without a reboot, left the previous agent running; probation waited for a version that never came and rejected a good release. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YY5BHXmNUhs48eubkKCNuE --- internal/selfupdate/selfupdate.go | 13 ++++++++++++- internal/selfupdate/selfupdate_test.go | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/internal/selfupdate/selfupdate.go b/internal/selfupdate/selfupdate.go index a9c504d..0fdea0c 100644 --- a/internal/selfupdate/selfupdate.go +++ b/internal/selfupdate/selfupdate.go @@ -390,7 +390,18 @@ func (u *Updater) startProbation(ctx context.Context, rec *Txn, resumed bool) (O } u.crashpoint("probation") - if !resumed { + restart := !resumed + if resumed { + // A run that stopped after the swap but before the restart, then + // resumed without a reboot, leaves the previous agent running. Only an + // agent that already reports the new version is left alone; one that + // is still starting is the installed (new) binary, so restarting it + // costs nothing. + if st, err := u.Status(ctx); err != nil || st.AgentVersion != rec.To { + restart = true + } + } + if restart { u.Logf("restarting %s", AgentUnit) if o, err := u.Run(ctx, "systemctl", "restart", AgentUnit); err != nil { if ctx.Err() != nil { diff --git a/internal/selfupdate/selfupdate_test.go b/internal/selfupdate/selfupdate_test.go index 51f9f05..8cf8a54 100644 --- a/internal/selfupdate/selfupdate_test.go +++ b/internal/selfupdate/selfupdate_test.go @@ -746,3 +746,26 @@ func TestFetchStatus(t *testing.T) { t.Error("connected backend not up") } } + +// A run stopped after the swap and resumed without a reboot finds the +// previous agent still running. It must start the new one, not wait for a +// version that never comes and then reject a good release. +func TestResumeWithoutRebootStartsTheNewAgent(t *testing.T) { + h := newHarness(t, true) + h.stage(t, "0.4.0", "arm64", releaseKey) + cutAt(t, h, "swapped") + if h.isNew() { + t.Fatal("harness: the new agent runs before any restart") + } + u := &Updater{Layout: h.u.Layout, TrustPath: h.u.TrustPath, Binary: h.u.Binary, NewBinary: h.u.NewBinary, + Prev: h.u.Prev, TempRoot: h.u.TempRoot, Arch: h.u.Arch, CurrentVersion: h.agent.version, Logf: t.Logf} + h.wire(u) + u.defaults() + out, err := u.Apply(context.Background()) + if err != nil || out.Status != StatusOK { + t.Fatalf("outcome = %+v, err = %v", out, err) + } + if rejected(t, u, "0.4.0") { + t.Error("a good release was rejected") + } +}