diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 78e3d53..dafefa8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,125 +1,136 @@
-name: CI
-
-on:
- push:
- branches: [ main ]
- paths-ignore:
- - 'custom_components/**'
- - '.github/workflows/release.yml'
- - '.github/workflows/release-ha.yml'
- - '.github/workflows/prerelease.yml'
- - '.github/workflows/slash-command.yml'
- - '**/*.md'
- - 'docs/**'
- - 'LICENSE'
- - '.gitignore'
- - '.gitattributes'
- - 'hacs.json'
- pull_request:
- paths-ignore:
- - 'custom_components/**'
- - '.github/workflows/release.yml'
- - '.github/workflows/release-ha.yml'
- - '.github/workflows/prerelease.yml'
- - '.github/workflows/slash-command.yml'
- - '**/*.md'
- - 'docs/**'
- - 'LICENSE'
- - '.gitignore'
- - '.gitattributes'
- - 'hacs.json'
-
-permissions:
- contents: read
-
-jobs:
- firmware:
- name: Firmware (${{ matrix.env }})
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- env: [c3-debug, s3-debug, esp32-debug]
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-node@v4
- with:
- node-version: 22
- cache: npm
- cache-dependency-path: frontend/package-lock.json
-
- - name: Build frontend (generates web_assets.h)
- run: npm ci && npm run build
- working-directory: frontend
-
- - uses: actions/setup-python@v5
- with:
- python-version: "3.12"
-
- - name: Cache PlatformIO
- uses: actions/cache@v4
- with:
- path: |
- ~/.platformio/platforms
- ~/.platformio/packages
- key: platformio-${{ matrix.env }}-${{ hashFiles('platformio.ini') }}
-
- - name: Install PlatformIO
- # intelhex is an unlisted runtime dep of the newer tool-esptoolpy
- # bundled with pioarduino 54.x; esptool imports it at module load.
- run: pip install platformio intelhex
-
- - name: Build firmware
- run: pio run -e ${{ matrix.env }}
-
- - name: Format check (clang-format)
- run: python scripts/check_format.py
-
- - name: Static analysis (clang-tidy)
- run: pio check -e ${{ matrix.env }} --fail-on-defect=low
-
- frontend:
- name: Frontend
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-node@v4
- with:
- node-version: 22
- cache: npm
- cache-dependency-path: frontend/package-lock.json
-
- - name: Install dependencies
- run: npm ci
- working-directory: frontend
-
- - name: Lint and format check
- run: npm run check
- working-directory: frontend
-
- - name: Build (lint + typecheck + vite + embed)
- run: npm run build
- working-directory: frontend
-
- flash-tool:
- name: Flash Tool
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-go@v5
- with:
- go-version-file: flash/go.mod
-
- - name: Lint
- uses: golangci/golangci-lint-action@v7
- with:
- version: v2.11.3
- args: ./...
- working-directory: flash
-
- - name: Build
- run: go build -o openneato-flash .
- working-directory: flash
+name: CI
+
+on:
+ push:
+ branches: [ main ]
+ paths-ignore:
+ - 'custom_components/**'
+ - '.github/workflows/release.yml'
+ - '.github/workflows/release-ha.yml'
+ - '.github/workflows/prerelease.yml'
+ - '.github/workflows/slash-command.yml'
+ - '**/*.md'
+ - 'docs/**'
+ - 'LICENSE'
+ - '.gitignore'
+ - '.gitattributes'
+ - 'hacs.json'
+ pull_request:
+ paths-ignore:
+ - 'custom_components/**'
+ - '.github/workflows/release.yml'
+ - '.github/workflows/release-ha.yml'
+ - '.github/workflows/prerelease.yml'
+ - '.github/workflows/slash-command.yml'
+ - '**/*.md'
+ - 'docs/**'
+ - 'LICENSE'
+ - '.gitignore'
+ - '.gitattributes'
+ - 'hacs.json'
+
+permissions:
+ contents: read
+
+jobs:
+ firmware:
+ name: Firmware (${{ matrix.env }})
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ env: [c3-debug, c6-debug, c6-ota, s3-debug, esp32-debug]
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 22
+ cache: npm
+ cache-dependency-path: frontend/package-lock.json
+
+ - name: Build frontend (generates web_assets.h)
+ run: npm ci && npm run build
+ working-directory: frontend
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Cache PlatformIO
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.platformio/platforms
+ ~/.platformio/packages
+ key: platformio-${{ matrix.env }}-${{ hashFiles('platformio.ini') }}
+
+ - name: Install PlatformIO
+ # intelhex is an unlisted runtime dep of the newer tool-esptoolpy
+ # bundled with pioarduino 54.x; esptool imports it at module load.
+ run: pip install platformio intelhex
+
+ - name: Build firmware
+ run: pio run -e ${{ matrix.env }}
+
+ - name: Upload firmware binary
+ if: matrix.env == 'c6-ota'
+ uses: actions/upload-artifact@v4
+ with:
+ name: firmware-${{ matrix.env }}
+ path: |
+ .pio/build/${{ matrix.env }}/firmware.bin
+ .pio/build/${{ matrix.env }}/partitions.bin
+ .pio/build/${{ matrix.env }}/bootloader.bin
+ retention-days: 7
+
+ - name: Format check (clang-format)
+ run: python scripts/check_format.py
+
+ - name: Static analysis (clang-tidy)
+ run: pio check -e ${{ matrix.env }} --fail-on-defect=low
+
+ frontend:
+ name: Frontend
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 22
+ cache: npm
+ cache-dependency-path: frontend/package-lock.json
+
+ - name: Install dependencies
+ run: npm ci
+ working-directory: frontend
+
+ - name: Lint and format check
+ run: npm run check
+ working-directory: frontend
+
+ - name: Build (lint + typecheck + vite + embed)
+ run: npm run build
+ working-directory: frontend
+
+ flash-tool:
+ name: Flash Tool
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: flash/go.mod
+
+ - name: Lint
+ uses: golangci/golangci-lint-action@v7
+ with:
+ version: v2.11.3
+ args: ./...
+ working-directory: flash
+
+ - name: Build
+ run: go build -o openneato-flash .
+ working-directory: flash
diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml
index c1759c0..d6c58d4 100644
--- a/.github/workflows/prerelease.yml
+++ b/.github/workflows/prerelease.yml
@@ -22,6 +22,7 @@ jobs:
pr_number: ${{ steps.resolve.outputs.pr_number }}
head_sha: ${{ steps.resolve.outputs.head_sha }}
head_ref: ${{ steps.resolve.outputs.head_ref }}
+ checkout_ref: ${{ steps.resolve.outputs.checkout_ref }}
release_tag: ${{ steps.resolve.outputs.tag }}
version: ${{ steps.resolve.outputs.version }}
@@ -37,9 +38,11 @@ jobs:
PR_JSON=$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json headRefOid,headRefName)
HEAD_SHA=$(echo "$PR_JSON" | jq -r '.headRefOid')
HEAD_REF=$(echo "$PR_JSON" | jq -r '.headRefName')
+ CHECKOUT_REF="refs/pull/$PR_NUMBER/head"
else
HEAD_SHA="$GITHUB_SHA"
HEAD_REF="$GITHUB_REF_NAME"
+ CHECKOUT_REF="$GITHUB_SHA"
fi
LATEST_TAG=$(gh release list --repo "$GITHUB_REPOSITORY" \
@@ -51,15 +54,18 @@ jobs:
if [ -n "$PR_NUMBER" ]; then
TAG="v${BASE}-pr${PR_NUMBER}.${SHORT_SHA}"
else
- SAFE_REF=$(echo "$HEAD_REF" | sed 's/[^a-zA-Z0-9._-]/-/g')
+ SAFE_REF="${HEAD_REF//[^a-zA-Z0-9._-]/-}"
TAG="v${BASE}-${SAFE_REF}.${SHORT_SHA}"
fi
- echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
- echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
- echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
- echo "tag=$TAG" >> "$GITHUB_OUTPUT"
- echo "version=${TAG#v}" >> "$GITHUB_OUTPUT"
+ {
+ echo "pr_number=$PR_NUMBER"
+ echo "head_sha=$HEAD_SHA"
+ echo "head_ref=$HEAD_REF"
+ echo "checkout_ref=$CHECKOUT_REF"
+ echo "tag=$TAG"
+ echo "version=${TAG#v}"
+ } >> "$GITHUB_OUTPUT"
echo "Release: $TAG ($HEAD_REF @ $SHORT_SHA)"
prerelease:
@@ -71,7 +77,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
- ref: ${{ needs.resolve.outputs.head_sha }}
+ ref: ${{ needs.resolve.outputs.checkout_ref }}
fetch-depth: 0
- name: Create release tag
@@ -122,10 +128,10 @@ jobs:
env:
FIRMWARE_VERSION: ${{ needs.resolve.outputs.version }}
run: |
- for env in $(grep -oE '^\[env:([a-zA-Z0-9_-]+-release)\]' platformio.ini | sed 's/\[env:\(.*\)\]/\1/'); do
+ while IFS= read -r env; do
echo "Building $env..."
pio run -e "$env"
- done
+ done < <(grep -oE '^\[env:([a-zA-Z0-9_-]+-release)\]' platformio.ini | sed 's/\[env:\(.*\)\]/\1/')
# --- Flash tool + GitHub Release ---
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 15a1ff7..884b341 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -58,7 +58,7 @@ jobs:
run: |
RELEASE_TAG="${{ github.event.inputs.release_tag }}"
VERSION=${RELEASE_TAG#v}
- echo "version=$VERSION" >> $GITHUB_OUTPUT
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "Setting version to: $VERSION"
- name: Save release notes
@@ -98,7 +98,7 @@ jobs:
run: npm ci
working-directory: frontend
- - name: Build frontend (lint + typecheck + vite + embed)
+ - name: Build frontend (lint + codegen + path-sync + typecheck + vite + embed)
run: npm run build
working-directory: frontend
@@ -128,10 +128,10 @@ jobs:
run: |
# Build all *-release environments; post-build hook packages
# firmware artifacts into each env's build dir for GoReleaser
- for env in $(grep -oE '^\[env:([a-zA-Z0-9_-]+-release)\]' platformio.ini | sed 's/\[env:\(.*\)\]/\1/'); do
+ while IFS= read -r env; do
echo "Building $env..."
pio run -e "$env"
- done
+ done < <(grep -oE '^\[env:([a-zA-Z0-9_-]+-release)\]' platformio.ini | sed 's/\[env:\(.*\)\]/\1/')
# --- Flash tool + GitHub Release ---
diff --git a/.github/workflows/slash-command.yml b/.github/workflows/slash-command.yml
index 95398d1..94952da 100644
--- a/.github/workflows/slash-command.yml
+++ b/.github/workflows/slash-command.yml
@@ -28,8 +28,7 @@ jobs:
-f content='rocket' --silent
PR_NUMBER="${{ github.event.issue.number }}"
- HEAD_REF=$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json headRefName --jq '.headRefName')
- gh workflow run prerelease.yml --repo "$GITHUB_REPOSITORY" --ref "$HEAD_REF" \
+ gh workflow run prerelease.yml --repo "$GITHUB_REPOSITORY" --ref "${{ github.event.repository.default_branch }}" \
-f pr_number="$PR_NUMBER" \
-f comment_id="${{ github.event.comment.id }}"
diff --git a/.gitignore b/.gitignore
index ddd21ec..8c96012 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,14 @@
.pio
.cache
+__pycache__/
+*.pyc
+compile_commands.json
firmware/compile_commands.json
firmware/src/web_assets.h
frontend/node_modules
frontend/dist
+frontend/mock/build-info.js
+frontend/src/types.generated.ts
flash/openneato-flash
release/
dist/
diff --git a/.goreleaser.yml b/.goreleaser.yml
index 31cb9d6..d330d06 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -5,6 +5,13 @@ project_name: openneato
env:
- ESPTOOL_VERSION=v5.2.0
+before:
+ # Stamp the release version into the OpenAPI spec shipped as a release
+ # artifact. The in-repo file carries "0.0.0-dev" as a placeholder.
+ hooks:
+ - "sed -i.bak 's/^ version:.*/ version: {{ .Version }}/' frontend/api/openapi.yaml"
+ - "rm -f frontend/api/openapi.yaml.bak"
+
builds:
- id: openneato-flash
dir: flash
@@ -16,6 +23,16 @@ builds:
env:
- >-
{{- if eq .Os "darwin" }}CGO_ENABLED=1{{- else }}CGO_ENABLED=0{{- end }}
+ # Pin macOS minimum to 11.0 (Big Sur), which is also Go's own floor as of
+ # Go 1.25+. Without this, the linker uses the build host's SDK as the
+ # deployment target, producing binaries that fail to load on anything
+ # older than the runner's macOS version.
+ - >-
+ {{- if eq .Os "darwin" }}MACOSX_DEPLOYMENT_TARGET=11.0{{- end }}
+ - >-
+ {{- if eq .Os "darwin" }}CGO_CFLAGS=-mmacosx-version-min=11.0{{- end }}
+ - >-
+ {{- if eq .Os "darwin" }}CGO_LDFLAGS=-mmacosx-version-min=11.0{{- end }}
goos:
- linux
- darwin
@@ -43,12 +60,14 @@ release:
extra_files:
- glob: .pio/build/*-release/*-firmware.bin
- glob: .pio/build/*-release/*-full.tar.gz
+ - glob: frontend/api/openapi.yaml
checksum:
name_template: checksums.txt
extra_files:
- glob: .pio/build/*-release/*-firmware.bin
- glob: .pio/build/*-release/*-full.tar.gz
+ - glob: frontend/api/openapi.yaml
changelog:
sort: asc
diff --git a/AGENTS.md b/AGENTS.md
index d55b2be..15b6634 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -25,7 +25,9 @@ Three top-level components: `firmware/` (ESP32 C/C++), `frontend/` (Preact SPA),
- Flash tool: Go CLI, cross-compiled via GoReleaser, uses esptool subprocess
- Mock server: `frontend/mock/server.js` Vite plugin, `SCENARIO` constant for state switching. Reset to `"ok"` before
committing.
-- Build pipeline: `npm run build` -> lint -> tsc -> vite -> `embed_frontend.js` generates `web_assets.h`
+- Build pipeline: `npm run build` -> lint -> `jscpd` duplicate check -> `openapi-typescript` regenerates `src/types.generated.ts` from
+ `frontend/api/openapi.yaml` -> `check_api_paths.js` enforces firmware↔spec route parity -> tsc -> vite ->
+ `embed_frontend.js` generates `web_assets.h`. `frontend/api/openapi.yaml` ships as a release asset.
### Data Logging
@@ -43,6 +45,13 @@ needs logging, add a new typed helper following the existing pattern. Log both
success and failure outcomes. At info level, only failures and state transitions
are logged; at debug level, all serial commands including raw responses are included.
+### API Documentation
+
+`frontend/api/openapi.yaml` is the single source of truth for the HTTP API.
+When adding/changing a route, edit it alongside `web_server.cpp` (the build
+fails if paths drift). Frontend types are generated from it; frontend-only
+types live in `frontend/src/types.ts`.
+
### Filesystem and Flash Wear
SPIFFS (not LittleFS). Buffer writes in RAM, never write to flash in a loop.
@@ -68,13 +77,17 @@ Verify frontend changes: `npm run check` + `npm run build` in `frontend/`.
Verify flash tool changes: `golangci-lint run ./...` + `go build` in `flash/`.
+Verify GitHub Actions workflow changes: install `actionlint` if missing, then run
+`actionlint` from the repository root.
+
### Frontend
```bash
cd frontend
npm run dev # Vite dev server with mock API
-npm run build # Lint + type check + build + embed into firmware header
+npm run build # Lint + duplicate check + type check + build + embed into firmware header
npm run check # Biome lint/format check
+npm run dupes # Duplicate code check via jscpd
npm run fix # Auto-fix safe issues
```
@@ -109,6 +122,9 @@ CSS frameworks, routing, or HTTP wrapper libraries.
- 4-space indent, double quotes, semicolons, 120-col (Biome)
- Named `interface`/`type` only, never inline object type literals
+- Reuse existing CSS utilities and component classes before adding new ones.
+ When two rules share a body, consolidate via a shared selector list, a
+ shared class, or a CSS custom property.
## Hardware
diff --git a/README.md b/README.md
index e0f4186..8abb4d7 100644
--- a/README.md
+++ b/README.md
@@ -1,28 +1,40 @@
-[](https://github.com/renjfk/OpenNeato/actions/workflows/ci.yml)
[](LICENSE)
-[](https://github.com/renjfk/OpenNeato/releases/latest)
-[](https://github.com/renjfk/OpenNeato/releases)
+[](https://github.com/Leicas/OpenNeato/releases/latest)
+[](https://github.com/renjfk/OpenNeato)
-# OpenNeato
+# OpenNeato — Home Assistant fork
-Open-source replacement for Neato's discontinued cloud and mobile app. An ESP32 bridge communicates with
-Botvac robots (D3-D7) over UART and exposes a local web UI over WiFi — no cloud, no app, no account required.
-
-> [!NOTE]
-> This is an early beta - things may break, rough edges are expected, and the API may change.
-> If you run into problems, a [Discussion](https://github.com/renjfk/OpenNeato/discussions)
-> or [issue](https://github.com/renjfk/OpenNeato/issues) is always welcome.
+**This is a [Leicas/OpenNeato](https://github.com/Leicas/OpenNeato) fork focused on the Home Assistant
+integration.** The HACS-installable integration under
+[`custom_components/openneato/`](custom_components/openneato/) is the reason this fork exists — it turns the
+upstream ESP32 bridge into a full HA device with vacuum / camera / sensor / switch / button entities, no YAML.
> [!IMPORTANT]
-> **Now in development:**
-[Guided Clean - zone cleaning, no-go lines, and map-based navigation](https://github.com/renjfk/OpenNeato/issues/68).
+> **If you want the standalone web UI**, use upstream [renjfk/OpenNeato](https://github.com/renjfk/OpenNeato)
+> directly — it has the original maintainer, the release cadence, and the live demo.
>
-> Select zones on a previously recorded map, draw no-go lines, and let the robot clean exactly where you want.
-> Follow the issue for progress updates and sub-task tracking.
+> **Use this fork if** you run Home Assistant and want the robot exposed as a first-class HA device. Firmware
+> changes in this fork are minor and exist only to support the integration (e.g. `/api/sensors`,
+> `/api/history` hardening, ntfy server/token/on-start extensions). The firmware, frontend, and flash tool
+> otherwise track upstream closely.
+
+The upstream project is an open-source replacement for Neato's discontinued cloud and mobile app: an ESP32
+bridge that talks to Botvac robots (D3-D7) over UART and exposes a local web UI over WiFi — no cloud, no app,
+no account required.
+
+> [!NOTE]
+> Both upstream and this fork are early beta. Rough edges are expected. For HA-integration bugs, open an
+> [issue here](https://github.com/Leicas/OpenNeato/issues); for firmware/frontend/flash-tool bugs, file
+> against [upstream](https://github.com/renjfk/OpenNeato/issues).
+
+> [!TIP]
+> Want to get a feel for the underlying web UI without hardware? Open the upstream
+> [live demo](https://openneato-demo.renjfk.com/). Demo states can be selected with `?scenario=...`; see
+> [mock scenarios](docs/mock-scenarios.md).
| Dashboard | Manual Drive | Cleaning History |
|:----------------------------------------:|:----------------------------------------------:|:------------------------------------------------------:|
@@ -32,13 +44,19 @@ Botvac robots (D3-D7) over UART and exposes a local web UI over WiFi — no clou
|:----------------------------------------:|:--------------------------------------:|:--------------------------------------:|
|  |  |  |
-## Motivation
+## About the underlying project (upstream)
+
+The sections below describe the upstream [renjfk/OpenNeato](https://github.com/renjfk/OpenNeato) project that
+this fork builds on — the ESP32 bridge firmware, standalone web UI, and flash tool. **None of this is unique
+to this fork**; it's reproduced here so HA users have full context on what's running behind the integration.
+If you only want the HA bits, you can skip ahead to the [Home Assistant section](#home-assistant-integration)
+below.
Neato shut down their cloud services and mobile app, leaving perfectly functional robot vacuums without remote
control or scheduling. OpenNeato brings them back to life with a small ESP32 board wired to the robot's debug
port, giving you a local web interface that works without any external dependencies.
-## Features
+### Upstream web UI features
- **Dashboard** with live robot status, battery level, cleaning state, WiFi signal, and storage usage
- **House and spot cleaning** with pause/resume/stop/dock controls that adapt to the current state
@@ -55,8 +73,8 @@ port, giving you a local web interface that works without any external dependenc
- **OTA firmware updates** from the browser with SHA-256 download verification (against published `checksums.txt`), MD5
transfer integrity, dual-partition layout with auto-rollback, and automatic new version notifications when a release
is available on GitHub
-- **Settings page** for hostname, timezone, motor presets, notification topics, UART pins, theme (dark/light/auto), and
- more
+- **Settings page** for hostname, timezone, motor presets, notification topics, UART pins, theme (dark/light/auto),
+ battery diagnostics (cycle count, voltage, temperature, and a "New Battery" calibration trigger), and more
- **Event logging** with configurable log levels (off/info/debug), compressed JSONL files on SPIFFS, browsable and
downloadable from the UI; optional remote syslog (UDP) for long-running diagnostics without flash wear; logging is
off by default
@@ -71,6 +89,90 @@ port, giving you a local web interface that works without any external dependenc
The frontend is a lightweight SPA that gets gzipped and embedded directly into the firmware binary, so a single
OTA update covers both firmware and UI. Mobile-friendly, dark theme by default.
+## Home Assistant Integration
+
+This fork ships a HACS-installable Home Assistant custom integration in
+[`custom_components/openneato/`](custom_components/openneato/). Once installed it discovers your OpenNeato bridge
+by IP/hostname and exposes the robot as a full HA device — no YAML, no extra add-ons.
+
+> [!NOTE]
+> The HA integration is the primary differentiator of this fork. The firmware, frontend, and flash tool track
+> upstream [`renjfk/OpenNeato`](https://github.com/renjfk/OpenNeato) closely. If you only want the standalone web
+> UI, use upstream directly.
+
+### Install via HACS
+
+1. In HACS, add this fork as a **custom repository**: `https://github.com/Leicas/OpenNeato` (category:
+ *Integration*).
+2. Search for **OpenNeato** in HACS and install.
+3. Restart Home Assistant.
+4. **Settings → Devices & Services → Add Integration → OpenNeato** and enter the bridge hostname or IP
+ (e.g. `neato.local` or `192.168.1.42`).
+
+The integration polls `/api/*` over your LAN every 5 seconds (`local_polling`). No cloud round-trip, no
+external dependencies. Requires firmware `1.0+`; battery diagnostics need firmware `0.13+` (upstream PR #121).
+
+### What you get
+
+A single device with the following entity groups:
+
+- **Vacuum** (`vacuum.openneato_`) — start/stop/pause/dock/locate/spot-clean, battery level, status,
+ fan speed presets (Eco/Auto/Intense), error reporting. Works with all the standard vacuum cards.
+- **Cameras** — `LIDAR map` (live 360° scan during cleaning) and `Cleaning replay` (animated GIF time-lapse
+ of the most recent completed session). Both are standard HA camera entities, compatible with
+ picture-entity, picture-glance, and vacuum-card.
+- **Sensors** — battery level/voltage/current/temperature, battery cycle count, cumulative cleaning time,
+ WiFi RSSI, free heap, storage used, uptime, motor RPMs, error code/message, plus "last clean" stats
+ (duration, area covered, distance, battery used, mode, end time) pulled from the on-device history.
+- **Binary sensors** — charging, external power, battery over-temp, battery failure, empty fuel, error
+ active, NTP synced, dustbin seated, left/right wheel lifted, dock contact.
+- **Switches** — eco mode, intense clean, bin-full detect, wall follower, schedule on/off, button-click
+ sounds, melodies, warnings, stealth LED, remote syslog, WiFi AP fallback, and per-event push
+ notifications (start/done/error/alert/docking).
+- **Text** — syslog server IP, ntfy topic, ntfy server, ntfy token (full push-notification config from HA).
+- **Numbers** — brush RPM, vacuum speed, side-brush power, stall threshold.
+- **Select** — navigation mode (Normal / Gentle / Deep / Quick).
+- **Buttons** — restart bridge, restart robot, shutdown robot, locate, clear errors, format filesystem
+ (diagnostic, disabled by default), **new battery** (resets fuel-gauge calibration after a physical pack
+ swap, disabled by default for safety).
+
+Every entity is translated via `strings.json`, and diagnostic-class entities (voltages, currents, raw
+sensor states) are tagged so they cluster cleanly under HA's Diagnostic section.
+
+### Notes for setup
+
+- **Camera entities and vacuum-card** — the `LIDAR map` camera self-manages a 2 s `/api/lidar` poll only
+ while the robot is actively cleaning, so it doesn't add load to the coordinator's 5 s cycle. When idle,
+ both cameras fall back to the most recent completed cleaning map.
+- **Coordinator resilience** — the integration tolerates a single hung endpoint without going into
+ "requires attention" state. State / charger / system are critical; anything else (errors, motors,
+ history) falls back to the last known value during transient ESP32 serial hangs.
+- **No Pillow declared dependency** — map rendering uses Pillow which already ships with HA Core, so the
+ integration's `manifest.json` keeps `"requirements": []`. Nothing extra to install.
+- **ntfy + custom servers** — point `ntfy_server` at a self-hosted instance and `ntfy_token` at a Bearer
+ token for authenticated push. Empty server defaults to `ntfy.sh`; empty token is unauthenticated.
+
+### Version history
+
+Full per-version notes live in [`custom_components/openneato/CHANGELOG.md`](custom_components/openneato/CHANGELOG.md).
+Highlights:
+
+- **1.11** — added `notify_on_start` and `ap_fallback_on_disconnect` switches; ntfy topic/server/token text
+ entities for full HA-side push config.
+- **1.10** — battery diagnostics (current, voltage, cycles, cumulative cleaning time) on top of firmware
+ PR #121, `New battery` calibration button, UTF-8-tolerant `/api/version` parsing.
+- **1.9** — fixed cameras stuck on the idle placeholder (`get_encoding()` crash on streamed bodies).
+- **1.6** — `Cleaning replay` camera (animated GIF time-lapse), `/api/history` corruption-tolerant parsing,
+ history filename validation + 2 MB response cap (LAN-MITM mitigation).
+- **1.3** — `LIDAR map` camera ported from the frontend renderer; self-managed polling.
+- **1.2** — last-clean stats sensors; dropped Pillow from declared deps.
+
+### Reporting integration bugs
+
+Use this fork's [issue tracker](https://github.com/Leicas/OpenNeato/issues) for anything that lives under
+`custom_components/openneato/`. For firmware, frontend, or flash-tool issues, upstream
+[renjfk/OpenNeato](https://github.com/renjfk/OpenNeato/issues) is the right place.
+
## Supported Robots
Neato Botvac D3 through D7. D8/D9/D10 are NOT supported (different board, password-locked serial port).
@@ -166,6 +268,10 @@ When creating issues, please follow our simple naming convention:
## Development
+For frontend development without hardware, use the mock API scenarios documented in
+[`docs/mock-scenarios.md`](docs/mock-scenarios.md). The same `?scenario=...` URLs work in local Vite dev and the
+Cloudflare demo.
+
### Release Process
Manual releases via opencode; see [RELEASE_PROCESS.md](RELEASE_PROCESS.md).
diff --git a/custom_components/openneato/CHANGELOG.md b/custom_components/openneato/CHANGELOG.md
index d20a60b..aa3181d 100644
--- a/custom_components/openneato/CHANGELOG.md
+++ b/custom_components/openneato/CHANGELOG.md
@@ -1,5 +1,61 @@
# Changelog
+## 1.11
+
+### Added
+- **`notify_on_start` switch** — surfaces fork PR #2's "notify on cleaning
+ started" setting (`ntfyOnStart`). Firmware + frontend already supported
+ the toggle; the matching HA entity was missing, so the switch row sat
+ among `notify_on_done`/`error`/`alert`/`docking` with no `start` peer.
+- **`ap_fallback_on_disconnect` switch** — upstream firmware #110 added
+ an `apFallbackOnDisconnect` setting that brings up the captive-portal
+ AP automatically when WiFi STA drops. Now configurable from HA.
+- **`ntfy_topic`, `ntfy_server`, `ntfy_token` text entities** — full ntfy
+ push-notification configuration is now editable from HA, no need to
+ open the device web UI. Empty topic disables notifications; empty
+ server defaults to `ntfy.sh`; empty token is unauthenticated.
+
+### Not done (deliberate deferral)
+- **Upstream About metadata not surfaced** — firmware commit bb7b541
+ added `name`, `repositoryUrl`, `license`, `model` fields to
+ `/api/firmware/version`. Threading `repository_url` through every
+ entity constructor for nine platforms (vacuum, sensor, switch,
+ binary_sensor, number, button, camera, select, text) just to set a
+ device-info attribute was too invasive for the value it adds, and the
+ `name` field already matches our hardcoded `manufacturer="OpenNeato"`.
+ A cleaner future approach is a dedicated diagnostic sensor that
+ exposes the full About payload as attributes.
+- **Zeroconf discovery not added** — `manifest.json` was eligible for a
+ `zeroconf` entry, but the firmware (`wifi_manager.cpp`) currently only
+ registers the generic `_http._tcp` service. Claiming that service for
+ the OpenNeato integration would match every random HTTP device on the
+ LAN. When the firmware advertises a dedicated service type (e.g.
+ `_openneato._tcp`), wire this up.
+
+## 1.10
+
+### Fixed
+- **Setup crash with new firmware (`utf-8 codec can't decode byte 0xab`)** —
+ Upstream firmware PR #121 added `smartBattery*` string fields to
+ `/api/version` that pass raw bytes from the robot's smart-battery
+ memory through the firmware's `jsonEscape()` (it only escapes bytes
+ below 0x20). When the pack reports a non-UTF-8 byte, aiohttp's
+ `response.json()` raised `UnicodeDecodeError` and the config entry
+ failed setup. Responses are now decoded with `errors="replace"`
+ before being parsed as JSON so one bad glyph can't take the
+ integration down.
+
+### Added
+- **Battery diagnostics from firmware PR #121** — three new diagnostic
+ sensors backed by the new `/api/analog` endpoint (battery voltage,
+ current, external voltage) and two backed by `/api/warranty`
+ (battery cycles, cumulative cleaning time). The existing
+ `Battery temperature` sensor was repointed from the now-removed
+ `/api/charger#battTempC` to `/api/analog#batteryTemperatureC`
+ (a finer-grained float in °C). A `New battery` button (disabled by
+ default) calls `POST /api/battery/new` to reset the fuel-gauge
+ calibration after a pack swap.
+
## 1.9
### Fixed
diff --git a/custom_components/openneato/api.py b/custom_components/openneato/api.py
index 05cfbd9..e0bf9c8 100644
--- a/custom_components/openneato/api.py
+++ b/custom_components/openneato/api.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
import logging
import re
from typing import Any
@@ -28,6 +29,19 @@ class OpenNeatoApiError(HomeAssistantError):
"""Error to indicate a non-connection API failure."""
+async def _read_json(response: aiohttp.ClientResponse) -> Any:
+ """Read a response body and parse it as JSON, tolerating stray non-UTF-8 bytes.
+
+ The firmware's jsonEscape() passes bytes >= 0x20 through verbatim, so the
+ /api/version response can contain raw bytes from the robot's smart-battery
+ memory (manufacturer name, serial, etc.) that aren't valid UTF-8. Replace
+ those bytes rather than raising — losing one glyph is better than failing
+ the whole config-entry setup.
+ """
+ raw = await response.read()
+ return json.loads(raw.decode("utf-8", errors="replace"))
+
+
class OpenNeatoApiClient:
"""Async HTTP client for OpenNeato."""
@@ -59,7 +73,7 @@ async def _get(self, path: str) -> dict[str, Any]:
path, response.status, response.content_type,
)
response.raise_for_status()
- return await response.json()
+ return await _read_json(response)
except aiohttp.ClientConnectionError as err:
_LOGGER.warning("Connection error on GET %s: %s", path, err)
raise OpenNeatoConnectionError(
@@ -92,7 +106,7 @@ async def _post(
response.raise_for_status()
content_type = response.content_type or ""
if "json" in content_type:
- return await response.json()
+ return await _read_json(response)
return await response.text()
except aiohttp.ClientConnectionError as err:
_LOGGER.warning("Connection error on POST %s: %s", path, err)
@@ -122,7 +136,7 @@ async def _put(self, path: str, json_data: dict[str, Any]) -> dict[str, Any]:
path, response.status, response.content_type,
)
response.raise_for_status()
- return await response.json()
+ return await _read_json(response)
except aiohttp.ClientConnectionError as err:
_LOGGER.warning("Connection error on PUT %s: %s", path, err)
raise OpenNeatoConnectionError(
@@ -149,6 +163,14 @@ async def get_charger(self) -> dict[str, Any]:
"""Get charger / battery information."""
return await self._get("/api/charger")
+ async def get_battery_analog(self) -> dict[str, Any]:
+ """Get analog battery readings (voltage, current, temperature)."""
+ return await self._get("/api/analog")
+
+ async def get_battery_warranty(self) -> dict[str, Any]:
+ """Get battery warranty data (cumulative cycles, runtime)."""
+ return await self._get("/api/warranty")
+
async def get_error(self) -> dict[str, Any]:
"""Get current error information."""
return await self._get("/api/error")
@@ -270,6 +292,10 @@ async def restart(self) -> dict[str, Any] | str:
"""Restart the robot controller."""
return await self._post("/api/system/restart")
+ async def new_battery(self) -> dict[str, Any] | str:
+ """Reset battery fuel-gauge calibration after physically replacing the pack."""
+ return await self._post("/api/battery/new")
+
async def format_fs(self) -> dict[str, Any] | str:
"""Format the filesystem."""
return await self._post("/api/system/format-fs")
diff --git a/custom_components/openneato/button.py b/custom_components/openneato/button.py
index f3425b3..1baa60e 100644
--- a/custom_components/openneato/button.py
+++ b/custom_components/openneato/button.py
@@ -81,6 +81,17 @@ class OpenNeatoButtonEntityDescription(ButtonEntityDescription):
entity_category=EntityCategory.DIAGNOSTIC,
press_fn=lambda api: api.clear_errors(),
),
+ OpenNeatoButtonEntityDescription(
+ # Resets the battery fuel-gauge calibration after a pack swap.
+ # Disabled by default — user must opt in to avoid accidental resets.
+ key="new_battery",
+ translation_key="new_battery",
+ name="New battery",
+ icon="mdi:battery-sync-outline",
+ entity_category=EntityCategory.CONFIG,
+ entity_registry_enabled_default=False,
+ press_fn=lambda api: api.new_battery(),
+ ),
)
diff --git a/custom_components/openneato/coordinator.py b/custom_components/openneato/coordinator.py
index fcf7e4a..bbf9ca7 100644
--- a/custom_components/openneato/coordinator.py
+++ b/custom_components/openneato/coordinator.py
@@ -41,12 +41,15 @@ async def _async_update_data(self) -> dict[str, Any]:
self.api.get_motors(),
self.api.get_history(),
self.api.get_sensors(),
+ self.api.get_battery_analog(),
+ self.api.get_battery_warranty(),
return_exceptions=True,
)
keys = (
"state", "charger", "error", "user_settings",
"system", "settings", "motors", "history", "sensors",
+ "analog", "warranty",
)
# Critical endpoints — if ALL of these fail we consider the robot
# unreachable. Non-critical endpoints (like /api/error, which can hang
diff --git a/custom_components/openneato/manifest.json b/custom_components/openneato/manifest.json
index 6c67297..e781d26 100644
--- a/custom_components/openneato/manifest.json
+++ b/custom_components/openneato/manifest.json
@@ -7,5 +7,5 @@
"documentation": "https://github.com/renjfk/OpenNeato",
"iot_class": "local_polling",
"requirements": [],
- "version": "1.9"
+ "version": "1.11"
}
diff --git a/custom_components/openneato/sensor.py b/custom_components/openneato/sensor.py
index 03a81c4..f50a5e9 100644
--- a/custom_components/openneato/sensor.py
+++ b/custom_components/openneato/sensor.py
@@ -15,6 +15,7 @@
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import (
EntityCategory,
+ UnitOfElectricCurrent,
UnitOfElectricPotential,
UnitOfLength,
UnitOfTemperature,
@@ -80,16 +81,17 @@ class OpenNeatoSensorEntityDescription(SensorEntityDescription):
entity_category=EntityCategory.DIAGNOSTIC,
),
OpenNeatoSensorEntityDescription(
+ # PR #121 moved battery temp out of /api/charger into the new
+ # /api/analog endpoint as a float in C (was an int with -1 sentinel).
key="charger_batt_temp",
translation_key="battery_temperature",
name="Battery temperature",
- section="charger",
- field="battTempC",
+ section="analog",
+ field="batteryTemperatureC",
device_class=SensorDeviceClass.TEMPERATURE,
native_unit_of_measurement=UnitOfTemperature.CELSIUS,
state_class=SensorStateClass.MEASUREMENT,
entity_category=EntityCategory.DIAGNOSTIC,
- value_fn=lambda v: v if v is not None and v >= 0 else None,
),
OpenNeatoSensorEntityDescription(
key="charger_discharge_mah",
@@ -113,6 +115,65 @@ class OpenNeatoSensorEntityDescription(SensorEntityDescription):
state_class=SensorStateClass.MEASUREMENT,
entity_category=EntityCategory.DIAGNOSTIC,
),
+ # ── Analog battery readings (PR #121, /api/analog) ──────────────────
+ OpenNeatoSensorEntityDescription(
+ key="analog_battery_voltage",
+ translation_key="analog_battery_voltage",
+ name="Battery voltage (analog)",
+ section="analog",
+ field="batteryVoltageV",
+ device_class=SensorDeviceClass.VOLTAGE,
+ native_unit_of_measurement=UnitOfElectricPotential.VOLT,
+ state_class=SensorStateClass.MEASUREMENT,
+ entity_category=EntityCategory.DIAGNOSTIC,
+ entity_registry_enabled_default=False,
+ ),
+ OpenNeatoSensorEntityDescription(
+ key="analog_battery_current",
+ translation_key="analog_battery_current",
+ name="Battery current",
+ section="analog",
+ field="batteryCurrentMA",
+ device_class=SensorDeviceClass.CURRENT,
+ native_unit_of_measurement=UnitOfElectricCurrent.MILLIAMPERE,
+ state_class=SensorStateClass.MEASUREMENT,
+ entity_category=EntityCategory.DIAGNOSTIC,
+ ),
+ OpenNeatoSensorEntityDescription(
+ key="analog_external_voltage",
+ translation_key="analog_external_voltage",
+ name="External voltage (analog)",
+ section="analog",
+ field="externalVoltageV",
+ device_class=SensorDeviceClass.VOLTAGE,
+ native_unit_of_measurement=UnitOfElectricPotential.VOLT,
+ state_class=SensorStateClass.MEASUREMENT,
+ entity_category=EntityCategory.DIAGNOSTIC,
+ entity_registry_enabled_default=False,
+ ),
+ # ── Battery warranty (PR #121, /api/warranty) ───────────────────────
+ OpenNeatoSensorEntityDescription(
+ key="warranty_battery_cycles",
+ translation_key="battery_cycles",
+ name="Battery cycles",
+ section="warranty",
+ field="cumulativeBatteryCycles",
+ icon="mdi:battery-sync",
+ state_class=SensorStateClass.TOTAL_INCREASING,
+ entity_category=EntityCategory.DIAGNOSTIC,
+ ),
+ OpenNeatoSensorEntityDescription(
+ key="warranty_cleaning_time",
+ translation_key="cumulative_cleaning_time",
+ name="Cumulative cleaning time",
+ section="warranty",
+ field="cumulativeCleaningTimeSeconds",
+ device_class=SensorDeviceClass.DURATION,
+ native_unit_of_measurement=UnitOfTime.SECONDS,
+ state_class=SensorStateClass.TOTAL_INCREASING,
+ icon="mdi:timer-sand",
+ entity_category=EntityCategory.DIAGNOSTIC,
+ ),
# ── Error ───────────────────────────────────────────────────────────
OpenNeatoSensorEntityDescription(
key="error_code",
diff --git a/custom_components/openneato/strings.json b/custom_components/openneato/strings.json
index 9f04d88..9684e3b 100644
--- a/custom_components/openneato/strings.json
+++ b/custom_components/openneato/strings.json
@@ -40,7 +40,12 @@
"last_clean_distance": { "name": "Last clean distance" },
"last_clean_battery_used": { "name": "Last clean battery used" },
"last_clean_mode": { "name": "Last clean mode" },
- "last_clean_ended": { "name": "Last clean ended" }
+ "last_clean_ended": { "name": "Last clean ended" },
+ "analog_battery_voltage": { "name": "Battery voltage (analog)" },
+ "analog_battery_current": { "name": "Battery current" },
+ "analog_external_voltage": { "name": "External voltage (analog)" },
+ "battery_cycles": { "name": "Battery cycles" },
+ "cumulative_cleaning_time": { "name": "Cumulative cleaning time" }
},
"binary_sensor": {
"charging": { "name": "Charging" },
@@ -66,10 +71,12 @@
"warning_sounds": { "name": "Warning sounds" },
"stealth_led": { "name": "Stealth LED" },
"notifications_enabled": { "name": "Notifications" },
+ "notify_on_start": { "name": "Notify on clean start" },
"notify_on_done": { "name": "Notify on clean done" },
"notify_on_error": { "name": "Notify on error" },
"notify_on_alert": { "name": "Notify on alert" },
"notify_on_docking": { "name": "Notify on docking" },
+ "ap_fallback_on_disconnect": { "name": "WiFi AP fallback" },
"remote_syslog": { "name": "Remote syslog" }
},
"button": {
@@ -78,7 +85,8 @@
"robot_restart": { "name": "Restart robot" },
"robot_shutdown": { "name": "Shutdown robot" },
"locate": { "name": "Locate robot" },
- "clear_errors": { "name": "Clear errors" }
+ "clear_errors": { "name": "Clear errors" },
+ "new_battery": { "name": "New battery" }
},
"number": {
"brush_rpm_setting": { "name": "Brush RPM" },
@@ -94,7 +102,10 @@
"navigation_mode": { "name": "Navigation mode" }
},
"text": {
- "syslog_ip": { "name": "Syslog server IP" }
+ "syslog_ip": { "name": "Syslog server IP" },
+ "ntfy_topic": { "name": "Notification topic" },
+ "ntfy_server": { "name": "Notification server" },
+ "ntfy_token": { "name": "Notification token" }
}
}
}
diff --git a/custom_components/openneato/switch.py b/custom_components/openneato/switch.py
index 5066bf1..16b4fce 100644
--- a/custom_components/openneato/switch.py
+++ b/custom_components/openneato/switch.py
@@ -131,6 +131,16 @@ class OpenNeatoSwitchEntityDescription(SwitchEntityDescription):
icon="mdi:bell",
entity_category=EntityCategory.CONFIG,
),
+ OpenNeatoSwitchEntityDescription(
+ key="notify_on_start",
+ translation_key="notify_on_start",
+ name="Notify on clean start",
+ section="settings",
+ field="ntfyOnStart",
+ settings_field="ntfyOnStart",
+ icon="mdi:bell-outline",
+ entity_category=EntityCategory.CONFIG,
+ ),
OpenNeatoSwitchEntityDescription(
key="notify_on_done",
translation_key="notify_on_done",
@@ -171,6 +181,16 @@ class OpenNeatoSwitchEntityDescription(SwitchEntityDescription):
icon="mdi:bell-plus",
entity_category=EntityCategory.CONFIG,
),
+ OpenNeatoSwitchEntityDescription(
+ key="ap_fallback_on_disconnect",
+ translation_key="ap_fallback_on_disconnect",
+ name="WiFi AP fallback",
+ section="settings",
+ field="apFallbackOnDisconnect",
+ settings_field="apFallbackOnDisconnect",
+ icon="mdi:wifi-strength-alert-outline",
+ entity_category=EntityCategory.CONFIG,
+ ),
OpenNeatoSwitchEntityDescription(
key="remote_syslog",
translation_key="remote_syslog",
diff --git a/custom_components/openneato/text.py b/custom_components/openneato/text.py
index a1fabfe..e0e252a 100644
--- a/custom_components/openneato/text.py
+++ b/custom_components/openneato/text.py
@@ -40,6 +40,33 @@ class OpenNeatoTextEntityDescription(TextEntityDescription):
native_max=15,
pattern=r"^(\d{1,3}\.){3}\d{1,3}$|^$",
),
+ OpenNeatoTextEntityDescription(
+ key="ntfy_topic",
+ translation_key="ntfy_topic",
+ name="Notification topic",
+ section="settings",
+ settings_field="ntfyTopic",
+ icon="mdi:bell-cog",
+ entity_category=EntityCategory.CONFIG,
+ ),
+ OpenNeatoTextEntityDescription(
+ key="ntfy_server",
+ translation_key="ntfy_server",
+ name="Notification server",
+ section="settings",
+ settings_field="ntfyServer",
+ icon="mdi:server",
+ entity_category=EntityCategory.CONFIG,
+ ),
+ OpenNeatoTextEntityDescription(
+ key="ntfy_token",
+ translation_key="ntfy_token",
+ name="Notification token",
+ section="settings",
+ settings_field="ntfyToken",
+ icon="mdi:key",
+ entity_category=EntityCategory.CONFIG,
+ ),
)
diff --git a/docs/mock-scenarios.md b/docs/mock-scenarios.md
new file mode 100644
index 0000000..4c985a6
--- /dev/null
+++ b/docs/mock-scenarios.md
@@ -0,0 +1,117 @@
+# Mock scenarios
+
+The frontend mock API supports scenario flags for testing robot states and failures without hardware. Scenarios are
+selected with the `scenario` query parameter and can be combined with `|`.
+
+Useful combinations:
+
+- `ok` - robot idle, online, battery 85%
+- `err|fa` - robot error, brush stuck, plus action faults
+- `low|fl|fs` - low battery plus log faults and settings fault
+- `man|llq` - manual clean plus low LIDAR quality
+
+When writing URLs, encode `|` as `%7C` if your browser or shell does not preserve it literally.
+
+## Local development
+
+Start the Vite dev server:
+
+```bash
+cd frontend
+npm run dev
+```
+
+Open the UI with a scenario:
+
+```text
+http://localhost:5173/?scenario=err%7Cfa
+http://localhost:5173/?scenario=low%7Cfl%7Cfs#/settings
+```
+
+The selected scenario is stored in the `openneato_scenario` cookie so later SPA API calls keep using it. Switch back to
+the normal state with:
+
+```text
+http://localhost:5173/?scenario=ok
+```
+
+## Demo site
+
+The Cloudflare demo uses the same mock API and the same query parameter:
+
+```text
+https://openneato-demo.renjfk.com/?scenario=err%7Cfa
+https://openneato-demo.renjfk.com/?scenario=man%7Cllq#/manual
+```
+
+The demo persists the selected scenario in the same `openneato_scenario` cookie. Firmware and history uploads are
+blocked server-side in demo mode.
+
+## Scenario keys
+
+Robot state:
+
+- `ok` - idle, online, battery 85%
+- `off` - device unreachable
+- `ident` - identifying robot during boot
+- `unsup` - unsupported robot model
+- `upd` - firmware v0.9, triggers update banner
+- `cls` - house cleaning
+- `spt` - spot cleaning
+- `dock` - docking, return to base
+- `rchg` - mid-clean recharge, on dock and charging
+- `chg` - charging, 62%
+- `ch2` - charging, 25%
+- `ful` - full, on dock
+- `mid` - battery 45%
+- `low` - battery 12%
+- `ded` - battery 0%
+- `err` - brush stuck error
+- `alrt` - alert only, brush change
+
+Manual clean, combinable with each other or fault scenarios:
+
+- `man` - manual mode active, no safety issues
+- `mlf` - manual mode active, robot lifted
+- `mbf` - manual mode active, front-left bumper contact
+- `mbs` - manual mode active, side-right bumper contact
+- `msf` - manual mode active, forward stall, reverse to clear
+- `msr` - manual mode active, rear stall, move forward to clear
+
+LIDAR quality, combinable with any state:
+
+- `llq` - low scan quality, fewer than 90 valid points
+- `lsl` - slow LDS rotation, 2.8 Hz
+- `lno` - LIDAR unavailable, `GET /api/lidar` returns an error
+
+Fault injection, combinable with any state:
+
+- `fa` - action faults, clean house, spot, stop, and return operations fail
+- `fs` - settings fault, NVS write error
+- `flr` - log read fault, list and content fail
+- `fld` - log delete fault, delete single and delete all fail
+- `fl` - log reads and deletes fail
+- `fps` - `/api/state` polling fails
+- `fpc` - `/api/charger` polling fails
+- `fpe` - `/api/error` polling fails
+- `fp` - all polling endpoints fail, state, charger, and error
+- `fhc` - history corruption, injects corrupted pose lines in session data
+- `fhl` - history list corruption, malformed JSON in `/api/history` response, triggers recovery panel
+- `fal` - all major faults combined
+
+WiFi:
+
+- `wap` - fallback AP active, STA disconnected and fallback enabled
+- `wnc` - no saved credentials, first boot path, AP always on
+- `wfo` - fallback AP setting off, combine with `wap` to test off plus disconnected
+- `fws` - scan fault, `/api/wifi/scan` returns 500
+- `fwn` - empty scan, `/api/wifi/scan` returns no networks
+- `fwc` - connect fault, `/api/wifi/connect` rejects with auth error
+
+## Examples
+
+- `err|fa` - robot error plus action failures
+- `low|fl|fs` - low battery, log failures, and settings failure
+- `man|llq` - manual mode with degraded LIDAR
+- `wap|wfo` - disconnected station with fallback AP setting disabled
+- `upd|fhl` - update banner plus corrupted history list recovery state
diff --git a/docs/user-guide.md b/docs/user-guide.md
index 8040331..b8d8d42 100644
--- a/docs/user-guide.md
+++ b/docs/user-guide.md
@@ -6,7 +6,6 @@ Everything you need to set up, configure, and troubleshoot OpenNeato.
- [Hardware Setup](#hardware-setup)
- [What You Need](#what-you-need)
- - [Opening the Robot](#opening-the-robot)
- [Debug Port Pinout](#debug-port-pinout)
- [Wiring](#wiring)
- [Flashing Firmware](#flashing-firmware)
@@ -14,16 +13,21 @@ Everything you need to set up, configure, and troubleshoot OpenNeato.
- [Basic Usage](#basic-usage)
- [What Happens Under the Hood](#what-happens-under-the-hood)
- [Command Reference](#command-reference)
+ - [Manual Flashing with esptool](#manual-flashing-with-esptool)
- [Troubleshooting Flash Issues](#troubleshooting-flash-issues)
- [First-Time WiFi Setup](#first-time-wifi-setup)
- - [Serial Monitor](#serial-monitor)
+ - [Option A: Fallback Access Point (no serial cable)](#option-a-fallback-access-point-no-serial-cable)
+ - [Option B: Serial Monitor](#option-b-serial-monitor)
- [WiFi Configuration Menu](#wifi-configuration-menu)
- [Verifying the Connection](#verifying-the-connection)
- [Quick Commands](#quick-commands)
+ - [Reconfiguring WiFi Later](#reconfiguring-wifi-later)
- [Troubleshooting](#troubleshooting)
- [Enabling Logging](#enabling-logging)
- [Collecting Logs](#collecting-logs)
+ - [Robot Stuck Starting a House Clean](#robot-stuck-starting-a-house-clean)
- [Downloading Cleaning Maps](#downloading-cleaning-maps)
+ - [Recovering Corrupted Cleaning History](#recovering-corrupted-cleaning-history)
- [Factory Reset](#factory-reset)
- [Reporting an Issue](#reporting-an-issue)
- [Multiple Robots](#multiple-robots)
@@ -37,12 +41,10 @@ Everything you need to set up, configure, and troubleshoot OpenNeato.
## Hardware Setup
> [!NOTE]
-> Hardware assembly is not the primary focus of this project. There are already comprehensive
-> teardown and wiring guides available — in particular
-> [Philip2809/neato-brainslug](https://github.com/Philip2809/neato-brainslug) which covers
-> the D3-D7 debug port in detail. This section shares my personal experience with minimal
-> photos and a bill of materials. If there's community interest I'll expand it further —
-> I'll be opening my own Neato D7 soon to replace the LIDAR O-ring.
+> Hardware assembly is not the primary focus of this project. For a comprehensive teardown
+> guide covering how to open the robot and reach the debug port, see
+> [Philip2809/neato-brainslug](https://github.com/Philip2809/neato-brainslug). This section
+> covers the parts list, debug port pinout, and wiring.
### What You Need
@@ -75,11 +77,6 @@ solder the JST connector wires directly to the board pads for the cleanest resul
The ESP32 is powered directly from the robot's 3.3V debug port — no separate USB power
supply needed during normal operation.
-### Opening the Robot
-
-
-
-
### Debug Port Pinout
The debug port connector on Botvac D3-D7 has four pins (left to right when looking at the
@@ -102,7 +99,7 @@ These are the **robot's** RX/TX labels, so you cross-connect to the ESP32:
The default TX/RX GPIOs depend on the chip (ESP32-C3: GPIO 3/4, ESP32-S3: GPIO 17/18,
original ESP32: GPIO 17/16) but are fully configurable from the web UI in
-**Settings -> Robot -> UART Pins** — so wire whichever GPIOs are convenient and update the
+**Settings -> Device -> UART Pins** — so wire whichever GPIOs are convenient and update the
setting to match.
### Wiring
@@ -155,6 +152,11 @@ These are standalone binaries — no extraction needed. On macOS/Linux you may n
> xattr -d com.apple.quarantine ~/Downloads/openneato-flash_Darwin_arm64
> ```
+> [!IMPORTANT]
+> macOS 11 (Big Sur) is the minimum supported version. This is the floor for the Go toolchain
+> the binary is built with — older macOS versions (Catalina, Mojave, and earlier) will fail to
+> load the binary or crash on first network request.
+
> [!WARNING]
> The flash tool has been primarily tested on macOS. Linux and Windows builds are provided but
> not battle-tested — if you run into issues on those platforms, please
@@ -215,6 +217,108 @@ tool refuses to flash.
> directory as the `.tar.gz` file. The tool verifies SHA-256 before extracting and will
> refuse to flash if the checksums file is missing or the hash doesn't match.
+### Manual Flashing with esptool
+
+If `openneato-flash` does not run on your system, you can flash the same firmware pack manually
+with Espressif's prebuilt `esptool` binary. This path does not require Python and uses the same
+files, offsets, baud rate, compression, and reset behavior as `openneato-flash`.
+
+1. Download the matching prebuilt `esptool` binary for your platform from
+ [Espressif's esptool releases](https://github.com/espressif/esptool/releases), then extract
+ it. To use the same version as `openneato-flash`, check `ESPTOOL_VERSION` in
+ [`.goreleaser.yml`](../.goreleaser.yml).
+
+2. Plug in the ESP32. If only one ESP device is connected, you can let `esptool` find the serial
+ port automatically by omitting `-p`.
+
+ If you have multiple serial devices connected, pass the port explicitly with `-p`. Common port
+ names are `/dev/cu.usbmodem*` on macOS, `/dev/ttyUSB*` or `/dev/ttyACM*` on Linux, and `COM3`
+ or similar on Windows.
+
+3. Detect the chip name:
+
+ ```bash
+ ./esptool chip-id
+ ```
+
+ Look for a line like `Detecting chip type... ESP32-C3`, then use the lowercase chip name in
+ the firmware filename: `esp32-c3`, `esp32-s3`, or `esp32`.
+
+4. Download the full firmware pack and `checksums.txt` from the
+ [OpenNeato Releases](https://github.com/renjfk/OpenNeato/releases) page.
+
+ For example, for an ESP32-C3 on the latest release:
+
+ ```bash
+ curl -LO https://github.com/renjfk/OpenNeato/releases/latest/download/openneato-esp32-c3-full.tar.gz
+ curl -LO https://github.com/renjfk/OpenNeato/releases/latest/download/checksums.txt
+ ```
+
+5. Verify the firmware pack checksum before extracting it.
+
+ macOS:
+
+ ```bash
+ shasum -a 256 -c checksums.txt --ignore-missing
+ ```
+
+ Linux:
+
+ ```bash
+ sha256sum -c checksums.txt --ignore-missing
+ ```
+
+ Windows PowerShell:
+
+ ```powershell
+ Select-String "openneato-esp32-c3-full.tar.gz" checksums.txt
+ Get-FileHash .\openneato-esp32-c3-full.tar.gz -Algorithm SHA256
+ ```
+
+ The hash printed by PowerShell must match the hash from `checksums.txt`.
+
+6. Extract the firmware pack and open `offsets.json`.
+
+ macOS/Linux:
+
+ ```bash
+ tar -xzf openneato-esp32-c3-full.tar.gz
+ ```
+
+ Windows PowerShell:
+
+ ```powershell
+ tar -xzf .\openneato-esp32-c3-full.tar.gz
+ ```
+
+7. Flash all images using the offsets from `offsets.json`.
+
+ macOS/Linux:
+
+ ```bash
+ ./esptool -b 921600 --before default-reset --after hard-reset write-flash -z \
+ BOOTLOADER_OFFSET bootloader.bin \
+ PARTITIONS_OFFSET partitions.bin \
+ OTADATA_OFFSET boot_app0.bin \
+ APP_OFFSET firmware.bin
+ ```
+
+ Windows PowerShell:
+
+ ```powershell
+ .\esptool.exe -b 921600 --before default-reset --after hard-reset write-flash -z `
+ BOOTLOADER_OFFSET .\bootloader.bin `
+ PARTITIONS_OFFSET .\partitions.bin `
+ OTADATA_OFFSET .\boot_app0.bin `
+ APP_OFFSET .\firmware.bin
+ ```
+
+ Replace the `*_OFFSET` placeholders with the values from `offsets.json`, and replace the
+ firmware pack name to match your chip. If auto-detection picks the wrong serial device or you
+ have multiple ESP devices connected, add `-p ` before `-b 921600`. Do not hard-code the
+ flash offsets from another source; `offsets.json` is generated with the release and is the
+ source of truth for the manual command.
+
### Troubleshooting Flash Issues
**"No USB serial ports found"** — Make sure the ESP32 is plugged in and your OS recognizes
@@ -234,23 +338,48 @@ Windows, close any other serial monitor that might have the port open.
## First-Time WiFi Setup
-After flashing, the tool opens a serial monitor where you'll configure WiFi.
+After flashing, the device has no saved WiFi credentials and won't be on your network yet.
+You have two ways to provision it: a browser via the fallback access point (no serial cable
+needed), or the serial menu that the flash tool opens for you.
+
+### Option A: Fallback Access Point (no serial cable)
+
+When the device has no saved credentials, it broadcasts an open WiFi network so you can
+configure it from any phone or laptop browser. This works even after you've unplugged the
+USB cable and tucked the ESP32 inside the robot.
-### Serial Monitor
+1. From your phone or laptop, connect to the WiFi network named **`neato-ap`** (or
+ `-ap` if you've changed the hostname). It's an open network , no password.
+2. Open a browser and go to `http://192.168.4.1`. You'll land on the OpenNeato dashboard.
+3. Open **Settings -> WiFi**, tap **Scan**, pick your home network from the dropdown, and
+ enter the password.
+4. After confirming, the device joins your home network and reboots. The `neato-ap` network
+ disappears at that point , reconnect your phone/laptop to your home WiFi to keep using
+ the web UI at `http://neato.local` (or the IP shown on the dashboard).
+
+> [!NOTE]
+> The fallback AP is unencrypted because there's no way to display a password to a user
+> who hasn't set one up yet. It only runs while the device has no saved credentials or
+> cannot reach your home network. Once connected, it shuts down automatically.
+
+### Option B: Serial Monitor
The serial monitor connects at 115,200 baud and shows the ESP32's boot output. You'll see
-the boot banner:
+the boot banner. With no credentials saved, the fallback AP comes up automatically and the
+banner reflects that:
```
========================================
OpenNeato v0.1
========================================
- WiFi: not configured
+ WiFi: AP mode, connect to neato-ap and open http://192.168.4.1
Press 'm' for menu, 's' for status
========================================
```
-If WiFi is not configured, the configuration menu appears automatically.
+You can finish provisioning either by joining `neato-ap` from a browser (Option A above) or
+by pressing `m` to open the WiFi configuration menu over serial , both end up in the same
+place.
### WiFi Configuration Menu
@@ -311,6 +440,21 @@ Once connected, you can type single-key commands in the serial monitor at any ti
| `m` | Open WiFi configuration menu |
| `s` | Print WiFi status (SSID, IP, MAC, RSSI) |
+### Reconfiguring WiFi Later
+
+Once the device is on your home network you can change networks from the web UI directly:
+**Settings -> WiFi -> Scan**, pick a new network, enter the password.
+
+If your home network goes down or you've moved house, the bridge falls back to the
+`-ap` access point automatically so you can re-provision it from a browser without
+opening up the robot. This behavior is controlled by the **Fallback AP on disconnect** toggle
+in the WiFi section , it's on by default. If you turn it off, recovery from a broken WiFi
+config requires the serial menu.
+
+To wipe credentials entirely and force the device back into first-time setup mode (always-on
+AP, no auto-reconnect), use **Settings -> WiFi -> Forget current network**. The device will
+broadcast `-ap` until you provision a new network.
+
---
## Troubleshooting
@@ -328,6 +472,11 @@ Go to **Settings -> Diagnostics** and set **Log Level**:
- **Debug** — everything in Info plus all serial commands with raw responses. Auto-reverts to
off after 10 minutes.
+The Diagnostics section also shows a **Battery diagnostics** card with live readings from the
+robot: charge percentage, voltage, current, temperature, cycle count, and chemistry details.
+If you replace the battery, use the **New Battery** button to reset the fuel gauge calibration
+(only after physically installing the new pack).
+
For long-running diagnostics (e.g. catching an intermittent error that only appears after
hours of idling), enable **Remote Syslog** in the same section. This sends all log output
over UDP (port 514) to a syslog receiver on your network instead of writing to flash. The
@@ -355,6 +504,41 @@ To download all logs: use the individual download buttons for each file you need
reporting an issue, include at least the `current.jsonl` and any archived log files from the
time the problem occurred.
+### Robot Stuck Starting a House Clean
+
+Some Botvac D7 robots can get stuck while starting a house clean with a robot notice like
+"Failed to load persistent map". Debug logs usually show `UI_ALERT_PM_LOAD_FAIL` (`234`) and
+the robot remains in `UIMGR_STATE_STARTHOUSECLEANING` while the robot state stays
+`ST_C_Standby`. This appears to be a robot firmware-side state rather than an OpenNeato
+scheduler failure. See [issue #24](https://github.com/renjfk/OpenNeato/issues/24) for the
+original field reports that led to documenting this recovery path.
+
+Try the recovery steps in this order:
+
+1. Open **Settings -> Diagnostics** and use **Clear Robot Errors**, then try starting a clean
+ again.
+2. If it remains stuck, open **Settings -> Robot** and use **Restart Robot**. This is equivalent
+ to a robot power cycle and has been the most reliable software recovery in reports so far.
+3. If the issue keeps returning, enable **Debug** logging or **Remote Syslog** before the next
+ scheduled run and save the logs when it happens.
+4. As an advanced hardware recovery step, fully remove power from the robot by disconnecting the
+ battery for a few minutes, then reconnect it. Some users reported the issue stopped after
+ maintenance that included a battery disconnect, cleaning sensors, clearing robot logs, and
+ removing debris from the brush or motor area.
+
+> [!CAUTION]
+> Disconnecting the robot battery requires opening the robot. Only do this if you are comfortable
+> with hardware disassembly, and avoid pulling on wires or shorting the battery connector.
+
+When reporting this issue, include the robot software version from:
+
+```bash
+curl -X POST 'http://neato.local/api/serial?cmd=GetVersion'
+```
+
+In known reports, affected robots were running Neato software `4.5.3.189`, but this is not yet a
+confirmed root cause.
+
### Downloading Cleaning Maps
If a cleaning session produced unexpected results (missed areas, strange paths, early
@@ -362,7 +546,36 @@ termination), download the cleaning history session from **History**. Each sessi
the robot's recorded path rendered as a coverage map, along with stats like duration, distance,
area covered, and battery usage.
-
+### Recovering Corrupted Cleaning History
+
+If the History page shows a "Cleaning history is corrupted" message, one of the stored sessions
+has malformed data and is preventing the list from loading. This can happen if a cleaning was
+interrupted by a power loss or unexpected reset. The following steps let you find and remove
+the bad session(s) without losing the rest. Replace `YOUR_ROBOT` with your bridge's hostname
+or IP address.
+
+1. **Fetch the session list** and inspect it visually:
+
+ ```sh
+ curl "http://YOUR_ROBOT/api/history"
+ ```
+
+ Paste the response into a JSON validator (for example
+ [jsonlint.com](https://jsonlint.com) or [jsonformatter.org](https://jsonformatter.org)).
+ The validator will flag the position of the malformed entry. Note the `name` field of the
+ bad session (e.g. `1776667071.jsonl.hs`).
+
+2. **Delete the bad session**:
+
+ ```sh
+ curl -X DELETE "http://YOUR_ROBOT/api/history/"
+ ```
+
+3. **Reload the History page**. If multiple sessions are corrupted, repeat steps 1-2 until the
+ list loads.
+
+If you'd rather not investigate, the History page also offers a **Delete all history** button
+that wipes every session in one go and restores the list view.
### Factory Reset
@@ -481,9 +694,9 @@ A few typical maintenance tasks:
```bash
# New battery installed (resets fuel gauge calibration, requires TestMode)
-curl -X POST 'http://neato.local/api/testmode?enable=1'
+curl -X POST 'http://neato.local/api/serial?cmd=TestMode%20On'
curl -X POST 'http://neato.local/api/serial?cmd=NewBattery'
-curl -X POST 'http://neato.local/api/testmode?enable=0'
+curl -X POST 'http://neato.local/api/serial?cmd=TestMode%20Off'
# Reset robot user settings to factory defaults
curl -X POST 'http://neato.local/api/serial?cmd=SetUserSettings%20Reset'
diff --git a/firmware/src/cleaning_history.cpp b/firmware/src/cleaning_history.cpp
index b68e7ed..25477a3 100644
--- a/firmware/src/cleaning_history.cpp
+++ b/firmware/src/cleaning_history.cpp
@@ -1,1299 +1,1299 @@
-#include "cleaning_history.h"
-#include "json_fields.h"
-#include "neato_serial.h"
-#include "system_manager.h"
-#include
-#include
-
-// Heatshrink decompression can drop a byte that merges two JSONL lines into
-// one, so substring matching isn't enough — validate the braces balance and
-// nothing trails the top-level object before embedding into /api/history.
-static bool isValidMetaLine(const String& line, const char* expectedTypePrefix) {
- if (line.length() < 2 || line[0] != '{' || line[line.length() - 1] != '}')
- return false;
- if (line.indexOf(expectedTypePrefix) < 0)
- return false;
- int depth = 0;
- bool inString = false;
- bool escape = false;
- for (size_t i = 0; i < line.length(); i++) {
- char c = line[i];
- if (escape) {
- escape = false;
- continue;
- }
- if (inString) {
- if (c == '\\')
- escape = true;
- else if (c == '"')
- inString = false;
- continue;
- }
- if (c == '"')
- inString = true;
- else if (c == '{')
- depth++;
- else if (c == '}') {
- depth--;
- if (depth < 0)
- return false;
- // Reject trailing content after the top-level object closes
- if (depth == 0 && i != line.length() - 1)
- return false;
- }
- }
- return depth == 0 && !inString;
-}
-
-CleaningHistory::CleaningHistory(NeatoSerial& neato, DataLogger& logger, SystemManager& sysMgr) :
- LoopTask(HISTORY_INTERVAL_IDLE_MS), neato(neato), dataLogger(logger), systemManager(sysMgr) {
- TaskRegistry::add(this);
-}
-
-void CleaningHistory::notifyCleanStart() {
- if (collecting)
- return; // Already recording
- setInterval(HISTORY_INTERVAL_ACTIVE_MS);
-}
-
-void CleaningHistory::tick() {
- // Run incremental compression when a session just finished
- if (compressing) {
- if (compressStep()) {
- // Compression done - remove raw source and cache metadata
- compressSrc.close();
- compressDst.close();
- SPIFFS.remove(compressSrcPath);
- LOG("HIST", "Compression done: %s", compressDstPath.c_str());
-
- // Cache session/summary from memory (avoids decompressing on next list request)
- String hsName = compressDstPath;
- int lastSlash = hsName.lastIndexOf('/');
- if (lastSlash >= 0)
- hsName = hsName.substring(lastSlash + 1);
- metaCache[hsName] = {pendingSessionJson, pendingSummaryJson};
- pendingSessionJson = "";
- pendingSummaryJson = "";
-
- compressing = false;
- setInterval(HISTORY_INTERVAL_IDLE_MS);
- }
- return;
- }
-
- if (fetchPending)
- return;
-
- if (collecting) {
- // Periodically flush buffered pose snapshots to disk
- if (!writeBuffer.empty() && millis() - lastFlushMs >= HISTORY_FLUSH_INTERVAL_MS) {
- flushWriteBuffer();
- }
- collectSnapshot();
- } else {
- checkState();
- enforceLimits();
- }
-}
-
-// -- State watching (idle mode) ----------------------------------------------
-
-void CleaningHistory::checkState() {
- fetchPending = true;
- neato.getState([this](bool ok, const RobotState& state) {
- fetchPending = false;
- if (!ok)
- return;
-
- bool wasCleaning = isCleaningState(prevUiState);
- bool nowCleaning = isCleaningState(state.uiState);
-
- // CLEANINGSUSPENDED + Charging_Cleaning means the robot is on the dock
- // recharging mid-clean and will resume automatically — treat as active
- bool isMidCleanRecharge = isSuspendedState(state.uiState) && state.robotState.indexOf("Charging_Cleaning") >= 0;
-
- // First poll after boot: if the robot is idle, finalize any orphan sessions
- // left by a previous crash/reboot so they don't get merged into the next clean
- if (prevUiState.isEmpty() && !nowCleaning && !isMidCleanRecharge && !recoveryAttempted) {
- recoveryAttempted = true;
- finalizeOrphanSessions();
- }
-
- if (!wasCleaning && (nowCleaning || isMidCleanRecharge) && !recoverCollection(state.uiState)) {
- startCollection(state.uiState);
- }
-
- prevUiState = state.uiState;
- });
-}
-
-bool CleaningHistory::isCleaningState(const String& uiState) {
- return uiState.indexOf("HOUSECLEANINGRUNNING") >= 0 || uiState.indexOf("HOUSECLEANINGPAUSED") >= 0 ||
- uiState.indexOf("SPOTCLEANINGRUNNING") >= 0 || uiState.indexOf("SPOTCLEANINGPAUSED") >= 0 ||
- uiState.indexOf("MANUALCLEANING") >= 0;
-}
-
-bool CleaningHistory::isPausedState(const String& uiState) {
- return uiState.indexOf("CLEANINGPAUSED") >= 0;
-}
-
-bool CleaningHistory::isDockingState(const String& uiState) {
- return uiState.indexOf("DOCKING") >= 0;
-}
-
-bool CleaningHistory::isSuspendedState(const String& uiState) {
- return uiState.indexOf("CLEANINGSUSPENDED") >= 0;
-}
-
-String CleaningHistory::cleanModeFromState(const String& uiState) {
- if (uiState.indexOf("HOUSECLEANING") >= 0)
- return "house";
- if (uiState.indexOf("SPOTCLEANING") >= 0)
- return "spot";
- if (uiState.indexOf("MANUALCLEANING") >= 0)
- return "manual";
- return "unknown";
-}
-
-void CleaningHistory::resetSession() {
- snapshotCount = 0;
- rechargeCount = 0;
- totalDistance = 0.0f;
- totalRotation = 0.0f;
- maxDistFromOrigin = 0.0f;
- errorsDuringClean = 0;
- prevHadError = false;
- hasPrevPose = false;
- prevX = 0.0f;
- prevY = 0.0f;
- prevTheta = 0.0f;
- originX = 0.0f;
- originY = 0.0f;
- visitedCells.clear();
- cleanMode = "";
- sessionStartTime = 0;
- batteryStart = -1;
- recharging = false;
- pendingSessionJson = "";
- pendingSummaryJson = "";
-}
-
-void CleaningHistory::startCollection(const String& uiState) {
- collecting = true;
- resetSession();
-
- cleanMode = cleanModeFromState(uiState);
- sessionStartTime = systemManager.now();
-
- // Create session file: /history/.jsonl
- activeFilePath = String(HISTORY_DIR) + "/" + String(static_cast(sessionStartTime)) + ".jsonl";
- activeFile = SPIFFS.open(activeFilePath, FILE_WRITE);
- if (!activeFile) {
- LOG("HIST", "Failed to create session file: %s", activeFilePath.c_str());
- collecting = false;
- return;
- }
-
- // Fetch battery level for session metadata, then write header
- neato.getCharger([this](bool ok, const ChargerData& charger) {
- if (ok) {
- batteryStart = charger.fuelPercent;
- }
- writeSessionHeader();
- });
-
- setInterval(HISTORY_INTERVAL_ACTIVE_MS);
- LOG("HIST", "Collection started (mode: %s, file: %s)", cleanMode.c_str(), activeFilePath.c_str());
- dataLogger.logGenericEvent("history_start", {{"mode", cleanMode, FIELD_STRING}});
-}
-
-void CleaningHistory::stopCollection() {
- // Flush any buffered snapshots before writing summary
- flushWriteBuffer();
-
- // Discard sessions that are too short to produce a useful map
- if (snapshotCount < HISTORY_MIN_SNAPSHOTS) {
- if (activeFile) {
- activeFile.close();
- }
- SPIFFS.remove(activeFilePath);
-
- LOG("HIST", "Session discarded (%u snapshots < %d minimum)", snapshotCount, HISTORY_MIN_SNAPSHOTS);
- dataLogger.logGenericEvent("history_discard", {{"snapshots", String(snapshotCount), FIELD_INT}});
-
- // Mark stats invalid so NotificationManager doesn't enrich a "done"
- // notification with stale data from the previous session.
- lastCleanStats.valid = false;
- lastCleanStats.sessionId = ++sessionCounter;
-
- collecting = false;
- recharging = false;
- setInterval(HISTORY_INTERVAL_IDLE_MS);
- return;
- }
-
- // Fetch final battery level for summary
- neato.getCharger([this](bool ok, const ChargerData& charger) {
- int batteryEnd = ok ? charger.fuelPercent : -1;
-
- writeSessionSummary(batteryEnd);
-
- // Close the raw file
- if (activeFile) {
- activeFile.close();
- }
-
- collecting = false;
- recharging = false;
- setInterval(HISTORY_INTERVAL_IDLE_MS);
-
- float areaCovered = static_cast(visitedCells.size()) * HISTORY_AREA_CELL_M * HISTORY_AREA_CELL_M;
-
- // Snapshot stats for notification enrichment (survives resetSession).
- // sessionId increments last so NotificationManager can detect that
- // the async charger fetch above has finalized the stats.
- time_t endTime = systemManager.now();
- lastCleanStats.valid = true;
- lastCleanStats.mode = cleanMode;
- lastCleanStats.durationSec = (sessionStartTime > 0 && endTime > sessionStartTime)
- ? static_cast(endTime - sessionStartTime)
- : 0;
- lastCleanStats.areaCoveredM2 = areaCovered;
- lastCleanStats.distanceM = totalDistance;
- lastCleanStats.batteryStart = batteryStart;
- lastCleanStats.batteryEnd = batteryEnd;
- lastCleanStats.recharges = rechargeCount;
- lastCleanStats.sessionId = ++sessionCounter;
-
- LOG("HIST", "Collection stopped (%u snapshots, %.1fm², %d recharges)", snapshotCount, areaCovered,
- rechargeCount);
- dataLogger.logGenericEvent("history_stop", {{"snapshots", String(snapshotCount), FIELD_INT},
- {"area_m2", String(areaCovered, 1), FIELD_FLOAT},
- {"recharges", String(rechargeCount), FIELD_INT},
- {"battery_end", String(batteryEnd), FIELD_INT}});
-
- // Start non-blocking compression: raw .jsonl -> .jsonl.hs
- compressSrcPath = activeFilePath;
- compressDstPath = activeFilePath + ".hs";
- compressSrc = SPIFFS.open(compressSrcPath, FILE_READ);
- compressDst = SPIFFS.open(compressDstPath, FILE_WRITE);
- if (compressSrc && compressDst) {
- heatshrink_encoder_reset(&compressEncoder);
- compressInputDone = false;
- compressing = true;
- setInterval(HISTORY_COMPRESS_INTERVAL_MS);
- LOG("HIST", "Starting compression: %s -> %s", compressSrcPath.c_str(), compressDstPath.c_str());
- } else {
- // Compression failed - keep raw file
- if (compressSrc)
- compressSrc.close();
- if (compressDst)
- compressDst.close();
- LOG("HIST", "Compression setup failed, keeping raw file");
- }
- });
-}
-
-// -- Incremental compression (called from tick) ------------------------------
-
-bool CleaningHistory::compressStep() {
- static const size_t CHUNK_SIZE = 512;
- uint8_t inBuf[CHUNK_SIZE];
- uint8_t outBuf[CHUNK_SIZE];
-
- if (!compressInputDone) {
- int bytesRead = compressSrc.read(inBuf, CHUNK_SIZE);
- if (bytesRead <= 0) {
- compressInputDone = true;
- } else {
- size_t offset = 0;
- while (offset < static_cast(bytesRead)) {
- size_t sunk = 0;
- HSE_sink_res sres =
- heatshrink_encoder_sink(&compressEncoder, inBuf + offset, bytesRead - offset, &sunk);
- if (sres < 0) {
- LOG("HIST", "Heatshrink sink error");
- compressSrc.close();
- compressDst.close();
- SPIFFS.remove(compressDstPath);
- compressing = false;
- return true;
- }
- offset += sunk;
-
- size_t outSz = 0;
- HSE_poll_res pres;
- do {
- pres = heatshrink_encoder_poll(&compressEncoder, outBuf, CHUNK_SIZE, &outSz);
- if (pres < 0) {
- LOG("HIST", "Heatshrink poll error");
- compressSrc.close();
- compressDst.close();
- SPIFFS.remove(compressDstPath);
- compressing = false;
- return true;
- }
- if (outSz > 0) {
- compressDst.write(outBuf, outSz);
- }
- } while (pres == HSER_POLL_MORE);
- }
- }
- return false;
- }
-
- // Input exhausted — finish encoding
- HSE_finish_res fres = heatshrink_encoder_finish(&compressEncoder);
- if (fres < 0) {
- LOG("HIST", "Heatshrink finish error");
- compressSrc.close();
- compressDst.close();
- SPIFFS.remove(compressDstPath);
- compressing = false;
- return true;
- }
-
- size_t outSz = 0;
- HSE_poll_res pres;
- do {
- pres = heatshrink_encoder_poll(&compressEncoder, outBuf, CHUNK_SIZE, &outSz);
- if (pres < 0) {
- LOG("HIST", "Heatshrink poll error during finish");
- compressSrc.close();
- compressDst.close();
- SPIFFS.remove(compressDstPath);
- compressing = false;
- return true;
- }
- if (outSz > 0) {
- compressDst.write(outBuf, outSz);
- }
- } while (pres == HSER_POLL_MORE);
-
- return (fres == HSER_FINISH_DONE);
-}
-
-// -- Session header/summary --------------------------------------------------
-
-void CleaningHistory::writeSessionHeader() {
- std::vector fields = {{"type", "session", FIELD_STRING}, {"mode", cleanMode, FIELD_STRING}};
- if (sessionStartTime > 0)
- fields.push_back({"time", String(static_cast(sessionStartTime)), FIELD_INT});
- if (batteryStart >= 0)
- fields.push_back({"battery", String(batteryStart), FIELD_INT});
- String json = fieldsToJson(fields);
- pendingSessionJson = json;
- writeLine(json);
-}
-
-void CleaningHistory::writeSessionSummary(int batteryEnd) {
- time_t endTime = systemManager.now();
- long duration =
- (sessionStartTime > 0 && endTime > sessionStartTime) ? static_cast(endTime - sessionStartTime) : 0;
- float areaCovered = static_cast(visitedCells.size()) * HISTORY_AREA_CELL_M * HISTORY_AREA_CELL_M;
- std::vector fields = {{"type", "summary", FIELD_STRING}};
- if (endTime > 0)
- fields.push_back({"time", String(static_cast(endTime)), FIELD_INT});
- fields.push_back({"duration", String(duration), FIELD_INT});
- fields.push_back({"mode", cleanMode, FIELD_STRING});
- fields.push_back({"recharges", String(rechargeCount), FIELD_INT});
- fields.push_back({"snapshots", String(static_cast(snapshotCount)), FIELD_INT});
- fields.push_back({"distanceTraveled", String(totalDistance, 2), FIELD_FLOAT});
- fields.push_back({"maxDistFromOrigin", String(maxDistFromOrigin, 2), FIELD_FLOAT});
- fields.push_back({"totalRotation", String(totalRotation, 1), FIELD_FLOAT});
- fields.push_back({"areaCovered", String(areaCovered, 2), FIELD_FLOAT});
- fields.push_back({"errorsDuringClean", String(errorsDuringClean), FIELD_INT});
- if (batteryStart >= 0)
- fields.push_back({"batteryStart", String(batteryStart), FIELD_INT});
- if (batteryEnd >= 0)
- fields.push_back({"batteryEnd", String(batteryEnd), FIELD_INT});
- String json = fieldsToJson(fields);
- pendingSummaryJson = json;
- writeLine(json);
-}
-
-// -- Buffered file writing ---------------------------------------------------
-// Pose snapshots accumulate in writeBuffer; flushed every HISTORY_FLUSH_INTERVAL_MS
-// or when the session ends (stopCollection / recovery). This reduces flash wear
-// from one write every 2s to one write every 30s during a cleaning session.
-// Session header, summary, and recharge markers flush immediately since they
-// are rare one-shot writes where crash safety matters.
-
-void CleaningHistory::writeLine(const String& line) {
- if (!activeFile)
- return;
- activeFile.println(line);
- activeFile.flush();
-}
-
-void CleaningHistory::bufferLine(const String& line) {
- if (!activeFile)
- return;
- writeBuffer.push_back(line);
-}
-
-void CleaningHistory::flushWriteBuffer() {
- if (writeBuffer.empty() || !activeFile)
- return;
- // Build a single string and write once to minimize SPIFFS COW metadata updates
- String batch;
- size_t total = 0;
- for (const auto& line: writeBuffer) {
- total += line.length() + 1;
- }
- batch.reserve(total);
- for (const auto& line: writeBuffer) {
- batch += line;
- batch += '\n';
- }
- activeFile.write(reinterpret_cast(batch.c_str()), batch.length());
- activeFile.flush();
- writeBuffer.clear();
- lastFlushMs = millis();
-}
-
-// -- Snapshot collection (active mode) ---------------------------------------
-
-static bool parsePose(const String& raw, float& x, float& y, float& theta, float& time) {
- int xPos = raw.indexOf("X=");
- int yPos = raw.indexOf("Y=");
- int tPos = raw.indexOf("Theta=");
- int tmPos = raw.indexOf("Time=");
- if (xPos < 0 || yPos < 0 || tPos < 0 || tmPos < 0)
- return false;
-
- x = raw.substring(xPos + 2).toFloat();
- y = raw.substring(yPos + 2).toFloat();
- theta = raw.substring(tPos + 6).toFloat();
- time = raw.substring(tmPos + 5).toFloat();
- return true;
-}
-
-// Parse a single JSONL line and update session accumulators (header, recharge, pose).
-// Returns true if the line was a session header.
-bool CleaningHistory::replayLine(const String& line) {
- auto fields = fieldsFromJson(line);
- if (fields.empty())
- return false;
-
- const Field *typeField = findField(fields, "type");
- if (typeField && typeField->value == "session") {
- const Field *modeField = findField(fields, "mode");
- if (modeField && !modeField->value.isEmpty())
- cleanMode = modeField->value;
-
- const Field *timeField = findField(fields, "time");
- if (timeField) {
- long t = timeField->value.toInt();
- if (t > 0)
- sessionStartTime = static_cast(t);
- }
-
- const Field *battField = findField(fields, "battery");
- if (battField)
- batteryStart = battField->value.toInt();
- return true;
- }
-
- if (typeField && typeField->value == "recharge") {
- rechargeCount++;
- return false;
- }
-
- // Pose snapshot line
- const Field *xf = findField(fields, "x");
- const Field *yf = findField(fields, "y");
- const Field *tf = findField(fields, "t");
- if (xf && yf && tf) {
- updateAccumulators(xf->value.toFloat(), yf->value.toFloat(), tf->value.toFloat());
- snapshotCount++;
- }
- return false;
-}
-
-bool CleaningHistory::recoverCollection(const String& uiState) {
- // Only attempt recovery once after boot — avoid scanning filesystem on every clean start
- if (recoveryAttempted)
- return false;
- recoveryAttempted = true;
-
- File root = SPIFFS.open(HISTORY_DIR);
- if (!root || !root.isDirectory())
- return false;
-
- // Collect all history filenames (both .jsonl and .jsonl.hs) and sort descending
- // (newest first). Then walk from newest to oldest: collect orphan .jsonl files
- // until we hit a compressed .jsonl.hs — that marks a completed session boundary,
- // so everything older belongs to previous cleans and should not be touched.
- // Also clean up any raw .jsonl that has a .jsonl.hs counterpart (stale leftover).
- std::vector allFiles;
- File entry = root.openNextFile();
- while (entry) {
- String path = String(entry.path());
- if (path.endsWith(".jsonl") || path.endsWith(".jsonl.hs"))
- allFiles.push_back(path);
- entry = root.openNextFile();
- }
-
- std::sort(allFiles.begin(), allFiles.end(), [](const String& a, const String& b) { return a > b; });
-
- std::vector orphans;
- for (const auto& path: allFiles) {
- if (path.endsWith(".jsonl.hs")) {
- // Hit a compressed file — this is the boundary of a completed session.
- // Stop collecting; everything older is from previous cleans.
- break;
- }
- // Raw .jsonl — check if a compressed copy exists (stale leftover)
- if (SPIFFS.exists(path + ".hs")) {
- SPIFFS.remove(path);
- LOG("HIST", "Removed stale raw file: %s (compressed copy exists)", path.c_str());
- continue;
- }
- // Check if it has a summary (completed but not yet compressed)
- String firstLine, lastLine;
- readFirstLastLines(path, false, firstLine, lastLine);
- if (lastLine.indexOf("\"type\":\"summary\"") >= 0)
- continue;
- orphans.push_back(path);
- }
-
- if (orphans.empty())
- return false;
-
- // orphans are newest-first; reverse to get chronological order (oldest first)
- std::reverse(orphans.begin(), orphans.end());
-
- // The oldest orphan becomes the target; newer ones are merged into it then deleted
- String targetPath = orphans[0];
-
- if (orphans.size() > 1) {
- File target = SPIFFS.open(targetPath, FILE_APPEND);
- if (target) {
- for (size_t i = 1; i < orphans.size(); i++) {
- File src = SPIFFS.open(orphans[i], FILE_READ);
- if (!src)
- continue;
- while (src.available()) {
- String line = src.readStringUntil('\n');
- line.trim();
- if (line.isEmpty())
- continue;
- // Skip duplicate session headers from newer orphans
- if (line.indexOf("\"type\":\"session\"") >= 0)
- continue;
- target.println(line);
- }
- src.close();
- SPIFFS.remove(orphans[i]);
- LOG("HIST", "Merged orphan %s into %s", orphans[i].c_str(), targetPath.c_str());
- }
- target.flush();
- target.close();
- }
- }
-
- // Now replay the merged file to rebuild accumulators
- resetSession();
- cleanMode = cleanModeFromState(uiState);
- activeFilePath = targetPath;
-
- File recoveryFile = SPIFFS.open(targetPath, FILE_READ);
- if (!recoveryFile)
- return false;
-
- bool hasSessionHeader = false;
- while (recoveryFile.available()) {
- String line = recoveryFile.readStringUntil('\n');
- line.trim();
- if (line.isEmpty())
- continue;
- if (replayLine(line))
- hasSessionHeader = true;
- }
- recoveryFile.close();
-
- // Fall back to extracting start time from the filename
- if (sessionStartTime <= 0) {
- String name = targetPath;
- int slash = name.lastIndexOf('/');
- if (slash >= 0)
- name = name.substring(slash + 1);
- int dot = name.indexOf('.');
- if (dot > 0)
- sessionStartTime = static_cast(name.substring(0, dot).toInt());
- }
-
- activeFile = SPIFFS.open(activeFilePath, FILE_APPEND);
- if (!activeFile) {
- activeFilePath = "";
- resetSession();
- return false;
- }
-
- collecting = true;
- setInterval(HISTORY_INTERVAL_ACTIVE_MS);
- LOG("HIST", "Recovered session: %s (%u snapshots, %zu orphans merged)", activeFilePath.c_str(), snapshotCount,
- orphans.size());
- dataLogger.logGenericEvent("history_recover", {{"path", activeFilePath, FIELD_STRING},
- {"snapshots", String(snapshotCount), FIELD_INT},
- {"orphans", String(static_cast(orphans.size())), FIELD_INT}});
-
- if (!hasSessionHeader) {
- writeSessionHeader();
- }
- return true;
-}
-
-void CleaningHistory::finalizeOrphanSessions() {
- // Called once at boot when the robot is idle. Finds orphan .jsonl files
- // (no summary line, no compressed counterpart) and finalizes them: replay
- // to compute summary stats, append summary line, then start compression.
- File root = SPIFFS.open(HISTORY_DIR);
- if (!root || !root.isDirectory())
- return;
-
- std::vector allFiles;
- File entry = root.openNextFile();
- while (entry) {
- String path = String(entry.path());
- if (path.endsWith(".jsonl") || path.endsWith(".jsonl.hs"))
- allFiles.push_back(path);
- entry = root.openNextFile();
- }
-
- std::sort(allFiles.begin(), allFiles.end(), [](const String& a, const String& b) { return a > b; });
-
- std::vector orphans;
- for (const auto& path: allFiles) {
- if (path.endsWith(".jsonl.hs"))
- break; // Completed session boundary
- if (SPIFFS.exists(path + ".hs")) {
- SPIFFS.remove(path);
- continue;
- }
- String firstLine, lastLine;
- readFirstLastLines(path, false, firstLine, lastLine);
- if (lastLine.indexOf("\"type\":\"summary\"") >= 0)
- continue;
- orphans.push_back(path);
- }
-
- if (orphans.empty())
- return;
-
- // Finalize each orphan: replay to rebuild stats, write summary, compress
- for (const auto& orphanPath: orphans) {
- resetSession();
-
- // Replay to rebuild accumulators
- File f = SPIFFS.open(orphanPath, FILE_READ);
- if (!f)
- continue;
- while (f.available()) {
- String line = f.readStringUntil('\n');
- line.trim();
- if (line.isEmpty())
- continue;
- replayLine(line);
- }
- f.close();
-
- // Extract start time from filename if not found in header
- if (sessionStartTime <= 0) {
- String name = orphanPath;
- int slash = name.lastIndexOf('/');
- if (slash >= 0)
- name = name.substring(slash + 1);
- int dot = name.indexOf('.');
- if (dot > 0)
- sessionStartTime = static_cast(name.substring(0, dot).toInt());
- }
-
- // Open for append and write summary
- activeFilePath = orphanPath;
- activeFile = SPIFFS.open(orphanPath, FILE_APPEND);
- if (!activeFile)
- continue;
-
- writeSessionSummary(-1); // Battery end unknown for interrupted sessions
- activeFile.close();
-
- LOG("HIST", "Finalized orphan: %s (%u snapshots)", orphanPath.c_str(), snapshotCount);
- dataLogger.logGenericEvent("history_finalize", {{"path", orphanPath, FIELD_STRING},
- {"snapshots", String(snapshotCount), FIELD_INT}});
-
- // Queue compression (only one at a time — first orphan wins, rest will
- // be picked up on next boot or left as uncompressed)
- if (!compressing) {
- compressSrcPath = orphanPath;
- compressDstPath = orphanPath + ".hs";
- compressSrc = SPIFFS.open(compressSrcPath, FILE_READ);
- compressDst = SPIFFS.open(compressDstPath, FILE_WRITE);
- if (compressSrc && compressDst) {
- heatshrink_encoder_reset(&compressEncoder);
- compressInputDone = false;
- compressing = true;
- setInterval(HISTORY_COMPRESS_INTERVAL_MS);
- } else {
- if (compressSrc)
- compressSrc.close();
- if (compressDst)
- compressDst.close();
- }
- }
- }
-
- activeFilePath = "";
- activeFile = File();
-}
-
-void CleaningHistory::collectSnapshot() {
- fetchPending = true;
-
- neato.getState([this](bool stateOk, const RobotState& state) {
- if (stateOk) {
- prevUiState = state.uiState;
-
- bool isDocking = isDockingState(state.uiState);
- bool isCleaning = isCleaningState(state.uiState);
- bool isSuspended = isSuspendedState(state.uiState);
- bool isChargingMidClean = state.robotState.indexOf("Charging_Cleaning") >= 0;
-
- // Mid-clean recharge: robot returns to base to charge before resuming.
- // The UI state can be DOCKING (on the way back) or CLEANINGSUSPENDED
- // (already on the dock and charging). Both combined with the
- // Charging_Cleaning robot state indicate a recharge-and-resume cycle.
- if ((isDocking || isSuspended) && isChargingMidClean) {
- if (!recharging) {
- recharging = true;
- rechargeCount++;
- LOG("HIST", "Recharge #%d detected — pausing collection", rechargeCount);
- dataLogger.logGenericEvent("history_recharge_start", {{"count", String(rechargeCount), FIELD_INT}});
-
- if (hasPrevPose) {
- writeLine(fieldsToJson({{"type", "recharge", FIELD_STRING},
- {"x", String(prevX, 3), FIELD_FLOAT},
- {"y", String(prevY, 3), FIELD_FLOAT}}));
- }
- }
- fetchPending = false;
- return;
- }
-
- if (recharging && isCleaning) {
- recharging = false;
- LOG("HIST", "Recharge done — resuming collection");
- dataLogger.logGenericEvent("history_recharge_end", {});
- }
-
- if (!isCleaning && !isDocking && !isSuspended) {
- fetchPending = false;
- stopCollection();
- return;
- }
-
- // Paused — keep session open but skip snapshot collection
- if (isPausedState(state.uiState)) {
- fetchPending = false;
- return;
- }
- }
-
- neato.getErr([this](bool errOk, const ErrorData& err) {
- if (errOk) {
- if (err.hasError && !prevHadError) {
- errorsDuringClean++;
- }
- prevHadError = err.hasError;
- }
-
- if (recharging) {
- fetchPending = false;
- return;
- }
-
- neato.getRobotPos(true, [this](bool posOk, const RobotPosData& pos) {
- fetchPending = false;
- if (!posOk)
- return;
-
- float x, y, theta, time;
- if (!parsePose(pos.raw, x, y, theta, time)) {
- LOG("HIST", "Failed to parse pose");
- return;
- }
-
- writeSnapshot(x, y, theta, time);
- });
- });
- });
-}
-
-void CleaningHistory::updateAccumulators(float x, float y, float theta) {
- if (!hasPrevPose) {
- originX = x;
- originY = y;
- prevX = x;
- prevY = y;
- prevTheta = theta;
- hasPrevPose = true;
- } else {
- float dx = x - prevX;
- float dy = y - prevY;
- totalDistance += sqrtf(dx * dx + dy * dy);
-
- float dTheta = theta - prevTheta;
- if (dTheta > 180.0f)
- dTheta -= 360.0f;
- if (dTheta < -180.0f)
- dTheta += 360.0f;
- totalRotation += fabsf(dTheta);
-
- prevX = x;
- prevY = y;
- prevTheta = theta;
- }
-
- float dox = x - originX;
- float doy = y - originY;
- float distFromOrigin = sqrtf(dox * dox + doy * doy);
- if (distFromOrigin > maxDistFromOrigin) {
- maxDistFromOrigin = distFromOrigin;
- }
-
- int ix = static_cast(floorf(x / HISTORY_AREA_CELL_M));
- int iy = static_cast(floorf(y / HISTORY_AREA_CELL_M));
- uint32_t cellKey = (static_cast(ix & 0xFFFF) << 16) | static_cast(iy & 0xFFFF);
- visitedCells.insert(cellKey);
-}
-
-void CleaningHistory::writeSnapshot(float x, float y, float theta, float time) {
- // Localization resets to origin right before session ends — drop if the
- // robot was far from origin (genuine return-to-base passes through gradually)
- if (hasPrevPose && fabsf(x) < 0.001f && fabsf(y) < 0.001f && fabsf(theta) < 0.1f) {
- float prevDist = sqrtf(prevX * prevX + prevY * prevY);
- if (prevDist > 1.0f) {
- return;
- }
- }
-
- String line = "{\"x\":" + String(x, 3) + ",\"y\":" + String(y, 3) + ",\"t\":" + String(theta, 1) +
- ",\"ts\":" + String(time, 1) + "}";
-
- updateAccumulators(x, y, theta);
- bufferLine(line);
- snapshotCount++;
-
- if (snapshotCount % 10 == 0) {
- LOG("HIST", "Snapshot #%u (%.1fm traveled, pose: %.2f,%.2f,%.0f)", snapshotCount, totalDistance, x, y, theta);
- }
-}
-
-// -- Storage enforcement (mirrors DataLogger::enforceLimits) -----------------
-
-void CleaningHistory::enforceLimits() {
- // Count session files, sum directory size, and find the oldest in one pass
- int fileCount = 0;
- size_t histDirBytes = 0;
- String oldest;
- File root = SPIFFS.open(HISTORY_DIR);
- if (!root || !root.isDirectory())
- return;
-
- File entry = root.openNextFile();
- while (entry) {
- String name = String(entry.name());
- histDirBytes += entry.size();
- if (name.endsWith(".jsonl") || name.endsWith(".jsonl.hs")) {
- fileCount++;
- if (oldest.isEmpty() || name < oldest) {
- oldest = name;
- }
- }
- entry = root.openNextFile();
- }
-
- if (oldest.isEmpty())
- return;
-
- // History budget: total filesystem cap minus non-history data, floored at minimum reserve
- size_t total = SPIFFS.totalBytes();
- size_t globalCap = (total * HISTORY_MAX_FS_PERCENT) / 100;
- size_t nonHistBytes = SPIFFS.usedBytes() > histDirBytes ? SPIFFS.usedBytes() - histDirBytes : 0;
- size_t available = globalCap > nonHistBytes ? globalCap - nonHistBytes : 0;
- size_t minReserved = (total * HISTORY_MIN_FS_PERCENT) / 100;
- size_t histBudget = available > minReserved ? available : minReserved;
-
- if (histDirBytes > histBudget || fileCount > HISTORY_MAX_FILES) {
- String fullPath = String(HISTORY_DIR) + "/" + oldest;
- LOG("HIST", "Limit: deleting %s (files=%d, histBytes=%u/%u)", fullPath.c_str(), fileCount, histDirBytes,
- histBudget);
- SPIFFS.remove(fullPath);
- metaCache.erase(oldest);
- }
-}
-
-// -- File management (for API) -----------------------------------------------
-
-void CleaningHistory::readFirstLastLines(const String& path, bool compressed, String& firstLine, String& lastLine) {
- firstLine = "";
- lastLine = "";
-
- if (compressed) {
- // Stream-decompress keeping only the first and last lines in memory
- // to avoid buffering the entire file (large sessions can exceed 100KB
- // decompressed which exhausts ESP32-C3 heap).
- File f = SPIFFS.open(path, FILE_READ);
- if (!f)
- return;
- CompressedLogReader reader(std::move(f));
- uint8_t buf[256];
- size_t n;
- bool firstDone = false;
- // Current incomplete line being assembled from chunks
- String current;
- while ((n = reader.read(buf, sizeof(buf))) > 0) {
- for (size_t i = 0; i < n; i++) {
- char c = static_cast(buf[i]);
- if (c == '\n') {
- current.trim();
- if (current.length() > 0) {
- if (!firstDone) {
- firstLine = current;
- firstDone = true;
- }
- // Always overwrite lastLine with the most recent non-empty line
- lastLine = current;
- }
- current = "";
- } else {
- current += c;
- }
- }
- }
- // Handle final chunk without trailing newline
- current.trim();
- if (current.length() > 0) {
- if (!firstDone) {
- firstLine = current;
- }
- lastLine = current;
- }
- } else {
- // Plain .jsonl — read first line directly, seek backward for last line
- File f = SPIFFS.open(path, FILE_READ);
- if (!f)
- return;
- // Read first line
- firstLine = f.readStringUntil('\n');
- firstLine.trim();
- // Seek backward from end to find last line
- size_t fileSize = f.size();
- if (fileSize < 2) {
- f.close();
- return;
- }
- int pos = static_cast(fileSize) - 2; // Skip trailing newline
- while (pos >= 0) {
- f.seek(pos);
- char c = static_cast(f.read());
- if (c == '\n') {
- break;
- }
- pos--;
- }
- // pos is at the newline before last line, or -1 if only one line
- if (pos < 0) {
- f.close();
- return; // Only one line
- }
- f.seek(pos + 1);
- lastLine = f.readStringUntil('\n');
- lastLine.trim();
- f.close();
- }
-}
-
-std::vector CleaningHistory::listSessions() {
- std::vector result;
-
- File root = SPIFFS.open(HISTORY_DIR);
- if (!root || !root.isDirectory())
- return result;
-
- File entry = root.openNextFile();
- while (entry) {
- String fullPath = String(entry.path());
- String name = fullPath;
- int lastSlash = name.lastIndexOf('/');
- if (lastSlash >= 0)
- name = name.substring(lastSlash + 1);
-
- if (name.endsWith(".jsonl") || name.endsWith(".jsonl.hs")) {
- // During compression both raw .jsonl and partial .jsonl.hs exist —
- // skip both until compression finishes and the raw source is deleted
- if (compressing && (fullPath == compressSrcPath || fullPath == compressDstPath)) {
- entry = root.openNextFile();
- continue;
- }
-
- // Skip the actively collecting file during disk enumeration.
- // We will manually append it at the end to avoid thread-safety /
- // concurrent modification issues with filesystem iterators.
- if (collecting && fullPath == activeFilePath) {
- entry = root.openNextFile();
- continue;
- }
-
- HistorySessionInfo info;
- info.name = name;
- info.size = entry.size();
- info.compressed = name.endsWith(".hs");
-
- // Use cached metadata if available (avoids decompressing .hs files)
- auto cached = metaCache.find(name);
- if (cached != metaCache.end()) {
- info.session = cached->second.session;
- info.summary = cached->second.summary;
- } else {
- String firstLine, lastLine;
- readFirstLastLines(fullPath, info.compressed, firstLine, lastLine);
- if (isValidMetaLine(firstLine, "\"type\":\"session\"")) {
- info.session = firstLine;
- }
- if (isValidMetaLine(lastLine, "\"type\":\"summary\"")) {
- info.summary = lastLine;
- }
- // Cache for subsequent requests
- metaCache[name] = {info.session, info.summary};
- }
-
- result.push_back(info);
- }
- entry = root.openNextFile();
- }
-
- // Always append the active session from memory to ensure exactly one entry
- if (collecting && !activeFilePath.isEmpty()) {
- String name = activeFilePath;
- int lastSlash = name.lastIndexOf('/');
- if (lastSlash >= 0)
- name = name.substring(lastSlash + 1);
-
- HistorySessionInfo info;
- info.name = name;
- info.size = activeFile ? activeFile.size() : 0;
- info.compressed = false;
- info.recording = true;
-
- String sessionJson = "{\"type\":\"session\",\"mode\":\"" + cleanMode + "\"";
- if (sessionStartTime > 0)
- sessionJson += ",\"time\":" + String(static_cast(sessionStartTime));
- if (batteryStart >= 0)
- sessionJson += ",\"battery\":" + String(batteryStart);
- sessionJson += "}";
- info.session = sessionJson;
- // No summary for active session
-
- result.push_back(info);
- }
-
- return result;
-}
-
-std::shared_ptr CleaningHistory::readSession(const String& filename) {
- String path = String(HISTORY_DIR) + "/" + filename;
-
- // Refuse to serve files involved in compression (partial .hs is corrupt)
- if (compressing && (path == compressSrcPath || path == compressDstPath))
- return nullptr;
-
- if (!SPIFFS.exists(path))
- return nullptr;
-
- File f = SPIFFS.open(path, FILE_READ);
- if (!f)
- return nullptr;
-
- if (filename.endsWith(".hs")) {
- return std::make_shared(std::move(f));
- }
- return std::make_shared(std::move(f));
-}
-
-bool CleaningHistory::deleteSession(const String& filename) {
- String path = String(HISTORY_DIR) + "/" + filename;
- if (!SPIFFS.exists(path))
- return false;
- metaCache.erase(filename);
- return SPIFFS.remove(path);
-}
-
-void CleaningHistory::deleteAllSessions() {
- File root = SPIFFS.open(HISTORY_DIR);
- if (!root || !root.isDirectory())
- return;
-
- std::vector paths;
- File entry = root.openNextFile();
- while (entry) {
- paths.push_back(String(entry.path()));
- entry = root.openNextFile();
- }
-
- for (const auto& p: paths) {
- SPIFFS.remove(p);
- }
- metaCache.clear();
- LOG("HIST", "Deleted %u session files", paths.size());
-}
-
-// -- Session import (compress-on-write from browser upload) -------------------
-
-bool CleaningHistory::beginImport(const String& filename) {
- importError = "";
-
- if (importing) {
- importError = "Import already in progress";
- return false;
- }
- if (collecting) {
- importError = "Cannot import while recording";
- return false;
- }
- if (compressing) {
- importError = "Cannot import while compressing";
- return false;
- }
-
- // Validate filename pattern: .jsonl
- if (!filename.endsWith(".jsonl")) {
- importError = "Invalid filename";
- return false;
- }
-
- // Check for duplicate (both raw and compressed)
- String rawPath = String(HISTORY_DIR) + "/" + filename;
- String hsPath = rawPath + ".hs";
- if (SPIFFS.exists(rawPath) || SPIFFS.exists(hsPath)) {
- importError = "Session already exists";
- return false;
- }
-
- // Open destination file for compressed output
- importFilePath = hsPath;
- importFile = SPIFFS.open(importFilePath, FILE_WRITE);
- if (!importFile) {
- importError = "Failed to create file";
- importFilePath = "";
- return false;
- }
-
- heatshrink_encoder_reset(&importEncoder);
- importBytesReceived = 0;
- importing = true;
- LOG("HIST", "Import started: %s", importFilePath.c_str());
- return true;
-}
-
-bool CleaningHistory::writeImportChunk(const uint8_t *data, size_t len) {
- if (!importing || !importFile) {
- importError = "No import in progress";
- return false;
- }
-
- importBytesReceived += len;
- if (importBytesReceived > HISTORY_IMPORT_MAX_BYTES) {
- importError = "File too large";
- importFile.close();
- SPIFFS.remove(importFilePath);
- importing = false;
- return false;
- }
-
- static const size_t OUT_BUF_SIZE = 512;
- uint8_t outBuf[OUT_BUF_SIZE];
- size_t offset = 0;
-
- while (offset < len) {
- size_t sunk = 0;
- HSE_sink_res sres = heatshrink_encoder_sink(&importEncoder, data + offset, len - offset, &sunk);
- if (sres < 0) {
- importError = "Compression error";
- importFile.close();
- SPIFFS.remove(importFilePath);
- importing = false;
- return false;
- }
- offset += sunk;
-
- // Drain compressed output
- size_t outSz = 0;
- HSE_poll_res pres;
- do {
- pres = heatshrink_encoder_poll(&importEncoder, outBuf, OUT_BUF_SIZE, &outSz);
- if (pres < 0) {
- importError = "Compression error";
- importFile.close();
- SPIFFS.remove(importFilePath);
- importing = false;
- return false;
- }
- if (outSz > 0) {
- importFile.write(outBuf, outSz);
- }
- } while (pres == HSER_POLL_MORE);
- }
- return true;
-}
-
-bool CleaningHistory::endImport() {
- if (!importing || !importFile) {
- importError = "No import in progress";
- return false;
- }
-
- static const size_t OUT_BUF_SIZE = 512;
- uint8_t outBuf[OUT_BUF_SIZE];
-
- // Finish the encoder — may require multiple poll rounds
- bool done = false;
- while (!done) {
- HSE_finish_res fres = heatshrink_encoder_finish(&importEncoder);
- if (fres < 0) {
- importError = "Compression finish error";
- importFile.close();
- SPIFFS.remove(importFilePath);
- importing = false;
- return false;
- }
- done = (fres == HSER_FINISH_DONE);
-
- size_t outSz = 0;
- HSE_poll_res pres;
- do {
- pres = heatshrink_encoder_poll(&importEncoder, outBuf, OUT_BUF_SIZE, &outSz);
- if (pres < 0) {
- importError = "Compression finish error";
- importFile.close();
- SPIFFS.remove(importFilePath);
- importing = false;
- return false;
- }
- if (outSz > 0) {
- importFile.write(outBuf, outSz);
- }
- } while (pres == HSER_POLL_MORE);
- }
-
- importFile.close();
- importing = false;
- LOG("HIST", "Import complete: %s", importFilePath.c_str());
- dataLogger.logGenericEvent("history_import", {{"path", importFilePath, FIELD_STRING}});
-
- // Delete oldest sessions if storage budget exceeded
- enforceLimits();
-
- return true;
-}
+#include "cleaning_history.h"
+#include "json_fields.h"
+#include "neato_serial.h"
+#include "system_manager.h"
+#include
+#include
+
+// Heatshrink decompression can drop a byte that merges two JSONL lines into
+// one, so substring matching isn't enough — validate the braces balance and
+// nothing trails the top-level object before embedding into /api/history.
+static bool isValidMetaLine(const String& line, const char *expectedTypePrefix) {
+ if (line.length() < 2 || line[0] != '{' || line[line.length() - 1] != '}')
+ return false;
+ if (line.indexOf(expectedTypePrefix) < 0)
+ return false;
+ int depth = 0;
+ bool inString = false;
+ bool escape = false;
+ for (size_t i = 0; i < line.length(); i++) {
+ char c = line[i];
+ if (escape) {
+ escape = false;
+ continue;
+ }
+ if (inString) {
+ if (c == '\\')
+ escape = true;
+ else if (c == '"')
+ inString = false;
+ continue;
+ }
+ if (c == '"')
+ inString = true;
+ else if (c == '{')
+ depth++;
+ else if (c == '}') {
+ depth--;
+ if (depth < 0)
+ return false;
+ // Reject trailing content after the top-level object closes
+ if (depth == 0 && i != line.length() - 1)
+ return false;
+ }
+ }
+ return depth == 0 && !inString;
+}
+
+CleaningHistory::CleaningHistory(NeatoSerial& neato, DataLogger& logger, SystemManager& sysMgr) :
+ LoopTask(HISTORY_INTERVAL_IDLE_MS), neato(neato), dataLogger(logger), systemManager(sysMgr) {
+ TaskRegistry::add(this);
+}
+
+void CleaningHistory::notifyCleanStart() {
+ if (collecting)
+ return; // Already recording
+ setInterval(HISTORY_INTERVAL_ACTIVE_MS);
+}
+
+void CleaningHistory::tick() {
+ // Run incremental compression when a session just finished
+ if (compressing) {
+ if (compressStep()) {
+ // Compression done - remove raw source and cache metadata
+ compressSrc.close();
+ compressDst.close();
+ SPIFFS.remove(compressSrcPath);
+ LOG("HIST", "Compression done: %s", compressDstPath.c_str());
+
+ // Cache session/summary from memory (avoids decompressing on next list request)
+ String hsName = compressDstPath;
+ int lastSlash = hsName.lastIndexOf('/');
+ if (lastSlash >= 0)
+ hsName = hsName.substring(lastSlash + 1);
+ metaCache[hsName] = {pendingSessionJson, pendingSummaryJson};
+ pendingSessionJson = "";
+ pendingSummaryJson = "";
+
+ compressing = false;
+ setInterval(HISTORY_INTERVAL_IDLE_MS);
+ }
+ return;
+ }
+
+ if (fetchPending)
+ return;
+
+ if (collecting) {
+ // Periodically flush buffered pose snapshots to disk
+ if (!writeBuffer.empty() && millis() - lastFlushMs >= HISTORY_FLUSH_INTERVAL_MS) {
+ flushWriteBuffer();
+ }
+ collectSnapshot();
+ } else {
+ checkState();
+ enforceLimits();
+ }
+}
+
+// -- State watching (idle mode) ----------------------------------------------
+
+void CleaningHistory::checkState() {
+ fetchPending = true;
+ neato.getState([this](bool ok, const RobotState& state) {
+ fetchPending = false;
+ if (!ok)
+ return;
+
+ bool wasCleaning = isCleaningState(prevUiState);
+ bool nowCleaning = isCleaningState(state.uiState);
+
+ // CLEANINGSUSPENDED + Charging_Cleaning means the robot is on the dock
+ // recharging mid-clean and will resume automatically — treat as active
+ bool isMidCleanRecharge = isSuspendedState(state.uiState) && state.robotState.indexOf("Charging_Cleaning") >= 0;
+
+ // First poll after boot: if the robot is idle, finalize any orphan sessions
+ // left by a previous crash/reboot so they don't get merged into the next clean
+ if (prevUiState.isEmpty() && !nowCleaning && !isMidCleanRecharge && !recoveryAttempted) {
+ recoveryAttempted = true;
+ finalizeOrphanSessions();
+ }
+
+ if (!wasCleaning && (nowCleaning || isMidCleanRecharge) && !recoverCollection(state.uiState)) {
+ startCollection(state.uiState);
+ }
+
+ prevUiState = state.uiState;
+ });
+}
+
+bool CleaningHistory::isCleaningState(const String& uiState) {
+ return uiState.indexOf("HOUSECLEANINGRUNNING") >= 0 || uiState.indexOf("HOUSECLEANINGPAUSED") >= 0 ||
+ uiState.indexOf("SPOTCLEANINGRUNNING") >= 0 || uiState.indexOf("SPOTCLEANINGPAUSED") >= 0 ||
+ uiState.indexOf("MANUALCLEANING") >= 0;
+}
+
+bool CleaningHistory::isPausedState(const String& uiState) {
+ return uiState.indexOf("CLEANINGPAUSED") >= 0;
+}
+
+bool CleaningHistory::isDockingState(const String& uiState) {
+ return uiState.indexOf("DOCKING") >= 0;
+}
+
+bool CleaningHistory::isSuspendedState(const String& uiState) {
+ return uiState.indexOf("CLEANINGSUSPENDED") >= 0;
+}
+
+String CleaningHistory::cleanModeFromState(const String& uiState) {
+ if (uiState.indexOf("HOUSECLEANING") >= 0)
+ return "house";
+ if (uiState.indexOf("SPOTCLEANING") >= 0)
+ return "spot";
+ if (uiState.indexOf("MANUALCLEANING") >= 0)
+ return "manual";
+ return "unknown";
+}
+
+void CleaningHistory::resetSession() {
+ snapshotCount = 0;
+ rechargeCount = 0;
+ totalDistance = 0.0f;
+ totalRotation = 0.0f;
+ maxDistFromOrigin = 0.0f;
+ errorsDuringClean = 0;
+ prevHadError = false;
+ hasPrevPose = false;
+ prevX = 0.0f;
+ prevY = 0.0f;
+ prevTheta = 0.0f;
+ originX = 0.0f;
+ originY = 0.0f;
+ visitedCells.clear();
+ cleanMode = "";
+ sessionStartTime = 0;
+ batteryStart = -1;
+ recharging = false;
+ pendingSessionJson = "";
+ pendingSummaryJson = "";
+}
+
+void CleaningHistory::startCollection(const String& uiState) {
+ collecting = true;
+ resetSession();
+
+ cleanMode = cleanModeFromState(uiState);
+ sessionStartTime = systemManager.now();
+
+ // Create session file: /history/.jsonl
+ activeFilePath = String(HISTORY_DIR) + "/" + String(static_cast(sessionStartTime)) + ".jsonl";
+ activeFile = SPIFFS.open(activeFilePath, FILE_WRITE);
+ if (!activeFile) {
+ LOG("HIST", "Failed to create session file: %s", activeFilePath.c_str());
+ collecting = false;
+ return;
+ }
+
+ // Fetch battery level for session metadata, then write header
+ neato.getCharger([this](bool ok, const ChargerData& charger) {
+ if (ok) {
+ batteryStart = charger.fuelPercent;
+ }
+ writeSessionHeader();
+ });
+
+ setInterval(HISTORY_INTERVAL_ACTIVE_MS);
+ LOG("HIST", "Collection started (mode: %s, file: %s)", cleanMode.c_str(), activeFilePath.c_str());
+ dataLogger.logGenericEvent("history_start", {{"mode", cleanMode, FIELD_STRING}});
+}
+
+void CleaningHistory::stopCollection() {
+ // Flush any buffered snapshots before writing summary
+ flushWriteBuffer();
+
+ // Discard sessions that are too short to produce a useful map
+ if (snapshotCount < HISTORY_MIN_SNAPSHOTS) {
+ if (activeFile) {
+ activeFile.close();
+ }
+ SPIFFS.remove(activeFilePath);
+
+ LOG("HIST", "Session discarded (%u snapshots < %d minimum)", snapshotCount, HISTORY_MIN_SNAPSHOTS);
+ dataLogger.logGenericEvent("history_discard", {{"snapshots", String(snapshotCount), FIELD_INT}});
+
+ // Mark stats invalid so NotificationManager doesn't enrich a "done"
+ // notification with stale data from the previous session.
+ lastCleanStats.valid = false;
+ lastCleanStats.sessionId = ++sessionCounter;
+
+ collecting = false;
+ recharging = false;
+ setInterval(HISTORY_INTERVAL_IDLE_MS);
+ return;
+ }
+
+ // Fetch final battery level for summary
+ neato.getCharger([this](bool ok, const ChargerData& charger) {
+ int batteryEnd = ok ? charger.fuelPercent : -1;
+
+ writeSessionSummary(batteryEnd);
+
+ // Close the raw file
+ if (activeFile) {
+ activeFile.close();
+ }
+
+ collecting = false;
+ recharging = false;
+ setInterval(HISTORY_INTERVAL_IDLE_MS);
+
+ float areaCovered = static_cast(visitedCells.size()) * HISTORY_AREA_CELL_M * HISTORY_AREA_CELL_M;
+
+ // Snapshot stats for notification enrichment (survives resetSession).
+ // sessionId increments last so NotificationManager can detect that
+ // the async charger fetch above has finalized the stats.
+ time_t endTime = systemManager.now();
+ lastCleanStats.valid = true;
+ lastCleanStats.mode = cleanMode;
+ lastCleanStats.durationSec = (sessionStartTime > 0 && endTime > sessionStartTime)
+ ? static_cast(endTime - sessionStartTime)
+ : 0;
+ lastCleanStats.areaCoveredM2 = areaCovered;
+ lastCleanStats.distanceM = totalDistance;
+ lastCleanStats.batteryStart = batteryStart;
+ lastCleanStats.batteryEnd = batteryEnd;
+ lastCleanStats.recharges = rechargeCount;
+ lastCleanStats.sessionId = ++sessionCounter;
+
+ LOG("HIST", "Collection stopped (%u snapshots, %.1fm², %d recharges)", snapshotCount, areaCovered,
+ rechargeCount);
+ dataLogger.logGenericEvent("history_stop", {{"snapshots", String(snapshotCount), FIELD_INT},
+ {"area_m2", String(areaCovered, 1), FIELD_FLOAT},
+ {"recharges", String(rechargeCount), FIELD_INT},
+ {"battery_end", String(batteryEnd), FIELD_INT}});
+
+ // Start non-blocking compression: raw .jsonl -> .jsonl.hs
+ compressSrcPath = activeFilePath;
+ compressDstPath = activeFilePath + ".hs";
+ compressSrc = SPIFFS.open(compressSrcPath, FILE_READ);
+ compressDst = SPIFFS.open(compressDstPath, FILE_WRITE);
+ if (compressSrc && compressDst) {
+ heatshrink_encoder_reset(&compressEncoder);
+ compressInputDone = false;
+ compressing = true;
+ setInterval(HISTORY_COMPRESS_INTERVAL_MS);
+ LOG("HIST", "Starting compression: %s -> %s", compressSrcPath.c_str(), compressDstPath.c_str());
+ } else {
+ // Compression failed - keep raw file
+ if (compressSrc)
+ compressSrc.close();
+ if (compressDst)
+ compressDst.close();
+ LOG("HIST", "Compression setup failed, keeping raw file");
+ }
+ });
+}
+
+// -- Incremental compression (called from tick) ------------------------------
+
+bool CleaningHistory::compressStep() {
+ static const size_t CHUNK_SIZE = 512;
+ uint8_t inBuf[CHUNK_SIZE];
+ uint8_t outBuf[CHUNK_SIZE];
+
+ if (!compressInputDone) {
+ int bytesRead = compressSrc.read(inBuf, CHUNK_SIZE);
+ if (bytesRead <= 0) {
+ compressInputDone = true;
+ } else {
+ size_t offset = 0;
+ while (offset < static_cast(bytesRead)) {
+ size_t sunk = 0;
+ HSE_sink_res sres =
+ heatshrink_encoder_sink(&compressEncoder, inBuf + offset, bytesRead - offset, &sunk);
+ if (sres < 0) {
+ LOG("HIST", "Heatshrink sink error");
+ compressSrc.close();
+ compressDst.close();
+ SPIFFS.remove(compressDstPath);
+ compressing = false;
+ return true;
+ }
+ offset += sunk;
+
+ size_t outSz = 0;
+ HSE_poll_res pres;
+ do {
+ pres = heatshrink_encoder_poll(&compressEncoder, outBuf, CHUNK_SIZE, &outSz);
+ if (pres < 0) {
+ LOG("HIST", "Heatshrink poll error");
+ compressSrc.close();
+ compressDst.close();
+ SPIFFS.remove(compressDstPath);
+ compressing = false;
+ return true;
+ }
+ if (outSz > 0) {
+ compressDst.write(outBuf, outSz);
+ }
+ } while (pres == HSER_POLL_MORE);
+ }
+ }
+ return false;
+ }
+
+ // Input exhausted — finish encoding
+ HSE_finish_res fres = heatshrink_encoder_finish(&compressEncoder);
+ if (fres < 0) {
+ LOG("HIST", "Heatshrink finish error");
+ compressSrc.close();
+ compressDst.close();
+ SPIFFS.remove(compressDstPath);
+ compressing = false;
+ return true;
+ }
+
+ size_t outSz = 0;
+ HSE_poll_res pres;
+ do {
+ pres = heatshrink_encoder_poll(&compressEncoder, outBuf, CHUNK_SIZE, &outSz);
+ if (pres < 0) {
+ LOG("HIST", "Heatshrink poll error during finish");
+ compressSrc.close();
+ compressDst.close();
+ SPIFFS.remove(compressDstPath);
+ compressing = false;
+ return true;
+ }
+ if (outSz > 0) {
+ compressDst.write(outBuf, outSz);
+ }
+ } while (pres == HSER_POLL_MORE);
+
+ return (fres == HSER_FINISH_DONE);
+}
+
+// -- Session header/summary --------------------------------------------------
+
+void CleaningHistory::writeSessionHeader() {
+ std::vector fields = {{"type", "session", FIELD_STRING}, {"mode", cleanMode, FIELD_STRING}};
+ if (sessionStartTime > 0)
+ fields.push_back({"time", String(static_cast(sessionStartTime)), FIELD_INT});
+ if (batteryStart >= 0)
+ fields.push_back({"battery", String(batteryStart), FIELD_INT});
+ String json = fieldsToJson(fields);
+ pendingSessionJson = json;
+ writeLine(json);
+}
+
+void CleaningHistory::writeSessionSummary(int batteryEnd) {
+ time_t endTime = systemManager.now();
+ long duration =
+ (sessionStartTime > 0 && endTime > sessionStartTime) ? static_cast(endTime - sessionStartTime) : 0;
+ float areaCovered = static_cast(visitedCells.size()) * HISTORY_AREA_CELL_M * HISTORY_AREA_CELL_M;
+ std::vector fields = {{"type", "summary", FIELD_STRING}};
+ if (endTime > 0)
+ fields.push_back({"time", String(static_cast(endTime)), FIELD_INT});
+ fields.push_back({"duration", String(duration), FIELD_INT});
+ fields.push_back({"mode", cleanMode, FIELD_STRING});
+ fields.push_back({"recharges", String(rechargeCount), FIELD_INT});
+ fields.push_back({"snapshots", String(static_cast(snapshotCount)), FIELD_INT});
+ fields.push_back({"distanceTraveled", String(totalDistance, 2), FIELD_FLOAT});
+ fields.push_back({"maxDistFromOrigin", String(maxDistFromOrigin, 2), FIELD_FLOAT});
+ fields.push_back({"totalRotation", String(totalRotation, 1), FIELD_FLOAT});
+ fields.push_back({"areaCovered", String(areaCovered, 2), FIELD_FLOAT});
+ fields.push_back({"errorsDuringClean", String(errorsDuringClean), FIELD_INT});
+ if (batteryStart >= 0)
+ fields.push_back({"batteryStart", String(batteryStart), FIELD_INT});
+ if (batteryEnd >= 0)
+ fields.push_back({"batteryEnd", String(batteryEnd), FIELD_INT});
+ String json = fieldsToJson(fields);
+ pendingSummaryJson = json;
+ writeLine(json);
+}
+
+// -- Buffered file writing ---------------------------------------------------
+// Pose snapshots accumulate in writeBuffer; flushed every HISTORY_FLUSH_INTERVAL_MS
+// or when the session ends (stopCollection / recovery). This reduces flash wear
+// from one write every 2s to one write every 30s during a cleaning session.
+// Session header, summary, and recharge markers flush immediately since they
+// are rare one-shot writes where crash safety matters.
+
+void CleaningHistory::writeLine(const String& line) {
+ if (!activeFile)
+ return;
+ activeFile.println(line);
+ activeFile.flush();
+}
+
+void CleaningHistory::bufferLine(const String& line) {
+ if (!activeFile)
+ return;
+ writeBuffer.push_back(line);
+}
+
+void CleaningHistory::flushWriteBuffer() {
+ if (writeBuffer.empty() || !activeFile)
+ return;
+ // Build a single string and write once to minimize SPIFFS COW metadata updates
+ String batch;
+ size_t total = 0;
+ for (const auto& line: writeBuffer) {
+ total += line.length() + 1;
+ }
+ batch.reserve(total);
+ for (const auto& line: writeBuffer) {
+ batch += line;
+ batch += '\n';
+ }
+ activeFile.write(reinterpret_cast(batch.c_str()), batch.length());
+ activeFile.flush();
+ writeBuffer.clear();
+ lastFlushMs = millis();
+}
+
+// -- Snapshot collection (active mode) ---------------------------------------
+
+static bool parsePose(const String& raw, float& x, float& y, float& theta, float& time) {
+ int xPos = raw.indexOf("X=");
+ int yPos = raw.indexOf("Y=");
+ int tPos = raw.indexOf("Theta=");
+ int tmPos = raw.indexOf("Time=");
+ if (xPos < 0 || yPos < 0 || tPos < 0 || tmPos < 0)
+ return false;
+
+ x = raw.substring(xPos + 2).toFloat();
+ y = raw.substring(yPos + 2).toFloat();
+ theta = raw.substring(tPos + 6).toFloat();
+ time = raw.substring(tmPos + 5).toFloat();
+ return true;
+}
+
+// Parse a single JSONL line and update session accumulators (header, recharge, pose).
+// Returns true if the line was a session header.
+bool CleaningHistory::replayLine(const String& line) {
+ auto fields = fieldsFromJson(line);
+ if (fields.empty())
+ return false;
+
+ const Field *typeField = findField(fields, "type");
+ if (typeField && typeField->value == "session") {
+ const Field *modeField = findField(fields, "mode");
+ if (modeField && !modeField->value.isEmpty())
+ cleanMode = modeField->value;
+
+ const Field *timeField = findField(fields, "time");
+ if (timeField) {
+ long t = timeField->value.toInt();
+ if (t > 0)
+ sessionStartTime = static_cast(t);
+ }
+
+ const Field *battField = findField(fields, "battery");
+ if (battField)
+ batteryStart = battField->value.toInt();
+ return true;
+ }
+
+ if (typeField && typeField->value == "recharge") {
+ rechargeCount++;
+ return false;
+ }
+
+ // Pose snapshot line
+ const Field *xf = findField(fields, "x");
+ const Field *yf = findField(fields, "y");
+ const Field *tf = findField(fields, "t");
+ if (xf && yf && tf) {
+ updateAccumulators(xf->value.toFloat(), yf->value.toFloat(), tf->value.toFloat());
+ snapshotCount++;
+ }
+ return false;
+}
+
+bool CleaningHistory::recoverCollection(const String& uiState) {
+ // Only attempt recovery once after boot — avoid scanning filesystem on every clean start
+ if (recoveryAttempted)
+ return false;
+ recoveryAttempted = true;
+
+ File root = SPIFFS.open(HISTORY_DIR);
+ if (!root || !root.isDirectory())
+ return false;
+
+ // Collect all history filenames (both .jsonl and .jsonl.hs) and sort descending
+ // (newest first). Then walk from newest to oldest: collect orphan .jsonl files
+ // until we hit a compressed .jsonl.hs — that marks a completed session boundary,
+ // so everything older belongs to previous cleans and should not be touched.
+ // Also clean up any raw .jsonl that has a .jsonl.hs counterpart (stale leftover).
+ std::vector allFiles;
+ File entry = root.openNextFile();
+ while (entry) {
+ String path = String(entry.path());
+ if (path.endsWith(".jsonl") || path.endsWith(".jsonl.hs"))
+ allFiles.push_back(path);
+ entry = root.openNextFile();
+ }
+
+ std::sort(allFiles.begin(), allFiles.end(), [](const String& a, const String& b) { return a > b; });
+
+ std::vector orphans;
+ for (const auto& path: allFiles) {
+ if (path.endsWith(".jsonl.hs")) {
+ // Hit a compressed file — this is the boundary of a completed session.
+ // Stop collecting; everything older is from previous cleans.
+ break;
+ }
+ // Raw .jsonl — check if a compressed copy exists (stale leftover)
+ if (SPIFFS.exists(path + ".hs")) {
+ SPIFFS.remove(path);
+ LOG("HIST", "Removed stale raw file: %s (compressed copy exists)", path.c_str());
+ continue;
+ }
+ // Check if it has a summary (completed but not yet compressed)
+ String firstLine, lastLine;
+ readFirstLastLines(path, false, firstLine, lastLine);
+ if (lastLine.indexOf("\"type\":\"summary\"") >= 0)
+ continue;
+ orphans.push_back(path);
+ }
+
+ if (orphans.empty())
+ return false;
+
+ // orphans are newest-first; reverse to get chronological order (oldest first)
+ std::reverse(orphans.begin(), orphans.end());
+
+ // The oldest orphan becomes the target; newer ones are merged into it then deleted
+ String targetPath = orphans[0];
+
+ if (orphans.size() > 1) {
+ File target = SPIFFS.open(targetPath, FILE_APPEND);
+ if (target) {
+ for (size_t i = 1; i < orphans.size(); i++) {
+ File src = SPIFFS.open(orphans[i], FILE_READ);
+ if (!src)
+ continue;
+ while (src.available()) {
+ String line = src.readStringUntil('\n');
+ line.trim();
+ if (line.isEmpty())
+ continue;
+ // Skip duplicate session headers from newer orphans
+ if (line.indexOf("\"type\":\"session\"") >= 0)
+ continue;
+ target.println(line);
+ }
+ src.close();
+ SPIFFS.remove(orphans[i]);
+ LOG("HIST", "Merged orphan %s into %s", orphans[i].c_str(), targetPath.c_str());
+ }
+ target.flush();
+ target.close();
+ }
+ }
+
+ // Now replay the merged file to rebuild accumulators
+ resetSession();
+ cleanMode = cleanModeFromState(uiState);
+ activeFilePath = targetPath;
+
+ File recoveryFile = SPIFFS.open(targetPath, FILE_READ);
+ if (!recoveryFile)
+ return false;
+
+ bool hasSessionHeader = false;
+ while (recoveryFile.available()) {
+ String line = recoveryFile.readStringUntil('\n');
+ line.trim();
+ if (line.isEmpty())
+ continue;
+ if (replayLine(line))
+ hasSessionHeader = true;
+ }
+ recoveryFile.close();
+
+ // Fall back to extracting start time from the filename
+ if (sessionStartTime <= 0) {
+ String name = targetPath;
+ int slash = name.lastIndexOf('/');
+ if (slash >= 0)
+ name = name.substring(slash + 1);
+ int dot = name.indexOf('.');
+ if (dot > 0)
+ sessionStartTime = static_cast(name.substring(0, dot).toInt());
+ }
+
+ activeFile = SPIFFS.open(activeFilePath, FILE_APPEND);
+ if (!activeFile) {
+ activeFilePath = "";
+ resetSession();
+ return false;
+ }
+
+ collecting = true;
+ setInterval(HISTORY_INTERVAL_ACTIVE_MS);
+ LOG("HIST", "Recovered session: %s (%u snapshots, %zu orphans merged)", activeFilePath.c_str(), snapshotCount,
+ orphans.size());
+ dataLogger.logGenericEvent("history_recover", {{"path", activeFilePath, FIELD_STRING},
+ {"snapshots", String(snapshotCount), FIELD_INT},
+ {"orphans", String(static_cast(orphans.size())), FIELD_INT}});
+
+ if (!hasSessionHeader) {
+ writeSessionHeader();
+ }
+ return true;
+}
+
+void CleaningHistory::finalizeOrphanSessions() {
+ // Called once at boot when the robot is idle. Finds orphan .jsonl files
+ // (no summary line, no compressed counterpart) and finalizes them: replay
+ // to compute summary stats, append summary line, then start compression.
+ File root = SPIFFS.open(HISTORY_DIR);
+ if (!root || !root.isDirectory())
+ return;
+
+ std::vector allFiles;
+ File entry = root.openNextFile();
+ while (entry) {
+ String path = String(entry.path());
+ if (path.endsWith(".jsonl") || path.endsWith(".jsonl.hs"))
+ allFiles.push_back(path);
+ entry = root.openNextFile();
+ }
+
+ std::sort(allFiles.begin(), allFiles.end(), [](const String& a, const String& b) { return a > b; });
+
+ std::vector orphans;
+ for (const auto& path: allFiles) {
+ if (path.endsWith(".jsonl.hs"))
+ break; // Completed session boundary
+ if (SPIFFS.exists(path + ".hs")) {
+ SPIFFS.remove(path);
+ continue;
+ }
+ String firstLine, lastLine;
+ readFirstLastLines(path, false, firstLine, lastLine);
+ if (lastLine.indexOf("\"type\":\"summary\"") >= 0)
+ continue;
+ orphans.push_back(path);
+ }
+
+ if (orphans.empty())
+ return;
+
+ // Finalize each orphan: replay to rebuild stats, write summary, compress
+ for (const auto& orphanPath: orphans) {
+ resetSession();
+
+ // Replay to rebuild accumulators
+ File f = SPIFFS.open(orphanPath, FILE_READ);
+ if (!f)
+ continue;
+ while (f.available()) {
+ String line = f.readStringUntil('\n');
+ line.trim();
+ if (line.isEmpty())
+ continue;
+ replayLine(line);
+ }
+ f.close();
+
+ // Extract start time from filename if not found in header
+ if (sessionStartTime <= 0) {
+ String name = orphanPath;
+ int slash = name.lastIndexOf('/');
+ if (slash >= 0)
+ name = name.substring(slash + 1);
+ int dot = name.indexOf('.');
+ if (dot > 0)
+ sessionStartTime = static_cast(name.substring(0, dot).toInt());
+ }
+
+ // Open for append and write summary
+ activeFilePath = orphanPath;
+ activeFile = SPIFFS.open(orphanPath, FILE_APPEND);
+ if (!activeFile)
+ continue;
+
+ writeSessionSummary(-1); // Battery end unknown for interrupted sessions
+ activeFile.close();
+
+ LOG("HIST", "Finalized orphan: %s (%u snapshots)", orphanPath.c_str(), snapshotCount);
+ dataLogger.logGenericEvent("history_finalize", {{"path", orphanPath, FIELD_STRING},
+ {"snapshots", String(snapshotCount), FIELD_INT}});
+
+ // Queue compression (only one at a time — first orphan wins, rest will
+ // be picked up on next boot or left as uncompressed)
+ if (!compressing) {
+ compressSrcPath = orphanPath;
+ compressDstPath = orphanPath + ".hs";
+ compressSrc = SPIFFS.open(compressSrcPath, FILE_READ);
+ compressDst = SPIFFS.open(compressDstPath, FILE_WRITE);
+ if (compressSrc && compressDst) {
+ heatshrink_encoder_reset(&compressEncoder);
+ compressInputDone = false;
+ compressing = true;
+ setInterval(HISTORY_COMPRESS_INTERVAL_MS);
+ } else {
+ if (compressSrc)
+ compressSrc.close();
+ if (compressDst)
+ compressDst.close();
+ }
+ }
+ }
+
+ activeFilePath = "";
+ activeFile = File();
+}
+
+void CleaningHistory::collectSnapshot() {
+ fetchPending = true;
+
+ neato.getState([this](bool stateOk, const RobotState& state) {
+ if (stateOk) {
+ prevUiState = state.uiState;
+
+ bool isDocking = isDockingState(state.uiState);
+ bool isCleaning = isCleaningState(state.uiState);
+ bool isSuspended = isSuspendedState(state.uiState);
+ bool isChargingMidClean = state.robotState.indexOf("Charging_Cleaning") >= 0;
+
+ // Mid-clean recharge: robot returns to base to charge before resuming.
+ // The UI state can be DOCKING (on the way back) or CLEANINGSUSPENDED
+ // (already on the dock and charging). Both combined with the
+ // Charging_Cleaning robot state indicate a recharge-and-resume cycle.
+ if ((isDocking || isSuspended) && isChargingMidClean) {
+ if (!recharging) {
+ recharging = true;
+ rechargeCount++;
+ LOG("HIST", "Recharge #%d detected — pausing collection", rechargeCount);
+ dataLogger.logGenericEvent("history_recharge_start", {{"count", String(rechargeCount), FIELD_INT}});
+
+ if (hasPrevPose) {
+ writeLine(fieldsToJson({{"type", "recharge", FIELD_STRING},
+ {"x", String(prevX, 3), FIELD_FLOAT},
+ {"y", String(prevY, 3), FIELD_FLOAT}}));
+ }
+ }
+ fetchPending = false;
+ return;
+ }
+
+ if (recharging && isCleaning) {
+ recharging = false;
+ LOG("HIST", "Recharge done — resuming collection");
+ dataLogger.logGenericEvent("history_recharge_end", {});
+ }
+
+ if (!isCleaning && !isDocking && !isSuspended) {
+ fetchPending = false;
+ stopCollection();
+ return;
+ }
+
+ // Paused — keep session open but skip snapshot collection
+ if (isPausedState(state.uiState)) {
+ fetchPending = false;
+ return;
+ }
+ }
+
+ neato.getErr([this](bool errOk, const ErrorData& err) {
+ if (errOk) {
+ if (err.hasError && !prevHadError) {
+ errorsDuringClean++;
+ }
+ prevHadError = err.hasError;
+ }
+
+ if (recharging) {
+ fetchPending = false;
+ return;
+ }
+
+ neato.getRobotPos(true, [this](bool posOk, const RobotPosData& pos) {
+ fetchPending = false;
+ if (!posOk)
+ return;
+
+ float x, y, theta, time;
+ if (!parsePose(pos.raw, x, y, theta, time)) {
+ LOG("HIST", "Failed to parse pose");
+ return;
+ }
+
+ writeSnapshot(x, y, theta, time);
+ });
+ });
+ });
+}
+
+void CleaningHistory::updateAccumulators(float x, float y, float theta) {
+ if (!hasPrevPose) {
+ originX = x;
+ originY = y;
+ prevX = x;
+ prevY = y;
+ prevTheta = theta;
+ hasPrevPose = true;
+ } else {
+ float dx = x - prevX;
+ float dy = y - prevY;
+ totalDistance += sqrtf(dx * dx + dy * dy);
+
+ float dTheta = theta - prevTheta;
+ if (dTheta > 180.0f)
+ dTheta -= 360.0f;
+ if (dTheta < -180.0f)
+ dTheta += 360.0f;
+ totalRotation += fabsf(dTheta);
+
+ prevX = x;
+ prevY = y;
+ prevTheta = theta;
+ }
+
+ float dox = x - originX;
+ float doy = y - originY;
+ float distFromOrigin = sqrtf(dox * dox + doy * doy);
+ if (distFromOrigin > maxDistFromOrigin) {
+ maxDistFromOrigin = distFromOrigin;
+ }
+
+ int ix = static_cast(floorf(x / HISTORY_AREA_CELL_M));
+ int iy = static_cast(floorf(y / HISTORY_AREA_CELL_M));
+ uint32_t cellKey = (static_cast(ix & 0xFFFF) << 16) | static_cast(iy & 0xFFFF);
+ visitedCells.insert(cellKey);
+}
+
+void CleaningHistory::writeSnapshot(float x, float y, float theta, float time) {
+ // Localization resets to origin right before session ends — drop if the
+ // robot was far from origin (genuine return-to-base passes through gradually)
+ if (hasPrevPose && fabsf(x) < 0.001f && fabsf(y) < 0.001f && fabsf(theta) < 0.1f) {
+ float prevDist = sqrtf(prevX * prevX + prevY * prevY);
+ if (prevDist > 1.0f) {
+ return;
+ }
+ }
+
+ String line = "{\"x\":" + String(x, 3) + ",\"y\":" + String(y, 3) + ",\"t\":" + String(theta, 1) +
+ ",\"ts\":" + String(time, 1) + "}";
+
+ updateAccumulators(x, y, theta);
+ bufferLine(line);
+ snapshotCount++;
+
+ if (snapshotCount % 10 == 0) {
+ LOG("HIST", "Snapshot #%u (%.1fm traveled, pose: %.2f,%.2f,%.0f)", snapshotCount, totalDistance, x, y, theta);
+ }
+}
+
+// -- Storage enforcement (mirrors DataLogger::enforceLimits) -----------------
+
+void CleaningHistory::enforceLimits() {
+ // Count session files, sum directory size, and find the oldest in one pass
+ int fileCount = 0;
+ size_t histDirBytes = 0;
+ String oldest;
+ File root = SPIFFS.open(HISTORY_DIR);
+ if (!root || !root.isDirectory())
+ return;
+
+ File entry = root.openNextFile();
+ while (entry) {
+ String name = String(entry.name());
+ histDirBytes += entry.size();
+ if (name.endsWith(".jsonl") || name.endsWith(".jsonl.hs")) {
+ fileCount++;
+ if (oldest.isEmpty() || name < oldest) {
+ oldest = name;
+ }
+ }
+ entry = root.openNextFile();
+ }
+
+ if (oldest.isEmpty())
+ return;
+
+ // History budget: total filesystem cap minus non-history data, floored at minimum reserve
+ size_t total = SPIFFS.totalBytes();
+ size_t globalCap = (total * HISTORY_MAX_FS_PERCENT) / 100;
+ size_t nonHistBytes = SPIFFS.usedBytes() > histDirBytes ? SPIFFS.usedBytes() - histDirBytes : 0;
+ size_t available = globalCap > nonHistBytes ? globalCap - nonHistBytes : 0;
+ size_t minReserved = (total * HISTORY_MIN_FS_PERCENT) / 100;
+ size_t histBudget = available > minReserved ? available : minReserved;
+
+ if (histDirBytes > histBudget || fileCount > HISTORY_MAX_FILES) {
+ String fullPath = String(HISTORY_DIR) + "/" + oldest;
+ LOG("HIST", "Limit: deleting %s (files=%d, histBytes=%u/%u)", fullPath.c_str(), fileCount, histDirBytes,
+ histBudget);
+ SPIFFS.remove(fullPath);
+ metaCache.erase(oldest);
+ }
+}
+
+// -- File management (for API) -----------------------------------------------
+
+void CleaningHistory::readFirstLastLines(const String& path, bool compressed, String& firstLine, String& lastLine) {
+ firstLine = "";
+ lastLine = "";
+
+ if (compressed) {
+ // Stream-decompress keeping only the first and last lines in memory
+ // to avoid buffering the entire file (large sessions can exceed 100KB
+ // decompressed which exhausts ESP32-C3 heap).
+ File f = SPIFFS.open(path, FILE_READ);
+ if (!f)
+ return;
+ CompressedLogReader reader(std::move(f));
+ uint8_t buf[256];
+ size_t n;
+ bool firstDone = false;
+ // Current incomplete line being assembled from chunks
+ String current;
+ while ((n = reader.read(buf, sizeof(buf))) > 0) {
+ for (size_t i = 0; i < n; i++) {
+ char c = static_cast(buf[i]);
+ if (c == '\n') {
+ current.trim();
+ if (current.length() > 0) {
+ if (!firstDone) {
+ firstLine = current;
+ firstDone = true;
+ }
+ // Always overwrite lastLine with the most recent non-empty line
+ lastLine = current;
+ }
+ current = "";
+ } else {
+ current += c;
+ }
+ }
+ }
+ // Handle final chunk without trailing newline
+ current.trim();
+ if (current.length() > 0) {
+ if (!firstDone) {
+ firstLine = current;
+ }
+ lastLine = current;
+ }
+ } else {
+ // Plain .jsonl — read first line directly, seek backward for last line
+ File f = SPIFFS.open(path, FILE_READ);
+ if (!f)
+ return;
+ // Read first line
+ firstLine = f.readStringUntil('\n');
+ firstLine.trim();
+ // Seek backward from end to find last line
+ size_t fileSize = f.size();
+ if (fileSize < 2) {
+ f.close();
+ return;
+ }
+ int pos = static_cast(fileSize) - 2; // Skip trailing newline
+ while (pos >= 0) {
+ f.seek(pos);
+ char c = static_cast(f.read());
+ if (c == '\n') {
+ break;
+ }
+ pos--;
+ }
+ // pos is at the newline before last line, or -1 if only one line
+ if (pos < 0) {
+ f.close();
+ return; // Only one line
+ }
+ f.seek(pos + 1);
+ lastLine = f.readStringUntil('\n');
+ lastLine.trim();
+ f.close();
+ }
+}
+
+std::vector CleaningHistory::listSessions() {
+ std::vector result;
+
+ File root = SPIFFS.open(HISTORY_DIR);
+ if (!root || !root.isDirectory())
+ return result;
+
+ File entry = root.openNextFile();
+ while (entry) {
+ String fullPath = String(entry.path());
+ String name = fullPath;
+ int lastSlash = name.lastIndexOf('/');
+ if (lastSlash >= 0)
+ name = name.substring(lastSlash + 1);
+
+ if (name.endsWith(".jsonl") || name.endsWith(".jsonl.hs")) {
+ // During compression both raw .jsonl and partial .jsonl.hs exist —
+ // skip both until compression finishes and the raw source is deleted
+ if (compressing && (fullPath == compressSrcPath || fullPath == compressDstPath)) {
+ entry = root.openNextFile();
+ continue;
+ }
+
+ // Skip the actively collecting file during disk enumeration.
+ // We will manually append it at the end to avoid thread-safety /
+ // concurrent modification issues with filesystem iterators.
+ if (collecting && fullPath == activeFilePath) {
+ entry = root.openNextFile();
+ continue;
+ }
+
+ HistorySessionInfo info;
+ info.name = name;
+ info.size = entry.size();
+ info.compressed = name.endsWith(".hs");
+
+ // Use cached metadata if available (avoids decompressing .hs files)
+ auto cached = metaCache.find(name);
+ if (cached != metaCache.end()) {
+ info.session = cached->second.session;
+ info.summary = cached->second.summary;
+ } else {
+ String firstLine, lastLine;
+ readFirstLastLines(fullPath, info.compressed, firstLine, lastLine);
+ if (isValidMetaLine(firstLine, "\"type\":\"session\"")) {
+ info.session = firstLine;
+ }
+ if (isValidMetaLine(lastLine, "\"type\":\"summary\"")) {
+ info.summary = lastLine;
+ }
+ // Cache for subsequent requests
+ metaCache[name] = {info.session, info.summary};
+ }
+
+ result.push_back(info);
+ }
+ entry = root.openNextFile();
+ }
+
+ // Always append the active session from memory to ensure exactly one entry
+ if (collecting && !activeFilePath.isEmpty()) {
+ String name = activeFilePath;
+ int lastSlash = name.lastIndexOf('/');
+ if (lastSlash >= 0)
+ name = name.substring(lastSlash + 1);
+
+ HistorySessionInfo info;
+ info.name = name;
+ info.size = activeFile ? activeFile.size() : 0;
+ info.compressed = false;
+ info.recording = true;
+
+ String sessionJson = "{\"type\":\"session\",\"mode\":\"" + cleanMode + "\"";
+ if (sessionStartTime > 0)
+ sessionJson += ",\"time\":" + String(static_cast(sessionStartTime));
+ if (batteryStart >= 0)
+ sessionJson += ",\"battery\":" + String(batteryStart);
+ sessionJson += "}";
+ info.session = sessionJson;
+ // No summary for active session
+
+ result.push_back(info);
+ }
+
+ return result;
+}
+
+std::shared_ptr CleaningHistory::readSession(const String& filename) {
+ String path = String(HISTORY_DIR) + "/" + filename;
+
+ // Refuse to serve files involved in compression (partial .hs is corrupt)
+ if (compressing && (path == compressSrcPath || path == compressDstPath))
+ return nullptr;
+
+ if (!SPIFFS.exists(path))
+ return nullptr;
+
+ File f = SPIFFS.open(path, FILE_READ);
+ if (!f)
+ return nullptr;
+
+ if (filename.endsWith(".hs")) {
+ return std::make_shared(std::move(f));
+ }
+ return std::make_shared(std::move(f));
+}
+
+bool CleaningHistory::deleteSession(const String& filename) {
+ String path = String(HISTORY_DIR) + "/" + filename;
+ if (!SPIFFS.exists(path))
+ return false;
+ metaCache.erase(filename);
+ return SPIFFS.remove(path);
+}
+
+void CleaningHistory::deleteAllSessions() {
+ File root = SPIFFS.open(HISTORY_DIR);
+ if (!root || !root.isDirectory())
+ return;
+
+ std::vector paths;
+ File entry = root.openNextFile();
+ while (entry) {
+ paths.push_back(String(entry.path()));
+ entry = root.openNextFile();
+ }
+
+ for (const auto& p: paths) {
+ SPIFFS.remove(p);
+ }
+ metaCache.clear();
+ LOG("HIST", "Deleted %u session files", paths.size());
+}
+
+// -- Session import (compress-on-write from browser upload) -------------------
+
+bool CleaningHistory::beginImport(const String& filename) {
+ importError = "";
+
+ if (importing) {
+ importError = "Import already in progress";
+ return false;
+ }
+ if (collecting) {
+ importError = "Cannot import while recording";
+ return false;
+ }
+ if (compressing) {
+ importError = "Cannot import while compressing";
+ return false;
+ }
+
+ // Validate filename pattern: .jsonl
+ if (!filename.endsWith(".jsonl")) {
+ importError = "Invalid filename";
+ return false;
+ }
+
+ // Check for duplicate (both raw and compressed)
+ String rawPath = String(HISTORY_DIR) + "/" + filename;
+ String hsPath = rawPath + ".hs";
+ if (SPIFFS.exists(rawPath) || SPIFFS.exists(hsPath)) {
+ importError = "Session already exists";
+ return false;
+ }
+
+ // Open destination file for compressed output
+ importFilePath = hsPath;
+ importFile = SPIFFS.open(importFilePath, FILE_WRITE);
+ if (!importFile) {
+ importError = "Failed to create file";
+ importFilePath = "";
+ return false;
+ }
+
+ heatshrink_encoder_reset(&importEncoder);
+ importBytesReceived = 0;
+ importing = true;
+ LOG("HIST", "Import started: %s", importFilePath.c_str());
+ return true;
+}
+
+bool CleaningHistory::writeImportChunk(const uint8_t *data, size_t len) {
+ if (!importing || !importFile) {
+ importError = "No import in progress";
+ return false;
+ }
+
+ importBytesReceived += len;
+ if (importBytesReceived > HISTORY_IMPORT_MAX_BYTES) {
+ importError = "File too large";
+ importFile.close();
+ SPIFFS.remove(importFilePath);
+ importing = false;
+ return false;
+ }
+
+ static const size_t OUT_BUF_SIZE = 512;
+ uint8_t outBuf[OUT_BUF_SIZE];
+ size_t offset = 0;
+
+ while (offset < len) {
+ size_t sunk = 0;
+ HSE_sink_res sres = heatshrink_encoder_sink(&importEncoder, data + offset, len - offset, &sunk);
+ if (sres < 0) {
+ importError = "Compression error";
+ importFile.close();
+ SPIFFS.remove(importFilePath);
+ importing = false;
+ return false;
+ }
+ offset += sunk;
+
+ // Drain compressed output
+ size_t outSz = 0;
+ HSE_poll_res pres;
+ do {
+ pres = heatshrink_encoder_poll(&importEncoder, outBuf, OUT_BUF_SIZE, &outSz);
+ if (pres < 0) {
+ importError = "Compression error";
+ importFile.close();
+ SPIFFS.remove(importFilePath);
+ importing = false;
+ return false;
+ }
+ if (outSz > 0) {
+ importFile.write(outBuf, outSz);
+ }
+ } while (pres == HSER_POLL_MORE);
+ }
+ return true;
+}
+
+bool CleaningHistory::endImport() {
+ if (!importing || !importFile) {
+ importError = "No import in progress";
+ return false;
+ }
+
+ static const size_t OUT_BUF_SIZE = 512;
+ uint8_t outBuf[OUT_BUF_SIZE];
+
+ // Finish the encoder — may require multiple poll rounds
+ bool done = false;
+ while (!done) {
+ HSE_finish_res fres = heatshrink_encoder_finish(&importEncoder);
+ if (fres < 0) {
+ importError = "Compression finish error";
+ importFile.close();
+ SPIFFS.remove(importFilePath);
+ importing = false;
+ return false;
+ }
+ done = (fres == HSER_FINISH_DONE);
+
+ size_t outSz = 0;
+ HSE_poll_res pres;
+ do {
+ pres = heatshrink_encoder_poll(&importEncoder, outBuf, OUT_BUF_SIZE, &outSz);
+ if (pres < 0) {
+ importError = "Compression finish error";
+ importFile.close();
+ SPIFFS.remove(importFilePath);
+ importing = false;
+ return false;
+ }
+ if (outSz > 0) {
+ importFile.write(outBuf, outSz);
+ }
+ } while (pres == HSER_POLL_MORE);
+ }
+
+ importFile.close();
+ importing = false;
+ LOG("HIST", "Import complete: %s", importFilePath.c_str());
+ dataLogger.logGenericEvent("history_import", {{"path", importFilePath, FIELD_STRING}});
+
+ // Delete oldest sessions if storage budget exceeded
+ enforceLimits();
+
+ return true;
+}
diff --git a/firmware/src/cleaning_history.h b/firmware/src/cleaning_history.h
index 12c987e..5db163f 100644
--- a/firmware/src/cleaning_history.h
+++ b/firmware/src/cleaning_history.h
@@ -1,198 +1,198 @@
-#ifndef CLEANING_HISTORY_H
-#define CLEANING_HISTORY_H
-
-#include
-#include