From cfcb5875beafa70706d45af05f15fb47344b01c9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 14:58:33 +0000 Subject: [PATCH 1/8] Initial plan From bfd10505b5795f56847f1f46b454a8980f90e65c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 15:17:24 +0000 Subject: [PATCH 2/8] Add network security testing + full agent capabilities to Debbie Agent-Logs-Url: https://github.com/magicalmutation-coder/DebbieDoesMobile/sessions/4e986b25-518d-47ad-bb1b-be6cf35d3018 Co-authored-by: magicalmutation-coder <246345154+magicalmutation-coder@users.noreply.github.com> --- .gitignore | 20 + README.md | 317 ++++++++++++++- companion-server/.env.example | 36 ++ companion-server/README.md | 63 +++ companion-server/agent_bridge.js | 94 +++++ companion-server/email_monitor.js | 103 +++++ companion-server/package.json | 29 ++ companion-server/server.js | 182 +++++++++ companion-server/spotify.js | 151 +++++++ companion-server/whatsapp.js | 85 ++++ firmware/CMakeLists.txt | 4 + firmware/main/CMakeLists.txt | 32 ++ firmware/main/audio_manager.c | 174 ++++++++ firmware/main/audio_manager.h | 43 ++ firmware/main/camera_manager.c | 147 +++++++ firmware/main/camera_manager.h | 37 ++ firmware/main/debbie.h | 73 ++++ firmware/main/display_manager.c | 592 ++++++++++++++++++++++++++++ firmware/main/display_manager.h | 53 +++ firmware/main/idf_component.yml | 32 ++ firmware/main/main.c | 527 +++++++++++++++++++++++++ firmware/main/notification_client.c | 205 ++++++++++ firmware/main/notification_client.h | 57 +++ firmware/main/openai_client.c | 528 +++++++++++++++++++++++++ firmware/main/openai_client.h | 91 +++++ firmware/main/settings.h | 117 ++++++ firmware/main/storage_manager.c | 115 ++++++ firmware/main/storage_manager.h | 18 + firmware/main/web_server.c | 250 ++++++++++++ firmware/main/web_server.h | 22 ++ firmware/main/wifi_manager.c | 148 +++++++ firmware/main/wifi_manager.h | 30 ++ firmware/partitions.csv | 8 + firmware/sdkconfig.defaults | 65 +++ 34 files changed, 4447 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 companion-server/.env.example create mode 100644 companion-server/README.md create mode 100644 companion-server/agent_bridge.js create mode 100644 companion-server/email_monitor.js create mode 100644 companion-server/package.json create mode 100644 companion-server/server.js create mode 100644 companion-server/spotify.js create mode 100644 companion-server/whatsapp.js create mode 100644 firmware/CMakeLists.txt create mode 100644 firmware/main/CMakeLists.txt create mode 100644 firmware/main/audio_manager.c create mode 100644 firmware/main/audio_manager.h create mode 100644 firmware/main/camera_manager.c create mode 100644 firmware/main/camera_manager.h create mode 100644 firmware/main/debbie.h create mode 100644 firmware/main/display_manager.c create mode 100644 firmware/main/display_manager.h create mode 100644 firmware/main/idf_component.yml create mode 100644 firmware/main/main.c create mode 100644 firmware/main/notification_client.c create mode 100644 firmware/main/notification_client.h create mode 100644 firmware/main/openai_client.c create mode 100644 firmware/main/openai_client.h create mode 100644 firmware/main/settings.h create mode 100644 firmware/main/storage_manager.c create mode 100644 firmware/main/storage_manager.h create mode 100644 firmware/main/web_server.c create mode 100644 firmware/main/web_server.h create mode 100644 firmware/main/wifi_manager.c create mode 100644 firmware/main/wifi_manager.h create mode 100644 firmware/partitions.csv create mode 100644 firmware/sdkconfig.defaults diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..65aeb00 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# ESP-IDF build artefacts +firmware/build/ +firmware/sdkconfig +firmware/dependencies.lock +firmware/managed_components/ + +# Node.js +companion-server/node_modules/ +companion-server/.env +companion-server/.wwebjs_auth/ +companion-server/.wwebjs_cache/ + +# OS +.DS_Store +Thumbs.db + +# Editor +.vscode/ +*.swp +*.swo diff --git a/README.md b/README.md index 067068a..ac39f7c 100644 --- a/README.md +++ b/README.md @@ -1 +1,316 @@ -# DebbieDoesMobile \ No newline at end of file +# โœจ DebbieDoesMobile + +> **Debbie** โ€” Your Portable Personal AI Friend on the Freenove Media Kit ESP32-S3 + +Debbie is a full-featured personal AI companion that lives on the Freenove Media Kit for ESP32-S3 (FNK0102). She can hold voice conversations, see through the camera, read your WhatsApp and email, control Spotify, and connect to your own AI agent. + +--- + +## ๐Ÿ“ธ Debbie's UI + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ ๐Ÿ“ถ OK ๐Ÿค– AI โ•‘ โœจ Debbie โ•‘ ๐Ÿ”‹ 85% ๐Ÿ”” 3 โ”‚ โ† status bar +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ โ”‚ โ”‚ +โ”‚ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ ๐Ÿ’ฌ Hi! I'm Debbie ๐Ÿ˜Š โ”‚ +โ”‚ ( โ—‰ โ—‰ ) โ”‚ โ”‚ +โ”‚ ( โ€ฟ ) โ”‚ I can chat, see what my โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ camera sees, read your โ”‚ +โ”‚ โ”‚ messages, and control music.โ”‚ +โ”‚ Ready! ๐Ÿ˜Š โ”‚ โ”‚ +โ”‚ โ”‚ Press centre button or โ”‚ +โ”‚ โ”‚ just say something! โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ ๐ŸŽต Artist โ€” Song Title โ–ถ โญ โ”‚ โ† Spotify +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +The face animates โ€” eyes open wide when listening, half-close when thinking, and blink periodically when idle. ๐Ÿ˜Š + +--- + +## ๐ŸŽฏ Features + +| Feature | Description | +|---------|-------------| +| ๐Ÿ—ฃ๏ธ **Voice Conversations** | Real-time bidirectional voice via OpenAI Realtime API (gpt-4o-realtime-preview) | +| ๐Ÿ“ท **Camera Vision** | Capture photos; Debbie describes what she sees using gpt-4o vision | +| ๐Ÿ’ฌ **WhatsApp Pager** | Receive WhatsApp messages as notifications on-device | +| ๐Ÿ“ง **Email Monitor** | IMAP email monitoring โ€” important emails pushed to Debbie | +| ๐ŸŽต **Spotify Control** | Play, pause, skip, search tracks by voice command | +| ๐Ÿ”” **Agent Notifications** | Connect to your own AI agent (e.g. D3881E) for custom notifications | +| โš™๏ธ **Captive-Portal Setup** | First-run web UI at `192.168.4.1` for easy configuration | +| ๐Ÿ”‹ **Battery Monitor** | Battery percentage shown in status bar | +| ๐ŸŒˆ **Animated LVGL UI** | Friendly face with expressions, blink animations, status bar | +| ๐Ÿ”„ **Auto-reconnect** | Automatically reconnects to WiFi and AI if connection drops | +| ๐Ÿ“ฆ **OTA Updates** | Dual-partition layout for over-the-air firmware updates | + +--- + +## ๐Ÿ›’ Hardware Requirements + +| Item | Details | +|------|---------| +| **Main Board** | [Freenove Media Kit ESP32-S3 (FNK0102)](https://www.freenove.com/FNK0102) | +| **Display** | 3.5" 480ร—320 TFT (ST7796) or 1.14" 135ร—240 TFT (ST7789) โ€” comes with kit | +| **Camera** | OV2640 โ€” included in kit | +| **Microphone** | MEMS microphone โ€” included | +| **Speaker** | 4ฮฉ/3W (3.5") or 8ฮฉ/1W (1.14") โ€” included | +| **USB Cable** | USB-C for flashing | +| **WiFi** | 2.4 GHz access point | +| **PC/Server** | To run the companion server (any machine on the same network) | + +--- + +## ๐Ÿš€ Quick Start + +### 1. Set Up the Firmware + +#### Prerequisites +- [ESP-IDF v5.2+](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s3/get-started/index.html) +- Python 3.8+, Git + +```bash +# Clone this repository +git clone https://github.com/magicalmutation-coder/DebbieDoesMobile.git +cd DebbieDoesMobile/firmware + +# Set target to ESP32-S3 +idf.py set-target esp32s3 + +# Build +idf.py build + +# Flash (replace /dev/ttyUSB0 with your port) +idf.py -p /dev/ttyUSB0 flash monitor +``` + +> **Windows:** Use `idf.py -p COM3 flash monitor` + +### 2. First-Run Setup + +1. After flashing, Debbie creates a WiFi hotspot called **"Debbie"** (no password). +2. Connect your phone or computer to **"Debbie"**. +3. Open a browser and navigate to **`http://192.168.4.1`**. +4. Fill in: + - Your home WiFi SSID and password + - Your [OpenAI API key](https://platform.openai.com/api-keys) (starts with `sk-`) + - Companion server URL (optional โ€” see below) +5. Click **Save & Connect**. + +Debbie will reboot, connect to your WiFi, and be ready to chat! ๐ŸŽ‰ + +### 3. Set Up the Companion Server (Optional but recommended) + +The companion server enables WhatsApp, email, and Spotify integration. + +```bash +cd companion-server + +# Install dependencies +npm install + +# Copy and edit environment variables +cp .env.example .env +nano .env # fill in your credentials + +# Start the server +npm start +``` + +The server runs on port 3001 by default. Enter its URL in Debbie's setup page: +`ws://YOUR_PC_IP:3001` + +--- + +## โš™๏ธ Configuration + +All settings are stored in flash (NVS) and can be updated via the web UI at `http://192.168.4.1/` (accessible via the Debbie AP, or on your local network at Debbie's IP address). + +| Setting | Description | Default | +|---------|-------------|---------| +| `wifi_ssid` | Your home WiFi network name | โ€” | +| `wifi_password` | WiFi password | โ€” | +| `openai_api_key` | OpenAI API key (sk-...) | โ€” | +| `agent_ws_url` | Custom agent WebSocket URL | โ€” | +| `companion_url` | Companion server WebSocket URL | โ€” | +| `debbie_name` | Name shown on screen | Debbie | +| `system_prompt` | Debbie's persona prompt | friendly AI companion | +| `speaker_volume` | Speaker volume 0โ€“100 | 75 | + +--- + +## ๐ŸŽฎ Controls + +| Button | Action | +|--------|--------| +| **Centre** | Capture photo + ask Debbie what she sees | +| **Up** | Volume up | +| **Down** | Volume down | +| **Long press** | Read notifications aloud | +| **Voice** | Just speak naturally โ€” Debbie listens automatically | + +--- + +## ๐Ÿ—๏ธ Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Debbie ESP32-S3 โ”‚ +โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” PCM24k โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ MEMS โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–บโ”‚ OpenAI Realtime API (WS) โ”‚ โ”‚ +โ”‚ โ”‚ Mic โ”‚ โ”‚ gpt-4o-realtime-preview โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ PCM24k โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ–ผ โ”‚ +โ”‚ โ”‚ Camera โ”‚โ”€ JPEG b64 โ”€โ–บ gpt-4o vision HTTP โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” WebSocket โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Display โ”‚โ—„โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–บโ”‚ Companion Server (Node.js) โ”‚ โ”‚ +โ”‚ โ”‚ (LVGL) โ”‚ notify โ”‚ WhatsApp โ”‚ Email โ”‚ Spotify โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +### Firmware Components + +| File | Purpose | +|------|---------| +| `main/main.c` | App entry, state machine, event routing | +| `main/openai_client.c` | OpenAI Realtime API + Chat/Vision HTTP | +| `main/audio_manager.c` | I2S mic capture + speaker playback | +| `main/camera_manager.c` | OV2640 capture + JPEG/base64 encoding | +| `main/display_manager.c` | LVGL UI โ€” Debbie's face + status | +| `main/notification_client.c` | WebSocket client for companion server | +| `main/wifi_manager.c` | WiFi STA+AP, auto-reconnect | +| `main/web_server.c` | HTTP config portal + REST API | +| `main/storage_manager.c` | NVS settings persistence | +| `main/settings.h` | All GPIO pin definitions | + +### Companion Server Modules + +| File | Purpose | +|------|---------| +| `server.js` | Express + WebSocket server, device registry | +| `whatsapp.js` | WhatsApp Web.js integration | +| `email_monitor.js` | IMAP email polling | +| `spotify.js` | Spotify Web API controller | +| `agent_bridge.js` | Custom agent WebSocket bridge | + +--- + +## ๐Ÿ”Œ Connecting Your Own Agent + +If you have your own AI agent (e.g. a custom LLM server), connect it via: + +1. **Set the companion server** to point to your agent: + ``` + AGENT_URL=ws://your-agent-host:8080 + ``` + +2. **Or point Debbie directly** to your agent's WebSocket: + ``` + Agent WS URL: ws://your-agent-host:8080 + ``` + When `use_custom_agent` is enabled, Debbie connects to your agent instead of OpenAI for conversations. + +3. **Your agent should send notifications** in this format: + ```json + { "type": "agent", "sender": "Your Agent", "preview": "Message here" } + ``` + +--- + +## ๐ŸŽต Spotify Setup + +1. Create a Spotify developer app at https://developer.spotify.com/dashboard +2. Add redirect URI: `http://localhost:3001/spotify/callback` +3. Add your Client ID and Secret to `companion-server/.env` +4. Start the companion server and visit `http://localhost:3001/spotify/auth` +5. Log in and authorise โ€” copy the refresh token to `.env` + +**Voice commands:** +- *"Play some jazz"* +- *"Skip to the next song"* +- *"Turn the volume up"* +- *"What's playing?"* +- *"Pause the music"* + +> **Audible:** Direct Audible API access is not publicly available. However, you can control Audible playback through your phone (it stays on the phone), and Debbie can serve as a voice remote via the companion server. + +--- + +## ๐Ÿ”ง Customising Debbie's Personality + +Edit the **System Prompt** field in the setup page: + +``` +You are Debbie, a warm and enthusiastic AI companion. You love helping with daily +tasks and always find the bright side of things. You speak in a friendly, casual +tone and use the occasional emoji. You remember what we've talked about and make +connections between conversations. +``` + +--- + +## ๐Ÿ“ Pin Reference (FNK0102) + +| Function | GPIO | +|----------|------| +| LCD MOSI | 35 | +| LCD CLK | 36 | +| LCD CS | 34 | +| LCD DC | 37 | +| LCD RST | 38 | +| LCD BL | 33 | +| Mic SCK | 14 | +| Mic WS | 13 | +| Mic SD | 12 | +| Spk BCK | 17 | +| Spk LRCK | 16 | +| Spk DATA | 15 | +| Cam XCLK | 10 | +| Nav Centre | 45 | +| Nav Up | 41 | +| Nav Down | 42 | +| RGB LED | 40 | +| Battery ADC | 4 | + +> For a different board variant, edit `firmware/main/settings.h` + +--- + +## ๐Ÿ› Troubleshooting + +| Problem | Solution | +|---------|---------| +| No audio | Check I2S pin assignments in `settings.h` | +| Camera black | Verify camera pins; check PSRAM is enabled | +| Can't connect to WiFi | Use 2.4 GHz (not 5 GHz); visit AP for reconfigure | +| OpenAI errors | Verify API key; check credit balance | +| No notifications | Ensure companion server is running and URL is correct | +| Display blank | Check SPI pins and `DEBBIE_DISPLAY_35` define | + +Enable verbose logging: +```bash +idf.py monitor -b 115200 +``` + +--- + +## ๐Ÿ”„ OTA Updates + +Debbie supports over-the-air firmware updates via the dual-OTA partition layout. A REST endpoint for OTA is included in the web server for future implementation. + +--- + +## ๐Ÿ“„ Licence + +MIT โ€” see [LICENSE](LICENSE) + +--- + +*Made with โค๏ธ for the Freenove Media Kit ESP32-S3 community.* \ No newline at end of file diff --git a/companion-server/.env.example b/companion-server/.env.example new file mode 100644 index 0000000..40c7dde --- /dev/null +++ b/companion-server/.env.example @@ -0,0 +1,36 @@ +# Debbie Companion Server โ€” Environment Variables +# Copy this file to .env and fill in your values + +# โ”€โ”€ Server โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +PORT=3001 + +# โ”€โ”€ WhatsApp โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Set to 'false' to disable WhatsApp integration +WHATSAPP_ENABLED=true + +# โ”€โ”€ Email (IMAP) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +EMAIL_HOST=imap.gmail.com +EMAIL_PORT=993 +EMAIL_TLS=true +EMAIL_USER=your@gmail.com +EMAIL_PASSWORD=your-app-password + +# Gmail: create an App Password at https://myaccount.google.com/apppasswords +# Outlook: use imap-mail.outlook.com, port 993 +# Yahoo: use imap.mail.yahoo.com, port 993 + +# โ”€โ”€ Spotify โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# 1. Create app at https://developer.spotify.com/dashboard +# 2. Add redirect URI: http://localhost:3001/spotify/callback +# 3. Start server and visit http://localhost:3001/spotify/auth +# 4. Paste the generated SPOTIFY_REFRESH_TOKEN here after first auth + +SPOTIFY_CLIENT_ID=your_spotify_client_id +SPOTIFY_CLIENT_SECRET=your_spotify_client_secret +SPOTIFY_REDIRECT_URI=http://localhost:3001/spotify/callback +SPOTIFY_REFRESH_TOKEN= + +# โ”€โ”€ Custom Agent (e.g. D3881E) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# WebSocket URL of your custom agent, if available +# Leave blank to use only OpenAI +AGENT_URL=ws://localhost:8080 diff --git a/companion-server/README.md b/companion-server/README.md new file mode 100644 index 0000000..c37a6e6 --- /dev/null +++ b/companion-server/README.md @@ -0,0 +1,63 @@ +# Debbie Companion Server + +The companion server bridges external services (WhatsApp, Email, Spotify, your custom AI agent) to your Debbie ESP32-S3 device over WebSocket. + +## Quick Start + +```bash +npm install +cp .env.example .env +# Edit .env with your credentials +npm start +``` + +The server listens on `ws://0.0.0.0:3001` by default. + +## Enabling Integrations + +### WhatsApp +No additional setup required โ€” on first run, a QR code will appear in the terminal. Scan it with WhatsApp on your phone (Settings โ†’ Linked Devices โ†’ Link a Device). + +### Email +Set the following in `.env`: +``` +EMAIL_HOST=imap.gmail.com # or your mail server +EMAIL_USER=you@gmail.com +EMAIL_PASSWORD=your-app-password +``` + +**Gmail:** Use an [App Password](https://myaccount.google.com/apppasswords), not your main password. + +### Spotify +1. Create an app at https://developer.spotify.com/dashboard +2. Add `http://localhost:3001/spotify/callback` as a redirect URI +3. Set `SPOTIFY_CLIENT_ID` and `SPOTIFY_CLIENT_SECRET` in `.env` +4. Start the server and visit `http://localhost:3001/spotify/auth` +5. Copy the printed refresh token into `.env` as `SPOTIFY_REFRESH_TOKEN` + +### Custom Agent +Set `AGENT_URL` in `.env` to your agent's WebSocket endpoint. + +Your agent should emit JSON in this format: +```json +{ "type": "agent", "sender": "Agent Name", "preview": "Notification text" } +``` + +## REST API + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/health` | Server status | +| POST | `/notify` | Send notification to all devices | +| POST | `/spotify` | Spotify command | + +## Production Deployment + +For a persistent server (e.g. on a Raspberry Pi or home server): + +```bash +npm install -g pm2 +pm2 start server.js --name debbie-companion +pm2 save +pm2 startup +``` diff --git a/companion-server/agent_bridge.js b/companion-server/agent_bridge.js new file mode 100644 index 0000000..cf3186b --- /dev/null +++ b/companion-server/agent_bridge.js @@ -0,0 +1,94 @@ +/** + * agent_bridge.js โ€” Bridge to your custom AI agent (D3881E or similar) + * + * Connects to your agent's WebSocket endpoint and forwards messages + * to connected Debbie devices. + * + * The agent should send JSON messages in the format: + * { "type": "agent", "sender": "Agent", "preview": "Message text" } + * + * Or standard notification format: + * { "type": "notification", "title": "...", "body": "..." } + */ + +const { WebSocket } = require('ws'); + +class AgentBridge { + constructor(agentUrl, onNotification) { + this.agentUrl = agentUrl; + this.onNotification = onNotification; + this.ws = null; + this.reconnectTimer = null; + this.shouldConnect = true; + } + + connect() { + if (!this.agentUrl) return; + console.log(`[agent] Connecting to ${this.agentUrl}...`); + + this.ws = new WebSocket(this.agentUrl, { + handshakeTimeout: 10000, + headers: { + 'User-Agent': 'Debbie-Companion/1.0', + }, + }); + + this.ws.on('open', () => { + console.log('[agent] โœ… Connected to agent'); + /* Announce ourselves */ + this.ws.send(JSON.stringify({ + type: 'register', + client: 'debbie-companion', + })); + }); + + this.ws.on('message', (data) => { + try { + const msg = JSON.parse(data.toString()); + + /* Normalise to Debbie notification format */ + const notification = { + type: 'agent', + sender: msg.sender || msg.title || 'Agent', + preview: msg.preview || msg.body || msg.text || msg.message + || JSON.stringify(msg), + }; + this.onNotification(notification); + } catch (e) { + console.error('[agent] Parse error:', e.message); + } + }); + + this.ws.on('close', () => { + console.warn('[agent] Disconnected โ€” reconnecting in 15s...'); + if (this.shouldConnect) { + this.reconnectTimer = setTimeout(() => this.connect(), 15000); + } + }); + + this.ws.on('error', (err) => { + console.error('[agent] WebSocket error:', err.message); + }); + } + + disconnect() { + this.shouldConnect = false; + clearTimeout(this.reconnectTimer); + if (this.ws) { + this.ws.close(); + this.ws = null; + } + } + + /** + * Send a message or command to your agent. + */ + send(message) { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(typeof message === 'string' + ? message : JSON.stringify(message)); + } + } +} + +module.exports = { AgentBridge }; diff --git a/companion-server/email_monitor.js b/companion-server/email_monitor.js new file mode 100644 index 0000000..99df540 --- /dev/null +++ b/companion-server/email_monitor.js @@ -0,0 +1,103 @@ +/** + * email_monitor.js โ€” IMAP email monitoring + * + * Polls for new unread emails in the INBOX and pushes them as notifications. + * Supports Gmail, Outlook, and any IMAP server. + */ + +const Imap = require('imap'); +const { simpleParser } = require('mailparser'); + +const POLL_INTERVAL_MS = 30000; // check every 30 seconds + +function startEmailMonitor(onNotification) { + const config = { + user: process.env.EMAIL_USER, + password: process.env.EMAIL_PASSWORD, + host: process.env.EMAIL_HOST || 'imap.gmail.com', + port: parseInt(process.env.EMAIL_PORT) || 993, + tls: process.env.EMAIL_TLS !== 'false', + tlsOptions: { rejectUnauthorized: false }, + authTimeout: 10000, + }; + + if (!config.user || !config.password) { + console.warn('[email] EMAIL_USER or EMAIL_PASSWORD not set โ€” skipping'); + return; + } + + let lastSeenUID = 0; + + function checkEmails() { + const imap = new Imap(config); + + imap.once('ready', () => { + imap.openBox('INBOX', false, (err, box) => { + if (err) { imap.end(); return; } + + /* Search for unseen emails */ + imap.search(['UNSEEN'], (searchErr, uids) => { + if (searchErr || !uids || uids.length === 0) { + imap.end(); + return; + } + + const newUIDs = uids.filter(uid => uid > lastSeenUID); + if (newUIDs.length === 0) { imap.end(); return; } + + const fetch = imap.fetch(newUIDs, { + bodies: 'HEADER.FIELDS (FROM SUBJECT)', + struct: true, + }); + + fetch.on('message', (msg, seqno) => { + msg.on('body', (stream) => { + let buffer = ''; + stream.on('data', (chunk) => { buffer += chunk.toString('utf8'); }); + stream.once('end', async () => { + try { + const parsed = await simpleParser(buffer); + const from = parsed.from?.text || 'Unknown'; + const subject = parsed.subject || '(no subject)'; + const preview = subject.length > 80 + ? subject.substring(0, 77) + '...' + : subject; + + onNotification({ + type: 'email', + sender: from.substring(0, 60), + preview: preview, + }); + } catch (e) { + console.error('[email] Parse error:', e.message); + } + }); + }); + + msg.once('attributes', (attrs) => { + if (attrs.uid > lastSeenUID) lastSeenUID = attrs.uid; + }); + }); + + fetch.once('end', () => { imap.end(); }); + fetch.once('error', () => { imap.end(); }); + }); + }); + }); + + imap.once('error', (err) => { + console.error('[email] IMAP error:', err.message); + }); + + imap.connect(); + } + + /* Initial check, then poll */ + checkEmails(); + const interval = setInterval(checkEmails, POLL_INTERVAL_MS); + + console.log(`[email] Monitoring ${config.user} every ${POLL_INTERVAL_MS / 1000}s`); + return () => clearInterval(interval); +} + +module.exports = { startEmailMonitor }; diff --git a/companion-server/package.json b/companion-server/package.json new file mode 100644 index 0000000..90e612d --- /dev/null +++ b/companion-server/package.json @@ -0,0 +1,29 @@ +{ + "name": "debbie-companion-server", + "version": "1.0.0", + "description": "Companion server for Debbie โ€” bridges WhatsApp, Email, Spotify and your custom agent to the ESP32-S3 device", + "main": "server.js", + "scripts": { + "start": "node server.js", + "dev": "nodemon server.js" + }, + "dependencies": { + "express": "^4.18.2", + "ws": "^8.14.2", + "whatsapp-web.js": "^1.23.0", + "qrcode-terminal": "^0.12.0", + "imap": "^0.8.19", + "mailparser": "^3.6.5", + "spotify-web-api-node": "^5.0.2", + "dotenv": "^16.3.1", + "axios": "^1.6.0", + "node-cron": "^3.0.3", + "cors": "^2.8.5" + }, + "devDependencies": { + "nodemon": "^3.0.2" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/companion-server/server.js b/companion-server/server.js new file mode 100644 index 0000000..21b8be1 --- /dev/null +++ b/companion-server/server.js @@ -0,0 +1,182 @@ +/** + * Debbie Companion Server + * + * Bridges notifications and services to the Debbie ESP32-S3 device. + * + * Architecture: + * โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” WebSocket โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + * โ”‚ Debbie ESP32โ”‚ โ—„โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–บ โ”‚ Companion Serverโ”‚ + * โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ push/cmd โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + * โ”‚ + * โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + * โ”‚ WhatsApp Web โ”‚ IMAP Email โ”‚ โ”‚ + * โ”‚ Spotify API โ”‚ Custom Agentโ”‚ โ”‚ + * โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + */ + +require('dotenv').config(); +const express = require('express'); +const http = require('http'); +const { WebSocketServer, WebSocket } = require('ws'); +const cors = require('cors'); + +const { startWhatsApp } = require('./whatsapp'); +const { startEmailMonitor } = require('./email_monitor'); +const { SpotifyController } = require('./spotify'); +const { AgentBridge } = require('./agent_bridge'); + +const app = express(); +const server = http.createServer(app); +const wss = new WebSocketServer({ server }); + +app.use(cors()); +app.use(express.json()); + +/* โ”€โ”€ Connected Debbie devices โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +const devices = new Set(); + +function broadcast(message) { + const payload = typeof message === 'string' ? message : JSON.stringify(message); + for (const ws of devices) { + if (ws.readyState === WebSocket.OPEN) { + ws.send(payload); + } + } +} + +/* โ”€โ”€ Spotify instance โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +let spotify = null; + +/* โ”€โ”€ WebSocket handler โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +wss.on('connection', (ws, req) => { + const ip = req.socket.remoteAddress; + console.log(`[ws] Debbie device connected from ${ip}`); + devices.add(ws); + + ws.send(JSON.stringify({ + type: 'system', + sender: 'companion', + preview: `Connected to Debbie companion server v${require('./package.json').version}`, + })); + + ws.on('message', async (data) => { + try { + const msg = JSON.parse(data.toString()); + console.log('[ws] Received:', msg); + + if (msg.type === 'spotify_command') { + if (spotify) { + const result = await spotify.handleCommand(msg.action); + ws.send(JSON.stringify({ + type: 'spotify', + sender: 'Spotify', + preview: result, + })); + } else { + ws.send(JSON.stringify({ + type: 'spotify', + sender: 'Spotify', + preview: 'Spotify not configured. Add SPOTIFY_* vars to .env', + })); + } + } + } catch (e) { + console.error('[ws] Message parse error:', e.message); + } + }); + + ws.on('close', () => { + console.log(`[ws] Device disconnected: ${ip}`); + devices.delete(ws); + }); + + ws.on('error', (err) => { + console.error('[ws] Error:', err.message); + devices.delete(ws); + }); +}); + +/* โ”€โ”€ REST API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/* Health check */ +app.get('/health', (req, res) => { + res.json({ + status: 'ok', + devices: devices.size, + uptime: process.uptime(), + version: require('./package.json').version, + }); +}); + +/* Send a manual notification to all connected devices */ +app.post('/notify', (req, res) => { + const { type = 'system', sender = 'Server', preview = '' } = req.body; + broadcast({ type, sender, preview }); + res.json({ ok: true, sent_to: devices.size }); +}); + +/* Spotify control via REST */ +app.post('/spotify', async (req, res) => { + if (!spotify) return res.status(503).json({ error: 'Spotify not configured' }); + const { action } = req.body; + const result = await spotify.handleCommand(action); + broadcast({ type: 'spotify', sender: 'Spotify', preview: result }); + res.json({ ok: true, result }); +}); + +/* โ”€โ”€ Start all integrations โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +async function main() { + const PORT = process.env.PORT || 3001; + + /* Spotify */ + if (process.env.SPOTIFY_CLIENT_ID && process.env.SPOTIFY_CLIENT_SECRET) { + spotify = new SpotifyController(); + await spotify.init(); + console.log('[spotify] Controller ready'); + } else { + console.warn('[spotify] No credentials in .env โ€” Spotify disabled'); + } + + /* WhatsApp */ + if (process.env.WHATSAPP_ENABLED !== 'false') { + try { + startWhatsApp((notification) => { + console.log('[whatsapp] Notification:', notification.sender); + broadcast(notification); + }); + } catch (e) { + console.warn('[whatsapp] Could not start:', e.message); + } + } + + /* Email */ + if (process.env.EMAIL_HOST && process.env.EMAIL_USER) { + try { + startEmailMonitor((notification) => { + console.log('[email] New email from:', notification.sender); + broadcast(notification); + }); + } catch (e) { + console.warn('[email] Could not start:', e.message); + } + } else { + console.warn('[email] No IMAP config in .env โ€” email disabled'); + } + + /* Custom agent bridge */ + if (process.env.AGENT_URL) { + const bridge = new AgentBridge(process.env.AGENT_URL, (notification) => { + broadcast(notification); + }); + bridge.connect(); + } + + server.listen(PORT, '0.0.0.0', () => { + console.log(`\n๐Ÿค– Debbie Companion Server running on port ${PORT}`); + console.log(` Devices can connect to: ws://:${PORT}`); + console.log(` Health check: http://localhost:${PORT}/health\n`); + }); +} + +main().catch(console.error); diff --git a/companion-server/spotify.js b/companion-server/spotify.js new file mode 100644 index 0000000..8c4db32 --- /dev/null +++ b/companion-server/spotify.js @@ -0,0 +1,151 @@ +/** + * spotify.js โ€” Spotify playback controller + * + * Uses the Spotify Web API via Client Credentials + Authorization Code flow. + * + * Setup: + * 1. Create an app at https://developer.spotify.com/dashboard + * 2. Set redirect URI to http://localhost:3001/spotify/callback + * 3. Add CLIENT_ID, CLIENT_SECRET, and REDIRECT_URI to .env + * 4. On first run, visit http://localhost:3001/spotify/auth to authorise + */ + +const SpotifyWebApi = require('spotify-web-api-node'); +const express = require('express'); + +class SpotifyController { + constructor() { + this.api = new SpotifyWebApi({ + clientId: process.env.SPOTIFY_CLIENT_ID, + clientSecret: process.env.SPOTIFY_CLIENT_SECRET, + redirectUri: process.env.SPOTIFY_REDIRECT_URI || 'http://localhost:3001/spotify/callback', + }); + + this.currentTrack = null; + this.tokenExpiresAt = 0; + this._refreshToken = process.env.SPOTIFY_REFRESH_TOKEN || null; + } + + async init() { + if (this._refreshToken) { + await this._refreshAccessToken(); + } else { + console.warn('[spotify] No refresh token โ€” visit /spotify/auth to authorise'); + } + } + + async _refreshAccessToken() { + try { + this.api.setRefreshToken(this._refreshToken); + const data = await this.api.refreshAccessToken(); + this.api.setAccessToken(data.body.access_token); + this.tokenExpiresAt = Date.now() + (data.body.expires_in - 60) * 1000; + console.log('[spotify] Access token refreshed'); + } catch (e) { + console.error('[spotify] Token refresh failed:', e.message); + } + } + + async _ensureToken() { + if (Date.now() > this.tokenExpiresAt && this._refreshToken) { + await this._refreshAccessToken(); + } + } + + async handleCommand(action) { + await this._ensureToken(); + + try { + if (action === 'play') { + await this.api.play(); + return 'โ–ถ Playing'; + + } else if (action === 'pause') { + await this.api.pause(); + return 'โธ Paused'; + + } else if (action === 'next') { + await this.api.skipToNext(); + return 'โญ Next track'; + + } else if (action === 'previous') { + await this.api.skipToPrevious(); + return 'โฎ Previous track'; + + } else if (action.startsWith('volume:')) { + const vol = parseInt(action.split(':')[1]); + await this.api.setVolume(vol); + return `๐Ÿ”Š Volume: ${vol}%`; + + } else if (action.startsWith('search:')) { + const query = action.substring(7).trim(); + const results = await this.api.searchTracks(query, { limit: 1 }); + const tracks = results.body?.tracks?.items; + if (tracks && tracks.length > 0) { + const track = tracks[0]; + await this.api.play({ uris: [track.uri] }); + const title = track.name; + const artist = track.artists[0]?.name || 'Unknown'; + this.currentTrack = { title, artist }; + return `โ–ถ Now playing: ${artist} โ€” ${title}`; + } + return `โŒ No results for: ${query}`; + + } else if (action === 'current') { + const playing = await this.api.getMyCurrentPlayingTrack(); + const item = playing.body?.item; + if (item) { + const title = item.name; + const artist = item.artists[0]?.name || 'Unknown'; + this.currentTrack = { title, artist }; + return `๐ŸŽต ${artist} โ€” ${title}`; + } + return 'โน Nothing playing'; + + } else { + return `โ“ Unknown command: ${action}`; + } + } catch (e) { + console.error('[spotify] Command error:', e.message); + return `โŒ Spotify error: ${e.message}`; + } + } + + /** Express router for OAuth callback */ + getRouter() { + const router = express.Router(); + + router.get('/auth', (req, res) => { + const scopes = [ + 'user-read-playback-state', + 'user-modify-playback-state', + 'user-read-currently-playing', + 'streaming', + 'playlist-read-private', + ]; + const url = this.api.createAuthorizeURL(scopes, 'debbie-state'); + res.redirect(url); + }); + + router.get('/callback', async (req, res) => { + const { code } = req.query; + try { + const data = await this.api.authorizationCodeGrant(code); + this.api.setAccessToken(data.body.access_token); + this._refreshToken = data.body.refresh_token; + this.tokenExpiresAt = Date.now() + (data.body.expires_in - 60) * 1000; + console.log('[spotify] โœ… Authorised!'); + console.log('[spotify] Add this to your .env:'); + console.log(`SPOTIFY_REFRESH_TOKEN=${this._refreshToken}`); + res.send('

โœ… Spotify linked to Debbie!

You can close this window.

'); + } catch (e) { + console.error('[spotify] Auth error:', e.message); + res.status(500).send('Auth failed: ' + e.message); + } + }); + + return router; + } +} + +module.exports = { SpotifyController }; diff --git a/companion-server/whatsapp.js b/companion-server/whatsapp.js new file mode 100644 index 0000000..37d12b0 --- /dev/null +++ b/companion-server/whatsapp.js @@ -0,0 +1,85 @@ +/** + * whatsapp.js โ€” WhatsApp integration via whatsapp-web.js + * + * On first run, a QR code is printed to the terminal. + * Scan it with WhatsApp on your phone to link the account. + * The session is persisted via LocalAuth so you only need to scan once. + */ + +const { Client, LocalAuth } = require('whatsapp-web.js'); +const qrcode = require('qrcode-terminal'); + +function startWhatsApp(onNotification) { + const client = new Client({ + authStrategy: new LocalAuth({ dataPath: './.wwebjs_auth' }), + puppeteer: { + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-gpu', + ], + }, + }); + + client.on('qr', (qr) => { + console.log('\n[whatsapp] โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•'); + console.log('[whatsapp] Scan this QR code with WhatsApp to link:'); + console.log('[whatsapp] โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•\n'); + qrcode.generate(qr, { small: true }); + console.log('\n[whatsapp] Waiting for scan...\n'); + }); + + client.on('ready', () => { + console.log('[whatsapp] โœ… WhatsApp connected!'); + }); + + client.on('disconnected', (reason) => { + console.warn('[whatsapp] Disconnected:', reason); + }); + + client.on('message', async (msg) => { + /* Only forward DMs and group mentions, skip status updates */ + if (msg.from === 'status@broadcast') return; + + try { + const contact = await msg.getContact(); + const name = contact.pushname || contact.number || msg.from; + + /* Truncate preview for display */ + let preview = msg.body || '[media]'; + if (preview.length > 100) preview = preview.substring(0, 97) + '...'; + + onNotification({ + type: 'whatsapp', + sender: name, + preview: preview, + }); + } catch (e) { + console.error('[whatsapp] Error processing message:', e.message); + } + }); + + /* Forward call notifications */ + client.on('call', async (call) => { + try { + const contact = await call.getContact(); + const name = contact.pushname || contact.number || call.from; + onNotification({ + type: 'whatsapp', + sender: name, + preview: `๐Ÿ“ž Incoming ${call.isVideo ? 'video' : 'voice'} call`, + }); + } catch (e) { /* ignore */ } + }); + + client.initialize().catch((e) => { + console.error('[whatsapp] Failed to initialize:', e.message); + console.warn('[whatsapp] Make sure Chromium/Puppeteer is available'); + }); + + return client; +} + +module.exports = { startWhatsApp }; diff --git a/firmware/CMakeLists.txt b/firmware/CMakeLists.txt new file mode 100644 index 0000000..29e917f --- /dev/null +++ b/firmware/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(debbie) diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt new file mode 100644 index 0000000..67e41d2 --- /dev/null +++ b/firmware/main/CMakeLists.txt @@ -0,0 +1,32 @@ +idf_component_register( + SRCS + "main.c" + "storage_manager.c" + "wifi_manager.c" + "audio_manager.c" + "camera_manager.c" + "display_manager.c" + "openai_client.c" + "notification_client.c" + "web_server.c" + INCLUDE_DIRS + "." + REQUIRES + nvs_flash + esp_wifi + esp_event + esp_netif + esp_http_client + esp_https_ota + esp_websocket_client + esp_http_server + esp_camera + esp_lcd + driver + lvgl + cJSON + mbedtls + esp_timer + freertos + log +) diff --git a/firmware/main/audio_manager.c b/firmware/main/audio_manager.c new file mode 100644 index 0000000..93b6536 --- /dev/null +++ b/firmware/main/audio_manager.c @@ -0,0 +1,174 @@ +#include "audio_manager.h" +#include "settings.h" +#include "esp_log.h" +#include "driver/i2s_std.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include +#include +#include + +static const char *TAG = "audio"; + +#define BUF_SAMPLES ((AUDIO_SAMPLE_RATE * AUDIO_BUF_MS) / 1000) +#define BUF_BYTES (BUF_SAMPLES * sizeof(int16_t)) + +static i2s_chan_handle_t s_rx_chan = NULL; +static i2s_chan_handle_t s_tx_chan = NULL; +static TaskHandle_t s_cap_task = NULL; +static audio_capture_cb_t s_cap_cb = NULL; +static uint8_t s_volume = 75; +static bool s_capturing = false; + +/* โ”€โ”€ Volume scaling โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void scale_volume(int16_t *buf, size_t count) +{ + float gain = s_volume / 100.0f; + for (size_t i = 0; i < count; i++) { + int32_t v = (int32_t)buf[i] * (int32_t)(gain * 256) >> 8; + buf[i] = (int16_t)(v > 32767 ? 32767 : (v < -32768 ? -32768 : v)); + } +} + +/* โ”€โ”€ Capture task โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void capture_task(void *pvParam) +{ + int16_t *buf = malloc(BUF_BYTES); + if (!buf) { ESP_LOGE(TAG, "OOM in capture task"); vTaskDelete(NULL); return; } + + while (s_capturing) { + size_t bytes_read = 0; + esp_err_t err = i2s_channel_read(s_rx_chan, buf, + BUF_BYTES, &bytes_read, + pdMS_TO_TICKS(100)); + if (err == ESP_OK && bytes_read > 0 && s_cap_cb) { + s_cap_cb(buf, bytes_read / sizeof(int16_t)); + } + } + free(buf); + vTaskDelete(NULL); +} + +/* โ”€โ”€ Public API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +esp_err_t audio_manager_init(void) +{ + /* โ”€โ”€ Microphone (RX) channel โ”€โ”€ */ + i2s_chan_config_t rx_cfg = I2S_CHANNEL_DEFAULT_CONFIG( + I2S_MIC_PORT, I2S_ROLE_MASTER); + ESP_ERROR_CHECK(i2s_new_channel(&rx_cfg, NULL, &s_rx_chan)); + + i2s_std_config_t rx_std = { + .clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(AUDIO_SAMPLE_RATE), + .slot_cfg = I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG( + I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO), + .gpio_cfg = { + .mclk = I2S_GPIO_UNUSED, + .bclk = I2S_MIC_SCK, + .ws = I2S_MIC_WS, + .dout = I2S_GPIO_UNUSED, + .din = I2S_MIC_SD, + .invert_flags = { .mclk_inv = false, .bclk_inv = false, .ws_inv = false }, + }, + }; + ESP_ERROR_CHECK(i2s_channel_init_std_mode(s_rx_chan, &rx_std)); + + /* โ”€โ”€ Speaker (TX) channel โ”€โ”€ */ + i2s_chan_config_t tx_cfg = I2S_CHANNEL_DEFAULT_CONFIG( + I2S_SPK_PORT, I2S_ROLE_MASTER); + ESP_ERROR_CHECK(i2s_new_channel(&tx_cfg, &s_tx_chan, NULL)); + + i2s_std_config_t tx_std = { + .clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(AUDIO_SAMPLE_RATE), + .slot_cfg = I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG( + I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO), + .gpio_cfg = { + .mclk = I2S_GPIO_UNUSED, + .bclk = I2S_SPK_BCK, + .ws = I2S_SPK_LRCK, + .dout = I2S_SPK_DATA, + .din = I2S_GPIO_UNUSED, + .invert_flags = { .mclk_inv = false, .bclk_inv = false, .ws_inv = false }, + }, + }; + ESP_ERROR_CHECK(i2s_channel_init_std_mode(s_tx_chan, &tx_std)); + ESP_ERROR_CHECK(i2s_channel_enable(s_tx_chan)); + + ESP_LOGI(TAG, "Audio initialised โ€” %d Hz, %d-bit", AUDIO_SAMPLE_RATE, AUDIO_BITS); + return ESP_OK; +} + +esp_err_t audio_manager_start_capture(audio_capture_cb_t callback) +{ + if (s_capturing) return ESP_OK; + s_cap_cb = callback; + s_capturing = true; + ESP_ERROR_CHECK(i2s_channel_enable(s_rx_chan)); + xTaskCreatePinnedToCore(capture_task, "audio_cap", 4096, NULL, + configMAX_PRIORITIES - 2, &s_cap_task, 1); + return ESP_OK; +} + +esp_err_t audio_manager_stop_capture(void) +{ + s_capturing = false; + if (s_cap_task) { + vTaskDelay(pdMS_TO_TICKS(200)); + s_cap_task = NULL; + } + i2s_channel_disable(s_rx_chan); + return ESP_OK; +} + +esp_err_t audio_manager_play_pcm(const int16_t *samples, size_t count) +{ + if (!s_tx_chan || !samples || count == 0) return ESP_ERR_INVALID_ARG; + + /* Make a mutable copy to apply volume */ + int16_t *buf = malloc(count * sizeof(int16_t)); + if (!buf) return ESP_ERR_NO_MEM; + memcpy(buf, samples, count * sizeof(int16_t)); + scale_volume(buf, count); + + size_t written = 0; + esp_err_t err = i2s_channel_write(s_tx_chan, buf, + count * sizeof(int16_t), + &written, portMAX_DELAY); + free(buf); + return err; +} + +esp_err_t audio_manager_beep(uint32_t freq_hz, uint32_t duration_ms) +{ + size_t samples = (AUDIO_SAMPLE_RATE * duration_ms) / 1000; + int16_t *buf = malloc(samples * sizeof(int16_t)); + if (!buf) return ESP_ERR_NO_MEM; + + for (size_t i = 0; i < samples; i++) { + double t = (double)i / AUDIO_SAMPLE_RATE; + buf[i] = (int16_t)(8000 * sin(2.0 * M_PI * freq_hz * t)); + } + esp_err_t err = audio_manager_play_pcm(buf, samples); + free(buf); + return err; +} + +esp_err_t audio_manager_set_volume(uint8_t volume) +{ + s_volume = volume > 100 ? 100 : volume; + return ESP_OK; +} + +bool audio_manager_vad(const int16_t *samples, size_t count) +{ + /* Simple RMS energy threshold โ€” tune THRESHOLD for your environment */ + const int32_t THRESHOLD = 800; + int64_t energy = 0; + for (size_t i = 0; i < count; i++) + energy += (int64_t)samples[i] * samples[i]; + int32_t rms = (int32_t)(energy / (int64_t)count); + return (rms > THRESHOLD * THRESHOLD); +} diff --git a/firmware/main/audio_manager.h b/firmware/main/audio_manager.h new file mode 100644 index 0000000..e1ef002 --- /dev/null +++ b/firmware/main/audio_manager.h @@ -0,0 +1,43 @@ +#pragma once +#include "esp_err.h" +#include +#include +#include + +/** + * @brief Initialise I2S microphone and speaker peripherals. + */ +esp_err_t audio_manager_init(void); + +/** + * @brief Start continuous microphone capture. + * Captured PCM frames are delivered via @p callback. + * + * @param callback Called from ISR-safe context with int16_t PCM samples. + * @p samples points to AUDIO_BUF_MS worth of mono 24 kHz data. + * The buffer is invalidated after the callback returns. + */ +typedef void (*audio_capture_cb_t)(const int16_t *samples, size_t count); +esp_err_t audio_manager_start_capture(audio_capture_cb_t callback); +esp_err_t audio_manager_stop_capture(void); + +/** + * @brief Play raw PCM data (int16, mono, 24 kHz) through the speaker. + * Blocks until the buffer has been sent to the I2S DMA. + */ +esp_err_t audio_manager_play_pcm(const int16_t *samples, size_t count); + +/** + * @brief Play a tone (helpful for status sounds). + */ +esp_err_t audio_manager_beep(uint32_t freq_hz, uint32_t duration_ms); + +/** + * @brief Set speaker volume 0โ€“100. + */ +esp_err_t audio_manager_set_volume(uint8_t volume); + +/** + * @brief Detect voice activity in a PCM frame (simple energy threshold). + */ +bool audio_manager_vad(const int16_t *samples, size_t count); diff --git a/firmware/main/camera_manager.c b/firmware/main/camera_manager.c new file mode 100644 index 0000000..91a6ece --- /dev/null +++ b/firmware/main/camera_manager.c @@ -0,0 +1,147 @@ +#include "camera_manager.h" +#include "settings.h" +#include "esp_log.h" +#include "esp_camera.h" +#include "mbedtls/base64.h" +#include +#include + +static const char *TAG = "camera"; +static bool s_initialised = false; + +/* -------------------------------------------------------------------------- */ + +esp_err_t camera_manager_init(void) +{ + camera_config_t config = { + .pin_pwdn = CAM_PIN_PWDN, + .pin_reset = CAM_PIN_RESET, + .pin_xclk = CAM_PIN_XCLK, + .pin_sccb_sda = CAM_PIN_SIOD, + .pin_sccb_scl = CAM_PIN_SIOC, + .pin_d7 = CAM_PIN_D7, + .pin_d6 = CAM_PIN_D6, + .pin_d5 = CAM_PIN_D5, + .pin_d4 = CAM_PIN_D4, + .pin_d3 = CAM_PIN_D3, + .pin_d2 = CAM_PIN_D2, + .pin_d1 = CAM_PIN_D1, + .pin_d0 = CAM_PIN_D0, + .pin_vsync = CAM_PIN_VSYNC, + .pin_href = CAM_PIN_HREF, + .pin_pclk = CAM_PIN_PCLK, + + .xclk_freq_hz = 20000000, + .ledc_timer = LEDC_TIMER_0, + .ledc_channel = LEDC_CHANNEL_0, + + .pixel_format = PIXFORMAT_JPEG, + .frame_size = FRAMESIZE_VGA, /* 640ร—480 โ€” balanced quality/size */ + .jpeg_quality = 12, /* 0โ€“63, lower = better quality */ + .fb_count = 2, + .fb_location = CAMERA_FB_IN_PSRAM, + .grab_mode = CAMERA_GRAB_WHEN_EMPTY, + }; + + esp_err_t err = esp_camera_init(&config); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Camera init failed: %s", esp_err_to_name(err)); + return err; + } + + /* Optimise sensor settings for indoor / handheld use */ + sensor_t *s = esp_camera_sensor_get(); + if (s) { + s->set_brightness(s, 0); + s->set_contrast(s, 0); + s->set_saturation(s, 0); + s->set_whitebal(s, 1); + s->set_awb_gain(s, 1); + s->set_wb_mode(s, 0); /* auto */ + s->set_exposure_ctrl(s, 1); + s->set_aec2(s, 0); + s->set_gain_ctrl(s, 1); + s->set_agc_gain(s, 0); + s->set_gainceiling(s, GAINCEILING_2X); + s->set_bpc(s, 0); + s->set_wpc(s, 1); + s->set_raw_gma(s, 1); + s->set_lenc(s, 1); + s->set_hmirror(s, 0); + s->set_vflip(s, 0); + s->set_dcw(s, 1); + } + + s_initialised = true; + ESP_LOGI(TAG, "Camera ready (OV2640, VGA JPEG)"); + return ESP_OK; +} + +esp_err_t camera_manager_capture_jpeg(uint8_t **jpeg_buf, size_t *jpeg_len) +{ + if (!s_initialised) return ESP_ERR_INVALID_STATE; + + camera_fb_t *fb = esp_camera_fb_get(); + if (!fb) { + ESP_LOGE(TAG, "Camera capture failed"); + return ESP_FAIL; + } + + *jpeg_buf = malloc(fb->len); + if (!*jpeg_buf) { + esp_camera_fb_return(fb); + return ESP_ERR_NO_MEM; + } + memcpy(*jpeg_buf, fb->buf, fb->len); + *jpeg_len = fb->len; + esp_camera_fb_return(fb); + + ESP_LOGD(TAG, "Captured JPEG: %zu bytes", *jpeg_len); + return ESP_OK; +} + +void camera_manager_free_frame(uint8_t *buf) +{ + free(buf); +} + +esp_err_t camera_manager_capture_base64(char **b64_out, size_t *b64_len) +{ + uint8_t *jpeg = NULL; + size_t jpeg_sz = 0; + + esp_err_t err = camera_manager_capture_jpeg(&jpeg, &jpeg_sz); + if (err != ESP_OK) return err; + + /* Calculate required base64 buffer size */ + size_t b64_sz = 0; + mbedtls_base64_encode(NULL, 0, &b64_sz, jpeg, jpeg_sz); + + *b64_out = malloc(b64_sz + 1); + if (!*b64_out) { + free(jpeg); + return ESP_ERR_NO_MEM; + } + + err = mbedtls_base64_encode((unsigned char *)*b64_out, b64_sz + 1, + b64_len, jpeg, jpeg_sz); + (*b64_out)[*b64_len] = '\0'; + free(jpeg); + + if (err != 0) { + free(*b64_out); + *b64_out = NULL; + return ESP_FAIL; + } + + ESP_LOGD(TAG, "Base64 frame: %zu chars", *b64_len); + return ESP_OK; +} + +esp_err_t camera_manager_set_power(bool on) +{ +#if CAM_PIN_PWDN >= 0 + gpio_set_level(CAM_PIN_PWDN, on ? 0 : 1); /* PWDN is active-low */ +#endif + return ESP_OK; +} diff --git a/firmware/main/camera_manager.h b/firmware/main/camera_manager.h new file mode 100644 index 0000000..8f7d23b --- /dev/null +++ b/firmware/main/camera_manager.h @@ -0,0 +1,37 @@ +#pragma once +#include "esp_err.h" +#include +#include + +/** + * @brief Initialise camera hardware. + */ +esp_err_t camera_manager_init(void); + +/** + * @brief Capture a JPEG snapshot. + * The returned buffer MUST be freed with camera_manager_free_frame(). + * + * @param[out] jpeg_buf Pointer to JPEG data. + * @param[out] jpeg_len Length of JPEG data in bytes. + */ +esp_err_t camera_manager_capture_jpeg(uint8_t **jpeg_buf, size_t *jpeg_len); + +/** + * @brief Free a frame previously returned by camera_manager_capture_jpeg(). + */ +void camera_manager_free_frame(uint8_t *buf); + +/** + * @brief Capture a JPEG, encode to base64, and return as a null-terminated + * string. Caller must free() the returned pointer. + * + * @param[out] b64_out Base64 string (malloc'd by this function). + * @param[out] b64_len Length of base64 string (excluding null terminator). + */ +esp_err_t camera_manager_capture_base64(char **b64_out, size_t *b64_len); + +/** + * @brief Enable or disable camera power (if PWDN pin is wired). + */ +esp_err_t camera_manager_set_power(bool on); diff --git a/firmware/main/debbie.h b/firmware/main/debbie.h new file mode 100644 index 0000000..4b11266 --- /dev/null +++ b/firmware/main/debbie.h @@ -0,0 +1,73 @@ +#pragma once +#include +#include +#include "esp_err.h" +#include "esp_event.h" + +/* โ”€โ”€ Debbie shared types โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +typedef enum { + DEBBIE_STATE_BOOT = 0, + DEBBIE_STATE_SETUP, /* captive-portal config */ + DEBBIE_STATE_CONNECTING, /* joining WiFi */ + DEBBIE_STATE_IDLE, /* waiting for user */ + DEBBIE_STATE_LISTENING, /* mic active */ + DEBBIE_STATE_THINKING, /* request sent, waiting */ + DEBBIE_STATE_SPEAKING, /* playing TTS audio */ + DEBBIE_STATE_CAMERA, /* camera preview / capture */ + DEBBIE_STATE_NOTIFICATION, /* showing incoming notification */ + DEBBIE_STATE_SPOTIFY, /* music playback control */ + DEBBIE_STATE_ERROR, +} debbie_state_t; + +typedef enum { + NOTIF_TYPE_WHATSAPP = 0, + NOTIF_TYPE_EMAIL, + NOTIF_TYPE_AGENT, + NOTIF_TYPE_SPOTIFY, + NOTIF_TYPE_SYSTEM, +} notif_type_t; + +typedef struct { + notif_type_t type; + char sender[64]; + char preview[128]; + int64_t timestamp_ms; + bool read; +} debbie_notification_t; + +/* Debbie configuration stored in NVS */ +typedef struct { + char wifi_ssid[64]; + char wifi_password[64]; + char openai_api_key[128]; + char agent_ws_url[256]; /* custom agent endpoint */ + char spotify_token[512]; /* Spotify OAuth token (via companion) */ + char companion_url[256]; /* companion server base URL */ + char debbie_name[32]; /* customisable name, default "Debbie" */ + char system_prompt[512]; /* custom persona prompt */ + bool use_custom_agent; /* use agent endpoint instead of OpenAI */ + uint8_t speaker_volume; /* 0โ€“100 */ + bool notifications_enabled; + bool camera_enabled; +} debbie_config_t; + +/* Event IDs published on the default event loop */ +/* Custom event base for the Debbie application */ +ESP_EVENT_DECLARE_BASE(DEBBIE_EVENT_BASE); + +typedef enum { + DEBBIE_EVT_STATE_CHANGE = 0, + DEBBIE_EVT_NOTIFICATION, + DEBBIE_EVT_WIFI_CONNECTED, + DEBBIE_EVT_WIFI_DISCONNECTED, + DEBBIE_EVT_AI_CONNECTED, + DEBBIE_EVT_AI_DISCONNECTED, + DEBBIE_EVT_CAMERA_FRAME, + DEBBIE_EVT_BUTTON_PRESS, + DEBBIE_EVT_SPOTIFY_UPDATE, +} debbie_event_id_t; + +/* Global state (set by main, read by display/audio/etc.) */ +extern volatile debbie_state_t g_debbie_state; +extern debbie_config_t g_debbie_config; diff --git a/firmware/main/display_manager.c b/firmware/main/display_manager.c new file mode 100644 index 0000000..f9b1ff5 --- /dev/null +++ b/firmware/main/display_manager.c @@ -0,0 +1,592 @@ +/* + * display_manager.c โ€” Debbie's LVGL UI + * + * Layout (3.5" 480ร—320): + * โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + * โ”‚ โ—‰ WiFi โ—‰ AI โ•‘ Debbie โ•‘ ๐Ÿ”‹ 85% ๐Ÿ”” 3 โ”‚ โ† status bar (32 px) + * โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค + * โ”‚ โ”‚ โ”‚ + * โ”‚ Debbie avatar โ”‚ Chat / notification โ”‚ + * โ”‚ (animated) โ”‚ panel โ”‚ + * โ”‚ โ”‚ โ”‚ + * โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค + * โ”‚ ๐ŸŽต Artist โ€” Song title [โ–ถ || โญ] โ”‚ โ† Spotify bar (36 px) + * โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + * + * For the 1.14" (240ร—135) version the avatar fills the screen with a + * simplified overlay. + */ + +#include "display_manager.h" +#include "settings.h" +#include "esp_log.h" +#include "driver/spi_master.h" +#include "driver/gpio.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include "lvgl.h" + +/* Conditional include for display driver */ +#if DEBBIE_DISPLAY_35 +# include "esp_lcd_st7796.h" +# define LCD_CMD_BITS 8 +# define LCD_PARAM_BITS 8 +#else +# include "esp_lcd_st7789.h" +# define LCD_CMD_BITS 8 +# define LCD_PARAM_BITS 8 +#endif + +#include "esp_lcd_panel_io.h" +#include "esp_lcd_panel_vendor.h" +#include "esp_lcd_panel_ops.h" +#include +#include + +static const char *TAG = "display"; + +/* โ”€โ”€ LVGL globals โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +static lv_disp_t *s_disp = NULL; +static SemaphoreHandle_t s_lvgl_mux = NULL; +static esp_lcd_panel_handle_t s_panel = NULL; + +/* โ”€โ”€ UI widget handles โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +/* Status bar */ +static lv_obj_t *s_status_bar = NULL; +static lv_obj_t *s_lbl_name = NULL; +static lv_obj_t *s_lbl_wifi = NULL; +static lv_obj_t *s_lbl_ai = NULL; +static lv_obj_t *s_lbl_battery = NULL; +static lv_obj_t *s_lbl_notif = NULL; + +/* Avatar panel */ +static lv_obj_t *s_avatar_cont = NULL; +static lv_obj_t *s_face_bg = NULL; /* circle */ +static lv_obj_t *s_eye_left = NULL; +static lv_obj_t *s_eye_right = NULL; +static lv_obj_t *s_mouth = NULL; +static lv_obj_t *s_state_lbl = NULL; + +/* Chat / notification panel */ +static lv_obj_t *s_chat_cont = NULL; +static lv_obj_t *s_chat_label = NULL; +static lv_obj_t *s_notif_cont = NULL; +static lv_obj_t *s_notif_label = NULL; + +/* Spotify bar */ +static lv_obj_t *s_spotify_bar = NULL; +static lv_obj_t *s_spotify_lbl = NULL; + +/* Camera preview */ +static lv_obj_t *s_cam_canvas = NULL; + +/* โ”€โ”€ Colour palette โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define CLR_BG_IDLE lv_color_hex(0x1A1A2E) /* deep navy */ +#define CLR_BG_LISTENING lv_color_hex(0x16213E) /* darker blue */ +#define CLR_BG_THINKING lv_color_hex(0x0F3460) /* mid blue */ +#define CLR_BG_SPEAKING lv_color_hex(0x533483) /* purple */ +#define CLR_BG_NOTIF lv_color_hex(0x2C3E50) /* slate */ +#define CLR_FACE lv_color_hex(0xFFD6A5) /* warm peach */ +#define CLR_EYE_OPEN lv_color_hex(0x5E548E) /* violet */ +#define CLR_EYE_CLOSED lv_color_hex(0x9E8FB2) +#define CLR_MOUTH lv_color_hex(0xFF6B6B) /* coral */ +#define CLR_ACCENT lv_color_hex(0x06D6A0) /* teal */ +#define CLR_TEXT_PRIMARY lv_color_hex(0xEEEEEE) +#define CLR_TEXT_SECONDARY lv_color_hex(0xAAAAAA) +#define CLR_SPOTIFY lv_color_hex(0x1DB954) /* Spotify green */ +#define CLR_WHATSAPP lv_color_hex(0x25D366) +#define CLR_EMAIL lv_color_hex(0x4285F4) +#define CLR_NOTIF_BADGE lv_color_hex(0xFF4444) + +/* โ”€โ”€ Eye blink animation โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +static lv_anim_t s_blink_anim; +static int32_t s_eye_height_px = 24; /* set from eye object after creation */ + +static void eye_height_cb(void *obj, int32_t val) +{ + lv_obj_set_height(s_eye_left, val); + lv_obj_set_height(s_eye_right, val); +} + +static void start_blink_animation(void) +{ + lv_anim_init(&s_blink_anim); + lv_anim_set_exec_cb(&s_blink_anim, eye_height_cb); + lv_anim_set_var(&s_blink_anim, s_eye_left); + lv_anim_set_values(&s_blink_anim, s_eye_height_px, 2); + lv_anim_set_time(&s_blink_anim, 120); + lv_anim_set_playback_time(&s_blink_anim, 120); + lv_anim_set_delay(&s_blink_anim, 4000); + lv_anim_set_repeat_count(&s_blink_anim, LV_ANIM_REPEAT_INFINITE); + lv_anim_set_repeat_delay(&s_blink_anim, 4500); + lv_anim_start(&s_blink_anim); +} + +/* โ”€โ”€ LCD flush callback โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +static bool on_color_trans_done(esp_lcd_panel_io_handle_t panel_io, + esp_lcd_panel_io_event_data_t *edata, + void *user_ctx) +{ + lv_disp_flush_ready(s_disp->driver); + return false; +} + +static void lvgl_flush_cb(lv_disp_drv_t *drv, const lv_area_t *area, + lv_color_t *color_map) +{ + esp_lcd_panel_draw_bitmap(s_panel, + area->x1, area->y1, + area->x2 + 1, area->y2 + 1, + color_map); +} + +/* โ”€โ”€ LVGL tick / task โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +static void lvgl_tick_inc(void *arg) +{ + lv_tick_inc(LVGL_TICK_MS); +} + +static void lvgl_task(void *pvParam) +{ + while (1) { + if (xSemaphoreTake(s_lvgl_mux, portMAX_DELAY)) { + lv_task_handler(); + xSemaphoreGive(s_lvgl_mux); + } + vTaskDelay(pdMS_TO_TICKS(5)); + } +} + +/* โ”€โ”€ UI construction โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void build_ui(void) +{ + lv_obj_t *screen = lv_scr_act(); + lv_obj_set_style_bg_color(screen, CLR_BG_IDLE, 0); + lv_obj_set_style_bg_opa(screen, LV_OPA_COVER, 0); + + /* โ”€โ”€ Status bar โ”€โ”€ */ + s_status_bar = lv_obj_create(screen); + lv_obj_set_size(s_status_bar, LCD_WIDTH, 32); + lv_obj_align(s_status_bar, LV_ALIGN_TOP_LEFT, 0, 0); + lv_obj_set_style_bg_color(s_status_bar, lv_color_hex(0x0D0D1A), 0); + lv_obj_set_style_border_width(s_status_bar, 0, 0); + lv_obj_set_style_radius(s_status_bar, 0, 0); + lv_obj_set_style_pad_all(s_status_bar, 4, 0); + lv_obj_clear_flag(s_status_bar, LV_OBJ_FLAG_SCROLLABLE); + + s_lbl_wifi = lv_label_create(s_status_bar); + lv_label_set_text(s_lbl_wifi, LV_SYMBOL_WIFI " --"); + lv_obj_set_style_text_color(s_lbl_wifi, CLR_TEXT_SECONDARY, 0); + lv_obj_align(s_lbl_wifi, LV_ALIGN_LEFT_MID, 4, 0); + + s_lbl_ai = lv_label_create(s_status_bar); + lv_label_set_text(s_lbl_ai, LV_SYMBOL_AUDIO " --"); + lv_obj_set_style_text_color(s_lbl_ai, CLR_TEXT_SECONDARY, 0); + lv_obj_align_to(s_lbl_ai, s_lbl_wifi, LV_ALIGN_OUT_RIGHT_MID, 8, 0); + + s_lbl_name = lv_label_create(s_status_bar); + lv_label_set_text(s_lbl_name, "โœจ Debbie"); + lv_obj_set_style_text_color(s_lbl_name, CLR_ACCENT, 0); + lv_obj_align(s_lbl_name, LV_ALIGN_CENTER, 0, 0); + + s_lbl_battery = lv_label_create(s_status_bar); + lv_label_set_text(s_lbl_battery, LV_SYMBOL_BATTERY_FULL " --"); + lv_obj_set_style_text_color(s_lbl_battery, CLR_TEXT_SECONDARY, 0); + lv_obj_align(s_lbl_battery, LV_ALIGN_RIGHT_MID, -50, 0); + + s_lbl_notif = lv_label_create(s_status_bar); + lv_label_set_text(s_lbl_notif, LV_SYMBOL_BELL); + lv_obj_set_style_text_color(s_lbl_notif, CLR_TEXT_SECONDARY, 0); + lv_obj_align(s_lbl_notif, LV_ALIGN_RIGHT_MID, -4, 0); + + /* โ”€โ”€ Avatar panel (left half below status bar) โ”€โ”€ */ + int avatar_w = LCD_WIDTH / 2; + int content_h = LCD_HEIGHT - 32 - 36; /* minus status + spotify bars */ + + s_avatar_cont = lv_obj_create(screen); + lv_obj_set_size(s_avatar_cont, avatar_w, content_h); + lv_obj_align(s_avatar_cont, LV_ALIGN_TOP_LEFT, 0, 32); + lv_obj_set_style_bg_color(s_avatar_cont, CLR_BG_IDLE, 0); + lv_obj_set_style_border_width(s_avatar_cont, 0, 0); + lv_obj_set_style_radius(s_avatar_cont, 0, 0); + lv_obj_clear_flag(s_avatar_cont, LV_OBJ_FLAG_SCROLLABLE); + + /* Face background circle */ + int face_r = (avatar_w < content_h ? avatar_w : content_h) / 2 - 16; + s_face_bg = lv_obj_create(s_avatar_cont); + lv_obj_set_size(s_face_bg, face_r * 2, face_r * 2); + lv_obj_align(s_face_bg, LV_ALIGN_CENTER, 0, -8); + lv_obj_set_style_radius(s_face_bg, LV_RADIUS_CIRCLE, 0); + lv_obj_set_style_bg_color(s_face_bg, CLR_FACE, 0); + lv_obj_set_style_border_color(s_face_bg, CLR_ACCENT, 0); + lv_obj_set_style_border_width(s_face_bg, 3, 0); + lv_obj_set_style_shadow_color(s_face_bg, CLR_ACCENT, 0); + lv_obj_set_style_shadow_width(s_face_bg, 20, 0); + lv_obj_clear_flag(s_face_bg, LV_OBJ_FLAG_SCROLLABLE); + + /* Eyes */ + int eye_w = face_r / 4; + int eye_h = face_r / 3; + s_eye_left = lv_obj_create(s_face_bg); + lv_obj_set_size(s_eye_left, eye_w, eye_h); + lv_obj_align(s_eye_left, LV_ALIGN_CENTER, -(face_r / 3), -(face_r / 6)); + lv_obj_set_style_radius(s_eye_left, LV_RADIUS_CIRCLE, 0); + lv_obj_set_style_bg_color(s_eye_left, CLR_EYE_OPEN, 0); + lv_obj_set_style_border_width(s_eye_left, 0, 0); + + s_eye_right = lv_obj_create(s_face_bg); + lv_obj_set_size(s_eye_right, eye_w, eye_h); + lv_obj_align(s_eye_right, LV_ALIGN_CENTER, (face_r / 3), -(face_r / 6)); + lv_obj_set_style_radius(s_eye_right, LV_RADIUS_CIRCLE, 0); + lv_obj_set_style_bg_color(s_eye_right, CLR_EYE_OPEN, 0); + lv_obj_set_style_border_width(s_eye_right, 0, 0); + + /* Mouth (arc approximation with a label for simplicity) */ + s_mouth = lv_arc_create(s_face_bg); + lv_arc_set_angles(s_mouth, 200, 340); /* lower half arc = smile */ + lv_arc_set_value(s_mouth, 0); + lv_obj_set_size(s_mouth, face_r, face_r / 2); + lv_obj_align(s_mouth, LV_ALIGN_CENTER, 0, face_r / 4); + lv_obj_set_style_arc_color(s_mouth, CLR_MOUTH, LV_PART_INDICATOR); + lv_obj_set_style_arc_color(s_mouth, CLR_MOUTH, LV_PART_MAIN); + lv_obj_set_style_arc_width(s_mouth, 4, LV_PART_MAIN); + lv_obj_set_style_arc_width(s_mouth, 0, LV_PART_INDICATOR); + lv_obj_set_style_bg_opa(s_mouth, LV_OPA_TRANSP, 0); + lv_obj_remove_style(s_mouth, NULL, LV_PART_KNOB); + lv_obj_clear_flag(s_mouth, LV_OBJ_FLAG_CLICKABLE); + + /* State label below avatar */ + s_state_lbl = lv_label_create(s_avatar_cont); + lv_label_set_text(s_state_lbl, "Hello! ๐Ÿ‘‹"); + lv_obj_set_style_text_color(s_state_lbl, CLR_ACCENT, 0); + lv_obj_align(s_state_lbl, LV_ALIGN_BOTTOM_MID, 0, -4); + + s_eye_height_px = eye_h; + + /* โ”€โ”€ Chat / notification panel (right half) โ”€โ”€ */ + s_chat_cont = lv_obj_create(screen); + lv_obj_set_size(s_chat_cont, LCD_WIDTH - avatar_w, content_h); + lv_obj_align(s_chat_cont, LV_ALIGN_TOP_RIGHT, 0, 32); + lv_obj_set_style_bg_color(s_chat_cont, lv_color_hex(0x12122A), 0); + lv_obj_set_style_border_width(s_chat_cont, 0, 0); + lv_obj_set_style_radius(s_chat_cont, 0, 0); + lv_obj_set_style_pad_all(s_chat_cont, 8, 0); + + s_chat_label = lv_label_create(s_chat_cont); + lv_label_set_long_mode(s_chat_label, LV_LABEL_LONG_WRAP); + lv_obj_set_width(s_chat_label, LCD_WIDTH - avatar_w - 16); + lv_label_set_text(s_chat_label, + "Hi! I'm Debbie ๐Ÿ˜Š\n" + "I can chat, show you what\n" + "my camera sees, read your\n" + "messages, and control music.\n\n" + "Press the centre button\nor just say something!"); + lv_obj_set_style_text_color(s_chat_label, CLR_TEXT_PRIMARY, 0); + lv_obj_align(s_chat_label, LV_ALIGN_TOP_LEFT, 0, 0); + + /* Spotify bar */ + s_spotify_bar = lv_obj_create(screen); + lv_obj_set_size(s_spotify_bar, LCD_WIDTH, 36); + lv_obj_align(s_spotify_bar, LV_ALIGN_BOTTOM_LEFT, 0, 0); + lv_obj_set_style_bg_color(s_spotify_bar, lv_color_hex(0x121212), 0); + lv_obj_set_style_border_width(s_spotify_bar, 0, 0); + lv_obj_set_style_radius(s_spotify_bar, 0, 0); + lv_obj_set_style_pad_all(s_spotify_bar, 4, 0); + lv_obj_clear_flag(s_spotify_bar, LV_OBJ_FLAG_SCROLLABLE); + + s_spotify_lbl = lv_label_create(s_spotify_bar); + lv_label_set_text(s_spotify_lbl, LV_SYMBOL_AUDIO " Not connected to Spotify"); + lv_obj_set_style_text_color(s_spotify_lbl, lv_color_hex(0x888888), 0); + lv_obj_align(s_spotify_lbl, LV_ALIGN_LEFT_MID, 4, 0); + + /* Start blink animation */ + start_blink_animation(); + + ESP_LOGI(TAG, "UI built โ€” LCD %dx%d", LCD_WIDTH, LCD_HEIGHT); +} + +/* โ”€โ”€ Public API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +esp_err_t display_manager_init(void) +{ + s_lvgl_mux = xSemaphoreCreateMutex(); + + /* โ”€โ”€ SPI bus โ”€โ”€ */ + spi_bus_config_t bus_cfg = { + .mosi_io_num = LCD_PIN_MOSI, + .miso_io_num = -1, + .sclk_io_num = LCD_PIN_CLK, + .quadwp_io_num = -1, + .quadhd_io_num = -1, + }; + ESP_ERROR_CHECK(spi_bus_initialize(LCD_SPI_HOST, &bus_cfg, SPI_DMA_CH_AUTO)); + + /* โ”€โ”€ Panel IO โ”€โ”€ */ + esp_lcd_panel_io_handle_t io; + esp_lcd_panel_io_spi_config_t io_cfg = { + .dc_gpio_num = LCD_PIN_DC, + .cs_gpio_num = LCD_PIN_CS, + .pclk_hz = LCD_SPI_FREQ_HZ, + .lcd_cmd_bits = LCD_CMD_BITS, + .lcd_param_bits = LCD_PARAM_BITS, + .spi_mode = 0, + .trans_queue_depth = 10, + .on_color_trans_done = on_color_trans_done, + }; + ESP_ERROR_CHECK(esp_lcd_new_panel_io_spi((esp_lcd_spi_bus_handle_t)LCD_SPI_HOST, + &io_cfg, &io)); + + /* โ”€โ”€ Panel driver โ”€โ”€ */ + esp_lcd_panel_dev_config_t panel_cfg = { + .reset_gpio_num = LCD_PIN_RST, + .color_space = ESP_LCD_COLOR_SPACE_BGR, + .bits_per_pixel = 16, + }; +#if DEBBIE_DISPLAY_35 + ESP_ERROR_CHECK(esp_lcd_new_panel_st7796(io, &panel_cfg, &s_panel)); +#else + ESP_ERROR_CHECK(esp_lcd_new_panel_st7789(io, &panel_cfg, &s_panel)); +#endif + ESP_ERROR_CHECK(esp_lcd_panel_reset(s_panel)); + ESP_ERROR_CHECK(esp_lcd_panel_init(s_panel)); + ESP_ERROR_CHECK(esp_lcd_panel_disp_on_off(s_panel, true)); + + /* Backlight on */ + gpio_set_direction(LCD_PIN_BL, GPIO_MODE_OUTPUT); + gpio_set_level(LCD_PIN_BL, 1); + + /* โ”€โ”€ LVGL โ”€โ”€ */ + lv_init(); + + static lv_disp_draw_buf_t draw_buf; + static lv_color_t buf1[LCD_WIDTH * 40]; + static lv_color_t buf2[LCD_WIDTH * 40]; + lv_disp_draw_buf_init(&draw_buf, buf1, buf2, LCD_WIDTH * 40); + + static lv_disp_drv_t disp_drv; + lv_disp_drv_init(&disp_drv); + disp_drv.hor_res = LCD_WIDTH; + disp_drv.ver_res = LCD_HEIGHT; + disp_drv.flush_cb = lvgl_flush_cb; + disp_drv.draw_buf = &draw_buf; + s_disp = lv_disp_drv_register(&disp_drv); + + /* LVGL tick timer */ + const esp_timer_create_args_t tick_args = { + .callback = lvgl_tick_inc, + .name = "lvgl_tick", + }; + esp_timer_handle_t tick_timer; + ESP_ERROR_CHECK(esp_timer_create(&tick_args, &tick_timer)); + ESP_ERROR_CHECK(esp_timer_start_periodic(tick_timer, + LVGL_TICK_MS * 1000 /* ยตs */)); + + /* Build the initial UI */ + if (xSemaphoreTake(s_lvgl_mux, portMAX_DELAY)) { + build_ui(); + xSemaphoreGive(s_lvgl_mux); + } + + /* LVGL task */ + xTaskCreatePinnedToCore(lvgl_task, "lvgl", 8192, NULL, + configMAX_PRIORITIES - 1, NULL, 1); + + ESP_LOGI(TAG, "Display ready โ€” %s %dx%d", LCD_DRIVER, LCD_WIDTH, LCD_HEIGHT); + return ESP_OK; +} + +/* โ”€โ”€ State โ†’ face expression mapping โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void set_face_for_state(debbie_state_t state) +{ + if (!s_face_bg) return; + lv_color_t bg; + const char *status_text; + lv_coord_t mouth_start, mouth_end; + + switch (state) { + case DEBBIE_STATE_LISTENING: + bg = CLR_BG_LISTENING; + status_text = "Listening... ๐Ÿ‘‚"; + mouth_start = 200; mouth_end = 340; /* big smile */ + /* Animate eyes โ€” wider when listening */ + lv_obj_set_height(s_eye_left, s_eye_height_px + 4); + lv_obj_set_height(s_eye_right, s_eye_height_px + 4); + break; + case DEBBIE_STATE_THINKING: + bg = CLR_BG_THINKING; + status_text = "Thinking... ๐Ÿค”"; + mouth_start = 220; mouth_end = 320; /* neutral */ + lv_obj_set_height(s_eye_left, s_eye_height_px / 2); + lv_obj_set_height(s_eye_right, s_eye_height_px / 2); + break; + case DEBBIE_STATE_SPEAKING: + bg = CLR_BG_SPEAKING; + status_text = "Speaking ๐Ÿ—ฃ๏ธ"; + mouth_start = 180; mouth_end = 360; /* open mouth */ + lv_obj_set_height(s_eye_left, s_eye_height_px); + lv_obj_set_height(s_eye_right, s_eye_height_px); + break; + case DEBBIE_STATE_NOTIFICATION: + bg = CLR_BG_NOTIF; + status_text = "Notification! ๐Ÿ””"; + mouth_start = 200; mouth_end = 340; + break; + case DEBBIE_STATE_CAMERA: + bg = CLR_BG_IDLE; + status_text = "Camera ๐Ÿ“ท"; + mouth_start = 200; mouth_end = 340; + break; + case DEBBIE_STATE_SPOTIFY: + bg = CLR_BG_IDLE; + status_text = "Music ๐ŸŽต"; + mouth_start = 200; mouth_end = 340; + break; + case DEBBIE_STATE_SETUP: + bg = lv_color_hex(0x2D3436); + status_text = "Setup mode โš™๏ธ"; + mouth_start = 210; mouth_end = 330; + break; + case DEBBIE_STATE_CONNECTING: + bg = lv_color_hex(0x2D3436); + status_text = "Connecting... ๐Ÿ”—"; + mouth_start = 210; mouth_end = 330; + break; + case DEBBIE_STATE_ERROR: + bg = lv_color_hex(0x5C1E1E); + status_text = "Oops! ๐Ÿ˜…"; + mouth_start = 200; mouth_end = 340; + break; + case DEBBIE_STATE_IDLE: + default: + bg = CLR_BG_IDLE; + status_text = "Ready! ๐Ÿ˜Š"; + mouth_start = 200; mouth_end = 340; + lv_obj_set_height(s_eye_left, s_eye_height_px); + lv_obj_set_height(s_eye_right, s_eye_height_px); + break; + } + + lv_obj_set_style_bg_color(lv_scr_act(), bg, 0); + lv_obj_set_style_bg_color(s_avatar_cont, bg, 0); + lv_arc_set_angles(s_mouth, mouth_start, mouth_end); + lv_label_set_text(s_state_lbl, status_text); +} + +void display_manager_set_state(debbie_state_t state) +{ + if (!s_lvgl_mux) return; + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + set_face_for_state(state); + xSemaphoreGive(s_lvgl_mux); + } +} + +void display_manager_show_notification(const debbie_notification_t *notif) +{ + if (!notif || !s_chat_label) return; + const char *icon; + switch (notif->type) { + case NOTIF_TYPE_WHATSAPP: icon = "๐Ÿ’ฌ"; break; + case NOTIF_TYPE_EMAIL: icon = "๐Ÿ“ง"; break; + case NOTIF_TYPE_SPOTIFY: icon = "๐ŸŽต"; break; + default: icon = "๐Ÿ””"; break; + } + char buf[220]; + snprintf(buf, sizeof(buf), "%s %s\n%s", icon, notif->sender, notif->preview); + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + lv_label_set_text(s_chat_label, buf); + xSemaphoreGive(s_lvgl_mux); + } +} + +void display_manager_show_text(const char *text) +{ + if (!text || !s_chat_label) return; + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + lv_label_set_text(s_chat_label, text); + xSemaphoreGive(s_lvgl_mux); + } +} + +void display_manager_show_camera_frame(const uint8_t *rgb565) +{ + /* For simplicity we reuse the chat panel area for the camera preview. + * A full implementation would use lv_canvas with a dedicated buffer. */ + if (!s_chat_label) return; + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + if (rgb565 == NULL) { + lv_label_set_text(s_chat_label, ""); + } else { + lv_label_set_text(s_chat_label, "๐Ÿ“ท Camera preview"); + /* TODO: Render rgb565 into lv_canvas for full preview */ + } + xSemaphoreGive(s_lvgl_mux); + } +} + +void display_manager_set_notif_count(int count) +{ + if (!s_lbl_notif) return; + char buf[16]; + if (count > 0) { + snprintf(buf, sizeof(buf), LV_SYMBOL_BELL " %d", count); + lv_obj_set_style_text_color(s_lbl_notif, CLR_NOTIF_BADGE, 0); + } else { + snprintf(buf, sizeof(buf), LV_SYMBOL_BELL); + lv_obj_set_style_text_color(s_lbl_notif, CLR_TEXT_SECONDARY, 0); + } + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + lv_label_set_text(s_lbl_notif, buf); + xSemaphoreGive(s_lvgl_mux); + } +} + +void display_manager_set_spotify_track(const char *artist, const char *title, + bool is_playing) +{ + if (!s_spotify_lbl) return; + char buf[128]; + snprintf(buf, sizeof(buf), "%s %s โ€” %s", + is_playing ? "โ–ถ" : "โธ", artist ? artist : "", title ? title : ""); + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + lv_label_set_text(s_spotify_lbl, buf); + lv_obj_set_style_text_color(s_spotify_lbl, CLR_SPOTIFY, 0); + xSemaphoreGive(s_lvgl_mux); + } +} + +void display_manager_set_battery(uint8_t percent) +{ + if (!s_lbl_battery) return; + const char *icon; + if (percent > 80) icon = LV_SYMBOL_BATTERY_FULL; + else if (percent > 60) icon = LV_SYMBOL_BATTERY_3; + else if (percent > 40) icon = LV_SYMBOL_BATTERY_2; + else if (percent > 15) icon = LV_SYMBOL_BATTERY_1; + else icon = LV_SYMBOL_BATTERY_EMPTY; + char buf[16]; + snprintf(buf, sizeof(buf), "%s %d%%", icon, percent); + lv_color_t col = percent > 20 ? CLR_TEXT_SECONDARY : lv_color_hex(0xFF4444); + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + lv_label_set_text(s_lbl_battery, buf); + lv_obj_set_style_text_color(s_lbl_battery, col, 0); + xSemaphoreGive(s_lvgl_mux); + } +} + +void display_manager_set_connection_status(bool wifi_ok, bool ai_ok) +{ + if (!s_lbl_wifi || !s_lbl_ai) return; + if (xSemaphoreTake(s_lvgl_mux, pdMS_TO_TICKS(50))) { + lv_label_set_text(s_lbl_wifi, wifi_ok ? LV_SYMBOL_WIFI " OK" : LV_SYMBOL_WIFI " --"); + lv_obj_set_style_text_color(s_lbl_wifi, + wifi_ok ? CLR_ACCENT : CLR_TEXT_SECONDARY, 0); + lv_label_set_text(s_lbl_ai, ai_ok ? LV_SYMBOL_AUDIO " AI" : LV_SYMBOL_AUDIO " --"); + lv_obj_set_style_text_color(s_lbl_ai, + ai_ok ? CLR_ACCENT : CLR_TEXT_SECONDARY, 0); + xSemaphoreGive(s_lvgl_mux); + } +} diff --git a/firmware/main/display_manager.h b/firmware/main/display_manager.h new file mode 100644 index 0000000..3a2e778 --- /dev/null +++ b/firmware/main/display_manager.h @@ -0,0 +1,53 @@ +#pragma once +#include "esp_err.h" +#include "debbie.h" +#include + +/** + * @brief Initialise the SPI LCD and LVGL graphics stack. + * Creates the LVGL timer task and renders the Debbie avatar UI. + */ +esp_err_t display_manager_init(void); + +/** + * @brief Update the UI to reflect the new application state. + * Safe to call from any task โ€” posts to the LVGL task internally. + */ +void display_manager_set_state(debbie_state_t state); + +/** + * @brief Show a notification badge and brief overlay. + */ +void display_manager_show_notification(const debbie_notification_t *notif); + +/** + * @brief Show a text bubble (AI transcript / message). + */ +void display_manager_show_text(const char *text); + +/** + * @brief Show a camera preview frame (RGB565, LCD_WIDTH ร— LCD_HEIGHT). + * Pass NULL to hide the preview. + */ +void display_manager_show_camera_frame(const uint8_t *rgb565); + +/** + * @brief Update the unread notification count badge. + */ +void display_manager_set_notif_count(int count); + +/** + * @brief Update the Spotify "now playing" info on screen. + */ +void display_manager_set_spotify_track(const char *artist, const char *title, + bool is_playing); + +/** + * @brief Update battery percentage shown in status bar. + */ +void display_manager_set_battery(uint8_t percent); + +/** + * @brief Update WiFi / AI connection icons in status bar. + */ +void display_manager_set_connection_status(bool wifi_ok, bool ai_ok); diff --git a/firmware/main/idf_component.yml b/firmware/main/idf_component.yml new file mode 100644 index 0000000..417fce8 --- /dev/null +++ b/firmware/main/idf_component.yml @@ -0,0 +1,32 @@ +## IDF Component Manager manifest +## https://docs.espressif.com/projects/idf-component-manager/ + +dependencies: + idf: ">=5.2.0" + + ## WebSocket client (for OpenAI Realtime API + companion server) + espressif/esp_websocket_client: + version: ">=1.1.0" + + ## LVGL v8 display library + lvgl/lvgl: + version: "~8.3.0" + + ## LCD panel drivers (ST7789 / ST7796) + espressif/esp_lcd_st7789: + version: ">=1.0.0" + + espressif/esp_lcd_st7796: + version: ">=1.0.0" + + ## ESP camera component + espressif/esp_camera: + version: ">=2.0.0" + + ## JSON library + espressif/cjson: + version: ">=1.7.14" + + ## HTTP client + idf_component_manager/idf_component_manager: + version: ">=2.0.0" diff --git a/firmware/main/main.c b/firmware/main/main.c new file mode 100644 index 0000000..f1e7e05 --- /dev/null +++ b/firmware/main/main.c @@ -0,0 +1,527 @@ +/* + * main.c โ€” Debbie: Portable Personal AI Friend + * + * Freenove Media Kit ESP32-S3 (FNK0102) + * + * Features: + * โœ… Voice conversations via OpenAI Realtime API (gpt-4o-realtime-preview) + * โœ… Camera vision โ€” capture and describe images using gpt-4o + * โœ… WhatsApp / Email / agent notification pager + * โœ… Spotify & Audible playback control + * โœ… Friendly LVGL avatar UI with animated expressions + * โœ… WiFi captive-portal setup + * โœ… Custom agent endpoint support (ws:// or wss://) + * โœ… Battery monitoring + * โœ… OTA-ready partition layout + */ + +#include "debbie.h" +#include "settings.h" +#include "storage_manager.h" +#include "wifi_manager.h" +#include "audio_manager.h" +#include "camera_manager.h" +#include "display_manager.h" +#include "openai_client.h" +#include "notification_client.h" +#include "web_server.h" + +#include "esp_log.h" +#include "esp_timer.h" +#include "cJSON.h" +#include "esp_system.h" +#include "driver/gpio.h" +#include "driver/adc.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/queue.h" +#include "freertos/event_groups.h" + +static const char *TAG = "debbie"; + +/* โ”€โ”€ Global state (declared in debbie.h) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +volatile debbie_state_t g_debbie_state = DEBBIE_STATE_BOOT; +debbie_config_t g_debbie_config = { 0 }; +ESP_EVENT_DEFINE_BASE(DEBBIE_EVENT_BASE); + +/* โ”€โ”€ Event queue for button / async events โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +typedef enum { + BTN_EVT_CENTER = 0, + BTN_EVT_UP, + BTN_EVT_DOWN, + BTN_EVT_LONG_PRESS, +} btn_event_t; + +static QueueHandle_t s_btn_queue = NULL; + +/* โ”€โ”€ Voice activity detection state โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +static bool s_user_speaking = false; +static uint32_t s_silence_ms = 0; +static uint32_t s_vad_start_ms = 0; +#define VAD_SILENCE_TIMEOUT_MS 1200 /* stop after 1.2 s of silence */ +#define VAD_MIN_SPEECH_MS 200 /* ignore very short bursts */ + +/* โ”€โ”€ Pending notification โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +static debbie_notification_t s_pending_notif = { 0 }; +static bool s_has_notif = false; + +/* -------------------------------------------------------------------------- */ + +static void set_state(debbie_state_t new_state) +{ + g_debbie_state = new_state; + display_manager_set_state(new_state); + esp_event_post(DEBBIE_EVENT_BASE, DEBBIE_EVT_STATE_CHANGE, + &new_state, sizeof(new_state), 0); +} + +/* โ”€โ”€ OpenAI event callback โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void on_oai_event(const oai_event_data_t *evt, void *ctx) +{ + switch (evt->type) { + case OAI_EVT_CONNECTED: + ESP_LOGI(TAG, "Connected to OpenAI Realtime API"); + display_manager_set_connection_status(true, true); + set_state(DEBBIE_STATE_IDLE); + audio_manager_beep(880, 120); + vTaskDelay(pdMS_TO_TICKS(80)); + audio_manager_beep(1100, 120); + break; + + case OAI_EVT_DISCONNECTED: + ESP_LOGW(TAG, "Disconnected from OpenAI"); + display_manager_set_connection_status(true, false); + set_state(DEBBIE_STATE_IDLE); + break; + + case OAI_EVT_SESSION_CREATED: + ESP_LOGI(TAG, "Session ready"); + break; + + case OAI_EVT_AUDIO_DELTA: + /* Play audio immediately as it streams */ + set_state(DEBBIE_STATE_SPEAKING); + audio_manager_play_pcm(evt->audio.pcm, evt->audio.count); + break; + + case OAI_EVT_TRANSCRIPT: + ESP_LOGI(TAG, "Transcript: %s", evt->transcript.text); + display_manager_show_text(evt->transcript.text); + break; + + case OAI_EVT_FUNCTION_CALL: + ESP_LOGI(TAG, "Function call: %s(%s)", evt->fn.name, evt->fn.args_json); + + if (strcmp(evt->fn.name, "capture_image") == 0) { + /* Capture a photo and send it back to the model */ + set_state(DEBBIE_STATE_CAMERA); + char *b64 = NULL; + size_t b64_len = 0; + if (camera_manager_capture_base64(&b64, &b64_len) == ESP_OK && b64) { + display_manager_show_text("๐Ÿ“ท Captured! Analysing..."); + openai_client_send_image(b64, + "Please describe in detail what you see in this image, " + "including objects, people, text, colours, and anything notable."); + free(b64); + openai_client_send_function_result( + "capture_image", + "{\"status\":\"ok\",\"message\":\"Image captured and sent for analysis\"}"); + } else { + openai_client_send_function_result( + "capture_image", + "{\"status\":\"error\",\"message\":\"Camera capture failed\"}"); + } + set_state(DEBBIE_STATE_THINKING); + + } else if (strcmp(evt->fn.name, "read_notifications") == 0) { + char *summary = notification_client_get_summary_json(); + if (summary) { + char result[512]; + snprintf(result, sizeof(result), + "{\"notifications\":%s}", summary); + openai_client_send_function_result("read_notifications", result); + free(summary); + notification_client_clear(); + display_manager_set_notif_count(0); + } else { + openai_client_send_function_result( + "read_notifications", + "{\"notifications\":[],\"message\":\"No pending notifications\"}"); + } + + } else if (strcmp(evt->fn.name, "spotify_control") == 0) { + /* Parse action from args_json */ + cJSON *args = cJSON_Parse(evt->fn.args_json); + cJSON *act = args ? cJSON_GetObjectItem(args, "action") : NULL; + if (act && cJSON_IsString(act)) { + notification_client_spotify_command(act->valuestring); + char result[128]; + snprintf(result, sizeof(result), + "{\"status\":\"ok\",\"action\":\"%s\"}", act->valuestring); + openai_client_send_function_result("spotify_control", result); + set_state(DEBBIE_STATE_SPOTIFY); + } + if (args) cJSON_Delete(args); + } + break; + + case OAI_EVT_ERROR: + ESP_LOGE(TAG, "OpenAI error: %s", evt->error.message); + display_manager_show_text("๐Ÿ˜… Something went wrong.\nPlease try again."); + set_state(DEBBIE_STATE_ERROR); + vTaskDelay(pdMS_TO_TICKS(2000)); + set_state(DEBBIE_STATE_IDLE); + break; + + default: + break; + } +} + +/* โ”€โ”€ Notification callback โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void on_notification(const debbie_notification_t *notif, void *ctx) +{ + s_pending_notif = *notif; + s_has_notif = true; + + int count = notification_client_unread_count(); + display_manager_set_notif_count(count); + display_manager_show_notification(notif); + set_state(DEBBIE_STATE_NOTIFICATION); + + /* Gentle beep for notifications */ + audio_manager_beep(660, 80); + + /* Tell the AI about it if connected */ + if (openai_client_is_connected()) { + char msg[256]; + const char *type_str; + switch (notif->type) { + case NOTIF_TYPE_WHATSAPP: type_str = "WhatsApp"; break; + case NOTIF_TYPE_EMAIL: type_str = "email"; break; + case NOTIF_TYPE_AGENT: type_str = "agent"; break; + default: type_str = "notification"; break; + } + snprintf(msg, sizeof(msg), + "[System: New %s from %s: \"%s\". " + "Please let the user know briefly.]", + type_str, notif->sender, notif->preview); + openai_client_send_text(msg); + } + + vTaskDelay(pdMS_TO_TICKS(3000)); + set_state(DEBBIE_STATE_IDLE); +} + +/* โ”€โ”€ Microphone / VAD callback โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void on_audio_capture(const int16_t *samples, size_t count) +{ + if (g_debbie_state != DEBBIE_STATE_LISTENING && + g_debbie_state != DEBBIE_STATE_IDLE) { + return; + } + + bool voice = audio_manager_vad(samples, count); + + if (voice) { + s_silence_ms = 0; + if (!s_user_speaking) { + s_user_speaking = true; + s_vad_start_ms = (uint32_t)(esp_timer_get_time() / 1000); + set_state(DEBBIE_STATE_LISTENING); + } + /* Stream audio to OpenAI */ + if (openai_client_is_connected()) + openai_client_send_audio(samples, count); + + } else if (s_user_speaking) { + s_silence_ms += AUDIO_BUF_MS; + if (s_silence_ms >= VAD_SILENCE_TIMEOUT_MS) { + uint32_t now_ms = (uint32_t)(esp_timer_get_time() / 1000); + uint32_t speech_ms = now_ms - s_vad_start_ms; + s_user_speaking = false; + s_silence_ms = 0; + + if (speech_ms >= VAD_MIN_SPEECH_MS && openai_client_is_connected()) { + set_state(DEBBIE_STATE_THINKING); + openai_client_commit_audio(); + } else { + set_state(DEBBIE_STATE_IDLE); + } + } + } +} + +/* โ”€โ”€ Button ISR / debounce task โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void IRAM_ATTR gpio_isr_handler(void *arg) +{ + btn_event_t evt = (btn_event_t)(uintptr_t)arg; + xQueueSendFromISR(s_btn_queue, &evt, NULL); +} + +static void button_task(void *pvParam) +{ + btn_event_t evt; + while (1) { + if (xQueueReceive(s_btn_queue, &evt, portMAX_DELAY)) { + vTaskDelay(pdMS_TO_TICKS(50)); /* debounce */ + + switch (evt) { + case BTN_EVT_CENTER: + /* Centre button: capture image or toggle listening */ + if (g_debbie_state == DEBBIE_STATE_IDLE || + g_debbie_state == DEBBIE_STATE_LISTENING) { + set_state(DEBBIE_STATE_CAMERA); + char *b64 = NULL; + size_t b64_len = 0; + if (camera_manager_capture_base64(&b64, &b64_len) == ESP_OK && b64) { + display_manager_show_text("๐Ÿ“ท What do you see, Debbie?"); + openai_client_send_image(b64, + "What can you see in this photo? Describe it naturally."); + free(b64); + set_state(DEBBIE_STATE_THINKING); + } else { + set_state(DEBBIE_STATE_IDLE); + } + } + break; + + case BTN_EVT_UP: + /* Volume up */ + if (g_debbie_config.speaker_volume < 100) + g_debbie_config.speaker_volume += 10; + audio_manager_set_volume(g_debbie_config.speaker_volume); + { + char msg[32]; + snprintf(msg, sizeof(msg), "๐Ÿ”Š Volume: %d%%", + g_debbie_config.speaker_volume); + display_manager_show_text(msg); + } + break; + + case BTN_EVT_DOWN: + /* Volume down */ + if (g_debbie_config.speaker_volume > 0) + g_debbie_config.speaker_volume -= 10; + audio_manager_set_volume(g_debbie_config.speaker_volume); + { + char msg[32]; + snprintf(msg, sizeof(msg), "๐Ÿ”‰ Volume: %d%%", + g_debbie_config.speaker_volume); + display_manager_show_text(msg); + } + break; + + case BTN_EVT_LONG_PRESS: + /* Long press: read notifications aloud */ + if (openai_client_is_connected()) { + char *summary = notification_client_get_summary_json(); + if (summary) { + char text[512]; + snprintf(text, sizeof(text), + "[System: Please read out the following notifications " + "to the user: %s]", summary); + openai_client_send_text(text); + free(summary); + notification_client_clear(); + display_manager_set_notif_count(0); + } else { + openai_client_send_text("Are there any notifications for me?"); + } + } + break; + } + } + } +} + +/* โ”€โ”€ Battery monitoring task โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void battery_task(void *pvParam) +{ + adc1_config_width(ADC_WIDTH_BIT_12); + adc1_config_channel_atten(ADC1_CHANNEL_3, ADC_ATTEN_DB_11); + + while (1) { + int raw = adc1_get_raw(ADC1_CHANNEL_3); + float v = (raw / 4095.0f) * 3.3f * BAT_DIVIDER_RATIO; + /* Approximate Li-Ion percentage: 4.2 V = 100%, 3.2 V = 0% */ + int percent = (int)((v - 3.2f) / (4.2f - 3.2f) * 100.0f); + if (percent > 100) percent = 100; + if (percent < 0) percent = 0; + display_manager_set_battery((uint8_t)percent); + vTaskDelay(pdMS_TO_TICKS(30000)); /* check every 30 s */ + } +} + +/* โ”€โ”€ GPIO setup โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void gpio_init(void) +{ + s_btn_queue = xQueueCreate(8, sizeof(btn_event_t)); + + gpio_config_t io_cfg = { + .mode = GPIO_MODE_INPUT, + .pull_up_en = GPIO_PULLUP_ENABLE, + .pull_down_en = GPIO_PULLDOWN_DISABLE, + .intr_type = GPIO_INTR_NEGEDGE, + }; + + io_cfg.pin_bit_mask = (1ULL << NAV_CENTER); + gpio_config(&io_cfg); + io_cfg.pin_bit_mask = (1ULL << NAV_UP); + gpio_config(&io_cfg); + io_cfg.pin_bit_mask = (1ULL << NAV_DOWN); + gpio_config(&io_cfg); + + gpio_install_isr_service(0); + gpio_isr_handler_add(NAV_CENTER, gpio_isr_handler, (void *)BTN_EVT_CENTER); + gpio_isr_handler_add(NAV_UP, gpio_isr_handler, (void *)BTN_EVT_UP); + gpio_isr_handler_add(NAV_DOWN, gpio_isr_handler, (void *)BTN_EVT_DOWN); + + xTaskCreate(button_task, "btn", 4096, NULL, 5, NULL); +} + +/* โ”€โ”€ Boot splash โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void show_boot_splash(void) +{ + display_manager_show_text( + "โœจ Debbie starting up...\n\n" + "Please wait while I connect\n" + "to WiFi and the AI...\n\n" + "This takes about 10 seconds."); + audio_manager_beep(440, 100); + vTaskDelay(pdMS_TO_TICKS(50)); + audio_manager_beep(550, 100); + vTaskDelay(pdMS_TO_TICKS(50)); + audio_manager_beep(660, 200); +} + +/* โ”€โ”€ App entry point โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +void app_main(void) +{ + ESP_LOGI(TAG, "โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—"); + ESP_LOGI(TAG, "โ•‘ Debbie โ€” v1.0.0 โ•‘"); + ESP_LOGI(TAG, "โ•‘ Portable Personal AI Friend โ•‘"); + ESP_LOGI(TAG, "โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•"); + + /* 1. NVS + configuration */ + ESP_ERROR_CHECK(storage_init()); + + /* 2. Display โ€” show boot splash as early as possible */ + ESP_ERROR_CHECK(display_manager_init()); + set_state(DEBBIE_STATE_BOOT); + show_boot_splash(); + + /* 3. Audio */ + ESP_ERROR_CHECK(audio_manager_init()); + audio_manager_set_volume(g_debbie_config.speaker_volume); + + /* 4. Camera */ + if (g_debbie_config.camera_enabled) { + esp_err_t cam_err = camera_manager_init(); + if (cam_err != ESP_OK) { + ESP_LOGW(TAG, "Camera init failed โ€” camera features disabled"); + g_debbie_config.camera_enabled = false; + } + } + + /* 5. GPIO / buttons */ + gpio_init(); + + /* 6. WiFi */ + set_state(DEBBIE_STATE_CONNECTING); + display_manager_show_text("๐Ÿ“ถ Connecting to WiFi..."); + display_manager_set_connection_status(false, false); + + esp_err_t wifi_err = wifi_manager_init(); + + /* 7. Always start the web server (for setup and status) */ + ESP_ERROR_CHECK(web_server_start()); + + if (wifi_err != ESP_OK) { + /* No WiFi โ€” enter setup mode */ + set_state(DEBBIE_STATE_SETUP); + display_manager_show_text( + "โš™๏ธ Setup needed!\n\n" + "Connect to WiFi: \"Debbie\"\n" + "Then visit: 192.168.4.1\n" + "to configure your settings."); + ESP_LOGI(TAG, "No WiFi โ€” in setup mode at %s", DEBBIE_AP_IP); + + /* Stay in setup mode loop */ + while (1) vTaskDelay(pdMS_TO_TICKS(1000)); + } + + display_manager_set_connection_status(true, false); + display_manager_show_text("โœ… Connected!\nConnecting to AI..."); + + /* 8. Companion server notifications */ + if (g_debbie_config.notifications_enabled && + strlen(g_debbie_config.companion_url) > 0) { + esp_err_t notif_err = notification_client_init( + g_debbie_config.companion_url, + on_notification, NULL); + if (notif_err == ESP_OK) { + ESP_LOGI(TAG, "Notification client connecting to %s", + g_debbie_config.companion_url); + } + } else { + ESP_LOGI(TAG, "Companion server not configured โ€” notifications disabled"); + } + + /* 9. Battery monitor */ + xTaskCreate(battery_task, "battery", 2048, NULL, 1, NULL); + + /* 10. Connect to AI */ + if (strlen(g_debbie_config.openai_api_key) > 0) { + esp_err_t ai_err = openai_client_connect( + g_debbie_config.openai_api_key, + g_debbie_config.system_prompt, + on_oai_event, NULL); + + if (ai_err != ESP_OK) { + ESP_LOGW(TAG, "Could not connect to OpenAI โ€” check API key"); + display_manager_show_text( + "โš ๏ธ AI connection failed.\n" + "Check your API key at\n" + "192.168.4.1 or say 'setup'."); + set_state(DEBBIE_STATE_IDLE); + } + } else { + ESP_LOGW(TAG, "No OpenAI API key โ€” visit 192.168.4.1 to configure"); + display_manager_show_text( + "๐Ÿ‘‹ Hi! I'm Debbie!\n\n" + "Please add your API key:\n" + "Visit 192.168.4.1 in\nyour browser.\n\n" + "Or connect me to your\nown AI agent!"); + set_state(DEBBIE_STATE_IDLE); + } + + /* 11. Start microphone โ€” continuous VAD listening */ + ESP_ERROR_CHECK(audio_manager_start_capture(on_audio_capture)); + ESP_LOGI(TAG, "Debbie is ready! ๐ŸŽ‰"); + + /* Main loop โ€” lightweight, most work is in callbacks and tasks */ + while (1) { + /* Periodic reconnect if AI disconnects */ + if (g_debbie_state != DEBBIE_STATE_SETUP && + wifi_manager_is_connected() && + !openai_client_is_connected() && + strlen(g_debbie_config.openai_api_key) > 0) { + ESP_LOGI(TAG, "Reconnecting to OpenAI..."); + display_manager_show_text("Reconnecting..."); + openai_client_connect( + g_debbie_config.openai_api_key, + g_debbie_config.system_prompt, + on_oai_event, NULL); + } + vTaskDelay(pdMS_TO_TICKS(15000)); + } +} diff --git a/firmware/main/notification_client.c b/firmware/main/notification_client.c new file mode 100644 index 0000000..0dee646 --- /dev/null +++ b/firmware/main/notification_client.c @@ -0,0 +1,205 @@ +#include "notification_client.h" +#include "settings.h" +#include "esp_log.h" +#include "esp_websocket_client.h" +#include "cJSON.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include +#include + +static const char *TAG = "notif"; + +#define MAX_NOTIFS 32 + +static esp_websocket_client_handle_t s_ws = NULL; +static notif_cb_t s_cb = NULL; +static void *s_ctx = NULL; +static bool s_connected = false; +static SemaphoreHandle_t s_mutex = NULL; + +static debbie_notification_t s_notifs[MAX_NOTIFS]; +static int s_notif_count = 0; +static int s_unread = 0; + +/* -------------------------------------------------------------------------- */ + +static void ws_event_handler(void *arg, esp_event_base_t base, + int32_t id, void *data) +{ + esp_websocket_event_data_t *evt = (esp_websocket_event_data_t *)data; + + switch (id) { + case WEBSOCKET_EVENT_CONNECTED: + s_connected = true; + ESP_LOGI(TAG, "Connected to companion server"); + break; + + case WEBSOCKET_EVENT_DISCONNECTED: + s_connected = false; + ESP_LOGW(TAG, "Disconnected from companion server"); + break; + + case WEBSOCKET_EVENT_DATA: { + if (!evt->data_ptr || evt->data_len == 0) break; + char *msg = malloc(evt->data_len + 1); + if (!msg) break; + memcpy(msg, evt->data_ptr, evt->data_len); + msg[evt->data_len] = '\0'; + + cJSON *json = cJSON_Parse(msg); + free(msg); + if (!json) break; + + cJSON *type_j = cJSON_GetObjectItem(json, "type"); + cJSON *sender_j = cJSON_GetObjectItem(json, "sender"); + cJSON *preview_j = cJSON_GetObjectItem(json, "preview"); + + if (type_j && cJSON_IsString(type_j)) { + debbie_notification_t notif = { 0 }; + + const char *type_str = type_j->valuestring; + if (strcmp(type_str, "whatsapp") == 0) notif.type = NOTIF_TYPE_WHATSAPP; + else if (strcmp(type_str, "email") == 0) notif.type = NOTIF_TYPE_EMAIL; + else if (strcmp(type_str, "agent") == 0) notif.type = NOTIF_TYPE_AGENT; + else if (strcmp(type_str, "spotify") == 0) notif.type = NOTIF_TYPE_SPOTIFY; + else notif.type = NOTIF_TYPE_SYSTEM; + + if (sender_j && cJSON_IsString(sender_j)) + strncpy(notif.sender, sender_j->valuestring, sizeof(notif.sender) - 1); + if (preview_j && cJSON_IsString(preview_j)) + strncpy(notif.preview, preview_j->valuestring, sizeof(notif.preview) - 1); + + notif.timestamp_ms = esp_timer_get_time() / 1000; + notif.read = false; + + /* Store in ring buffer */ + if (xSemaphoreTake(s_mutex, pdMS_TO_TICKS(100))) { + int idx = s_notif_count % MAX_NOTIFS; + s_notifs[idx] = notif; + s_notif_count++; + s_unread++; + xSemaphoreGive(s_mutex); + } + + ESP_LOGI(TAG, "Notification [%s] from %s: %s", + type_str, notif.sender, notif.preview); + + if (s_cb) s_cb(¬if, s_ctx); + } + + cJSON_Delete(json); + break; + } + + case WEBSOCKET_EVENT_ERROR: + ESP_LOGE(TAG, "Companion server WebSocket error"); + break; + + default: + break; + } +} + +/* -------------------------------------------------------------------------- */ + +esp_err_t notification_client_init(const char *server_url, + notif_cb_t cb, + void *user_ctx) +{ + if (!server_url || strlen(server_url) == 0) { + ESP_LOGW(TAG, "No companion server URL โ€” notifications disabled"); + return ESP_ERR_INVALID_ARG; + } + + s_cb = cb; + s_ctx = user_ctx; + s_mutex = xSemaphoreCreateMutex(); + + const esp_websocket_client_config_t ws_cfg = { + .uri = server_url, + .disable_auto_reconnect = false, + .reconnect_timeout_ms = 10000, + .network_timeout_ms = 5000, + .buffer_size = 4096, + .task_stack = 4096, + }; + + s_ws = esp_websocket_client_init(&ws_cfg); + if (!s_ws) return ESP_FAIL; + + ESP_ERROR_CHECK(esp_websocket_register_events(s_ws, WEBSOCKET_EVENT_ANY, + ws_event_handler, NULL)); + ESP_ERROR_CHECK(esp_websocket_client_start(s_ws)); + + ESP_LOGI(TAG, "Connecting to companion server: %s", server_url); + return ESP_OK; +} + +esp_err_t notification_client_deinit(void) +{ + if (s_ws) { + esp_websocket_client_stop(s_ws); + esp_websocket_client_destroy(s_ws); + s_ws = NULL; + } + s_connected = false; + return ESP_OK; +} + +bool notification_client_is_connected(void) { return s_connected; } + +int notification_client_unread_count(void) { return s_unread; } + +void notification_client_clear(void) +{ + if (xSemaphoreTake(s_mutex, pdMS_TO_TICKS(100))) { + s_unread = 0; + for (int i = 0; i < MAX_NOTIFS; i++) s_notifs[i].read = true; + xSemaphoreGive(s_mutex); + } +} + +char *notification_client_get_summary_json(void) +{ + cJSON *arr = cJSON_CreateArray(); + if (xSemaphoreTake(s_mutex, pdMS_TO_TICKS(200))) { + int count = s_notif_count < MAX_NOTIFS ? s_notif_count : MAX_NOTIFS; + for (int i = 0; i < count; i++) { + debbie_notification_t *n = &s_notifs[i]; + if (n->read) continue; + cJSON *obj = cJSON_CreateObject(); + const char *type_str; + switch (n->type) { + case NOTIF_TYPE_WHATSAPP: type_str = "whatsapp"; break; + case NOTIF_TYPE_EMAIL: type_str = "email"; break; + case NOTIF_TYPE_AGENT: type_str = "agent"; break; + case NOTIF_TYPE_SPOTIFY: type_str = "spotify"; break; + default: type_str = "system"; break; + } + cJSON_AddStringToObject(obj, "type", type_str); + cJSON_AddStringToObject(obj, "sender", n->sender); + cJSON_AddStringToObject(obj, "preview", n->preview); + cJSON_AddItemToArray(arr, obj); + } + xSemaphoreGive(s_mutex); + } + char *out = cJSON_PrintUnformatted(arr); + cJSON_Delete(arr); + return out; +} + +esp_err_t notification_client_spotify_command(const char *action) +{ + if (!s_connected || !action) return ESP_ERR_INVALID_STATE; + cJSON *cmd = cJSON_CreateObject(); + cJSON_AddStringToObject(cmd, "type", "spotify_command"); + cJSON_AddStringToObject(cmd, "action", action); + char *str = cJSON_PrintUnformatted(cmd); + cJSON_Delete(cmd); + if (!str) return ESP_ERR_NO_MEM; + esp_websocket_client_send_text(s_ws, str, strlen(str), portMAX_DELAY); + free(str); + return ESP_OK; +} diff --git a/firmware/main/notification_client.h b/firmware/main/notification_client.h new file mode 100644 index 0000000..754b73d --- /dev/null +++ b/firmware/main/notification_client.h @@ -0,0 +1,57 @@ +#pragma once +#include "esp_err.h" +#include "debbie.h" +#include + +/** + * @brief Connect to the companion server WebSocket for push notifications. + * + * The companion server (Node.js) bridges WhatsApp, email, Spotify, and + * custom agent notifications. Messages arrive as JSON on the WebSocket + * and are surfaced via the callback set in notification_client_init(). + */ + +typedef void (*notif_cb_t)(const debbie_notification_t *notif, void *user_ctx); + +/** + * @brief Initialise and connect to the companion server. + * + * @param server_url WebSocket URL of companion server, e.g. "ws://192.168.1.10:3001" + * @param cb Callback invoked on new notification (task context). + * @param user_ctx Passed through to callback. + */ +esp_err_t notification_client_init(const char *server_url, + notif_cb_t cb, + void *user_ctx); + +/** + * @brief Disconnect from the companion server. + */ +esp_err_t notification_client_deinit(void); + +/** + * @brief Return true if connected to companion server. + */ +bool notification_client_is_connected(void); + +/** + * @brief Get the count of unread notifications. + */ +int notification_client_unread_count(void); + +/** + * @brief Mark all notifications as read. + */ +void notification_client_clear(void); + +/** + * @brief Get a summary of pending notifications as a JSON string. + * Caller must free() the returned pointer. + */ +char *notification_client_get_summary_json(void); + +/** + * @brief Send a Spotify command to the companion server. + * action examples: "play", "pause", "next", "search:lofi beats" + */ +esp_err_t notification_client_spotify_command(const char *action); diff --git a/firmware/main/openai_client.c b/firmware/main/openai_client.c new file mode 100644 index 0000000..5cbb3c7 --- /dev/null +++ b/firmware/main/openai_client.c @@ -0,0 +1,528 @@ +/* + * openai_client.c โ€” OpenAI Realtime API WebSocket client + * + * Voice path : ESP32 mic โ†’ PCM16/24kHz โ†’ OpenAI โ†’ PCM16/24kHz โ†’ speaker + * Vision path : Camera JPEG โ†’ base64 โ†’ Chat Completions vision endpoint + * + * Protocol reference: + * https://platform.openai.com/docs/guides/realtime-model-capabilities + */ + +#include "openai_client.h" +#include "settings.h" +#include "debbie.h" +#include "esp_log.h" +#include "esp_websocket_client.h" +#include "esp_http_client.h" +#include "cJSON.h" +#include "mbedtls/base64.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include +#include + +static const char *TAG = "oai"; + +/* โ”€โ”€ State โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +static esp_websocket_client_handle_t s_ws = NULL; +static oai_event_cb_t s_cb = NULL; +static void *s_ctx = NULL; +static SemaphoreHandle_t s_mutex = NULL; +static bool s_connected = false; +static char s_api_key[128]; +static char s_current_response_id[64]; + +/* โ”€โ”€ Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/* Encode int16 PCM โ†’ base64 for the Realtime protocol */ +static char *pcm_to_base64(const int16_t *pcm, size_t count, size_t *out_len) +{ + size_t raw_bytes = count * sizeof(int16_t); + size_t b64_len = 0; + mbedtls_base64_encode(NULL, 0, &b64_len, (uint8_t *)pcm, raw_bytes); + char *b64 = malloc(b64_len + 1); + if (!b64) return NULL; + mbedtls_base64_encode((unsigned char *)b64, b64_len + 1, out_len, + (const uint8_t *)pcm, raw_bytes); + b64[*out_len] = '\0'; + return b64; +} + +/* Decode base64 โ†’ int16 PCM */ +static int16_t *base64_to_pcm(const char *b64, size_t *sample_count) +{ + size_t b64_len = strlen(b64); + size_t raw_bytes = 0; + mbedtls_base64_decode(NULL, 0, &raw_bytes, (const uint8_t *)b64, b64_len); + uint8_t *raw = malloc(raw_bytes); + if (!raw) return NULL; + mbedtls_base64_decode(raw, raw_bytes, &raw_bytes, + (const uint8_t *)b64, b64_len); + *sample_count = raw_bytes / sizeof(int16_t); + return (int16_t *)raw; +} + +/* -------------------------------------------------------------------------- */ + +static void send_json(cJSON *root) +{ + char *str = cJSON_PrintUnformatted(root); + if (str && s_ws && s_connected) { + esp_websocket_client_send_text(s_ws, str, strlen(str), portMAX_DELAY); + } + free(str); +} + +/* โ”€โ”€ Session initialisation โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void send_session_update(const char *system_prompt) +{ + cJSON *root = cJSON_CreateObject(); + cJSON *sess = cJSON_CreateObject(); + cJSON *tools = cJSON_CreateArray(); + + /* Built-in tools Debbie exposes to the model */ + cJSON *t_cam = cJSON_CreateObject(); + cJSON_AddStringToObject(t_cam, "type", "function"); + cJSON_AddStringToObject(t_cam, "name", "capture_image"); + cJSON_AddStringToObject(t_cam, "description", + "Capture a photo from the device camera and describe what you see. " + "Use when the user asks 'what do you see?', 'look at this', etc."); + cJSON_AddItemToObject(t_cam, "parameters", + cJSON_CreateObject()); /* no params */ + cJSON_AddItemToArray(tools, t_cam); + + cJSON *t_notif = cJSON_CreateObject(); + cJSON_AddStringToObject(t_notif, "type", "function"); + cJSON_AddStringToObject(t_notif, "name", "read_notifications"); + cJSON_AddStringToObject(t_notif, "description", + "Read any pending WhatsApp, email, or other notifications."); + cJSON_AddItemToObject(t_notif, "parameters", cJSON_CreateObject()); + cJSON_AddItemToArray(tools, t_notif); + + cJSON *t_spot = cJSON_CreateObject(); + cJSON_AddStringToObject(t_spot, "type", "function"); + cJSON_AddStringToObject(t_spot, "name", "spotify_control"); + cJSON_AddStringToObject(t_spot, "description", + "Control Spotify playback โ€” play, pause, next, previous, or search."); + cJSON *t_spot_params = cJSON_CreateObject(); + cJSON *t_spot_props = cJSON_CreateObject(); + cJSON *t_action = cJSON_CreateObject(); + cJSON_AddStringToObject(t_action, "type", "string"); + cJSON_AddStringToObject(t_action, "description", + "One of: play, pause, next, previous, search:, volume:<0-100>"); + cJSON_AddItemToObject(t_spot_props, "action", t_action); + cJSON_AddItemToObject(t_spot_params, "properties", t_spot_props); + cJSON *required_arr = cJSON_CreateArray(); + cJSON_AddItemToArray(required_arr, cJSON_CreateString("action")); + cJSON_AddItemToObject(t_spot_params, "required", required_arr); + cJSON_AddStringToObject(t_spot_params, "type", "object"); + cJSON_AddItemToObject(t_spot, "parameters", t_spot_params); + cJSON_AddItemToArray(tools, t_spot); + + cJSON_AddStringToObject(root, "type", "session.update"); + cJSON_AddStringToObject(sess, "modalities", "audio"); + cJSON_AddStringToObject(sess, "voice", "alloy"); + cJSON_AddStringToObject(sess, "instructions", system_prompt); + cJSON_AddStringToObject(sess, "input_audio_format", "pcm16"); + cJSON_AddStringToObject(sess, "output_audio_format", "pcm16"); + cJSON_AddBoolToObject(sess, "turn_detection", false); /* manual VAD */ + cJSON_AddItemToObject(sess, "tools", tools); + cJSON_AddStringToObject(sess, "tool_choice", "auto"); + cJSON_AddItemToObject(root, "session", sess); + + send_json(root); + cJSON_Delete(root); + ESP_LOGI(TAG, "Session updated with persona + tools"); +} + +/* โ”€โ”€ WebSocket event handler โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void ws_event_handler(void *handler_args, esp_event_base_t base, + int32_t event_id, void *event_data) +{ + esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; + + switch (event_id) { + case WEBSOCKET_EVENT_CONNECTED: + s_connected = true; + ESP_LOGI(TAG, "WebSocket connected to OpenAI Realtime API"); + { + oai_event_data_t evt = { .type = OAI_EVT_CONNECTED }; + if (s_cb) s_cb(&evt, s_ctx); + } + break; + + case WEBSOCKET_EVENT_DISCONNECTED: + s_connected = false; + ESP_LOGW(TAG, "WebSocket disconnected"); + { + oai_event_data_t evt = { .type = OAI_EVT_DISCONNECTED }; + if (s_cb) s_cb(&evt, s_ctx); + } + break; + + case WEBSOCKET_EVENT_DATA: { + if (!data->data_ptr || data->data_len == 0) break; + + /* Null-terminate */ + char *msg = malloc(data->data_len + 1); + if (!msg) break; + memcpy(msg, data->data_ptr, data->data_len); + msg[data->data_len] = '\0'; + + cJSON *json = cJSON_Parse(msg); + free(msg); + if (!json) break; + + cJSON *type_j = cJSON_GetObjectItem(json, "type"); + if (!type_j || !cJSON_IsString(type_j)) { cJSON_Delete(json); break; } + const char *type = type_j->valuestring; + + if (strcmp(type, "session.created") == 0 || + strcmp(type, "session.updated") == 0) { + oai_event_data_t evt = { .type = OAI_EVT_SESSION_CREATED }; + if (s_cb) s_cb(&evt, s_ctx); + } + else if (strcmp(type, "response.audio.delta") == 0) { + cJSON *delta_j = cJSON_GetObjectItem(json, "delta"); + if (delta_j && cJSON_IsString(delta_j)) { + size_t sample_count = 0; + int16_t *pcm = base64_to_pcm(delta_j->valuestring, &sample_count); + if (pcm) { + oai_event_data_t evt = { + .type = OAI_EVT_AUDIO_DELTA, + .audio.pcm = pcm, + .audio.count = sample_count, + }; + if (s_cb) s_cb(&evt, s_ctx); + free(pcm); + } + } + } + else if (strcmp(type, "response.audio_transcript.delta") == 0 || + strcmp(type, "conversation.item.input_audio_transcription.completed") == 0) { + cJSON *text_j = cJSON_GetObjectItem(json, "transcript"); + if (!text_j) text_j = cJSON_GetObjectItem(json, "delta"); + if (text_j && cJSON_IsString(text_j)) { + oai_event_data_t evt = { + .type = OAI_EVT_TRANSCRIPT, + .transcript.text = text_j->valuestring, + }; + if (s_cb) s_cb(&evt, s_ctx); + } + } + else if (strcmp(type, "response.function_call_arguments.done") == 0) { + cJSON *name_j = cJSON_GetObjectItem(json, "name"); + cJSON *args_j = cJSON_GetObjectItem(json, "arguments"); + cJSON *id_j = cJSON_GetObjectItem(json, "call_id"); + if (name_j && cJSON_IsString(name_j) && id_j) { + strncpy(s_current_response_id, id_j->valuestring, + sizeof(s_current_response_id) - 1); + oai_event_data_t evt = { + .type = OAI_EVT_FUNCTION_CALL, + .fn.name = name_j->valuestring, + .fn.args_json = args_j ? args_j->valuestring : "{}", + }; + if (s_cb) s_cb(&evt, s_ctx); + } + } + else if (strcmp(type, "error") == 0) { + cJSON *err_j = cJSON_GetObjectItem(json, "error"); + cJSON *msg_j = err_j ? cJSON_GetObjectItem(err_j, "message") : NULL; + oai_event_data_t evt = { + .type = OAI_EVT_ERROR, + .error.message = msg_j ? msg_j->valuestring : "unknown error", + }; + ESP_LOGE(TAG, "API error: %s", evt.error.message); + if (s_cb) s_cb(&evt, s_ctx); + } + + cJSON_Delete(json); + break; + } + + case WEBSOCKET_EVENT_ERROR: + ESP_LOGE(TAG, "WebSocket error"); + break; + + default: + break; + } +} + +/* โ”€โ”€ Public API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +esp_err_t openai_client_connect(const char *api_key, + const char *system_prompt, + oai_event_cb_t cb, + void *user_ctx) +{ + if (!api_key || strlen(api_key) == 0) { + ESP_LOGE(TAG, "No API key provided"); + return ESP_ERR_INVALID_ARG; + } + + strncpy(s_api_key, api_key, sizeof(s_api_key) - 1); + s_cb = cb; + s_ctx = user_ctx; + + if (!s_mutex) s_mutex = xSemaphoreCreateMutex(); + + /* + * Build the combined headers string required by the OpenAI Realtime API. + * esp_websocket_client_config_t accepts a single multi-line header string. + * Format: "Key1: Value1\r\nKey2: Value2\r\n" + */ + char headers[320]; + snprintf(headers, sizeof(headers), + "Authorization: Bearer %s\r\nOpenAI-Beta: realtime=v1\r\n", + api_key); + + const esp_websocket_client_config_t ws_cfg = { + .uri = "wss://" OPENAI_REALTIME_HOST OPENAI_REALTIME_PATH, + .headers = headers, + .cert_pem = NULL, /* Uses bundled CA certificates */ + .disable_auto_reconnect = false, + .reconnect_timeout_ms = 5000, + .network_timeout_ms = 10000, + .buffer_size = 8192, + .task_stack = 8192, + .task_prio = configMAX_PRIORITIES - 3, + }; + + s_ws = esp_websocket_client_init(&ws_cfg); + if (!s_ws) { + ESP_LOGE(TAG, "Failed to init WebSocket client"); + return ESP_FAIL; + } + + ESP_ERROR_CHECK(esp_websocket_register_events(s_ws, WEBSOCKET_EVENT_ANY, + ws_event_handler, NULL)); + ESP_ERROR_CHECK(esp_websocket_client_start(s_ws)); + + /* Wait for connection up to 10 s */ + int waited = 0; + while (!s_connected && waited < 100) { + vTaskDelay(pdMS_TO_TICKS(100)); + waited++; + } + + if (!s_connected) { + ESP_LOGE(TAG, "Timed out waiting for WebSocket connection"); + esp_websocket_client_stop(s_ws); + esp_websocket_client_destroy(s_ws); + s_ws = NULL; + return ESP_FAIL; + } + + /* Configure the session */ + send_session_update(system_prompt ? system_prompt : + "You are Debbie, a helpful and friendly AI companion."); + return ESP_OK; +} + +esp_err_t openai_client_disconnect(void) +{ + if (s_ws) { + esp_websocket_client_stop(s_ws); + esp_websocket_client_destroy(s_ws); + s_ws = NULL; + } + s_connected = false; + return ESP_OK; +} + +esp_err_t openai_client_send_audio(const int16_t *pcm, size_t count) +{ + if (!s_connected || !pcm || count == 0) return ESP_ERR_INVALID_STATE; + + size_t b64_len = 0; + char *b64 = pcm_to_base64(pcm, count, &b64_len); + if (!b64) return ESP_ERR_NO_MEM; + + cJSON *root = cJSON_CreateObject(); + cJSON_AddStringToObject(root, "type", "input_audio_buffer.append"); + cJSON_AddStringToObject(root, "audio", b64); + send_json(root); + cJSON_Delete(root); + free(b64); + return ESP_OK; +} + +esp_err_t openai_client_commit_audio(void) +{ + if (!s_connected) return ESP_ERR_INVALID_STATE; + + /* Commit the audio buffer */ + cJSON *commit = cJSON_CreateObject(); + cJSON_AddStringToObject(commit, "type", "input_audio_buffer.commit"); + send_json(commit); + cJSON_Delete(commit); + + /* Request a response */ + cJSON *respond = cJSON_CreateObject(); + cJSON_AddStringToObject(respond, "type", "response.create"); + send_json(respond); + cJSON_Delete(respond); + + return ESP_OK; +} + +esp_err_t openai_client_send_text(const char *text) +{ + if (!s_connected || !text) return ESP_ERR_INVALID_STATE; + + /* Create a conversation item with the user's text */ + cJSON *root = cJSON_CreateObject(); + cJSON *item = cJSON_CreateObject(); + cJSON *content = cJSON_CreateArray(); + cJSON *part = cJSON_CreateObject(); + + cJSON_AddStringToObject(part, "type", "input_text"); + cJSON_AddStringToObject(part, "text", text); + cJSON_AddItemToArray(content, part); + cJSON_AddStringToObject(item, "type", "message"); + cJSON_AddStringToObject(item, "role", "user"); + cJSON_AddItemToObject(item, "content", content); + cJSON_AddStringToObject(root, "type", "conversation.item.create"); + cJSON_AddItemToObject(root, "item", item); + send_json(root); + cJSON_Delete(root); + + /* Request a response */ + cJSON *respond = cJSON_CreateObject(); + cJSON_AddStringToObject(respond, "type", "response.create"); + send_json(respond); + cJSON_Delete(respond); + + return ESP_OK; +} + +esp_err_t openai_client_send_image(const char *jpeg_b64, const char *prompt) +{ + if (!jpeg_b64 || !s_api_key[0]) return ESP_ERR_INVALID_ARG; + + /* + * Vision via Chat Completions API (gpt-4o). + * The Realtime API does not yet support direct image input, so we + * make a separate HTTPS request and inject the description back + * into the realtime session as a text message. + */ + cJSON *body = cJSON_CreateObject(); + cJSON *msgs = cJSON_CreateArray(); + cJSON *user_m = cJSON_CreateObject(); + cJSON *content = cJSON_CreateArray(); + + /* Text part */ + cJSON *txt_part = cJSON_CreateObject(); + cJSON_AddStringToObject(txt_part, "type", "text"); + cJSON_AddStringToObject(txt_part, "text", + prompt ? prompt : "Describe what you see in this image in detail."); + cJSON_AddItemToArray(content, txt_part); + + /* Image part */ + cJSON *img_part = cJSON_CreateObject(); + cJSON *img_url_o = cJSON_CreateObject(); + char data_url[32]; + snprintf(data_url, sizeof(data_url), "data:image/jpeg;base64,"); + /* Build full data URL โ€” note: large strings may need heap allocation */ + char *full_url = malloc(strlen(data_url) + strlen(jpeg_b64) + 1); + if (!full_url) { cJSON_Delete(body); return ESP_ERR_NO_MEM; } + strcpy(full_url, data_url); + strcat(full_url, jpeg_b64); + cJSON_AddStringToObject(img_url_o, "url", full_url); + cJSON_AddStringToObject(img_url_o, "detail", "auto"); + cJSON_AddStringToObject(img_part, "type", "image_url"); + cJSON_AddItemToObject(img_part, "image_url", img_url_o); + cJSON_AddItemToArray(content, img_part); + free(full_url); + + cJSON_AddStringToObject(user_m, "role", "user"); + cJSON_AddItemToObject(user_m, "content", content); + cJSON_AddItemToArray(msgs, user_m); + cJSON_AddStringToObject(body, "model", OPENAI_VISION_MODEL); + cJSON_AddItemToObject(body, "messages", msgs); + cJSON_AddNumberToObject(body, "max_tokens", 512); + + char *body_str = cJSON_PrintUnformatted(body); + cJSON_Delete(body); + + /* HTTPS POST */ + char auth_hdr[160]; + snprintf(auth_hdr, sizeof(auth_hdr), "Bearer %s", s_api_key); + + esp_http_client_config_t http_cfg = { + .host = OPENAI_CHAT_HOST, + .path = OPENAI_CHAT_PATH, + .transport_type = HTTP_TRANSPORT_OVER_SSL, + .method = HTTP_METHOD_POST, + }; + esp_http_client_handle_t client = esp_http_client_init(&http_cfg); + esp_http_client_set_header(client, "Content-Type", "application/json"); + esp_http_client_set_header(client, "Authorization", auth_hdr); + esp_http_client_set_post_field(client, body_str, strlen(body_str)); + + esp_err_t err = esp_http_client_perform(client); + if (err == ESP_OK) { + int status = esp_http_client_get_status_code(client); + if (status == 200) { + int content_len = esp_http_client_get_content_length(client); + char *resp_buf = malloc(content_len + 1); + if (resp_buf) { + esp_http_client_read_response(client, resp_buf, content_len); + resp_buf[content_len] = '\0'; + + cJSON *resp = cJSON_Parse(resp_buf); + if (resp) { + cJSON *choices = cJSON_GetObjectItem(resp, "choices"); + cJSON *choice0 = cJSON_GetArrayItem(choices, 0); + cJSON *msg = cJSON_GetObjectItem(choice0, "message"); + cJSON *resp_txt = cJSON_GetObjectItem(msg, "content"); + if (resp_txt && cJSON_IsString(resp_txt)) { + /* Inject description into realtime conversation */ + char desc[1024]; + snprintf(desc, sizeof(desc), + "[Camera sees]: %s", resp_txt->valuestring); + openai_client_send_text(desc); + } + cJSON_Delete(resp); + } + free(resp_buf); + } + } else { + ESP_LOGW(TAG, "Vision API returned HTTP %d", status); + } + } + + esp_http_client_cleanup(client); + free(body_str); + return err; +} + +esp_err_t openai_client_send_function_result(const char *call_id, + const char *result_json) +{ + if (!s_connected) return ESP_ERR_INVALID_STATE; + + cJSON *root = cJSON_CreateObject(); + cJSON *item = cJSON_CreateObject(); + cJSON_AddStringToObject(root, "type", "conversation.item.create"); + cJSON_AddStringToObject(item, "type", "function_call_output"); + cJSON_AddStringToObject(item, "call_id", call_id); + cJSON_AddStringToObject(item, "output", result_json); + cJSON_AddItemToObject(root, "item", item); + send_json(root); + cJSON_Delete(root); + + /* Ask the model to continue with a response */ + cJSON *respond = cJSON_CreateObject(); + cJSON_AddStringToObject(respond, "type", "response.create"); + send_json(respond); + cJSON_Delete(respond); + + return ESP_OK; +} + +bool openai_client_is_connected(void) { return s_connected; } diff --git a/firmware/main/openai_client.h b/firmware/main/openai_client.h new file mode 100644 index 0000000..f1228ff --- /dev/null +++ b/firmware/main/openai_client.h @@ -0,0 +1,91 @@ +#pragma once +#include "esp_err.h" +#include +#include +#include + +/** + * @brief Initialise the OpenAI Realtime (WebSocket) client. + * + * - Uses WebSocket to wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview + * - Audio is streamed bidirectionally as PCM16 mono 24 kHz. + * - Callbacks deliver events to the application layer. + */ + +typedef enum { + OAI_EVT_CONNECTED = 0, + OAI_EVT_DISCONNECTED, + OAI_EVT_AUDIO_DELTA, /* AI speaking โ€” PCM chunk ready */ + OAI_EVT_TRANSCRIPT, /* text transcript available */ + OAI_EVT_FUNCTION_CALL, /* function/tool call requested by model */ + OAI_EVT_SESSION_CREATED, + OAI_EVT_ERROR, +} oai_event_t; + +typedef struct { + oai_event_t type; + union { + struct { const int16_t *pcm; size_t count; } audio; /* AUDIO_DELTA */ + struct { const char *text; } transcript; + struct { const char *name; const char *args_json; } fn;/* FUNCTION_CALL */ + struct { const char *message; } error; + }; +} oai_event_data_t; + +typedef void (*oai_event_cb_t)(const oai_event_data_t *evt, void *user_ctx); + +/** + * @brief Connect to OpenAI Realtime API. + * + * @param api_key OpenAI API key. + * @param system_prompt Optional persona/system prompt. + * @param cb Event callback invoked from a dedicated task. + * @param user_ctx Passed through to the callback. + */ +esp_err_t openai_client_connect(const char *api_key, + const char *system_prompt, + oai_event_cb_t cb, + void *user_ctx); + +/** + * @brief Disconnect and clean up. + */ +esp_err_t openai_client_disconnect(void); + +/** + * @brief Stream a chunk of microphone PCM audio to the API. + * Call this continuously while the user is speaking. + */ +esp_err_t openai_client_send_audio(const int16_t *pcm, size_t count); + +/** + * @brief Signal end of user speech (triggers model response). + */ +esp_err_t openai_client_commit_audio(void); + +/** + * @brief Send a text message (alternative to voice). + */ +esp_err_t openai_client_send_text(const char *text); + +/** + * @brief Send a camera image to the model for visual analysis. + * The image is sent as a vision user message via the Chat API + * (OpenAI Realtime API does not yet support image input directly). + * + * @param jpeg_b64 Base64-encoded JPEG image. + * @param prompt User prompt to accompany the image. + */ +esp_err_t openai_client_send_image(const char *jpeg_b64, + const char *prompt); + +/** + * @brief Return a function-call result so the model can continue. + */ +esp_err_t openai_client_send_function_result(const char *call_id, + const char *result_json); + +/** + * @brief Return true if the WebSocket is currently connected. + */ +bool openai_client_is_connected(void); diff --git a/firmware/main/settings.h b/firmware/main/settings.h new file mode 100644 index 0000000..546c99e --- /dev/null +++ b/firmware/main/settings.h @@ -0,0 +1,117 @@ +#pragma once + +/* ============================================================================= + * Debbie โ€” Portable Personal AI Friend + * Freenove Media Kit ESP32-S3 (FNK0102) โ€” Hardware Pin Definitions + * ============================================================================= + * + * Change DEBBIE_DISPLAY_SIZE to match your kit variant: + * DEBBIE_DISPLAY_35 โ†’ 3.5" 480ร—320 (ST7796) โ€” recommended + * DEBBIE_DISPLAY_114 โ†’ 1.14" 135ร—240 (ST7789) + */ + +/* โ”€โ”€ Display variant โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define DEBBIE_DISPLAY_35 1 /* set to 0 to use 1.14" version */ +#define DEBBIE_DISPLAY_114 0 + +/* โ”€โ”€ SPI Display (ST7796 / ST7789) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€*/ +#define LCD_SPI_HOST SPI2_HOST +#define LCD_PIN_MOSI 35 +#define LCD_PIN_CLK 36 +#define LCD_PIN_CS 34 +#define LCD_PIN_DC 37 +#define LCD_PIN_RST 38 +#define LCD_PIN_BL 33 +#define LCD_SPI_FREQ_HZ (40 * 1000 * 1000) + +#if DEBBIE_DISPLAY_35 +# define LCD_WIDTH 480 +# define LCD_HEIGHT 320 +# define LCD_DRIVER "ST7796" +#else +# define LCD_WIDTH 240 +# define LCD_HEIGHT 135 +# define LCD_DRIVER "ST7789" +#endif + +/* โ”€โ”€ Camera (OV2640) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define CAM_PIN_PWDN -1 +#define CAM_PIN_RESET -1 +#define CAM_PIN_XCLK 10 +#define CAM_PIN_SIOD 21 +#define CAM_PIN_SIOC 22 +#define CAM_PIN_D7 9 +#define CAM_PIN_D6 8 +#define CAM_PIN_D5 47 +#define CAM_PIN_D4 7 +#define CAM_PIN_D3 6 +#define CAM_PIN_D2 5 +#define CAM_PIN_D1 4 +#define CAM_PIN_D0 3 +#define CAM_PIN_VSYNC 2 +#define CAM_PIN_HREF 1 +#define CAM_PIN_PCLK 0 + +/* โ”€โ”€ I2S Microphone (MEMS, e.g. MSM261S4030H0) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define I2S_MIC_PORT I2S_NUM_0 +#define I2S_MIC_SCK 14 /* bit clock */ +#define I2S_MIC_WS 13 /* word select / LRCK */ +#define I2S_MIC_SD 12 /* serial data in */ + +/* โ”€โ”€ I2S Speaker (I2S DAC, e.g. MAX98357) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define I2S_SPK_PORT I2S_NUM_1 +#define I2S_SPK_BCK 17 /* bit clock */ +#define I2S_SPK_LRCK 16 /* word select */ +#define I2S_SPK_DATA 15 /* serial data out */ +#define I2S_SPK_SD_EN -1 /* shutdown pin (optional) */ + +/* โ”€โ”€ Audio parameters โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define AUDIO_SAMPLE_RATE 24000 /* Hz โ€” matches OpenAI Realtime API */ +#define AUDIO_BITS 16 +#define AUDIO_CHANNELS 1 +#define AUDIO_BUF_MS 60 /* ms per audio chunk */ + +/* โ”€โ”€ WS2812 RGB LED โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define RGB_LED_PIN 40 +#define RGB_LED_COUNT 1 + +/* โ”€โ”€ 5-way navigation switch โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define NAV_UP 41 +#define NAV_DOWN 42 +#define NAV_LEFT 43 +#define NAV_RIGHT 44 +#define NAV_CENTER 45 /* press to capture image / confirm */ + +/* โ”€โ”€ Battery ADC โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define BAT_ADC_PIN 4 /* ADC1_CH3 โ€” check your board variant */ +#define BAT_DIVIDER_RATIO 2.0f /* voltage divider (R1=R2) */ + +/* โ”€โ”€ SD Card (SPI) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define SD_PIN_CLK 36 /* shared SPI bus */ +#define SD_PIN_MOSI 35 +#define SD_PIN_MISO 39 +#define SD_PIN_CS 46 + +/* โ”€โ”€ Connectivity defaults โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define DEBBIE_AP_SSID "Debbie" +#define DEBBIE_AP_PASSWORD "" /* open network for first-run setup */ +#define DEBBIE_AP_IP "192.168.4.1" + +/* โ”€โ”€ OpenAI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define OPENAI_REALTIME_HOST "api.openai.com" +#define OPENAI_REALTIME_PATH "/v1/realtime?model=gpt-4o-realtime-preview" +#define OPENAI_CHAT_HOST "api.openai.com" +#define OPENAI_CHAT_PATH "/v1/chat/completions" +#define OPENAI_VISION_MODEL "gpt-4o" + +/* โ”€โ”€ Agent / companion server defaults โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define AGENT_WS_DEFAULT_URL "ws://YOUR_COMPANION_SERVER:3001" + +/* โ”€โ”€ Spotify (handled by companion server) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +/* The device sends commands to the companion server which calls Spotify API */ + +/* โ”€โ”€ NVS namespace โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define NVS_NAMESPACE "debbie" + +/* โ”€โ”€ LVGL tick rate โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define LVGL_TICK_MS 5 diff --git a/firmware/main/storage_manager.c b/firmware/main/storage_manager.c new file mode 100644 index 0000000..87af280 --- /dev/null +++ b/firmware/main/storage_manager.c @@ -0,0 +1,115 @@ +#include "storage_manager.h" +#include "debbie.h" +#include "settings.h" + +#include "nvs_flash.h" +#include "nvs.h" +#include "esp_log.h" +#include + +static const char *TAG = "storage"; + +/* -------------------------------------------------------------------------- */ + +esp_err_t storage_init(void) +{ + esp_err_t err = nvs_flash_init(); + if (err == ESP_ERR_NVS_NO_FREE_PAGES || + err == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_LOGW(TAG, "NVS partition truncated โ€” erasingโ€ฆ"); + ESP_ERROR_CHECK(nvs_flash_erase()); + err = nvs_flash_init(); + } + ESP_ERROR_CHECK(err); + + /* populate defaults */ + memset(&g_debbie_config, 0, sizeof(g_debbie_config)); + strncpy(g_debbie_config.debbie_name, "Debbie", sizeof(g_debbie_config.debbie_name) - 1); + strncpy(g_debbie_config.system_prompt, + "You are Debbie, a warm, helpful, and friendly personal AI companion. " + "You are running on a small portable device and love to help with daily tasks, " + "chat, answer questions, read notifications, and more. Keep responses concise " + "and conversational unless the user asks for detail.", + sizeof(g_debbie_config.system_prompt) - 1); + g_debbie_config.speaker_volume = 75; + g_debbie_config.notifications_enabled = true; + g_debbie_config.camera_enabled = true; + g_debbie_config.use_custom_agent = false; + strncpy(g_debbie_config.agent_ws_url, AGENT_WS_DEFAULT_URL, + sizeof(g_debbie_config.agent_ws_url) - 1); + + /* try to load from NVS */ + nvs_handle_t nvs; + err = nvs_open(NVS_NAMESPACE, NVS_READONLY, &nvs); + if (err == ESP_ERR_NVS_NOT_FOUND) { + ESP_LOGI(TAG, "No saved config โ€” using defaults"); + return ESP_OK; + } + ESP_ERROR_CHECK(err); + +#define NVS_GET_STR(key, dst) do { \ + size_t _len = sizeof(dst); \ + nvs_get_str(nvs, key, dst, &_len); \ + } while (0) +#define NVS_GET_U8(key, dst) nvs_get_u8(nvs, key, &(dst)) + + NVS_GET_STR("wifi_ssid", g_debbie_config.wifi_ssid); + NVS_GET_STR("wifi_pass", g_debbie_config.wifi_password); + NVS_GET_STR("oai_key", g_debbie_config.openai_api_key); + NVS_GET_STR("agent_url", g_debbie_config.agent_ws_url); + NVS_GET_STR("companion_url", g_debbie_config.companion_url); + NVS_GET_STR("name", g_debbie_config.debbie_name); + NVS_GET_STR("sys_prompt", g_debbie_config.system_prompt); + NVS_GET_STR("spotify_tok", g_debbie_config.spotify_token); + NVS_GET_U8("volume", g_debbie_config.speaker_volume); + + uint8_t tmp; + if (nvs_get_u8(nvs, "use_agent", &tmp) == ESP_OK) + g_debbie_config.use_custom_agent = (tmp != 0); + if (nvs_get_u8(nvs, "notifs_en", &tmp) == ESP_OK) + g_debbie_config.notifications_enabled = (tmp != 0); + if (nvs_get_u8(nvs, "cam_en", &tmp) == ESP_OK) + g_debbie_config.camera_enabled = (tmp != 0); + + nvs_close(nvs); + ESP_LOGI(TAG, "Config loaded โ€” WiFi SSID: %s", g_debbie_config.wifi_ssid); + return ESP_OK; +} + +esp_err_t storage_save_config(void) +{ + nvs_handle_t nvs; + ESP_ERROR_CHECK(nvs_open(NVS_NAMESPACE, NVS_READWRITE, &nvs)); + +#define NVS_SET_STR(key, src) nvs_set_str(nvs, key, src) +#define NVS_SET_U8(key, val) nvs_set_u8(nvs, key, val) + + NVS_SET_STR("wifi_ssid", g_debbie_config.wifi_ssid); + NVS_SET_STR("wifi_pass", g_debbie_config.wifi_password); + NVS_SET_STR("oai_key", g_debbie_config.openai_api_key); + NVS_SET_STR("agent_url", g_debbie_config.agent_ws_url); + NVS_SET_STR("companion_url", g_debbie_config.companion_url); + NVS_SET_STR("name", g_debbie_config.debbie_name); + NVS_SET_STR("sys_prompt", g_debbie_config.system_prompt); + NVS_SET_STR("spotify_tok", g_debbie_config.spotify_token); + NVS_SET_U8("volume", g_debbie_config.speaker_volume); + NVS_SET_U8("use_agent", g_debbie_config.use_custom_agent ? 1 : 0); + NVS_SET_U8("notifs_en", g_debbie_config.notifications_enabled ? 1 : 0); + NVS_SET_U8("cam_en", g_debbie_config.camera_enabled ? 1 : 0); + + esp_err_t err = nvs_commit(nvs); + nvs_close(nvs); + ESP_LOGI(TAG, "Config saved"); + return err; +} + +esp_err_t storage_factory_reset(void) +{ + nvs_handle_t nvs; + ESP_ERROR_CHECK(nvs_open(NVS_NAMESPACE, NVS_READWRITE, &nvs)); + esp_err_t err = nvs_erase_all(nvs); + nvs_commit(nvs); + nvs_close(nvs); + ESP_LOGI(TAG, "Factory reset complete"); + return err; +} diff --git a/firmware/main/storage_manager.h b/firmware/main/storage_manager.h new file mode 100644 index 0000000..d2387c0 --- /dev/null +++ b/firmware/main/storage_manager.h @@ -0,0 +1,18 @@ +#pragma once +#include "esp_err.h" + +/** + * @brief Initialise NVS flash and load saved configuration. + * If no config exists the defaults in settings.h are used. + */ +esp_err_t storage_init(void); + +/** + * @brief Persist the current g_debbie_config to NVS. + */ +esp_err_t storage_save_config(void); + +/** + * @brief Erase all Debbie NVS keys (factory reset). + */ +esp_err_t storage_factory_reset(void); diff --git a/firmware/main/web_server.c b/firmware/main/web_server.c new file mode 100644 index 0000000..4a1d56d --- /dev/null +++ b/firmware/main/web_server.c @@ -0,0 +1,250 @@ +#include "web_server.h" +#include "debbie.h" +#include "settings.h" +#include "storage_manager.h" +#include "wifi_manager.h" +#include "camera_manager.h" +#include "esp_log.h" +#include "esp_http_server.h" +#include "cJSON.h" +#include +#include + +static const char *TAG = "web"; + +/* -------------------------------------------------------------------------- */ + +/* Inline HTML for the setup portal โ€” friendly, mobile-first design */ +static const char SETUP_HTML[] = +"" +"" +"Debbie Setup โœจ" +"" +"
" +"

โœจ Hi, I'm Debbie!

" +"

Your portable personal AI companion โ€” let's get set up!

" +"
" + +"
" +"

๐Ÿ“ถ Wi-Fi Connection

" +"" +"" +"
" + +"
" +"

๐Ÿค– AI Settings

" +"" +"" +"" +"" +"" +"" +"
" + +"
" +"

๐ŸŽ€ Debbie's Personality

" +"" +"" +"" +"" +"" +"
" + +"
" +"" +"" +"
" +"
" + +""; + +/* โ”€โ”€ Handlers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static esp_err_t handler_root(httpd_req_t *req) +{ + httpd_resp_set_type(req, "text/html"); + httpd_resp_send(req, SETUP_HTML, strlen(SETUP_HTML)); + return ESP_OK; +} + +static esp_err_t handler_status(httpd_req_t *req) +{ + cJSON *json = cJSON_CreateObject(); + cJSON_AddStringToObject(json, "name", g_debbie_config.debbie_name); + cJSON_AddStringToObject(json, "sys_prompt", g_debbie_config.system_prompt); + cJSON_AddStringToObject(json, "agent_url", g_debbie_config.agent_ws_url); + cJSON_AddStringToObject(json, "companion_url",g_debbie_config.companion_url); + cJSON_AddNumberToObject(json, "volume", g_debbie_config.speaker_volume); + cJSON_AddBoolToObject(json, "wifi_ok", wifi_manager_is_connected()); + cJSON_AddStringToObject(json, "state", + g_debbie_state == DEBBIE_STATE_IDLE ? "idle" : + g_debbie_state == DEBBIE_STATE_LISTENING ? "listening" : + g_debbie_state == DEBBIE_STATE_THINKING ? "thinking" : + g_debbie_state == DEBBIE_STATE_SPEAKING ? "speaking" : "other"); + + char *str = cJSON_PrintUnformatted(json); + cJSON_Delete(json); + httpd_resp_set_type(req, "application/json"); + httpd_resp_send(req, str, strlen(str)); + free(str); + return ESP_OK; +} + +static esp_err_t handler_configure(httpd_req_t *req) +{ + char *buf = malloc(req->content_len + 1); + if (!buf) { httpd_resp_send_500(req); return ESP_FAIL; } + + int received = httpd_req_recv(req, buf, req->content_len); + if (received <= 0) { free(buf); httpd_resp_send_500(req); return ESP_FAIL; } + buf[received] = '\0'; + + cJSON *json = cJSON_Parse(buf); + free(buf); + if (!json) { httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "Invalid JSON"); return ESP_FAIL; } + +#define GET_STR(key, dst) do { \ + cJSON *j = cJSON_GetObjectItem(json, key); \ + if (j && cJSON_IsString(j) && strlen(j->valuestring) > 0) \ + strncpy(dst, j->valuestring, sizeof(dst) - 1); \ +} while (0) + + GET_STR("ssid", g_debbie_config.wifi_ssid); + GET_STR("pass", g_debbie_config.wifi_password); + GET_STR("oai_key", g_debbie_config.openai_api_key); + GET_STR("agent_url", g_debbie_config.agent_ws_url); + GET_STR("companion_url",g_debbie_config.companion_url); + GET_STR("name", g_debbie_config.debbie_name); + GET_STR("sys_prompt", g_debbie_config.system_prompt); + + cJSON *vol = cJSON_GetObjectItem(json, "volume"); + if (vol && cJSON_IsNumber(vol)) + g_debbie_config.speaker_volume = (uint8_t)vol->valuedouble; + + cJSON_Delete(json); + storage_save_config(); + + const char *resp = "{\"ok\":true,\"msg\":\"Saved! Connecting to WiFi...\"}"; + httpd_resp_set_type(req, "application/json"); + httpd_resp_send(req, resp, strlen(resp)); + + /* Reconnect with new credentials in background */ + vTaskDelay(pdMS_TO_TICKS(500)); + wifi_manager_reconnect(); + + return ESP_OK; +} + +static esp_err_t handler_reset(httpd_req_t *req) +{ + storage_factory_reset(); + httpd_resp_send(req, "{\"ok\":true}", strlen("{\"ok\":true}")); + vTaskDelay(pdMS_TO_TICKS(1000)); + esp_restart(); + return ESP_OK; +} + +static esp_err_t handler_snapshot(httpd_req_t *req) +{ + if (!g_debbie_config.camera_enabled) { + httpd_resp_send_err(req, HTTPD_404_NOT_FOUND, "Camera disabled"); + return ESP_FAIL; + } + uint8_t *jpeg = NULL; + size_t len = 0; + if (camera_manager_capture_jpeg(&jpeg, &len) != ESP_OK) { + httpd_resp_send_500(req); + return ESP_FAIL; + } + httpd_resp_set_type(req, "image/jpeg"); + httpd_resp_send(req, (char *)jpeg, len); + camera_manager_free_frame(jpeg); + return ESP_OK; +} + +/* -------------------------------------------------------------------------- */ + +static httpd_handle_t s_server = NULL; + +esp_err_t web_server_start(void) +{ + httpd_config_t cfg = HTTPD_DEFAULT_CONFIG(); + cfg.max_uri_handlers = 8; + cfg.stack_size = 8192; + + ESP_ERROR_CHECK(httpd_start(&s_server, &cfg)); + + const httpd_uri_t routes[] = { + { .uri = "/", .method = HTTP_GET, .handler = handler_root }, + { .uri = "/status", .method = HTTP_GET, .handler = handler_status }, + { .uri = "/configure", .method = HTTP_POST, .handler = handler_configure }, + { .uri = "/reset", .method = HTTP_POST, .handler = handler_reset }, + { .uri = "/snapshot", .method = HTTP_GET, .handler = handler_snapshot }, + }; + + for (int i = 0; i < (int)(sizeof(routes) / sizeof(routes[0])); i++) { + ESP_ERROR_CHECK(httpd_register_uri_handler(s_server, &routes[i])); + } + + ESP_LOGI(TAG, "Web server started at http://%s/", DEBBIE_AP_IP); + return ESP_OK; +} + +esp_err_t web_server_stop(void) +{ + if (s_server) { + httpd_stop(s_server); + s_server = NULL; + } + return ESP_OK; +} diff --git a/firmware/main/web_server.h b/firmware/main/web_server.h new file mode 100644 index 0000000..c9f37f9 --- /dev/null +++ b/firmware/main/web_server.h @@ -0,0 +1,22 @@ +#pragma once +#include "esp_err.h" + +/** + * @brief Start the HTTP configuration server. + * + * Serves the captive-portal setup page at http://192.168.4.1/ and provides + * REST endpoints for configuration, status, and device control. + * + * Endpoints: + * GET / โ†’ Setup / status HTML page + * POST /configure โ†’ Save WiFi, API key, companion URL, persona + * GET /status โ†’ JSON device status + * POST /reset โ†’ Factory reset + * GET /snapshot โ†’ JPEG camera snapshot (if camera enabled) + */ +esp_err_t web_server_start(void); + +/** + * @brief Stop the HTTP server. + */ +esp_err_t web_server_stop(void); diff --git a/firmware/main/wifi_manager.c b/firmware/main/wifi_manager.c new file mode 100644 index 0000000..b19a264 --- /dev/null +++ b/firmware/main/wifi_manager.c @@ -0,0 +1,148 @@ +#include "wifi_manager.h" +#include "debbie.h" +#include "settings.h" +#include "storage_manager.h" + +#include "esp_wifi.h" +#include "esp_event.h" +#include "esp_log.h" +#include "esp_netif.h" +#include "freertos/FreeRTOS.h" +#include "freertos/event_groups.h" +#include + +static const char *TAG = "wifi"; + +#define WIFI_CONNECTED_BIT BIT0 +#define WIFI_FAIL_BIT BIT1 +#define WIFI_MAX_RETRY 5 + +static EventGroupHandle_t s_wifi_event_group; +static int s_retry_count = 0; +static bool s_connected = false; + +/* -------------------------------------------------------------------------- */ + +static void event_handler(void *arg, esp_event_base_t base, + int32_t id, void *data) +{ + if (base == WIFI_EVENT) { + switch (id) { + case WIFI_EVENT_STA_START: + esp_wifi_connect(); + break; + case WIFI_EVENT_STA_DISCONNECTED: + s_connected = false; + if (s_retry_count < WIFI_MAX_RETRY) { + esp_wifi_connect(); + s_retry_count++; + ESP_LOGI(TAG, "Retrying WiFiโ€ฆ (%d/%d)", + s_retry_count, WIFI_MAX_RETRY); + } else { + xEventGroupSetBits(s_wifi_event_group, WIFI_FAIL_BIT); + } + esp_event_post(DEBBIE_EVENT_BASE, DEBBIE_EVT_WIFI_DISCONNECTED, + NULL, 0, portMAX_DELAY); + break; + case WIFI_EVENT_AP_STACONNECTED: + ESP_LOGI(TAG, "Client connected to Debbie AP"); + break; + default: + break; + } + } else if (base == IP_EVENT && id == IP_EVENT_STA_GOT_IP) { + ip_event_got_ip_t *evt = (ip_event_got_ip_t *)data; + ESP_LOGI(TAG, "Got IP: " IPSTR, IP2STR(&evt->ip_info.ip)); + s_retry_count = 0; + s_connected = true; + xEventGroupSetBits(s_wifi_event_group, WIFI_CONNECTED_BIT); + esp_event_post(DEBBIE_EVENT_BASE, DEBBIE_EVT_WIFI_CONNECTED, + NULL, 0, portMAX_DELAY); + } +} + +/* -------------------------------------------------------------------------- */ + +esp_err_t wifi_manager_init(void) +{ + s_wifi_event_group = xEventGroupCreate(); + ESP_ERROR_CHECK(esp_netif_init()); + ESP_ERROR_CHECK(esp_event_loop_create_default()); + esp_netif_create_default_wifi_sta(); + esp_netif_create_default_wifi_ap(); + + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_wifi_init(&cfg)); + ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM)); + + ESP_ERROR_CHECK(esp_event_handler_instance_register( + WIFI_EVENT, ESP_EVENT_ANY_ID, event_handler, NULL, NULL)); + ESP_ERROR_CHECK(esp_event_handler_instance_register( + IP_EVENT, IP_EVENT_STA_GOT_IP, event_handler, NULL, NULL)); + + /* AP config โ€” always on for setup */ + wifi_config_t ap_config = { + .ap = { + .ssid = DEBBIE_AP_SSID, + .ssid_len = strlen(DEBBIE_AP_SSID), + .password = DEBBIE_AP_PASSWORD, + .max_connection = 4, + .authmode = WIFI_AUTH_OPEN, + }, + }; + + if (strlen(g_debbie_config.wifi_ssid) > 0) { + /* Try STA + AP together */ + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_APSTA)); + wifi_config_t sta_config = { 0 }; + strncpy((char *)sta_config.sta.ssid, g_debbie_config.wifi_ssid, + sizeof(sta_config.sta.ssid) - 1); + strncpy((char *)sta_config.sta.password, g_debbie_config.wifi_password, + sizeof(sta_config.sta.password) - 1); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &sta_config)); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_AP, &ap_config)); + ESP_ERROR_CHECK(esp_wifi_start()); + + /* Wait up to 10 s for connection */ + EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group, + WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, + pdFALSE, pdFALSE, + pdMS_TO_TICKS(10000)); + if (bits & WIFI_CONNECTED_BIT) { + ESP_LOGI(TAG, "Connected to WiFi: %s", g_debbie_config.wifi_ssid); + return ESP_OK; + } + ESP_LOGW(TAG, "Could not connect to saved WiFi โ€” entering setup mode"); + } else { + /* AP-only setup mode */ + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_AP)); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_AP, &ap_config)); + ESP_ERROR_CHECK(esp_wifi_start()); + ESP_LOGI(TAG, "AP-only mode โ€” connect to '%s' and visit %s", + DEBBIE_AP_SSID, DEBBIE_AP_IP); + } + return ESP_FAIL; +} + +bool wifi_manager_is_connected(void) { return s_connected; } + +esp_err_t wifi_manager_reconnect(void) +{ + s_retry_count = 0; + xEventGroupClearBits(s_wifi_event_group, WIFI_CONNECTED_BIT | WIFI_FAIL_BIT); + esp_wifi_disconnect(); + esp_wifi_connect(); + EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group, + WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, + pdFALSE, pdFALSE, + pdMS_TO_TICKS(15000)); + return (bits & WIFI_CONNECTED_BIT) ? ESP_OK : ESP_FAIL; +} + +esp_err_t wifi_manager_set_credentials(const char *ssid, const char *password) +{ + strncpy(g_debbie_config.wifi_ssid, ssid, sizeof(g_debbie_config.wifi_ssid) - 1); + strncpy(g_debbie_config.wifi_password, password, sizeof(g_debbie_config.wifi_password) - 1); + storage_save_config(); + return wifi_manager_reconnect(); +} diff --git a/firmware/main/wifi_manager.h b/firmware/main/wifi_manager.h new file mode 100644 index 0000000..e68a6b1 --- /dev/null +++ b/firmware/main/wifi_manager.h @@ -0,0 +1,30 @@ +#pragma once +#include "esp_err.h" +#include + +/** + * @brief Initialise WiFi in station + AP (dual) mode. + * + * - If WiFi credentials are saved, try to connect immediately. + * - Always start the soft-AP "Debbie" for first-run configuration. + * - Blocks until the STA connection succeeds, or times out and + * leaves the device in AP-only setup mode. + * + * @return ESP_OK on successful STA connection, ESP_FAIL if setup required. + */ +esp_err_t wifi_manager_init(void); + +/** + * @brief Return true when the device has an IP address on the STA interface. + */ +bool wifi_manager_is_connected(void); + +/** + * @brief Attempt to (re)connect using stored credentials. + */ +esp_err_t wifi_manager_reconnect(void); + +/** + * @brief Store new WiFi credentials and reconnect. + */ +esp_err_t wifi_manager_set_credentials(const char *ssid, const char *password); diff --git a/firmware/partitions.csv b/firmware/partitions.csv new file mode 100644 index 0000000..8b0f592 --- /dev/null +++ b/firmware/partitions.csv @@ -0,0 +1,8 @@ +# Debbie โ€” Custom Partition Table +# 16 MB flash: app0 + app1 (OTA), NVS, storage +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +otadata, data, ota, 0xf000, 0x2000, +app0, app, ota_0, 0x20000, 0x5A0000, +app1, app, ota_1, 0x5C0000, 0x5A0000, +storage, data, spiffs, 0xB80000, 0x480000, diff --git a/firmware/sdkconfig.defaults b/firmware/sdkconfig.defaults new file mode 100644 index 0000000..47a3b73 --- /dev/null +++ b/firmware/sdkconfig.defaults @@ -0,0 +1,65 @@ +# Debbie โ€” ESP-IDF default configuration +# Adjust for your specific hardware variant + +# Target +CONFIG_IDF_TARGET="esp32s3" + +# Flash +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + +# PSRAM (required for camera frame buffers) +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_USE_MALLOC=y +CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 + +# CPU +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y + +# WiFi +CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=16 +CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=64 +CONFIG_ESP_WIFI_TX_BUFFER_TYPE_DYNAMIC=y + +# โ”€โ”€โ”€ TLS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# SECURITY WARNING: The settings below are for development only. +# For production deployments, remove CONFIG_ESP_TLS_INSECURE and +# CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY to enable full certificate validation. +# Without these, the device is vulnerable to man-in-the-middle attacks. +# Development only: +CONFIG_ESP_TLS_INSECURE=y +CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y + +# NVS encryption (disabled for simplicity โ€” enable for production) +CONFIG_NVS_ENCRYPTION=n + +# Stack sizes +CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 + +# Logging +CONFIG_LOG_DEFAULT_LEVEL_INFO=y +CONFIG_LOG_COLORS=y + +# FreeRTOS +CONFIG_FREERTOS_HZ=1000 +CONFIG_FREERTOS_UNICORE=n + +# OTA +CONFIG_OTA_ALLOW_HTTP=y + +# ADC (battery monitoring) +CONFIG_ADC_CAL_EFUSE_TP_ENABLE=y + +# I2S +CONFIG_I2S_SUPPRESS_DEPRECATE_WARN=y + +# HTTP server +CONFIG_HTTPD_MAX_REQ_HDR_LEN=1024 +CONFIG_HTTPD_MAX_URI_LEN=512 + +# Camera +CONFIG_CAMERA_CORE0=n +CONFIG_CAMERA_CORE1=y From d12849a18b45d9502bf49c9a3d88afff07a768d4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 15:30:22 +0000 Subject: [PATCH 3/8] Add network security scanning, self-agent, and fix all security issues Agent-Logs-Url: https://github.com/magicalmutation-coder/DebbieDoesMobile/sessions/4e986b25-518d-47ad-bb1b-be6cf35d3018 Co-authored-by: magicalmutation-coder <246345154+magicalmutation-coder@users.noreply.github.com> --- README.md | 84 +++- companion-server/network_tools.js | 450 +++++++++++++++++ companion-server/server.js | 25 +- companion-server/spotify.js | 25 +- firmware/main/CMakeLists.txt | 5 + firmware/main/debbie.h | 4 + firmware/main/main.c | 56 ++- firmware/main/network_scanner.c | 776 ++++++++++++++++++++++++++++++ firmware/main/network_scanner.h | 196 ++++++++ firmware/main/openai_client.c | 8 +- firmware/main/openai_client.h | 6 +- firmware/main/self_agent.c | 597 +++++++++++++++++++++++ firmware/main/self_agent.h | 61 +++ firmware/main/settings.h | 22 + firmware/main/storage_manager.c | 5 +- firmware/main/vuln_reporter.c | 586 ++++++++++++++++++++++ firmware/main/vuln_reporter.h | 88 ++++ 17 files changed, 2964 insertions(+), 30 deletions(-) create mode 100644 companion-server/network_tools.js create mode 100644 firmware/main/network_scanner.c create mode 100644 firmware/main/network_scanner.h create mode 100644 firmware/main/self_agent.c create mode 100644 firmware/main/self_agent.h create mode 100644 firmware/main/vuln_reporter.c create mode 100644 firmware/main/vuln_reporter.h diff --git a/README.md b/README.md index ac39f7c..dc8f24f 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,89 @@ All settings are stored in flash (NVS) and can be updated via the web UI at `htt --- -## ๐Ÿ”Œ Connecting Your Own Agent +## ๐Ÿ”’ Network Security Testing + +> โš ๏ธ **AUTHORISED USE ONLY** โ€” These tools are for testing **your own networks** or networks you have **explicit written permission** to test. Unauthorised network scanning is illegal under the Computer Misuse Act (UK), CFAA (US), and similar laws worldwide. + +Debbie includes a full ethical network security toolkit, voice-commanded through the AI. + +### What Debbie Can Scan + +| Tool | Description | +|------|-------------| +| ๐Ÿ” **WiFi Scanner** | Discover all visible networks, check encryption (flags WEP/open APs) | +| ๐Ÿ“ก **ARP Host Discovery** | Find all live devices on your subnet | +| ๐Ÿšช **Port Scanner** | Scan 65+ common service ports per host | +| ๐Ÿท๏ธ **Service Fingerprinting** | Identify running services from banners | +| ๐Ÿ•ต๏ธ **Vulnerability Analysis** | Check against 20+ known-dangerous configurations | +| ๐Ÿ“Š **Risk Scoring** | 0โ€“100 risk score per device with remediation | +| ๐ŸŒ **Web Probe** | Detect HTTP/HTTPS, grab server headers and page titles | +| ๐Ÿ”ฌ **CVE Lookup** | Query the NVD database for specific CVE details | +| ๐Ÿ—บ๏ธ **DNS Lookup** | Resolve hostnames from the device | + +### Checks Performed + +| Check | Severity | +|-------|---------| +| Open WiFi networks | HIGH | +| WEP encryption (crackable in seconds) | CRITICAL | +| Telnet exposed (plaintext credentials) | HIGH | +| Docker API on port 2375 (no auth) | CRITICAL | +| Redis/MongoDB/Elasticsearch (often unauthenticated) | HIGH | +| Modbus/S7 industrial protocols exposed | CRITICAL | +| VNC remote desktop exposed | HIGH | +| MQTT broker without authentication | MEDIUM | +| SNMP with default community strings | MEDIUM | +| HTTP without HTTPS | LOW | +| OpenSSH version fingerprinting | MEDIUM | + +### Voice Commands + +Just say it to Debbie โ€” no typing needed: + +- *"Scan my network"* โ†’ Full scan (WiFi + all devices + ports) +- *"What devices are on my network?"* โ†’ Quick host discovery +- *"Scan the WiFi networks nearby"* โ†’ WiFi-only scan +- *"Check 192.168.1.1 for vulnerabilities"* โ†’ Port scan a specific device +- *"What's the risk score for my router?"* โ†’ Risk assessment +- *"Look up CVE-2021-44228"* โ†’ Log4Shell CVE details +- *"Give me the security report"* โ†’ Full vulnerability summary +- *"What are my highest-risk devices?"* โ†’ Prioritised findings +- *"How do I fix the Redis vulnerability?"* โ†’ Remediation guidance + +### Companion Server nmap Integration + +For more advanced scanning, install nmap on the companion server host: + +```bash +# Linux +sudo apt install nmap + +# macOS +brew install nmap +``` + +Then request a scan via REST: +```bash +curl -X POST http://localhost:3001/network/scan \ + -H "Content-Type: application/json" \ + -d '{"target": "192.168.1.0/24", "flags": ["-sV", "-O"]}' +``` + +Generate an HTML report at `http://localhost:3001/network/report`. + +### Self-Assessment + +Debbie also audits herself: +- TLS certificate verification status +- ESP-IDF firmware version +- Enabled features and potential risks + +Say *"Check yourself for vulnerabilities"* or *"Run a self-assessment"*. + +--- + + If you have your own AI agent (e.g. a custom LLM server), connect it via: diff --git a/companion-server/network_tools.js b/companion-server/network_tools.js new file mode 100644 index 0000000..a495793 --- /dev/null +++ b/companion-server/network_tools.js @@ -0,0 +1,450 @@ +/** + * network_tools.js โ€” Companion server network security tools + * + * โš ๏ธ AUTHORISED USE ONLY โš ๏ธ + * Only use on networks you own or have explicit written permission to test. + * Unauthorised network scanning is illegal in most jurisdictions. + * + * Provides: + * โ€ข nmap wrapper (if installed) for advanced scanning + * โ€ข NVD CVE API queries + * โ€ข Shodan-style local device fingerprinting + * โ€ข HTML vulnerability report generation + * โ€ข MAC vendor OUI lookup + * โ€ข Network topology mapping + */ + +const { execFile } = require('child_process'); +const https = require('https'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); + +/* โ”€โ”€ nmap wrapper โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/** + * Run nmap on the given target and return parsed JSON results. + * Requires nmap to be installed: apt install nmap (Linux) or https://nmap.org + * + * @param {string} target - IP, CIDR, or hostname (e.g. "192.168.1.0/24") + * @param {string[]} flags - Extra nmap flags (e.g. ['-sV', '-O']) + * @returns {Promise} Parsed nmap XML results as JSON + */ +async function nmapScan(target, flags = []) { + return new Promise((resolve, reject) => { + const args = [ + '-oX', '-', /* XML to stdout */ + '--stats-every', '5s', + '-T4', /* Aggressive timing */ + '--open', /* Only show open ports */ + ...flags, + '--', + target, + ]; + + execFile('nmap', args, { maxBuffer: 5 * 1024 * 1024, timeout: 300000 }, + (err, stdout, stderr) => { + if (err && !stdout) { + if (err.code === 'ENOENT') { + reject(new Error( + 'nmap not found. Install with: sudo apt install nmap\n' + + 'or download from https://nmap.org')); + } else { + reject(new Error(`nmap failed: ${err.message}`)); + } + return; + } + try { + resolve(parseNmapXml(stdout)); + } catch (e) { + reject(new Error(`nmap XML parse failed: ${e.message}`)); + } + } + ); + }); +} + +/** + * Parse nmap XML output to a clean JSON structure. + */ +function parseNmapXml(xml) { + const hosts = []; + /* Simple regex-based extraction โ€” use xml2js for production */ + const hostRegex = /]*>([\s\S]*?)<\/host>/g; + let hostMatch; + + while ((hostMatch = hostRegex.exec(xml)) !== null) { + const hostXml = hostMatch[1]; + + /* State */ + const stateMatch = /]*)vendor="([^"]*)"/.exec(hostXml); + const hostnameMatch = /([\s\S]*?)<\/port>/g; + let portMatch; + while ((portMatch = portRegex.exec(hostXml)) !== null) { + const portXml = portMatch[3]; + const pStateMatch = /]*)product="([^"]*)"(?:[^>]*)version="([^"]*)"/.exec(portXml); + const bannerMatch = /"; + +/* โ”€โ”€ Handlers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static esp_err_t handler_root(httpd_req_t *req) +{ + httpd_resp_set_type(req, "text/html"); + httpd_resp_send(req, SETUP_HTML, strlen(SETUP_HTML)); + return ESP_OK; +} + +static esp_err_t handler_status(httpd_req_t *req) +{ + cJSON *json = cJSON_CreateObject(); + cJSON_AddStringToObject(json, "name", g_debbie_config.debbie_name); + cJSON_AddStringToObject(json, "sys_prompt", g_debbie_config.system_prompt); + cJSON_AddStringToObject(json, "agent_url", g_debbie_config.agent_ws_url); + cJSON_AddStringToObject(json, "companion_url",g_debbie_config.companion_url); + cJSON_AddStringToObject(json, "llm_provider", g_debbie_config.llm_provider); + cJSON_AddStringToObject(json, "llm_model", g_debbie_config.llm_model); + cJSON_AddStringToObject(json, "local_llm_url",g_debbie_config.local_llm_url); + cJSON_AddStringToObject(json, "ssid", g_debbie_config.wifi_ssid); + cJSON_AddStringToObject(json, "ssid2", g_debbie_config.wifi_ssid2); + cJSON_AddStringToObject(json, "ssid3", g_debbie_config.wifi_ssid3); + cJSON_AddStringToObject(json, "ble_name", g_debbie_config.ble_device_name); + cJSON_AddStringToObject(json, "voice_style", g_debbie_config.voice_style); + cJSON_AddStringToObject(json, "resp_len", g_debbie_config.response_length); + cJSON_AddNumberToObject(json, "volume", g_debbie_config.speaker_volume); + cJSON_AddNumberToObject(json, "vad_threshold",g_debbie_config.vad_threshold); + cJSON_AddBoolToObject(json, "wifi_ok", wifi_manager_is_connected()); + cJSON_AddBoolToObject(json, "bt_en", g_debbie_config.bluetooth_enabled); + cJSON_AddBoolToObject(json, "bt_on", g_debbie_config.bluetooth_enabled); + cJSON_AddBoolToObject(json, "use_agent", g_debbie_config.use_custom_agent); + cJSON_AddBoolToObject(json, "notifs_en", g_debbie_config.notifications_enabled); + cJSON_AddBoolToObject(json, "notif_wa", g_debbie_config.notif_whatsapp); + cJSON_AddBoolToObject(json, "notif_em", g_debbie_config.notif_email); + cJSON_AddBoolToObject(json, "notif_sp", g_debbie_config.notif_spotify); + cJSON_AddBoolToObject(json, "cam_en", g_debbie_config.camera_enabled); + cJSON_AddStringToObject(json, "state", + g_debbie_state == DEBBIE_STATE_IDLE ? "idle" : + g_debbie_state == DEBBIE_STATE_LISTENING ? "listening" : + g_debbie_state == DEBBIE_STATE_THINKING ? "thinking" : + g_debbie_state == DEBBIE_STATE_SPEAKING ? "speaking" : "other"); + + char *str = cJSON_PrintUnformatted(json); + cJSON_Delete(json); + httpd_resp_set_type(req, "application/json"); + httpd_resp_send(req, str, strlen(str)); + free(str); + return ESP_OK; +} + +static esp_err_t handler_configure(httpd_req_t *req) +{ + char *buf = malloc(req->content_len + 1); + if (!buf) { httpd_resp_send_500(req); return ESP_FAIL; } + + int received = httpd_req_recv(req, buf, req->content_len); + if (received <= 0) { free(buf); httpd_resp_send_500(req); return ESP_FAIL; } + buf[received] = '\0'; + + cJSON *json = cJSON_Parse(buf); + free(buf); + if (!json) { httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "Invalid JSON"); return ESP_FAIL; } + +#define GET_STR(key, dst) do { \ + cJSON *j = cJSON_GetObjectItem(json, key); \ + if (j && cJSON_IsString(j) && strlen(j->valuestring) > 0) \ + strncpy(dst, j->valuestring, sizeof(dst) - 1); \ +} while (0) + +#define GET_BOOL(key, dst) do { \ + cJSON *j = cJSON_GetObjectItem(json, key); \ + if (j) dst = cJSON_IsTrue(j); \ +} while (0) + + /* Network */ + GET_STR("ssid", g_debbie_config.wifi_ssid); + GET_STR("pass", g_debbie_config.wifi_password); + GET_STR("ssid2", g_debbie_config.wifi_ssid2); + GET_STR("pass2", g_debbie_config.wifi_password2); + GET_STR("ssid3", g_debbie_config.wifi_ssid3); + GET_STR("pass3", g_debbie_config.wifi_password3); + GET_STR("ble_name", g_debbie_config.ble_device_name); + GET_STR("companion_url", g_debbie_config.companion_url); + GET_BOOL("bt_en", g_debbie_config.bluetooth_enabled); + + /* LLM */ + GET_STR("llm_provider", g_debbie_config.llm_provider); + GET_STR("llm_model", g_debbie_config.llm_model); + GET_STR("oai_key", g_debbie_config.openai_api_key); + GET_STR("anthropic_key", g_debbie_config.anthropic_api_key); + GET_STR("groq_key", g_debbie_config.groq_api_key); + GET_STR("or_key", g_debbie_config.openrouter_api_key); + GET_STR("local_llm_url", g_debbie_config.local_llm_url); + GET_STR("local_llm_model",g_debbie_config.local_llm_model); + GET_STR("agent_url", g_debbie_config.agent_ws_url); + GET_BOOL("use_agent", g_debbie_config.use_custom_agent); + + /* Personality */ + GET_STR("name", g_debbie_config.debbie_name); + GET_STR("sys_prompt", g_debbie_config.system_prompt); + GET_STR("voice_style", g_debbie_config.voice_style); + GET_STR("resp_len", g_debbie_config.response_length); + + /* Notifications */ + GET_BOOL("notifs_en", g_debbie_config.notifications_enabled); + GET_BOOL("notif_wa", g_debbie_config.notif_whatsapp); + GET_BOOL("notif_em", g_debbie_config.notif_email); + GET_BOOL("notif_sp", g_debbie_config.notif_spotify); + + /* Advanced */ + cJSON *vol = cJSON_GetObjectItem(json, "volume"); + if (vol && cJSON_IsNumber(vol)) + g_debbie_config.speaker_volume = (uint8_t)vol->valuedouble; + + cJSON *vad = cJSON_GetObjectItem(json, "vad_threshold"); + if (vad && cJSON_IsNumber(vad)) + g_debbie_config.vad_threshold = (uint16_t)vad->valuedouble; + + GET_BOOL("cam_en", g_debbie_config.camera_enabled); + + cJSON_Delete(json); + storage_save_config(); + + const char *resp = "{\"ok\":true,\"msg\":\"Settings saved!\"}"; + httpd_resp_set_type(req, "application/json"); + httpd_resp_send(req, resp, strlen(resp)); + + /* Reconnect with new credentials in background */ + vTaskDelay(pdMS_TO_TICKS(500)); + wifi_manager_reconnect(); + + return ESP_OK; +} + +static esp_err_t handler_reset(httpd_req_t *req) +{ + storage_factory_reset(); + httpd_resp_send(req, "{\"ok\":true}", strlen("{\"ok\":true}")); + vTaskDelay(pdMS_TO_TICKS(1000)); + esp_restart(); + return ESP_OK; +} + +static esp_err_t handler_snapshot(httpd_req_t *req) +{ + if (!g_debbie_config.camera_enabled) { + httpd_resp_send_err(req, HTTPD_404_NOT_FOUND, "Camera disabled"); + return ESP_FAIL; + } + uint8_t *jpeg = NULL; + size_t len = 0; + if (camera_manager_capture_jpeg(&jpeg, &len) != ESP_OK) { + httpd_resp_send_500(req); + return ESP_FAIL; + } + httpd_resp_set_type(req, "image/jpeg"); + httpd_resp_send(req, (char *)jpeg, len); + camera_manager_free_frame(jpeg); + return ESP_OK; +} + /* -------------------------------------------------------------------------- */ -/* Inline HTML for the setup portal โ€” friendly, mobile-first design */ -static const char SETUP_HTML[] = +static httpd_handle_t s_server = NULL; + +esp_err_t web_server_start(void) +{ + httpd_config_t cfg = HTTPD_DEFAULT_CONFIG(); + cfg.max_uri_handlers = 8; + cfg.stack_size = 12288; /* larger stack for big HTML responses */ + + ESP_ERROR_CHECK(httpd_start(&s_server, &cfg)); + + const httpd_uri_t routes[] = { + { .uri = "/", .method = HTTP_GET, .handler = handler_root }, + { .uri = "/status", .method = HTTP_GET, .handler = handler_status }, + { .uri = "/configure", .method = HTTP_POST, .handler = handler_configure }, + { .uri = "/reset", .method = HTTP_POST, .handler = handler_reset }, + { .uri = "/snapshot", .method = HTTP_GET, .handler = handler_snapshot }, + }; + + for (int i = 0; i < (int)(sizeof(routes) / sizeof(routes[0])); i++) { + ESP_ERROR_CHECK(httpd_register_uri_handler(s_server, &routes[i])); + } + + ESP_LOGI(TAG, "Web server started at http://%s/", DEBBIE_AP_IP); + return ESP_OK; +} + +esp_err_t web_server_stop(void) +{ + if (s_server) { + httpd_stop(s_server); + s_server = NULL; + } + return ESP_OK; +} "" "" "Debbie Setup โœจ" diff --git a/firmware/main/wifi_manager.c b/firmware/main/wifi_manager.c index b19a264..6964946 100644 --- a/firmware/main/wifi_manager.c +++ b/firmware/main/wifi_manager.c @@ -63,6 +63,34 @@ static void event_handler(void *arg, esp_event_base_t base, /* -------------------------------------------------------------------------- */ +/** + * @brief Try connecting to a single SSID/password combination. + * Returns ESP_OK if connected within timeout, ESP_FAIL otherwise. + */ +static esp_err_t try_connect(const char *ssid, const char *password) +{ + if (!ssid || strlen(ssid) == 0) return ESP_FAIL; + + ESP_LOGI(TAG, "Trying WiFi network: %s", ssid); + xEventGroupClearBits(s_wifi_event_group, WIFI_CONNECTED_BIT | WIFI_FAIL_BIT); + s_retry_count = 0; + + wifi_config_t sta_config = { 0 }; + strncpy((char *)sta_config.sta.ssid, ssid, sizeof(sta_config.sta.ssid) - 1); + strncpy((char *)sta_config.sta.password, password, sizeof(sta_config.sta.password) - 1); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &sta_config)); + esp_wifi_disconnect(); + esp_wifi_connect(); + + EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group, + WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, + pdFALSE, pdFALSE, + pdMS_TO_TICKS(10000)); + return (bits & WIFI_CONNECTED_BIT) ? ESP_OK : ESP_FAIL; +} + +/* -------------------------------------------------------------------------- */ + esp_err_t wifi_manager_init(void) { s_wifi_event_group = xEventGroupCreate(); @@ -80,7 +108,7 @@ esp_err_t wifi_manager_init(void) ESP_ERROR_CHECK(esp_event_handler_instance_register( IP_EVENT, IP_EVENT_STA_GOT_IP, event_handler, NULL, NULL)); - /* AP config โ€” always on for setup */ + /* AP config โ€” always on so the config portal is reachable */ wifi_config_t ap_config = { .ap = { .ssid = DEBBIE_AP_SSID, @@ -91,34 +119,42 @@ esp_err_t wifi_manager_init(void) }, }; - if (strlen(g_debbie_config.wifi_ssid) > 0) { - /* Try STA + AP together */ + /* Check if any network is configured */ + bool has_network = (strlen(g_debbie_config.wifi_ssid) > 0 || + strlen(g_debbie_config.wifi_ssid2) > 0 || + strlen(g_debbie_config.wifi_ssid3) > 0); + + if (has_network) { ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_APSTA)); - wifi_config_t sta_config = { 0 }; - strncpy((char *)sta_config.sta.ssid, g_debbie_config.wifi_ssid, - sizeof(sta_config.sta.ssid) - 1); - strncpy((char *)sta_config.sta.password, g_debbie_config.wifi_password, - sizeof(sta_config.sta.password) - 1); - ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &sta_config)); - ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_AP, &ap_config)); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_AP, &ap_config)); ESP_ERROR_CHECK(esp_wifi_start()); - /* Wait up to 10 s for connection */ - EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group, - WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, - pdFALSE, pdFALSE, - pdMS_TO_TICKS(10000)); - if (bits & WIFI_CONNECTED_BIT) { - ESP_LOGI(TAG, "Connected to WiFi: %s", g_debbie_config.wifi_ssid); - return ESP_OK; + /* Try each saved network in priority order */ + const char *ssids[3] = { + g_debbie_config.wifi_ssid, + g_debbie_config.wifi_ssid2, + g_debbie_config.wifi_ssid3, + }; + const char *passwords[3] = { + g_debbie_config.wifi_password, + g_debbie_config.wifi_password2, + g_debbie_config.wifi_password3, + }; + + for (int i = 0; i < 3; i++) { + if (try_connect(ssids[i], passwords[i]) == ESP_OK) { + ESP_LOGI(TAG, "Connected to WiFi: %s", ssids[i]); + return ESP_OK; + } } - ESP_LOGW(TAG, "Could not connect to saved WiFi โ€” entering setup mode"); + + ESP_LOGW(TAG, "All saved networks failed โ€” staying in AP+STA mode"); } else { /* AP-only setup mode */ ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_AP)); ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_AP, &ap_config)); ESP_ERROR_CHECK(esp_wifi_start()); - ESP_LOGI(TAG, "AP-only mode โ€” connect to '%s' and visit %s", + ESP_LOGI(TAG, "AP-only mode โ€” connect to '%s' and visit http://%s/", DEBBIE_AP_SSID, DEBBIE_AP_IP); } return ESP_FAIL; @@ -128,15 +164,22 @@ bool wifi_manager_is_connected(void) { return s_connected; } esp_err_t wifi_manager_reconnect(void) { - s_retry_count = 0; - xEventGroupClearBits(s_wifi_event_group, WIFI_CONNECTED_BIT | WIFI_FAIL_BIT); - esp_wifi_disconnect(); - esp_wifi_connect(); - EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group, - WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, - pdFALSE, pdFALSE, - pdMS_TO_TICKS(15000)); - return (bits & WIFI_CONNECTED_BIT) ? ESP_OK : ESP_FAIL; + const char *ssids[3] = { + g_debbie_config.wifi_ssid, + g_debbie_config.wifi_ssid2, + g_debbie_config.wifi_ssid3, + }; + const char *passwords[3] = { + g_debbie_config.wifi_password, + g_debbie_config.wifi_password2, + g_debbie_config.wifi_password3, + }; + + for (int i = 0; i < 3; i++) { + if (try_connect(ssids[i], passwords[i]) == ESP_OK) + return ESP_OK; + } + return ESP_FAIL; } esp_err_t wifi_manager_set_credentials(const char *ssid, const char *password) @@ -146,3 +189,4 @@ esp_err_t wifi_manager_set_credentials(const char *ssid, const char *password) storage_save_config(); return wifi_manager_reconnect(); } + diff --git a/firmware/sdkconfig.defaults b/firmware/sdkconfig.defaults index 47a3b73..de3c848 100644 --- a/firmware/sdkconfig.defaults +++ b/firmware/sdkconfig.defaults @@ -24,6 +24,20 @@ CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=16 CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=64 CONFIG_ESP_WIFI_TX_BUFFER_TYPE_DYNAMIC=y +# โ”€โ”€โ”€ Bluetooth (BLE only โ€” ESP32-S3 does not support Classic BT) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +CONFIG_BT_ENABLED=y +CONFIG_BT_BLE_ENABLED=y +CONFIG_BT_BLUEDROID_ENABLED=y +CONFIG_BT_CLASSIC_ENABLED=n +CONFIG_BT_BLE_42_FEATURES_SUPPORTED=y +CONFIG_BT_BLE_50_FEATURES_SUPPORTED=n +CONFIG_BT_GATTS_ENABLE=y +CONFIG_BT_GATT_MAX_SR_PROFILES=8 +CONFIG_BT_GATT_MAX_SR_ATTRIBUTES=100 +# Reserve enough heap for BLE stack +CONFIG_BT_CTRL_BLE_MAX_ACT=10 +CONFIG_BT_CTRL_BLE_STATIC_ACL_TX_BUF_NB=0 + # โ”€โ”€โ”€ TLS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # SECURITY WARNING: The settings below are for development only. # For production deployments, remove CONFIG_ESP_TLS_INSECURE and @@ -36,8 +50,8 @@ CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y # NVS encryption (disabled for simplicity โ€” enable for production) CONFIG_NVS_ENCRYPTION=n -# Stack sizes -CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 +# Stack sizes (increase main task to accommodate BLE + WiFi coexistence) +CONFIG_ESP_MAIN_TASK_STACK_SIZE=12288 # Logging CONFIG_LOG_DEFAULT_LEVEL_INFO=y @@ -63,3 +77,4 @@ CONFIG_HTTPD_MAX_URI_LEN=512 # Camera CONFIG_CAMERA_CORE0=n CONFIG_CAMERA_CORE1=y + From f31c0fca23b5aa07efa007115f5a0a467c3bc95c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 17:02:15 +0000 Subject: [PATCH 6/8] Fix code review issues: proper NUS UUIDs, consistent key names, remove dead code Agent-Logs-Url: https://github.com/magicalmutation-coder/DebbieDoesMobile/sessions/22321e97-0113-4701-b064-f09c0b0ab890 Co-authored-by: magicalmutation-coder <246345154+magicalmutation-coder@users.noreply.github.com> --- firmware/main/bluetooth_manager.c | 54 ++++--- firmware/main/web_server.c | 234 +----------------------------- 2 files changed, 35 insertions(+), 253 deletions(-) diff --git a/firmware/main/bluetooth_manager.c b/firmware/main/bluetooth_manager.c index bca4db9..efcf091 100644 --- a/firmware/main/bluetooth_manager.c +++ b/firmware/main/bluetooth_manager.c @@ -32,14 +32,27 @@ static const char *TAG = "ble"; -/* โ”€โ”€ Nordic UART Service UUIDs โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ -/* NUS service: 6E400001-B5A3-F393-E0A9-E50E24DCCA9E */ -/* NUS RX (write): 6E400002-B5A3-F393-E0A9-E50E24DCCA9E */ -/* NUS TX (notify): 6E400003-B5A3-F393-E0A9-E50E24DCCA9E */ - -#define NUS_SERVICE_UUID 0xABCD /* abbreviated handles for simplicity */ -#define NUS_RX_CHAR_UUID 0xABCE -#define NUS_TX_CHAR_UUID 0xABCF +/* โ”€โ”€ Nordic UART Service โ€” official 128-bit UUIDs โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +/* These are the registered NUS UUIDs. Standard BLE Serial apps (nRF Toolbox, + * Serial Bluetooth Terminal, etc.) look for exactly these UUIDs. + * UUID bytes are stored in ESP-IDF little-endian order. + * + * Service: 6E400001-B5A3-F393-E0A9-E50E24DCCA9E + * RX(write):6E400002-B5A3-F393-E0A9-E50E24DCCA9E + * TX(notify):6E400003-B5A3-F393-E0A9-E50E24DCCA9E + */ +static const uint8_t NUS_SVC_UUID128[16] = { + 0x9E,0xCA,0xDC,0x24,0xE2,0x50,0xA9,0xE0, + 0x93,0xF3,0xA3,0xB5,0x01,0x00,0x40,0x6E +}; +static const uint8_t NUS_RX_UUID128[16] = { + 0x9E,0xCA,0xDC,0x24,0xE2,0x50,0xA9,0xE0, + 0x93,0xF3,0xA3,0xB5,0x02,0x00,0x40,0x6E +}; +static const uint8_t NUS_TX_UUID128[16] = { + 0x9E,0xCA,0xDC,0x24,0xE2,0x50,0xA9,0xE0, + 0x93,0xF3,0xA3,0xB5,0x03,0x00,0x40,0x6E +}; /* Device Information Service */ #define DIS_SERVICE_UUID 0x180A @@ -113,7 +126,7 @@ static void process_ble_command(const char *json_str) GET_STR("groq_key", g_debbie_config.groq_api_key); GET_STR("or_key", g_debbie_config.openrouter_api_key); GET_STR("local_llm_url", g_debbie_config.local_llm_url); - GET_STR("local_llm_mdl", g_debbie_config.local_llm_model); + GET_STR("local_llm_model", g_debbie_config.local_llm_model); /* Agent */ GET_STR("agent_url", g_debbie_config.agent_ws_url); @@ -178,14 +191,14 @@ static void gatts_event_handler(esp_gatts_cb_event_t event, esp_ble_gap_config_adv_data_raw(adv_data, sizeof(adv_data)); } - /* Create Nordic UART Service */ + /* Create Nordic UART Service with official 128-bit UUID */ { esp_gatt_srvc_id_t svc = { .is_primary = true, .id = { .inst_id = 0, - .uuid = { .len = ESP_UUID_LEN_16, - .uuid.uuid16 = NUS_SERVICE_UUID } } + .uuid = { .len = ESP_UUID_LEN_128 } } }; + memcpy(svc.id.uuid.uuid.uuid128, NUS_SVC_UUID128, 16); esp_ble_gatts_create_service(gatts_if, &svc, GATTS_NUM_HANDLE); } break; @@ -194,18 +207,18 @@ static void gatts_event_handler(esp_gatts_cb_event_t event, uint16_t svc_handle = param->create.service_handle; esp_ble_gatts_start_service(svc_handle); - /* RX characteristic (write without response) */ - esp_bt_uuid_t rx_uuid = { .len = ESP_UUID_LEN_16, - .uuid.uuid16 = NUS_RX_CHAR_UUID }; + /* RX characteristic (write without response) โ€” 128-bit NUS RX UUID */ + esp_bt_uuid_t rx_uuid = { .len = ESP_UUID_LEN_128 }; + memcpy(rx_uuid.uuid.uuid128, NUS_RX_UUID128, 16); esp_ble_gatts_add_char(svc_handle, &rx_uuid, ESP_GATT_PERM_WRITE, ESP_GATT_CHAR_PROP_BIT_WRITE | ESP_GATT_CHAR_PROP_BIT_WRITE_NR, NULL, NULL); - /* TX characteristic (notify) */ - esp_bt_uuid_t tx_uuid = { .len = ESP_UUID_LEN_16, - .uuid.uuid16 = NUS_TX_CHAR_UUID }; + /* TX characteristic (notify) โ€” 128-bit NUS TX UUID */ + esp_bt_uuid_t tx_uuid = { .len = ESP_UUID_LEN_128 }; + memcpy(tx_uuid.uuid.uuid128, NUS_TX_UUID128, 16); esp_ble_gatts_add_char(svc_handle, &tx_uuid, ESP_GATT_PERM_READ, ESP_GATT_CHAR_PROP_BIT_NOTIFY, @@ -214,9 +227,10 @@ static void gatts_event_handler(esp_gatts_cb_event_t event, } case ESP_GATTS_ADD_CHAR_EVT: - if (param->add_char.char_uuid.uuid.uuid16 == NUS_TX_CHAR_UUID) { + if (param->add_char.char_uuid.len == ESP_UUID_LEN_128 && + memcmp(param->add_char.char_uuid.uuid.uuid128, NUS_TX_UUID128, 16) == 0) { s_tx_handle = param->add_char.attr_handle; - /* Add CCCD for TX */ + /* Add CCCD (2902) for TX notifications */ esp_bt_uuid_t cccd_uuid = { .len = ESP_UUID_LEN_16, .uuid.uuid16 = ESP_GATT_UUID_CHAR_CLIENT_CONFIG }; esp_ble_gatts_add_char_descr(param->add_char.service_handle, diff --git a/firmware/main/web_server.c b/firmware/main/web_server.c index ff9d7b1..351f660 100644 --- a/firmware/main/web_server.c +++ b/firmware/main/web_server.c @@ -682,239 +682,7 @@ esp_err_t web_server_start(void) { httpd_config_t cfg = HTTPD_DEFAULT_CONFIG(); cfg.max_uri_handlers = 8; - cfg.stack_size = 12288; /* larger stack for big HTML responses */ - - ESP_ERROR_CHECK(httpd_start(&s_server, &cfg)); - - const httpd_uri_t routes[] = { - { .uri = "/", .method = HTTP_GET, .handler = handler_root }, - { .uri = "/status", .method = HTTP_GET, .handler = handler_status }, - { .uri = "/configure", .method = HTTP_POST, .handler = handler_configure }, - { .uri = "/reset", .method = HTTP_POST, .handler = handler_reset }, - { .uri = "/snapshot", .method = HTTP_GET, .handler = handler_snapshot }, - }; - - for (int i = 0; i < (int)(sizeof(routes) / sizeof(routes[0])); i++) { - ESP_ERROR_CHECK(httpd_register_uri_handler(s_server, &routes[i])); - } - - ESP_LOGI(TAG, "Web server started at http://%s/", DEBBIE_AP_IP); - return ESP_OK; -} - -esp_err_t web_server_stop(void) -{ - if (s_server) { - httpd_stop(s_server); - s_server = NULL; - } - return ESP_OK; -} -"" -"" -"Debbie Setup โœจ" -"" -"
" -"

โœจ Hi, I'm Debbie!

" -"

Your portable personal AI companion โ€” let's get set up!

" -"
" - -"
" -"

๐Ÿ“ถ Wi-Fi Connection

" -"" -"" -"
" - -"
" -"

๐Ÿค– AI Settings

" -"" -"" -"" -"" -"" -"" -"
" - -"
" -"

๐ŸŽ€ Debbie's Personality

" -"" -"" -"" -"" -"" -"
" - -"
" -"" -"" -"
" -"
" - -""; - -/* โ”€โ”€ Handlers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ - -static esp_err_t handler_root(httpd_req_t *req) -{ - httpd_resp_set_type(req, "text/html"); - httpd_resp_send(req, SETUP_HTML, strlen(SETUP_HTML)); - return ESP_OK; -} - -static esp_err_t handler_status(httpd_req_t *req) -{ - cJSON *json = cJSON_CreateObject(); - cJSON_AddStringToObject(json, "name", g_debbie_config.debbie_name); - cJSON_AddStringToObject(json, "sys_prompt", g_debbie_config.system_prompt); - cJSON_AddStringToObject(json, "agent_url", g_debbie_config.agent_ws_url); - cJSON_AddStringToObject(json, "companion_url",g_debbie_config.companion_url); - cJSON_AddNumberToObject(json, "volume", g_debbie_config.speaker_volume); - cJSON_AddBoolToObject(json, "wifi_ok", wifi_manager_is_connected()); - cJSON_AddStringToObject(json, "state", - g_debbie_state == DEBBIE_STATE_IDLE ? "idle" : - g_debbie_state == DEBBIE_STATE_LISTENING ? "listening" : - g_debbie_state == DEBBIE_STATE_THINKING ? "thinking" : - g_debbie_state == DEBBIE_STATE_SPEAKING ? "speaking" : "other"); - - char *str = cJSON_PrintUnformatted(json); - cJSON_Delete(json); - httpd_resp_set_type(req, "application/json"); - httpd_resp_send(req, str, strlen(str)); - free(str); - return ESP_OK; -} - -static esp_err_t handler_configure(httpd_req_t *req) -{ - char *buf = malloc(req->content_len + 1); - if (!buf) { httpd_resp_send_500(req); return ESP_FAIL; } - - int received = httpd_req_recv(req, buf, req->content_len); - if (received <= 0) { free(buf); httpd_resp_send_500(req); return ESP_FAIL; } - buf[received] = '\0'; - - cJSON *json = cJSON_Parse(buf); - free(buf); - if (!json) { httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "Invalid JSON"); return ESP_FAIL; } - -#define GET_STR(key, dst) do { \ - cJSON *j = cJSON_GetObjectItem(json, key); \ - if (j && cJSON_IsString(j) && strlen(j->valuestring) > 0) \ - strncpy(dst, j->valuestring, sizeof(dst) - 1); \ -} while (0) - - GET_STR("ssid", g_debbie_config.wifi_ssid); - GET_STR("pass", g_debbie_config.wifi_password); - GET_STR("oai_key", g_debbie_config.openai_api_key); - GET_STR("agent_url", g_debbie_config.agent_ws_url); - GET_STR("companion_url",g_debbie_config.companion_url); - GET_STR("name", g_debbie_config.debbie_name); - GET_STR("sys_prompt", g_debbie_config.system_prompt); - - cJSON *vol = cJSON_GetObjectItem(json, "volume"); - if (vol && cJSON_IsNumber(vol)) - g_debbie_config.speaker_volume = (uint8_t)vol->valuedouble; - - cJSON_Delete(json); - storage_save_config(); - - const char *resp = "{\"ok\":true,\"msg\":\"Saved! Connecting to WiFi...\"}"; - httpd_resp_set_type(req, "application/json"); - httpd_resp_send(req, resp, strlen(resp)); - - /* Reconnect with new credentials in background */ - vTaskDelay(pdMS_TO_TICKS(500)); - wifi_manager_reconnect(); - - return ESP_OK; -} - -static esp_err_t handler_reset(httpd_req_t *req) -{ - storage_factory_reset(); - httpd_resp_send(req, "{\"ok\":true}", strlen("{\"ok\":true}")); - vTaskDelay(pdMS_TO_TICKS(1000)); - esp_restart(); - return ESP_OK; -} - -static esp_err_t handler_snapshot(httpd_req_t *req) -{ - if (!g_debbie_config.camera_enabled) { - httpd_resp_send_err(req, HTTPD_404_NOT_FOUND, "Camera disabled"); - return ESP_FAIL; - } - uint8_t *jpeg = NULL; - size_t len = 0; - if (camera_manager_capture_jpeg(&jpeg, &len) != ESP_OK) { - httpd_resp_send_500(req); - return ESP_FAIL; - } - httpd_resp_set_type(req, "image/jpeg"); - httpd_resp_send(req, (char *)jpeg, len); - camera_manager_free_frame(jpeg); - return ESP_OK; -} - -/* -------------------------------------------------------------------------- */ - -static httpd_handle_t s_server = NULL; - -esp_err_t web_server_start(void) -{ - httpd_config_t cfg = HTTPD_DEFAULT_CONFIG(); - cfg.max_uri_handlers = 8; - cfg.stack_size = 8192; + cfg.stack_size = 12288; /* increased for BLE + WiFi coexistence overhead */ ESP_ERROR_CHECK(httpd_start(&s_server, &cfg)); From 4479a40f9905a93236b1bdca256f3ea8b5d44ad0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 17:15:22 +0000 Subject: [PATCH 7/8] Add memory manager, RAG, and persistence across firmware and companion server Agent-Logs-Url: https://github.com/magicalmutation-coder/DebbieDoesMobile/sessions/579a96ff-263f-4aae-b142-0e518af43f49 Co-authored-by: magicalmutation-coder <246345154+magicalmutation-coder@users.noreply.github.com> --- .gitignore | 1 + companion-server/.env.example | 4 + companion-server/memory_store.js | 302 +++++++++++++++++ companion-server/package.json | 3 +- companion-server/server.js | 4 + firmware/main/CMakeLists.txt | 40 +++ firmware/main/debbie.h | 5 + firmware/main/main.c | 48 ++- firmware/main/memory_manager.c | 534 +++++++++++++++++++++++++++++++ firmware/main/memory_manager.h | 140 ++++++++ firmware/main/openai_client.c | 49 ++- firmware/main/openai_client.h | 7 +- firmware/main/settings.h | 5 + firmware/main/storage_manager.c | 13 + firmware/main/web_server.c | 133 +++++++- 15 files changed, 1273 insertions(+), 15 deletions(-) create mode 100644 companion-server/memory_store.js create mode 100644 firmware/main/memory_manager.c create mode 100644 firmware/main/memory_manager.h diff --git a/.gitignore b/.gitignore index 65aeb00..57ce3bf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ companion-server/node_modules/ companion-server/.env companion-server/.wwebjs_auth/ companion-server/.wwebjs_cache/ +companion-server/debbie_memory.db # OS .DS_Store diff --git a/companion-server/.env.example b/companion-server/.env.example index 40c7dde..e860285 100644 --- a/companion-server/.env.example +++ b/companion-server/.env.example @@ -34,3 +34,7 @@ SPOTIFY_REFRESH_TOKEN= # WebSocket URL of your custom agent, if available # Leave blank to use only OpenAI AGENT_URL=ws://localhost:8080 + +# โ”€โ”€ Memory & RAG โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Path for the SQLite memory database (default: companion-server/debbie_memory.db) +# MEMORY_DB_PATH=/path/to/debbie_memory.db diff --git a/companion-server/memory_store.js b/companion-server/memory_store.js new file mode 100644 index 0000000..a2571ba --- /dev/null +++ b/companion-server/memory_store.js @@ -0,0 +1,302 @@ +/** + * memory_store.js โ€” Debbie's persistent memory & RAG backend + * + * Uses SQLite (better-sqlite3) with: + * - `facts` table: long-term key-value facts + * - `turns` FTS5 table: conversation history with full-text search + * + * REST endpoints mounted by addMemoryRoutes(): + * GET /memory/query?q=&limit= โ€” retrieve relevant memories + * POST /memory/turn โ€” store a conversation turn + * POST /memory/fact โ€” store / update a fact + * GET /memory/stats โ€” counts + recent summary + * DELETE /memory/clear โ€” wipe all memory + */ + +'use strict'; + +const path = require('path'); +const rateLimit = require('express-rate-limit'); + +let db = null; + +/* โ”€โ”€ DB setup โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +function initDb() { + if (db) return db; + + let Database; + try { + Database = require('better-sqlite3'); + } catch (e) { + console.warn('[memory] better-sqlite3 not installed โ€” memory store disabled.'); + console.warn('[memory] Run: cd companion-server && npm install'); + return null; + } + + const dbPath = process.env.MEMORY_DB_PATH || + path.join(__dirname, 'debbie_memory.db'); + + db = new Database(dbPath); + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = NORMAL'); + + /* Long-term facts */ + db.exec(` + CREATE TABLE IF NOT EXISTS facts ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + importance INTEGER DEFAULT 5, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + `); + + /* Conversation turns โ€” FTS5 for full-text retrieval */ + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS turns USING fts5( + role, + text, + session_id UNINDEXED, + ts UNINDEXED, + tokenize = 'porter ascii' + ); + `); + + /* Regular table to track row count efficiently */ + db.exec(` + CREATE TABLE IF NOT EXISTS turn_meta ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + role TEXT, + ts INTEGER + ); + `); + + console.log(`[memory] SQLite memory store ready at ${dbPath}`); + return db; +} + +/* โ”€โ”€ Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +function nowMs() { + return Date.now(); +} + +function sanitiseText(t) { + if (typeof t !== 'string') return ''; + return t.slice(0, 2000).replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ''); +} + +function sanitiseKey(k) { + if (typeof k !== 'string') return ''; + return k.slice(0, 64).replace(/[^a-zA-Z0-9_\-\.]/g, '_'); +} + +/* โ”€โ”€ Core operations โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/** + * Store / update a long-term fact. + */ +function saveFact(key, value, importance = 5) { + if (!db) return false; + const k = sanitiseKey(key); + const v = sanitiseText(value); + const imp = Math.min(10, Math.max(0, parseInt(importance, 10) || 5)); + const now = nowMs(); + + db.prepare(` + INSERT INTO facts (key, value, importance, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(key) DO UPDATE SET + value = excluded.value, + importance = excluded.importance, + updated_at = excluded.updated_at + `).run(k, v, imp, now, now); + return true; +} + +/** + * Store a conversation turn. + * Prunes oldest turns when count exceeds maxTurns. + */ +function saveTurn(role, text, sessionId = 'default', maxTurns = 500) { + if (!db) return false; + const r = role === 'assistant' ? 'assistant' : 'user'; + const t = sanitiseText(text); + const now = nowMs(); + + db.prepare(`INSERT INTO turns (role, text, session_id, ts) VALUES (?, ?, ?, ?)`) + .run(r, t, sessionId, now); + db.prepare(`INSERT INTO turn_meta (role, ts) VALUES (?, ?)`) + .run(r, now); + + /* Prune if over limit */ + const countRow = db.prepare(`SELECT COUNT(*) AS n FROM turn_meta`).get(); + if (countRow && countRow.n > maxTurns) { + const overage = countRow.n - maxTurns; + /* Delete oldest from FTS โ€” need rowids */ + const oldRows = db.prepare( + `SELECT id FROM turn_meta ORDER BY id ASC LIMIT ?` + ).all(overage); + const stmt = db.prepare(`DELETE FROM turns WHERE rowid = ?`); + const del = db.prepare(`DELETE FROM turn_meta WHERE id = ?`); + for (const row of oldRows) { + stmt.run(row.id); + del.run(row.id); + } + } + return true; +} + +/** + * Full-text search over turns and facts, returning the top `limit` results. + * Falls back to recency-based retrieval when the query is empty. + */ +function queryMemory(queryText, limit = 5) { + if (!db) return []; + const lim = Math.min(20, Math.max(1, parseInt(limit, 10) || 5)); + const results = []; + + if (queryText && queryText.trim().length > 0) { + /* FTS5 search over turns โ€” wrap in try/catch for malformed queries */ + try { + const q = sanitiseText(queryText).trim(); + const rows = db.prepare(` + SELECT role, text, ts, + rank AS score + FROM turns + WHERE turns MATCH ? + ORDER BY rank + LIMIT ? + `).all(q, lim); + + for (const r of rows) { + results.push({ + source: 'conversation', + role: r.role, + text: r.text, + timestamp: r.ts, + }); + } + } catch (_) { + /* Fall through to recency fallback */ + } + + /* Also search facts */ + const factRows = db.prepare(` + SELECT key, value, importance, updated_at + FROM facts + WHERE LOWER(key) LIKE '%' || LOWER(?) || '%' + OR LOWER(value) LIKE '%' || LOWER(?) || '%' + ORDER BY importance DESC, updated_at DESC + LIMIT ? + `).all(queryText, queryText, lim); + + for (const r of factRows) { + results.push({ + source: 'fact', + key: r.key, + text: `${r.key}: ${r.value}`, + timestamp: r.updated_at, + }); + } + } else { + /* No query โ€” return most recent turns */ + const rows = db.prepare(` + SELECT role, text, ts FROM turns + ORDER BY ts DESC + LIMIT ? + `).all(lim); + for (const r of rows) { + results.push({ + source: 'conversation', + role: r.role, + text: r.text, + timestamp: r.ts, + }); + } + } + + /* Sort by relevance then recency */ + results.sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0)); + return results.slice(0, lim); +} + +/** + * Return summary statistics. + */ +function getStats() { + if (!db) return { enabled: false }; + const factCount = db.prepare(`SELECT COUNT(*) AS n FROM facts`).get().n; + const turnCount = db.prepare(`SELECT COUNT(*) AS n FROM turn_meta`).get().n; + const recent = db.prepare(` + SELECT role, text, ts FROM turns ORDER BY ts DESC LIMIT 5 + `).all(); + return { enabled: true, facts: factCount, turns: turnCount, recent }; +} + +/** + * Wipe all stored memory. + */ +function clearAll() { + if (!db) return; + db.exec(`DELETE FROM facts; DELETE FROM turns; DELETE FROM turn_meta;`); + console.log('[memory] All memory cleared'); +} + +/* โ”€โ”€ Express route handler โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +const memLimiter = rateLimit({ windowMs: 60000, max: 120 }); + +function addMemoryRoutes(app) { + /* Lazy-initialise the DB */ + const database = initDb(); + if (!database) { + /* If better-sqlite3 is missing, mount stub routes that return 503 */ + app.use('/memory', (req, res) => { + res.status(503).json({ + error: 'Memory store not available. Run: npm install', + }); + }); + return; + } + + /* GET /memory/query?q=&limit= */ + app.get('/memory/query', memLimiter, (req, res) => { + const q = typeof req.query.q === 'string' ? req.query.q : ''; + const limit = parseInt(req.query.limit, 10) || 5; + const memories = queryMemory(q, limit); + res.json({ ok: true, query: q, memories }); + }); + + /* POST /memory/turn โ€” { role, text, session_id? } */ + app.post('/memory/turn', memLimiter, (req, res) => { + const { role = 'user', text = '', session_id = 'default' } = req.body || {}; + if (!text) return res.status(400).json({ error: 'text required' }); + saveTurn(role, text, session_id); + res.json({ ok: true }); + }); + + /* POST /memory/fact โ€” { key, value, importance? } */ + app.post('/memory/fact', memLimiter, (req, res) => { + const { key = '', value = '', importance = 5 } = req.body || {}; + if (!key || !value) return res.status(400).json({ error: 'key and value required' }); + const ok = saveFact(key, value, importance); + res.json({ ok }); + }); + + /* GET /memory/stats */ + app.get('/memory/stats', memLimiter, (req, res) => { + res.json(getStats()); + }); + + /* DELETE /memory/clear */ + app.delete('/memory/clear', memLimiter, (req, res) => { + clearAll(); + res.json({ ok: true }); + }); + + console.log('[memory] Routes mounted: /memory/{query,turn,fact,stats,clear}'); +} + +module.exports = { addMemoryRoutes, saveFact, saveTurn, queryMemory, getStats, clearAll, initDb }; diff --git a/companion-server/package.json b/companion-server/package.json index fd45c26..769bf63 100644 --- a/companion-server/package.json +++ b/companion-server/package.json @@ -19,7 +19,8 @@ "dotenv": "^16.3.1", "axios": "^1.6.0", "node-cron": "^3.0.3", - "cors": "^2.8.5" + "cors": "^2.8.5", + "better-sqlite3": "^9.4.3" }, "devDependencies": { "nodemon": "^3.0.2" diff --git a/companion-server/server.js b/companion-server/server.js index 4763834..fc7fdd2 100644 --- a/companion-server/server.js +++ b/companion-server/server.js @@ -26,6 +26,7 @@ const { startEmailMonitor } = require('./email_monitor'); const { SpotifyController } = require('./spotify'); const { AgentBridge } = require('./agent_bridge'); const { addNetworkRoutes, storeScanResults } = require('./network_tools'); +const { addMemoryRoutes } = require('./memory_store'); const app = express(); const server = http.createServer(app); @@ -185,6 +186,9 @@ async function main() { /* Network security tools */ addNetworkRoutes(app, broadcast); + /* Memory & RAG store */ + addMemoryRoutes(app); + server.listen(PORT, '0.0.0.0', () => { console.log(`\n๐Ÿค– Debbie Companion Server running on port ${PORT}`); console.log(` Devices can connect to: ws://:${PORT}`); diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt index 2b6d005..edc39c5 100644 --- a/firmware/main/CMakeLists.txt +++ b/firmware/main/CMakeLists.txt @@ -1,4 +1,44 @@ idf_component_register( + SRCS + "main.c" + "storage_manager.c" + "wifi_manager.c" + "bluetooth_manager.c" + "audio_manager.c" + "camera_manager.c" + "display_manager.c" + "openai_client.c" + "notification_client.c" + "web_server.c" + "network_scanner.c" + "vuln_reporter.c" + "self_agent.c" + "memory_manager.c" + INCLUDE_DIRS + "." + REQUIRES + nvs_flash + esp_wifi + esp_event + esp_netif + esp_http_client + esp_https_ota + esp_websocket_client + esp_http_server + esp_camera + esp_lcd + driver + lvgl + cJSON + mbedtls + esp_timer + freertos + log + lwip + ping + bt +) + SRCS "main.c" "storage_manager.c" diff --git a/firmware/main/debbie.h b/firmware/main/debbie.h index 63c0aa5..4164734 100644 --- a/firmware/main/debbie.h +++ b/firmware/main/debbie.h @@ -95,6 +95,11 @@ typedef struct { bool notif_whatsapp; bool notif_email; bool notif_spotify; + + /* โ”€โ”€ Memory & RAG โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + bool memory_enabled; /* master on/off for conversation memory */ + bool memory_rag_enabled; /* query companion server for RAG retrieval */ + uint8_t memory_max_turns; /* number of recent turns to keep (default 20) */ } debbie_config_t; /* Event IDs published on the default event loop */ diff --git a/firmware/main/main.c b/firmware/main/main.c index b4e2408..7e84069 100644 --- a/firmware/main/main.c +++ b/firmware/main/main.c @@ -29,6 +29,7 @@ #include "network_scanner.h" #include "vuln_reporter.h" #include "self_agent.h" +#include "memory_manager.h" #include "esp_log.h" #include "esp_timer.h" @@ -112,6 +113,16 @@ static void on_oai_event(const oai_event_data_t *evt, void *ctx) case OAI_EVT_TRANSCRIPT: ESP_LOGI(TAG, "Transcript: %s", evt->transcript.text); display_manager_show_text(evt->transcript.text); + /* Record AI turn in memory */ + memory_manager_add_turn("assistant", evt->transcript.text); + memory_manager_sync_turn("assistant", evt->transcript.text); + break; + + case OAI_EVT_USER_TRANSCRIPT: + ESP_LOGI(TAG, "User said: %s", evt->transcript.text); + /* Record user turn in memory */ + memory_manager_add_turn("user", evt->transcript.text); + memory_manager_sync_turn("user", evt->transcript.text); break; case OAI_EVT_FUNCTION_CALL: @@ -170,6 +181,28 @@ static void on_oai_event(const oai_event_data_t *evt, void *ctx) if (args) cJSON_Delete(args); /* โ”€โ”€ Network security & self-agent (all other tools) โ”€โ”€ */ + } else if (strcmp(evt->fn.name, "save_memory") == 0) { + /* AI can explicitly save a fact to long-term memory */ + cJSON *args = cJSON_Parse(evt->fn.args_json); + cJSON *key = args ? cJSON_GetObjectItem(args, "key") : NULL; + cJSON *val = args ? cJSON_GetObjectItem(args, "value") : NULL; + cJSON *imp = args ? cJSON_GetObjectItem(args, "importance") : NULL; + if (key && val && cJSON_IsString(key) && cJSON_IsString(val)) { + uint8_t importance = imp ? (uint8_t)imp->valuedouble : 7; + memory_manager_save_fact(key->valuestring, + val->valuestring, + importance); + char result[256]; + snprintf(result, sizeof(result), + "{\"status\":\"ok\",\"key\":\"%s\",\"value\":\"%s\"}", + key->valuestring, val->valuestring); + openai_client_send_function_result(evt->fn.call_id, result); + } else { + openai_client_send_function_result(evt->fn.call_id, + "{\"error\":\"key and value are required\"}"); + } + if (args) cJSON_Delete(args); + } else { /* Defer to the self_agent dispatcher for: * network_scan, get_vuln_report, system_info, @@ -438,6 +471,9 @@ void app_main(void) /* 1. NVS + configuration */ ESP_ERROR_CHECK(storage_init()); + /* 1b. Memory manager โ€” loads persisted facts & recent turns from NVS */ + memory_manager_init(); + /* 2. Display โ€” show boot splash as early as possible */ ESP_ERROR_CHECK(display_manager_init()); set_state(DEBBIE_STATE_BOOT); @@ -509,13 +545,21 @@ void app_main(void) /* 10. Battery monitor */ xTaskCreate(battery_task, "battery", 2048, NULL, 1, NULL); - /* 11. Connect to AI */ + /* 11. Connect to AI (inject memory context into system prompt) */ if (strlen(g_debbie_config.openai_api_key) > 0) { + /* Build memory-enriched system prompt */ + char *enriched_prompt = memory_manager_enrich_prompt( + g_debbie_config.system_prompt); + const char *prompt_to_use = enriched_prompt + ? enriched_prompt : g_debbie_config.system_prompt; + esp_err_t ai_err = openai_client_connect( g_debbie_config.openai_api_key, - g_debbie_config.system_prompt, + prompt_to_use, on_oai_event, NULL); + free(enriched_prompt); /* safe even if NULL */ + if (ai_err != ESP_OK) { ESP_LOGW(TAG, "Could not connect to OpenAI โ€” check API key"); display_manager_show_text( diff --git a/firmware/main/memory_manager.c b/firmware/main/memory_manager.c new file mode 100644 index 0000000..9b66238 --- /dev/null +++ b/firmware/main/memory_manager.c @@ -0,0 +1,534 @@ +/* + * memory_manager.c โ€” Debbie's short-term & long-term memory + RAG + * + * Short-term : circular buffer of recent turns in RAM. + * Long-term : key-value facts persisted to NVS flash. + * RAG : companion server queried via HTTP for richer context. + */ + +#include "memory_manager.h" +#include "debbie.h" +#include "settings.h" +#include "esp_log.h" +#include "esp_http_client.h" +#include "esp_timer.h" +#include "nvs_flash.h" +#include "nvs.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "cJSON.h" +#include +#include +#include + +static const char *TAG = "mem"; + +#define NVS_MEM_NS "debbie_mem" + +/* โ”€โ”€ State โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/* Short-term: ring buffer */ +static memory_turn_t s_turns[MEMORY_MAX_TURNS]; +static int s_turn_head = 0; /* next write position */ +static int s_turn_count = 0; /* how many are populated */ + +/* Long-term facts */ +static memory_fact_t s_facts[MEMORY_MAX_FACTS]; +static int s_fact_count = 0; + +/* โ”€โ”€ NVS helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +static void nvs_write_facts(nvs_handle_t nvs) +{ + nvs_set_u8(nvs, "mem_n_facts", (uint8_t)s_fact_count); + char key[16]; + for (int i = 0; i < s_fact_count; i++) { + /* Serialise each fact as compact JSON */ + cJSON *j = cJSON_CreateObject(); + cJSON_AddStringToObject(j, "k", s_facts[i].key); + cJSON_AddStringToObject(j, "v", s_facts[i].value); + cJSON_AddNumberToObject(j, "i", s_facts[i].importance); + cJSON_AddNumberToObject(j, "t", (double)s_facts[i].timestamp_ms); + char *str = cJSON_PrintUnformatted(j); + cJSON_Delete(j); + if (!str) continue; + snprintf(key, sizeof(key), "mf%d", i); + nvs_set_str(nvs, key, str); + free(str); + } +} + +static void nvs_write_recent_turns(nvs_handle_t nvs) +{ + /* Persist last 5 turns so memory survives a reboot */ + int keep = s_turn_count < 5 ? s_turn_count : 5; + nvs_set_u8(nvs, "mem_n_turns", (uint8_t)keep); + char key[16]; + for (int i = 0; i < keep; i++) { + int idx = ((s_turn_head - keep + i) + MEMORY_MAX_TURNS) % MEMORY_MAX_TURNS; + cJSON *j = cJSON_CreateObject(); + cJSON_AddStringToObject(j, "r", s_turns[idx].role); + cJSON_AddStringToObject(j, "t", s_turns[idx].text); + cJSON_AddNumberToObject(j, "s", (double)s_turns[idx].timestamp_ms); + char *str = cJSON_PrintUnformatted(j); + cJSON_Delete(j); + if (!str) continue; + snprintf(key, sizeof(key), "mt%d", i); + nvs_set_str(nvs, key, str); + free(str); + } +} + +static void nvs_read_facts(nvs_handle_t nvs) +{ + uint8_t count = 0; + if (nvs_get_u8(nvs, "mem_n_facts", &count) != ESP_OK) return; + if (count > MEMORY_MAX_FACTS) count = MEMORY_MAX_FACTS; + + char key[16]; + for (int i = 0; i < (int)count; i++) { + snprintf(key, sizeof(key), "mf%d", i); + char buf[300]; + size_t len = sizeof(buf); + if (nvs_get_str(nvs, key, buf, &len) != ESP_OK) continue; + + cJSON *j = cJSON_Parse(buf); + if (!j) continue; + cJSON *k = cJSON_GetObjectItem(j, "k"); + cJSON *v = cJSON_GetObjectItem(j, "v"); + cJSON *imp = cJSON_GetObjectItem(j, "i"); + cJSON *ts = cJSON_GetObjectItem(j, "t"); + if (k && v && cJSON_IsString(k) && cJSON_IsString(v)) { + strncpy(s_facts[s_fact_count].key, k->valuestring, + sizeof(s_facts[s_fact_count].key) - 1); + strncpy(s_facts[s_fact_count].value, v->valuestring, + sizeof(s_facts[s_fact_count].value) - 1); + s_facts[s_fact_count].importance = imp ? (uint8_t)imp->valuedouble : 5; + s_facts[s_fact_count].timestamp_ms = ts ? (int64_t)ts->valuedouble : 0; + s_fact_count++; + } + cJSON_Delete(j); + } + ESP_LOGI(TAG, "Loaded %d long-term facts from NVS", s_fact_count); +} + +static void nvs_read_turns(nvs_handle_t nvs) +{ + uint8_t count = 0; + if (nvs_get_u8(nvs, "mem_n_turns", &count) != ESP_OK) return; + if (count > 5) count = 5; + + char key[16]; + for (int i = 0; i < (int)count; i++) { + snprintf(key, sizeof(key), "mt%d", i); + char buf[600]; + size_t len = sizeof(buf); + if (nvs_get_str(nvs, key, buf, &len) != ESP_OK) continue; + + cJSON *j = cJSON_Parse(buf); + if (!j) continue; + cJSON *r = cJSON_GetObjectItem(j, "r"); + cJSON *t = cJSON_GetObjectItem(j, "t"); + cJSON *ts = cJSON_GetObjectItem(j, "s"); + if (r && t && cJSON_IsString(r) && cJSON_IsString(t)) { + strncpy(s_turns[s_turn_head].role, r->valuestring, + sizeof(s_turns[s_turn_head].role) - 1); + strncpy(s_turns[s_turn_head].text, t->valuestring, + sizeof(s_turns[s_turn_head].text) - 1); + s_turns[s_turn_head].timestamp_ms = ts ? (int64_t)ts->valuedouble : 0; + s_turn_head = (s_turn_head + 1) % MEMORY_MAX_TURNS; + s_turn_count++; + } + cJSON_Delete(j); + } + ESP_LOGI(TAG, "Loaded %d recent turns from NVS", s_turn_count); +} + +/* โ”€โ”€ HTTP helper โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +typedef struct { char *buf; int len; int cap; } http_resp_t; + +static esp_err_t http_on_data(esp_http_client_event_t *evt) +{ + http_resp_t *r = (http_resp_t *)evt->user_data; + if (evt->event_id == HTTP_EVENT_ON_DATA && r) { + int needed = r->len + evt->data_len + 1; + if (needed > r->cap) { + int newcap = needed + 512; + char *tmp = realloc(r->buf, newcap); + if (!tmp) return ESP_FAIL; + r->buf = tmp; + r->cap = newcap; + } + memcpy(r->buf + r->len, evt->data, evt->data_len); + r->len += evt->data_len; + r->buf[r->len] = '\0'; + } + return ESP_OK; +} + +/* Fire-and-forget HTTP POST with a JSON body; returns true on 2xx. */ +static bool http_post_json(const char *url, const char *json_body) +{ + esp_http_client_config_t cfg = { + .url = url, + .timeout_ms = 4000, + .method = HTTP_METHOD_POST, + .skip_cert_common_name_check = true, + }; + esp_http_client_handle_t cli = esp_http_client_init(&cfg); + if (!cli) return false; + esp_http_client_set_header(cli, "Content-Type", "application/json"); + esp_http_client_set_post_field(cli, json_body, (int)strlen(json_body)); + esp_err_t err = esp_http_client_perform(cli); + int status = esp_http_client_get_status_code(cli); + esp_http_client_cleanup(cli); + return (err == ESP_OK && status >= 200 && status < 300); +} + +/* HTTP GET returning heap-allocated body, or NULL. Caller must free(). */ +static char *http_get(const char *url, int max_bytes) +{ + if (max_bytes <= 0 || max_bytes > 8192) max_bytes = 2048; + http_resp_t resp = { .buf = malloc(512), .len = 0, .cap = 512 }; + if (!resp.buf) return NULL; + resp.buf[0] = '\0'; + + esp_http_client_config_t cfg = { + .url = url, + .timeout_ms = 5000, + .event_handler = http_on_data, + .user_data = &resp, + .skip_cert_common_name_check = true, + }; + esp_http_client_handle_t cli = esp_http_client_init(&cfg); + if (!cli) { free(resp.buf); return NULL; } + + esp_err_t err = esp_http_client_perform(cli); + int status = esp_http_client_get_status_code(cli); + esp_http_client_cleanup(cli); + + if (err != ESP_OK || status < 200 || status >= 300) { + free(resp.buf); + return NULL; + } + + /* Truncate to max_bytes */ + if (resp.len > max_bytes) { + resp.buf[max_bytes] = '\0'; + } + return resp.buf; +} + +/* Build HTTP companion URL from base (ws://... or http://...) + path */ +static void make_companion_url(char *out, size_t out_sz, const char *path) +{ + /* companion_url may be ws:// or http:// โ€” normalise to http:// */ + const char *base = g_debbie_config.companion_url; + if (strncmp(base, "ws://", 5) == 0) { + snprintf(out, out_sz, "http://%s%s", base + 5, path); + } else if (strncmp(base, "wss://", 6) == 0) { + snprintf(out, out_sz, "https://%s%s", base + 6, path); + } else { + snprintf(out, out_sz, "%s%s", base, path); + } +} + +/* โ”€โ”€ Public API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +esp_err_t memory_manager_init(void) +{ + memset(s_turns, 0, sizeof(s_turns)); + memset(s_facts, 0, sizeof(s_facts)); + s_turn_head = 0; + s_turn_count = 0; + s_fact_count = 0; + + if (!g_debbie_config.memory_enabled) { + ESP_LOGI(TAG, "Memory disabled โ€” skipping load"); + return ESP_OK; + } + + nvs_handle_t nvs; + esp_err_t err = nvs_open(NVS_MEM_NS, NVS_READONLY, &nvs); + if (err == ESP_ERR_NVS_NOT_FOUND) { + ESP_LOGI(TAG, "No saved memory โ€” starting fresh"); + return ESP_OK; + } + if (err != ESP_OK) return err; + + nvs_read_facts(nvs); + nvs_read_turns(nvs); + nvs_close(nvs); + + ESP_LOGI(TAG, "Memory initialised: %d facts, %d recent turns", + s_fact_count, s_turn_count); + return ESP_OK; +} + +esp_err_t memory_manager_save(void) +{ + if (!g_debbie_config.memory_enabled) return ESP_OK; + + nvs_handle_t nvs; + esp_err_t err = nvs_open(NVS_MEM_NS, NVS_READWRITE, &nvs); + if (err != ESP_OK) return err; + + nvs_write_facts(nvs); + nvs_write_recent_turns(nvs); + nvs_commit(nvs); + nvs_close(nvs); + return ESP_OK; +} + +esp_err_t memory_manager_clear(void) +{ + memset(s_turns, 0, sizeof(s_turns)); + memset(s_facts, 0, sizeof(s_facts)); + s_turn_head = 0; + s_turn_count = 0; + s_fact_count = 0; + + nvs_handle_t nvs; + esp_err_t err = nvs_open(NVS_MEM_NS, NVS_READWRITE, &nvs); + if (err != ESP_OK) return err; + nvs_erase_all(nvs); + nvs_commit(nvs); + nvs_close(nvs); + ESP_LOGI(TAG, "Memory cleared"); + return ESP_OK; +} + +void memory_manager_add_turn(const char *role, const char *text) +{ + if (!g_debbie_config.memory_enabled) return; + if (!role || !text || strlen(text) == 0) return; + + memory_turn_t *slot = &s_turns[s_turn_head]; + strncpy(slot->role, role, sizeof(slot->role) - 1); + strncpy(slot->text, text, sizeof(slot->text) - 1); + slot->text[sizeof(slot->text) - 1] = '\0'; + slot->timestamp_ms = (int64_t)(esp_timer_get_time() / 1000); + + s_turn_head = (s_turn_head + 1) % MEMORY_MAX_TURNS; + if (s_turn_count < MEMORY_MAX_TURNS) s_turn_count++; + + /* Persist asynchronously every 4 turns to avoid NVS wear */ + if (s_turn_count % 4 == 0) { + memory_manager_save(); + } +} + +const memory_turn_t *memory_manager_get_turns(void) { return s_turns; } +int memory_manager_turn_count(void) { return s_turn_count; } + +esp_err_t memory_manager_save_fact(const char *key, + const char *value, + uint8_t importance) +{ + if (!key || !value || strlen(key) == 0) return ESP_ERR_INVALID_ARG; + + /* Update existing fact if key matches */ + for (int i = 0; i < s_fact_count; i++) { + if (strcmp(s_facts[i].key, key) == 0) { + strncpy(s_facts[i].value, value, sizeof(s_facts[i].value) - 1); + s_facts[i].importance = importance; + s_facts[i].timestamp_ms = (int64_t)(esp_timer_get_time() / 1000); + ESP_LOGI(TAG, "Updated fact: %s = %s", key, value); + return memory_manager_save(); + } + } + + /* Add new fact */ + if (s_fact_count >= MEMORY_MAX_FACTS) { + /* Evict the oldest, least important fact */ + int evict = 0; + for (int i = 1; i < s_fact_count; i++) { + if (s_facts[i].importance < s_facts[evict].importance) + evict = i; + } + memmove(&s_facts[evict], &s_facts[evict + 1], + sizeof(memory_fact_t) * (s_fact_count - evict - 1)); + s_fact_count--; + } + + strncpy(s_facts[s_fact_count].key, key, sizeof(s_facts[s_fact_count].key) - 1); + strncpy(s_facts[s_fact_count].value, value, sizeof(s_facts[s_fact_count].value) - 1); + s_facts[s_fact_count].importance = importance; + s_facts[s_fact_count].timestamp_ms = (int64_t)(esp_timer_get_time() / 1000); + s_fact_count++; + + ESP_LOGI(TAG, "Saved fact [%d]: %s = %s", s_fact_count - 1, key, value); + return memory_manager_save(); +} + +const memory_fact_t *memory_manager_get_facts(int *count_out) +{ + if (count_out) *count_out = s_fact_count; + return s_facts; +} + +char *memory_manager_build_context(void) +{ + if (!g_debbie_config.memory_enabled) return NULL; + if (s_turn_count == 0 && s_fact_count == 0) return NULL; + + char *ctx = malloc(MEMORY_CONTEXT_MAX); + if (!ctx) return NULL; + ctx[0] = '\0'; + int pos = 0; + +#define CTX_APPEND(fmt, ...) \ + do { \ + int _n = snprintf(ctx + pos, MEMORY_CONTEXT_MAX - pos, fmt, ##__VA_ARGS__); \ + if (_n > 0) pos += _n; \ + } while (0) + + CTX_APPEND("\n\n--- MEMORY ---\n"); + + /* Long-term facts */ + if (s_fact_count > 0) { + CTX_APPEND("Known facts:"); + for (int i = 0; i < s_fact_count && pos < MEMORY_CONTEXT_MAX - 64; i++) { + CTX_APPEND(" %s=%s;", s_facts[i].key, s_facts[i].value); + } + CTX_APPEND("\n"); + } + + /* Recent conversation โ€” last min(s_turn_count, 10) turns */ + if (s_turn_count > 0) { + int show = s_turn_count < 10 ? s_turn_count : 10; + CTX_APPEND("Recent conversation:\n"); + for (int i = 0; i < show && pos < MEMORY_CONTEXT_MAX - 128; i++) { + int idx = ((s_turn_head - show + i) + MEMORY_MAX_TURNS) % MEMORY_MAX_TURNS; + CTX_APPEND(" %s: %s\n", s_turns[idx].role, s_turns[idx].text); + } + } + + CTX_APPEND("--- END MEMORY ---\n"); + return ctx; +} + +char *memory_manager_enrich_prompt(const char *base_prompt) +{ + char *ctx = memory_manager_build_context(); + if (!ctx) { + /* No memory โ€” just duplicate the base prompt */ + return base_prompt ? strdup(base_prompt) : NULL; + } + + size_t base_len = base_prompt ? strlen(base_prompt) : 0; + size_t ctx_len = strlen(ctx); + char *out = malloc(base_len + ctx_len + 4); + if (!out) { free(ctx); return NULL; } + + if (base_prompt) { + memcpy(out, base_prompt, base_len); + out[base_len] = '\0'; + } else { + out[0] = '\0'; + } + strncat(out, ctx, ctx_len); + free(ctx); + return out; +} + +char *memory_manager_query_rag(const char *query) +{ + if (!g_debbie_config.memory_enabled || + !g_debbie_config.memory_rag_enabled) return NULL; + if (!query || strlen(query) == 0) return NULL; + if (strlen(g_debbie_config.companion_url) == 0) return NULL; + + /* Build URL: http:///memory/query?q=&limit=5 */ + char url[512]; + char companion_http[256]; + make_companion_url(companion_http, sizeof(companion_http), ""); + /* Strip trailing slash from base */ + int base_len = (int)strlen(companion_http); + while (base_len > 0 && companion_http[base_len - 1] == '/') { + companion_http[--base_len] = '\0'; + } + + /* Simple percent-encoding of the query (spaces โ†’ %20) */ + char encoded_query[256] = {0}; + int qi = 0, ei = 0; + while (query[qi] && ei < (int)sizeof(encoded_query) - 4) { + char c = query[qi++]; + if (c == ' ') { + encoded_query[ei++] = '%'; + encoded_query[ei++] = '2'; + encoded_query[ei++] = '0'; + } else if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || c == '-' || c == '_' || + c == '.' || c == '~') { + encoded_query[ei++] = c; + } else { + ei += snprintf(encoded_query + ei, sizeof(encoded_query) - ei, + "%%%02X", (unsigned char)c); + } + } + + snprintf(url, sizeof(url), "%s/memory/query?q=%s&limit=5", + companion_http, encoded_query); + + char *body = http_get(url, 2048); + if (!body) return NULL; + + /* The companion returns JSON: {"memories":[{"text":"..."},...]} + * Extract into a plain text summary */ + cJSON *json = cJSON_Parse(body); + free(body); + if (!json) return NULL; + + cJSON *mems = cJSON_GetObjectItem(json, "memories"); + if (!mems || !cJSON_IsArray(mems) || cJSON_GetArraySize(mems) == 0) { + cJSON_Delete(json); + return NULL; + } + + char *summary = malloc(1024); + if (!summary) { cJSON_Delete(json); return NULL; } + summary[0] = '\0'; + strncat(summary, "Relevant memories: ", 1023); + + int n = cJSON_GetArraySize(mems); + for (int i = 0; i < n; i++) { + cJSON *m = cJSON_GetArrayItem(mems, i); + cJSON *text = cJSON_GetObjectItem(m, "text"); + if (text && cJSON_IsString(text)) { + int rem = (int)(1023 - strlen(summary)); + if (rem > 4) { + strncat(summary, text->valuestring, rem - 2); + if (i < n - 1) strncat(summary, ". ", 2); + } + } + } + cJSON_Delete(json); + return summary; +} + +void memory_manager_sync_turn(const char *role, const char *text) +{ + if (!g_debbie_config.memory_enabled || + !g_debbie_config.memory_rag_enabled) return; + if (strlen(g_debbie_config.companion_url) == 0) return; + if (!role || !text || strlen(text) == 0) return; + + /* Build JSON body */ + cJSON *j = cJSON_CreateObject(); + cJSON_AddStringToObject(j, "role", role); + cJSON_AddStringToObject(j, "text", text); + cJSON_AddNumberToObject(j, "ts", + (double)(esp_timer_get_time() / 1000)); + char *body = cJSON_PrintUnformatted(j); + cJSON_Delete(j); + if (!body) return; + + char url[256]; + make_companion_url(url, sizeof(url), "/memory/turn"); + + /* Best-effort: ignore errors โ€” we don't want to block audio pipeline */ + http_post_json(url, body); + free(body); +} diff --git a/firmware/main/memory_manager.h b/firmware/main/memory_manager.h new file mode 100644 index 0000000..5d69d8f --- /dev/null +++ b/firmware/main/memory_manager.h @@ -0,0 +1,140 @@ +#pragma once +/* + * memory_manager.h โ€” Debbie's short-term & long-term memory + * + * Short-term memory: a circular buffer of the last MEMORY_MAX_TURNS + * conversation turns (user + assistant) kept in RAM. + * + * Long-term memory: up to MEMORY_MAX_FACTS key-value "facts" (e.g. + * "user_name=Alice", "user_likes=jazz") persisted to NVS flash. + * + * RAG: on request, queries the companion server's /memory/query endpoint + * and appends the retrieved context to the system prompt. + */ + +#include "esp_err.h" +#include +#include + +/* โ”€โ”€ Limits โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define MEMORY_MAX_TURNS 20 /* recent turns kept in RAM */ +#define MEMORY_MAX_FACTS 50 /* long-term facts in NVS */ +#define MEMORY_FACT_KEY_MAX 48 /* max chars for a fact key */ +#define MEMORY_FACT_VAL_MAX 192 /* max chars for a fact value */ +#define MEMORY_TURN_TEXT_MAX 512 /* max chars per turn */ +#define MEMORY_CONTEXT_MAX 4096 /* max chars in built context blob */ + +/* โ”€โ”€ Types โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +typedef struct { + char role[12]; /* "user" or "assistant" */ + char text[MEMORY_TURN_TEXT_MAX]; + int64_t timestamp_ms; +} memory_turn_t; + +typedef struct { + char key[MEMORY_FACT_KEY_MAX]; + char value[MEMORY_FACT_VAL_MAX]; + int64_t timestamp_ms; + uint8_t importance; /* 0-10 */ +} memory_fact_t; + +/* โ”€โ”€ Lifecycle โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/** + * @brief Initialise memory manager. + * Loads saved facts (and last-N turns) from NVS. + * Call once after storage_init(). + */ +esp_err_t memory_manager_init(void); + +/** + * @brief Persist current facts (and last 5 turns) to NVS. + * Called automatically after each turn; call explicitly on shutdown. + */ +esp_err_t memory_manager_save(void); + +/** + * @brief Erase all in-RAM and NVS memory. + */ +esp_err_t memory_manager_clear(void); + +/* โ”€โ”€ Short-term memory โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/** + * @brief Append a conversation turn to the in-RAM circular buffer. + * + * @param role "user" or "assistant" + * @param text Transcribed text + */ +void memory_manager_add_turn(const char *role, const char *text); + +/** + * @brief Return a pointer to the turn circular buffer (read-only). + * Use memory_manager_turn_count() to know how many are valid. + */ +const memory_turn_t *memory_manager_get_turns(void); + +/** @brief Number of turns currently in the buffer (0 โ€“ MEMORY_MAX_TURNS). */ +int memory_manager_turn_count(void); + +/* โ”€โ”€ Long-term facts โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/** + * @brief Store / update a long-term fact in NVS. + * + * @param key Identifier (e.g. "user_name", "user_location") + * @param value Value string + * @param importance 0-10 (higher = kept longer / shown first) + */ +esp_err_t memory_manager_save_fact(const char *key, + const char *value, + uint8_t importance); + +/** + * @brief Return all stored facts and their count. + */ +const memory_fact_t *memory_manager_get_facts(int *count_out); + +/* โ”€โ”€ Context building (for system-prompt injection) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/** + * @brief Build a memory context string to prepend to the system prompt. + * + * Format: + * [Memory] Facts: user_name=Alice; user_location=London + * [Memory] Recent conversation: + * user: what's the weather like? + * assistant: It's sunny in London today. + * + * @return Heap-allocated string โ€” caller must free(). + * Returns NULL on allocation failure. + */ +char *memory_manager_build_context(void); + +/** + * @brief Build an enriched system prompt: base_prompt + memory context. + * Returns a heap-allocated string โ€” caller must free(). + */ +char *memory_manager_enrich_prompt(const char *base_prompt); + +/* โ”€โ”€ Companion RAG โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +/** + * @brief Query the companion server for memories relevant to `query`. + * Returns NULL if companion is not reachable or not configured. + * Caller must free() the returned string (it contains a brief + * plain-text summary of retrieved memories). + * + * @param query The user's latest utterance. + */ +char *memory_manager_query_rag(const char *query); + +/** + * @brief Sync a new turn to the companion server's persistent store. + * Non-blocking โ€” fires-and-forgets using a short HTTP POST. + * + * @param role "user" or "assistant" + * @param text Turn text + */ +void memory_manager_sync_turn(const char *role, const char *text); diff --git a/firmware/main/openai_client.c b/firmware/main/openai_client.c index fb8b538..8207a24 100644 --- a/firmware/main/openai_client.c +++ b/firmware/main/openai_client.c @@ -121,6 +121,42 @@ static void send_session_update(const char *system_prompt) cJSON_AddItemToObject(t_spot, "parameters", t_spot_params); cJSON_AddItemToArray(tools, t_spot); + cJSON *t_mem = cJSON_CreateObject(); + cJSON_AddStringToObject(t_mem, "type", "function"); + cJSON_AddStringToObject(t_mem, "name", "save_memory"); + cJSON_AddStringToObject(t_mem, "description", + "Save an important fact or piece of information to long-term memory " + "so you can recall it in future conversations. Use when the user shares " + "personal details, preferences, or important context (e.g. their name, " + "location, interests, schedule, or preferences)."); + { + cJSON *params = cJSON_CreateObject(); + cJSON *props = cJSON_CreateObject(); + cJSON *p_key = cJSON_CreateObject(); + cJSON *p_val = cJSON_CreateObject(); + cJSON *p_imp = cJSON_CreateObject(); + cJSON_AddStringToObject(p_key, "type", "string"); + cJSON_AddStringToObject(p_key, "description", + "Short snake_case identifier for this fact, e.g. user_name, user_location, " + "favourite_colour"); + cJSON_AddStringToObject(p_val, "type", "string"); + cJSON_AddStringToObject(p_val, "description", "The value to store"); + cJSON_AddStringToObject(p_imp, "type", "integer"); + cJSON_AddStringToObject(p_imp, "description", + "Importance score 0-10 (10 = critical, 5 = normal)"); + cJSON_AddItemToObject(props, "key", p_key); + cJSON_AddItemToObject(props, "value", p_val); + cJSON_AddItemToObject(props, "importance", p_imp); + cJSON_AddItemToObject(params, "properties", props); + cJSON *req_arr = cJSON_CreateArray(); + cJSON_AddItemToArray(req_arr, cJSON_CreateString("key")); + cJSON_AddItemToArray(req_arr, cJSON_CreateString("value")); + cJSON_AddItemToObject(params, "required", req_arr); + cJSON_AddStringToObject(params, "type", "object"); + cJSON_AddItemToObject(t_mem, "parameters", params); + } + cJSON_AddItemToArray(tools, t_mem); + cJSON_AddStringToObject(root, "type", "session.update"); cJSON_AddStringToObject(sess, "modalities", "audio"); cJSON_AddStringToObject(sess, "voice", "alloy"); @@ -201,8 +237,7 @@ static void ws_event_handler(void *handler_args, esp_event_base_t base, } } } - else if (strcmp(type, "response.audio_transcript.delta") == 0 || - strcmp(type, "conversation.item.input_audio_transcription.completed") == 0) { + else if (strcmp(type, "response.audio_transcript.delta") == 0) { cJSON *text_j = cJSON_GetObjectItem(json, "transcript"); if (!text_j) text_j = cJSON_GetObjectItem(json, "delta"); if (text_j && cJSON_IsString(text_j)) { @@ -213,6 +248,16 @@ static void ws_event_handler(void *handler_args, esp_event_base_t base, if (s_cb) s_cb(&evt, s_ctx); } } + else if (strcmp(type, "conversation.item.input_audio_transcription.completed") == 0) { + cJSON *text_j = cJSON_GetObjectItem(json, "transcript"); + if (text_j && cJSON_IsString(text_j)) { + oai_event_data_t evt = { + .type = OAI_EVT_USER_TRANSCRIPT, + .transcript.text = text_j->valuestring, + }; + if (s_cb) s_cb(&evt, s_ctx); + } + } else if (strcmp(type, "response.function_call_arguments.done") == 0) { cJSON *name_j = cJSON_GetObjectItem(json, "name"); cJSON *args_j = cJSON_GetObjectItem(json, "arguments"); diff --git a/firmware/main/openai_client.h b/firmware/main/openai_client.h index b7f48ad..2c3fc1a 100644 --- a/firmware/main/openai_client.h +++ b/firmware/main/openai_client.h @@ -15,9 +15,10 @@ typedef enum { OAI_EVT_CONNECTED = 0, OAI_EVT_DISCONNECTED, - OAI_EVT_AUDIO_DELTA, /* AI speaking โ€” PCM chunk ready */ - OAI_EVT_TRANSCRIPT, /* text transcript available */ - OAI_EVT_FUNCTION_CALL, /* function/tool call requested by model */ + OAI_EVT_AUDIO_DELTA, /* AI speaking โ€” PCM chunk ready */ + OAI_EVT_TRANSCRIPT, /* AI text transcript available */ + OAI_EVT_USER_TRANSCRIPT, /* User speech transcript (for memory) */ + OAI_EVT_FUNCTION_CALL, /* function/tool call requested by model */ OAI_EVT_SESSION_CREATED, OAI_EVT_ERROR, } oai_event_t; diff --git a/firmware/main/settings.h b/firmware/main/settings.h index a94fa70..6eec855 100644 --- a/firmware/main/settings.h +++ b/firmware/main/settings.h @@ -125,6 +125,11 @@ #define DEFAULT_RESPONSE_LENGTH "normal" #define DEFAULT_VAD_THRESHOLD 300 +/* โ”€โ”€ Memory & RAG defaults โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +#define MEMORY_ENABLED_DEFAULT true +#define MEMORY_RAG_ENABLED_DEFAULT true +#define MEMORY_MAX_TURNS_DEFAULT 20 + /* โ”€โ”€ Spotify (handled by companion server) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ /* The device sends commands to the companion server which calls Spotify API */ diff --git a/firmware/main/storage_manager.c b/firmware/main/storage_manager.c index 99be4c1..7a1b9c6 100644 --- a/firmware/main/storage_manager.c +++ b/firmware/main/storage_manager.c @@ -49,6 +49,9 @@ esp_err_t storage_init(void) sizeof(g_debbie_config.local_llm_url) - 1); strncpy(g_debbie_config.local_llm_model, LOCAL_LLM_DEFAULT_MODEL, sizeof(g_debbie_config.local_llm_model) - 1); + g_debbie_config.memory_enabled = MEMORY_ENABLED_DEFAULT; + g_debbie_config.memory_rag_enabled = MEMORY_RAG_ENABLED_DEFAULT; + g_debbie_config.memory_max_turns = MEMORY_MAX_TURNS_DEFAULT; strncpy(g_debbie_config.voice_style, DEFAULT_VOICE_STYLE, sizeof(g_debbie_config.voice_style) - 1); strncpy(g_debbie_config.response_length, DEFAULT_RESPONSE_LENGTH, @@ -109,6 +112,13 @@ esp_err_t storage_init(void) if (nvs_get_u8(nvs, "bt_en", &tmp) == ESP_OK) g_debbie_config.bluetooth_enabled = (tmp != 0); + if (nvs_get_u8(nvs, "mem_en", &tmp) == ESP_OK) + g_debbie_config.memory_enabled = (tmp != 0); + if (nvs_get_u8(nvs, "mem_rag", &tmp) == ESP_OK) + g_debbie_config.memory_rag_enabled = (tmp != 0); + if (nvs_get_u8(nvs, "mem_turns", &tmp) == ESP_OK) + g_debbie_config.memory_max_turns = tmp ? tmp : MEMORY_MAX_TURNS_DEFAULT; + uint16_t vad; if (nvs_get_u16(nvs, "vad_thresh", &vad) == ESP_OK) g_debbie_config.vad_threshold = vad; @@ -156,6 +166,9 @@ esp_err_t storage_save_config(void) NVS_SET_U8("notif_sp", g_debbie_config.notif_spotify ? 1 : 0); NVS_SET_U8("cam_en", g_debbie_config.camera_enabled ? 1 : 0); NVS_SET_U8("bt_en", g_debbie_config.bluetooth_enabled ? 1 : 0); + NVS_SET_U8("mem_en", g_debbie_config.memory_enabled ? 1 : 0); + NVS_SET_U8("mem_rag", g_debbie_config.memory_rag_enabled ? 1 : 0); + NVS_SET_U8("mem_turns", g_debbie_config.memory_max_turns); nvs_set_u16(nvs, "vad_thresh", g_debbie_config.vad_threshold); esp_err_t err = nvs_commit(nvs); diff --git a/firmware/main/web_server.c b/firmware/main/web_server.c index 351f660..d102b6e 100644 --- a/firmware/main/web_server.c +++ b/firmware/main/web_server.c @@ -4,6 +4,7 @@ #include "storage_manager.h" #include "wifi_manager.h" #include "camera_manager.h" +#include "memory_manager.h" #include "esp_log.h" #include "esp_http_server.h" #include "cJSON.h" @@ -107,6 +108,7 @@ static const char SETUP_HTML[] = "
๐ŸŽ€ Personality
" "
๐Ÿ”” Notifications
" "
โš™๏ธ Advanced
" +"
๐Ÿง  Memory
" "" /* โ”€โ”€ Page 0: Network โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ @@ -364,6 +366,52 @@ static const char SETUP_HTML[] = "" "" +/* โ”€โ”€ Page 5: Memory & RAG โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ +"
" + +"
" +"
๐Ÿง  Conversation Memory
" +"
" +"
Enable Memory
" +"
Remember recent conversations and facts across sessions
" +"

" +"
" +"
Companion RAG
" +"
Query companion server for richer context retrieval
" +"

" +"" +"" +"

Stored in NVS flash; survives reboots. Companion server stores unlimited history.

" +"
" + +"
" +"
๐Ÿ“Š Memory Stats
" +"
Loading...
" +"
" +"
" + +"
" +"
๐Ÿ’ก How Debbie's Memory Works
" +"

" +"Short-term: Last 20 turns (user + AI) kept in RAM " +"and injected into every new conversation as context, so Debbie always knows what " +"was just discussed.

" +"Long-term facts: You can say 'Remember that my name is Alice' " +"or 'Remember I have a cat named Pixel' and Debbie will save it to NVS flash " +"โ€” it persists across reboots.

" +"Companion RAG: When a companion server is configured, " +"all conversation history is stored there in a searchable SQLite database. When you ask " +"something, relevant past conversations are retrieved and added to the AI context " +"โ€” making Debbie smarter over time.

" +"Voice commands: 'Remember that...' / " +"'What do you know about me?' / 'Forget everything'" +"

" +"
" +"
" + /* Sticky save bar */ "
" "" @@ -397,6 +445,7 @@ static const char SETUP_HTML[] = " tabs.forEach(function(t,j){t.className='tab'+(i===j?' active':'');});" " pages.forEach(function(p,j){p.className='page'+(i===j?' active':'');});" " if(i===4)loadStatus();" +" if(i===5)loadMemStats();" "}" /* โ”€โ”€ Helpers โ”€โ”€โ”€ */ @@ -439,7 +488,10 @@ static const char SETUP_HTML[] = " notif_em:gc('notif_em'),notif_sp:gc('notif_sp')," " volume:parseInt(g('volume'))||75," " vad_threshold:parseInt(g('vad_thresh'))||300," -" cam_en:gc('cam_en')" +" cam_en:gc('cam_en')," +" mem_en:gc('mem_en')," +" mem_rag:gc('mem_rag')," +" mem_turns:parseInt(g('mem_turns'))||20" " };" " fetch('/configure',{method:'POST'," " headers:{'Content-Type':'application/json'},body:JSON.stringify(d)})" @@ -508,8 +560,33 @@ static const char SETUP_HTML[] = " if(j.volume!=null)s('volume',j.volume);" " if(j.vad_threshold!=null)s('vad_thresh',j.vad_threshold);" " sc('cam_en',j.cam_en);" +" sc('mem_en',j.mem_en);" +" sc('mem_rag',j.mem_rag);" +" if(j.mem_turns!=null)s('mem_turns',j.mem_turns);" " }).catch(function(){});" "}" + +/* โ”€โ”€ Memory stats & clear โ”€โ”€โ”€ */ +"function loadMemStats(){" +" var sb=document.getElementById('mem_stats');" +" if(sb)sb.innerHTML='Loading...';" +" fetch('/memory_stats').then(function(r){return r.json();})" +" .then(function(j){" +" if(!sb)return;" +" sb.innerHTML=" +" '๐Ÿ’ฌ Turns: '+j.turn_count+''" +" +'๐Ÿ“Œ Facts: '+j.fact_count+''" +" +'" +" ๐Ÿ” Companion RAG: '+(j.companion_rag?'Active':'Off')+'';" +" }).catch(function(){if(sb)sb.innerHTML='Stats unavailable';});" +"}" +"function clearMem(){" +" if(!confirm('Clear all Debbie memory? This cannot be undone.'))return;" +" fetch('/memory_clear',{method:'POST'})" +" .then(function(){showMsg('ok','Memory cleared!');loadMemStats();})" +" .catch(function(e){showMsg('err',''+e);});" +"}" + "loadStatus();" ""; @@ -549,6 +626,9 @@ static esp_err_t handler_status(httpd_req_t *req) cJSON_AddBoolToObject(json, "notif_em", g_debbie_config.notif_email); cJSON_AddBoolToObject(json, "notif_sp", g_debbie_config.notif_spotify); cJSON_AddBoolToObject(json, "cam_en", g_debbie_config.camera_enabled); + cJSON_AddBoolToObject(json, "mem_en", g_debbie_config.memory_enabled); + cJSON_AddBoolToObject(json, "mem_rag", g_debbie_config.memory_rag_enabled); + cJSON_AddNumberToObject(json, "mem_turns", g_debbie_config.memory_max_turns); cJSON_AddStringToObject(json, "state", g_debbie_state == DEBBIE_STATE_IDLE ? "idle" : g_debbie_state == DEBBIE_STATE_LISTENING ? "listening" : @@ -633,6 +713,15 @@ static esp_err_t handler_configure(httpd_req_t *req) GET_BOOL("cam_en", g_debbie_config.camera_enabled); + /* Memory */ + GET_BOOL("mem_en", g_debbie_config.memory_enabled); + GET_BOOL("mem_rag", g_debbie_config.memory_rag_enabled); + { + cJSON *mt = cJSON_GetObjectItem(json, "mem_turns"); + if (mt && cJSON_IsNumber(mt) && mt->valuedouble >= 5 && mt->valuedouble <= 50) + g_debbie_config.memory_max_turns = (uint8_t)mt->valuedouble; + } + cJSON_Delete(json); storage_save_config(); @@ -674,6 +763,34 @@ static esp_err_t handler_snapshot(httpd_req_t *req) return ESP_OK; } +static esp_err_t handler_memory_stats(httpd_req_t *req) +{ + int fact_count = 0; + memory_manager_get_facts(&fact_count); + cJSON *json = cJSON_CreateObject(); + cJSON_AddNumberToObject(json, "turn_count", memory_manager_turn_count()); + cJSON_AddNumberToObject(json, "fact_count", fact_count); + cJSON_AddBoolToObject(json, "memory_enabled", g_debbie_config.memory_enabled); + cJSON_AddBoolToObject(json, "companion_rag", + g_debbie_config.memory_rag_enabled && + strlen(g_debbie_config.companion_url) > 0); + char *str = cJSON_PrintUnformatted(json); + cJSON_Delete(json); + httpd_resp_set_type(req, "application/json"); + httpd_resp_send(req, str, strlen(str)); + free(str); + return ESP_OK; +} + +static esp_err_t handler_memory_clear(httpd_req_t *req) +{ + memory_manager_clear(); + const char *resp = "{\"ok\":true,\"msg\":\"Memory cleared\"}"; + httpd_resp_set_type(req, "application/json"); + httpd_resp_send(req, resp, strlen(resp)); + return ESP_OK; +} + /* -------------------------------------------------------------------------- */ static httpd_handle_t s_server = NULL; @@ -681,17 +798,19 @@ static httpd_handle_t s_server = NULL; esp_err_t web_server_start(void) { httpd_config_t cfg = HTTPD_DEFAULT_CONFIG(); - cfg.max_uri_handlers = 8; + cfg.max_uri_handlers = 10; cfg.stack_size = 12288; /* increased for BLE + WiFi coexistence overhead */ ESP_ERROR_CHECK(httpd_start(&s_server, &cfg)); const httpd_uri_t routes[] = { - { .uri = "/", .method = HTTP_GET, .handler = handler_root }, - { .uri = "/status", .method = HTTP_GET, .handler = handler_status }, - { .uri = "/configure", .method = HTTP_POST, .handler = handler_configure }, - { .uri = "/reset", .method = HTTP_POST, .handler = handler_reset }, - { .uri = "/snapshot", .method = HTTP_GET, .handler = handler_snapshot }, + { .uri = "/", .method = HTTP_GET, .handler = handler_root }, + { .uri = "/status", .method = HTTP_GET, .handler = handler_status }, + { .uri = "/configure", .method = HTTP_POST, .handler = handler_configure }, + { .uri = "/reset", .method = HTTP_POST, .handler = handler_reset }, + { .uri = "/snapshot", .method = HTTP_GET, .handler = handler_snapshot }, + { .uri = "/memory_stats", .method = HTTP_GET, .handler = handler_memory_stats }, + { .uri = "/memory_clear", .method = HTTP_POST, .handler = handler_memory_clear }, }; for (int i = 0; i < (int)(sizeof(routes) / sizeof(routes[0])); i++) { From fac8c55862498b9056e7dfa26791dc6431ff0dfc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 17:17:18 +0000 Subject: [PATCH 8/8] Fix duplicate CMakeLists.txt entry and improve rate limiter config Agent-Logs-Url: https://github.com/magicalmutation-coder/DebbieDoesMobile/sessions/579a96ff-263f-4aae-b142-0e518af43f49 Co-authored-by: magicalmutation-coder <246345154+magicalmutation-coder@users.noreply.github.com> --- companion-server/memory_store.js | 10 +++++++- firmware/main/CMakeLists.txt | 39 -------------------------------- 2 files changed, 9 insertions(+), 40 deletions(-) diff --git a/companion-server/memory_store.js b/companion-server/memory_store.js index a2571ba..36637a9 100644 --- a/companion-server/memory_store.js +++ b/companion-server/memory_store.js @@ -246,7 +246,15 @@ function clearAll() { /* โ”€โ”€ Express route handler โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ -const memLimiter = rateLimit({ windowMs: 60000, max: 120 }); +const memLimiter = rateLimit({ + windowMs: 60000, + max: 120, + standardHeaders: true, + legacyHeaders: false, + handler: (req, res) => { + res.status(429).json({ error: 'Too many memory requests, please slow down.' }); + }, +}); function addMemoryRoutes(app) { /* Lazy-initialise the DB */ diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt index edc39c5..5e49b62 100644 --- a/firmware/main/CMakeLists.txt +++ b/firmware/main/CMakeLists.txt @@ -38,42 +38,3 @@ idf_component_register( ping bt ) - - SRCS - "main.c" - "storage_manager.c" - "wifi_manager.c" - "bluetooth_manager.c" - "audio_manager.c" - "camera_manager.c" - "display_manager.c" - "openai_client.c" - "notification_client.c" - "web_server.c" - "network_scanner.c" - "vuln_reporter.c" - "self_agent.c" - INCLUDE_DIRS - "." - REQUIRES - nvs_flash - esp_wifi - esp_event - esp_netif - esp_http_client - esp_https_ota - esp_websocket_client - esp_http_server - esp_camera - esp_lcd - driver - lvgl - cJSON - mbedtls - esp_timer - freertos - log - lwip - ping - bt -)