From 11d72f1e752351a67a879ccfe1ebcb8dee53b0e8 Mon Sep 17 00:00:00 2001 From: pasketti Date: Sun, 19 Apr 2026 02:40:41 -0400 Subject: [PATCH] =?UTF-8?q?Initial=20release=20=E2=80=94=20Re-Commander=20?= =?UTF-8?q?v1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local web-based control interface for the Jibo social robot via the ROM WebSocket API (port 8160) and on-device ASR (port 8088). Features head navigation via click-to-look and arrow keys, speech/listen/Voice-AI loop, display control, camera/photo capture, and entity tracking — no cloud dependency required. Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 11 + .gitignore | 4 + README.md | 226 +++++++++++ package-lock.json | 861 ++++++++++++++++++++++++++++++++++++++++ package.json | 14 + public/app.js | 579 +++++++++++++++++++++++++++ public/index.html | 592 ++++++++++++++++++++++++++++ public/style.css | 376 ++++++++++++++++++ server.js | 973 ++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 3636 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 public/app.js create mode 100644 public/index.html create mode 100644 public/style.css create mode 100644 server.js diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..32a1397 --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +# OpenAI-compatible completions endpoint (e.g. Ollama, LM Studio, OpenAI) +LLM_ENDPOINT=http://localhost:11434/v1/chat/completions + +# Model name passed to the endpoint +LLM_MODEL=llama3 + +# Optional API key (sent as Bearer token) — leave blank for local servers +LLM_API_KEY= + +# Default system prompt for the voice AI loop +LLM_SYSTEM_PROMPT=You are Jibo, a friendly social robot. Keep responses brief and conversational. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50c6113 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.env +photos/ +.claude/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..26c1a88 --- /dev/null +++ b/README.md @@ -0,0 +1,226 @@ +# Re-Commander + +A local web-based control interface for the Jibo social robot. Re-Commander connects directly to Jibo's on-device ROM (Robot Operating Module) over your local network, giving you a browser UI to control head movement, speech, listening, display, camera, and an LLM voice-AI loop — all without any cloud dependency. + +![Re-Commander UI](docs/screenshot.png) + +--- + +## Requirements + +- **Node.js** 18 or later +- **Jibo robot** on the same local network, running in **`int-developer` mode** (see below) +- A modern browser (Chrome, Firefox, Edge) + +--- + +## Robot prerequisite — `int-developer` mode + +> **This is required. Re-Commander will not work without it.** + +Jibo must be placed into `int-developer` mode before the server can open a WebSocket session with it. In this mode the robot exposes its ROM WebSocket API on port 8160 and its local ASR service on port 8088. + +To enable it: + +1. On your phone, open the Jibo app and connect to your robot. +2. Navigate to **Settings → Developer Options** and enable **Developer Mode**. +3. On the robot itself (or via the app), switch the operating mode to **`int-developer`**. +4. The robot will reboot. Once it is back up and on Wi-Fi, its ROM API will be accessible at `:8160`. + +If you are unsure whether your robot is in the right mode, try `curl http://:8160/request` — if you get a JSON response (even an error), the port is open. + +--- + +## Installation + +```bash +git clone https://github.com/youruser/re-commander.git +cd re-commander +npm install +``` + +--- + +## Configuration + +### 1. Robot IP address + +Open `server.js` and set `JIBO_HOST` to your robot's local IP: + +```js +const JIBO_HOST = '192.168.1.217'; // ← change this +const JIBO_PORT = 8160; // leave as-is +``` + +Find your robot's IP in your router's device list, or check the Jibo app under **Settings → Wi-Fi**. + +### 2. Environment variables (optional) + +Create a `.env` file in the project root to configure the LLM integration and server port: + +```env +# Port the web UI is served on (default: 3000) +PORT=3000 + +# OpenAI-compatible LLM endpoint (default: local Ollama) +LLM_ENDPOINT=http://localhost:11434/v1/chat/completions + +# Model name passed to the endpoint +LLM_MODEL=llama3 + +# API key — set if your endpoint requires one (e.g. OpenAI, Anthropic proxy) +LLM_API_KEY=sk-... +``` + +If no `.env` is present the server defaults to port 3000 and a local Ollama instance. The LLM system prompt is embedded in `server.js` and pre-configured to make Jibo respond in ESML (Embodied Speech Markup Language) with animations and sound effects. + +--- + +## Running + +```bash +npm start +``` + +Then open `http://localhost:3000` in your browser. + +The server immediately begins connecting to the robot. The status indicator in the top-left of the UI turns green once a session is established. If the robot is unreachable the server retries every 3 seconds automatically. + +--- + +## UI overview + +The interface is divided into three panels. + +### Left panel — Controls + +**Head Navigation** +- Arrow pad (↑ ↓ ← →): moves Jibo's head. Each press sends one step command and waits for the robot to acknowledge before sending the next, so there is no command queuing or drift. +- Clicking anywhere on the camera feed makes Jibo look at that point in the scene. +- The **Track** checkbox keeps the robot tracking that point continuously. + +**Say** +- Type any text (plain or ESML markup) and press **▶ Say**. +- **✕ Stop** cancels mid-speech. + +**Listen** +- Triggers Jibo's on-device ASR (port 8088, no cloud required). +- Configure the no-speech and max-speech timeouts before pressing **🎙 Listen**. +- The transcribed result appears below the buttons. +- **Auto-listen on hotword**: when checked, Re-Commander listens automatically every time the wake word ("Hey Jibo") is detected. + +**Attention Mode** +- Sets Jibo's attention state: `OFF`, `IDLE`, `DISENGAGE`, `ENGAGED`, `SPEAKING`, `FIXATED`, `ATTRACTABLE`, `COMMAND`. + +**Volume** +- Drag the slider and press **Set Volume** (0 – 100%). + +**Voice AI** +- Enables a continuous listen → LLM → speak loop. +- The LLM receives the transcribed speech and returns ESML; Jibo speaks the reply with matching animations. +- Configure the endpoint, model, and system prompt in `.env` or via the UI fields. + +### Center panel — Camera & photos + +- The camera feed displays the live MJPEG stream from Jibo when video is active, or the most recent photo otherwise. +- Click anywhere on the feed to make Jibo look at that point. +- The **Photo Strip** below the feed shows all photos taken this session. Click any thumbnail to open it full-size. + +### Right panel — Tabs + +**Camera tab** +- **Start / Stop Video**: starts or stops the live MJPEG stream from Jibo's right camera. +- **Take Photo**: captures a still from the selected camera at the selected resolution. Photos are automatically saved to the `photos/` directory in the project folder and served from `/photos/`. +- **Subscriptions**: toggle Entity (person detection), Motion, and Head Touch event streams on/off. + +**Display tab** +- **Show Eye**: displays Jibo's animated eye graphic on his screen. +- **Play Animation**: select and play any of Jibo's built-in eye animations (blinks, expressions, emoji, transitions, and more) from a curated dropdown. +- **Show Text / Show Image**: display a text string or a remote image URL on Jibo's screen. + +**Entities tab** +- Live list of people Jibo's vision system has detected, with entity ID, confidence, and screen coordinates. +- Head Touch display shows which pads on Jibo's head are currently being touched. + +**Log tab** +- Real-time event log of every message received from the robot (LookAt events, touch events, ASR results, errors, etc.). + +--- + +## Keyboard shortcuts + +| Key | Action | +|-----|--------| +| `↑` `↓` `←` `→` | Move Jibo's head | +| `Space` | Center Jibo's head | + +Arrow keys are ignored when a text input is focused. + +--- + +## Photos + +Every photo taken is saved to `/photos/` as `photo_.jpg`. The directory is created automatically on startup. Photos are served at `http://localhost:3000/photos/` and persist across server restarts. + +--- + +## LLM / Voice AI integration + +Re-Commander proxies LLM requests server-side so your API key never touches the browser. Any OpenAI-compatible endpoint works: + +| Provider | `LLM_ENDPOINT` | Notes | +|----------|---------------|-------| +| Local Ollama | `http://localhost:11434/v1/chat/completions` | Default; no key needed | +| OpenAI | `https://api.openai.com/v1/chat/completions` | Set `LLM_API_KEY` | +| Anthropic (via proxy) | your proxy URL | Set `LLM_API_KEY` | +| Any OpenAI-compatible | any URL | Set `LLM_API_KEY` if required | + +The built-in system prompt instructs the model to respond exclusively in ESML — Jibo's markup language that simultaneously drives speech, body animations, screen graphics, and audio effects. You can override it by setting `LLM_SYSTEM_PROMPT` in `.env`. + +--- + +## Architecture + +``` +Browser (app.js) + │ WebSocket /ws REST /api/* + ▼ ▼ +server.js (Node/Express) + │ + ├─ JiboClient ──── WebSocket ──► Jibo ROM :8160 + │ └─ WakewordWatcher ─── WebSocket ──► Jibo ASR :8088 + │ + └─ /photos (static file serving) +``` + +- The server maintains a persistent WebSocket to the robot and reconnects automatically. +- A heartbeat (`GetConfig` every 9 s) keeps the session alive past Jibo's 10 s inactivity timeout. +- The wakeword watcher maintains a separate persistent connection to the always-on ASR task and forwards hotphrase events to the browser. +- All robot events are broadcast to every connected browser tab over the `/ws` WebSocket. + +--- + +## Troubleshooting + +**"Connecting…" never turns green** +- Confirm the robot is in `int-developer` mode and on the same network. +- Check that `JIBO_HOST` in `server.js` matches the robot's IP. +- Try `curl http://:8160/request` from the machine running the server. + +**Listen / ASR does nothing** +- The local ASR service runs on port 8088. Confirm the robot is in `int-developer` mode (it exposes the ASR service only in that mode). + +**LLM responses don't work** +- Check `LLM_ENDPOINT` and `LLM_MODEL` in `.env`. +- For local Ollama, make sure the model is pulled: `ollama pull llama3`. +- For cloud endpoints, verify `LLM_API_KEY` is set correctly. + +**Photos are not appearing** +- The `photos/` directory is created automatically. Check the server console for `[photo] saved:` log lines. +- If the robot disconnects immediately after taking a photo, the fetch from port 8160 may time out — reconnect and try again. + +--- + +## License + +MIT diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..1825c1a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,861 @@ +{ + "name": "re-commander", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "re-commander", + "version": "1.0.0", + "dependencies": { + "dotenv": "^17.4.2", + "express": "^4.18.2", + "ws": "^8.14.2" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..4e15789 --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "re-commander", + "version": "1.0.0", + "description": "Jibo ROM Commander — local Node.js recreation using port 8160", + "main": "server.js", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "dotenv": "^17.4.2", + "express": "^4.18.2", + "ws": "^8.14.2" + } +} diff --git a/public/app.js b/public/app.js new file mode 100644 index 0000000..b104787 --- /dev/null +++ b/public/app.js @@ -0,0 +1,579 @@ +'use strict'; + +// ── WebSocket to server ────────────────────────────────────────────────────── + +let ws; +let connected = false; +let sessionActive = false; +let currentAngles = [0, 0]; +let lastSayTx = null; +let lastListenTx = null; +let videoActive = false; +let entities = {}; // entityId → track data + +function connectWS() { + const proto = location.protocol === 'https:' ? 'wss' : 'ws'; + ws = new WebSocket(`${proto}://${location.host}/ws`); + + ws.onopen = () => { + connected = true; + }; + + ws.onclose = () => { + connected = false; + setStatus(false, 'Disconnected — reconnecting…'); + setTimeout(connectWS, 2000); + }; + + ws.onerror = () => {}; + + ws.onmessage = (e) => { + let msg; + try { msg = JSON.parse(e.data); } catch { return; } + handleServerMessage(msg); + }; +} + +function handleServerMessage(msg) { + if (msg.type === 'status') { + sessionActive = !!msg.sessionID; + currentAngles = msg.angles || [0, 0]; + setStatus(msg.connected && sessionActive, + msg.connected + ? (sessionActive ? 'Connected • ' + msg.sessionID.slice(0, 8) + '…' : 'Connecting…') + : 'Disconnected'); + return; + } + + if (msg.type === 'jiboEvent') { + handleJiboEvent(msg.body, msg.txId); + } +} + +function handleJiboEvent(body, txId) { + if (!body) return; + const evt = body.Event || body.ResponseString || '?'; + if (evt === 'onConfig') return; // heartbeat GetConfig response — suppress from log + logEvent(evt, body, txId); + + switch (body.Event) { + case 'onHotWordHeard': + flashHotword(body.utterance || 'hey jibo', body.score); + if (document.getElementById('auto-listen-toggle').checked) doListen(); + break; + + case 'onStart': + if (txId === lastListenTx) { + clearListenTimeout(); + document.getElementById('listen-result').textContent = '(listening…)'; + } + break; + + case 'onListenResult': { + clearListenTimeout(); + const speech = body.Speech || ''; + document.getElementById('listen-result').textContent = '"' + speech + '"'; + if (speech && document.getElementById('llm-toggle').checked) runLLMLoop(speech); + break; + } + + case 'onPhotoSaved': + if (body.url) addPhoto(body.url); + break; + + case 'onVideoReady': + if (body.URI) startVideoFeed(body.URI); + break; + + case 'onEntityGained': + case 'onEntityUpdate': + if (body.Tracks) body.Tracks.forEach(t => { entities[t.EntityID] = t; }); + renderEntities(); + break; + + case 'onEntityLost': + if (body.Tracks) body.Tracks.forEach(t => { delete entities[t.EntityID]; }); + renderEntities(); + break; + + case 'onHeadTouch': + if (body.Pads) renderHeadTouch(body.Pads); + break; + + case 'onLookAtAchieved': + break; + + case 'onStop': + if (txId === lastListenTx) { + clearListenTimeout(); + const reason = body.StopReason || body.ListenStopReason || 'stopped'; + document.getElementById('listen-result').textContent = '(stopped: ' + reason + ')'; + } + break; + + case 'onError': + if (txId === lastListenTx) { + clearListenTimeout(); + const errStr = body.EventError?.ErrorString || body.ErrorString || 'unknown error'; + document.getElementById('listen-result').textContent = '(error: ' + errStr + ')'; + } + break; + } +} + +// ── REST helpers ───────────────────────────────────────────────────────────── + +async function api(method, path, body) { + try { + const opts = { method, headers: { 'Content-Type': 'application/json' } }; + if (body) opts.body = JSON.stringify(body); + const res = await fetch(path, opts); + return await res.json(); + } catch (err) { + logEvent('api-error: ' + path, { error: err.message }, null); + } +} + +const post = (path, body) => api('POST', path, body); +const get = (path) => api('GET', path); + +// ── Status ─────────────────────────────────────────────────────────────────── + +function setStatus(ok, label) { + const dot = document.getElementById('status-dot'); + const lbl = document.getElementById('status-label'); + dot.className = ok ? 'ok' : ''; + lbl.textContent = label; +} + +// ── Arrow pad ──────────────────────────────────────────────────────────────── + +// ── Directional step ────────────────────────────────────────────────────────── +// Left/right: angle nudge (preserves psi exactly, no vertical drift). +// Up/down: screen-coord click 30% from center (same math as click-to-look). +// Both endpoints block server-side until the robot acks, so the loop has +// exactly one command in-flight at all times. Gen counter kills a stale loop +// on direction change after at most one more in-flight step completes. + +const STEP_FRAC = 0.30; + +let moveGen = 0; +let moveHeld = false; + + +// Horizontal steps use pctY slightly above true center to compensate for +// a small downward bias in the camera's optical axis vs the head neutral. +// Tune HORIZ_CENTER_Y if left/right still drifts up or down. +const HORIZ_CENTER_Y = 0.38; + +async function runMoveLoop(dir, gen) { + while (moveHeld && gen === moveGen) { + let pctX, pctY; + if (dir === 'left') { pctX = 0.5 - STEP_FRAC; pctY = HORIZ_CENTER_Y; } + else if (dir === 'right') { pctX = 0.5 + STEP_FRAC; pctY = HORIZ_CENTER_Y; } + else if (dir === 'up') { pctX = 0.5; pctY = 0.5 - STEP_FRAC; } + else { pctX = 0.5; pctY = 0.5 + STEP_FRAC; } + const x = Math.round((1 - pctX) * 640); + const y = Math.round(pctY * 480); + await post('/api/look/step', { x, y }); + } +} + +function startMove(dir) { + moveHeld = true; + runMoveLoop(dir, ++moveGen); +} + +function stopMove() { + moveHeld = false; +} + +function addMoveButton(id, dir) { + const btn = document.getElementById(id); + btn.addEventListener('pointerdown', (e) => { e.preventDefault(); startMove(dir); }); + btn.addEventListener('pointerup', stopMove); + btn.addEventListener('pointercancel', stopMove); + btn.addEventListener('pointerleave', stopMove); +} + +addMoveButton('btn-up', 'up'); +addMoveButton('btn-down', 'down'); +addMoveButton('btn-left', 'left'); +addMoveButton('btn-right', 'right'); + +// Keyboard arrow keys — browser key-repeat ignored; our await loop is the repeat +const KEY_TO_DIR = { ArrowLeft: 'left', ArrowRight: 'right', ArrowUp: 'up', ArrowDown: 'down' }; +const heldKeys = new Set(); + +document.addEventListener('keydown', (e) => { + if (['INPUT', 'TEXTAREA', 'SELECT'].includes(document.activeElement.tagName)) return; + if (e.key === ' ') { e.preventDefault(); post('/api/look/angle', { theta: 0, psi: 0 }); return; } + const dir = KEY_TO_DIR[e.key]; + if (!dir || e.repeat || heldKeys.has(e.key)) return; + e.preventDefault(); + heldKeys.add(e.key); + startMove(dir); +}); + +document.addEventListener('keyup', (e) => { + if (!KEY_TO_DIR[e.key]) return; + heldKeys.delete(e.key); + stopMove(); +}); + +// ── Click-to-look ───────────────────────────────────────────────────────────── + +const cameraWrap = document.getElementById('camera-wrap'); +const clickDot = document.getElementById('click-dot'); + +cameraWrap.addEventListener('click', (e) => { + const rect = cameraWrap.getBoundingClientRect(); + const x = e.clientX - rect.left; + const y = e.clientY - rect.top; + const pctX = x / rect.width; + const pctY = y / rect.height; + + // Show click indicator + clickDot.style.left = x + 'px'; + clickDot.style.top = y + 'px'; + clickDot.style.display = 'block'; + setTimeout(() => { clickDot.style.display = 'none'; }, 800); + + // Mirror X to match the horizontally-flipped display + const camX = Math.round((1 - pctX) * 640); + const camY = Math.round(pctY * 480); + const track = document.getElementById('track-flag').checked; + + post('/api/look/screen', { x: camX, y: camY, track }); +}); + +// ── Say ─────────────────────────────────────────────────────────────────────── + +document.getElementById('btn-say').addEventListener('click', async () => { + const text = document.getElementById('say-text').value.trim(); + if (!text) return; + const r = await post('/api/say', { text }); + if (r) lastSayTx = r.txId; +}); + +document.getElementById('btn-say-cancel').addEventListener('click', () => { + if (lastSayTx) post('/api/cancel', { txId: lastSayTx }); +}); + +// ── Listen ──────────────────────────────────────────────────────────────────── + +let listenClientTimer = null; + +function clearListenTimeout() { + if (listenClientTimer) { clearTimeout(listenClientTimer); listenClientTimer = null; } +} + +async function doListen() { + clearListenTimeout(); + document.getElementById('listen-result').textContent = '(waiting for robot…)'; + const maxNoSpeech = parseInt(document.getElementById('listen-max-nosp').value) || 5000; + const maxSpeech = parseInt(document.getElementById('listen-max-speech').value) || 10000; + const r = await post('/api/listen', { maxSpeech, maxNoSpeech }); + if (r) { + lastListenTx = r.txId; + const clientTimeout = Math.max(maxSpeech, maxNoSpeech) + 16000; + listenClientTimer = setTimeout(() => { + document.getElementById('listen-result').textContent = + '(timed out — cloud ASR may be unavailable)'; + }, clientTimeout); + } +} + +document.getElementById('btn-listen').addEventListener('click', doListen); + +document.getElementById('btn-listen-cancel').addEventListener('click', () => { + clearListenTimeout(); + if (lastListenTx) post('/api/cancel', { txId: lastListenTx }); + document.getElementById('listen-result').textContent = '(cancelled)'; +}); + +// ── Auto-listen + Voice AI ──────────────────────────────────────────────────── + +let llmHistory = []; // [{role:'user'|'assistant', content:string}] + +function llmStatus(msg) { + document.getElementById('llm-status').textContent = msg; +} + +async function runLLMLoop(speechText) { + llmHistory.push({ role: 'user', content: speechText }); + llmStatus('Thinking…'); + + const endpoint = document.getElementById('llm-endpoint').value.trim(); + const model = document.getElementById('llm-model').value.trim(); + const systemPrompt = document.getElementById('llm-system-prompt').value.trim(); + + const r = await post('/api/llm/chat', { + messages: llmHistory, + endpoint: endpoint || undefined, + model: model || undefined, + systemPrompt: systemPrompt || undefined, + }); + + if (!r || r.error) { + llmStatus('LLM error: ' + (r?.error || 'no response')); + llmHistory.pop(); // undo the user push so history stays consistent + return; + } + + const reply = r.reply; + llmHistory.push({ role: 'assistant', content: reply }); + llmStatus(`[${llmHistory.length / 2} turns] Last: "${reply.slice(0, 60)}${reply.length > 60 ? '…' : ''}"`); + + // Fill say box so user can see what Jibo is about to say + document.getElementById('say-text').value = reply; + await post('/api/say', { text: reply }); +} + +document.getElementById('btn-llm-clear').addEventListener('click', () => { + llmHistory = []; + llmStatus('Conversation cleared.'); +}); + +// ── Attention ───────────────────────────────────────────────────────────────── + +document.querySelectorAll('.attn-btn').forEach(btn => { + btn.addEventListener('click', function () { + document.querySelectorAll('.attn-btn').forEach(b => b.classList.remove('active')); + this.classList.add('active'); + post('/api/attention', { mode: this.dataset.mode }); + }); +}); + +// ── Volume ──────────────────────────────────────────────────────────────────── + +document.getElementById('volume-slider').addEventListener('input', function () { + document.getElementById('volume-label').textContent = + Math.round(this.value * 100) + '%'; +}); + +document.getElementById('btn-set-volume').addEventListener('click', () => { + const level = parseFloat(document.getElementById('volume-slider').value); + post('/api/volume', { level }); +}); + +// ── Camera / Video ──────────────────────────────────────────────────────────── + +let videoTxId = null; + +document.getElementById('btn-video-start').addEventListener('click', async () => { + const r = await post('/api/video/start', { duration: 0 }); + if (r) videoTxId = r.txId; + document.getElementById('video-status').textContent = 'Waiting for VideoReady…'; +}); + +document.getElementById('btn-video-stop').addEventListener('click', () => { + post('/api/video/stop', {}); + stopVideoFeed(); +}); + +function startVideoFeed(uri) { + const feed = document.getElementById('camera-feed'); + const noFeed = document.getElementById('camera-no-feed'); + feed.src = '/proxy/stream?uri=' + encodeURIComponent(uri); + feed.style.display = 'block'; + noFeed.style.display = 'none'; + document.getElementById('video-status').textContent = 'Streaming'; + videoActive = true; + + feed.onerror = () => { + stopVideoFeed(); + document.getElementById('video-status').textContent = 'Stream error'; + }; +} + +function stopVideoFeed() { + const feed = document.getElementById('camera-feed'); + feed.src = ''; + feed.style.display = 'none'; + document.getElementById('camera-no-feed').style.display = ''; + document.getElementById('video-status').textContent = ''; + videoActive = false; +} + +// ── Take Photo ──────────────────────────────────────────────────────────────── + +document.getElementById('btn-photo').addEventListener('click', () => { + post('/api/photo', { + camera: document.getElementById('photo-camera').value, + resolution: document.getElementById('photo-res').value + }); +}); + +function addPhoto(url) { + const strip = document.getElementById('photo-strip'); + const img = document.createElement('img'); + img.src = url; + img.title = url; + img.addEventListener('click', () => openPhotoModal(img.src)); + strip.prepend(img); + // Show photo in camera area if not streaming video + if (!videoActive) { + const feed = document.getElementById('camera-feed'); + const noFeed = document.getElementById('camera-no-feed'); + feed.src = img.src; + feed.style.display = 'block'; + noFeed.style.display = 'none'; + } +} + +// ── Photo modal ─────────────────────────────────────────────────────────────── + +function openPhotoModal(src) { + document.getElementById('photo-modal-img').src = src; + document.getElementById('photo-modal').classList.add('open'); +} + +document.getElementById('photo-modal').addEventListener('click', (e) => { + if (e.target === e.currentTarget || e.target.id === 'photo-modal-close') { + document.getElementById('photo-modal').classList.remove('open'); + } +}); + +// ── Display controls ────────────────────────────────────────────────────────── + +document.getElementById('btn-eye').addEventListener('click', () => { + post('/api/display/eye', {}); +}); + +document.getElementById('btn-play-anim').addEventListener('click', () => { + const name = document.getElementById('eye-anim-select').value; + if (name) post('/api/display/anim', { name }); +}); + +document.getElementById('btn-display-text').addEventListener('click', () => { + const text = document.getElementById('display-text').value.trim(); + if (!text) return; + post('/api/display/text', { text }); +}); + +document.getElementById('btn-display-image').addEventListener('click', () => { + const src = document.getElementById('display-img-src').value.trim(); + if (!src) return; + post('/api/display/image', { src }); +}); + +// ── Entities ────────────────────────────────────────────────────────────────── + +function renderEntities() { + const list = document.getElementById('entity-list'); + const ids = Object.keys(entities); + if (ids.length === 0) { + list.innerHTML = '
No entities tracked
'; + return; + } + list.innerHTML = ids.map(id => { + const t = entities[id]; + const sc = t.ScreenCoords ? t.ScreenCoords.map(v => v.toFixed(0)).join(',') : '?'; + return `
+ ${t.Type || 'Person'} #${id} (${t.Confidence || 0}%)
+ screen: [${sc}] + +
`; + }).join(''); +} + +window.trackEntity = function (entityId) { + post('/api/look/entity', { entityId, track: true }); +}; + +// ── Head Touch ──────────────────────────────────────────────────────────────── + +const padNames = ['frontLeft', 'middleLeft', 'backLeft', 'frontRight', 'middleRight', 'backRight']; + +function renderHeadTouch(pads) { + padNames.forEach((name, i) => { + const el = document.querySelector(`.pad-indicator[data-pad="${name}"]`); + if (el) el.classList.toggle('active', !!pads[i]); + }); + // Clear after 1s + setTimeout(() => padNames.forEach(name => { + const el = document.querySelector(`.pad-indicator[data-pad="${name}"]`); + if (el) el.classList.remove('active'); + }), 1000); +} + +// ── Subscriptions (toggle) ──────────────────────────────────────────────────── + +// Subscriptions are always-on by default (server subscribes on session start). +// Buttons here allow re-subscribing if needed. +['entity', 'motion', 'headtouch'].forEach(sub => { + document.getElementById('btn-sub-' + sub).addEventListener('click', function () { + const map = { entity: 'Entity', motion: 'Motion', headtouch: 'HeadTouch' }; + post('/api/look/angle', {}); // ping/no-op to keep conn; subscriptions are server-managed + }); +}); + +// ── Tabs ────────────────────────────────────────────────────────────────────── + +document.querySelectorAll('.tab-btn').forEach(btn => { + btn.addEventListener('click', function () { + document.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('active')); + document.querySelectorAll('.tab-panel').forEach(p => p.classList.remove('active')); + this.classList.add('active'); + document.getElementById(this.dataset.tab).classList.add('active'); + }); +}); + +// ── Event log ───────────────────────────────────────────────────────────────── + +const MAX_LOG = 200; + +function logEvent(eventName, body, txId) { + const log = document.getElementById('event-log'); + const now = new Date().toLocaleTimeString(); + + // Pick color class + let cls = 'evt'; + if (eventName.toLowerCase().includes('error')) cls = 'evt-error'; + else if (eventName === 'MotionDetected') cls = 'evt-motion'; + else if (['TrackGained', 'TrackUpdate', 'TrackLost'].includes(eventName)) cls = 'evt-entity'; + + // Short summary of body + let detail = ''; + if (body.Speech) detail = ' "' + body.Speech + '"'; + else if (body.URI) detail = ' ' + body.URI; + else if (body.ErrorString) detail = ' ' + body.ErrorString; + + const el = document.createElement('div'); + el.className = 'log-entry'; + el.innerHTML = `${now} ${eventName}${detail}`; + log.prepend(el); + + // Trim + while (log.children.length > MAX_LOG) log.removeChild(log.lastChild); +} + +document.getElementById('btn-clear-log').addEventListener('click', () => { + document.getElementById('event-log').innerHTML = ''; +}); + +// ── Hotword indicator ───────────────────────────────────────────────────────── + +let hotwordTimer = null; + +function flashHotword(utterance, score) { + const el = document.getElementById('hotword-indicator'); + if (!el) return; + el.textContent = '🎙 "' + utterance + '" (score: ' + (score || 0).toFixed(0) + ')'; + el.classList.add('active'); + clearTimeout(hotwordTimer); + hotwordTimer = setTimeout(() => el.classList.remove('active'), 3000); +} + +// ── Init ────────────────────────────────────────────────────────────────────── + +connectWS(); + +// Populate LLM fields from server config (.env defaults) +get('/api/config').then(cfg => { + if (!cfg) return; + if (cfg.llmEndpoint) document.getElementById('llm-endpoint').value = cfg.llmEndpoint; + if (cfg.llmModel) document.getElementById('llm-model').value = cfg.llmModel; + if (cfg.llmSystemPrompt) document.getElementById('llm-system-prompt').value = cfg.llmSystemPrompt; +}); diff --git a/public/index.html b/public/index.html new file mode 100644 index 0000000..67db010 --- /dev/null +++ b/public/index.html @@ -0,0 +1,592 @@ + + + + + + Re-Commander — Jibo + + + + +
+
+

Re-Commander

+ Connecting… + +
+ +
+ + +
+ + +
+
Head Navigation
+
+
+
+ +
+ +
+ +
+ +
+
+
+
+ + +
+
Say
+
+ +
+
+ + +
+
+ + +
+
Listen
+
+ + + +
+
+ + + +
+
+ + +
+
(result appears here)
+
+ + +
+
+ + +
+
Voice AI
+
+ + + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
Attention Mode
+
+ + + + + + + + +
+
+ + +
+
Volume
+
+ + 75% +
+ +
+ +
+ + +
+ + +
+ +
+
📷
+
No camera feed
+
Use the Camera tab to start video or take a photo
+
+
+
+ + +
+
+
Click on feed → look there
+
+ Track: + + +
+
+
+ + +
+
Photo Strip
+
+
+ +
+ + +
+
+ + + + +
+ + +
+
Video Stream
+
+ + +
+ +
Take Photo
+
+ + +
+
+ + +
+ + +
Subscriptions
+
+ + + +
+ +
+ + +
+
Eye
+ + +
Play Animation
+
+ +
+ + +
Display Text
+
+ +
+ + +
Display Image
+
+ +
+ + +
+ + +
+
Detected Entities
+
Waiting for entity events…
+
Head Touch
+
+
FL
+
ML
+
BL
+
FR
+
MR
+
BR
+
+ +
+ + +
+
+ Event Log + +
+
+
+ +
+
+ + +
+
✕
+ Photo +
+ + + + diff --git a/public/style.css b/public/style.css new file mode 100644 index 0000000..e169858 --- /dev/null +++ b/public/style.css @@ -0,0 +1,376 @@ +:root { + --bg: #0e0e0e; + --surface: #1a1a1a; + --surface2: #242424; + --border: #333; + --accent: #00d4ff; + --accent2: #0099bb; + --text: #e8e8e8; + --muted: #888; + --danger: #ff4444; + --success: #44dd88; + --warn: #ffaa00; +} + +* { box-sizing: border-box; margin: 0; padding: 0; } + +body { + background: var(--bg); + color: var(--text); + font-family: 'Segoe UI', system-ui, sans-serif; + font-size: 14px; + height: 100vh; + overflow: hidden; + display: flex; + flex-direction: column; +} + +header { + display: flex; + align-items: center; + gap: 12px; + padding: 8px 16px; + background: var(--surface); + border-bottom: 1px solid var(--border); + flex-shrink: 0; +} + +header h1 { font-size: 16px; font-weight: 600; letter-spacing: 0.05em; } +header h1 span { color: var(--accent); } + +#status-dot { + width: 10px; height: 10px; + border-radius: 50%; + background: var(--danger); + flex-shrink: 0; +} +#status-dot.ok { background: var(--success); } +#status-label { font-size: 12px; color: var(--muted); } +#angle-display { margin-left: auto; font-size: 12px; color: var(--muted); font-family: monospace; } + +#hotword-indicator { + font-size: 12px; color: var(--muted); font-style: italic; + opacity: 0; transition: opacity 0.2s; + white-space: nowrap; +} +#hotword-indicator.active { opacity: 1; color: var(--accent); } + +.main { + display: grid; + grid-template-columns: 340px 1fr 300px; + gap: 0; + flex: 1; + overflow: hidden; +} + +/* ── Left panel ── */ +.left-panel { + background: var(--surface); + border-right: 1px solid var(--border); + display: flex; + flex-direction: column; + overflow-y: auto; +} + +/* ── Center panel ── */ +.center-panel { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 12px; + gap: 10px; + background: var(--bg); + overflow: hidden; +} + +/* ── Right panel ── */ +.right-panel { + background: var(--surface); + border-left: 1px solid var(--border); + display: flex; + flex-direction: column; + overflow: hidden; +} + +/* ── Sections ── */ +.section { + border-bottom: 1px solid var(--border); + padding: 10px 12px; +} + +.section-title { + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--muted); + margin-bottom: 8px; +} + +/* ── Camera feed ── */ +#camera-wrap { + position: relative; + width: 100%; + max-width: 640px; + aspect-ratio: 640 / 480; + background: #000; + border-radius: 6px; + overflow: hidden; + cursor: crosshair; + border: 1px solid var(--border); + flex-shrink: 0; +} + +#camera-feed { + width: 100%; + height: 100%; + object-fit: cover; + display: block; + transform: scaleX(-1); +} + +#camera-overlay { + position: absolute; + inset: 0; + pointer-events: none; +} + +#click-dot { + position: absolute; + width: 16px; height: 16px; + border-radius: 50%; + border: 2px solid var(--accent); + transform: translate(-50%, -50%); + display: none; + pointer-events: none; + box-shadow: 0 0 8px var(--accent); +} + +#camera-no-feed { + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + flex-direction: column; + gap: 8px; + color: var(--muted); + font-size: 13px; +} + +#camera-no-feed canvas { display: none; } + +/* ── Arrow pad ── */ +.arrow-pad { + display: grid; + grid-template-columns: repeat(3, 40px); + grid-template-rows: repeat(3, 40px); + gap: 4px; +} + +.arrow-pad button { + width: 40px; height: 40px; + background: var(--surface2); + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text); + font-size: 16px; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: background 0.1s; + user-select: none; +} +.arrow-pad button:hover { background: var(--accent2); border-color: var(--accent); } +.arrow-pad button:active { background: var(--accent); } + +/* ── Controls row ── */ +.controls-row { + display: flex; + flex-wrap: wrap; + gap: 6px; + align-items: center; +} + +/* ── Buttons ── */ +button, .btn { + background: var(--surface2); + border: 1px solid var(--border); + border-radius: 5px; + color: var(--text); + padding: 5px 10px; + cursor: pointer; + font-size: 13px; + transition: background 0.1s, border-color 0.1s; + white-space: nowrap; +} +button:hover, .btn:hover { background: var(--accent2); border-color: var(--accent); } +button.active { background: var(--accent); border-color: var(--accent); color: #000; font-weight: 600; } +button.danger { border-color: var(--danger); } +button.danger:hover { background: var(--danger); } + +/* ── Inputs ── */ +input[type="text"], input[type="number"], select, textarea { + background: var(--surface2); + border: 1px solid var(--border); + border-radius: 5px; + color: var(--text); + padding: 5px 8px; + font-size: 13px; + outline: none; + width: 100%; +} +input:focus, select:focus, textarea:focus { border-color: var(--accent); } + +input[type="range"] { + width: 100%; + accent-color: var(--accent); +} + +label { font-size: 12px; color: var(--muted); display: block; margin-bottom: 3px; } + +.row { display: flex; gap: 6px; align-items: center; margin-bottom: 6px; } +.row label { margin: 0; flex-shrink: 0; } +.row input, .row select { flex: 1; } + +.field { margin-bottom: 8px; } + +/* ── Photo strip ── */ +#photo-strip { + display: flex; + flex-wrap: wrap; + gap: 6px; + max-height: 200px; + overflow-y: auto; +} +#photo-strip img { + width: 80px; height: 60px; + object-fit: cover; + border-radius: 4px; + border: 1px solid var(--border); + cursor: pointer; + transition: border-color 0.1s; +} +#photo-strip img:hover { border-color: var(--accent); } + +/* ── Event log ── */ +#event-log { + flex: 1; + overflow-y: auto; + font-family: monospace; + font-size: 11px; + padding: 8px; +} +.log-entry { + padding: 2px 0; + border-bottom: 1px solid var(--border); + word-break: break-all; +} +.log-entry .ts { color: var(--muted); } +.log-entry .evt { color: var(--accent); } +.log-entry .evt-error { color: var(--danger); } +.log-entry .evt-motion { color: var(--warn); } +.log-entry .evt-entity { color: var(--success); } + +/* ── Tabs ── */ +.tabs { + display: flex; + border-bottom: 1px solid var(--border); + flex-shrink: 0; +} +.tab-btn { + flex: 1; + padding: 7px 4px; + background: none; + border: none; + border-bottom: 2px solid transparent; + color: var(--muted); + font-size: 12px; + cursor: pointer; + border-radius: 0; +} +.tab-btn:hover { color: var(--text); background: none; border-color: transparent; } +.tab-btn.active { color: var(--accent); border-bottom-color: var(--accent); background: none; font-weight: 600; } + +.tab-panel { display: none; padding: 10px 12px; overflow-y: auto; flex: 1; } +.tab-panel.active { display: block; } + +/* ── Attention grid ── */ +.attention-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 4px; +} + +/* ── Listen result ── */ +#listen-result { + background: var(--surface2); + border: 1px solid var(--border); + border-radius: 5px; + padding: 6px 8px; + min-height: 40px; + font-style: italic; + color: var(--accent); + font-size: 13px; +} + +/* ── Entity list ── */ +#entity-list { font-size: 12px; } +.entity-item { + padding: 4px 6px; + background: var(--surface2); + border-radius: 4px; + margin-bottom: 4px; + cursor: pointer; + display: flex; + justify-content: space-between; + align-items: center; + border: 1px solid transparent; +} +.entity-item:hover { border-color: var(--accent); } +.entity-item .track-btn { font-size: 11px; padding: 2px 6px; } + +/* ── Scrollbars ── */ +::-webkit-scrollbar { width: 6px; height: 6px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } +::-webkit-scrollbar-thumb:hover { background: var(--muted); } + +/* ── Photo modal ── */ +#photo-modal { + display: none; + position: fixed; + inset: 0; + background: rgba(0,0,0,0.85); + z-index: 100; + align-items: center; + justify-content: center; +} +#photo-modal.open { display: flex; } +#photo-modal img { + max-width: 90vw; + max-height: 90vh; + border-radius: 6px; + border: 1px solid var(--border); +} +#photo-modal-close { + position: absolute; + top: 16px; right: 16px; + font-size: 28px; + cursor: pointer; + color: var(--text); + background: var(--surface); + border: 1px solid var(--border); + border-radius: 50%; + width: 36px; height: 36px; + display: flex; + align-items: center; + justify-content: center; + line-height: 1; +} + +/* ── Video area ── */ +.video-controls { display: flex; gap: 6px; align-items: center; } +#video-status { font-size: 12px; color: var(--muted); } diff --git a/server.js b/server.js new file mode 100644 index 0000000..b6236cd --- /dev/null +++ b/server.js @@ -0,0 +1,973 @@ +'use strict'; + +const express = require('express'); +const http = require('http'); +const https = require('https'); +const { WebSocketServer, WebSocket } = require('ws'); +const crypto = require('crypto'); +const httpModule = require('http'); +const path = require('path'); +const fs = require('fs'); + +require('dotenv').config(); + +const JIBO_HOST = '192.168.1.217'; +const JIBO_PORT = 8160; +const APP_PORT = process.env.PORT || 3000; + +const LLM_SYSTEM_PROMPT = `You are Jibo, a small expressive home robot. Every reply MUST be written in ESML +(Embodied Speech Markup Language). ESML is an XML dialect that simultaneously +drives Jibo's body animations, screen graphics, audio effects, and TTS voice. +Respond ONLY with the final spoken output annotated with ESML tags. +No reasoning, no blocks, no preamble — only what Jibo will say and do. + +== ANIMATION TAGS == +Use for body/screen animations from Jibo's built-in library (preferred). +Use when you also need to blend in SSA or SFX in the same tag. + +Blocking (Jibo freezes speech while it plays, resumes after): + following text here + following text here + +Bounded non-blocking (animation duration stretches to match the enclosed text): + text spoken during animation + +Unbounded non-blocking (animation plays at native length alongside text that follows): + text spoken at the same time + +Common attributes: + cat='CATEGORY' select animation by emotional category (preferred) + name='AnimName' select exact animation by its library name + nonBlocking='true' play alongside TTS instead of blocking it + endNeutral='true' snap back to neutral pose when done (use this by default) + loop='0' repeat to fill bounded duration (bounded mode only) + loop='N' repeat N times (unbounded mode only) + filter='!ssa-only' exclude audio-only animations from the category pick + layers='!screen' use only body layer (drop screen graphics) + +Animation categories (cat= values): + affection confused dance embarrassed excited frustrated + happy laughing no proud relieved sad scared surprised worried yes + +== EMOJIS (Screen Graphics) == +Use with the emoji category and specific filters to display a graphic on Jibo's screen. +Always use nonBlocking='true' for emojis. +Syntax: + +Available EMOJIS (EMOJI_NAME): + airplane basketball beach car disco-spin football soccer trophy + music question-mark star beer cake cheese drumstick coffee fork + fish groceries burger hotdog icecream pizza wine christmas-tree + fireworks halloween hanukkah thanksgiving clover valentines chocolate + bicycle cat laptop dog gift house laundry lightbulb money popcorn + party phone robot sunglasses toilet-paper trash umbrella video-game + bird cow earth flower lightning-bolt moon mountain mouse penguin + pig bunny rainbow baby heart + +== DANCES == +Use with the dance category to make Jibo dance. You can choose to include music or not. +Syntax (with music): +Syntax (without music): + +Available DANCES (DANCE_NAME): + rom-upbeat rom-ballroom rom-silly rom-slowdance rom-eletronic rom-twerk + +== SSA (Semi-Speech Audio — emotional vocal sounds) == +Always self-closing. Play before, after, or between sentences; never inside . + + + + + + + + +== SFX (Sound effects) == +Always self-closing. Good for punctuating facts, transitions, or reactions. + + + + + + +== VOICE / SPEECH TAGS == +Pause: (length in seconds) +Style: + Styles: neutral enthusiastic sheepish confused confident +Pitch: text (±semitones from baseline) + text (pitch multiplier) + text (Hz offset) + text (vibrance/bandwidth) +Duration: text (>1 = slower, <1 = faster) + text (exact duration in seconds) +Spell: (spells each letter) +Phoneme: Bono + +== RULES == +1. ALWAYS use ESML. Plain text is valid ESML — but add tags whenever they make + Jibo more expressive and natural. +2. Keep total response length SHORT: one or two sentences maximum. +3. Opening animations set the emotional tone before speech: + Oh, cool! +4. Bounded animations sync motion to the most important words: + I really love that idea! +5. Use for non-verbal emotional sounds (gasps, laughs, hums). +6. Use + +User: "What's 2 plus 2?" + That's 4! Easy one. + +User: "Wow, that's surprising!" + I know, right?! + +User: "Do you like cats?" + I love them! + +User: "Show me a dance." + Watch these moves!` + +// ── Jibo client ────────────────────────────────────────────────────────────── + +class JiboClient { + constructor() { + this.ws = null; + this.sessionID = ''; + this.version = '1.0'; + this.connected = false; + this.pendingTx = new Map(); // txId → {resolve, reject, timer} + this.subscribers = new Set(); // browser WebSocket connections + this.currentAngles = [0, 0]; // [theta, psi] + this.reconnectTimer = null; + this.videoStreamActive = false; + this.videoTxId = null; + this._heartbeatTimer = null; + this._heartbeatTxIds = new Set(); // suppress these from browser broadcast + this._lookInFlight = false; // true while waiting for robot to ack a LookAt angle + this._lookPending = null; // [theta, psi] – latest desired angles while in-flight + this._lookAckTimer = null; // safety timeout in case ack never arrives + } + + // POST /request to Jibo before WebSocket to supply a full ACO. + // Without this the @be falls back to a default ACO that omits Listen, + // SetAttention, Display, FetchAsset, SetConfig, HeadTouch, ScreenGesture. + _postRequest() { + return new Promise((resolve) => { + const body = JSON.stringify({ + aco: { + version: '1.0', + sourceId: 'ReCommander', + commandSet: [ + 'StartSession', 'GetConfig', 'SetConfig', 'Cancel', + 'SetAttention', 'Say', 'Listen', 'LookAt', + 'TakePhoto', 'Video', 'Display', 'FetchAsset', 'UnloadAsset', 'Subscribe' + ], + streamSet: ['Entity', 'Motion', 'HeadTouch', 'ScreenGesture', 'HotWord'], + keepAliveTimeout: 10000, + recoveryTimeout: 20000, + remoteConfig: { hideVisualCue: false, inactivityTimeout: 3600000 } + } + }); + const req = httpModule.request({ + host: JIBO_HOST, port: JIBO_PORT, + path: '/request', method: 'POST', + headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) } + }, (res) => { + let data = ''; + res.on('data', d => data += d); + res.on('end', () => { + console.log('[jibo] /request response:', data); + resolve(); + }); + }); + req.on('error', (err) => { + console.warn('[jibo] /request error (continuing anyway):', err.message); + resolve(); + }); + req.write(body); + req.end(); + }); + } + + connect() { + if (this.ws) { + try { this.ws.terminate(); } catch (_) {} + } + console.log(`[jibo] posting ACO to /request then connecting WebSocket`); + this._postRequest().then(() => { + this.ws = new WebSocket(`ws://${JIBO_HOST}:${JIBO_PORT}`); + + this.ws.on('open', () => { + console.log('[jibo] connected'); + this.connected = true; + this.sessionID = ''; + this._send({ Type: 'StartSession' }); + + // Respond explicitly to robot's WebSocket-level pings (belt-and-suspenders; + // ws library auto-pongs, but this ensures the robot's FLATLINE check never fires). + this.ws.on('ping', () => { + if (this.ws) try { this.ws.pong(); } catch (_) {} + }); + }); + + this.ws.on('message', (data) => { + let msg; + try { msg = JSON.parse(data); } catch (e) { return; } + this._handleMessage(msg); + }); + + this.ws.on('close', () => { + console.log('[jibo] disconnected — reconnecting in 3s'); + this.connected = false; + this.sessionID = ''; + this.videoStreamActive = false; + this._lookInFlight = false; + this._lookPending = null; + clearTimeout(this._lookAckTimer); + this._stopHeartbeat(); + this._broadcastStatus(); + clearTimeout(this.reconnectTimer); + this.reconnectTimer = setTimeout(() => this.connect(), 3000); + }); + + this.ws.on('error', (err) => { + console.error('[jibo] ws error:', err.message); + }); + }); + } + + _txId() { + return crypto.createHash('md5') + .update(Date.now().toString() + Math.random().toString()) + .digest('hex'); + } + + _send(command, expectAsync = false) { + const txId = this._txId(); + const msg = { + ClientHeader: { + TransactionID: txId, + SessionID: this.sessionID, + AppID: 'ImmaLittleTeapot', + Credentials: '', + Version: this.version + }, + Command: command + }; + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify(msg)); + } + return txId; + } + + _handleMessage(msg) { + // StartSession response + if (msg.Response?.ResponseBody?.SessionID && !this.sessionID) { + this.sessionID = msg.Response.ResponseBody.SessionID; + this.version = msg.Response.ResponseBody.Version || '1.0'; + console.log('[jibo] session started:', this.sessionID); + this._broadcastStatus(); + // Re-subscribe to entity/motion/headtouch after reconnect + this._send({ Type: 'Subscribe', StreamType: 'Entity' }); + this._send({ Type: 'Subscribe', StreamType: 'Motion' }); + this._send({ Type: 'Subscribe', StreamType: 'HeadTouch', StreamFilter: {} }); + this._send({ Type: 'Subscribe', StreamType: 'ScreenGesture', + StreamFilter: { Type: 'Tap', Area: { x: 0, y: 0, width: 1, height: 1 } } }); + this._startHeartbeat(); + return; + } + + // Suppress heartbeat (GetConfig) responses from reaching the browser. + // GetConfig sends two messages per txId (ack + onConfig event) so we keep + // the txId in the set until the pruning threshold clears it. + const incomingTxId = msg.EventHeader?.TransactionID || msg.ResponseHeader?.TransactionID; + if (incomingTxId && this._heartbeatTxIds.has(incomingTxId)) return; + + // Resolve any pending ack waiting on this txId + const txId = msg.EventHeader?.TransactionID || msg.ResponseHeader?.TransactionID; + if (txId && this.pendingTx.has(txId)) { + const evt = msg.EventBody?.Event; + // Terminal events for async commands + if (evt === 'onLookAtAchieved' || evt === 'onStop' || evt === 'onError') { + const { resolve, timer } = this.pendingTx.get(txId); + clearTimeout(timer); + this.pendingTx.delete(txId); + resolve(msg); + } + } + + // Release the in-flight lock when our angle command finishes (any terminal event). + // This must happen before the suppression below so "Target overwritten" still clears it. + const evtName = msg.EventBody?.Event; + if (txId && txId === this._lookActiveTxId && + (evtName === 'onLookAtAchieved' || evtName === 'onStop' || evtName === 'onError')) { + this._onLookAngleDone(); + } + + // Suppress "Target overwritten" — not a real error; don't pollute the event log. + if (evtName === 'onError' && + msg.EventBody?.EventError?.ErrorString === 'Target overwritten') return; + + // Photo — fetch from Jibo and save locally; browser gets onPhotoSaved with local URL. + if (msg.EventBody?.Event === 'onTakePhoto' && msg.EventBody?.URI) { + this._savePhoto(msg.EventBody.URI); + return; // suppress the raw onTakePhoto; browser gets onPhotoSaved instead + } + + // VideoReady — capture URI for proxy (event name is "onVideoReady") + if (msg.EventBody?.Event === 'onVideoReady') { + this.videoStreamActive = true; + this.videoURI = msg.EventBody.URI; + console.log('[jibo] onVideoReady URI:', this.videoURI); + } + + // Broadcast all events to browser clients + const envelope = { + type: 'jiboEvent', + txId: msg.EventHeader?.TransactionID || msg.ResponseHeader?.TransactionID, + body: msg.EventBody || msg.Response + }; + this._broadcastToClients(JSON.stringify(envelope)); + } + + _broadcastStatus() { + const status = JSON.stringify({ + type: 'status', + connected: this.connected, + sessionID: this.sessionID, + angles: this.currentAngles + }); + this._broadcastToClients(status); + } + + _broadcastToClients(data) { + for (const client of this.subscribers) { + if (client.readyState === WebSocket.OPEN) { + client.send(data); + } + } + } + + addSubscriber(ws) { + this.subscribers.add(ws); + // Send current status immediately + ws.send(JSON.stringify({ + type: 'status', + connected: this.connected, + sessionID: this.sessionID, + angles: this.currentAngles + })); + } + + removeSubscriber(ws) { + this.subscribers.delete(ws); + } + + // ── Heartbeat ───────────────────────────────────────────────────────────── + // Sends GetConfig every 9 s to reset the robot's inactivity timer. + // The robot enforces keepAliveTimeout=10s (app-level) and a 20s flatline + // check at the WebSocket level — this satisfies both. + _startHeartbeat() { + this._stopHeartbeat(); + this._heartbeatTimer = setInterval(() => { + if (this.connected && this.sessionID) { + const txId = this._send({ Type: 'GetConfig' }); + if (txId) this._heartbeatTxIds.add(txId); + // Prune old txIds so the set doesn't grow unbounded + if (this._heartbeatTxIds.size > 20) { + const first = this._heartbeatTxIds.values().next().value; + this._heartbeatTxIds.delete(first); + } + } + }, 9000); + } + + _stopHeartbeat() { + if (this._heartbeatTimer) { + clearInterval(this._heartbeatTimer); + this._heartbeatTimer = null; + } + this._heartbeatTxIds.clear(); + } + + // ── Public command methods ──────────────────────────────────────────────── + + lookAt(target, trackFlag = false, levelHeadFlag = false) { + return this._send({ Type: 'LookAt', LookAtTarget: target, TrackFlag: trackFlag, LevelHeadFlag: levelHeadFlag }); + } + + lookAtAngle(theta, psi, track = false) { + theta = Math.max(-180, Math.min(180, theta)); + psi = Math.max(-30, Math.min(30, psi)); + this.currentAngles = [theta, psi]; + this._broadcastStatus(); + if (this._lookInFlight) { + // Robot is still processing the last command — just update desired target, + // don't queue another message into its receive buffer. + this._lookPending = [theta, psi, track]; + return null; + } + return this._fireLookAngle(theta, psi, track); + } + + _fireLookAngle(theta, psi, track) { + this._lookInFlight = true; + this._lookPending = null; + const DEG = Math.PI / 180; + const txId = this.lookAt({ Angle: [theta * DEG, psi * DEG] }, track); + this._lookActiveTxId = txId; + // Safety release: if we never hear back within 400 ms, unblock anyway. + clearTimeout(this._lookAckTimer); + this._lookAckTimer = setTimeout(() => this._onLookAngleDone(), 400); + return txId; + } + + _onLookAngleDone() { + clearTimeout(this._lookAckTimer); + this._lookInFlight = false; + this._lookActiveTxId = null; + if (this._lookPending) { + const [t, p, track] = this._lookPending; + this._lookPending = null; + this._fireLookAngle(t, p, track); + } + } + + lookAtScreen(x, y, track = false) { + return this.lookAt({ ScreenCoords: [x, y] }, track, false); + } + + lookAtPosition(x, y, z, track = false) { + return this.lookAt({ Position: [x, y, z] }, track, false); + } + + lookAtEntity(entityId, track = true) { + return this.lookAt({ Entity: entityId }, track, false); + } + + say(esml) { + return this._send({ Type: 'Say', ESML: esml }); + } + + listen(maxSpeech = 10000, maxNoSpeech = 5000, lang = 'en-US') { + return this._send({ Type: 'Listen', MaxSpeechTimeout: maxSpeech, MaxNoSpeechTimeout: maxNoSpeech, LanguageCode: lang }); + } + + // Local STT via jibo-asr-service (port 8088) — no cloud needed. + // Mirrors the approach in @be/be/be/ai-bridge.js. + listenLocalASR(maxNoSpeech, maxSpeech) { + const ASR_HTTP = `http://${JIBO_HOST}:8088`; + const ASR_WS = `ws://${JIBO_HOST}:8088/simple_port`; + const taskId = 're-cmd-' + Date.now() + '-' + Math.floor(Math.random() * 1e9); + const reqId = 'start-' + Date.now(); + const timeoutMs = Math.max(maxNoSpeech, maxSpeech) + 2000; + const self = this; + + // Send the ROM Listen for light ring / attention visuals, ignore its result + const romTxId = this._send({ Type: 'Listen', MaxSpeechTimeout: maxSpeech, MaxNoSpeechTimeout: maxNoSpeech, LanguageCode: 'en-US' }); + + const startPayload = JSON.stringify({ + command: 'start', + task_id: taskId, + request_id: reqId, + audio_source_id: 'alsa1', + hotphrase: 'none', + speech_to_text: true, + }); + + function stopASR() { + const stopBody = JSON.stringify({ command: 'stop', task_id: taskId, request_id: 'stop-' + Date.now() }); + const req = httpModule.request({ + host: JIBO_HOST, port: 8088, path: '/asr_simple_interface', method: 'POST', + headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(stopBody) } + }); + req.on('error', () => {}); + req.write(stopBody); + req.end(); + } + + let wsClient = null; + let timer = null; + let done = false; + + function finish(speech) { + if (done) return; + done = true; + clearTimeout(timer); + if (wsClient) { try { wsClient.terminate(); } catch (e) {} wsClient = null; } + stopASR(); + // Cancel ROM listen + self._send({ Type: 'Cancel', ID: romTxId }); + // Broadcast result as if it were a normal jiboEvent + const evt = speech + ? { Event: 'onListenResult', Speech: speech, LanguageCode: 'en-US' } + : { Event: 'onStop', StopReason: 'NoInput' }; + self._broadcastToClients(JSON.stringify({ type: 'jiboEvent', txId: romTxId, body: evt })); + } + + // Connect WS first, then POST start + wsClient = new WebSocket(ASR_WS); + wsClient.on('open', () => { + // POST start to kick off recognition + const req = httpModule.request({ + host: JIBO_HOST, port: 8088, path: '/asr_simple_interface', method: 'POST', + headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(startPayload) } + }, (res) => { res.resume(); }); + req.on('error', (e) => { console.error('[asr] start error:', e.message); finish(null); }); + req.write(startPayload); + req.end(); + + // Overall timeout + timer = setTimeout(() => { finish(null); }, timeoutMs); + + console.log('[asr] local listen started, task:', taskId); + }); + + wsClient.on('message', (data) => { + let evt; + try { evt = JSON.parse(String(data)); } catch (e) { return; } + const evType = evt.event_type || evt.eventType || evt.event || evt.type; + if (evType !== 'speech_to_text_final') return; + + // Match by task/request id if present + const evTask = evt.task_id || evt.taskId || (evt.payload && evt.payload.task_id); + const evReq = evt.request_id || evt.requestId || (evt.payload && evt.payload.request_id); + if ((evTask || evReq) && evTask !== taskId && evReq !== reqId) return; + + const utterances = evt.utterances || evt.Utterances || (evt.payload && evt.payload.utterances); + // Utterance objects use .utterance as the primary text field (ai-bridge.js: pickBestAsrUtterance) + function pickUtterance(u) { + if (!u) return ''; + if (typeof u === 'string') return u; + return String(u.utterance || u.Utterance || u.text || ''); + } + const text = Array.isArray(utterances) + ? pickUtterance(utterances[0]) + : (typeof utterances === 'string' ? utterances : ''); + + const speech = text ? String(text).trim() : null; + console.log('[asr] speech_to_text_final:', speech || '(empty)'); + if (speech) finish(speech); + }); + + wsClient.on('error', (e) => { + console.error('[asr] ws error:', e.message); + finish(null); + }); + + wsClient.on('close', () => { if (!done) finish(null); }); + + return romTxId; + } + + takePhoto(camera = 'Right', resolution = 'HighRes', distortion = false) { + return this._send({ Type: 'TakePhoto', Camera: camera, Resolution: resolution, Distortion: distortion }); + } + + startVideo() { + // VideoType must be uppercase enum value; Duration is not in server schema + this.videoTxId = this._send({ Type: 'Video', VideoType: 'NORMAL' }); + return this.videoTxId; + } + + cancelVideo() { + if (this.videoTxId) { + this._send({ Type: 'Cancel', ID: this.videoTxId }); + this.videoTxId = null; + this.videoStreamActive = false; + } + } + + displayEye() { + return this._send({ Type: 'Display', View: { Type: 'Eye', Name: 'default' } }); + } + + playEyeAnim(animName) { + return this._send({ Type: 'Say', ESML: `` }); + } + + displayText(text, name = 'reCmd') { + return this._send({ Type: 'Display', View: { Type: 'Text', Name: name, Text: text } }); + } + + displayImage(src, name = 'reCmd') { + return this._send({ Type: 'Display', View: { Type: 'Image', Name: name, Image: { src, name, set: '' } } }); + } + + setAttention(mode) { + return this._send({ Type: 'SetAttention', Mode: mode }); + } + + setVolume(level) { + return this._send({ Type: 'SetConfig', Options: { Mixer: Math.max(0, Math.min(1, level)) } }); + } + + getConfig() { + return this._send({ Type: 'GetConfig' }); + } + + cancel(txId) { + return this._send({ Type: 'Cancel', ID: txId }); + } + + subscribe(streamType, filter = null) { + const cmd = { Type: 'Subscribe', StreamType: streamType }; + if (filter) cmd.StreamFilter = filter; + return this._send(cmd); + } + + nudge(dTheta, dPsi) { + const [theta, psi] = this.currentAngles; + return this.lookAtAngle(theta + dTheta, psi + dPsi); + } + + // Returns a Promise that resolves when the robot acks txId, or after timeoutMs. + _savePhoto(jiboUri) { + const url = `http://${JIBO_HOST}:${JIBO_PORT}${jiboUri}`; + const filename = `photo_${Date.now()}.jpg`; + const filepath = path.join(PHOTOS_DIR, filename); + const file = fs.createWriteStream(filepath); + httpModule.get(url, (jiboRes) => { + jiboRes.pipe(file); + file.on('finish', () => { + file.close(); + console.log('[photo] saved:', filename); + // Rebroadcast with local URL so the browser doesn't need the proxy + this._broadcastToClients(JSON.stringify({ + type: 'jiboEvent', + txId: null, + body: { Event: 'onPhotoSaved', url: `/photos/${filename}`, filename } + })); + }); + }).on('error', (err) => { + fs.unlink(filepath, () => {}); + console.error('[photo] save failed:', err.message); + }); + } + + awaitAck(txId, timeoutMs = 2000) { + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.pendingTx.delete(txId); + resolve(null); + }, timeoutMs); + this.pendingTx.set(txId, { resolve, timer }); + }); + } +} + +// ── Video proxy ────────────────────────────────────────────────────────────── + +function proxyJiboStream(uri, res) { + const url = `http://${JIBO_HOST}:${JIBO_PORT}${uri}`; + console.log('[proxy] streaming:', url); + const req = httpModule.get(url, (jiboRes) => { + res.writeHead(jiboRes.statusCode, jiboRes.headers); + jiboRes.pipe(res); + res.on('close', () => req.destroy()); + }); + req.on('error', (err) => { + if (!res.headersSent) res.status(502).json({ error: err.message }); + }); +} + +function proxyJiboFetch(uri, res) { + const url = `http://${JIBO_HOST}:${JIBO_PORT}${uri}`; + const req = httpModule.get(url, (jiboRes) => { + res.writeHead(jiboRes.statusCode, jiboRes.headers); + jiboRes.pipe(res); + res.on('close', () => req.destroy()); + }); + req.on('error', (err) => { + if (!res.headersSent) res.status(502).json({ error: err.message }); + }); +} + +// ── Wakeword watcher ───────────────────────────────────────────────────────── +// Maintains a persistent connection to the always-on resident ASR task (task0) +// and forwards every "hotphrase" event to browser clients as onHotWordHeard. + +class WakewordWatcher { + constructor(broadcastFn) { + this._broadcast = broadcastFn; + this._ws = null; + this._reconnectTimer = null; + this._connect(); + } + + _connect() { + const url = `ws://${JIBO_HOST}:8088/simple_port`; + this._ws = new WebSocket(url); + + this._ws.on('open', () => { + console.log('[wakeword] connected to ASR WebSocket'); + }); + + this._ws.on('message', (data) => { + let evt; + try { evt = JSON.parse(String(data)); } catch (e) { return; } + if (evt.event_type !== 'hotphrase') return; + + const utterance = evt.utterances && evt.utterances[0]; + const score = utterance ? utterance.score : 0; + console.log('[wakeword] heard! score:', score); + + this._broadcast(JSON.stringify({ + type: 'jiboEvent', + txId: null, + body: { + Event: 'onHotWordHeard', + utterance: utterance ? utterance.utterance : 'hey jibo', + score: score, + timestamp: evt.timestamp || new Date().toISOString() + } + })); + }); + + this._ws.on('close', () => { + console.log('[wakeword] disconnected — reconnecting in 3s'); + clearTimeout(this._reconnectTimer); + this._reconnectTimer = setTimeout(() => this._connect(), 3000); + }); + + this._ws.on('error', (err) => { + console.error('[wakeword] error:', err.message); + }); + } +} + +// ── App setup ──────────────────────────────────────────────────────────────── + +const jibo = new JiboClient(); +const app = express(); +app.use(express.json()); +const PHOTOS_DIR = path.join(__dirname, 'photos'); +fs.mkdirSync(PHOTOS_DIR, { recursive: true }); + +app.use(express.static(path.join(__dirname, 'public'))); +app.use('/photos', express.static(PHOTOS_DIR)); + +// ── REST API ───────────────────────────────────────────────────────────────── + +app.post('/api/look/angle', (req, res) => { + const { theta = 0, psi = 0, track = false } = req.body; + const txId = jibo.lookAtAngle(parseFloat(theta), parseFloat(psi), !!track); + res.json({ txId }); +}); + +app.post('/api/look/screen', (req, res) => { + const { x, y, track = false } = req.body; + const txId = jibo.lookAtScreen(parseFloat(x), parseFloat(y), !!track); + res.json({ txId }); +}); + +// Blocking screen-coord step (up/down navigation). +app.post('/api/look/step', async (req, res) => { + const { x, y } = req.body; + const txId = jibo.lookAtScreen(parseFloat(x), parseFloat(y)); + await jibo.awaitAck(txId, 2000); + res.json({ txId }); +}); + + +app.post('/api/look/position', (req, res) => { + const { x = 0, y = 0, z = 500, track = false } = req.body; + const txId = jibo.lookAtPosition(parseFloat(x), parseFloat(y), parseFloat(z), !!track); + res.json({ txId }); +}); + +app.post('/api/look/entity', (req, res) => { + const { entityId, track = true } = req.body; + const txId = jibo.lookAtEntity(entityId, !!track); + res.json({ txId }); +}); + +app.post('/api/look/nudge', (req, res) => { + const { dTheta = 0, dPsi = 0 } = req.body; + const txId = jibo.nudge(parseFloat(dTheta), parseFloat(dPsi)); + res.json({ txId, angles: jibo.currentAngles }); +}); + +app.post('/api/say', (req, res) => { + const { text } = req.body; + if (!text) return res.status(400).json({ error: 'text required' }); + const txId = jibo.say(text); + res.json({ txId }); +}); + +app.post('/api/listen', (req, res) => { + const { maxSpeech = 10000, maxNoSpeech = 5000 } = req.body; + // Use local ASR service (port 8088) — bypasses offline Google cloud ASR + const txId = jibo.listenLocalASR(maxNoSpeech, maxSpeech); + res.json({ txId }); +}); + +app.post('/api/photo', (req, res) => { + const { camera = 'Right', resolution = 'HighRes' } = req.body; + const txId = jibo.takePhoto(camera, resolution); + res.json({ txId }); +}); + +app.post('/api/video/start', (req, res) => { + const txId = jibo.startVideo(); + res.json({ txId }); +}); + +app.post('/api/video/stop', (req, res) => { + jibo.cancelVideo(); + res.json({ ok: true }); +}); + +app.post('/api/display/eye', (req, res) => { + const txId = jibo.displayEye(); + res.json({ txId }); +}); + +app.post('/api/display/anim', (req, res) => { + const { name } = req.body; + if (!name) return res.status(400).json({ error: 'name required' }); + const txId = jibo.playEyeAnim(name); + res.json({ txId }); +}); + +app.post('/api/display/text', (req, res) => { + const { text } = req.body; + if (!text) return res.status(400).json({ error: 'text required' }); + const txId = jibo.displayText(text); + res.json({ txId }); +}); + +app.post('/api/display/image', (req, res) => { + const { src } = req.body; + if (!src) return res.status(400).json({ error: 'src required' }); + const txId = jibo.displayImage(src); + res.json({ txId }); +}); + +app.post('/api/attention', (req, res) => { + const { mode } = req.body; + if (!mode) return res.status(400).json({ error: 'mode required' }); + const txId = jibo.setAttention(mode); + res.json({ txId }); +}); + +app.post('/api/volume', (req, res) => { + const { level } = req.body; + if (level == null) return res.status(400).json({ error: 'level required' }); + const txId = jibo.setVolume(parseFloat(level)); + res.json({ txId }); +}); + +app.post('/api/cancel', (req, res) => { + const { txId } = req.body; + if (!txId) return res.status(400).json({ error: 'txId required' }); + jibo.cancel(txId); + res.json({ ok: true }); +}); + + +app.get('/api/config', (req, res) => { + res.json({ + llmEndpoint: process.env.LLM_ENDPOINT || '', + llmModel: process.env.LLM_MODEL || '', + llmSystemPrompt: LLM_SYSTEM_PROMPT || '', + }); +}); + +// Proxy OpenAI-compatible chat completions — keeps API key off the browser +function httpPost(urlStr, reqHeaders, body) { + return new Promise((resolve, reject) => { + const u = new URL(urlStr); + const mod = u.protocol === 'https:' ? https : httpModule; + const payload = JSON.stringify(body); + const req = mod.request({ + hostname: u.hostname, + port: u.port || (u.protocol === 'https:' ? 443 : 80), + path: u.pathname + u.search, + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload), ...reqHeaders } + }, (res) => { + let data = ''; + res.on('data', d => data += d); + res.on('end', () => { + try { resolve(JSON.parse(data)); } + catch (e) { reject(new Error('LLM non-JSON response: ' + data.slice(0, 300))); } + }); + }); + req.on('error', reject); + req.write(payload); + req.end(); + }); +} + +app.post('/api/llm/chat', async (req, res) => { + const { messages = [], endpoint, model, systemPrompt } = req.body; + const url = endpoint || process.env.LLM_ENDPOINT || 'http://localhost:11434/v1/chat/completions'; + const mdl = model || process.env.LLM_MODEL || 'llama3'; + const sysProm = systemPrompt || LLM_SYSTEM_PROMPT || ''; + const apiKey = process.env.LLM_API_KEY || ''; + + const allMessages = sysProm + ? [{ role: 'system', content: sysProm }, ...messages] + : messages; + + const headers = {}; + if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`; + + try { + const result = await httpPost(url, headers, { model: mdl, messages: allMessages, stream: false }); + const reply = result.choices?.[0]?.message?.content?.trim() || ''; + res.json({ reply }); + } catch (err) { + console.error('[llm] error:', err.message); + res.status(502).json({ error: err.message }); + } +}); + +app.get('/api/status', (req, res) => { + res.json({ + connected: jibo.connected, + sessionID: jibo.sessionID, + angles: jibo.currentAngles, + videoStreamActive: jibo.videoStreamActive + }); +}); + +// Proxy Jibo's video/photo byte streams through the server +app.get('/proxy/stream', (req, res) => { + const { uri } = req.query; + if (!uri || !uri.startsWith('/')) return res.status(400).json({ error: 'invalid uri' }); + proxyJiboStream(uri, res); +}); + +app.get('/proxy/photo', (req, res) => { + const { uri } = req.query; + if (!uri || !uri.startsWith('/')) return res.status(400).json({ error: 'invalid uri' }); + proxyJiboFetch(uri, res); +}); + +// ── HTTP + WebSocket server ─────────────────────────────────────────────────── + +const server = http.createServer(app); + +const wss = new WebSocketServer({ server, path: '/ws' }); +wss.on('connection', (ws) => { + jibo.addSubscriber(ws); + ws.on('close', () => jibo.removeSubscriber(ws)); + ws.on('error', () => jibo.removeSubscriber(ws)); +}); + +server.listen(APP_PORT, () => { + console.log(`Re-Commander running at http://localhost:${APP_PORT}`); + jibo.connect(); + new WakewordWatcher((msg) => jibo._broadcastToClients(msg)); +});