From 03b51ffd0cd4d85d0a299b3ae08d56ff3d5dcd15 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 24 Mar 2026 18:23:39 +0200 Subject: [PATCH] talking works & menus work plus ai documentation : ) --- V3.1/build/hub-shim/HUB-SHIM_DEPLOYMENT.md | 57 ++++++++++ V3.1/build/hub-shim/HUB-SHIM_PROTOCOL.md | 66 +++++++++++ .../Jibo/Skills/@be/be/be/BE-API_USAGE.md | 45 ++++++++ .../opt/jibo/Jibo/Skills/@be/be/be/be-api.js | 105 ++++++++++++++++++ .../jibo/Jibo/Skills/@be/be/be/postinit.js | 17 +++ .../opt/jibo/Jibo/Skills/@be/be/index.js | 4 + 6 files changed, 294 insertions(+) create mode 100644 V3.1/build/hub-shim/HUB-SHIM_DEPLOYMENT.md create mode 100644 V3.1/build/hub-shim/HUB-SHIM_PROTOCOL.md create mode 100644 V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/BE-API_USAGE.md create mode 100644 V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/be-api.js diff --git a/V3.1/build/hub-shim/HUB-SHIM_DEPLOYMENT.md b/V3.1/build/hub-shim/HUB-SHIM_DEPLOYMENT.md new file mode 100644 index 00000000..60559d0b --- /dev/null +++ b/V3.1/build/hub-shim/HUB-SHIM_DEPLOYMENT.md @@ -0,0 +1,57 @@ +# Jibo Hub-Shim Server: Deployment & Usage Guide + +## Quick Start +1. **Install dependencies** (if needed): + - Node.js v16+ (recommended v18+) + - `npm install` (if package.json present) +2. **Configure**: + - Edit `config.json` (or use `config.example.json`): + - `listen.port`: Port to listen on (default: 9000) + - `listen.path`: WebSocket path (default: `/v1/listen`) + - `asr.baseUrl`: Local ASR HTTP endpoint + - `logging.level`: `info`, `debug`, etc. +3. **Run the server**: + ```sh + node index.js config.json + ``` + - Or set `JIBO_HUB_SHIM_CONFIG` env var. + +## Integration with Jibo Robot +- The robot's `jibo-jetstream-service` binary must be configured to connect to the hub-shim: + - `hub_port`: 9000 (or your chosen port) + - `hub_hostname`: IP of the machine running hub-shim + - `listen_url`: `/v1/listen` +- Confirm with logs: hub-shim should show `ws connected` and `listen result` for each turn. + +## How It Works +- The robot opens a WebSocket to hub-shim for every listen turn. +- hub-shim receives CONTEXT and LISTEN messages, runs ASR/NLU, and sends a TURN_RESULT. +- The TURN_RESULT is always local (`global: false`), with `status: 'SUCCEEDED'` and the correct intent/entities. +- This resolves the skill's local turn and allows menu/intro flows to advance by voice. + +## Troubleshooting +- **No skill advancement?** + - Check that TURN_RESULT is sent with `status: 'SUCCEEDED'`, `global: false`, and correct `requestID` (should match transID). + - Use robot-logger (`/tmp/jibo-skills.log` or web panel) to trace intent and state. +- **ASR timeouts?** + - Ensure ASR service is running and reachable from hub-shim. +- **NLU not matching?** + - Edit `inferNluFromText()` in `index.js` to adjust rule patterns. + +## Advanced +- **Custom NLU**: You can extend `inferNluFromText()` for more complex intent/entity extraction. +- **Client-supplied ASR/NLU**: Send CLIENT_ASR or CLIENT_NLU messages before LISTEN to inject results. +- **Logging**: Adjust `logging.level` in config for more/less verbosity. + +## File Locations +- Main server: `index.js` +- Config: `config.json` or `config.example.json` +- Protocol doc: `HUB-SHIM_PROTOCOL.md` + +## Support +- For protocol or integration issues, see HUB-SHIM_PROTOCOL.md and the source code comments. +- For robot-side debugging, use robot-logger and check `/tmp/jibo-skills.log`. + +--- + +# Enjoy fully offline Jibo skills and menus! diff --git a/V3.1/build/hub-shim/HUB-SHIM_PROTOCOL.md b/V3.1/build/hub-shim/HUB-SHIM_PROTOCOL.md new file mode 100644 index 00000000..ac0ac321 --- /dev/null +++ b/V3.1/build/hub-shim/HUB-SHIM_PROTOCOL.md @@ -0,0 +1,66 @@ +# Jibo Hub-Shim Server: Architecture & Protocol + +## Overview +The hub-shim server emulates the Jibo cloud hub's `/v1/listen` WebSocket protocol, allowing the robot to run skills and menu flows entirely offline. It acts as a bridge between the robot's jetstream service (binary) and local ASR/NLU logic, providing the expected protocol and responses for seamless skill operation. + +## Key Features +- Listens on a configurable port (default: 9000) and path (`/v1/listen`) +- Accepts WebSocket connections from the robot's jetstream service +- Handles CONTEXT, LISTEN, CLIENT_ASR, and CLIENT_NLU messages +- Runs local ASR (via HTTP) and NLU (via rule-based inference) +- Sends TURN_RESULT responses that resolve the robot's local turn promise +- Fully mimics the cloud hub protocol, including transID/requestID echo + +## Protocol Flow +1. **Connection**: Jetstream binary opens a WebSocket to `/v1/listen`. +2. **CONTEXT**: Robot sends context (skill, runtime info). +3. **LISTEN**: Robot requests a listen turn, providing rules and mode. +4. **ASR**: hub-shim runs local ASR (or accepts client-supplied text). +5. **NLU**: hub-shim infers intent/entities from text and rules. +6. **TURN_RESULT**: hub-shim sends a TURN_RESULT with status `SUCCEEDED`, echoing the transID as requestID, and including asr/nlu/match objects. +7. **Skill Advances**: The robot's local turn promise resolves, and the skill/menu flow continues. + +## Message Types +- **CONTEXT**: Sets skill/runtime context for the turn. +- **LISTEN**: Initiates a listen turn; includes rules and mode. +- **CLIENT_ASR/CLIENT_NLU**: (Optional) Supplies ASR text or NLU result directly. +- **TURN_RESULT**: Final result; must have `status: 'SUCCEEDED'`, `global: false`, and correct asr/nlu/match. + +## Key Implementation Details +- **requestID**: Always set to the incoming transID for local turns. +- **asr/nlu/match**: Structured to match the robot's expected ListenResult format. +- **No global-only results**: All results are sent as local TURN_RESULTs to resolve the skill's local turn. +- **Skill-specific NLU**: Rule-based inference for menu/intro/yes-no flows. + +## Configuration +- `config.json` controls port, ASR service URL, and logging. +- ASR service must be running and reachable by hub-shim. + +## Debugging +- Logs all connections, requests, and results. +- Use robot-logger on the robot to trace skill flow and intent resolution. + +## Example TURN_RESULT +``` +{ + "type": "TURN_RESULT", + "msgID": "...", + "transID": "...", + "ts": 1234567890, + "requestID": "...", // matches transID + "data": { + "status": "SUCCEEDED", + "global": false, + "result": { + "asr": { "text": "face", "confidence": 1 }, + "nlu": { "intent": "face", ... }, + "match": { "onRobot": true } + } + }, + "final": true +} +``` + +--- + +# See also: hub-shim source code for full protocol details. diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/BE-API_USAGE.md b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/BE-API_USAGE.md new file mode 100644 index 00000000..274d33d1 --- /dev/null +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/BE-API_USAGE.md @@ -0,0 +1,45 @@ +# BE-API: External Eye/Lightring Trigger + +This API lets you trigger Jibo's eye (and, in the future, lightring) from any external tool or script via HTTP. + +## Endpoint + +- **URL:** `http://:15141/be-eye` +- **Method:** `POST` +- **Content-Type:** `application/json` + +## Actions + +### 1. Trigger Eye (Blue) +- **Request Body:** + ```json + { "action": "eye" } + ``` +- **Effect:** Forces Jibo's eye to blue (listening state). +- **Response:** + - `200 OK` with `{ "ok": true }` if successful + - `500` if failed + +### 2. (Planned) Trigger Lightring +- **Request Body:** + ```json + { "action": "lightring", "color": "blue" } + ``` +- **Effect:** (Not implemented yet) +- **Response:** + - `501` with `{ "ok": false, "msg": "not implemented" }` + +## Example: curl + +``` +curl -X POST -H 'Content-Type: application/json' \ + -d '{"action":"eye"}' \ + http://:15141/be-eye +``` + +## Logging +- All API activity is logged to the skills log and web log panel. + +## Notes +- The API server starts automatically with the BE skill. +- Lightring support is a placeholder for future hardware/API support. diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/be-api.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/be-api.js new file mode 100644 index 00000000..8bd5e1f2 --- /dev/null +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/be-api.js @@ -0,0 +1,105 @@ +// Simple HTTP API for BE skill to control eye and lightring +// Usage: POST /be-eye { "action": "eye" | "lightring", "color": "blue" } + +const http = require('http'); + +let jibo = null; +let beLogger = null; +let rlog = null; + +function setJiboRef(jiboInstance, logger, robotLogger) { + jibo = jiboInstance; + beLogger = logger || null; + try { + if (!robotLogger) { + robotLogger = require('./robot-logger'); + } + rlog = robotLogger; + } catch (e) { + rlog = null; + } +} + +function forceEyeView() { + if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE-API] forceEyeView called'); + if (jibo && jibo.loader && typeof jibo.loader.load === 'function') { + // This is the animation file loaded when "hey jibo" is triggered + const animPath = '/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo-anim-db-animations/animations/eye-globals/perceptual/hj-dsp-transition-to-pop-ns.keys'; + try { + jibo.loader.load({ + id: animPath, + cache: 'global', + options: { cache: 'global', dofs: {} }, + data: null, + src: animPath, + upload: true, + root: '/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo-anim-db-animations', + type: 'keys' + }); + if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE-API] forceEyeView animation loaded'); + return true; + } catch (e) { + if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE-API] forceEyeView animation load failed: ' + String(e && (e.stack || e.message || e))); + return false; + } + } + if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE-API] forceEyeView failed'); + return false; +} + +// TODO: Implement lightring control if API is available +function setLightring(color) { + // Placeholder: No direct API found + return false; +} + +const server = http.createServer((req, res) => { + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE-API] Incoming request ' + req.method + ' ' + req.url); + } else if (beLogger && typeof beLogger.info === 'function') { + beLogger.info('[BE-API] Incoming request', { method: req.method, url: req.url }); + } else { + console.log('[BE-API] Incoming request', req.method, req.url); + } + if (req.method === 'POST' && req.url === '/be-eye') { + let body = ''; + req.on('data', chunk => { body += chunk; }); + req.on('end', () => { + try { + const data = JSON.parse(body); + if (data.action === 'eye') { + const ok = forceEyeView(); + res.writeHead(ok ? 200 : 500); + res.end(JSON.stringify({ ok })); + } else if (data.action === 'lightring') { + const ok = setLightring(data.color); + res.writeHead(ok ? 200 : 501); + res.end(JSON.stringify({ ok, msg: ok ? 'ok' : 'not implemented' })); + } else { + res.writeHead(400); + res.end(JSON.stringify({ error: 'invalid action' })); + } + } catch (e) { + res.writeHead(400); + res.end(JSON.stringify({ error: 'bad json' })); + } + }); + } else { + res.writeHead(404); + res.end(); + } +}); + +function start(port = 15141) { + server.listen(port, () => { + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE-API] Listening on port ' + port); + } else if (beLogger && typeof beLogger.info === 'function') { + beLogger.info('[BE-API] Listening on port ' + port); + } else { + console.log('[BE-API] Listening on port', port); + } + }); +} + +module.exports = { setJiboRef, start }; diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js index 11d21e69..54d4f948 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js @@ -9,6 +9,8 @@ try { // ignore } +const beApi = require('./be-api'); + exports.postInit = function (err) { this.log.debug('postInit !!'); @@ -23,6 +25,21 @@ exports.postInit = function (err) { // ignore } + // Start BE HTTP API for eye/lightring control (always attempt, even if error) + try { + beApi.setJiboRef(jibo, this.log, rlog); + beApi.start(); + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE-API] HTTP server started'); + } else { + this.log.info('BE-API HTTP server started'); + } + } catch (e) { + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE-API] HTTP server failed to start: ' + String(e && (e.stack || e.message || e))); + } + this.log.warn('BE-API HTTP server failed to start', e); + } if (err) { this.log.error(err); this.initDoneCallback(err); diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/index.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/index.js index 01186907..a6e07109 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/index.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/index.js @@ -128,6 +128,10 @@ class Be { } catch (err) { this.log.warn('Error setting up listening for log level notifications', err); + // Explicitly call postInit on main BE skill instance to ensure BE-API server starts + if (typeof this.postInit === 'function') { + this.postInit(); + } } } if (this.packageInfo.jibo.debug.resourceLeak) {