From 5617886ebed9970e8288d98a626cffd05c7e4021 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 24 Mar 2026 02:56:27 +0200 Subject: [PATCH] Fixed the ai server & disabled the old ai-bridge BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP BACKFLIP --- V3.1/build/hub-shim/index.js | 174 ++++++++++++------ .../Skills/@be/be/be/ai-bridge-config.json | 6 +- .../jibo/Jibo/Skills/@be/be/be/ai-bridge.js | 6 + .../jibo/Jibo/Skills/@be/be/be/postinit.js | 1 + .../jetstream-client/lib/jetstream-client.js | 4 + .../@be/be/node_modules/jibo/lib/jibo.js | 9 + .../usr/local/etc/jibo-system-manager.json | 4 +- 7 files changed, 143 insertions(+), 61 deletions(-) diff --git a/V3.1/build/hub-shim/index.js b/V3.1/build/hub-shim/index.js index fedf5631..1f5fbbf0 100644 --- a/V3.1/build/hub-shim/index.js +++ b/V3.1/build/hub-shim/index.js @@ -111,42 +111,88 @@ function hasRule(rules, want) { } function inferNluFromText(text, rules) { - // Minimal, rule-ish: only try to satisfy common skills. - const yn = classifyYesNo(text); - if (yn) { - // Many flows use the intent string as grammar tag. - return buildNluResult(yn, rules, {}); - } - - // If Jetstream provided tutorial/global rules, prefer returning the rule itself as intent. - // This matches flows that transition on a specific grammar tag. - if (Array.isArray(rules)) { - const t = normalizeText(text).toLowerCase(); - // Many builds pass short rule names (e.g., 'launch', 'dance') instead of full paths. - if (hasRule(rules, 'dance') && /\bdance\b/.test(t)) return buildNluResult('dance', rules, {}); - if (hasRule(rules, 'take_photo') && /\b(photo|picture|take a photo|take a picture)\b/.test(t)) return buildNluResult('take_photo', rules, {}); - // Launch/global command grammar: some builds only provide 'launch' + globals/global_commands_launch - // for many commands. When the utterance clearly matches a known command, emit that intent so flows - // do not treat it as a generic launch request. - if (hasRule(rules, 'launch')) { - // Many global command handlers expect a launch intent with a target skill in entities.skill. - if (/\bdance\b/.test(t)) return buildNluResult('launch', rules, { skill: 'dance', query: normalizeText(text) }, ['skill']); - if (/\b(photo|picture|take a photo|take a picture|selfie)\b/.test(t)) return buildNluResult('launch', rules, { skill: 'photobooth', query: normalizeText(text) }, ['skill']); - // Otherwise: route to chitchat so the robot actually speaks an answer. - // We keep intent='launch' but provide a concrete skill target. - return buildNluResult('launch', rules, { skill: '@be/chitchat', domain: 'chitchat', query: normalizeText(text) }, ['skill']); - } - } - - // Simple tutorial-ish intents. const t = normalizeText(text).toLowerCase(); - if (/\b(dance|do a dance)\b/.test(t)) return buildNluResult('dance', rules, {}); - if (/\b(photo|picture|take a photo|take a picture)\b/.test(t)) return buildNluResult('take_photo', rules, {}); - // If rules indicate yes/no, but we couldn't classify, mark noMatch. - if (Array.isArray(rules) && rules.some((r) => /yes[_-]?no/i.test(r))) { + // No text → empty intent (ListenResultState.noInput → Mim re-prompts or times out). + if (!t) return buildNluResult('', rules, {}); + + // Yes/No detection — common across many MIM types. + const yn = classifyYesNo(text); + + if (!Array.isArray(rules) || !rules.length) { + return yn ? buildNluResult(yn, rules, {}) : buildNluResult('', rules, {}); + } + + // ── Skill-specific rule matching (checked BEFORE global catch-all) ── + + // introductions/recognition_type_menu: expects face, name, voice, all + if (hasRule(rules, 'recognition_type_menu')) { + if (/\bface\b/.test(t)) return buildNluResult('face', rules, {}); + if (/\bname\b/.test(t)) return buildNluResult('name', rules, {}); + if (/\bvoice\b/.test(t)) return buildNluResult('voice', rules, {}); + if (/\b(all|everything|everyone)\b/.test(t)) return buildNluResult('all', rules, {}); + if (yn) return buildNluResult(yn, rules, {}); return buildNluResult('', rules, {}); } + + // introductions/voice_face_training_menu: similar recognition type choices + if (hasRule(rules, 'voice_face_training_menu')) { + if (/\bface\b/.test(t)) return buildNluResult('face', rules, {}); + if (/\bname\b/.test(t)) return buildNluResult('name', rules, {}); + if (/\bvoice\b/.test(t)) return buildNluResult('voice', rules, {}); + if (/\b(all|everything)\b/.test(t)) return buildNluResult('all', rules, {}); + if (yn) return buildNluResult(yn, rules, {}); + return buildNluResult('', rules, {}); + } + + // introductions yes/no questions: face_capture_ready, did_i_hear_name, + // did_i_pronounce_name, any_more_intros, recognition_any_more + if (rules.some((r) => /face_capture|did_i_hear|did_i_pronounce|any_more|recognition_any_more/.test(r))) { + if (yn) return buildNluResult(yn, rules, {}); + return buildNluResult('', rules, {}); + } + + // introductions/intro_looper: expects loopmember intent with loopMemberReferent entity. + // Without real NLU we cannot resolve the entity → return noMatch so the MIM re-prompts. + if (hasRule(rules, 'intro_looper')) { + if (yn) return buildNluResult(yn, rules, {}); + return buildNluResult('', rules, {}); + } + + // main-menu/execute_main_menu: map spoken words to menu items. + if (hasRule(rules, 'execute_main_menu')) { + if (/\bintroduc/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'introductions' }); + if (/\bsurprise/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'surprise-me' }); + if (/\b(time|clock)\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'clock' }); + if (/\bphoto\s*booth\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'photobooth' }); + if (/\bgallery\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'gallery' }); + if (/\b(exercise|workout)\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'exercise' }); + if (/\b(radio|music)\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'radio' }); + if (/\bsettings?\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'settings' }); + if (/\btips?\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'tips-tricks' }); + if (/\bfun\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'fun-stuff' }); + if (/\bcreate\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'create' }); + if (/\b(report|personal)\b/.test(t)) return buildNluResult('loadMenu', rules, { loadMenu: 'personal-report' }); + // Fall through to generic patterns below. + } + + // ── Generic patterns ── + + // Yes/No (applies to any MIM type that accepts it) + if (yn) return buildNluResult(yn, rules, {}); + + // Dance / Photo (tutorial rules) + if (hasRule(rules, 'dance') && /\bdance\b/.test(t)) return buildNluResult('dance', rules, {}); + if (hasRule(rules, 'take_photo') && /\b(photo|picture|take a photo|take a picture)\b/.test(t)) return buildNluResult('take_photo', rules, {}); + + // Global launch commands — only for specific well-known commands. + // Do NOT catch-all to chitchat; that breaks in-skill NLU. + if (hasRule(rules, 'launch')) { + if (/\bdance\b/.test(t)) return buildNluResult('launch', rules, { skill: 'dance', query: normalizeText(text) }, ['skill']); + if (/\b(photo|picture|take a photo|take a picture|selfie)\b/.test(t)) return buildNluResult('launch', rules, { skill: 'photobooth', query: normalizeText(text) }, ['skill']); + } + + // Default: no match — lets the Mim framework re-prompt or handle noMatch. return buildNluResult('', rules, {}); } @@ -588,6 +634,7 @@ function createHubShim(configPath) { let binaryBytes = 0; let lastContext = null; + let lastContextMsg = null; let pendingListen = null; async function maybeHandleListen() { @@ -600,25 +647,24 @@ function createHubShim(configPath) { if (mode === 'CLIENT_NLU' && !listenMsg._clientNlu) return; pendingListen = null; - const t0 = nowMs(); - const base = { - msgID: uuid(), - ts: nowMs(), - }; + // Derive transID: echo back the transID the jetstream-service sent. + // It may appear on either the LISTEN or CONTEXT message. + const msgTransID = listenMsg.transID || (lastContextMsg && lastContextMsg.transID) || transID || ''; - // Emit SOS immediately so the robot transitions into listening. - send(ws, { ...base, type: 'SOS', data: null, final: false }); + const t0 = nowMs(); let text = ''; try { if (mode === 'CLIENT_ASR') { text = normalizeText(listenMsg._clientAsrText); + } else if (mode === 'CLIENT_NLU') { + // No ASR needed for client-supplied NLU. } else { text = await runAsrQueued(() => asrServiceSttOnce(asrBaseUrl, wsPath, timeoutMs, audioSourceId, logger)); } } catch (e) { logger.warn('asr failed', { connId, err: String(e && (e.stack || e.message || e)) }); - // Still emit EOS and an empty listen result. + // Still send an empty listen result. } const rules = Array.isArray(listenMsg?.data?.rules) ? listenMsg.data.rules : []; @@ -628,8 +674,16 @@ function createHubShim(configPath) { : ((config?.nlu?.enabled === false) ? buildNluResult('', rules, {}) : inferNluFromText(text, rules)); const asrRes = buildAsrResult(text); + // Build the match object (mirrors what the cloud hub returns). + const skillID = (lastContext && lastContext.skill && lastContext.skill.skillID) + ? lastContext.skill.skillID + : (lastContext && typeof lastContext.skill === 'string' ? lastContext.skill : ''); + const matchObj = { onRobot: true }; + if (skillID) matchObj.skillID = skillID; + logger.info('listen result', { connId, + transID: msgTransID || undefined, text: String(text || '').slice(0, 120), intent: nluRes && nluRes.intent, slot0: nluRes && Array.isArray(nluRes.slotActions) ? nluRes.slotActions[0] : undefined, @@ -639,26 +693,25 @@ function createHubShim(configPath) { rules: Array.isArray(rules) ? rules.slice(0, 6) : [], }); - // Optionally provide incremental ASR/NLU; Jetstream consumers often listen for these. - send(ws, { ...base, type: 'ASR', data: asrRes, final: false }); - send(ws, { ...base, type: 'NLU', data: nluRes, final: false }); - - // Final listen response. - const listenResp = { - type: 'LISTEN', - msgID: uuid(), + // Send a local TURN_RESULT (not just LISTEN) so the skill's local turn resolves. + const turnResult = { + type: 'TURN_RESULT', + msgID: listenMsg.msgID || uuid(), + transID: msgTransID, ts: nowMs(), + requestID: msgTransID, // local turn uses transID as requestID data: { - asr: asrRes, - nlu: nluRes, + status: 'SUCCEEDED', + global: false, + result: { + asr: asrRes, + nlu: nluRes, + match: matchObj, + }, }, final: true, - timings: { total: nowMs() - t0 }, }; - send(ws, listenResp); - - // Emit EOS to complete the listen lifecycle. - send(ws, { ...base, type: 'EOS', data: null, final: true }); + send(ws, turnResult); } ws.on('message', async (data, isBinary) => { @@ -690,8 +743,10 @@ function createHubShim(configPath) { switch (msg.type) { case 'CONTEXT': lastContext = msg.data; + lastContextMsg = msg; logger.debug('context', { connId, + transID: msg.transID || undefined, hasSkill: !!(msg.data && msg.data.skill), hasRuntime: !!(msg.data && msg.data.runtime), }); @@ -700,6 +755,7 @@ function createHubShim(configPath) { pendingListen = msg; logger.debug('listen req', { connId, + transID: msg.transID || undefined, hotphrase: !!(msg.data && msg.data.hotphrase), mode: msg.data && msg.data.mode, rules: Array.isArray(msg.data && msg.data.rules) ? msg.data.rules : [], @@ -707,11 +763,15 @@ function createHubShim(configPath) { break; case 'CLIENT_ASR': // Accept client-provided text (requires a LISTEN message too). - pendingListen = pendingListen || { type: 'LISTEN', msgID: uuid(), ts: nowMs(), data: { rules: [], mode: 'CLIENT_ASR' } }; + if (!pendingListen) { + pendingListen = { type: 'LISTEN', msgID: uuid(), transID: msg.transID, ts: nowMs(), data: { rules: [], mode: 'CLIENT_ASR' } }; + } pendingListen._clientAsrText = msg.data?.text; break; case 'CLIENT_NLU': - pendingListen = pendingListen || { type: 'LISTEN', msgID: uuid(), ts: nowMs(), data: { rules: [], mode: 'CLIENT_NLU' } }; + if (!pendingListen) { + pendingListen = { type: 'LISTEN', msgID: uuid(), transID: msg.transID, ts: nowMs(), data: { rules: [], mode: 'CLIENT_NLU' } }; + } pendingListen._clientNlu = msg.data; break; default: diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json index 60c61ccd..54555848 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json @@ -6,7 +6,7 @@ "recordSeconds": 5, "useDumpStateAudio": true, - "useAsrServiceStt": true, + "useAsrServiceStt": false, "asrServiceHost": "127.0.0.1", "asrServicePort": 8088, "asrAudioSourceId": "alsa1", @@ -28,8 +28,8 @@ "suppressWakeGreetings": true, - "jetstreamOfflineFallbackEnabled": true, - "jetstreamInjectOnHjHeard": true, + "jetstreamOfflineFallbackEnabled": false, + "jetstreamInjectOnHjHeard": false, "aiForwardingOnlyAllowedSkills": true, "aiForwardingAllowedSkills": ["@be/main-menu", "@be/idle"], diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js index 27f9c217..7487c18f 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js @@ -3122,6 +3122,12 @@ AIBridge.prototype.setupTunables = function () { }; AIBridge.prototype.start = function () { + // ── DISABLED: skip all hooks/listeners to eliminate interference ── + if (rlog) { + rlog.info("ai-bridge", "start() SKIPPED – ai-bridge disabled for hub-shim debugging", {}); + } + return; + // ── end disabled block ── try { this._loadConfig(); } catch (e0) { diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js index c9641c1b..11d21e69 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js @@ -4,6 +4,7 @@ const jibo = require('jibo'); let rlog = null; try { rlog = require('./robot-logger'); + if (rlog) global.__rlog = rlog; } catch (e) { // ignore } diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/@jibo/jetstream-client/lib/jetstream-client.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/@jibo/jetstream-client/lib/jetstream-client.js index f777f344..e922f090 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/@jibo/jetstream-client/lib/jetstream-client.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/@jibo/jetstream-client/lib/jetstream-client.js @@ -319,6 +319,7 @@ class Client { } break; case types.ServiceEventType.TURN_RESULT: + try { global.__rlog.info('js-client', 'TURN_RESULT raw', {status:event.data.status,resultType:typeof event.data.result,requestID:event.requestID,transID:event.transID,global:!!event.data.global}); } catch(_e){} if (event.data.status === types.TurnResultType.SUCCEEDED && typeof event.data.result === 'string') { event.data.result = JSON.parse(event.data.result); } @@ -326,6 +327,7 @@ class Client { if (result && 'asr' in result) { event.data.result = new types.ListenResult(result.asr, result.nlu, result.match); } + try { global.__rlog.info('js-client', 'TURN_RESULT parsed', {intent:event.data.result&&event.data.result.intent,text:event.data.result&&event.data.result.text,state:event.data.result&&event.data.result.state,nluIntent:result&&result.nlu&&result.nlu.intent,nluRules:result&&result.nlu&&result.nlu.rules}); } catch(_e){} event.data.transID = event.transID; if (event.data.global || event.requestID === types.GLOBAL_REQUEST) { const data = event.data; @@ -377,6 +379,7 @@ class Client { } if (shouldPassToRequest) { const request = this._requests.get(event.requestID); + try { global.__rlog.info('js-client', 'passToRequest', {type:event.type,requestID:event.requestID,found:!!request,isGlobal:event.requestID===types.GLOBAL_REQUEST}); } catch(_e){} if (request) { if (event.type === types.ServiceEventType.ERROR) { request.error.emit(new Error(`Received error: ${event.data.message}`)); @@ -387,6 +390,7 @@ class Client { } else { if (event.requestID !== types.GLOBAL_REQUEST) { + try { global.__rlog.warn('js-client', 'request NOT FOUND for requestID', {requestID:event.requestID,type:event.type,mapSize:this._requests.size}); } catch(_e){} } } } diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo/lib/jibo.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo/lib/jibo.js index e2dcea77..d7dc239f 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo/lib/jibo.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo/lib/jibo.js @@ -2032,6 +2032,7 @@ class Mim extends Behavior_1.default { listen.failedToGetListener = false; listen.listener.on(ListenEvent_1.default.FINISHED, () => { if (this.status !== Status_1.default.IN_PROGRESS) { + try { global.__rlog.warn('mim', 'FINISHED but status!=IN_PROGRESS', {status:this.status}); } catch(_e){} return; } listen.listener = null; @@ -2039,25 +2040,31 @@ class Mim extends Behavior_1.default { if (!this.asrResults) { this.asrResults = new jetstream_client_1.types.ListenResult(null); } + try { global.__rlog.info('mim', 'FINISHED routing', {state:this.asrResults.state,intent:this.asrResults.intent,text:this.asrResults.text,domain:this.asrResults.entities&&this.asrResults.entities.domain,elapsed:Date.now()-listen.startTime,timeout:listen.timeout}); } catch(_e){} if (this.checkResult) { this.checkResult(this.asrResults); } if (this.asrResults.state === jetstream_client_1.types.ListenResultState.noInput && Date.now() - listen.startTime < listen.timeout) { + try { global.__rlog.info('mim', 'FINISHED → restartListen (noInput, under timeout)'); } catch(_e){} this.states.listen.transitionTo(this.states.restartListen); } else if (this.asrResults.entities.domain === 'mim_global') { + try { global.__rlog.info('mim', 'FINISHED → analyzeMimGlobal'); } catch(_e){} this.states.listen.transitionTo(this.states.analyzeMimGlobal); } else if (this.asrResults && (this.asrResults.entities.domain === 'gui_command' || this.asrResults.entities.domain === 'menu_global')) { + try { global.__rlog.info('mim', 'FINISHED → analyzeMenuGlobal'); } catch(_e){} this.states.listen.transitionTo(this.states.analyzeMenuGlobal); } else { + try { global.__rlog.info('mim', 'FINISHED → analyzeResults', {isNoInput:this.asrResults.state===jetstream_client_1.types.ListenResultState.noInput}); } catch(_e){} this.states.listen.transitionTo(this.states.analyzeResults, this.asrResults.state === jetstream_client_1.types.ListenResultState.noInput); } } }); listen.listener.on(ListenEvent_1.default.CLOUD, (asrResultsData) => { + try { global.__rlog.info('mim', 'CLOUD event', {status:asrResultsData.status,intent:asrResultsData.result&&asrResultsData.result.intent,text:asrResultsData.result&&asrResultsData.result.text,state:asrResultsData.result&&asrResultsData.result.state,nlu:asrResultsData.result&&asrResultsData.result.nlu}); } catch(_e){} this.asrResults = asrResultsData.result; this.exitInputType = analytics.INPUT.SPEECH; }); @@ -5118,6 +5125,7 @@ class GlobalListener extends events_1.EventEmitter { Runtime_1.default.instance.jetstream.events.localTurnStarted.emit(); try { const data = yield this.turn.promise; + try { global.__rlog.info('mim', 'turn resolved', {status:data.status,hasResult:!!data.result,intent:data.result&&data.result.intent,text:data.result&&data.result.text,state:data.result&&data.result.state}); } catch(_e){} if (this.stopped) { return; } @@ -6496,6 +6504,7 @@ class FlowMim extends ActivityImplementation { firstGrammarTag = result.firstGrammarTag = result.asrResults.intent; result.grammarResults = result.asrResults.entities; } + try { global.__rlog.info('mim', 'FlowMim onSuccess', {activityClass:this.activityClass,firstGrammarTag:firstGrammarTag,intent:result.asrResults&&result.asrResults.intent,text:result.asrResults&&result.asrResults.text,state:result.asrResults&&result.asrResults.state,entities:result.asrResults&&result.asrResults.entities}); } catch(_e){} let transition = options.innerOnSuccess(result); if (transition === undefined) { transition = firstGrammarTag; diff --git a/V3.1/build/usr/local/etc/jibo-system-manager.json b/V3.1/build/usr/local/etc/jibo-system-manager.json index 2407c4ec..d862af3e 100644 --- a/V3.1/build/usr/local/etc/jibo-system-manager.json +++ b/V3.1/build/usr/local/etc/jibo-system-manager.json @@ -504,7 +504,9 @@ "environment": { "DISPLAY": ":0", "XAUTHORITY": "/tmp/.Xauthority", - "XDG_CONFIG_HOME": "/opt/home/jibo-skill/.config" + "XDG_CONFIG_HOME": "/opt/home/jibo-skill/.config", + "JIBO_HUB_SHIM_HOST": "192.168.1.28", + "JIBO_GQA_ENDPOINT": "http://192.168.1.28:8080" }, "path": { "jibo": [