'use strict'; // ── WebSocket to server ────────────────────────────────────────────────────── let ws; let connected = false; let sessionActive = false; let currentAngles = [0, 0]; let lastSayTx = null; let lastListenTx = null; let videoActive = false; let entities = {}; // entityId → track data function connectWS() { const proto = location.protocol === 'https:' ? 'wss' : 'ws'; ws = new WebSocket(`${proto}://${location.host}/ws`); ws.onopen = () => { connected = true; }; ws.onclose = () => { connected = false; setStatus(false, 'Disconnected — reconnecting…'); setTimeout(connectWS, 2000); }; ws.onerror = () => {}; ws.onmessage = (e) => { let msg; try { msg = JSON.parse(e.data); } catch { return; } handleServerMessage(msg); }; } function handleServerMessage(msg) { if (msg.type === 'status') { sessionActive = !!msg.sessionID; currentAngles = msg.angles || [0, 0]; setStatus(msg.connected && sessionActive, msg.connected ? (sessionActive ? 'Connected • ' + msg.sessionID.slice(0, 8) + '…' : 'Connecting…') : 'Disconnected'); return; } if (msg.type === 'jiboEvent') { handleJiboEvent(msg.body, msg.txId); } } function handleJiboEvent(body, txId) { if (!body) return; const evt = body.Event || body.ResponseString || '?'; if (evt === 'onConfig') return; // heartbeat GetConfig response — suppress from log logEvent(evt, body, txId); switch (body.Event) { case 'onHotWordHeard': flashHotword(body.utterance || 'hey jibo', body.score); if (document.getElementById('auto-listen-toggle').checked) post('/api/interrupt').then(() => doListen()); break; case 'onStart': if (txId === lastListenTx) { clearListenTimeout(); document.getElementById('listen-result').textContent = '(listening…)'; } break; case 'onListenResult': { clearListenTimeout(); const speech = body.Speech || ''; document.getElementById('listen-result').textContent = '"' + speech + '"'; if (speech && document.getElementById('llm-toggle').checked) runLLMLoop(speech); break; } case 'onPhotoSaved': if (body.url) addPhoto(body.url); break; case 'onVideoReady': if (body.URI) startVideoFeed(body.URI); break; case 'onEntityGained': case 'onEntityUpdate': if (body.Tracks) body.Tracks.forEach(t => { entities[t.EntityID] = t; }); renderEntities(); break; case 'onEntityLost': if (body.Tracks) body.Tracks.forEach(t => { delete entities[t.EntityID]; }); renderEntities(); break; case 'onHeadTouch': if (body.Pads) renderHeadTouch(body.Pads); break; case 'onLookAtAchieved': break; case 'onStop': if (txId === lastListenTx) { clearListenTimeout(); const reason = body.StopReason || body.ListenStopReason || 'stopped'; document.getElementById('listen-result').textContent = '(stopped: ' + reason + ')'; } break; case 'onError': if (txId === lastListenTx) { clearListenTimeout(); const errStr = body.EventError?.ErrorString || body.ErrorString || 'unknown error'; document.getElementById('listen-result').textContent = '(error: ' + errStr + ')'; } break; } } // ── REST helpers ───────────────────────────────────────────────────────────── async function api(method, path, body) { try { const opts = { method, headers: { 'Content-Type': 'application/json' } }; if (body) opts.body = JSON.stringify(body); const res = await fetch(path, opts); return await res.json(); } catch (err) { logEvent('api-error: ' + path, { error: err.message }, null); } } const post = (path, body) => api('POST', path, body); const get = (path) => api('GET', path); // ── Status ─────────────────────────────────────────────────────────────────── function setStatus(ok, label) { const dot = document.getElementById('status-dot'); const lbl = document.getElementById('status-label'); dot.className = ok ? 'ok' : ''; lbl.textContent = label; } // ── Arrow pad ──────────────────────────────────────────────────────────────── // ── Directional step ────────────────────────────────────────────────────────── // Left/right: angle nudge (preserves psi exactly, no vertical drift). // Up/down: screen-coord click 30% from center (same math as click-to-look). // Both endpoints block server-side until the robot acks, so the loop has // exactly one command in-flight at all times. Gen counter kills a stale loop // on direction change after at most one more in-flight step completes. const STEP_FRAC = 0.30; let moveGen = 0; let moveHeld = false; // Horizontal steps use pctY slightly above true center to compensate for // a small downward bias in the camera's optical axis vs the head neutral. // Tune HORIZ_CENTER_Y if left/right still drifts up or down. const HORIZ_CENTER_Y = 0.38; async function runMoveLoop(dir, gen) { while (moveHeld && gen === moveGen) { let pctX, pctY; if (dir === 'left') { pctX = 0.5 - STEP_FRAC; pctY = HORIZ_CENTER_Y; } else if (dir === 'right') { pctX = 0.5 + STEP_FRAC; pctY = HORIZ_CENTER_Y; } else if (dir === 'up') { pctX = 0.5; pctY = 0.5 - STEP_FRAC; } else { pctX = 0.5; pctY = 0.5 + STEP_FRAC; } const x = Math.round((1 - pctX) * 640); const y = Math.round(pctY * 480); await post('/api/look/step', { x, y }); } } function startMove(dir) { moveHeld = true; runMoveLoop(dir, ++moveGen); } function stopMove() { moveHeld = false; } function addMoveButton(id, dir) { const btn = document.getElementById(id); btn.addEventListener('pointerdown', (e) => { e.preventDefault(); startMove(dir); }); btn.addEventListener('pointerup', stopMove); btn.addEventListener('pointercancel', stopMove); btn.addEventListener('pointerleave', stopMove); } addMoveButton('btn-up', 'up'); addMoveButton('btn-down', 'down'); addMoveButton('btn-left', 'left'); addMoveButton('btn-right', 'right'); // Keyboard arrow keys — browser key-repeat ignored; our await loop is the repeat const KEY_TO_DIR = { ArrowLeft: 'left', ArrowRight: 'right', ArrowUp: 'up', ArrowDown: 'down' }; const heldKeys = new Set(); document.addEventListener('keydown', (e) => { if (['INPUT', 'TEXTAREA', 'SELECT'].includes(document.activeElement.tagName)) return; if (e.key === ' ') { e.preventDefault(); post('/api/look/angle', { theta: 0, psi: 0 }); return; } const dir = KEY_TO_DIR[e.key]; if (!dir || e.repeat || heldKeys.has(e.key)) return; e.preventDefault(); heldKeys.add(e.key); startMove(dir); }); document.addEventListener('keyup', (e) => { if (!KEY_TO_DIR[e.key]) return; heldKeys.delete(e.key); stopMove(); }); // ── Click-to-look ───────────────────────────────────────────────────────────── const cameraWrap = document.getElementById('camera-wrap'); const clickDot = document.getElementById('click-dot'); cameraWrap.addEventListener('click', (e) => { const rect = cameraWrap.getBoundingClientRect(); const x = e.clientX - rect.left; const y = e.clientY - rect.top; const pctX = x / rect.width; const pctY = y / rect.height; // Show click indicator clickDot.style.left = x + 'px'; clickDot.style.top = y + 'px'; clickDot.style.display = 'block'; setTimeout(() => { clickDot.style.display = 'none'; }, 800); // Mirror X to match the horizontally-flipped display const camX = Math.round((1 - pctX) * 640); const camY = Math.round(pctY * 480); const track = document.getElementById('track-flag').checked; post('/api/look/screen', { x: camX, y: camY, track }); }); // ── Say ─────────────────────────────────────────────────────────────────────── document.getElementById('btn-say').addEventListener('click', async () => { const text = document.getElementById('say-text').value.trim(); if (!text) return; const r = await post('/api/say', { text }); if (r) lastSayTx = r.txId; }); document.getElementById('btn-say-cancel').addEventListener('click', () => { post('/api/say/cancel'); }); // ── Listen ──────────────────────────────────────────────────────────────────── let listenClientTimer = null; function clearListenTimeout() { if (listenClientTimer) { clearTimeout(listenClientTimer); listenClientTimer = null; } } async function doListen() { clearListenTimeout(); document.getElementById('listen-result').textContent = '(waiting for robot…)'; const maxNoSpeech = parseInt(document.getElementById('listen-max-nosp').value) || 5000; const maxSpeech = parseInt(document.getElementById('listen-max-speech').value) || 10000; const r = await post('/api/listen', { maxSpeech, maxNoSpeech }); if (r) { lastListenTx = r.txId; const clientTimeout = Math.max(maxSpeech, maxNoSpeech) + 16000; listenClientTimer = setTimeout(() => { document.getElementById('listen-result').textContent = '(timed out — cloud ASR may be unavailable)'; }, clientTimeout); } } document.getElementById('btn-listen').addEventListener('click', doListen); document.getElementById('btn-listen-cancel').addEventListener('click', () => { clearListenTimeout(); post('/api/listen/cancel'); document.getElementById('listen-result').textContent = '(cancelled)'; }); // ── Auto-listen + Voice AI ──────────────────────────────────────────────────── let llmHistory = []; // [{role:'user'|'assistant', content:string}] let llmSessionMode = false; // true when server uses LLM_SESSION_KEY (OpenClaw session) let llmTurnCount = 0; function llmStatus(msg) { document.getElementById('llm-status').textContent = msg; } async function runLLMLoop(speechText) { // In session mode send only the latest message — history lives on the server. // In history mode accumulate the full thread and send it each time. const messages = llmSessionMode ? [{ role: 'user', content: speechText }] : [...llmHistory, { role: 'user', content: speechText }]; if (!llmSessionMode) llmHistory.push({ role: 'user', content: speechText }); llmStatus('Thinking…'); const endpoint = document.getElementById('llm-endpoint').value.trim(); const model = document.getElementById('llm-model').value.trim(); const systemPrompt = document.getElementById('llm-system-prompt').value.trim(); const r = await post('/api/llm/chat', { messages, endpoint: endpoint || undefined, model: model || undefined, systemPrompt: systemPrompt || undefined, }); if (!r || r.error) { if (r?.error === 'cancelled') { llmStatus('Interrupted.'); if (!llmSessionMode) llmHistory.pop(); return; } llmStatus('LLM error: ' + (r?.error || 'no response')); if (!llmSessionMode) llmHistory.pop(); return; } const reply = r.reply; if (!llmSessionMode) llmHistory.push({ role: 'assistant', content: reply }); llmTurnCount++; const modeTag = llmSessionMode ? 'session' : 'local'; llmStatus(`[${modeTag} · ${llmTurnCount} turns] Last: "${reply.slice(0, 50)}${reply.length > 50 ? '…' : ''}"`); // Fill say box so user can see what Jibo is about to say document.getElementById('say-text').value = reply; const sayResult = await post('/api/say', { text: reply }); // If the reply ends with a question and wasn't interrupted, listen for the user's answer const endsWithQuestion = /\?[^a-zA-Z0-9]*$/.test(reply.trim()); if (endsWithQuestion && !sayResult?.aborted && document.getElementById('llm-toggle').checked) { doListen(); } } document.getElementById('btn-llm-cancel').addEventListener('click', () => { post('/api/llm/cancel'); llmStatus('Cancelled.'); }); document.getElementById('btn-llm-clear').addEventListener('click', () => { llmHistory = []; llmTurnCount = 0; llmStatus(llmSessionMode ? 'Session turn counter reset (server session persists).' : 'Conversation cleared.'); }); // ── Attention ───────────────────────────────────────────────────────────────── document.querySelectorAll('.attn-btn').forEach(btn => { btn.addEventListener('click', function () { document.querySelectorAll('.attn-btn').forEach(b => b.classList.remove('active')); this.classList.add('active'); post('/api/attention', { mode: this.dataset.mode }); }); }); // ── Volume ──────────────────────────────────────────────────────────────────── document.getElementById('volume-slider').addEventListener('input', function () { document.getElementById('volume-label').textContent = Math.round(this.value * 100) + '%'; }); document.getElementById('btn-set-volume').addEventListener('click', () => { const level = parseFloat(document.getElementById('volume-slider').value); post('/api/volume', { level }); }); // ── Camera / Video ──────────────────────────────────────────────────────────── let videoTxId = null; document.getElementById('btn-video-start').addEventListener('click', async () => { const r = await post('/api/video/start', { duration: 0 }); if (r) videoTxId = r.txId; document.getElementById('video-status').textContent = 'Waiting for VideoReady…'; }); document.getElementById('btn-video-stop').addEventListener('click', () => { post('/api/video/stop', {}); stopVideoFeed(); }); function startVideoFeed(uri) { const feed = document.getElementById('camera-feed'); const noFeed = document.getElementById('camera-no-feed'); feed.src = '/proxy/stream?uri=' + encodeURIComponent(uri); feed.style.display = 'block'; noFeed.style.display = 'none'; document.getElementById('video-status').textContent = 'Streaming'; videoActive = true; feed.onerror = () => { stopVideoFeed(); document.getElementById('video-status').textContent = 'Stream error'; }; } function stopVideoFeed() { const feed = document.getElementById('camera-feed'); feed.src = ''; feed.style.display = 'none'; document.getElementById('camera-no-feed').style.display = ''; document.getElementById('video-status').textContent = ''; videoActive = false; } // ── Take Photo ──────────────────────────────────────────────────────────────── document.getElementById('btn-photo').addEventListener('click', () => { post('/api/photo', { camera: document.getElementById('photo-camera').value, resolution: document.getElementById('photo-res').value }); }); function addPhoto(url) { const strip = document.getElementById('photo-strip'); const img = document.createElement('img'); img.src = url; img.title = url; img.addEventListener('click', () => openPhotoModal(img.src)); strip.prepend(img); // Show photo in camera area if not streaming video if (!videoActive) { const feed = document.getElementById('camera-feed'); const noFeed = document.getElementById('camera-no-feed'); feed.src = img.src; feed.style.display = 'block'; noFeed.style.display = 'none'; } } // ── Photo modal ─────────────────────────────────────────────────────────────── function openPhotoModal(src) { document.getElementById('photo-modal-img').src = src; document.getElementById('photo-modal').classList.add('open'); } document.getElementById('photo-modal').addEventListener('click', (e) => { if (e.target === e.currentTarget || e.target.id === 'photo-modal-close') { document.getElementById('photo-modal').classList.remove('open'); } }); // ── Display controls ────────────────────────────────────────────────────────── document.getElementById('btn-eye').addEventListener('click', () => { post('/api/display/eye', {}); }); document.getElementById('btn-play-anim').addEventListener('click', () => { const name = document.getElementById('eye-anim-select').value; if (name) post('/api/display/anim', { name }); }); document.getElementById('btn-display-text').addEventListener('click', () => { const text = document.getElementById('display-text').value.trim(); if (!text) return; post('/api/display/text', { text }); }); document.getElementById('btn-display-image').addEventListener('click', () => { const src = document.getElementById('display-img-src').value.trim(); if (!src) return; post('/api/display/image', { src }); }); // ── Entities ────────────────────────────────────────────────────────────────── function renderEntities() { const list = document.getElementById('entity-list'); const ids = Object.keys(entities); if (ids.length === 0) { list.innerHTML = '