diff --git a/V3.1/build/ai_bridge_server/README.md b/V3.1/build/ai_bridge_server/README.md new file mode 100644 index 00000000..8d82282e --- /dev/null +++ b/V3.1/build/ai_bridge_server/README.md @@ -0,0 +1,46 @@ +# Jibo AI Bridge Server + +This is a small companion server you run on your PC (same machine as Ollama). +It gives the robot a stable HTTP target and keeps the on-robot code modular. + +## Endpoints + +- `POST /v1/chat/text` JSON: `{ "text": "..." }` → `{ "reply": "..." }` +- `POST /v1/chat/audio` JSON: `{ "wav_base64": "..." }` → `{ "reply": "...", "text": "" }` + +## Requirements + +- Python 3.9+ +- Ollama running locally + - default Ollama chat URL: `http://127.0.0.1:11434/api/chat` + +Optional (for AUDIO mode): +- `faster-whisper` + `ffmpeg` + +## Run + +From this folder: + +- `python3 server.py --host 0.0.0.0 --port 8020` + +Environment variables (optional): + +- `OLLAMA_MODEL` (default `phi3.5`) +- `OLLAMA_URL` (default `http://127.0.0.1:11434/api/chat`) +- `WHISPER_MODEL` (default `base`) + +Note: Ollama can stay bound to `127.0.0.1:11434` on your PC; the robot only talks to this bridge server (`:8020`). + +Install optional STT deps: + +- `pip install faster-whisper` +- install `ffmpeg` (platform-specific) + +## Robot configuration + +Open the tunables UI (`http://:3333`) and set: + +- **Jibo AI Bridge → Server URL**: `http://:8020` +- **Jibo AI Bridge → Input**: + - `AUDIO` (records a short WAV clip on the robot and sends it) + - `TEXT` (uses `globalTurnResult` ASR text if available) diff --git a/V3.1/build/ai_bridge_server/__pycache__/server.cpython-313.pyc b/V3.1/build/ai_bridge_server/__pycache__/server.cpython-313.pyc new file mode 100644 index 00000000..20f2acd0 Binary files /dev/null and b/V3.1/build/ai_bridge_server/__pycache__/server.cpython-313.pyc differ diff --git a/V3.1/build/ai_bridge_server/jibo_last.wav b/V3.1/build/ai_bridge_server/jibo_last.wav new file mode 100644 index 00000000..48444193 Binary files /dev/null and b/V3.1/build/ai_bridge_server/jibo_last.wav differ diff --git a/V3.1/build/ai_bridge_server/server.py b/V3.1/build/ai_bridge_server/server.py new file mode 100644 index 00000000..421d5809 --- /dev/null +++ b/V3.1/build/ai_bridge_server/server.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Minimal AI Bridge server for Jibo. + +Endpoints: +- POST /v1/chat/text {"text": "..."} -> {"reply": "..."} +- POST /v1/chat/audio {"wav_base64": "...", "sample_rate": 16000} -> {"reply": "...", "text": ""} + +LLM: +- Uses Ollama Chat API by default: http://localhost:11434/api/chat + Env: + OLLAMA_URL (default: http://127.0.0.1:11434/api/chat) + OLLAMA_MODEL (default: phi3.5) + +STT (optional, for /audio): +- If `faster-whisper` is installed, it will be used. + Env: + WHISPER_MODEL (default: base) + +Run: + python3 server.py --host 0.0.0.0 --port 8020 +""" + +from __future__ import annotations + +import argparse +import array +import base64 +import io +import json +import os +import tempfile +import time +import traceback +import wave +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.request import Request, urlopen + + +def _ts() -> str: + # ISO-ish timestamp in local time; good enough for debugging + return time.strftime("%Y-%m-%d %H:%M:%S") + + +def _log(msg: str): + # Server previously muted logs; we want visibility while debugging. + print(f"[{_ts()}] {msg}", flush=True) + + +def _json_response(handler: BaseHTTPRequestHandler, status: int, payload: dict): + body = json.dumps(payload).encode("utf-8") + handler.send_response(status) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + +def _read_json(handler: BaseHTTPRequestHandler) -> dict: + length = int(handler.headers.get("Content-Length", "0")) + raw = handler.rfile.read(length) if length else b"{}" + return json.loads(raw.decode("utf-8")) + + +def _ollama_chat(user_text: str) -> str: + ollama_url = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434/api/chat") + model = os.environ.get("OLLAMA_MODEL", "phi3.5") + + req_body = { + "model": model, + "stream": False, + "messages": [ + {"role": "system", "content": "You are Jibo, a friendly home robot. Keep replies short and spoken."}, + {"role": "user", "content": user_text}, + ], + } + + req = Request( + ollama_url, + data=json.dumps(req_body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + + with urlopen(req, timeout=60) as resp: + data = json.loads(resp.read().decode("utf-8")) + + msg = data.get("message") or {} + content = msg.get("content") + if not content: + raise RuntimeError(f"Unexpected Ollama response: {data}") + return content.strip() + + +def _wav_diagnostics(wav_bytes: bytes) -> dict: + """Best-effort WAV parsing + signal stats for debugging mic capture.""" + info: dict = {"bytes": len(wav_bytes)} + try: + with wave.open(io.BytesIO(wav_bytes), "rb") as wf: # type: ignore[name-defined] + nch = wf.getnchannels() + sw = wf.getsampwidth() + fr = wf.getframerate() + nframes = wf.getnframes() + info.update({"channels": nch, "sample_width": sw, "frame_rate": fr, "frames": nframes}) + + # Read up to ~3 seconds of audio for stats (avoid huge CPU) + max_frames = min(nframes, fr * 3) + frames = wf.readframes(max_frames) + except Exception as e: + info["parse_error"] = str(e) + return info + + # Only compute stats for 16-bit PCM (most common). + if info.get("sample_width") != 2: + return info + + try: + samples = array.array("h") + samples.frombytes(frames) + if not samples: + return info + + mn = min(samples) + mx = max(samples) + zeros = sum(1 for s in samples if s == 0) + # RMS over interleaved samples (good enough for quick signal presence) + n = float(len(samples)) + rms = (sum(float(s) * float(s) for s in samples) / n) ** 0.5 + # Per-channel RMS (helps debug mic arrays) + ch = int(info.get("channels") or 1) + channel_rms = None + if ch > 1: + channel_rms = [] + for c in range(ch): + chan = samples[c::ch] + if not chan: + channel_rms.append(0.0) + else: + nn = float(len(chan)) + channel_rms.append((sum(float(s) * float(s) for s in chan) / nn) ** 0.5) + info.update( + { + "min": int(mn), + "max": int(mx), + "rms": float(rms), + "zero_frac": float(zeros) / n, + "channel_rms": channel_rms, + } + ) + except Exception as e: + info["stats_error"] = str(e) + return info + + +def _to_loudest_channel_mono_wav(wav_bytes: bytes) -> tuple[bytes, dict]: + """If WAV is multi-channel 16-bit PCM, pick loudest channel and return mono WAV bytes.""" + try: + with wave.open(io.BytesIO(wav_bytes), "rb") as wf: + nch = wf.getnchannels() + sw = wf.getsampwidth() + fr = wf.getframerate() + nframes = wf.getnframes() + frames = wf.readframes(nframes) + except Exception as e: + return wav_bytes, {"convert_error": str(e)} + + if nch <= 1 or sw != 2: + return wav_bytes, {"converted": False, "channels": nch, "sample_width": sw} + + samples = array.array("h") + samples.frombytes(frames) + if not samples: + return wav_bytes, {"converted": False, "reason": "empty_samples"} + + # Choose loudest channel by RMS + rms_list: list[float] = [] + for c in range(nch): + chan = samples[c::nch] + if not chan: + rms_list.append(0.0) + else: + nn = float(len(chan)) + rms_list.append((sum(float(s) * float(s) for s in chan) / nn) ** 0.5) + best = max(range(nch), key=lambda i: rms_list[i]) + mono = samples[best::nch] + + out = io.BytesIO() + with wave.open(out, "wb") as ow: + ow.setnchannels(1) + ow.setsampwidth(2) + ow.setframerate(fr) + ow.writeframes(mono.tobytes()) + + return out.getvalue(), {"converted": True, "picked_channel": best, "channel_rms": rms_list, "frame_rate": fr} + + +class _Whisper: + def __init__(self): + self._model = None + + def available(self) -> bool: + try: + import faster_whisper # noqa: F401 + + return True + except Exception: + return False + + def transcribe_wav_bytes(self, wav_bytes: bytes) -> str: + try: + from faster_whisper import WhisperModel + except Exception as e: + raise RuntimeError( + "Audio mode requires `faster-whisper` (pip install faster-whisper) and ffmpeg on your PC" + ) from e + + model_name = os.environ.get("WHISPER_MODEL", "base") + if self._model is None: + # CPU-friendly default; user can override via WHISPER_MODEL and faster-whisper params if needed. + self._model = WhisperModel(model_name, device="cpu", compute_type="int8") + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as f: + f.write(wav_bytes) + f.flush() + segments, info = self._model.transcribe(f.name) + text = "".join(seg.text for seg in segments).strip() + return text + + +_whisper = _Whisper() + + +class Handler(BaseHTTPRequestHandler): + server_version = "JiboAIBridge/1.0" + + def do_POST(self): + try: + client = f"{self.client_address[0]}:{self.client_address[1]}" + length = int(self.headers.get("Content-Length", "0") or "0") + _log(f"{client} POST {self.path} len={length}") + + if self.path == "/v1/chat/text": + payload = _read_json(self) + text = (payload.get("text") or "").strip() + if not text: + _json_response(self, 400, {"error": "Missing 'text'"}) + return + _log(f"{client} /text prompt_chars={len(text)} prompt={text[:200]!r}") + reply = _ollama_chat(text) + _log(f"{client} /text ok reply_chars={len(reply)}") + _json_response(self, 200, {"reply": reply}) + return + + if self.path == "/v1/chat/audio": + payload = _read_json(self) + b64 = payload.get("wav_base64") + if not b64: + _json_response(self, 400, {"error": "Missing 'wav_base64'"}) + return + + if not _whisper.available(): + _log(f"{client} /audio STT unavailable (faster-whisper not installed)") + _json_response( + self, + 503, + { + "error": "STT unavailable: install faster-whisper and ffmpeg on this PC", + "hint": "pip install faster-whisper (and install ffmpeg)", + }, + ) + return + + wav_bytes = base64.b64decode(b64) + diag = _wav_diagnostics(wav_bytes) + wav_for_stt, conv = _to_loudest_channel_mono_wav(wav_bytes) + if conv.get("converted"): + diag_mono = _wav_diagnostics(wav_for_stt) + _log( + f"{client} /audio wav_diag={json.dumps(diag, sort_keys=True)} " + f"mono={json.dumps(diag_mono, sort_keys=True)} conv={json.dumps(conv, sort_keys=True)}" + ) + else: + _log(f"{client} /audio wav_diag={json.dumps(diag, sort_keys=True)} conv={json.dumps(conv, sort_keys=True)}") + try: + # Save for debugging (overwrite each time) + out_path = os.environ.get("AI_BRIDGE_LAST_WAV", "jibo_last.wav") + with open(out_path, "wb") as f: + f.write(wav_bytes) + _log(f"{client} /audio decoded bytes={len(wav_bytes)} saved={out_path}") + except Exception as e: + _log(f"{client} /audio failed saving wav: {e}") + + transcript = _whisper.transcribe_wav_bytes(wav_for_stt) + if not transcript: + _log(f"{client} /audio empty transcript") + _json_response(self, 200, {"reply": "I didn't catch that. Could you say it again?", "text": ""}) + return + _log(f"{client} /audio transcript_chars={len(transcript)} transcript={transcript[:200]!r}") + reply = _ollama_chat(transcript) + _log(f"{client} /audio ok text_chars={len(transcript)} reply_chars={len(reply)}") + _json_response(self, 200, {"reply": reply, "text": transcript}) + return + + _json_response(self, 404, {"error": "Not found"}) + + except Exception as e: + _log(f"ERROR {self.path}: {e}\n{traceback.format_exc()}") + _json_response( + self, + 500, + { + "error": str(e), + "trace": traceback.format_exc(), + }, + ) + + def log_message(self, format, *args): + # Keep BaseHTTPRequestHandler from double-logging; we do our own. + return + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--host", default="0.0.0.0") + ap.add_argument("--port", type=int, default=8020) + args = ap.parse_args() + + server = ThreadingHTTPServer((args.host, args.port), Handler) + _log(f"AI Bridge server listening on http://{args.host}:{args.port}") + _log( + "Ollama: " + + os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434/api/chat") + + " model=" + + os.environ.get("OLLAMA_MODEL", "phi3.5") + ) + if not _whisper.available(): + _log("STT: faster-whisper not installed; /v1/chat/audio will return 503") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js new file mode 100644 index 00000000..5be1243a --- /dev/null +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js @@ -0,0 +1,1697 @@ +"use strict"; + +// Modular AI bridge for Jibo BE. +// - TEXT mode: uses jetstream turn results (ASR text) and forwards to a server. +// - AUDIO mode: records a short WAV clip (arecord) and forwards to a server. +// Server is expected to return JSON: { reply: "..." }. + +var http = require("http"); +var https = require("https"); +var urlLib = require("url"); +var fs = require("fs"); +var childProcess = require("child_process"); +var spawn = childProcess.spawn; + +var rlog = null; +try { + rlog = require("./robot-logger"); +} catch (e) { + // ignore +} + +var tunable = null; +try { + // Optional: drives the :3333 panel + tunable = require("jibo-tunable"); +} catch (e) { + // ignore +} + +var beFramework = null; +try { + beFramework = require("@be/be-framework"); +} catch (e) { + // ignore +} + +// Many builds already have Tunables under the "Jibo Embodied Dialog" window. +// Keeping AI bridge controls there makes them easier to find. +var WIN = "Jibo Embodied Dialog"; +var LABEL = { + enabled: "AI Bridge: Enabled", + input: "AI Bridge: Input", + audioCheckbox: "AI Bridge: Audio input (off = text)", + serverUrl: "AI Bridge: Server URL", + recordSeconds: "AI Bridge: Record seconds", + alsaDevice: "AI Bridge: ALSA device (arecord -D)", + audioSampleRate: "AI Bridge: Audio sample rate", + audioChannels: "AI Bridge: Audio channels", + audioFormat: "AI Bridge: Audio format (arecord -f)", + debugAudio: "AI Bridge: Debug audio capture", + useDumpStateAudio: "AI Bridge: Use audio-service dump-state (fix mic busy)", +}; + +var InputMode = { + TEXT: "TEXT", + AUDIO: "AUDIO", +}; + +function httpJsonPost(urlString, payload, timeoutMs) { + timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 15000; + + var parsed = urlLib.parse(urlString); + var isHttps = parsed.protocol === "https:"; + var bodyStr = JSON.stringify(payload || {}); + // Buffer.from is not available on some older Node builds + var body = new Buffer(bodyStr, "utf8"); + + var requestOptions = { + protocol: parsed.protocol, + hostname: parsed.hostname, + port: parsed.port || (isHttps ? 443 : 80), + path: parsed.path || "/", + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": body.length, + }, + timeout: timeoutMs, + }; + + return new Promise(function (resolve, reject) { + var req = (isHttps ? https : http).request(requestOptions, function (res) { + var chunks = []; + res.on("data", function (d) { + chunks.push(d); + }); + res.on("end", function () { + var raw = Buffer.concat(chunks).toString("utf8"); + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + try { + resolve(JSON.parse(raw)); + } catch (e) { + reject(new Error("AI Bridge: bad JSON response (" + res.statusCode + "): " + raw)); + } + return; + } + reject(new Error("AI Bridge: HTTP " + res.statusCode + ": " + raw)); + }); + }); + + req.on("error", reject); + req.on("timeout", function () { + req.destroy(new Error("AI Bridge: request timeout")); + }); + + req.write(body); + req.end(); + }); +} + +function recordWavWithArecord(durationSec, sampleRate, channels, device, format, debug) { + durationSec = typeof durationSec === "number" ? durationSec : 5; + sampleRate = typeof sampleRate === "number" ? sampleRate : 16000; + channels = typeof channels === "number" ? channels : 1; + device = typeof device === "string" ? device : ""; + format = typeof format === "string" ? format : "S16_LE"; + debug = !!debug; + if (["S16_LE", "S32_LE"].indexOf(format) < 0) { + format = "S16_LE"; + } + // Uses `arecord` to capture a short wav to stdout. + // Args chosen to be broadly compatible with embedded ALSA. + var args = [ + // Quiet unless debugging; arecord prints useful device/rate info on stderr. + // (When not quiet, we still only log stderr when debug=true.) + // "-q", + "-f", + format, + "-r", + String(sampleRate), + "-c", + String(channels), + "-d", + String(durationSec), + "-t", + "wav", + ]; + + if (!debug) { + args.unshift("-q"); + } + if (device && device.trim()) { + args.unshift(device.trim()); + args.unshift("-D"); + } + + return new Promise(function (resolve, reject) { + var child = spawn("arecord", args, { stdio: ["ignore", "pipe", "pipe"] }); + var stdout = []; + var stderr = []; + + child.stdout.on("data", function (d) { + stdout.push(d); + }); + child.stderr.on("data", function (d) { + stderr.push(d); + }); + child.on("error", function (err) { + reject(err); + }); + child.on("close", function (code) { + if (code !== 0) { + var errText = Buffer.concat(stderr).toString("utf8"); + reject(new Error("arecord failed (" + code + "): " + errText)); + return; + } + if (debug && rlog) { + try { + var s = Buffer.concat(stderr).toString("utf8"); + if (s && s.trim()) { + rlog.debug("ai-bridge", "arecord stderr", { text: s.slice(0, 500) }); + } + } catch (e) { + // ignore + } + } + resolve(Buffer.concat(stdout)); + }); + }); +} + +function wavSignalStats16(wavBuffer, maxSamples) { + // Best-effort parser for PCM 16-bit WAV emitted by arecord. + // Returns { ok, samples, zeroFrac, rms, min, max } or { ok:false, err }. + maxSamples = typeof maxSamples === "number" ? maxSamples : 16000; + try { + if (!wavBuffer || !Buffer.isBuffer(wavBuffer) || wavBuffer.length < 44) { + return { ok: false, err: "no buffer" }; + } + if (wavBuffer.toString("ascii", 0, 4) !== "RIFF" || wavBuffer.toString("ascii", 8, 12) !== "WAVE") { + return { ok: false, err: "not wav" }; + } + var offset = 12; + var dataOffset = -1; + var dataSize = 0; + var bitsPerSample = null; + while (offset + 8 <= wavBuffer.length) { + var chunkId = wavBuffer.toString("ascii", offset, offset + 4); + var chunkSize = wavBuffer.readUInt32LE(offset + 4); + offset += 8; + if (chunkId === "fmt ") { + if (offset + 16 <= wavBuffer.length) { + bitsPerSample = wavBuffer.readUInt16LE(offset + 14); + } + } else if (chunkId === "data") { + dataOffset = offset; + dataSize = chunkSize; + break; + } + offset += chunkSize; + if (chunkSize % 2 === 1) offset += 1; // padding + } + if (dataOffset < 0) return { ok: false, err: "no data chunk" }; + if (bitsPerSample !== 16) return { ok: false, err: "bitsPerSample=" + String(bitsPerSample) }; + + var available = Math.min(dataSize, wavBuffer.length - dataOffset); + var sampleCount = Math.floor(available / 2); + sampleCount = Math.max(0, Math.min(sampleCount, maxSamples)); + if (sampleCount <= 0) return { ok: false, err: "no samples" }; + + var zeros = 0; + var min = 32767; + var max = -32768; + var sumSq = 0; + for (var i = 0; i < sampleCount; i++) { + var s = wavBuffer.readInt16LE(dataOffset + i * 2); + if (s === 0) zeros += 1; + if (s < min) min = s; + if (s > max) max = s; + sumSq += s * s; + } + var rms = Math.sqrt(sumSq / sampleCount); + return { + ok: true, + samples: sampleCount, + zeroFrac: zeros / sampleCount, + rms: rms, + min: min, + max: max, + }; + } catch (e) { + return { ok: false, err: String(e && (e.stack || e.message || e)) }; + } +} + +function parseWavHeader(wavBuffer) { + // Minimal WAV header parser (RIFF/WAVE + fmt/data). Used for logging/metadata only. + try { + if (!wavBuffer || !Buffer.isBuffer(wavBuffer) || wavBuffer.length < 44) return { ok: false, err: "short" }; + if (wavBuffer.toString("ascii", 0, 4) !== "RIFF" || wavBuffer.toString("ascii", 8, 12) !== "WAVE") return { ok: false, err: "not wav" }; + + var offset = 12; + var channels = null; + var sampleRate = null; + var bitsPerSample = null; + var dataBytes = null; + while (offset + 8 <= wavBuffer.length) { + var chunkId = wavBuffer.toString("ascii", offset, offset + 4); + var chunkSize = wavBuffer.readUInt32LE(offset + 4); + offset += 8; + if (chunkId === "fmt ") { + if (offset + 16 <= wavBuffer.length) { + channels = wavBuffer.readUInt16LE(offset + 2); + sampleRate = wavBuffer.readUInt32LE(offset + 4); + bitsPerSample = wavBuffer.readUInt16LE(offset + 14); + } + } else if (chunkId === "data") { + dataBytes = chunkSize; + break; + } + offset += chunkSize; + if (chunkSize % 2 === 1) offset += 1; + } + + return { ok: true, channels: channels, sampleRate: sampleRate, bitsPerSample: bitsPerSample, dataBytes: dataBytes }; + } catch (e) { + return { ok: false, err: String(e && (e.stack || e.message || e)) }; + } +} + +function runAudioDumpState(cwd, timeoutMs) { + // Runs /usr/local/bin/jibo-audio-dump-state (on-robot) which emits sin.wav/sout.wav/etc. + cwd = typeof cwd === "string" && cwd ? cwd : "/tmp"; + timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 7000; + + var trimDir = function (d) { + return String(d || "/tmp").replace(/\/+$/, "") || "/tmp"; + }; + var fileExistsNonEmpty = function (p) { + try { + var st = fs.statSync(p); + return !!(st && st.isFile && st.isFile() && st.size > 0); + } catch (e) { + return false; + } + }; + // Audio service appears to write its state log as /audiolog.bin + // (see strings in /usr/local/bin/jibo-audio-service). + var dir = trimDir(cwd); + // Some builds appear to write the log at an absolute path: /audiolog.bin. + var candidates = [dir + "/audiolog.bin", "/tmp/audiolog.bin", "/opt/tmp/audiolog.bin", "/audiolog.bin"]; + var infile = null; + for (var i = 0; i < candidates.length; i += 1) { + if (fileExistsNonEmpty(candidates[i])) { + infile = candidates[i]; + break; + } + } + var args = []; + if (infile) { + args = ["--infile", infile]; + } + + return new Promise(function (resolve, reject) { + var child; + try { + child = spawn("/usr/local/bin/jibo-audio-dump-state", args, { stdio: ["ignore", "pipe", "pipe"], cwd: cwd }); + } catch (e) { + reject(e); + return; + } + var out = []; + var err = []; + var killed = false; + var timer = setTimeout(function () { + killed = true; + try { + child.kill("SIGKILL"); + } catch (e) { + // ignore + } + }, timeoutMs); + + child.stdout.on("data", function (d) { + out.push(d); + }); + child.stderr.on("data", function (d) { + err.push(d); + }); + child.on("error", function (e) { + clearTimeout(timer); + reject(e); + }); + child.on("close", function (code) { + clearTimeout(timer); + resolve({ + code: code, + timedOut: killed, + infile: infile, + stdout: Buffer.concat(out).toString("utf8"), + stderr: Buffer.concat(err).toString("utf8"), + }); + }); + }); +} + +function getAudioLogInfo(dir) { + dir = typeof dir === "string" && dir ? dir : "/tmp"; + var base = dir.replace(/\/+$/, ""); + var candidates = [base + "/audiolog.bin", "/tmp/audiolog.bin", "/opt/tmp/audiolog.bin", "/audiolog.bin"]; + for (var i = 0; i < candidates.length; i += 1) { + var p = candidates[i]; + try { + var st = fs.statSync(p); + if (st && st.isFile && st.isFile() && st.size > 0) { + return { path: p, exists: true, bytes: st.size, mtimeMs: st.mtime ? st.mtime.getTime() : null }; + } + } catch (e) { + // ignore + } + } + return { path: candidates[0], exists: false, candidates: candidates }; +} + +function httpTextGet(urlString, timeoutMs) { + timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 3000; + var parsed = urlLib.parse(urlString); + var isHttps = parsed.protocol === "https:"; + var requestOptions = { + protocol: parsed.protocol, + hostname: parsed.hostname, + port: parsed.port || (isHttps ? 443 : 80), + path: parsed.path || "/", + method: "GET", + headers: { Accept: "application/json,text/plain,*/*" }, + timeout: timeoutMs, + }; + return new Promise(function (resolve, reject) { + var req = (isHttps ? https : http).request(requestOptions, function (res) { + var chunks = []; + res.on("data", function (d) { + chunks.push(d); + }); + res.on("end", function () { + var raw = Buffer.concat(chunks).toString("utf8"); + resolve({ statusCode: res.statusCode, headers: res.headers || {}, body: raw }); + }); + }); + req.on("error", reject); + req.on("timeout", function () { + req.destroy(new Error("GET timeout")); + }); + req.end(); + }); +} + +function listTmpAudioFiles(dir) { + dir = typeof dir === "string" && dir ? dir : "/tmp"; + try { + var names = fs.readdirSync(dir); + var out = []; + names.forEach(function (n) { + if (!/\.(raw|wav)$/i.test(n)) return; + try { + var p = dir.replace(/\/+$/, "") + "/" + n; + var st = fs.statSync(p); + if (!st || !st.isFile || !st.isFile()) return; + out.push({ name: n, bytes: st.size, mtimeMs: st.mtime ? st.mtime.getTime() : null }); + } catch (e) { + // ignore + } + }); + out.sort(function (a, b) { + return (b.mtimeMs || 0) - (a.mtimeMs || 0); + }); + return out; + } catch (e) { + return []; + } +} + +function pickBestDumpWav(dir) { + // Prefer multi-mic capture if present. + dir = typeof dir === "string" && dir ? dir : "/tmp"; + var candidates = ["sin.wav", "sout.wav", "ref.wav", "rin.wav", "rout.wav"]; + for (var i = 0; i < candidates.length; i += 1) { + var p = dir.replace(/\/+$/, "") + "/" + candidates[i]; + try { + var st = fs.statSync(p); + if (st && st.isFile && st.isFile() && st.size > 64) { + return { path: p, wav: fs.readFileSync(p) }; + } + } catch (e) { + // ignore + } + } + return null; +} + +function extractAsrText(turnResult) { + // Different builds/components surface listen results differently. + // Prefer the "real" ASR transcript, but accept multiple shapes. + try { + if (!turnResult) return null; + // Already a string + if (typeof turnResult === "string") return turnResult; + // jetstream global/local turn result (common) + if (turnResult.result && turnResult.result.asr && typeof turnResult.result.asr.text === "string") { + return turnResult.result.asr.text; + } + // Sometimes the payload is already the listen result + if (turnResult.asr && typeof turnResult.asr.text === "string") { + return turnResult.asr.text; + } + // Some command-library style payloads use Speech + if (typeof turnResult.Speech === "string") { + return turnResult.Speech; + } + // Some internal events embed options + if (turnResult.options && turnResult.options.asr && typeof turnResult.options.asr.text === "string") { + return turnResult.options.asr.text; + } + // Some events wrap details under `message` + if (turnResult.message) { + var msg = turnResult.message; + // JSON string? + if (typeof msg === "string") { + var trimmed = msg.trim(); + if (trimmed && (trimmed[0] === "{" || trimmed[0] === "[")) { + try { + var parsed = JSON.parse(trimmed); + var nested = extractAsrText(parsed); + if (nested) return nested; + } catch (e) { + // ignore parse errors + } + } + // Sometimes the message is literally the transcript + if (trimmed) return trimmed; + } + if (typeof msg === "object") { + var nested2 = extractAsrText(msg); + if (nested2) return nested2; + } + } + // Defensive last resort: asr.text + if (turnResult.asr && typeof turnResult.asr === "object" && typeof turnResult.asr.text === "string") { + return turnResult.asr.text; + } + return null; + } catch (e) { + return null; + } +} + +function safePreview(obj, limit) { + limit = typeof limit === "number" ? limit : 400; + try { + var s = JSON.stringify(obj); + if (typeof s !== "string") return null; + if (s.length > limit) return s.slice(0, limit) + "..."; + return s; + } catch (e) { + return null; + } +} + +function normalizeServerBaseUrl(input) { + if (typeof input !== "string") return null; + var s = input.replace(/\s+/g, "").trim(); + if (!s) return null; + + // Common paste mistakes from the tunables UI (fires on each keystroke) + // - http://http://... + // - http://host/:8020 + s = s.replace(/^https?:\/\/https?:\/\//i, function (m) { + // keep the first scheme only + var scheme = m.toLowerCase().indexOf("https://") === 0 ? "https://" : "http://"; + return scheme; + }); + if (!/^https?:\/\//i.test(s)) { + s = "http://" + s; + } + // Fix "/:8020" after host + s = s.replace(/\/:(\d{2,5})(\/|$)/, ":$1$2"); + // Remove any path; we only want scheme://host:port + var m = s.match(/^(https?:\/\/[^\/]+)(\/.*)?$/i); + if (m && m[1]) { + s = m[1]; + } + // Strip trailing slash + s = s.replace(/\/+$/, ""); + + // Validate very loosely: scheme://host[:port] + if (!/^https?:\/\/[^\/:]+(:\d{2,5})?$/i.test(s)) { + return null; + } + + return s; +} + +function isCloudAsrFailureText(text) { + if (typeof text !== "string") return false; + var s = text.trim(); + if (!s) return false; + // Heuristics: avoid sending cloud connectivity errors as "ASR text". + // Observed examples: + // CloudConnection::open (poco exception): Host not found: neo-hub.jibo.com + if (/CloudConnection::open/i.test(s)) return true; + if (/poco exception/i.test(s) && /Host not found/i.test(s)) return true; + if (/neo-hub\.jibo\.com/i.test(s)) return true; + return false; +} + +function isMicBusyError(err) { + var s = String(err || ""); + return /Device or resource busy/i.test(s) || /audio device is busy/i.test(s); +} + +function looksLikeHeyJiboIntent(results) { + try { + if (!results || typeof results !== "object") return false; + var nlu = results.nlu; + if (!nlu || typeof nlu !== "object") return false; + var intent = nlu.intent || nlu.intentName || nlu.topIntent; + if (typeof intent !== "string") return false; + return /heyjibo/i.test(intent); + } catch (e) { + return false; + } +} + + +function AIBridge(skill, jibo) { + this.skill = skill; + this.jibo = jibo; + + this.enabled = true; + // Default to TEXT because the built-in ASR pipeline already owns the mic + // on many builds; AUDIO/arecord often fails with "Device or resource busy". + this.mode = InputMode.TEXT; + this.serverBaseUrl = "http://127.0.0.1:8020"; + this.recordSeconds = 5; + this.arecordDevice = ""; + this.audioSampleRate = 16000; + this.audioChannels = 1; + this.audioFormat = "S16_LE"; + this.debugAudioCapture = false; + // Prefer grabbing audio from the system audio service (avoids ALSA mic lock). + this.useDumpStateAudio = true; + this.dumpStateDir = "/tmp"; + + this._inFlight = false; + this._lastHJAt = 0; + this._hjWindowMs = 15000; + this._pendingTextFallback = false; + this._allowAnyTextNoHJ = false; + this._lastSentText = ""; + this._lastSentAt = 0; + this._openHookInstalled = false; + this._offlineAudioAttempts = 0; + this._audioFallbackTimer = null; + + var self = this; + this._onHJHeard = function () { + return self._handleHJHeard(); + }; + this._onGlobalTurnResult = function (data) { + return self._handleTurnResult(data, "globalTurnResult"); + }; + this._onLocalTurnResult = function (data) { + return self._handleTurnResult(data, "localTurnResult"); + }; +} + +AIBridge.prototype._scheduleOfflineAudioFallback = function (reason, delayMs) { + var self = this; + if (!self.enabled) return; + if (self._offlineAudioAttempts >= 2) return; + if (!self._allowAnyTextNoHJ && Date.now() - self._lastHJAt > self._hjWindowMs) return; + if (self._inFlight) return; + + if (self._audioFallbackTimer) { + clearTimeout(self._audioFallbackTimer); + self._audioFallbackTimer = null; + } + + delayMs = typeof delayMs === "number" ? delayMs : 1200; + if (rlog) { + rlog.warn("ai-bridge", "scheduling offline audio fallback", { reason: String(reason || ""), delayMs: delayMs }); + } + + self._audioFallbackTimer = setTimeout(function () { + self._audioFallbackTimer = null; + self._offlineAudioAttempts += 1; + self + ._captureAndSendAudio("offlineFallback:" + String(reason || "")) + .catch(function (e) { + if (rlog) { + rlog.warn("ai-bridge", "offline audio fallback failed", { err: String(e && (e.stack || e.message || e)) }); + } + // Common case: mic is still owned by the built-in pipeline. Retry once. + if (isMicBusyError(String(e && (e.stack || e.message || e)))) { + self._scheduleOfflineAudioFallback("retryMicBusy:" + String(reason || ""), 2400); + } + }); + }, delayMs); +}; + +AIBridge.prototype._captureAndSendAudio = function (source) { + var self = this; + if (!self.enabled) { + return Promise.resolve(); + } + if (self._inFlight) { + return Promise.resolve(); + } + self._inFlight = true; + + var t0 = Date.now(); + var rate = typeof self.audioSampleRate === "number" ? self.audioSampleRate : 16000; + var ch = typeof self.audioChannels === "number" ? self.audioChannels : 1; + var dev = typeof self.arecordDevice === "string" ? self.arecordDevice : ""; + var fmt = typeof self.audioFormat === "string" ? self.audioFormat : "S16_LE"; + if (["S16_LE", "S32_LE"].indexOf(fmt) < 0) fmt = "S16_LE"; + var dbg = !!self.debugAudioCapture; + + var dumpDir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; + + var captureViaDumpState = function () { + var tDump0 = Date.now(); + return runAudioDumpState(dumpDir, 7000).then(function (res) { + if (rlog) { + rlog.info("ai-bridge", "dump-state complete", { + ms: Date.now() - tDump0, + code: res.code, + timedOut: res.timedOut, + infile: res.infile || null, + stdout: String(res.stdout || "").slice(0, 700), + stderr: String(res.stderr || "").slice(0, 700), + }); + } + var picked = pickBestDumpWav(dumpDir); + if (!picked || !picked.wav) { + throw new Error( + "dump-state produced no wavs; files=" + + JSON.stringify(listTmpAudioFiles(dumpDir).slice(0, 12)) + + " audiolog=" + + JSON.stringify(getAudioLogInfo(dumpDir)) + ); + } + var hdr = parseWavHeader(picked.wav); + return { wav: picked.wav, meta: { method: "dump-state", path: picked.path, header: hdr } }; + }); + }; + + var captureViaArecord = function () { + return recordWavWithArecord(self.recordSeconds, rate, ch, dev, fmt, dbg).then(function (wav) { + return { wav: wav, meta: { method: "arecord", device: dev || "(default)", rate: rate, ch: ch, format: fmt } }; + }); + }; + + var capturePromise; + if (self.useDumpStateAudio) { + capturePromise = captureViaDumpState().catch(function (e) { + if (rlog) { + rlog.warn("ai-bridge", "dump-state capture failed; falling back to arecord", { err: String(e && (e.stack || e.message || e)) }); + } + return captureViaArecord(); + }); + } else { + capturePromise = captureViaArecord().catch(function (e) { + if (isMicBusyError(String(e && (e.stack || e.message || e)))) { + return captureViaDumpState(); + } + throw e; + }); + } + + return capturePromise + .then(function (cap) { + var wav = cap && cap.wav ? cap.wav : null; + var meta = cap && cap.meta ? cap.meta : null; + if (!wav) throw new Error("no audio captured"); + var url = self.serverBaseUrl.replace(/\/+$/, "") + "/v1/chat/audio"; + if (rlog) { + rlog.info("ai-bridge", "audio captured", { + bytes: wav.length, + ms: Date.now() - t0, + url: url, + source: source || "audio", + capture: meta || { method: "unknown" }, + }); + } + var t1 = Date.now(); + return httpJsonPost(url, { + wav_base64: wav.toString("base64"), + source: source || "audio", + capture: meta || null, + }).then(function (resp) { + if (rlog) { + rlog.info("ai-bridge", "audio request complete", { ms: Date.now() - t1, ok: !!(resp && resp.reply), source: source || "audio" }); + } + if (rlog && resp && typeof resp.text === "string") { + rlog.info("ai-bridge", "server transcript", { + chars: resp.text.length, + text: resp.text.slice(0, 160), + }); + } + if (resp && resp.reply) { + return self._speak(resp.reply); + } + }); + }) + .catch(function (e) { + self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge audio capture error", e && (e.stack || e.message || e)); + if (rlog) { + rlog.warn("ai-bridge", "audio capture error", { err: String(e && (e.stack || e.message || e)), source: source || "audio" }); + } + throw e; + }) + .then( + function (v) { + self._inFlight = false; + return v; + }, + function (e) { + self._inFlight = false; + throw e; + } + ); +}; + +AIBridge.prototype._installOpenHook = function () { + var self = this; + if (self._openHookInstalled) return; + self._openHookInstalled = true; + + if (!beFramework || !beFramework.BeSkill || typeof beFramework.BeSkill.registerOpenHook !== "function") { + if (rlog) { + rlog.warn("ai-bridge", "be-framework open hook unavailable; cannot intercept skill launches", {}); + } + return; + } + + try { + beFramework.BeSkill.registerOpenHook(function (lastSkill, nextSkill, results) { + return function (resolve) { + try { + if (!self.enabled) return resolve(); + if (self.mode !== InputMode.TEXT && !self._pendingTextFallback) return resolve(); + if (!self._allowAnyTextNoHJ && Date.now() - self._lastHJAt > self._hjWindowMs) return resolve(); + + var text = extractAsrText(results); + if (!text) { + if (rlog) { + rlog.debug("ai-bridge", "open hook saw no asr text", { + nextSkill: String(nextSkill || ""), + keys: results && typeof results === "object" ? Object.keys(results).slice(0, 25) : [], + preview: safePreview(results, 320), + }); + } + // If cloud ASR is down, Hey-Jibo intents often have empty asr.text. + // Try a one-shot offline audio fallback after the pipeline releases the mic. + if (looksLikeHeyJiboIntent(results)) { + self._scheduleOfflineAudioFallback("openHookNoText", 1400); + } + return resolve(); + } + text = String(text); + if (!text.trim()) return resolve(); + if (isCloudAsrFailureText(text)) { + if (rlog) { + rlog.warn("ai-bridge", "ignoring cloud ASR error text (open hook)", { text: text.slice(0, 200) }); + } + self._scheduleOfflineAudioFallback("cloudAsrErrorText", 1400); + return resolve(); + } + + if (rlog) { + rlog.info("ai-bridge", "open hook asr", { + nextSkill: String(nextSkill || ""), + text: text.slice(0, 160), + }); + } + self._sendText(text, "openHook:" + String(nextSkill || "")); + return resolve(); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "open hook error", { err: String(e && (e.stack || e.message || e)) }); + } + return resolve(); + } + }; + }); + + if (rlog) { + rlog.info("ai-bridge", "installed BeSkill open hook", {}); + } + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "failed to install open hook", { err: String(e && (e.stack || e.message || e)) }); + } + } +}; + +AIBridge.prototype.setupTunables = function () { + if (!tunable || !tunable.Tunable) { + if (rlog && typeof rlog.raw === "function") { + rlog.raw("[AI-BRIDGE] jibo-tunable not available; UI controls disabled"); + } + return; + } + + var self = this; + var Tunable = tunable.Tunable; + var safe = function (fn) { + try { + return fn(); + } catch (e) { + if (rlog && typeof rlog.raw === "function") { + rlog.raw("[AI-BRIDGE] Tunable registration failed: " + String(e && (e.stack || e.message || e))); + } + return null; + } + }; + + safe(function () { + return Tunable.getCheckboxField(LABEL.enabled, self.enabled, WIN).events.change.on(function (v) { + self.enabled = !!v; + if (rlog) { + rlog.info("ai-bridge", "enabled changed", { enabled: self.enabled }); + } + }); + }); + + // Prefer dropdown if available; otherwise fallback to a checkbox. + if (typeof Tunable.getDropdownField === "function") { + safe(function () { + return Tunable.getDropdownField(LABEL.input, [InputMode.AUDIO, InputMode.TEXT], 0, WIN).events.change.on(function (idxOrVal) { + var options = [InputMode.AUDIO, InputMode.TEXT]; + self.mode = options[idxOrVal] || idxOrVal || InputMode.AUDIO; + if (rlog) { + rlog.info("ai-bridge", "mode changed", { mode: self.mode }); + } + }); + }); + } else { + safe(function () { + return Tunable.getCheckboxField(LABEL.audioCheckbox, self.mode === InputMode.AUDIO, WIN).events.change.on(function (v) { + self.mode = v ? InputMode.AUDIO : InputMode.TEXT; + if (rlog) { + rlog.info("ai-bridge", "mode changed", { mode: self.mode }); + } + }); + }); + } + + safe(function () { + return Tunable.getStringField(LABEL.serverUrl, self.serverBaseUrl, WIN).events.change.on(function (v) { + var normalized = normalizeServerBaseUrl(v); + if (normalized) { + self.serverBaseUrl = normalized; + if (rlog) { + rlog.info("ai-bridge", "server url changed", { url: self.serverBaseUrl }); + } + } else { + if (rlog) { + rlog.warn("ai-bridge", "ignored invalid server url", { url: String(v).slice(0, 120) }); + } + } + }); + }); + + // Audio capture tuning for offline fallback and AUDIO mode. + safe(function () { + return Tunable.getStringField(LABEL.alsaDevice, self.arecordDevice, WIN).events.change.on(function (v) { + self.arecordDevice = String(v || "").trim(); + if (rlog) { + rlog.info("ai-bridge", "alsa device changed", { device: self.arecordDevice || "(default)" }); + } + }); + }); + + var setRate = function (v) { + var n = Number(v); + if (Number.isNaN(n)) return; + self.audioSampleRate = Math.max(8000, Math.min(48000, Math.floor(n))); + if (rlog) { + rlog.info("ai-bridge", "audio sample rate changed", { hz: self.audioSampleRate }); + } + }; + if (typeof Tunable.getNumberField === "function") { + safe(function () { + return Tunable.getNumberField(LABEL.audioSampleRate, self.audioSampleRate, 8000, 48000, 1000, WIN).events.change.on(setRate); + }); + } else { + safe(function () { + return Tunable.getStringField(LABEL.audioSampleRate, String(self.audioSampleRate), WIN).events.change.on(setRate); + }); + } + + var setCh = function (v) { + var n = Number(v); + if (Number.isNaN(n)) return; + self.audioChannels = Math.max(1, Math.min(8, Math.floor(n))); + if (rlog) { + rlog.info("ai-bridge", "audio channels changed", { ch: self.audioChannels }); + } + }; + if (typeof Tunable.getNumberField === "function") { + safe(function () { + return Tunable.getNumberField(LABEL.audioChannels, self.audioChannels, 1, 8, 1, WIN).events.change.on(setCh); + }); + } else { + safe(function () { + return Tunable.getStringField(LABEL.audioChannels, String(self.audioChannels), WIN).events.change.on(setCh); + }); + } + + // Sample format (default S16_LE). Keep it whitelisted. + if (typeof Tunable.getDropdownField === "function") { + safe(function () { + var opts = ["S16_LE", "S32_LE"]; + var idx = Math.max(0, opts.indexOf(self.audioFormat)); + return Tunable.getDropdownField(LABEL.audioFormat, opts, idx, WIN).events.change.on(function (idxOrVal) { + var v = opts[idxOrVal] || idxOrVal; + v = String(v || "").trim(); + if (opts.indexOf(v) >= 0) { + self.audioFormat = v; + if (rlog) { + rlog.info("ai-bridge", "audio format changed", { format: self.audioFormat }); + } + } + }); + }); + } else { + safe(function () { + return Tunable.getStringField(LABEL.audioFormat, String(self.audioFormat), WIN).events.change.on(function (v) { + v = String(v || "").trim(); + if (["S16_LE", "S32_LE"].indexOf(v) >= 0) { + self.audioFormat = v; + if (rlog) { + rlog.info("ai-bridge", "audio format changed", { format: self.audioFormat }); + } + } else if (rlog) { + rlog.warn("ai-bridge", "ignored invalid audio format", { format: String(v).slice(0, 32) }); + } + }); + }); + } + + safe(function () { + return Tunable.getCheckboxField(LABEL.debugAudio, self.debugAudioCapture, WIN).events.change.on(function (v) { + self.debugAudioCapture = !!v; + if (rlog) { + rlog.info("ai-bridge", "debug audio capture changed", { enabled: self.debugAudioCapture }); + } + }); + }); + + // Prefer the system audio service's dump-state output over arecord (avoids mic busy). + safe(function () { + return Tunable.getCheckboxField(LABEL.useDumpStateAudio, self.useDumpStateAudio, WIN).events.change.on(function (v) { + self.useDumpStateAudio = !!v; + if (rlog) { + rlog.info("ai-bridge", "use dump-state audio changed", { enabled: self.useDumpStateAudio }); + } + }); + }); + + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Dump /tmp audio files", WIN).events.change.on(function () { + try { + var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; + var files = listTmpAudioFiles(dir); + if (rlog) { + rlog.info("ai-bridge", "tmp audio files", { dir: dir, files: files.slice(0, 60), audiolog: getAudioLogInfo(dir) }); + } + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "dump tmp audio files failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Run audio dump-state now", WIN).events.change.on(function () { + try { + var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; + var t0 = Date.now(); + runAudioDumpState(dir, 8000) + .then(function (res) { + var picked = pickBestDumpWav(dir); + if (rlog) { + rlog.info("ai-bridge", "manual dump-state", { + ms: Date.now() - t0, + code: res.code, + timedOut: res.timedOut, + infile: res.infile || null, + picked: picked ? picked.path : null, + files: listTmpAudioFiles(dir).slice(0, 20), + audiolog: getAudioLogInfo(dir), + stderr: String(res.stderr || "").slice(0, 500), + }); + } + }) + .catch(function (e) { + if (rlog) { + rlog.warn("ai-bridge", "manual dump-state failed", { err: String(e && (e.stack || e.message || e)) }); + } + }); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "manual dump-state crashed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + // Dump hardware params for the currently-selected settings. + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Dump hw params", WIN).events.change.on(function () { + try { + var dev = (self.arecordDevice || "").trim(); + var rate = typeof self.audioSampleRate === "number" ? self.audioSampleRate : 16000; + var ch = typeof self.audioChannels === "number" ? self.audioChannels : 1; + var fmt = (self.audioFormat || "S16_LE").trim(); + if (["S16_LE", "S32_LE"].indexOf(fmt) < 0) fmt = "S16_LE"; + var args = ["--dump-hw-params", "-d", "1", "-t", "wav", "-f", fmt, "-r", String(rate), "-c", String(ch)]; + if (dev) { + args.unshift(dev); + args.unshift("-D"); + } + args.push("/dev/null"); + var child = spawn("arecord", args, { stdio: ["ignore", "pipe", "pipe"] }); + var out = []; + var err = []; + child.stdout.on("data", function (d) { + out.push(d); + }); + child.stderr.on("data", function (d) { + err.push(d); + }); + child.on("close", function (code) { + var o = Buffer.concat(out).toString("utf8"); + var e = Buffer.concat(err).toString("utf8"); + if (rlog) { + rlog.info("ai-bridge", "arecord hw params", { + code: code, + device: dev || "(default)", + rate: rate, + ch: ch, + format: fmt, + stdout: (o || "").slice(0, 1400), + stderr: (e || "").slice(0, 1400), + }); + } + }); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "dump hw params failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + // Quick probes for the built-in audio service HTTP API. + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: AudioSvc stream_state", WIN).events.change.on(function () { + try { + var u = "http://127.0.0.1:8383/stream_state"; + var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; + httpTextGet(u, 2500) + .then(function (res) { + if (rlog) { + rlog.info("ai-bridge", "audiosvc stream_state", { + url: u, + status: res.statusCode, + body: String(res.body || "").slice(0, 1200), + audiolog: getAudioLogInfo(dir), + }); + } + }) + .catch(function (e) { + if (rlog) { + rlog.warn("ai-bridge", "audiosvc stream_state failed", { err: String(e && (e.stack || e.message || e)) }); + } + }); + } catch (e) { + if (rlog) rlog.warn("ai-bridge", "audiosvc stream_state crashed", { err: String(e && (e.stack || e.message || e)) }); + } + }); + }); + } + + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: AudioSvc log", WIN).events.change.on(function () { + try { + var u = "http://127.0.0.1:8383/log"; + var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; + httpTextGet(u, 2500) + .then(function (res) { + if (rlog) { + rlog.info("ai-bridge", "audiosvc log", { + url: u, + status: res.statusCode, + body: String(res.body || "").slice(0, 1200), + audiolog: getAudioLogInfo(dir), + }); + } + }) + .catch(function (e) { + if (rlog) { + rlog.warn("ai-bridge", "audiosvc log failed", { err: String(e && (e.stack || e.message || e)) }); + } + }); + } catch (e) { + if (rlog) rlog.warn("ai-bridge", "audiosvc log crashed", { err: String(e && (e.stack || e.message || e)) }); + } + }); + }); + } + + // Probe a few candidate devices to find a usable mic tap when the real ADC is busy. + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Probe audio devices", WIN).events.change.on(function () { + try { + if (self._inFlight) { + if (rlog) rlog.warn("ai-bridge", "probe skipped; in flight", {}); + return; + } + var rate = typeof self.audioSampleRate === "number" ? self.audioSampleRate : 16000; + var ch = typeof self.audioChannels === "number" ? self.audioChannels : 1; + var fmt = typeof self.audioFormat === "string" ? self.audioFormat : "S16_LE"; + if (["S16_LE", "S32_LE"].indexOf(fmt) < 0) fmt = "S16_LE"; + // Signal stats helper only understands 16-bit PCM, so probe with S16_LE even if + // the main audio capture is set to S32_LE. + var probeFmt = "S16_LE"; + var devices = []; + // 1) configured + devices.push((self.arecordDevice || "").trim()); + // 2) default + devices.push(""); + // 3) Try shareable plugin PCMs (may work even if hw is busy) + devices.push("dsnoop"); + devices.push("dsnoop:CARD=ADC,DEV=0"); + devices.push("dsnoop:CARD=1,DEV=0"); + devices.push("plughw:1,0"); + devices.push("hw:1,0"); + // 4) ALSA Loopback common capture endpoints + devices.push("plughw:2,0"); + devices.push("plughw:2,1"); + // 5) Dummy (should be silence) + devices.push("plughw:3,0"); + // De-dupe + var seen = {}; + devices = devices.filter(function (d) { + d = String(d || ""); + if (seen[d]) return false; + seen[d] = true; + return true; + }); + + if (rlog) { + rlog.info("ai-bridge", "probe starting", { rate: rate, ch: ch, format: fmt, probeFormat: probeFmt, devices: devices }); + } + + var p = Promise.resolve(); + devices.forEach(function (dev) { + p = p.then(function () { + return recordWavWithArecord(1, rate, ch, dev, probeFmt, true) + .then(function (wav) { + var stats = wavSignalStats16(wav, Math.max(8000, rate)); + if (rlog) { + rlog.info("ai-bridge", "probe result", { + device: dev || "(default)", + bytes: wav.length, + format: probeFmt, + stats: stats, + }); + } + }); + }).catch(function (e) { + if (rlog) { + rlog.warn("ai-bridge", "probe error", { device: dev || "(default)", err: String(e && (e.stack || e.message || e)) }); + } + }); + }); + + p.then(function () { + if (rlog) rlog.info("ai-bridge", "probe done", {}); + }); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "probe failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + // Helper for finding the right -D device string. + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Dump ALSA (arecord -l)", WIN).events.change.on(function () { + try { + var child = spawn("arecord", ["-l"], { stdio: ["ignore", "pipe", "pipe"] }); + var out = []; + var err = []; + child.stdout.on("data", function (d) { + out.push(d); + }); + child.stderr.on("data", function (d) { + err.push(d); + }); + child.on("close", function (code) { + var o = Buffer.concat(out).toString("utf8"); + var e = Buffer.concat(err).toString("utf8"); + if (rlog) { + rlog.info("ai-bridge", "arecord -l", { code: code, stdout: (o || "").slice(0, 2000), stderr: (e || "").slice(0, 500) }); + } + }); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "arecord -l failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + // Helper for listing all ALSA PCMs (useful for discovering dsnoop/plug routes). + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Dump ALSA PCMs (arecord -L)", WIN).events.change.on(function () { + try { + var child = spawn("arecord", ["-L"], { stdio: ["ignore", "pipe", "pipe"] }); + var out = []; + var err = []; + child.stdout.on("data", function (d) { + out.push(d); + }); + child.stderr.on("data", function (d) { + err.push(d); + }); + child.on("close", function (code) { + var o = Buffer.concat(out).toString("utf8"); + var e = Buffer.concat(err).toString("utf8"); + if (rlog) { + rlog.info("ai-bridge", "arecord -L", { code: code, stdout: (o || "").slice(0, 4000), stderr: (e || "").slice(0, 500) }); + } + }); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "arecord -L failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + // Dump who owns the Tobor ADC capture device, via /proc/asound and (if available) fuser. + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Dump ADC owner (/proc/asound)", WIN).events.change.on(function () { + try { + var fs = require("fs"); + function readText(path) { + try { + return fs.readFileSync(path, "utf8"); + } catch (e) { + return ""; + } + } + + var statusPath = "/proc/asound/card1/pcm0c/sub0/status"; + var hwParamsPath = "/proc/asound/card1/pcm0c/sub0/hw_params"; + var infoPath = "/proc/asound/card1/pcm0c/sub0/info"; + var pcmPath = "/proc/asound/pcm"; + var cardsPath = "/proc/asound/cards"; + var idPath = "/proc/asound/card1/id"; + + var status = readText(statusPath); + var hwParams = readText(hwParamsPath); + var info = readText(infoPath); + var pcm = readText(pcmPath); + var cards = readText(cardsPath); + var cardId = readText(idPath); + + // Extract owning PID if present. + var pid = null; + var m = (status || "").match(/owner_pid\s*:\s*(\d+)/); + if (m && m[1]) pid = Number(m[1]); + var cmdline = ""; + if (pid && pid > 0) { + cmdline = readText("/proc/" + pid + "/cmdline").replace(/\u0000/g, " ").trim(); + } + + if (rlog) { + rlog.info("ai-bridge", "adc owner", { + cardId: (cardId || "").trim(), + status: (status || "").trim().slice(0, 2000), + hw_params: (hwParams || "").trim().slice(0, 2000), + info: (info || "").trim().slice(0, 2000), + pcm: (pcm || "").trim().slice(0, 2000), + cards: (cards || "").trim().slice(0, 2000), + owner_pid: pid || null, + owner_cmdline: cmdline || "", + }); + } + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "dump adc owner failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Dump ADC owner (fuser)", WIN).events.change.on(function () { + try { + var devs = ["/dev/snd/pcmC1D0c", "/dev/snd/controlC1", "/dev/snd/timer"]; // best-effort + var child = spawn("fuser", ["-v"].concat(devs), { stdio: ["ignore", "pipe", "pipe"] }); + var out = []; + var err = []; + child.stdout.on("data", function (d) { + out.push(d); + }); + child.stderr.on("data", function (d) { + err.push(d); + }); + child.on("close", function (code) { + var o = Buffer.concat(out).toString("utf8"); + var e = Buffer.concat(err).toString("utf8"); + if (rlog) { + rlog.info("ai-bridge", "fuser adc", { code: code, stdout: (o || "").slice(0, 2000), stderr: (e || "").slice(0, 2000) }); + } + }); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "fuser failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + // Simple one-shot connectivity test. + if (typeof Tunable.getButtonField === "function") { + safe(function () { + return Tunable.getButtonField("AI Bridge: Test server", WIN).events.change.on(function () { + try { + self._sendText("(test) hello from Jibo", "test"); + } catch (e) { + if (rlog) { + rlog.warn("ai-bridge", "test send failed", { err: String(e && (e.stack || e.message || e)) }); + } + } + }); + }); + } + + if (typeof Tunable.getNumberField === "function") { + safe(function () { + return Tunable.getNumberField(LABEL.recordSeconds, self.recordSeconds, 1, 15, 1, WIN).events.change.on(function (v) { + var n = Number(v); + if (!Number.isNaN(n)) { + self.recordSeconds = Math.max(1, Math.min(15, Math.floor(n))); + if (rlog) { + rlog.info("ai-bridge", "record seconds changed", { seconds: self.recordSeconds }); + } + } + }); + }); + } else { + safe(function () { + return Tunable.getStringField(LABEL.recordSeconds, String(self.recordSeconds), WIN).events.change.on(function (v) { + var n = Number(v); + if (!Number.isNaN(n)) { + self.recordSeconds = Math.max(1, Math.min(15, Math.floor(n))); + if (rlog) { + rlog.info("ai-bridge", "record seconds changed", { seconds: self.recordSeconds }); + } + } + }); + }); + } + }; + +AIBridge.prototype.start = function () { + try { + this.setupTunables(); + } catch (e) { + if (rlog && typeof rlog.raw === "function") { + rlog.raw("[AI-BRIDGE] setupTunables crashed: " + String(e && (e.stack || e.message || e))); + } + } + try { + var ev = this.jibo && this.jibo.jetstream && this.jibo.jetstream.events; + if (ev && ev.hjHeard && typeof ev.hjHeard.on === "function") { + ev.hjHeard.on(this._onHJHeard); + } + if (ev && ev.globalTurnResult && typeof ev.globalTurnResult.on === "function") { + ev.globalTurnResult.on(this._onGlobalTurnResult); + } + if (ev && ev.localTurnResult && typeof ev.localTurnResult.on === "function") { + ev.localTurnResult.on(this._onLocalTurnResult); + } + if (rlog) { + rlog.info("ai-bridge", "started", { mode: this.mode, serverUrl: this.serverBaseUrl, recordSeconds: this.recordSeconds }); + } + if (rlog && typeof rlog.raw === "function") { + rlog.raw("[AI-BRIDGE] started mode=" + this.mode + " url=" + this.serverBaseUrl); + } + this._installOpenHook(); + } catch (e) { + this.skill.log && this.skill.log.warn && this.skill.log.warn("AI Bridge failed to attach listeners", e); + if (rlog) { + rlog.error("ai-bridge", "failed to attach listeners", { err: String(e && (e.stack || e.message || e)) }); + } + if (rlog && typeof rlog.raw === "function") { + rlog.raw("[AI-BRIDGE] failed to attach listeners: " + String(e && (e.stack || e.message || e))); + } + } + }; + +AIBridge.prototype._speak = function (text) { + var self = this; + if (!text || !String(text).trim()) { + return Promise.resolve(); + } + + // Keep TTS inputs conservative: some builds have fragile service-client JSON parsing + // on long or multi-line payloads. + try { + text = String(text) + .replace(/[\r\n\t]+/g, " ") + .replace(/\s{2,}/g, " ") + .trim(); + if (text.length > 280) { + text = text.slice(0, 280); + } + } catch (e) { + // ignore + } + + try { + if (rlog) { + rlog.info("ai-bridge", "speaking", { chars: String(text).length }); + } + } catch (e) { + // ignore + } + + var ret; + try { + ret = self.jibo.tts.speak(String(text), { mode: self.jibo.tts.TTSMode.TEXT }); + } catch (e) { + self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge TTS error", e); + if (rlog) { + rlog.warn("ai-bridge", "tts error", { err: String(e && (e.stack || e.message || e)) }); + } + return Promise.resolve(); + } + + return Promise.resolve(ret).catch(function (e) { + self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge TTS error", e); + if (rlog) { + rlog.warn("ai-bridge", "tts error", { err: String(e && (e.stack || e.message || e)) }); + } + }); +}; + +AIBridge.prototype._handleHJHeard = function () { + var self = this; + self._lastHJAt = Date.now(); + // Clear any prior fallback request window. + self._pendingTextFallback = false; + self._offlineAudioAttempts = 0; + if (self._audioFallbackTimer) { + clearTimeout(self._audioFallbackTimer); + self._audioFallbackTimer = null; + } + if (rlog) { + rlog.debug("ai-bridge", "hjHeard", { mode: self.mode, enabled: self.enabled }); + } + if (!self.enabled) { + return; + } + if (self.mode !== InputMode.AUDIO) { + return; + } + if (self._inFlight) { + return; + } + + return self._captureAndSendAudio("hjHeard") + .then(function () { + // done + }) + .catch(function (e) { + // _captureAndSendAudio already logs; keep compatibility with existing fallback behavior + var msg = String(e && (e.stack || e.message || e)); + if (isMicBusyError(msg)) { + self._scheduleOfflineAudioFallback("micBusyAfterHJ", 1600); + } + self._pendingTextFallback = true; + }); +}; + +AIBridge.prototype._handleTurnResult = function (data, eventName) { + var self = this; + if (!self.enabled) { + return; + } + if (rlog) { + try { + rlog.debug("ai-bridge", "turn result", { event: eventName }); + } catch (e) { + // ignore + } + } + // Normal behavior: only send ASR in TEXT mode. + // Fallback behavior: if AUDIO failed during this HJ window, allow a one-shot + // ASR send even though the mode is AUDIO. + if (self.mode !== InputMode.TEXT && !self._pendingTextFallback) { + return; + } + if (!self._allowAnyTextNoHJ && Date.now() - self._lastHJAt > self._hjWindowMs) { + return; + } + + var text = extractAsrText(data); + if (!text) { + if (rlog) { + try { + var t = typeof data; + var keys = data && typeof data === "object" ? Object.keys(data).slice(0, 25) : []; + rlog.debug("ai-bridge", "turn result had no text", { event: eventName, type: t, keys: keys }); + } catch (e) { + // ignore + } + } + return; + } + text = String(text); + if (!text.trim()) { + return; + } + if (isCloudAsrFailureText(text)) { + if (rlog) { + rlog.warn("ai-bridge", "ignoring cloud ASR error text", { event: eventName, text: text.slice(0, 220) }); + } + self._scheduleOfflineAudioFallback("cloudAsrErrorText:" + String(eventName || "turnResult"), 1400); + return; + } + + // Deduplicate rapid double-emits (some builds fire both local & global). + var now = Date.now(); + if (self._lastSentText === text && now - self._lastSentAt < 2000) { + if (rlog) { + rlog.debug("ai-bridge", "deduped transcript", { event: eventName }); + } + return; + } + self._lastSentText = text; + self._lastSentAt = now; + + if (rlog) { + rlog.info("ai-bridge", "asr text", { event: eventName, text: String(text).slice(0, 160) }); + } + if (self._inFlight) { + return; + } + + return self._sendText(text, "asr:" + String(eventName || "turnResult")); +}; + +AIBridge.prototype._sendText = function (text, source) { + var self = this; + if (!text) { + return Promise.resolve(); + } + if (self._inFlight) { + return Promise.resolve(); + } + + self._inFlight = true; + self._pendingTextFallback = false; + + var t0 = Date.now(); + var url = self.serverBaseUrl.replace(/\/+$/, "") + "/v1/chat/text"; + if (rlog) { + rlog.info("ai-bridge", "sending text", { + source: source || "text", + chars: String(text).length, + url: url, + text: String(text).slice(0, 160), + }); + } + + return httpJsonPost(url, { text: text }) + .then(function (resp) { + if (rlog) { + rlog.info("ai-bridge", "text request complete", { ms: Date.now() - t0, ok: !!(resp && resp.reply) }); + } + if (resp && resp.reply) { + return self._speak(resp.reply); + } + }) + .catch(function (e) { + self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge text mode error", e && (e.stack || e.message || e)); + if (rlog) { + rlog.warn("ai-bridge", "text mode error", { err: String(e && (e.stack || e.message || e)) }); + } + }) + .then(function () { + self._inFlight = false; + }); +}; + +exports.InputMode = InputMode; +exports.AIBridge = AIBridge; + +exports.initAIBridge = function initAIBridge(skill, jibo) { + try { + var bridge = new AIBridge(skill, jibo); + bridge.start(); + return bridge; + } catch (e) { + skill && skill.log && skill.log.warn && skill.log.warn("AI Bridge init failed", e); + return null; + } +}; diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js index a796252e..c9641c1b 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/be/postinit.js @@ -1,8 +1,27 @@ "use strict"; const jibo = require('jibo'); +let rlog = null; +try { + rlog = require('./robot-logger'); +} catch (e) { + // ignore +} + exports.postInit = function (err) { this.log.debug('postInit !!'); + + try { + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE] postInit starting'); + } + if (rlog) { + rlog.info('be', 'postInit starting'); + } + } catch (e) { + // ignore + } + if (err) { this.log.error(err); this.initDoneCallback(err); @@ -34,6 +53,31 @@ exports.postInit = function (err) { catch (e) { this.log.warn('Dynamic skills patch failed (non-fatal):', e.message || e); } + + // Optional: AI Bridge (modular; can run models off-robot for now) + try { + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE] initializing AI bridge'); + } + require('./ai-bridge').initAIBridge(this, jibo); + this.log.info('AI bridge initialized'); + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE] AI bridge initialized'); + } + } + catch (e) { + this.log.warn('AI bridge init failed (non-fatal):', e.message || e); + try { + if (rlog && typeof rlog.raw === 'function') { + rlog.raw('[BE] AI bridge init failed: ' + String(e && (e.stack || e.message || e))); + } + if (rlog) { + rlog.warn('be', 'ai bridge init failed', { err: String(e && (e.stack || e.message || e)) }); + } + } catch (e2) { + // ignore + } + } jibo.face.views.changeView({ removeAll: true, leaveEmpty: true }, () => { this.selectFirstSkill(this.launchFirstSkill.bind(this)); diff --git a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo-embodied-dialog/lib/jibo-embodied-dialog.js b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo-embodied-dialog/lib/jibo-embodied-dialog.js index d0129aca..92253f1a 100644 --- a/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo-embodied-dialog/lib/jibo-embodied-dialog.js +++ b/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/jibo-embodied-dialog/lib/jibo-embodied-dialog.js @@ -684,6 +684,7 @@ const log = logs.listenLog; const USE_CENTER_ROBOT = false; const TIMEOUT = 3000; const ENGAGE_HJ_FAILSAFE_TIMEOUT_MS = 6000; +const LISTENING_FAILSAFE_TIMEOUT_MS = 12000; const ANIM_CLEAR_TIMEOUT = 'ANIM_CLEAR_TIMEOUT'; class EmbodiedListen extends jibo_state_machine_1.StateMachine { constructor(_ed) { @@ -781,6 +782,8 @@ class EmbodiedListen extends jibo_state_machine_1.StateMachine { this._idle.onEntry = () => { log.debug('Entering Idle'); this._jibo.performance.log('EmbodiedListenReturnedToIdleSkill'); + // Safety: ensure we don't stay in a listening LED state indefinitely. + Utils_1.Utils.setLED(this._jibo, Assets_1.Led.OFF); this.eventsOut.finished.emit(); this._jibo.timer.removeListener('update', updateBind); if (this._pendingActiveMode) { @@ -966,19 +969,59 @@ class EmbodiedListen extends jibo_state_machine_1.StateMachine { this._listening.addTypedEventTransition(this.eventsIn.cloudFinished, this._offExpression); this._listening.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); let incrementalHandler = this._handleIncremental.bind(this); + let listeningFailsafe = null; this._listening.onEntry = () => { log.debug('Entering Listening'); this.eventsIn.sos.on(incrementalHandler); + if (listeningFailsafe) { + clearTimeout(listeningFailsafe); + } + listeningFailsafe = setTimeout(() => { + try { + if (this.current === this._listening) { + Utils_1.Utils.setLED(this._jibo, Assets_1.Led.OFF); + this._listening.transitionTo(this._idle); + } + } + catch (e) { + // ignore + } + }, LISTENING_FAILSAFE_TIMEOUT_MS); }; this._listening.onExit = () => { this.eventsIn.sos.removeListener(incrementalHandler); + if (listeningFailsafe) { + clearTimeout(listeningFailsafe); + listeningFailsafe = null; + } }; this._thinking.addTypedEventTransition(this.eventsIn.cloudFinished, this._waitForAnimFinish); this._thinking.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); + let thinkingFailsafe = null; this._thinking.onEntry = () => __awaiter(this, void 0, void 0, function* () { log.debug('Entering Thinking (+ Off Expression)'); Utils_1.Utils.setLED(jibo, Assets_1.Led.OFF); + if (thinkingFailsafe) { + clearTimeout(thinkingFailsafe); + } + thinkingFailsafe = setTimeout(() => { + try { + if (this.current === this._thinking) { + Utils_1.Utils.setLED(this._jibo, Assets_1.Led.OFF); + this._thinking.transitionTo(this._idle); + } + } + catch (e) { + // ignore + } + }, LISTENING_FAILSAFE_TIMEOUT_MS); }); + this._thinking.onExit = () => { + if (thinkingFailsafe) { + clearTimeout(thinkingFailsafe); + thinkingFailsafe = null; + } + }; }); } dispose() { diff --git a/V3.1/build/usr/local/bin/jibo-audio-convert b/V3.1/build/usr/local/bin/jibo-audio-convert new file mode 100644 index 00000000..e0fb16f8 Binary files /dev/null and b/V3.1/build/usr/local/bin/jibo-audio-convert differ diff --git a/V3.1/build/usr/local/bin/jibo-audio-convertall b/V3.1/build/usr/local/bin/jibo-audio-convertall new file mode 100644 index 00000000..6f54d920 --- /dev/null +++ b/V3.1/build/usr/local/bin/jibo-audio-convertall @@ -0,0 +1,28 @@ +#!/bin/sh + +CONVERT="jibo-audio-convert -f planar -F interleaved -w" + +if [ -z "$1" ]; then + LOC="/tmp" +else + LOC="$1" +fi + +if [ -s $LOC/sin.raw ]; then + $CONVERT --infile $LOC/sin.raw --outfile $LOC/sin.wav -c 6 -r 16000 +fi +if [ -s $LOC/sout.raw ]; then + $CONVERT --infile $LOC/sout.raw --outfile $LOC/sout.wav -c 2 -r 16000 +fi +if [ -s $LOC/ref.raw ]; then + $CONVERT --infile $LOC/ref.raw --outfile $LOC/ref.wav -c 2 -r 16000 +fi +if [ -s $LOC/rin.raw ]; then + $CONVERT --infile $LOC/rin.raw --outfile $LOC/rin.wav -c 1 -r 48000 +fi +if [ -s $LOC/rout.raw ]; then + $CONVERT --infile $LOC/rout.raw --outfile $LOC/rout.wav -c 1 -r 48000 +fi +if [ -s $LOC/hotphrase.raw ]; then + $CONVERT --infile $LOC/hotphrase.raw --outfile $LOC/hotphrase.wav -c 2 -r 16000 +fi diff --git a/V3.1/build/usr/local/bin/jibo-audio-dump-state b/V3.1/build/usr/local/bin/jibo-audio-dump-state new file mode 100644 index 00000000..9b44eeca Binary files /dev/null and b/V3.1/build/usr/local/bin/jibo-audio-dump-state differ diff --git a/V3.1/build/usr/local/bin/jibo-audio-ping-test b/V3.1/build/usr/local/bin/jibo-audio-ping-test new file mode 100644 index 00000000..c0ba773a Binary files /dev/null and b/V3.1/build/usr/local/bin/jibo-audio-ping-test differ diff --git a/V3.1/build/usr/local/bin/jibo-audio-pong-test b/V3.1/build/usr/local/bin/jibo-audio-pong-test new file mode 100644 index 00000000..15e5fb31 Binary files /dev/null and b/V3.1/build/usr/local/bin/jibo-audio-pong-test differ diff --git a/V3.1/build/usr/local/bin/jibo-audio-pump-device b/V3.1/build/usr/local/bin/jibo-audio-pump-device new file mode 100644 index 00000000..c7194174 Binary files /dev/null and b/V3.1/build/usr/local/bin/jibo-audio-pump-device differ diff --git a/V3.1/build/usr/local/bin/jibo-audio-service b/V3.1/build/usr/local/bin/jibo-audio-service new file mode 100644 index 00000000..00571239 Binary files /dev/null and b/V3.1/build/usr/local/bin/jibo-audio-service differ diff --git a/V3.1/build/usr/local/etc/jibo-asr-service.json b/V3.1/build/usr/local/etc/jibo-asr-service.json new file mode 100644 index 00000000..3dd50652 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-asr-service.json @@ -0,0 +1,84 @@ +{ + "webCore" : { + "serverPort": 8088, + "fileRoot": "/usr/local/var/www/asrservice", + "requestLogging": false + }, + "AsrService" : { + "cloud_establish_http_timeout": 5000, + "language": "en-US", + "post_to_performance_service": true, + "log_audio": true, + "log_text" : true, + "log_path" : "/var/log/asr", + "log_level" : "INFO", + "log_server_url" : "https://speech-logging.jibo.com/logdrop/logdrop.py", + "speaker_id_resource_path" : "/var/jibo/asr/", + "name_learning_resource_path" : "/usr/local/share/asr/namelearning", + "name_learning_temp_path" : "/var/jibo/asr/namelearning_temp/", + "name_learning_nbest" : 70, + "active_sleep_duration" : 5000, + "idle_sleep_duration" : 50000, + "block_duration" : 50, + "audio_loop_sleep_us": 10000, + "use_nuance_upload_voc": false, + "dictation_type" : "dictation", + "nuance_uId" : "b8fb02f2c5794963aaafb8c716ef384c", + "contacts_checksum": "", + "loop_checksum": "1", + "customs_checksum": "", + "upload_voc_url": "ws.nuancemobility.net", + "cloud_url": "https://jibo-ncs-engusa-http.nuancemobility.net/NmspServlet/", + "cloud_appid": "HTTP_NMDPPRODUCTION_Jibo_Jibo_Robot_20151231124503", + "cloud_appkey": "a8c18159a8e3ca49471c56d867552bc77693ccdcc041375ee97b7c867160ae1a212f73c9123d1359596931c0be5c8734ef5310af95470d7ec3890434e9b24e0b", + "upload_voc_rootcert": "", + "google_credential": "/usr/local/share/asr/google_asr/credentials-key.json", + "fadeout_duration": 5000000, + "max_logfile_size": 1000, + "log_upload_time_interval": 60000, + "min_available_log_partition_space": 5000, + "max_asr_log_dir_size_before_upload_trigger": 10000, + "max_asr_log_dir_size": 12000, + "size_to_free_up_when_dir_overflowing": 1000, + "asr_resource_path" : "/usr/local/share/asr/", + "max_memory": 150000, + "wipable_files": ["/var/log/asr/*.pcm", "/var/log/asr/*.wav", + "/var/log/asr/*.log", "/var/jibo/asr/sensory_data_td/client_model.bin", "/var/jibo/asr/sensory_data_td/audio/*", "/var/jibo/asr/namelearning_temp/*" + ], + "resident_task" : "{\"command\":\"start\",\"task_id\":\"task0\",\"audio_source_id\":\"alsa1\",\"hotphrase\":\"hey_jibo\",\"request_id\":\"resident_hey_jibo_self_start\",\"residency\":true}", + "resident_audio_channel" : "{\"action\":\"start\", \"audio_source_id\":\"alsa1\", \"wav_files\":[], \"audio_source\":\"alsa\", \"request_id\":\"self_start_audio_source_request_id\"}", + "task_templates" : { + "hey_jibo_resident": {"input_template": "{\"name\":\"hey jibo\",\"path\":\"/usr/local/share/asr/hey_jibo\",\"timeout\":0} * {\"name\":\"Speaker ID TD\",\"path\":\"/usr/local/share/asr/sensory_spkr_id_td\",\"audio_tail_length\":1}", + "emitting_recogs": ["hey jibo", "Speaker ID TD"]}, + "hey_jibo": {"input_template": "{\"name\":\"hey jibo\",\"path\":\"/usr/local/share/asr/hey_jibo\",\"timeout\":0} * ({\"name\":\"pcmwriter\",\"path\":\"/usr/local/share/asr/pcm_writer\",\"timeout\":0,\"audio_tail_length\":0, \"audio_overshoot_duration\":0} | {\"name\":\"Speaker ID TD\",\"path\":\"/usr/local/share/asr/sensory_spkr_id_td\",\"audio_tail_length\":1})", + "emitting_recogs": ["hey jibo", "Speaker ID TD"] + }, + "cloud": {"input_template":"({\"name\":\"google_asr\",\"path\":\"/usr/local/share/asr/google_asr\",\"timeout\":14000,\"bargein\":false,\"nbest\":1,\"speaker_name\":\"\",\"incremental\":false,\"audio_tail_length\":300}| {\"name\":\"sensory_sdet\",\"path\":\"/usr/local/share/asr/jibo_energy_fake_eos\",\"timeout\":50000,\"bargein\":false,\"nbest\":1,\"speaker_name\":\"\",\"incremental\":false})", + "emitting_recogs": ["google_asr","sensory_sdet"] + }, + "hey_jibo_cloud": {"input_template":"{\"name\":\"hey jibo\",\"path\":\"/usr/local/share/asr/hey_jibo\",\"timeout\":0,\"bargein\":true,\"nbest\":1,\"speaker_name\":\"\",\"incremental\":false,\"speaker_id\":true} * ({\"name\":\"Speaker ID TD\",\"path\":\"/usr/local/share/asr/sensory_spkr_id_td\",\"audio_tail_length\":1} & ({\"name\":\"pcmwriter\",\"path\":\"/usr/local/share/asr/pcm_writer\",\"timeout\":0,\"audio_tail_length\":400, \"audio_overshoot_duration\":0, \"prebuffer\":true} | {\"name\":\"google_asr\",\"path\":\"/usr/local/share/asr/google_asr\",\"timeout\":14000,\"bargein\":false,\"nbest\":1,\"speaker_name\":\"\",\"incremental\":false,\"trim_audio_tail\":true,\"wakeup_phrase_detection\":false,\"audio_tail_length\":0}|{\"name\":\"sensory_sdet\",\"path\":\"/usr/local/share/asr/jibo_energy_fake_eos\",\"timeout\":50000,\"bargein\":false,\"nbest\":1,\"speaker_name\":\"\",\"incremental\":false}))", + "emitting_recogs": ["hey jibo", "Speaker ID TD", "google_asr","sensory_sdet"] + } + }, + "rewrite_rules" : { + "log_audio_no_trigger" : "^(?!.*?hey_jibo)(.*)->{\"name\":\"pcmwriter\",\"path\":\"/usr/local/share/asr/pcm_writer\",\"timeout\":0,\"audio_tail_length\":300, \"audio_overshoot_duration\":0,\"prebuffer\":false} | ($1)", + "namelearning_EOS" : "^(.*?\"name\":\"\\s*name_learning.*)->{\"name\":\"jibo_energy_eos\",\"path\":\"/usr/local/share/asr/jibo_energy_eos\",\"timeout\":10000,\"bargein\":false,\"nbest\":1,\"speaker_name\":\"\",\"incremental\":false} | ($1)" + } + }, + "logging" : { + "jibo_message_prefix": "C", + "loggers" : { + "root": { + "level": "information" + }, + "l1" : { + "name" : "ASRService", + "level" : "information" + }, + "l2" : { + "name" : "Application", + "level" : "information" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-audio-service.json b/V3.1/build/usr/local/etc/jibo-audio-service.json new file mode 100644 index 00000000..4b413ebf --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-audio-service.json @@ -0,0 +1,111 @@ +{ + "WebCore" : { + "serverPort" : 8383, + "fileRoot": "/usr/local/var/www/audioservice", + "requestLogging": false + }, + "AudioService": { + "registryPort": 8181, + "serverPort": 8383, + "AlsaAudio": { + "energy": { + "high_freq": 1000, + "high_q": 2.0, + "mid_freq": 500, + "mid_q": 2.0, + "low_freq": 200, + "low_q": 1.0 + }, + "LocationEstimate": { + "adjust_confidence_scale": 0.9, + "alternate_confidence_scale": 0.5 + }, + "LocationHistory": { + "adjust_confidence_scale": 0.9, + "alternate_confidence_scale": 0.5 + }, + "PlaybackDetector": { + "quiet_threshold": 50, + "sub_samples": 10, + "consec_active_threshold": 2, + "consec_inactive_threshold": 10 + } + }, + "alsaCaptureDevice": "hw:ADC", + "alsaPlaybackDevice": "hw:TLV320DAC3100", + "routerCaptureDevice": "services_sink", + "routerCaptureLatency": 64000, + "routerPlaybackDevice": "as_source", + "routerPlaybackLatency": 64000, + "useMMFx": true, + "SEDiags": { + "injector_white_noise_rx_magnitude_q15": [2048], + "injector_white_noise_tx_magnitude_q15": [2048], + "senr_res_echo_fullband_weight_factor": [3] + }, + "kinematic_model": "/usr/local/etc/jibo-kinematic-model.json", + "hotphrase_begin_time": 0.800, + "hotphrase_end_time": 0.200, + "log_hotphrase_audio": false, + "enable_vis_sockets": false, + "ping_pong_test": false, + "log_directory": "/tmp", + "moving_noise_enabled": false, + "moving_noise_file": "/usr/local/share/audioservice/brown_noise.wav", + "moving_noise_duration": 2.0, + "moving_noise_min_vel": 0.05 + }, + "ErrorTracker": { + "views": [ + { + "name": "AudioService", + "errors": [ + "ALSA_CAPTURE_XRUN", + "ALSA_PLAYBACK_XRUN", + "ROUTER_CAPTURE_XRUN", + "ROUTER_CAPTURE_SHORT", + "ROUTER_PLAYBACK_XRUN", + "ROUTER_PLAYBACK_SHORT", + "ROUTER_GET_UNDERRUN", + "ROUTER_PUT_OVERRUN", + "ROUTER_READ_OVERRUN", + "ROUTER_WRITE_UNDERRUN", + "ROUTER_ON_STREAM_OVERFLOW", + "ROUTER_ON_STREAM_UNDERFLOW" + ] + } + ] + }, + "logging": { + "loggers": { + "root": { + "level": "notice" + }, + "pr": { + "name": "PulseRouter", + "level": "notice" + }, + "ar": { + "name": "AlsaRouter", + "level": "notice" + }, + "as": { + "name": "AudioService", + "level": "notice" + }, + "aa": { + "name": "AlsaAudio", + "level": "notice" + }, + "aalh": { + "name": "AlsaAudio.LocationHistory", + "level": "notice" + }, + "aala": { + "name": "AlsaAudio.LocationAccumulator", + "level": "notice" + } + } + } +} + diff --git a/V3.1/build/usr/local/etc/jibo-body-service.json b/V3.1/build/usr/local/etc/jibo-body-service.json new file mode 100644 index 00000000..e3891ac7 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-body-service.json @@ -0,0 +1,177 @@ +{ + "WebCore" : { + "serverPort" : 8282, + "fileRoot": "/usr/local/var/www/bodyservice", + "requestLogging": false + }, + "bodyBoard" : { + "pelvisDevice" : "/dev/ttyTHS1", + "pelvisOffset" : 2.742, + "pelvisFlipped" : true, + "pelvisVelLimit" : 10.0, + "pelvisAccLimit" : 30.0, + "torsoDevice" : "/dev/ttyTHS1", + "torsoOffset" : 0.052, + "torsoFlipped" : true, + "torsoVelLimit" : 10.0, + "torsoAccLimit" : 30.0, + "neckDevice" : "/dev/ttyTHS0", + "neckOffset" : 0.061, + "neckFlipped" : true, + "neckVelLimit" : 10.0, + "neckAccLimit" : 30.0, + "headTouchMapping" : [3, 5, 4, 0, 2, 1], + "hatchLowThreshold": 1.0, + "hatchHighThreshold": 1.5, + "lowBattCapHiThresh": 0.42, + "lowBattCapLowThresh": 0.35, + "battMinTemp": 0.0, + "battMaxTemp": 47.0, + "motorCmdTimeout": 500, + "ledCmdTimeout": 5000 + }, + "BodyService" : { + "registryPort": 8181, + "serverPort": 8282, + "backlightDutyFile": "/sys/class/backlight/backlight.9/brightness", + "backlightPeriodFile": "/sys/class/backlight/backlight.9/max_brightness", + "fanControl": false, + "fanPWMFile": "/sys/class/hwmon/hwmon5/pwm1", + "fanPWMMax": 255, + "fanOnTemp" : 70, + "fanOffTemp" : 60, + "mainBoardTempFile": "/sys/class/thermal/thermal_zone0/temp", + "cpuThermalZone": "/sys/class/thermal/thermal_zone1", + "fanCurrentFile": "/sys/class/hwmon/hwmon6/in3_input", + "fanCurrentResistor": 0.5, + "pmicTempFile": "/sys/class/hwmon/hwmon6/temp1_input", + "cpuDCDC1TempFile": "/sys/class/hwmon/hwmon6/temp2_input", + "cpuDCDC2TempFile": "/sys/class/hwmon/hwmon6/temp3_input", + "coreDCDCTempFile": "/sys/class/hwmon/hwmon6/temp4_input", + "gpuDCDC1TempFile": "/sys/class/hwmon/hwmon6/temp5_input", + "gpuDCDC2TempFile": "/sys/class/hwmon/hwmon6/temp6_input", + "minCommandRate": 4.0, + "maxCommandRate": 40.0, + "commandOKCount": 25, + "cpuTempHiThresh": 90, + "cpuTempLowThres": 85, + "kinematic_model": "/usr/local/etc/jibo-kinematic-model.json" + }, + "motionLimiter" : { + "maxTorque": 70, + "maxTime": 0.250, + "velAlphaDn": 0.80, + "velAlphaUp": 0.05, + "kVelLim": 0.25, + "logFile": "/tmp/motion-limiter.log", + "logEnable": false, + "disable": false, + "unindexedAcc": 10.0, + "unindexedVel": 1.0 + }, + "imu" : { + "driver": { + "deviceName": "/dev/spidev1.0", + "calibrationFile": "/var/jibo/imu/imu-cal.json" + }, + "fallDetector": { + "low_g_start_thresh": 2.0, + "low_g_end_thresh": 4.0, + "high_g_start_thresh": 30.0, + "high_g_end_thresh": 25.0, + "max_low_high_sep": 0.1, + "min_low": 0.1, + "max_high": 0.1, + "min_falling": 1.0 + }, + "gyroOffset": { + "avg_samples": 250, + "still_seconds": 2.0, + "thresh_acc": 0.20, + "thresh_rot": 0.05 + }, + "movingDetector": { + "a_acc": 0.01, + "a_rot": 0.02, + "thresh_acc": 0.30, + "thresh_rot": 0.25 + }, + "upEstimator": { + "alpha": 0.5 + }, + "upMovingDetector": { + "a_acc": 0.50, + "a_rot": 0.05, + "thresh_acc": 0.15, + "thresh_rot": 0.05 + }, + "tip_start_thresh": 0.7, + "tip_end_thresh": 0.9 + }, + "ErrorTracker":{ + "views": [ + { + "name": "BodyService", + "errors": [ + "NECK_THERMISTOR_HIGH_FAULT", + "NECK_THERMISTOR_LOW_FAULT", + "NECK_ENCODER_FAULT", + "NECK_STALL_FAULT", + "NECK_BOARD_TIMEOUT", + "NECK_POWER_FAULT_COUNT", + "NECK_STALL_FAULT_COUNT", + "NECK_ENCODER_FAULT_COUNT", + "NECK_ENCODER_BACKWARDS_FAULT_COUNT", + "NECK_THERMISTOR_FAULT_COUNT", + "NECK_POWER_FAULT", + "NECK_ENCODER_BACKWARDS_FAULT", + + "TORSO_THERMISTOR_HIGH_FAULT", + "TORSO_THERMISTOR_LOW_FAULT", + "TORSO_ENCODER_FAULT", + "TORSO_STALL_FAULT", + "TORSO_BOARD_TIMEOUT", + "TORSO_POWER_FAULT_COUNT", + "TORSO_STALL_FAULT_COUNT", + "TORSO_ENCODER_FAULT_COUNT", + "TORSO_ENCODER_BACKWARDS_FAULT_COUNT", + "TORSO_THERMISTOR_FAULT_COUNT", + "TORSO_POWER_FAULT", + "TORSO_ENCODER_BACKWARDS_FAULT", + + "PELVIS_THERMISTOR_HIGH_FAULT", + "PELVIS_THERMISTOR_LOW_FAULT", + "PELVIS_ENCODER_FAULT", + "PELVIS_STALL_FAULT", + "PELVIS_BOARD_TIMEOUT", + "PELVIS_POWER_FAULT_COUNT", + "PELVIS_STALL_FAULT_COUNT", + "PELVIS_ENCODER_FAULT_COUNT", + "PELVIS_ENCODER_BACKWARDS_FAULT_COUNT", + "PELVIS_THERMISTOR_FAULT_COUNT", + "PELVIS_POWER_FAULT", + "PELVIS_ENCODER_BACKWARDS_FAULT", + + "NOT_UPRIGHT_LOCKOUT", + "NO_BATTERY_LOCKOUT", + "FALLING_LOCKOUT", + "CPU_TEMP_HIGH", + "BATTERY_TEMP_LOW", + "BATTERY_TEMP_HIGH", + "BATTERY_LOW", + "FALL_DETECTED", + "BB_PACKET_INVALID", + "BB_PACKET_WRONG_DEST", + "BB_PACKET_WRONG_SRC" + ] + } + ] + }, + "logging": { + "loggers": { + "root": { + "level": "notice" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-camera-calibrator.json b/V3.1/build/usr/local/etc/jibo-camera-calibrator.json new file mode 100644 index 00000000..91e4026f --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-camera-calibrator.json @@ -0,0 +1,208 @@ +{ + "calibrator" : { + "setup" : { + "output": "/var/jibo/lps", + "num_distances": 3, + "circleDistance": 0.117, + "pixSize": 4e-6, + "recordDelay" : 1000, + "maxTries": 5, + "num_calibrators": 2, + "calibrators" : [ + { + "filename": "CameraModelParamsR.json", + "frame_name" : "right_eye", + "debug":false + }, + { + "filename": "CameraModelParamsL.json", + "frame_name" : "left_eye", + "debug":false + } + ], + "num_transforms": 1, + "transforms" : [ + { + "filename": "InterCameraTransform.json", + "cameraA": 0, + "cameraB": 1 + + } + ], + "num_positions" : 10, + "positions": [ + { + "message" : "Wide Position 1 - press Enter", + "num_cameras": 1, + "cameras" : [ 0 ], + "saveBlobData" : true, + "fileMessage" : "wide_pos_1", + "saveImage" : true, + "saveBlobToCal": true + }, + { + "message" : "Wide Position 2 - press Enter", + "num_cameras": 1, + "cameras" : [ 0 ], + "saveBlobData" : true, + "fileMessage" : "wide_pos_2", + "saveImage" : true, + "saveBlobToCal": true + }, + { + "message" : "Wide Position 3 - press Enter", + "num_cameras": 1, + "cameras" : [ 0 ], + "saveBlobData" : true, + "fileMessage" : "wide_pos_3", + "saveImage" : true, + "saveBlobToCal": true + }, + { + "message" : "Wide Position 4 - press Enter", + "num_cameras": 1, + "cameras" : [ 0 ], + "saveBlobData" : true, + "fileMessage" : "wide_pos_4", + "saveImage" : true, + "saveBlobToCal" : true + }, + { + "message" : "Wide Position 5- press Enter", + "num_cameras": 1, + "cameras" : [ 0 ], + "saveBlobData" : true, + "fileMessage" : "wide_pos_5", + "saveImage" : true, + "saveBlobToCal" : true + }, + { + "message" : "Both Position 1 - press Enter", + "num_cameras": 2, + "cameras" : [ 0, 1 ], + "saveBlobData" : true, + "fileMessage" : "both_pos_1", + "saveImage" : true, + "saveBlobToCal": true + }, + { + "message" : "Both Position 2 - press Enter", + "num_cameras": 2, + "cameras" : [ 0, 1 ], + "saveBlobData" : true, + "fileMessage" : "both_pos_2", + "saveImage" : true, + "saveBlobToCal": true + }, + { + "message" : "Both Position 3 - press Enter", + "num_cameras": 2, + "cameras" : [ 0, 1 ], + "saveBlobData" : true, + "fileMessage" : "both_pos_3", + "saveImage" : true, + "saveBlobToCal": true + }, + { + "message" : "Both Position 4 - press Enter", + "num_cameras": 2, + "cameras" : [ 0, 1 ], + "saveBlobData" : true, + "fileMessage" : "both_pos_4", + "saveImage" : true, + "saveBlobToCal": true + }, + { + "message" : "Both Position 5 - press Enter", + "num_cameras": 2, + "cameras" : [ 0, 1 ], + "saveBlobData" : true, + "fileMessage" : "both_pos_5", + "saveImage" : true, + "saveBlobToCal": true + } + ] + }, + "camera": { + "type": "CUDA", + "cuda": { + "devices": [ + { + "enabled": true, + "name": "Camera-0", + "path": "/dev/video1", + "bufferPoolSize": 4, + "width": 1280, + "height": 720, + "hFlip":true, + "vFlip":true, + "gamma" : { + "R": 1.0, + "G": 1.0, + "B": 1.0 + }, + "ae" : { + "exposureP":0.05, + "gainP":0.02, + "targetY":0.35, + "errorY":0.0001 + }, + "awb" : { + "grayThreshold":0.3, + "numSamples":1000, + "P":0.15, + "targetU":0.5, + "targetV":0.53, + "seed":0 + } + }, + { + "enabled": true, + "name": "Camera-1", + "path": "/dev/video0", + "bufferPoolSize": 4, + "width": 1280, + "height": 720, + "hFlip":true, + "vFlip":true, + "gamma" : { + "R": 1.0, + "G": 1.0, + "B": 1.0 + }, + "ae" : { + "exposureP":0.05, + "gainP":0.02, + "targetY":0.35, + "errorY":0.0001 + }, + "awb" : { + "grayThreshold":0.3, + "numSamples":1000, + "P":0.15, + "targetU":0.5, + "targetV":0.53, + "seed":0 + } + } + ], + "cuda" : { + + } + }, + "file": { + "videos": [ + "videos/video-000/video.json", + "videos/video-001/video.json" + ], + "format": "YUV420p" + }, + "v4l2": { + "devices": { + "left": "/dev/video0", + "right": "/dev/video1" + } + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-certification-service.json b/V3.1/build/usr/local/etc/jibo-certification-service.json new file mode 100644 index 00000000..416ff40c --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-certification-service.json @@ -0,0 +1,32 @@ +{ + "WebCore": { + "serverPort": 9292, + "fileRoot": "/usr/local/var/www/certificationservice", + "requestLogging": false + }, + "CertificationService": { + "registryPort": 8181, + "serverPort": 9292, + "enableDebugSocket": true + }, + "ErrorTracker": { + "views": [ + { + "name": "CertificationService", + "errors": [] + } + ] + }, + "logging": { + "channels": { + "splitter": { + "channels": "syslog" + } + }, + "loggers": { + "root": { + "level": "notice" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-identity-service.json b/V3.1/build/usr/local/etc/jibo-identity-service.json new file mode 100644 index 00000000..27f205c3 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-identity-service.json @@ -0,0 +1,87 @@ +{ + "WebCore": { + "serverPort": 8489, + "fileRoot": "/usr/local/var/www/identity" + }, + "IdentityService":{ + "registryPort":8181, + "serverPort":8489, + "global" : { + "face_landmark_68" : { + "file" : + "/usr/local/share/lps/shape_predictor_68_face_landmarks.dat" + } + }, + "engine" : { + "processor":{ + "max_requests":10, + "interval":1000 + }, + "manager" : { + "criteria" : { }, + "kind" : "face", + "max_subjects" : 16, + "max_examples" : 5, + "max_attempts" : 3, + "directory" : "/var/jibo/identity/faces", + "detector": { + "criteria": {}, + "min_hint_width" : 40, + "min_hint_height" : 40 + } + }, + "identifier" : { + "identifier_type": "deepid", + "eigenfaces" : { + "type" : "eigenfaces_identifier", + "name" : "eigenfaces", + "kind" : "face", + "size" : 128, + "radius" : 5, + "attenuation" : 100.0, + "filename" : "/var/jibo/identity/eigenfaces.data" + }, + "deepid" : { + "type" : "deepid_identifier", + "name" : "deepid", + "kind" : "face", + "solver" : "/usr/local/share/lps/deepid/CASIA_solver.prototxt", + "model" : "/usr/local/share/lps/deepid/CASIA_train_test.prototxt", + "snapshot" : "/usr/local/share/lps/deepid", + "weights" : "/usr/local/share/lps/deepid/CASIA_iter_666000.caffemodel", + "iterations" : 50000, + "size" : 128, + "sigint_effect" : "stop", + "sighup_effect" : "snapshot", + "jlf" : "/var/jibo/identity/deepid", + "filename" : "/var/jibo/identity/deepid.data", + "gpu_id":0 + }, + "resnetfaceid" : { + "type" : "resnetfaceid_identifier", + "name" : "resnetfaceid", + "kind" : "face", + "model" : "/usr/local/share/identity/resnet/dlib_face_recognition_resnet_model_v1.dat", + "size" : 150, + "jlf" : "/var/jibo/identity/resnet", + "filename" : "/var/jibo/identity/resnetfaceid.data" + } + } + } + }, + "logging" : { + "loggers" : { + "root": { + "level": "warning" + }, + "l8" : { + "name" : "IdentificationManager", + "level" : "information" + }, + "l11" : { + "name" : "ImageIdentifier", + "level" : "information" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-jetstream-service.json b/V3.1/build/usr/local/etc/jibo-jetstream-service.json new file mode 100644 index 00000000..eb303abc --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-jetstream-service.json @@ -0,0 +1,158 @@ +{ + "webCore": { + "serverPort": 8090, + "fileRoot": "/usr/local/var/www/jetstream", + "requestLogging": false + }, + "JetstreamService": { + "stacktracing": false, + "registryPort": 8181, + "serverPort": 8090 + }, + "JetService": { + "log_directory": "/var/log/jetstream" + }, + "HubClient": { + "proactive_url": "/v1/proactive", + "listen_url": "/v1/listen", + "listen_language": "en-US", + "xxx_override": { + "hub_port": 9000, + "hub_hostname": "172.24.84.137", + "entrypoint_hostname": "dev-entrypoint.jibo.com" + }, + "region-settings": { + "comment": "This is a switch, selected by the 'region' field setting in the robot's /var/jibo/credentials.json file", + "dev-entrypoint": { + "hub_port": 443, + "hub_hostname": "dev-hub.jibo.com", + "entrypoint_hostname": "dev-entrypoint.jibo.com" + }, + "alpha-entrypoint": { + "hub_port": 443, + "hub_hostname": "alpha-hub.jibo.com", + "entrypoint_hostname": "alpha-entrypoint.jibo.com" + }, + "stg-entrypoint": { + "hub_port": 443, + "hub_hostname": "stg-hub.jibo.com", + "entrypoint_hostname": "stg-entrypoint.jibo.com" + }, + "preprod-entrypoint": { + "hub_port": 443, + "hub_hostname": "preprod-hub.jibo.com", + "entrypoint_hostname": "preprod-entrypoint.jibo.com" + }, + "api": { + "hub_port": 443, + "hub_hostname": "neo-hub.jibo.com", + "entrypoint_hostname": "api.jibo.com" + } + }, + "encoding_type_comment": "This can be either LINEAR16, FLAC, or OGG_OPUS", + "encoding_type": "OGG_OPUS", + "encoding-settings": { + "OGG_OPUS": { + "streaming_rate": 1.2, + "channels": 1, + "sample_rate": 16000, + "bitrate": 64000, + "vbr": true + }, + "FLAC": { + "streaming_rate": 3.0, + "channels": 1, + "sample_rate": 16000, + "bps": 16 + } + } + }, + "AudioChannel": { + "block_duration_ms": 50, + "period_size": 85, + "buffer_size": 2048 + }, + "HJLogger": { + "speech_analytics_log_path": "/var/log/jetstream/hj_logs", + "comment-logging-fraction": "The probability that an HJ utterance will be logged -- range 0-1", + "logging_probability": 0.05, + "start_margin_ms": 500, + "end_margin_ms": 500 + }, + "RecogHJ": { + "config_path": "/usr/local/share/asr/hey_jibo", + "comment": "Positive margin values widen the HJ Phrase spotter's standard endpoints. The end_margin_ms also compensates for the Phrase Spotter's premature endpoint, which is around the beginning O in jibO", + "hj_start_margin_ms": 0, + "hj_end_margin_ms": 100 + }, + "HubAsr": { + "global_sosTimeout_sec": 3, + "global_maxSpeechTimeout_sec": 20, + "local_sosTimeout_sec": 2, + "local_maxSpeechTimeout_sec": 20 + }, + "RecogSpeakerID": { + "ubm_path": "/usr/local/share/asr/sensory_data_td/", + "client_path": "/var/jibo/asr/sensory_data_td/", + "comment": "change next property to 'sensory_log_path' to enable Sensory authenticator logging, 'xxxsensory_log_path' to disable", + "xxxsensory_log_path": "/var/jibo/asr/sensory_auth_logs", + "threshold": -1.2, + "confidence_margin": 2, + "margin-comment": "positive values for start and end margins widen the space around the speaker-id HJ", + "id_start_margin_ms": 250, + "comment2":"id_end_margin is used when retrying to ID with an audio segment, it also controls the end margin of the logged utts", + "id_end_margin_ms": 300, + "PHRASEver": "OFF" + }, + "RecogSpeakerEnroll": { + "ubm_path": "/usr/local/share/asr/sensory_data_td/", + "client_path": "/var/jibo/asr/sensory_data_td/", + "comment": "change next property to 'sensory_log_path' to enable Sensory enroller logging, 'xxxsensory_log_path' to disable", + "xxxsensory_log_path": "/var/jibo/asr/sensory_enroller_logs", + "speech_analytics_log_path": "/var/log/jetstream/enroller_logs", + "minEnrollUtts": 6, + "checkQuality": "LOW", + "margin-comment": "positive values for the start margin increase the amount of silence fed to the enroller before the HJ occurs", + "enroll_start_margin_ms": 250 + }, + "RecogEOS": { + "resource_path": "/usr/local/share/asr/jibo_energy_eos" + }, + "RecogNameLearning": { + "resource_path": "/usr/local/share/asr/namelearning", + "temp_path": "/var/jibo/asr/namelearning_temp/", + "speech_analytics_log_path": "/var/log/jetstream/namelearning_logs", + "g2p_service": "http://localhost:8089/tts_nbest_prons" + }, + "logging": { + "jibo_message_prefix": "C", + "channels": { + "console": { + "class": "ConsoleChannel", + "pattern": "%Y-%m-%d %H:%M:%S %s: [%p] %t" + }, + "syslog": { + "class": "RFC_5424_Channel", + "name": "jibo-jetstream-service", + "facility": "SYSLOG_DAEMON" + }, + "splitter": { + "class": "SplitterChannel", + "channels": "syslog" + } + }, + "loggers": { + "root": { + "level": "debug" + }, + "l1": { + "name": "JetService", + "level": "debug" + }, + "l2": { + "name": "Application", + "level": "debug" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-kinematic-model.json b/V3.1/build/usr/local/etc/jibo-kinematic-model.json new file mode 100644 index 00000000..e59bba89 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-kinematic-model.json @@ -0,0 +1,220 @@ +{ + "transforms": [ + { + "name": "root", + "frame_type": "STATIC", + "transform_type": "DH", + "parent": "", + "parameters": { + "d": 0, + "r": 0, + "a": 0, + "initial": 0, + "offset": 0 + } + }, + { + "name": "base", + "frame_type": "STATIC", + "transform_type": "DH", + "parent": "root", + "parameters": { + "d": 0, + "r": 0, + "a": 0, + "initial": 0, + "offset": 0, + "multiplier": 1 + }, + "mass": 1.541647, + "center_of_mass": { + "x": 0.004438, + "y": 0.000080, + "z": 0.021875 + }, + "inertia_tensor": { + "xx": 2.336131, + "yy": 1.869121, + "zz": 3.886624, + "xy": -0.006218, + "xz": 0.030549, + "yz": 0.003324 + } + }, + { + "name": "pelvis", + "frame_type": "DYNAMIC", + "transform_type": "DH", + "parent": "base", + "parameters": { + "d": 0.043125, + "r": 0.0, + "a": 0.2269, + "initial": 0, + "offset": 0 + }, + "mass": 0.486722, + "center_of_mass": { + "x": -0.005595, + "y": 0.001393, + "z": 0.020862 + }, + "inertia_tensor": { + "xx": 0.934733, + "yy": 1.049176, + "zz": 1.600895, + "xy": 0.005263, + "xz": 0.014859, + "yz": 0.006845 + } + }, + { + "name": "torso", + "frame_type": "DYNAMIC", + "transform_type": "DH", + "parent": "pelvis", + "parameters": { + "d": 0.056129, + "r": 0.0, + "a": -0.37874, + "initial": 0, + "offset": 0 + }, + "mass": 0.405103, + "center_of_mass": { + "x": 0.001299, + "y": 0.001032, + "z": 0.020476 + }, + "inertia_tensor": { + "xx": 0.765517, + "yy": 0.847899, + "zz": 1.027389, + "xy": -0.021990, + "xz": 0.005756, + "yz": -0.013292 + } + }, + { + "name": "head", + "frame_type": "DYNAMIC", + "transform_type": "DH", + "parent": "torso", + "parameters": { + "d": 0.073181, + "r": 0, + "a": 0, + "initial": 0, + "offset": 0 + }, + "mass": 1.105841, + "center_of_mass": { + "x": -0.006929, + "y": -0.000665, + "z": 0.047582 + }, + "inertia_tensor": { + "xx": 4.256305, + "yy": 3.001682, + "zz": 3.980863, + "xy": 0.028528, + "xz": -0.070913, + "yz": 0.025871 + } + }, + { + "name": "upper_head", + "frame_type": "STATIC", + "transform_type": "DH", + "parent": "head", + "parameters": { + "d": 0.108090, + "r": -0.005976, + "a": -0.32882, + "initial": 0, + "offset": 0 + } + }, + { + "name": "center_face", + "frame_type": "STATIC", + "transform_type": "DH", + "parent": "head", + "parameters": { + "d": 0.060362, + "r": 0.035301, + "a": -0.32882, + "initial": 0, + "offset": 0 + } + }, + { + "name": "imu_dh", + "frame_type": "STATIC", + "transform_type": "DH", + "parent": "center_face", + "parameters": { + "d": 0.0, + "r": 0.0, + "a": -1.5708, + "initial": 0, + "offset": 0 + } + }, + { + "name": "imu", + "frame_type": "STATIC", + "transform_type": "DOF6", + "parent": "imu_dh", + "parameters": { + "dx": -0.0410, + "dy": -0.0315, + "dz": 0.0150, + "rx": 0, + "ry": 0, + "rz": 0 + } + }, + { + "name": "top_head", + "frame_type": "STATIC", + "transform_type": "DH", + "parent": "head", + "parameters": { + "d": 0.142443, + "r": 0.007506, + "a": 0, + "initial": 0, + "offset": 0 + } + }, + { + "name": "left_eye", + "frame_type": "STATIC", + "transform_type": "DOF6", + "parent": "upper_head", + "parameters": { + "dx": 0, + "dy": 0.041, + "dz": 0, + "rx": 0, + "ry": 0, + "rz": 0 + } + }, + { + "name": "right_eye", + "frame_type": "STATIC", + "transform_type": "DOF6", + "parent": "upper_head", + "parameters": { + "dx": 0, + "dy": -0.041, + "dz": 0, + "rx": 0, + "ry": 0, + "rz": 0 + } + } + ] +} diff --git a/V3.1/build/usr/local/etc/jibo-lps-service.json b/V3.1/build/usr/local/etc/jibo-lps-service.json new file mode 100644 index 00000000..860729b0 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-lps-service.json @@ -0,0 +1,235 @@ +{ + "WebCoreLPS" : { + "serverPort" : 8484, + "fileRoot" : "/usr/local/var/www/lps" + }, + "LPSService" : { + "registryPort" : 8181, + "serverPort" : 8484, + "channels": [ + "PFSearchRegion", + "CapDevCUDAExpMeta", + "CapDevCUDAExp_0", + "CapDevCUDAExp_1", + "CapDevCUDAExpStat" + ] + }, + "WebCoreMedia" : { + "serverPort" : 8486, + "fileRoot" : "/usr/local/var/www/media" + }, + "MediaService" : { + "registryPort" : 8181, + "serverPort" : 8486, + "photographer" : { + "compressor" : { + "quality" : 92, + "interval" : 1000 + }, + "oneMP" : { "width" : 1280, "height" : 720 }, + "fourMP" : { "width" : 2688, "height" : 1520 }, + "request_queue_max_size" : 5 + }, + "photo_path" : "/opt/jibo/Photos/", + "recording_path" : "/opt/jibo/Recordings/", + "audio_source" : "MediaIn.monitor", + "future_wait_ms" : 8000 + }, + "AudioSubsystem" : { + "registryPort" : 8181, + "player" : false + }, + "BodySubsystem" : { + "registryPort" : 8181, + "player" : false + }, + "CaptureSubsystem" : { + "player" : false, + "watchdog" : { + "enabled": false, + "timeout": 5000, + "period" : 3000, + "start" : 10000 + }, + "camera_config_file" : "/usr/local/etc/lps/cameras.json" + }, + "EngineSubsystem" : { + "global" : { + "face_landmark_68" : { + "file" : + "/usr/local/share/lps/shape_predictor_68_face_landmarks.dat" + } + }, + "schemas" : { + "normal" : "/usr/local/etc/lps/schemas/normal.json", + "focused" : "/usr/local/etc/lps/schemas/focused.json", + "minimal" : "/usr/local/etc/lps/schemas/minimal.json" + }, + "engine" : { + "update_period" : 60, + "log" : { + "schema" : { + "time" : { + "stats" : false, + "skip" : 600 + }, + "audio" : { + "stats" : false, + "skip" : 600 + }, + "axis" : { + "stats" : false, + "skip" : 900 + }, + "image" : { + "stats" : false, + "skip" : 600 + } + } + }, + "state" : { + "entity_config_file" : "/usr/local/etc/lps/entityConfig.json", + "awareness" : { + "num_sectors" : 8, + "lighting" : { + "criteria" : { + "cameraIds" :[0], + "dimensions" : [ + { "width" : 1280, "height" : 720 } + ], + "outputTypes" :[0], + "outputLevels" :[0] + }, + "max_luminance" : 1.0, + "max_gain" : 16.0, + "max_exposure" : 0.066667, + "sensor" : [ + { "exposure" : 0.01067, "quality" : 0 }, + { "exposure" : 0.05333, "quality" : 1 }, + { "exposure" : 0.90667, "quality" : 1 }, + { "exposure" : 1.01333, "quality" : 0 } + ], + "quality" : [ + { "lighting" : 0.1, "quality" : 0 }, + { "lighting" : 0.2, "quality" : 1 }, + { "lighting" : 1.0, "quality" : 1 }, + { "lighting" : 1.5, "quality" : 0 } + ] + } + }, + "identification" : { + "face" : { + "type" : "network_face_identification", + "kind": "face", + "registry_host": "127.0.0.1", + "registry_port": 8181, + "connection_timeout":60000, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 1280, "height" : 720 }], + "outputTypes" : [0] + } + } + }, + "geometry" : { + "kinematic_model" : + "/usr/local/etc/jibo-kinematic-model.json", + "left_camera_file" : "/var/jibo/lps/CameraModelParamsL.json", + "right_camera_file" : + "/var/jibo/lps/CameraModelParamsR.json", + "inter_cam_transform_file" : + "/var/jibo/lps/InterCameraTransform.json" + } + } + } + }, + "RecorderSubsystem" : { + "root" : "/opt/jibo/data/recorder", + "audio" : { + "detections" : { + "directory" : "/opt/jibo/data/recorder/audio/detections", + "padding" : 7 + } + }, + "body" : { + "axis" : { + "directory" : "/opt/jibo/data/recorder/body/axis", + "padding" : 7 + } + }, + "capture" : { + "frames" : { + "directory" : "/opt/jibo/data/recorder/capture/frames", + "padding" : 7, + "criteria" : { + "cameraIds" :[], + "dimensions" :[ + {"width" : 640,"height" : 360} + ], + "outputTypes" :[0] + } + } + }, + "lps" : { + "visual-awareness" : { + "directory" : "/opt/jibo/data/recorder/lps/visual-awareness/", + "padding" : 7 + }, + "audio-awareness" : { + "directory" : "/opt/jibo/data/recorder/lps/audio-awareness/", + "padding" : 7 + } + } + }, + "PlayerSubsystem" : { + "root" : "/opt/jibo/data/player", + "audio" : { + "detections" : { + "directory" : "/opt/jibo/data/player/audio/detections" + } + }, + "body" : { + "axis" : { + "directory" : "/opt/jibo/data/player/body/axis" + } + }, + "capture" : { + "frames" : { + "directory" : "/opt/jibo/data/player/capture/frames" + } + } + }, + "ErrorTracker":{ + "views": [ + { + "name": "LPSService", + "errors": [ + "CAMERA_FAILURE" + ] + }, + { + "name": "MediaService", + "errors": [ ] + } + ] + }, + "logging" : { + "loggers" : { + "root": { + "level": "warning" + }, + "l1" : { + "name" : "ImageIdentifier", + "level" : "information" + }, + "l2" : { + "name" : "IdentityFusion", + "level" : "information" + }, + "l3" : { + "name" : "Util.Channel", + "level" : "warning" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-media-service.json b/V3.1/build/usr/local/etc/jibo-media-service.json new file mode 100644 index 00000000..8add45cc --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-media-service.json @@ -0,0 +1,88 @@ +{ + "WebCore": { + "serverPort": 7979, + "fileRoot": "/usr/local/var/www/media" + }, + "logging" : { + "channels" : { + "console" : { + "class" : "ConsoleChannel", + "pattern" : "%Y-%m-%d %H:%M:%S %s: [%p] %t" + }, + "syslog" : { + "class" : "SyslogChannel", + "name" : "jibo-media-service", + "facility" : "SYSLOG_DAEMON", + "pattern" : "%s: [%p] %t" + }, + "splitter" : { + "class" : "SplitterChannel", + "channels" : "syslog" + } + }, + "loggers" : { + "l1" : { + "name" : "Application", + "level" : "debug", + "channel" : "splitter" + } + } + }, + "MediaService": { + "registryPort":8181, + "serverPort":7979, + "camera": { + "type": "CUDA", + "cuda": { + "devices": [ + { + "enabled": true, + "name": "Camera-0", + "path": "/dev/video0", + "bufferPoolSize": 4, + "width": 1280, + "height": 720, + "format": 3, + "hFlip":true, + "vFlip":true, + "gamma" : { + "Y":0.5, + "U":1.0, + "V":1.0 + }, + "ae" : { + "exposureP":0.1, + "gainP":0.05, + "targetY":0.35, + "errorY":0.0001 + }, + "awb" : { + "grayThreshold":0.3, + "numSamples":1000, + "P":0.15, + "targetU":0.5, + "targetV":0.53, + "seed":0 + } + } + ], + "cuda" : { + + } + }, + "file": { + "videos": [ + "videos/video-000/video.json", + "videos/video-001/video.json" + ], + "format": "YUV420p" + }, + "v4l2": { + "devices": { + "left": "/dev/video0", + "right": "/dev/video1" + } + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-nlu-service.json b/V3.1/build/usr/local/etc/jibo-nlu-service.json new file mode 100644 index 00000000..ec6f7adc --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-nlu-service.json @@ -0,0 +1,50 @@ +{ + "webCore" : { + "serverPort": 8787, + "requestLogging": false + }, + "Service": { + "version":"v2.7.6", + "name":"nlu", + "host":"localhost", + "port":8787, + "path":"/", + "ttl":30, + "reg_timer":10000, + "tls":"", + "handler":"nlu_interface", + "log_ws":"nlu_logs", + "reset_memory_ws":"reset_memory", + "nlu_data_dir":"/usr/local/share/nlu-data", + "default_locale":"en-us", + "post_to_perf_monitor_service":true + }, + "parsing": { + "cmp_params": { + "log_weights": false, + "minimize_fst": true, + "union_minimize_fst": false + }, + "parsing_params": { + "max_num_states": 50, + "use_v8_interpreter": true + } + }, + + "logging" : { + "jibo_message_prefix": "C", + "loggers" : { + "root": { + "level": "information" + }, + "l1" : { + "name" : "NluService", + "level" : "information" + }, + "l2" : { + "name" : "Application", + "level" : "information" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-server-service.json b/V3.1/build/usr/local/etc/jibo-server-service.json new file mode 100644 index 00000000..09b491a3 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-server-service.json @@ -0,0 +1,37 @@ +{ + "WebCore": { + "serverPort": 8888, + "fileRoot": "/usr/local/var/www/server" + }, + "ServerService": { + "registryPort":8181, + "serverPort":8888 + }, + "NotificationSubsystem":{ + "registryPort":8181, + "refreshInterval": 15000, + "serverURLSuffix": "-socket.jibo.com" + }, + "ErrorTracker":{ + "views": [ + { + "name": "ServerService", + "errors": [ + "CANNOT_CONNECT_TO_SERVER" + ] + } + ] + }, + "logging" : { + "loggers" : { + "root": { + "level": "warning" + }, + "l2": { + "name": "NotificationSubsystem", + "level": "information", + "channel": "splitter" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-service-center-service.json b/V3.1/build/usr/local/etc/jibo-service-center-service.json new file mode 100644 index 00000000..3503f735 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-service-center-service.json @@ -0,0 +1,35 @@ +{ + "WebCore": { + "serverPort": 9797, + "fileRoot": "/usr/local/var/www/servicecenterservice", + "requestLogging": false + }, + "ServiceCenterService": { + "registryPort": 8181, + "serverPort": 9797 + }, + "ErrorTracker": { + "views": [ + { + "name": "ServiceCenterService", + "errors": [] + } + ] + }, + "logging": { + "channels": { + "splitter": { + "channels": "syslog" + } + }, + "loggers": { + "root": { + "level": "notice" + }, + "ServiceCenterService": { + "name": "ServiceCenterService", + "level": "notice" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-service-registry.json b/V3.1/build/usr/local/etc/jibo-service-registry.json new file mode 100644 index 00000000..a4ddfe51 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-service-registry.json @@ -0,0 +1,23 @@ +{ + "WebCore" : { + "serverPort": 8181, + "fileRoot": "/usr/local/var/www/service-registry", + "requestLogging": false + }, + "ManagementCore": { + "authenticate": false, + "validate": false, + "fileRoot": "/usr/local/var/www/service-management" + }, + "ServiceRegistry": { + "registryPort": 8181, + "serverPort": 8181 + }, + "logging": { + "loggers": { + "root": { + "level": "notice" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-developer.json b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-developer.json new file mode 100644 index 00000000..310e498f --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-developer.json @@ -0,0 +1,69 @@ +{ + "platformVersion": ">=3.1.0", + "services": { + "KBService": { + "port": 8778 + }, + "GlobalManagerService": { + "port": 8338 + }, + "SkillsService": { + "singleSkill": true, + "port": 8779 + }, + "NotificationsService": { + "port": 8001 + }, + "PerformanceService": { + "port": 10003 + }, + "PerformanceServiceSim": {}, + "ErrorService": { + "port": 10004 + }, + "SchedulerService": { + "port": 10005, + "otaFilter": "fcs" + }, + "RemoteService": { + "port": 10321 + }, + "WifiService": { + "port": 8668, + "region": "api" + }, + "DevShell": { + "port": 8686, + "syncPort": 8989, + "debugPort": 9191, + "skillDest": "/opt/jibo/Skills", + "skillUser": "jibo-skill", + "sdkDest": "bin/on-robot/" + } + }, + "RegistryClient": { + "port": 8181, + "host": "127.0.0.1" + }, + "logging": { + "logUncaughtExceptions": true, + "logUnhandledRejections": true, + "stackTraceLimit": 30, + "outputs": { + "console": { + "outputFileAndLine": false + }, + "syslog": { + "port": 514, + "target": "127.0.0.1", + "outputFileAndLine": false + } + }, + "namespaces": { + "": { + "console": "info", + "syslog": "info" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-int-developer.json b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-int-developer.json new file mode 100644 index 00000000..1c181956 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-int-developer.json @@ -0,0 +1,71 @@ +{ + "platformVersion": ">=3.1.0", + "services": { + "KBService": { + "port": 8778 + }, + "GlobalManagerService": { + "port": 8338 + }, + "SkillsService": { + "singleSkill": true, + "port": 8779 + }, + "NotificationsService": { + "port": 8001 + }, + "PerformanceService": { + "port": 10003 + }, + "PerformanceServiceSim": {}, + "ErrorService": { + "port": 10004 + }, + "SchedulerService": { + "port": 10005, + "otaFilter": "fcs" + }, + "RemoteService": { + "port": 10321 + }, + "WifiService": { + "port": 8668, + "region": "api" + }, + "DevShell": { + "port": 8686, + "syncPort": 8989, + "debugPort": 9191, + "skillDest": "/opt/jibo/Jibo/Skills", + "skillUser": "jibo-skill", + "sdkDest": "bin/on-robot/" + } + }, + "RegistryClient": { + "port": 8181, + "host": "127.0.0.1" + }, + "logging": { + "logUncaughtExceptions": true, + "logUnhandledRejections": true, + "stackTraceLimit": 30, + "outputs": { + "console": { + "outputFileAndLine": false + }, + "syslog": { + "port": 514, + "target": "127.0.0.1", + "outputFileAndLine": false + } + }, + "namespaces": { + "SSM.Client.ASR": {"console": "info", "syslog": "debug" }, + "C.AsrService": {"console": "info", "syslog": "debug" }, + "": { + "console": "info", + "syslog": "info" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-normal.json b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-normal.json new file mode 100644 index 00000000..692eb22a --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-normal.json @@ -0,0 +1,60 @@ +{ + "platformVersion": ">=3.1.0", + "services": { + "KBService": { + "port": 8778 + }, + "GlobalManagerService": { + "port": 8338 + }, + "SkillsService": { + "startSkill": "@be/be", + "singleSkill": true, + "port": 8779 + }, + "NotificationsService": { + "port": 8001 + }, + "ErrorService": { + "port": 10004 + }, + "SchedulerService": { + "port": 10005, + "otaFilter": "fcs" + }, + + "PerformanceServiceSim": {}, + "RemoteService": { + "port": 10321 + }, + "WifiService": { + "port": 8668, + "region": "api" + } + }, + "RegistryClient": { + "port": 8181, + "host": "127.0.0.1" + }, + "logging": { + "logUncaughtExceptions": true, + "logUnhandledRejections": true, + "stackTraceLimit": 30, + "outputs": { + "console": { + "outputFileAndLine": false + }, + "syslog": { + "port": 514, + "target": "127.0.0.1", + "outputFileAndLine": false + } + }, + "namespaces": { + "": { + "console": "none", + "syslog": "info" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-oobe.json b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-oobe.json new file mode 100644 index 00000000..ff438248 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-ssm/jibo-ssm-oobe.json @@ -0,0 +1,57 @@ +{ + "platformVersion": ">=3.1.0", + "services": { + "KBService": { + "port": 8778 + }, + "GlobalManagerService": { + "port": 8338 + }, + "SkillsService": { + "singleSkill": true, + "startSkill": "oobe-config", + "port": 8779 + }, + "NotificationsService": { + "port": 8001 + }, + "ErrorService": { + "port": 10004 + }, + "SchedulerService": { + "port": 10005, + "otaFilter": "fcs" + }, + + "PerformanceServiceSim": {}, + "WifiService": { + "port": 8668, + "region": "api" + } + }, + "RegistryClient": { + "port": 8181, + "host": "127.0.0.1" + }, + "logging": { + "logUncaughtExceptions": true, + "logUnhandledRejections": true, + "stackTraceLimit": 30, + "outputs": { + "console": { + "outputFileAndLine": false + }, + "syslog": { + "port": 514, + "target": "127.0.0.1", + "outputFileAndLine": false + } + }, + "namespaces": { + "": { + "console": "none", + "syslog": "info" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-sts.json b/V3.1/build/usr/local/etc/jibo-sts.json new file mode 100644 index 00000000..e69de29b diff --git a/V3.1/build/usr/local/etc/jibo-system-manager.json b/V3.1/build/usr/local/etc/jibo-system-manager.json new file mode 100644 index 00000000..d497e4c1 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-system-manager.json @@ -0,0 +1,676 @@ +{ + "WebCore": { + "serverPort": 8585, + "fileRoot": "/usr/local/var/www/system" + }, + "ManagementCore": { + "authenticate": false, + "validate": false + }, + "SystemManager": { + "registryPort": 8181, + "serverPort": 8585, + "service": { + "connection_timeout": 210000, + "backup":{ + "file": "/opt/tmp/backup.tar.bz2", + "directory" : "/opt/tmp/backup", + "executable" : "/usr/local/bin/jibo-system-backup" + }, + "restore":{ + "file" : "/opt/tmp/restore.tar.bz2", + "directory": "/opt/tmp/restore", + "executable": "/usr/local/bin/jibo-system-restore" + }, + "services" : [ + { + "name": "body", + "order": 0, + "executable": "/usr/local/bin/jibo-body-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-body-service.json"], + "environment": { + "DISPLAY": ":0", + "XAUTHORITY": "/tmp/.Xauthority" + }, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_BODY" + }, + { + "name": "audio", + "order": 1, + "executable": "/usr/local/bin/jibo-audio-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-audio-service.json"], + "environment": {}, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_AUDIO" + }, + { + "name": "tts", + "order": 2, + "executable": "/usr/local/bin/jibo-tts-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-tts-service.json"], + "environment": {}, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_TTS" + }, + { + "name": "asr", + "order": 2, + "executable": "/usr/local/bin/jibo-asr-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-asr-service.json"], + "environment": {}, + "directory": "", + "enabled":false + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_ASR" + }, + { + "name": "jetstream", + "order": 2, + "executable": "/usr/local/bin/jibo-jetstream-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-jetstream-service.json"], + "environment": {}, + "directory": "", + "enabled": true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": true + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_JETSTREAM" + }, + { + "name": "nlu", + "order": 2, + "executable": "/usr/local/bin/jibo-nlu-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-nlu-service.json"], + "environment": {}, + "directory": "", + "enabled":false + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_NLU" + }, + { + "name": "identity", + "order": 2, + "executable": "/usr/local/bin/jibo-identity-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-identity-service.json"], + "environment": { + "GLOG_minloglevel": "2" + }, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_IDENTITY" + }, + { + "name": "lps", + "order": 3, + "executable": "/usr/local/bin/jibo-lps-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-lps-service.json"], + "environment": { + "DISPLAY": ":0", + "XAUTHORITY": "/tmp/.Xauthority", + "GST_REGISTRY": "/usr/share/gstreamer-1.0/registry.$(uname -m).bin", + "GST_GL_XINITTHREADS": "1", + "GST_REGISTRY_UPDATE": "no" + }, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + } + }, + "startup_timeout":60000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_LPS" + }, + { + "name": "secure-transfer", + "order": 4, + "executable": "/usr/bin/node", + "modes": { + "default": { + "arguments": ["/usr/local/bin/jibo-sts/index.js"], + "environment": {}, + "directory": "", + "enabled":true + }, + "int-developer": { + "arguments": [ + "--inspect=10775", + "/usr/local/bin/jibo-sts/index.js" + ], + "environment": {}, + "directory" : "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_SECURE_TRANSFER" + }, + { + "name": "security-controller-service", + "order": 4, + "executable": "/usr/bin/node", + "modes": { + "default": { + "arguments": [ + "/usr/local/bin/jibo-ssm/jibo-scs.js" + ], + "environment": {}, + "directory": "", + "enabled":true + }, + "int-developer": { + "arguments": [ + "--inspect=10226", + "/usr/local/bin/jibo-ssm/jibo-scs.js" + ], + "environment": {}, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_SECURITY_CONTROLLER_SERVICE" + }, + { + "name": "media-manager", + "order": 4, + "executable": "/usr/bin/node", + "modes": { + "default": { + "arguments": [ + "/usr/local/bin/jibo-ssm/jibo-mms.js" + ], + "environment": {}, + "directory": "", + "enabled":true + }, + "int-developer": { + "arguments": [ + "--inspect=10225", + "/usr/local/bin/jibo-ssm/jibo-mms.js" + ], + "environment": {}, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_MEDIA_MANAGER" + }, + { + "name": "monitor", + "order": 4, + "executable": "/usr/local/bin/jibo-system-monitoring-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-system-monitoring-service.json"], + "environment": {}, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_MONITOR" + }, + { + "name": "server", + "order": 4, + "executable": "/usr/local/bin/jibo-server-service", + "modes": { + "default": { + "arguments": ["-c","/usr/local/etc/jibo-server-service.json"], + "environment": {}, + "directory": "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":10000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_SERVER" + }, + { + "name": "ssm", + "order": 5, + "executable": "/usr/bin/node", + "modes": { + "default": { + "arguments": [ + "/usr/local/bin/jibo-ssm/jibo-ssm.js" + ], + "environment": { + "DISPLAY": ":0", + "XAUTHORITY": "/tmp/.Xauthority", + "NODE_PATH": "/usr/lib/node_modules", + "RUNMODE": "ON_ROBOT", + "XDG_CONFIG_HOME": "/tmp/.config" + }, + "directory" : "", + "enabled":true + }, + "int-developer": { + "arguments": [ + "--inspect=10223", + "/usr/local/bin/jibo-ssm/jibo-ssm.js" + ], + "environment": { + "DISPLAY": ":0", + "XAUTHORITY": "/tmp/.Xauthority", + "NODE_PATH": "/usr/lib/node_modules", + "RUNMODE": "ON_ROBOT", + "XDG_CONFIG_HOME": "/tmp/.config" + }, + "directory" : "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":60000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_SSM" + }, + { + "name": "expression", + "order": 4, + "executable": "/usr/bin/node", + "modes": { + "default": { + "arguments": [ + "/usr/local/bin/jibo-ssm/lib/expression-process.js" + ], + "environment": { + "NODE_PATH": "/usr/lib/node_modules", + "RUNMODE": "ON_ROBOT" + }, + "directory" : "", + "enabled":true + }, + "int-developer": { + "arguments": [ + "--inspect=10224", + "/usr/local/bin/jibo-ssm/jibo-expression.js" + ], + "environment": { + "NODE_PATH": "/usr/lib/node_modules", + "RUNMODE": "ON_ROBOT" + }, + "directory" : "", + "enabled":true + }, + "certification": { + "enabled": false + }, + "service": { + "enabled": false + } + }, + "startup_timeout":60000, + "shutdown_timeout":10000, + "crash_code": "SERVICE_CRASH_EXPRESSION" + }, + { + "name": "certification", + "order": 0, + "executable": "/usr/local/bin/jibo-certification-service", + "modes": { + "default": { + "arguments": [ + "-c", "/usr/local/etc/jibo-certification-service.json" + ], + "environment": {}, + "directory": "", + "enabled": false + }, + "certification": { + "arguments": [ + "-c", "/usr/local/etc/jibo-certification-service.json" + ], + "environment": {}, + "directory": "", + "enabled": true + }, + "service": { + "arguments": [ + "-c", "/usr/local/etc/jibo-certification-service.json" + ], + "environment": {}, + "directory": "", + "enabled": true + } + }, + "startup_timeout": 10000, + "shutdown_timeout": 10000, + "crash_code": "SERVICE_CRASH_CERTIFICATION" + }, + { + "name": "service-center", + "order": 5, + "executable": "/usr/local/bin/jibo-service-center-service", + "modes": { + "default": { + "arguments": [ + "-c", "/usr/local/etc/jibo-service-center-service.json" + ], + "environment": { + "DISPLAY": ":0", + "XAUTHORITY": "/tmp/.Xauthority" + }, + "environment": {}, + "directory": "", + "enabled": false + }, + "service": { + "arguments": [ + "-c", "/usr/local/etc/jibo-service-center-service.json" + ], + "environment": { + "DISPLAY": ":0", + "XAUTHORITY": "/tmp/.Xauthority" + }, + "directory": "", + "enabled": true + } + }, + "startup_timeout": 10000, + "shutdown_timeout": 10000, + "crash_code": "SERVICE_CRASH_SERVICE_CENTER" + } + ] + }, + "session": { + + }, + "skill":{ + "executable": "/usr/bin/electron/electron", + "arguments": [ + "--remote-debugging-port=9222", + "/usr/local/bin/jibo-ssm/skill-main.js" + ], + "environment": { + "DISPLAY": ":0", + "XAUTHORITY": "/tmp/.Xauthority", + "XDG_CONFIG_HOME": "/opt/home/jibo-skill/.config" + }, + "path": { + "jibo": [ + "/opt/jibo/Jibo/Skills/@be", + "/opt/jibo/Jibo/Skills" + ], + "devs": [ + "/opt/jibo/Skills" + ] + }, + "modes":{ + "normal" : { + "user" : "jibo-skill" + }, + "oobe" : { + "user" : "jibo-skill" + }, + "int-developer" : { + "user" : "jibo-skill" + }, + "developer" : { + "user" : "jibo-skill" + } + } + }, + "storage" : { + "mount_entries_file": "/proc/mounts", + "semantic": { + "jibo": "/opt/jibo/Jibo", + "skills": "/opt/jibo/Skills", + "photos": "/opt/jibo/Photos", + "recordings": "/opt/jibo/Recordings", + "knowledge": "/opt/jibo/Knowledge" + } + }, + "credentials" : { + "path" : "/var/jibo/credentials.json" + }, + "wifi" : { + "wpa" : { + "interface": "wlan0", + "runtime" : "/var/run/wpa_supplicant", + "reconnect_interval": 15000, + "monitor_interval": 1000 + }, + "udhcpc" : { + "executable" : "udhcpc", + "options" : "-R -b -t 3 -T 5 -A 5" + } + }, + "powerButton" : { + "path" : "/dev/input/event0", + "duration" : 2000, + "warning" : 10000 + }, + "powerManager" : { + "warning" : 10000 + }, + "time": { + "timezone_path": "/var/etc/timezone", + "localtime_path": "/var/etc/localtime", + "zoneinfo_path": "/usr/share/zoneinfo", + "ntpd_init": "/etc/init.d/S66ntp", + "ntpdate": { + "command": "/usr/bin/ntpdate", + "arguments": [ + "-b", + "-t", + "3", + "0.north-america.pool.ntp.org" + ] + }, + "hwclock": { + "command": "/sbin/hwclock", + "arguments": [ + "--systohc" + ] + } + }, + "update": { + "subsystems": { + "os": { + "version_check": "os", + "apply_method": "system", + "default_order": 1 + }, + "services": { + "version_check": "services", + "apply_method": "system", + "default_order": 2 + }, + "jibo-diagnostics": { + "version_check": "skill", + "apply_method": "skill", + "default_order": 3 + }, + "fin-goods-test": { + "version_check": "skill", + "apply_method": "skill", + "default_order": 4 + }, + "oobe-config": { + "version_check": "skill", + "apply_method": "skill", + "default_order": 5 + }, + "@be/be": { + "version_check": "skill", + "apply_method": "skill", + "default_order": 6 + }, + "jibo-rhino": { + "version_check": "skill", + "apply_method": "skill", + "default_order": 7 + }, + "jibo-trivia": { + "version_check": "skill", + "apply_method": "skill", + "default_order": 8 + }, + "jibo-tbd": { + "version_check": "skill", + "apply_method": "skill", + "default_order": 9 + } + } + }, + "firewall": { + "mode_commands": { + "remote_operation": ["-p tcp --syn --dport 7160 -j ACCEPT"] + } + } + }, + "ErrorTracker":{ + "views": [ + { + "name": "SystemManager", + "errors": [ + "SERVICE_CRASH_BODY", + "SERVICE_CRASH_AUDIO", + "SERVICE_CRASH_TTS", + "SERVICE_CRASH_ASR", + "SERVICE_CRASH_NLU", + "SERVICE_CRASH_IDENTITY", + "SERVICE_CRASH_LPS", + "SERVICE_CRASH_MEDIA_MANAGER", + "SERVICE_CRASH_SECURE_TRANSFER", + "SERVICE_CRASH_MONITOR", + "SERVICE_CRASH_SERVER", + "SERVICE_CRASH_SSM", + "SERVICE_CRASH_EXPRESSION", + "WPA_CONTROL_INTERFACE_DOWN" + ] + } + ] + }, + "logging" : { + "loggers" : { + "root": { + "level": "warning" + }, + "l4" : { + "name" : "SystemManager", + "level" : "information" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-system-monitoring-service.json b/V3.1/build/usr/local/etc/jibo-system-monitoring-service.json new file mode 100644 index 00000000..3673106f --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-system-monitoring-service.json @@ -0,0 +1,57 @@ +{ + "WebCore": { + "serverPort": 4111, + "fileRoot": "/usr/local/var/www/systemMonitor" + }, + "SystemMonitoringService": { + "registryPort": 8181, + "serverPort": 4111, + "storage" : { + "mount_entries_file": "/proc/mounts", + "semantic": { + "jibo": "/opt/jibo/Jibo", + "skills": "/opt/jibo/Skills", + "photos": "/opt/jibo/Photos", + "recordings": "/opt/jibo/Recordings", + "knowledge": "/opt/jibo/Knowledge", + "tmp": "/tmp" + }, + "low_storage_byte": 1073741824, + "polling_interval": 60000 + }, + "error" : { + "connection_timeout":5000, + "polling_interval":10000 + }, + "health" : { + "connection_timeout":30000, + "polling_interval": 1800000, + "upload" : { + "executable": "/usr/bin/jibo-log-client-async", + "arguments": [ + "--credentials","/var/jibo/credentials.json", + "--kind","HEALTH" + ], + "environment": {}, + "directory": "" + } + } + }, + "ErrorTracker": { + "views": [ + { + "name": "SystemMonitoringService", + "errors": [ + "LOW_ROBOT_STORAGE" + ] + } + ] + }, + "logging" : { + "loggers" : { + "root": { + "level": "warning" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-test-capture-service.json b/V3.1/build/usr/local/etc/jibo-test-capture-service.json new file mode 100644 index 00000000..8f6fb141 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-test-capture-service.json @@ -0,0 +1,97 @@ +{ + "WebCore": { + "serverPort": 7979, + "fileRoot": "/usr/local/var/www/capture-tools" + }, + "TestCaptureService": { + "registryPort": 8181, + "serverPort": 7979, + "camera": { + "type": "CUDA", + "cuda": { + "devices": [ + { + "enabled": true, + "name": "Right Wide Eye", + "path": "/dev/video1", + "bufferPoolSize": 4, + "width": 672, + "height": 380, + "outputBufferFormats" : 2, + "hFlip":true, + "vFlip":true, + "gamma" : { + "R":0.75, + "G":0.7, + "B":0.7 + }, + "ae" : { + "exposureP":0.05, + "gainP":0.02, + "targetY":0.6, + "skinSegmentedTargetY" : 0.5, + "skinSegmentedAvgYThreshold" : 0.8, + "skinRatioThreshold" : 0.010, + "errorY":0.0001 + }, + "awb" : { + "grayThreshold":0.3, + "numSamples":1000, + "P":0.15, + "targetU":0.505, + "targetV":0.50 + } + }, + { + "enabled": true, + "name": "Left Narrow Eye", + "path": "/dev/video0", + "bufferPoolSize": 4, + "width": 672, + "height": 380, + "outputBufferFormats" : 2, + "hFlip":true, + "vFlip":true, + "gamma" : { + "R":0.75, + "G":0.7, + "B":0.7 + }, + "ae" : { + "exposureP":0.05, + "gainP":0.02, + "targetY":0.6, + "skinSegmentedTargetY" : 0.5, + "skinSegmentedAvgYThreshold" : 0.8, + "skinRatioThreshold" : 0.010, + "errorY":0.0001 + }, + "awb" : { + "grayThreshold":0.3, + "numSamples":1000, + "P":0.15, + "targetU":0.505, + "targetV":0.50 + } + } + ], + "cuda" : { + + } + }, + "file": { + "videos": [ + "videos/video-000/video.json", + "videos/video-001/video.json" + ], + "format": "YUV420p" + }, + "v4l2": { + "devices": { + "left": "/dev/video0", + "right": "/dev/video1" + } + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-test-capture.json b/V3.1/build/usr/local/etc/jibo-test-capture.json new file mode 100644 index 00000000..3409a7cd --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-test-capture.json @@ -0,0 +1,117 @@ +{ + "test" : { + "handler" : { + "num_frames" : 0, + "awb_lock_count" : 190, + "ae_lock_count" : 190, + "min_exposure" : 0.012, + "max_exposure" : 0.033, + "record" : { + "enabled" : false, + "directory" : "/tmp/jibo-cap/", + "padding" : 5, + "skip_count" : 180 + }, + "recordAxes" : { + "enabled" : false, + "uri" : "http://localhost:8282/axis_state", + "padding" : 5 + }, + "display" : { + "enabled" : true + }, + "displayCycle" : { + "enabled" : false + }, + "profiler" : { + "enabled" : true + }, + "fps" : { + "enabled" : false + } + }, + "camera" : { + "type" : "CUDA", + "cuda" : { + "devices" : [ + { + "enabled":true, + "name" : "Right Wide Eye", + "path" : "/dev/video1", + "bufferPoolSize" : 4, + "width" : 1280, + "height" : 720, + "outputBufferFormats" : 0, + "hFlip" : true, + "vFlip" : true, + "gamma" : { + "R" : 0.6, + "G" : 0.6, + "B" : 0.6 + }, + "ae" : { + "exposureP" : 0.05, + "gainP" : 0.02, + "targetY" : 0.25, + "errorY" : 0.0001 + }, + "awb" : { + "grayThreshold" : 0.3, + "numSamples" : 1000, + "P" : 0.15, + "targetU" : 0.505, + "targetV" : 0.50, + "seed" : 0 + } + }, + { + "enabled":true, + "name" : "Left Narrow Eye", + "path" : "/dev/video0", + "bufferPoolSize" : 4, + "width" : 672, + "height" : 380, + "outputBufferFormats" : 0, + "hFlip" : true, + "vFlip" : true, + "gamma" : { + "R" : 0.6, + "G" : 0.6, + "B" : 0.6 + }, + "ae" : { + "exposureP" : 0.05, + "gainP" : 0.02, + "targetY" : 0.25, + "errorY" : 0.0001 + }, + "awb" : { + "grayThreshold" : 0.3, + "numSamples" : 1000, + "P" : 0.15, + "targetU" : 0.505, + "targetV" : 0.50, + "seed" : 0 + } + } + ], + "cuda" : { + + } + }, + "file" : { + "videos" : [ + "videos/video-000/video.json", + "videos/video-001/video.json" + ], + "format" : "YUV420p" + }, + "v4l2" : { + "devices" : { + "left" : "/dev/video0", + "right" : "/dev/video1" + } + } + } + } +} diff --git a/V3.1/build/usr/local/etc/jibo-tts-service.json b/V3.1/build/usr/local/etc/jibo-tts-service.json new file mode 100644 index 00000000..90b14dd6 --- /dev/null +++ b/V3.1/build/usr/local/etc/jibo-tts-service.json @@ -0,0 +1,110 @@ +{ + "webCore" : { + "serverPort": 8089, + "fileRoot": "/usr/local/var/www/ttsservice", + "requestLogging": false + }, + "TTSService" : { + "alsaPlaybackDevice" : "TTSOut", + "audioDriver" : "alsa", + "effectsDir" : "/usr/local/share/ttsservice/effects", + "enableFadeOut" : "FALSE", + "enableJiboDefaultSwitch" : "FALSE", + "enablePD" : "FALSE", + "postToPerformanceService" : "TRUE", + "enableTextNorm" : "TRUE", + "maxRSSMemory" : 250000, + "frontEnd" : "JIBO", + "resourcePath" : "/usr/local/share/ttsservice/voices/en_us_world/" + }, + "voiceParams" : { + "postFilter_b" : "0.40", + "samplerate_s" : "48000", + "framerate_p" : "240", + "pitchbw_jf" : "0.40", + "halftone_fm" : "3", + "unvoicedvoice_u" : "0.15", + "allPass_a" : "0.76", + "gvMCEP_jm" : "0.9", + "speed_r" : "1", + "mode" : "text", + "outmode" : "stream", + "mgc_order" : "60", + "frontendResource" : "/usr/local/share/ttsservice/voices/en_us/griffin/fsts/", + "buffer_size" : "1024", + "fft_size" : "1024", + "volume_linear" : "0.65", + "maxChars" : "1000" + }, + "PostFilterMap" : { + "pitchshiftSwitch" : "pitchStrength", + "freqshiftSwitch" : "freqshiftAmount", + "autotuneSwitch" : "autotunePitch", + "vocoderSwitch" : "vocoderChord", + "phaserSwitch" : "phaserAmount", + "resonatorSwitch" : "resonatorPitch", + "ringmodSwitch" : "ringmodFreq", + "aliasSwitch" : "aliasAmount", + "flangerSwitch" : "flangerAmount", + "chorusSwitch" : "chorusAmount", + "masteringSwitch" : "masteringAmount", + "excitedSwitch" : "excitedAmount", + "worriedSwitch" : "worriedAmount", + "disapproveSwitch" : "disapproveAmount", + "neutralSwitch" : "neutralAmount", + "affectionateSwitch" : "affectionateAmount", + "sorrowSwitch" : "sorrowAmount", + "laughterSwitch" : "laughterAmount", + "listSwitch" : "listAmount", + "emphasisSwitch" : "emphasisAmount", + "fadeSwitch" : "fadeAmount", + "makeSimilar" : "makeSimilar", + "analysisSwitch" : "analysisSwitch", + "oo" : "1", + "wow" : "2", + "perfect" : "3", + "ok" : "4", + "ooo" : "5", + "ah" : "6", + "oh" : "7", + "your_welcome" : "8", + "cool" : "9", + "woo_hoo_hoo" : "10", + "laugh" : "11", + "laugh2" : "12", + "sweet" : "13", + "done" : "14", + "what" : "15", + "aw" : "16", + "my_bad" : "17", + "oops" : "18", + "um" : "19", + "huh" : "20", + "whoa" : "21", + "argh" : "22", + "nm_um" : "23", + "i_love_to_1" : "24", + "i_love_to_2" : "25", + "i_love_to_3" : "26", + "i_love_to_4" : "27", + "i_love_to_5" : "28", + "i_love_to_6" : "29", + "triggerJibonics" : "bang" + }, + "logging" : { + "jibo_message_prefix": "C", + "loggers" : { + "root": { + "level": "warning" + }, + "l1" : { + "name" : "TTSService", + "level" : "information" + }, + "l2" : { + "name" : "Application", + "level" : "information" + } + } + } +} diff --git a/V3.1/build/usr/local/etc/lps/cameras.json b/V3.1/build/usr/local/etc/lps/cameras.json new file mode 100644 index 00000000..5478f21e --- /dev/null +++ b/V3.1/build/usr/local/etc/lps/cameras.json @@ -0,0 +1,168 @@ +{ + "type" : "CUDA", + "cuda" : { + "devices" : [ + { + "enabled" : true, + "name" : "Camera-0", + "path" : "/dev/video1", + "bufferPoolSize" : 10, + "width" : 1280, + "height" : 720, + "outputBufferFormats" : 2, + "hFlip" : true, + "vFlip" : true, + "gamma" : { + "R" : 0.7, + "G" : 0.7, + "B" : 0.7 + }, + "numSamples" : 10000, + "ae" : { + "exposureP" : 0.02, + "gainP" : 0.02, + "targetY" : 0.6, + "avgYThreshold" : 0.8, + "skinSegmentedTargetY" : 0.43, + "skinSegmentedAvgYThreshold" : 0.8, + "skinRatioThreshold" : 0.010, + "numSamples" : 1000, + "numRegions" : 64, + "errorY" : 0.0001 + }, + "awb" : { + "grayThreshold" : 0.3, + "numSamples" : 1000, + "P" : 0.15, + "targetU" : 0.505, + "targetV" : 0.50, + "seed" : 0 + } + }, + { + "enabled" : true, + "name" : "Camera-1", + "path" : "/dev/video0", + "bufferPoolSize" : 4, + "width" : 1280, + "height" : 720, + "outputBufferFormats" : 2, + "hFlip" : true, + "vFlip" : true, + "gamma" : { + "R" : 0.7, + "G" : 0.7, + "B" : 0.7 + }, + "numSamples" : 10000, + "ae" : { + "exposureP" : 0.02, + "gainP" : 0.02, + "targetY" : 0.6, + "avgYThreshold" : 0.8, + "skinSegmentedTargetY" : 0.46, + "skinSegmentedAvgYThreshold" : 0.8, + "skinRatioThreshold" : 0.010, + "numSamples" : 1000, + "errorY" : 0.0001 + }, + "awb" : { + "grayThreshold" : 0.3, + "numSamples" : 1000, + "P" : 0.15, + "targetU" : 0.505, + "targetV" : 0.50, + "seed" : 0 + } + } + ], + "cuda" : { + + } + }, + "file" : { + "videos" : [ + "videos/video-000/video.json", + "videos/video-001/video.json" + ], + "format" : "YUV420p" + }, + "v4l2" : { + "devices" : { + "left" : "/dev/video0", + "right" : "/dev/video1" + } + }, + "controls" : [ + { + "width" : 1280, + "height" : 720, + "hFlip" : true, + "vFlip" : true, + "ae" : { + "lock" : false, + "manual" : false, + "time" : { + "low" : 0.0016667, + "high" : 0.033 + }, + "regions" : [], + "targetY" : 0.4, + "manualMode" : "default", + "manualExposure" : 0.03333, + "manualGain" : 1, + "ev" : 0 + }, + "outputBuffers" : [], + "outputBuffersConfIndex" : 2, + "awb" : { + "lock" : false, + "manual" : false, + "gammaRed" : 0.7, + "gammaGreen" : 0.7, + "gammaBlue" : 0.7, + "regions" : [], + "manualRedGain" : 1024, + "manualGreenGain" : 1024, + "manualBlueGain" : 1024, + "gammaCr" : 0.78, + "gammaCb" : 0.78 + } + }, + { + "width" : 1280, + "height" : 720, + "hFlip" : true, + "vFlip" : true, + "ae" : { + "lock" : false, + "manual" : false, + "time" : { + "low" : 0.0016667, + "high" : 0.033 + }, + "regions" : [], + "targetY" : 0.4, + "manualMode" : "default", + "manualExposure" : 0.03333, + "manualGain" : 1, + "ev" : 0 + }, + "outputBuffers" : [], + "outputBuffersConfIndex" : 2, + "awb" : { + "lock" : false, + "manual" : false, + "gammaRed" : 0.7, + "gammaGreen" : 0.7, + "gammaBlue" : 0.7, + "regions" : [], + "manualRedGain" : 1024, + "manualGreenGain" : 1024, + "manualBlueGain" : 1024, + "gammaCr" : 0.78, + "gammaCb" : 0.78 + } + } + ] +} diff --git a/V3.1/build/usr/local/etc/lps/entityConfig.json b/V3.1/build/usr/local/etc/lps/entityConfig.json new file mode 100644 index 00000000..b181b910 --- /dev/null +++ b/V3.1/build/usr/local/etc/lps/entityConfig.json @@ -0,0 +1,318 @@ +{ + "default_person" : { + "debug" : false, + "description" : "person", + "name" : "person", + "max_track_age" : 300.0, + "is_static" : false, + "min_part_confidence" : 0.001, + "max_part_measure_age" : 5.0, + "max_audio_measure_age" : 10.0, + "min_audio_confidence" : 0.4, + "out_fov_age_factor" : 1.5, + "in_fov_margin" : 0.10, + "min_detect_request_age" : 5.0, + "speaking_timeout" : 1.5, + "speaking_threshold" : 0.25, + "head" : { + "name" : "head", + "debug" : false, + "identity" : { + "alpha" : 0.75, + "cull" : 0.1 + }, + "enable_back_projection" : true, + "default_ramge" : 2.5, + "use_pseudo_range" : true, + "extent_x" : 0.25, + "extent_y" : 0.25, + "extent_z" : 0.25, + "vel_decay" : 0.5, + "vel_max" : 1.9, + "linear_psd" : 10.0, + "min_state_covariance" : 0.0001, + "pseudo_range_covariance" : 0.9, + "auto_add_tracker" : false, + "auto_add_tracker_skip" : 20, + "pose_filter":0.0, + "tracker_type" : "tracker_2", + "tracker_1" : { + "tracker_class" : "color_tracker", + "image_tracker" : { + "debug" : false, + "image_debug" : false, + "name" : "A", + "use_image_velocity" : true, + "vel_scale" : 0.9, + "vel_filter" : 0.75, + "base_covar" : 220.0, + "image_width" : 640, + "image_height" : 360, + "track_confidence_filter" : 0.6, + "model_confidence_filter" : 0.9, + "min_confidence" : 0.05, + "measure_interval": 0.067 + }, + "color_tracker" : { + "model_debug" : false, + "track_debug" : false, + "drop_area_too_big" : 120000.0, + "drop_area_too_small" : 600.0, + "search_size_horizontal_scale" : 20, + "search_size_vertical_scale" : 20, + "area_filter" : 0.6, + "model" : { + "y_min" : 25, + "y_max" : 250, + "color_threshold_min" : 10, + "color_threshold_max" : 25 + }, + "camshift" : { + "max_iterations" : 6, + "area_scale" : 10.0, + "area_aspect" : 1.2 + }, + "kmeans" : { + "K" : 2, + "max_iterations" : 10, + "convergence" : 0.01 + } + } + }, + "tracker_2" : { + "tracker_class" : "abcshift_tracker", + "image_tracker" : { + "debug" : false, + "image_debug" : false, + "name" : "A", + "use_image_velocity" : true, + "vel_scale" : 0.9, + "vel_filter" : 0.75, + "base_covar" : 220.0, + "image_width" : 640, + "image_height" : 360, + "track_confidence_filter" : 0.6, + "model_confidence_filter" : 0.9, + "min_confidence" : 0.05, + "measure_interval": 0.067 + }, + "abcshift_tracker" : { + "drop_area_too_big" : 120000.0, + "drop_area_too_small" : 600.0, + "search_size_horizontal_scale" : 20, + "search_size_vertical_scale" : 20, + "area_filter" : 0.6, + "max_iterations" : 6, + "area_scale" : 1.3, + "area_aspect" : 1.2, + "bhattacharyya_threshold" : 0.5, + "model" : { + "y_min" : 5, + "y_max" : 250, + "color_threshold_min" : 5, + "color_threshold_max" : 10 + }, + "background" : { + "minX" : 0, + "maxX" : 255, + "binsX" : 256, + "minY" : 0, + "maxY" : 255, + "binsY" : 256 + }, + "foreground" : { + "minX" : 0, + "maxX" : 255, + "binsX" : 256, + "minY" : 0, + "maxY" : 255, + "binsY" : 256 + } + } + }, + "tracker_3" : { + "tracker_class" : "correlation_tracker", + "image_tracker" : { + "debug" : false, + "image_debug" : false, + "name" : "A", + "use_image_velocity" : true, + "vel_scale" : 0.9, + "vel_filter" : 0.75, + "base_covar" : 220.0, + "image_width" : 640, + "image_height" : 360, + "track_confidence_filter" : 0.6, + "model_confidence_filter" : 0.9, + "min_confidence" : 0.05, + "measure_interval": 0.067 + }, + "correlation_tracker" : { + "filter_size" : 6, + "num_scale_levels" : 5, + "scale_window_size" : 23, + "regularizer_space" : 0.001, + "nu_space" : 0.025, + "regularizer_scale" : 0.001, + "nu_scale" : 0.025, + "scale_pyramid_alpha" : 1.020, + "side_lobe_ratio_threshold": 12.0 + } + }, + "tracker_4" : { + "tracker_class" : "face_landmark_tracker", + "image_tracker" : { + "debug" : false, + "image_debug" : false, + "name" : "A", + "use_image_velocity" : true, + "vel_scale" : 0.9, + "vel_filter" : 0.75, + "base_covar" : 220.0, + "image_width" : 640, + "image_height" : 360, + "track_confidence_filter" : 0.6, + "model_confidence_filter" : 0.9, + "min_confidence" : 0.05, + "measure_interval": 0.067 + }, + "face_landmark_tracker" : { } + }, + "tracker_5" : { + "tracker_class" : "composite_tracker", + "image_tracker" : { + "debug" : false, + "image_debug" : false, + "name" : "A", + "use_image_velocity" : true, + "vel_scale" : 0.9, + "vel_filter" : 0.75, + "base_covar" : 220.0, + "image_width" : 1280, + "image_height" : 720, + "track_confidence_filter" : 0.6, + "model_confidence_filter" : 0.9, + "min_confidence" : 0.05, + "measure_interval": 0.067 + }, + "composite_tracker" : { + "trackers" : { + "color" : { + "tracker_class" : "abcshift_tracker", + "image_tracker" : { + "debug" : false, + "image_debug" : false, + "name" : "A", + "use_image_velocity" : true, + "vel_scale" : 0.9, + "vel_filter" : 0.75, + "base_covar" : 220.0, + "image_width" : 640, + "image_height" : 360, + "track_confidence_filter" : 0.6, + "model_confidence_filter" : 0.9, + "min_confidence" : 0.05, + "measure_interval": 0.067 + }, + "abcshift_tracker" : { + "drop_area_too_big" : 120000.0, + "drop_area_too_small" : 600.0, + "search_size_horizontal_scale" : 20, + "search_size_vertical_scale" : 20, + "area_filter" : 0.6, + "max_iterations" : 6, + "area_scale" : 1.3, + "area_aspect" : 1.2, + "bhattacharyya_threshold" : 0.5, + "model" : { + "y_min" : 5, + "y_max" : 250, + "color_threshold_min" : 5, + "color_threshold_max" : 25 + }, + "background" : { + "minX" : 0, + "maxX" : 255, + "binsX" : 64, + "minY" : 0, + "maxY" : 255, + "binsY" : 64 + }, + "foreground" : { + "minX" : 0, + "maxX" : 255, + "binsX" : 64, + "minY" : 0, + "maxY" : 255, + "binsY" : 64 + } + } + } + } + } + } + } + }, + "default_motion" : { + "debug" : false, + "description" : "motion", + "name" : "motion", + "max_track_age" : 20.0, + "is_static" : false, + "max_part_measure_age" : 1.0, + "max_audio_measure_age" : 0.1, + "out_fov_age_factor" : 0.1, + "motion" : { + "name" : "motion", + "debug" : false, + "enable_back_projection" : true, + "default_ramge" : 2.5, + "use_pseudo_range" : true, + "extent_x" : 0.25, + "extent_y" : 0.25, + "extent_z" : 0.25, + "vel_decay" : 0.999, + "vel_limit" : 3.0, + "linear_psd" : 0.10, + "min_state_covariance" : 0.001, + "pseudo_range_covariance" : 0.5, + "fixed_pseudo_range" : 2.0, + "auto_add_tracker" : false, + "auto_add_tracker_skip" : 5, + "confidence_filter" : 0.9, + "pose_filter":0.0, + "tracker_type" : "tracker_1", + "tracker_1" : { + "tracker_class" : "motion_tracker", + "image_tracker" : { + "debug" : false, + "image_debug" : false, + "name" : "A", + "use_image_velocity" : false, + "vel_scale" : 0.9, + "vel_filter" : 0.75, + "base_covar" : 200.0, + "image_width" : 128, + "image_height" : 128, + "track_confidence_filter" : 0.6, + "model_confidence_filter" : 0.5, + "min_confidence" : 0.001, + "measure_interval": 0.001 + }, + "motion_tracker" : { + "background_threshold" : 5, + "drop_area_too_big" : 1600.0, + "drop_area_too_small" : 100.0, + "search_size_horizontal_scale" : 5, + "search_size_vertical_scale" : 5, + "camshift" : { + "max_iterations" : 6, + "area_scale" : 3.0, + "area_aspect" : 2.0 + } + } + } + } + } +} + diff --git a/V3.1/build/usr/local/etc/lps/schemas/focused.json b/V3.1/build/usr/local/etc/lps/schemas/focused.json new file mode 100644 index 00000000..2624f4cd --- /dev/null +++ b/V3.1/build/usr/local/etc/lps/schemas/focused.json @@ -0,0 +1,475 @@ +{ + "schemas" :[ + { + "name" : "FaceDetectSchema", + "type" : "image_update_schema", + "enable" : true, + "predicate" : { + "type" : "face_detection_predicate", + "enable" : true, + "start" : 0, + "skip_with_tracks" : 150, + "skip_without_tracks" : 10, + "criteria" : { + "cameraIds" : [0] + } + }, + "actions" :[ + { + "name" : "FaceDetectAction", + "type" : "face_detection_action", + "enable" : true, + "policy_detector" : { + "criteria" : { + "cameraIds" : [0], + "dimensions" : [ + { "width" : 640, "height" : 360 }, + { "width" : 80, "height" : 45 } + ], + "formats" : [3, 1], + "outputTypes" : [0, 2] + }, + "detector" : { + "min_hint_width" : 40, + "min_hint_height" : 40 + }, + "policies" : [ + { + "weight": 0.75, + "kind": "skincolor", + "sub_image_width" : 200, + "sub_image_height" : 200, + "extra_horizontal" : 20, + "extra_vertical" : 20 + }, + { + "weight": 0.25, + "kind": "gaussian", + "sub_image_width" : 200, + "sub_image_height" : 200, + "extra_horizontal" : 20, + "extra_vertical" : 20 + } + ] + } + }, + { + "name" : "FaceTrackerInitAction", + "type" : "face_tracker_init_action", + "enable" : true, + "max_tracked" : 1 + }, + { + "name" : "FaceRecognitionAction", + "type" : "face_recognition_action", + "enable" : true, + "turn_threshold" : 0.5 + } + ] + }, + { + "name" : "FaceTrackSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "FaceTrackerUpdate", + "type" : "face_tracker_update_action", + "enable" : true + } + ] + }, + { + "name" : "FaceTrackPoseEstimateSchema", + "type" : "image_update_schema", + "enable" : false, + "predicate" : { + "type" : "face_tracker_pose_estimate_predicate", + "enable" : true, + "start" : 4, + "skip" : 11 + }, + "actions" :[ + { + "name" : "FaceTrackerPoseEstimate", + "type" : "face_tracker_pose_estimate_action", + "enable" : true, + "hflip" : true, + "vflip" : false, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + }, + "detector" : { + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + }, + "min_hint_width" : 80, + "min_hint_height" : 80 + } + } + ] + }, + { + "name" : "FaceTrackReevaluateSchema", + "type" : "image_update_schema", + "enable" : true, + "predicate" : { + "type" : "face_reevaluate_predicate", + "enable" : true, + "start" : 0, + "skip" : 89 + }, + "actions" :[ + { + "name" : "FaceTrackerRedetectAction", + "type" : "face_tracker_redetect_action", + "enable" : true, + "detector" : { + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + }, + "min_hint_width" : 80, + "min_hint_height" : 80 + } + } + ] + }, + { + "name" : "FaceTrainSchema", + "type" : "image_update_schema", + "enable" : true, + "predicate" : { + "type" : "face_training_predicate", + "enable" : true, + "criteria" : { + "cameraIds" : [0] + } + }, + "actions" :[ + { + "name" : "FaceTrainingAction", + "type" : "face_training_action", + "enable" : true, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 640, "height" : 360 }], + "outputTypes" : [0] + } + } + ] + }, + { + "name" : "EntityDetailSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "EntityDetailAction", + "type" : "entity_detail_action", + "enable" : true, + "detector" : { + "criteria" : { + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + } + } + } + ] + }, + { + "name" : "VisualAwarenessSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" : [ + { + "name" : "VisualAwarenessUpdateAction", + "type" : "visual_awareness_update_action", + "enable" : true + } + ] + }, + { + "name" : "BarcodeSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "BarcodeRequestAction", + "type" : "barcode_request_action", + "enable" : true, + "reader":{ + "types": { + "EAN8" : false, + "UPCE" : false, + "ISBN10" : false, + "UPCA" : false, + "EAN13" : false, + "ISBN13" : false, + "I25" : false, + "CODE39" : false, + "PDF417" : false, + "QRCODE" : true, + "CODE128" : false + } + }, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 1280, "height" : 720 }], + "outputTypes" : [0] + } + } + ] + }, + { + "name" : "ControlChangeSchema", + "type" : "image_update_schema", + "enable": true, + "actions": [ + { + "name" : "ControlChangeRequestAction", + "type" : "control_change_request_action", + "enable" : true, + "defaultResetTimeout": 7000 + } + ] + }, + { + "name" : "MotionSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "MotionImagerAction", + "type" : "motion_imager_action", + "enable" : true, + "moving_delay" : 400, + "image_width" : 128, + "image_height" : 128, + "accumulator_alpha" : 0.75, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 320, "height" : 180 }], + "outputTypes" : [0] + }, + "motion_timeout" : 100 + }, + { + "name" : "MotionDetectionAction", + "type" : "motion_detection_action", + "enable" : true, + "moving_delay" : 400, + "detector": { + "min_width" : 10, + "min_height" : 10, + "criteria" : { + "cameraIds" : [0] + }, + "threshold" : { + "dynamic": true, + "low": 5, + "high": 50, + "ae":{ + "gain":16.0, + "time":0.0333 + }, + "awb":{ + "red":4.0, + "green":4.0, + "blue": 4.0 + } + } + } + }, + { + "name" : "MotionTrackerInitAction", + "type" : "motion_tracker_init_action", + "enable" : true, + "moving_delay" : 400, + "max_tracked" : 2, + "pool" : { + "format" : 1, + "width" : 128, + "height" : 128, + "size" : 8 + } + }, + { + "name" : "MotionTrackerUpdateAction", + "type" : "motion_tracker_update_action", + "enable" : true, + "moving_delay" : 400, + "pool" : { + "format" : 1, + "width" : 128, + "height" : 128, + "size" : 8 + } + } + ] + }, + { + "name" : "FaceTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "FaceTimeAction", + "type" : "face_time_action", + "enable" : true, + "drop_radius" : 0.3, + "fence_radius" : 4.0, + "ceiling_height": 2.0, + "floor_height": -2.0 + } + ] + }, + { + "name" : "FaceEvaluateTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "predicate" : { + "type" : "min_period_predicate", + "enable" : true, + "min_period" : 1.1 + }, + "actions" :[ + { + "name" : "PersonEvaluationAction", + "type" : "person_evaluation_action", + "enable" : true, + "unknown_age_limit":1.5, + "not_trained_age_limit":20.0, + "refresh_limit":50.0 + }, + { + "name" : "PresenceTimeAction", + "type" : "presence_time_action", + "enable" : true + } + ] + }, + { + "name" : "SpeakerTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "SpeakerTimeAction", + "type" : "speaker_time_action", + "enable" : true, + "expiration": 60000 + } + ] + }, + { + "name" : "AwarenessTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AwarenessTimeAction", + "type" : "awareness_time_action", + "enable" : true + } + ] + }, + { + "name" : "MotionTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "MotionTimeAction", + "type" : "motion_time_action", + "enable" : true, + "drop_radius" : 0.3, + "fence_radius" : 4.0 + } + ] + }, + { + "name" : "AudioTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioTimeAction", + "type" : "audio_time_action", + "enable" : true, + "event_timeout" : 5.0 + } + ] + }, + { + "name" : "PresenceTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "PresenceTimeAction", + "type" : "presence_time_action", + "enable" : true + } + ] + }, + { + "name" : "AudioDetectSchema", + "type" : "audio_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioDetectionAction", + "type" : "audio_detection_action", + "enable" : true + }, + { + "name" : "AudioMeasurementAction", + "type" : "audio_measurement_action", + "enable" : true, + "audio_angle_tolerance" : 0.5, + "audio_time_tolerance" : 0.35, + "in_fov_margin" : 0.1 + } + ] + }, + { + "name" : "AudioAwarenessSchema", + "type" : "audio_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioAwarenessAction", + "type" : "audio_awareness_action", + "enable" : true + } + ] + }, + { + "name" : "AxisUpdateSchema", + "type" : "axis_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "GeometryUpdateAction", + "type" : "geometry_update_action", + "enable" : true + }, + { + "name" : "MotionTrackerAxisUpdateAction", + "type" : "motion_tracker_axis_update_action", + "enable" : true + } + ] + } + ] +} diff --git a/V3.1/build/usr/local/etc/lps/schemas/minimal.json b/V3.1/build/usr/local/etc/lps/schemas/minimal.json new file mode 100644 index 00000000..cb902aec --- /dev/null +++ b/V3.1/build/usr/local/etc/lps/schemas/minimal.json @@ -0,0 +1,212 @@ +{ + "schemas" :[ + { + "name" : "EntityDetailSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "EntityDetailAction", + "type" : "entity_detail_action", + "enable" : true, + "detector" : { + "criteria" : { + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + } + } + } + ] + }, + { + "name" : "VisualAwarenessSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" : [ + { + "name" : "VisualAwarenessUpdateAction", + "type" : "visual_awareness_update_action", + "enable" : true + } + ] + }, + { + "name" : "BarcodeSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "BarcodeRequestAction", + "type" : "barcode_request_action", + "enable" : true, + "reader":{ + "types": { + "EAN8" : false, + "UPCE" : false, + "ISBN10" : false, + "UPCA" : false, + "EAN13" : false, + "ISBN13" : false, + "I25" : false, + "CODE39" : false, + "PDF417" : false, + "QRCODE" : true, + "CODE128" : false + } + }, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 1280, "height" : 720 }], + "outputTypes" : [0] + } + } + ] + }, + { + "name" : "ControlChangeSchema", + "type" : "image_update_schema", + "enable": true, + "actions": [ + { + "name" : "ControlChangeRequestAction", + "type" : "control_change_request_action", + "enable" : true, + "defaultResetTimeout": 7000 + } + ] + }, + { + "name" : "FaceTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "FaceTimeAction", + "type" : "face_time_action", + "enable" : true, + "drop_radius" : 0.3, + "fence_radius" : 4.0, + "ceiling_height": 2.0, + "floor_height": -2.0 + } + ] + }, + { + "name" : "AwarenessTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AwarenessTimeAction", + "type" : "awareness_time_action", + "enable" : true + } + ] + }, + { + "name" : "MotionTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "MotionTimeAction", + "type" : "motion_time_action", + "enable" : true, + "drop_radius" : 0.3, + "fence_radius" : 4.0 + } + ] + }, + { + "name" : "AudioTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioTimeAction", + "type" : "audio_time_action", + "enable" : true, + "event_timeout" : 5.0 + } + ] + }, + { + "name" : "PresenceTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "PresenceTimeAction", + "type" : "presence_time_action", + "enable" : true + } + ] + }, + { + "name" : "AudioDetectSchema", + "type" : "audio_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioDetectionAction", + "type" : "audio_detection_action", + "enable" : true + }, + { + "name" : "AudioMeasurementAction", + "type" : "audio_measurement_action", + "enable" : true, + "audio_angle_tolerance" : 0.5, + "audio_time_tolerance" : 0.35, + "in_fov_margin" : 0.1 + } + ] + }, + { + "name" : "AudioPresenceSchema", + "type" : "audio_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "SpeakerPresenceAudioAction", + "type" : "speaker_presence_audio_action", + "enable" : true, + "audio_angle_tolerance" : 0.5, + "audio_time_tolerance" : 0.35, + "in_fov_margin" : 0.1, + "presence_timeout" : 10.0 + } + ] + }, + { + "name" : "AudioAwarenessSchema", + "type" : "audio_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioAwarenessAction", + "type" : "audio_awareness_action", + "enable" : true + } + ] + }, + { + "name" : "AxisUpdateSchema", + "type" : "axis_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "GeometryUpdateAction", + "type" : "geometry_update_action", + "enable" : true + }, + { + "name" : "MotionTrackerAxisUpdateAction", + "type" : "motion_tracker_axis_update_action", + "enable" : true + } + ] + } + ] +} diff --git a/V3.1/build/usr/local/etc/lps/schemas/normal.json b/V3.1/build/usr/local/etc/lps/schemas/normal.json new file mode 100644 index 00000000..dada7d77 --- /dev/null +++ b/V3.1/build/usr/local/etc/lps/schemas/normal.json @@ -0,0 +1,495 @@ +{ + "schemas" :[ + { + "name" : "FaceDetectSchema", + "type" : "image_update_schema", + "enable" : true, + "predicate" : { + "type" : "face_detection_predicate", + "enable" : true, + "start" : 0, + "skip_with_tracks" : 30, + "skip_without_tracks" : 10, + "criteria" : { + "cameraIds" : [0] + } + }, + "actions" :[ + { + "name" : "FaceDetectAction", + "type" : "face_detection_action", + "enable" : true, + "policy_detector" : { + "criteria" : { + "cameraIds" : [0], + "dimensions" : [ + { "width" : 1280, "height" : 720 }, + { "width" : 80, "height" : 45 } + ], + "formats" : [3, 1], + "outputTypes" : [0, 2] + }, + "detector" : { + "min_hint_width" : 40, + "min_hint_height" : 40 + }, + "policies" : [ + { + "weight": 0.75, + "kind": "skincolor", + "sub_image_width" : 200, + "sub_image_height" : 200, + "extra_horizontal" : 20, + "extra_vertical" : 20 + }, + { + "weight": 0.25, + "kind": "gaussian", + "sub_image_width" : 200, + "sub_image_height" : 200, + "extra_horizontal" : 20, + "extra_vertical" : 20 + } + ] + } + }, + { + "name" : "FaceTrackerInitAction", + "type" : "face_tracker_init_action", + "enable" : true, + "max_tracked" : 3 + }, + { + "name": "FaceTrackerIDAction", + "type": "face_tracker_id_action", + "enable": true, + "criteria": { + "cameraIds" : [0], + "dimensions" : [{ "width" : 1280, "height" : 720 }], + "formats" : [3], + "outputTypes" : [0] + } + } + ] + }, + { + "name" : "FaceTrackSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "FaceTrackerUpdate", + "type" : "face_tracker_update_action", + "enable" : true + } + ] + }, + { + "name" : "FaceTrackPoseEstimateSchema", + "type" : "image_update_schema", + "enable" : false, + "predicate" : { + "type" : "face_tracker_pose_estimate_predicate", + "enable" : true, + "start" : 4, + "skip" : 11 + }, + "actions" :[ + { + "name" : "FaceTrackerPoseEstimate", + "type" : "face_tracker_pose_estimate_action", + "enable" : true, + "hflip" : true, + "vflip" : false, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + }, + "detector" : { + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + }, + "min_hint_width" : 80, + "min_hint_height" : 80 + } + } + ] + }, + { + "name" : "FaceTrackRedetectSchema", + "type" : "image_update_schema", + "enable" : true, + "predicate" : { + "type" : "face_reevaluate_predicate", + "enable" : true, + "start" : 0, + "skip" : 3 + }, + "actions" :[ + { + "name" : "PersonRedetectAction", + "type" : "person_redetect_action", + "enable" : true, + "detector" : { + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 1280, "height" : 720 }], + "formats" : [3], + "outputTypes" : [0] + }, + "min_hint_width" : 80, + "min_hint_height" : 80 + } + } + ] + }, + { + "name" : "DeferredDetectSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "DeferredDetectionAction", + "type" : "deferred_detection_action", + "enable" : true, + "detector" : { + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + }, + "min_hint_width" : 80, + "min_hint_height" : 80 + }, + "max_tracked": 3 + } + ] + }, + { + "name" : "FaceTrainSchema", + "type" : "image_update_schema", + "enable" : true, + "predicate" : { + "type" : "face_training_predicate", + "enable" : true + }, + "actions" :[ + { + "name" : "FaceTrainingAction", + "type" : "face_training_action", + "enable" : true + } + ] + }, + { + "name" : "EntityDetailSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "EntityDetailAction", + "type" : "entity_detail_action", + "enable" : true, + "detector" : { + "criteria" : { + "dimensions" : [{ "width" : 640, "height" : 360 }], + "formats" : [3], + "outputTypes" : [0] + } + } + } + ] + }, + { + "name" : "VisualAwarenessSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" : [ + { + "name" : "VisualAwarenessUpdateAction", + "type" : "visual_awareness_update_action", + "enable" : true + } + ] + }, + { + "name" : "BarcodeSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "BarcodeRequestAction", + "type" : "barcode_request_action", + "enable" : true, + "reader":{ + "types": { + "EAN8" : false, + "UPCE" : false, + "ISBN10" : false, + "UPCA" : false, + "EAN13" : false, + "ISBN13" : false, + "I25" : false, + "CODE39" : false, + "PDF417" : false, + "QRCODE" : true, + "CODE128" : false + } + }, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 1280, "height" : 720 }], + "outputTypes" : [0] + } + } + ] + }, + { + "name" : "ControlChangeSchema", + "type" : "image_update_schema", + "enable": true, + "actions": [ + { + "name" : "ControlChangeRequestAction", + "type" : "control_change_request_action", + "enable" : true, + "defaultResetTimeout": 7000 + } + ] + }, + { + "name" : "MotionSchema", + "type" : "image_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "MotionImagerAction", + "type" : "motion_imager_action", + "enable" : true, + "moving_delay" : 400, + "image_width" : 128, + "image_height" : 128, + "accumulator_alpha" : 0.75, + "criteria" : { + "cameraIds" : [0], + "dimensions" : [{ "width" : 320, "height" : 180 }], + "outputTypes" : [0] + }, + "motion_timeout" : 100 + }, + { + "name" : "MotionDetectionAction", + "type" : "motion_detection_action", + "enable" : true, + "moving_delay" : 400, + "detector": { + "min_width" : 10, + "min_height" : 10, + "criteria" : { + "cameraIds" : [0] + }, + "threshold" : { + "dynamic": true, + "low": 5, + "high": 50, + "ae":{ + "gain":16.0, + "time":0.0333 + }, + "awb":{ + "red":4.0, + "green":4.0, + "blue": 4.0 + } + } + } + }, + { + "name" : "MotionTrackerInitAction", + "type" : "motion_tracker_init_action", + "enable" : true, + "moving_delay" : 400, + "max_tracked" : 3, + "pool" : { + "format" : 1, + "width" : 128, + "height" : 128, + "size" : 8 + } + }, + { + "name" : "MotionTrackerUpdateAction", + "type" : "motion_tracker_update_action", + "enable" : true, + "moving_delay" : 400, + "pool" : { + "format" : 1, + "width" : 128, + "height" : 128, + "size" : 8 + } + } + ] + }, + { + "name" : "FaceTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "FaceTimeAction", + "type" : "face_time_action", + "enable" : true, + "drop_radius" : 0.3, + "fence_radius": 4.0, + "ceiling_height": 2.0, + "floor_height": -2.0 + } + ] + }, + { + "name" : "FaceEvaluateTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "predicate" : { + "type" : "min_period_predicate", + "enable" : true, + "min_period" : 1.1 + }, + "actions" :[ + { + "name" : "PersonEvaluationAction", + "type" : "person_evaluation_action", + "enable" : true, + "unknown_age_limit":10.0, + "not_trained_age_limit":20.0, + "refresh_limit":50.0 + }, + { + "name" : "PresenceTimeAction", + "type" : "presence_time_action", + "enable" : true + } + ] + }, + { + "name" : "SpeakerTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "SpeakerTimeAction", + "type" : "speaker_time_action", + "enable" : true, + "expiration": 60000 + } + ] + }, + { + "name" : "AwarenessTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AwarenessTimeAction", + "type" : "awareness_time_action", + "enable" : true + } + ] + }, + { + "name" : "MotionTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "MotionTimeAction", + "type" : "motion_time_action", + "enable" : true, + "drop_radius" : 0.3, + "fence_radius": 4.0 + } + ] + }, + { + "name" : "AudioTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioTimeAction", + "type" : "audio_time_action", + "enable" : true, + "event_timeout" : 5.0 + } + ] + }, + { + "name" : "PresenceTimeSchema", + "type" : "time_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "PresenceTimeAction", + "type" : "presence_time_action", + "enable" : true + } + ] + }, + { + "name" : "AudioDetectSchema", + "type" : "audio_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioDetectionAction", + "type" : "audio_detection_action", + "enable" : true + }, + { + "name" : "AudioMeasurementAction", + "type" : "audio_measurement_action", + "enable" : true, + "audio_angle_tolerance" : 0.5, + "audio_time_tolerance" : 0.35, + "in_fov_margin" : 0.1 + } + ] + }, + { + "name" : "AudioAwarenessSchema", + "type" : "audio_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "AudioAwarenessAction", + "type" : "audio_awareness_action", + "enable" : true + } + ] + }, + { + "name" : "AxisUpdateSchema", + "type" : "axis_update_schema", + "enable" : true, + "actions" :[ + { + "name" : "GeometryUpdateAction", + "type" : "geometry_update_action", + "enable" : true + }, + { + "name" : "MotionTrackerAxisUpdateAction", + "type" : "motion_tracker_axis_update_action", + "enable" : true + } + ] + } + ] +}