(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.jiboEmbodiedDialog = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o embodied.listen.debugPerceptualEye = value); Tunable.getButtonField('Person found', WIN).events.change.on(() => { embodied.listen.eventsIn.personFound.emit(); }); Tunable.getButtonField('Person lost', WIN).events.change.on(() => { embodied.listen.eventsIn.personLost.emit(); }); Tunable.getButtonField('Demo engaged ON', WIN).events.change.on(() => { embodied.listen.demoEngaged(true); }); Tunable.getButtonField('Demo engaged OFF', WIN).events.change.on(() => { embodied.listen.demoEngaged(false); }); Tunable.getButtonField('Disable once', WIN).events.change.on(() => { embodied.listen.disableOnce(); }); } } } exports.TunableDebug = TunableDebug; },{"jibo-tunable":undefined}],4:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const EmbodiedDialog_1 = require("./EmbodiedDialog"); exports.EmbodiedState = EmbodiedDialog_1.EmbodiedState; const speech_ns = require("./speech"); const listen_ns = require("./listen"); const TunableDebug_1 = require("./TunableDebug"); const logs = require("./log"); const log = logs.dialogLog; let _jibo; function init(jibo, options, autoRuleConfig) { return __awaiter(this, void 0, void 0, function* () { _jibo = jibo; exports._embodied = new EmbodiedDialog_1.EmbodiedDialog(); yield exports._embodied.init(_jibo, options, autoRuleConfig); _jibo.jetstream.events.hjHeard.on(() => { exports._embodied.listen.eventsIn.hjHeard.emit(); process.nextTick(() => exports._embodied.listen.eventsIn.oriented.emit()); }); _jibo.jetstream.events.localTurnStarted.on(() => { exports._embodied.listen.eventsIn.startListen.emit(); }); _jibo.jetstream.events.globalTurnResult.on(() => { exports._embodied.listen.eventsIn.cloudFinished.emit(); }); _jibo.globalEvents.shared.hjOnly.on(() => { exports._embodied.listen.eventsIn.cloudFinished.emit(); }); _jibo.jetstream.events.localTurnResult.on(() => { exports._embodied.listen.eventsIn.cloudFinished.emit(); }); _jibo.jetstream.events.eos.on(() => { exports._embodied.listen.eventsIn.eos.emit(); }); _jibo.jetstream.events.sos.on(() => { exports._embodied.listen.eventsIn.sos.emit(); }); let personVisible; if (_jibo.lps.identity.getVisibleFaces().size === 0) { personVisible = false; exports._embodied.listen.eventsIn.personLost.emit(); } else { personVisible = true; exports._embodied.listen.eventsIn.personFound.emit(); } _jibo.lps.identity.events.visibleFaceStarted.on(data => { if (!personVisible) { personVisible = true; exports._embodied.listen.eventsIn.personFound.emit(); } }); _jibo.lps.identity.events.visibleFaceStopped.on(data => { const visible = _jibo.lps.identity.getVisibleFaces(); if (visible.size === 0 || (visible.size === 1 && visible.has(data.id))) { personVisible = false; exports._embodied.listen.eventsIn.personLost.emit(); } }); TunableDebug_1.TunableDebug.setup(exports._embodied); exports._embodied.listen.start(); }); } exports.init = init; function isInitialized() { return !!exports._embodied; } exports.isInitialized = isInitialized; function getState() { return exports._embodied.getState(); } exports.getState = getState; var speech; (function (speech) { speech.RequestStatus = speech_ns.common.RequestStatus; function speak(textToSpeak, speakOptions, autoRuleConfig) { return exports._embodied.speech.speak(textToSpeak, speakOptions, autoRuleConfig); } speech.speak = speak; function stop() { return exports._embodied.speech.stop(); } speech.stop = stop; function setPaths(assetPack) { let resourceRoot = null; if (assetPack) { try { resourceRoot = _jibo.utils.PathUtils.getAssetUri('', assetPack); } catch (e) { log.warn('Unable to set resourceRoot, path-based assets will be unavailable: ', e); } } exports._embodied.speech.setResourceRoots(resourceRoot); } speech.setPaths = setPaths; function isActive() { return !!exports._embodied.speech.isActive(); } speech.isActive = isActive; function waitForAnimDispatch() { return exports._embodied.speech.waitForAnimDispatch(); } speech.waitForAnimDispatch = waitForAnimDispatch; })(speech = exports.speech || (exports.speech = {})); var listen; (function (listen) { listen.AmbientListenMode = listen_ns.AmbientListenMode; listen.ActiveListenMode = listen_ns.ActiveListenMode; listen.RequestStatus = listen_ns.RequestStatus; function getAmbientMode() { return exports._embodied.listen.getAmbientMode(); } listen.getAmbientMode = getAmbientMode; function getActiveMode() { return exports._embodied.listen.getActiveMode(); } listen.getActiveMode = getActiveMode; function setAmbientMode(mode) { exports._embodied.listen.setAmbientMode(mode); } listen.setAmbientMode = setAmbientMode; function enterActiveMode(mode, options) { return exports._embodied.listen.enterActiveMode(mode, options); } listen.enterActiveMode = enterActiveMode; function exitActiveMode(fastForward) { return exports._embodied.listen.exitActiveMode(fastForward); } listen.exitActiveMode = exitActiveMode; function dispose() { exports._embodied.listen.dispose(); } listen.dispose = dispose; function waitForIdle(fastForward) { return exports._embodied.listen.waitForIdle(fastForward); } listen.waitForIdle = waitForIdle; function disableOnce() { exports._embodied.listen.disableOnce(); } listen.disableOnce = disableOnce; function demoEngaged(turnOn) { exports._embodied.listen.demoEngaged(turnOn); } listen.demoEngaged = demoEngaged; })(listen = exports.listen || (exports.listen = {})); },{"./EmbodiedDialog":2,"./TunableDebug":3,"./listen":10,"./log":11,"./speech":15}],5:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const findRoot = require('find-root'); const ROOT = findRoot(__filename); const SOUNDS_ROOT = ROOT + '/resources/audio/'; exports.Anims = { ENGAGE: `User_Speak_Begin_01`, DISENGAGE: `User_Speak_End_01`, THINKING: `Thinking_Short_01`, BLINK: `Eye_Quick_Blink_01`, HEAD_BOB: [ `Listen_Head_Bob_01`, `Speak_Posture_Shift_01`, ], }; var EyeTransitionStyle; (function (EyeTransitionStyle) { EyeTransitionStyle[EyeTransitionStyle["POP"] = 0] = "POP"; EyeTransitionStyle[EyeTransitionStyle["BLINK"] = 1] = "BLINK"; })(EyeTransitionStyle = exports.EyeTransitionStyle || (exports.EyeTransitionStyle = {})); var EyeTransitionDirection; (function (EyeTransitionDirection) { EyeTransitionDirection[EyeTransitionDirection["TO"] = 0] = "TO"; EyeTransitionDirection[EyeTransitionDirection["FROM"] = 1] = "FROM"; })(EyeTransitionDirection = exports.EyeTransitionDirection || (exports.EyeTransitionDirection = {})); class AnimSelector { static getPerceptualEyeTransition(seePerson, style, direction, sound = true) { let nameComps = []; nameComps.push(seePerson ? 'hj-sp-transition-' : 'hj-dsp-transition-'); nameComps.push(direction === EyeTransitionDirection.FROM ? 'from-' : 'to-'); nameComps.push(style === EyeTransitionStyle.POP ? 'pop' : 'blink'); if (direction === EyeTransitionDirection.TO) { nameComps.push(sound ? '' : '-ns'); } return nameComps.join(''); } static getPerceptualEyePose(seePerson, listening) { let nameComps = []; nameComps.push(listening ? 'l-' : 'nl-'); nameComps.push(seePerson ? 'sp' : 'dsp'); return nameComps.join(''); } } exports.AnimSelector = AnimSelector; exports.AnimConfigs = { 'NO_BODY': { layers: { 'Body': false } } }; const LISTENING = [0.05, 0.73, 0.94]; exports.Led = { LISTENING: LISTENING, LISTENING_OR: LISTENING, OFF: [0, 0, 0], }; exports.Sound = { ON: SOUNDS_ROOT + `speak-cue.mp3` }; },{"find-root":undefined}],6:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_cai_utils_1 = require("jibo-cai-utils"); const jibo_state_machine_1 = require("jibo-state-machine"); const Types_1 = require("./Types"); const Events_1 = require("./Events"); const Utils_1 = require("./Utils"); const Utils_2 = require("../utils/Utils"); const Assets_1 = require("./Assets"); const logs = require("../log"); const log = logs.listenLog; const HJ_BLINK_INTERVAL = 25 * 1000; const USE_CENTER_ROBOT = false; const TIMEOUT = 3000; const ANIM_CLEAR_TIMEOUT = 'ANIM_CLEAR_TIMEOUT'; class EmbodiedListen extends jibo_state_machine_1.StateMachine { constructor(_ed) { super(); this._ed = _ed; this._queue = new jibo_cai_utils_1.PromiseQueue(); this._idle = new jibo_state_machine_1.State(this, 'Idle'); this._reset = new jibo_state_machine_1.State(this, 'Reset'); this._hjExpression = new jibo_state_machine_1.State(this, 'HJ Expression'); this._nonHjExpression = new jibo_state_machine_1.State(this, 'NonHJ Expression'); this._offExpression = new jibo_state_machine_1.State(this, 'Off Expression'); this._engage = new jibo_state_machine_1.State(this, 'Engage'); this._listening = new jibo_state_machine_1.State(this, 'Listen'); this._thinking = new jibo_state_machine_1.State(this, 'Off Expression + Thinking'); this._waitForAnimFinish = new jibo_state_machine_1.State(this, 'Wait for anim finish'); this._active = new jibo_state_machine_1.State(this, 'Active Mode'); this.listenState = Types_1.ListeningState.NOT_LISTENING; this.debugPerceptualEye = false; this.eventsIn = new Events_1.IncomingEvents(); this.eventsOut = new Events_1.OutgoingEvents(); this.eventsInternal = new Events_1.InternalEvents(); this._disableOnce = false; this._lastHeadBob = -1; this._nextHeadBob = 0; this._ambientMode = Types_1.AmbientListenMode.NORMAL; this._activeMode = null; this._lastHJBlinkTime = -1; } init(jibo, debug = false) { return __awaiter(this, void 0, void 0, function* () { if (jibo) { this._jibo = jibo; } else { throw new Error('Cannot instantiate without Jibo instances!'); } const dofs = this._jibo.expression.dofs; this.DOFS_ALL_MINUS_EYE_TRANSLATE = dofs.ALL.minus(dofs.EYE_TRANSLATE); this.DOFS_EYE_AND_OVERLAY_MINUS_EYE_TRANSLATE = (dofs.EYE.plus(dofs.OVERLAY)).minus(dofs.EYE_TRANSLATE); this.DOFS_TO_CENTER = dofs.ALL.minus(dofs.LED).minus(dofs.BODY).minus(dofs.EYE_TRANSLATE); this._debug = debug; jibo.timer.once('update', () => { log.debug('Caching core Embodied Listen animations.'); const seePerson = true; const eyeStyles = [Assets_1.EyeTransitionStyle.POP, Assets_1.EyeTransitionStyle.BLINK]; const dir = Assets_1.EyeTransitionDirection.TO; const listening = true; const useSound = false; eyeStyles.forEach((styleKey) => __awaiter(this, void 0, void 0, function* () { let anim; try { anim = Assets_1.AnimSelector.getPerceptualEyeTransition(seePerson, styleKey, dir, useSound); yield Utils_2.Utils.loadAnimationIntoCache(jibo, anim, jibo_cai_utils_1.CacheUtils.GlobalCacheName); anim = Assets_1.AnimSelector.getPerceptualEyePose(seePerson, listening); yield Utils_2.Utils.loadAnimationIntoCache(jibo, anim, jibo_cai_utils_1.CacheUtils.GlobalCacheName); } catch (err) { log.warn(`Unable to load animation ${anim} into cache ${jibo_cai_utils_1.CacheUtils.GlobalCacheName}`); } })); }); [ this._hjExpression, this._nonHjExpression, this._offExpression, this._engage, this._listening, this._thinking, this._waitForAnimFinish, ].forEach(state => { state.addTypedEventTransition(this.eventsInternal.reset, this._reset); }); this._reset.addDoneTransition(this._idle); this._reset.onEntry = () => __awaiter(this, void 0, void 0, function* () { log.debug('Entering Reset'); this._queue.clear(); const waitResult = yield Utils_2.timeout(this._queue.waitUntilEmpty(), TIMEOUT, { log: log, logMessage: 'embodiedListen.animQueue.waitUntilEmpty', timeoutValue: ANIM_CLEAR_TIMEOUT }); if (typeof waitResult === 'string') { if (waitResult === ANIM_CLEAR_TIMEOUT) { this._queue = new jibo_cai_utils_1.PromiseQueue(); } } if (this._reset.isCurrent()) { Utils_1.Utils.setLED(this._jibo, Assets_1.Led.OFF); yield this.stopListeningEye(); } }); const updateBind = this.update.bind(this); updateBind.isGlobalTimer = true; this._idle.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); this._idle.addTypedEventTransition(this.eventsIn.startListen, this._nonHjExpression); this._idle.addTypedEventTransition(this.eventsInternal.enterActiveMode, this._active); this._idle.onEntry = () => { log.debug('Entering Idle'); this._jibo.performance.log('EmbodiedListenReturnedToIdleSkill'); this.eventsOut.finished.emit(); this._jibo.timer.removeListener('update', updateBind); if (this._pendingActiveMode) { this.eventsInternal.enterActiveMode.emit(); } }; this._idle.onExit = () => { log.debug('Exiting Idle'); this._jibo.timer.on('update', updateBind); }; this._active.addTypedEventTransition(this.eventsInternal.exitActiveMode, this._idle); this._active.addTypedEventTransition(this.eventsInternal.enterActiveMode, this._active); this._active.onEntry = () => { this._activeMode = this._pendingActiveMode; this._pendingActiveModeExtPr.resolve(Types_1.RequestStatus.SUCCEEDED); this._pendingActiveMode = null; log.debug(`Entering Active Mode: ${this._activeMode}`); switch (this._activeMode) { case Types_1.ActiveListenMode.UI: Utils_1.Utils.setLED(this._jibo, Assets_1.Led.LISTENING); break; case Types_1.ActiveListenMode.OPTIONAL_RESPONSE: Utils_1.Utils.setLED(this._jibo, Assets_1.Led.LISTENING_OR); this.startListeningEye(true); break; } }; this._active.onExit = () => __awaiter(this, void 0, void 0, function* () { log.debug(`Exiting Active Mode: ${this._activeMode}`); switch (this._activeMode) { case Types_1.ActiveListenMode.UI: Utils_1.Utils.setLED(this._jibo, Assets_1.Led.OFF); break; case Types_1.ActiveListenMode.OPTIONAL_RESPONSE: Utils_1.Utils.setLED(this._jibo, Assets_1.Led.OFF); yield this.stopListeningEye(); break; } this._activeMode = null; }); this._waitForAnimFinish.addInternalTransition('Finished', this._idle); this._waitForAnimFinish.onEntry = () => { log.debug('Entering Wait for Anim Finished'); this._jibo.performance.log('EmbodiedListenWaitForAnimFinishSkill'); if (USE_CENTER_ROBOT) { this._addAnimToQueue(() => { return this._jibo.expression.centerRobot({ requestor: EmbodiedListen.NONHJ_OWNER_INFORMATION, dofs: this.DOFS_TO_CENTER, centerGlobally: false }); }); } this._queue.waitUntilEmpty() .then(() => { if (this._waitForAnimFinish._isCurrent) { this._waitForAnimFinish.transitionTo(this._idle); } }).catch(e => { log.error(e); if (this._waitForAnimFinish._isCurrent) { this._waitForAnimFinish.transitionTo(this._idle); } }); }; this._offExpression.addDoneTransition(this._waitForAnimFinish); this._offExpression.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); this._offExpression.onEntry = () => __awaiter(this, void 0, void 0, function* () { log.debug('Entering Off Expression'); Utils_1.Utils.setLED(jibo, Assets_1.Led.OFF); yield this.stopListeningEye(); }); this._hjExpression.addTypedEventTransition(this.eventsIn.oriented, this._engage); this._hjExpression.addTypedEventTransition(this.eventsIn.cloudFinished, this._offExpression); this._hjExpression.addInternalTransition('Disable once', this._idle); this._hjExpression.onEntry = () => { log.debug('Entering HJ Expression (Eye+LED)'); this._jibo.performance.log('EmbodiedListenStartingHJHeardSkill'); if (this._disableOnce) { this._disableOnce = false; this._hjExpression.transitionTo(this._idle); } else { switch (this._ambientMode) { case Types_1.AmbientListenMode.NORMAL: case Types_1.AmbientListenMode.NO_BODY: Utils_1.Utils.setLED(this._jibo, Assets_1.Led.LISTENING); this.startListeningEye(true); break; } } }; this._nonHjExpression.addInternalTransition('ES Dispatch Done', this._engage); this._nonHjExpression.addInternalTransition('Disable once', this._idle); this._nonHjExpression.addTypedEventTransition(this.eventsIn.cloudFinished, this._offExpression); this._nonHjExpression.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); this._nonHjExpression.onEntry = () => { log.debug('Entering non-HJ Expression (Eye+LED)'); this._jibo.performance.log('EmbodiedListenStartingNonHJSkill'); if (this._disableOnce) { this._disableOnce = false; this._nonHjExpression.transitionTo(this._idle); } else { switch (this._ambientMode) { case Types_1.AmbientListenMode.NORMAL: case Types_1.AmbientListenMode.NO_BODY: Utils_1.Utils.setLED(this._jibo, Assets_1.Led.LISTENING); this.startListeningEye(true); break; } this._ed.speech.waitForAnimDispatch().then(() => { if (this.current === this._nonHjExpression) { this._nonHjExpression.transitionTo(this._engage); } }); } }; this._engage.addTypedEventTransition(this.eventsIn.sos, this._listening); this._engage.addTypedEventTransition(this.eventsIn.eos, this._thinking); this._engage.addTypedEventTransition(this.eventsIn.cloudFinished, this._offExpression); this._engage.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); let nextBlinkTime = -1; this._engage.onEntry = () => { log.debug('Entering Engage'); nextBlinkTime = Date.now() + 2000; switch (this._ambientMode) { case Types_1.AmbientListenMode.NORMAL: break; case Types_1.AmbientListenMode.NO_BODY: break; } }; this._engage.onUpdate = () => { let now = Date.now(); if (now >= nextBlinkTime) { nextBlinkTime = now + Math.floor(2500 + Math.random() * 1000); } }; this._engage.onExit = () => { switch (this._ambientMode) { case Types_1.AmbientListenMode.NORMAL: break; case Types_1.AmbientListenMode.NO_BODY: break; } }; this._listening.addTypedEventTransition(this.eventsIn.eos, this._thinking); this._listening.addTypedEventTransition(this.eventsIn.cloudFinished, this._offExpression); this._listening.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); let incrementalHandler = this._handleIncremental.bind(this); this._listening.onEntry = () => { log.debug('Entering Listening'); this.eventsIn.sos.on(incrementalHandler); }; this._listening.onExit = () => { this.eventsIn.sos.removeListener(incrementalHandler); }; this._thinking.addTypedEventTransition(this.eventsIn.cloudFinished, this._waitForAnimFinish); this._thinking.addTypedEventTransition(this.eventsIn.hjHeard, this._hjExpression); this._thinking.onEntry = () => __awaiter(this, void 0, void 0, function* () { log.debug('Entering Thinking (+ Off Expression)'); Utils_1.Utils.setLED(jibo, Assets_1.Led.OFF); yield this.stopListeningEye(); }); }); } dispose() { } waitForIdle(fastForward = false) { if (this.current === this._idle) { return Promise.resolve(); } else { return new Promise(resolve => { this.eventsOut.finished.once(resolve); if (fastForward) { this.eventsInternal.reset.emit(); } }); } } demoEngaged(turnOn) { if (turnOn) { Utils_1.Utils.setLED(this._jibo, Assets_1.Led.LISTENING); return Promise.all([this.startListeningEye(true)]); } else { Utils_1.Utils.setLED(this._jibo, Assets_1.Led.OFF); return Promise.all([this.stopListeningEye()]); } } disableOnce() { this._disableOnce = true; } getAmbientMode() { return this._ambientMode; } getActiveMode() { return this._activeMode; } setAmbientMode(mode) { const newMode = Types_1.AmbientListenMode[mode]; if (!newMode) { log.warn(`Requested invalid Ambient Listen Mode '${mode}'`); } else { this._ambientMode = newMode; } } enterActiveMode(mode, options) { const newMode = Types_1.ActiveListenMode[mode]; if (!newMode) { log.warn(`Requested invalid Active Listen Mode '${mode}'`); return Promise.resolve(Types_1.RequestStatus.REJECTED); } else { if (this._activeMode) { if (newMode !== this._activeMode) { this._pendingActiveMode = newMode; this._pendingActiveModeExtPr = new Utils_2.ExtPromise(); this.eventsInternal.enterActiveMode.emit(); return this._pendingActiveModeExtPr.promise; } else { return Promise.resolve(Types_1.RequestStatus.SUCCEEDED); } } else { if (this._pendingActiveMode) { this._pendingActiveMode = null; this._pendingActiveModeExtPr.resolve(Types_1.RequestStatus.INTERRUPTED); } this._pendingActiveMode = newMode; this._pendingActiveModeExtPr = new Utils_2.ExtPromise(); this.waitForIdle(true); this.eventsInternal.enterActiveMode.emit(); return this._pendingActiveModeExtPr.promise; } } } exitActiveMode(fastforward = true) { if (this._activeMode) { if (this._pendingActiveMode) { this._pendingActiveMode = null; this._pendingActiveModeExtPr.resolve(Types_1.RequestStatus.CANCELLED); } this.eventsInternal.exitActiveMode.emit(); return this.waitForIdle(fastforward); } else { return Promise.resolve(); } } startListeningEye(heyJibo) { const seePerson = true; const dir = Assets_1.EyeTransitionDirection.TO; const useSound = false; const now = Date.now(); if (this.listenState !== Types_1.ListeningState.LISTENING) { this.listenState = Types_1.ListeningState.LISTENING; if (heyJibo && ((now - this._lastHJBlinkTime) > HJ_BLINK_INTERVAL)) { this._lastHJBlinkTime = now; this.addEyeAnimationToQueue(Assets_1.AnimSelector.getPerceptualEyeTransition(seePerson, Assets_1.EyeTransitionStyle.BLINK, dir, useSound), heyJibo); } else { this.addEyeAnimationToQueue(Assets_1.AnimSelector.getPerceptualEyeTransition(seePerson, Assets_1.EyeTransitionStyle.POP, dir, useSound), heyJibo); } } } stopListeningEye() { return __awaiter(this, void 0, void 0, function* () { if (this.listenState !== Types_1.ListeningState.NOT_LISTENING) { this.listenState = Types_1.ListeningState.NOT_LISTENING; yield this._jibo.expression.cleanup({ requestor: EmbodiedListen.HJ_OWNER_INFORMATION, dofs: this.DOFS_TO_CENTER }); } }); } _handleIncremental() { if (this._ambientMode !== Types_1.AmbientListenMode.NO_BODY) { const now = Date.now(); if (now - this._lastHeadBob > this._nextHeadBob) { this._nextHeadBob = Math.floor(1000 + Math.random() * 1000); this._lastHeadBob = now; } } } addEyeAnimationToQueue(name, heyJibo = false) { const baseConfig = { cache: jibo_cai_utils_1.CacheUtils.GlobalCacheName, dofs: this.DOFS_EYE_AND_OVERLAY_MINUS_EYE_TRANSLATE, }; const nonhjConfig = Object.assign({}, baseConfig, { layer: "beat" }); const config = heyJibo ? baseConfig : nonhjConfig; const baseOptions = { screenCenterOverride: true, disableSetFaceAnim: true }; const hjOptions = Object.assign({}, baseOptions, { ownerInformation: EmbodiedListen.HJ_OWNER_INFORMATION, disableSetFaceAnim: false, }); const options = heyJibo ? hjOptions : baseOptions; this._addAnimToQueue(name, config, options); } _addAnimToQueue(anim, config, options) { if (typeof anim === 'function') { this._queue.add(anim); } else { this._queue.add(() => __awaiter(this, void 0, void 0, function* () { const playbackResult = Utils_1.Utils.playAnimation({ jibo: this._jibo, animName: anim, config: config, options: options, }); this._lastAnimInstance = playbackResult.playback; return playbackResult.result; })); } } } EmbodiedListen.NONHJ_OWNER_INFORMATION = 'EmbodiedListen'; EmbodiedListen.HJ_OWNER_INFORMATION = 'BargeIn'; exports.EmbodiedListen = EmbodiedListen; },{"../log":11,"../utils/Utils":65,"./Assets":5,"./Events":7,"./Types":8,"./Utils":9,"jibo-cai-utils":undefined,"jibo-state-machine":undefined}],7:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_typed_events_1 = require("jibo-typed-events"); class IncomingEvents extends jibo_typed_events_1.EventContainer { constructor() { super(); this.hjHeard = new jibo_typed_events_1.Event('Hey Jibo heard'); this.startListen = new jibo_typed_events_1.Event('Started listening'); this.oriented = new jibo_typed_events_1.Event('Oriented'); this.sos = new jibo_typed_events_1.Event('Start of speech'); this.cloudFinished = new jibo_typed_events_1.Event('Cloud finished'); this.eos = new jibo_typed_events_1.Event('End of speech'); this.personFound = new jibo_typed_events_1.Event('Person visually found'); this.personLost = new jibo_typed_events_1.Event('Person visually lost'); } } exports.IncomingEvents = IncomingEvents; class OutgoingEvents extends jibo_typed_events_1.EventContainer { constructor() { super(); this.finished = new jibo_typed_events_1.Event('Embodied Listen finished'); } } exports.OutgoingEvents = OutgoingEvents; class InternalEvents extends jibo_typed_events_1.EventContainer { constructor() { super(); this.reset = new jibo_typed_events_1.Event(`Reset`); this.enterActiveMode = new jibo_typed_events_1.Event(`Enter Active Mode`); this.exitActiveMode = new jibo_typed_events_1.Event(`Exit Active Mode`); } } exports.InternalEvents = InternalEvents; },{"jibo-typed-events":undefined}],8:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var RequestStatus; (function (RequestStatus) { RequestStatus["SUCCEEDED"] = "SUCCEEDED"; RequestStatus["INTERRUPTED"] = "INTERRUPTED"; RequestStatus["CANCELLED"] = "CANCELLED"; RequestStatus["REJECTED"] = "REJECTED"; })(RequestStatus = exports.RequestStatus || (exports.RequestStatus = {})); var VisualPerceptionState; (function (VisualPerceptionState) { VisualPerceptionState["PERSON"] = "PERSON"; VisualPerceptionState["NO_PERSON"] = "NO_PERSON"; })(VisualPerceptionState = exports.VisualPerceptionState || (exports.VisualPerceptionState = {})); var ListeningState; (function (ListeningState) { ListeningState["LISTENING"] = "LISTENING"; ListeningState["NOT_LISTENING"] = "NOT_LISTENING"; })(ListeningState = exports.ListeningState || (exports.ListeningState = {})); var FadeDirection; (function (FadeDirection) { FadeDirection["IN"] = "IN"; FadeDirection["OUT"] = "OUT"; })(FadeDirection = exports.FadeDirection || (exports.FadeDirection = {})); var AmbientListenMode; (function (AmbientListenMode) { AmbientListenMode["NORMAL"] = "NORMAL"; AmbientListenMode["NO_BODY"] = "NO_BODY"; })(AmbientListenMode = exports.AmbientListenMode || (exports.AmbientListenMode = {})); var ActiveListenMode; (function (ActiveListenMode) { ActiveListenMode["OPTIONAL_RESPONSE"] = "OPTIONAL_RESPONSE"; ActiveListenMode["UI"] = "UI"; })(ActiveListenMode = exports.ActiveListenMode || (exports.ActiveListenMode = {})); },{}],9:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const EmbodiedListen_1 = require("./EmbodiedListen"); const jibo_cai_utils_1 = require("jibo-cai-utils"); exports.WIN_NAME = 'Embodied Listen'; let ALL_MINUS_BASE; class Utils { static playAnimation(options) { const jibo = options.jibo; if (!ALL_MINUS_BASE) { ALL_MINUS_BASE = jibo.expression.dofs.ALL.minus(jibo.expression.dofs.BASE); } const asset = jibo.animDB.getAnimByName(options.animName); if (!asset) { throw new Error(`No animation of name '${options.animName}' found in animDB`); } const hasDOFs = !!(options.config && options.config.dofs); const config = hasDOFs ? options.config : Object.assign(options.config || { cache: jibo_cai_utils_1.CacheUtils.GlobalCacheName }, { dofs: ALL_MINUS_BASE }); const hasOwner = !!(options.options && options.options.ownerInformation); const playbackOptions = hasOwner ? options.options : Object.assign(options.options || {}, { ownerInformation: EmbodiedListen_1.EmbodiedListen.NONHJ_OWNER_INFORMATION, }); return asset.play(config, playbackOptions); } static setLED(jibo, color) { jibo.performance.log('EmbodiedListenSettingLEDSkill', (color[0] !== 0) ? 'ON' : 'OFF'); jibo.expression.setLEDColor(color); } static playSound(jibo, path) { return new Promise((resolve, reject) => { jibo.performance.log('EmbodiedListenLoadSoundSkill'); const sound = jibo.loader.cached(path, jibo_cai_utils_1.CacheUtils.GlobalCacheName); if (sound) { jibo.performance.log('EmbodiedListenPlaySoundSkill', path); sound.play(() => resolve()); } else { jibo.loader.load({ type: 'sound', id: path, cache: jibo_cai_utils_1.CacheUtils.GlobalCacheName, src: path, complete: (err, sound) => { if (err) { return reject(err); } jibo.performance.log('EmbodiedListenPlaySoundSkill', path); sound.play(() => resolve()); } }); } }); } static packRGB(rbgArray) { return (rbgArray[0] & 0xff) << 16 | (rbgArray[1] & 0xff) << 8 | (rbgArray[2] & 0xff); } static unpackRGB(rgbPacked) { const r = (rgbPacked >>> 16) & 0xff; const g = (rgbPacked >>> 8) & 0xff; const b = (rgbPacked) & 0xff; return [r, g, b]; } } exports.Utils = Utils; },{"./EmbodiedListen":6,"jibo-cai-utils":undefined}],10:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const EmbodiedListen_1 = require("./EmbodiedListen"); exports.EmbodiedListen = EmbodiedListen_1.EmbodiedListen; const Types_1 = require("./Types"); exports.AmbientListenMode = Types_1.AmbientListenMode; exports.ActiveListenMode = Types_1.ActiveListenMode; exports.RequestStatus = Types_1.RequestStatus; },{"./EmbodiedListen":6,"./Types":8}],11:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_log_1 = require("jibo-log"); const dialogLog = new jibo_log_1.Log('Jibo.Embodied'); exports.dialogLog = dialogLog; const speechLog = dialogLog.createChild('Speech'); exports.speechLog = speechLog; const listenLog = dialogLog.createChild('Listen'); exports.listenLog = listenLog; },{"jibo-log":undefined}],12:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const uuid = require("uuid"); const path = require("path"); const jibo_cai_utils_1 = require("jibo-cai-utils"); const jibo_state_machine_1 = require("jibo-state-machine"); const Types_1 = require("./common/Types"); const InputProcessor_1 = require("./processing/InputProcessor"); const NLParser_1 = require("./processing/parsers/NLParser"); const AutoRuleManager_1 = require("./processing/auto_rules/AutoRuleManager"); const TimelineManager_1 = require("./timelines/TimelineManager"); const Utils_1 = require("../utils/Utils"); const Validation_1 = require("./utils/Validation"); const states_1 = require("./states"); const logs = require("../log"); const log = logs.speechLog; const DEFAULT_FRAME_BUFFER_MS = 0; class EmbodiedSpeech { constructor(_ed) { this.log = log; this.ownerInformation = 'EmbodiedSpeech'; this.playingKeys = []; this.animationDispatchPr = Promise.resolve(); this.active = null; this.sm = new jibo_state_machine_1.StateMachine(); this.createTimelineState = new states_1.CreateTimelineState(this.sm, this, 'Create Timeline'); this.buildDispatchableState = new states_1.BuildDispatchableState(this.sm, this, 'Build Dispatchable'); this.dispatchState = new states_1.DispatchState(this.sm, this, 'Dispatch'); this.completeState = new states_1.CompleteState(this.sm, this, 'Complete'); this.queue = new jibo_cai_utils_1.PromiseQueue(); this.unresolved = new Set(); this.isStopping = false; this.ed = _ed; this.sm.setInitial(this.createTimelineState); this.createTimelineState.addDoneTransition(this.buildDispatchableState); this.buildDispatchableState.addDoneTransition(this.dispatchState); this.dispatchState.addDoneTransition(this.completeState); } init(jibo, debug = false, options, autoRuleConfig) { return __awaiter(this, void 0, void 0, function* () { if (jibo) { this.jibo = jibo; } else { throw new Error('Cannot instantiate without Jibo instances!'); } jibo.timer.once('update', () => { log.debug('Caching common Embodied Speech AutoRule animations.'); const blinksQuery = { categories: ['blinks'], includeSomeMeta: ['single', 'double'], excludeMeta: ['sfx-only', 'ssa-only'] }; const beatQuery = { categories: ['poses'], includeMeta: ['initiation'], excludeMeta: ['sfx-only', 'ssa-only'] }; const commaQuery = { categories: ['comma'], excludeMeta: ['sfx-only', 'ssa-only'] }; const questionQuery = { categories: ['question'], excludeMeta: ['eye-only', 'sfx-only', 'ssa-only'] }; [blinksQuery, beatQuery, commaQuery, questionQuery].forEach((query) => __awaiter(this, void 0, void 0, function* () { try { yield Utils_1.Utils.loadAnimationIntoCache(jibo, query, jibo_cai_utils_1.CacheUtils.GlobalCacheName); } catch (err) { log.warn(`Unable to load animations from query ${JSON.stringify(query)} into cache ${jibo_cai_utils_1.CacheUtils.GlobalCacheName}`); } })); }); const dofsAll = this.jibo.expression.dofs.ALL; this.dofsToCenter = dofsAll.minus(this.jibo.expression.dofs.LED) .minus(this.jibo.expression.dofs.BODY).minus(this.jibo.expression.dofs.EYE_TRANSLATE); const defaultSpeakOptions = { disableAutoRules: false, dispatchComplete: Types_1.DispatchType.ALL, enableLayerMerging: true, frameBuffer: Utils_1.Utils.millisecToSec(DEFAULT_FRAME_BUFFER_MS), followedByListen: false, debugMode: debug }; this._speakOptions = Validation_1.Validation.validateSpeakOptions(defaultSpeakOptions, options); const allowAutoRule = !this._speakOptions.disableAutoRules; const defaultAutoRuleConfig = { interSentenceTiming: false, intraSentenceTiming: false, hotWords: allowAutoRule, punctuation: allowAutoRule, structure: false, voice: allowAutoRule, beat: allowAutoRule, themeRheme: false }; this._autoRuleConfig = Validation_1.Validation.validateAutoRuleConfig(defaultAutoRuleConfig, autoRuleConfig); this.inputProcessor = new InputProcessor_1.InputProcessor(this.jibo); this.nlParser = new NLParser_1.NLParser(this.jibo); this.autoRuleManager = new AutoRuleManager_1.AutoRuleManager(this.jibo); let resourceRoot; if (options && options.resourceRoot) { resourceRoot = options.resourceRoot; } this.setResourceRoots(resourceRoot); }); } speak(textToSpeak, speakOptions, autoRuleConfig) { return __awaiter(this, void 0, void 0, function* () { log.info("Speak request received"); this.jibo.performance.log('EmbodiedSpeechStartSkill', 'When ES is called'); if (this.active) { this.stop(); } return new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () { const session = { resolve, reject, id: uuid.v4(), wasStopped: false }; this.unresolved.add(session); this.queue.add(() => __awaiter(this, void 0, void 0, function* () { this.active = session; this.unresolved.delete(session); try { yield this._speak(textToSpeak, session, speakOptions, autoRuleConfig); } catch (error) { reject(error); } finally { this.active = null; log.info("Ready for next speak request"); if (session.wasStopped) { this.isStopping = false; resolve(Types_1.RequestStatus.STOPPED); } else { resolve(Types_1.RequestStatus.SUCCEEDED); } } })); })); }); } _speak(textToSpeak, session, speakOptions, autoRuleConfig) { return __awaiter(this, void 0, void 0, function* () { log.info("Processing speak request"); const validatedOptions = Validation_1.Validation.validateSpeakOptions(this._speakOptions, speakOptions); log.debug("Validated Speak Options: ", validatedOptions); const validatedConfig = Validation_1.Validation.validateAutoRuleConfig(this._autoRuleConfig, autoRuleConfig); log.debug("Validated AutoRule Config: ", validatedConfig); const data = { session: session.id, textToSpeak: textToSpeak, speakOptions: validatedOptions, autoRuleConfig: validatedConfig, centeringEnabled: false }; log.info("Starting new speak session: ", session.id); return this.sm.start(data); }); } stop() { return __awaiter(this, void 0, void 0, function* () { log.info("Stop request received"); if (this.active && !this.isStopping) { this.isStopping = true; this.active.wasStopped = true; this._stop(); } this.queue.clear(); const temp = this.unresolved; this.unresolved = new Set(); temp.forEach(session => { log.info("Stopping queued session: ", session.id); session.resolve(Types_1.RequestStatus.STOPPED); }); return this.queue.waitUntilEmpty() .then(() => null); }); } _stop() { return __awaiter(this, void 0, void 0, function* () { if (this.active) { log.info("Processing stop request on active session: ", this.active.id); return this.sm.stop(); } }); } setResourceRoots(root) { const resourceRoot = root; let animResourceRoot; let audioResourceRoot; if (resourceRoot) { log.debug(`Setting new resource roots based off of: ${root}`); animResourceRoot = path.join(resourceRoot, 'animations'); audioResourceRoot = path.join(resourceRoot, 'audio'); } else { log.debug(`Clearing resource roots`); } this.timelineManager = new TimelineManager_1.TimelineManager(this.jibo, resourceRoot, animResourceRoot, audioResourceRoot); } isActive() { return !!this.active; } waitForAnimDispatch() { return this.animationDispatchPr; } } exports.EmbodiedSpeech = EmbodiedSpeech; },{"../log":11,"../utils/Utils":65,"./common/Types":13,"./processing/InputProcessor":19,"./processing/auto_rules/AutoRuleManager":23,"./processing/parsers/NLParser":51,"./states":58,"./timelines/TimelineManager":60,"./utils/Validation":63,"jibo-cai-utils":undefined,"jibo-state-machine":undefined,"path":undefined,"uuid":undefined}],13:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var RequestStatus; (function (RequestStatus) { RequestStatus["SUCCEEDED"] = "SUCCEEDED"; RequestStatus["STOPPED"] = "STOPPED"; })(RequestStatus = exports.RequestStatus || (exports.RequestStatus = {})); var SSMLNodeType; (function (SSMLNodeType) { SSMLNodeType["BREAK"] = "BREAK"; SSMLNodeType["AUDIO_BREAK"] = "AUDIO_BREAK"; SSMLNodeType["PHONEME"] = "PHONEME"; SSMLNodeType["PITCH"] = "PITCH"; SSMLNodeType["DURATION"] = "DURATION"; SSMLNodeType["SAY_AS"] = "SAY-AS"; SSMLNodeType["STYLE"] = "STYLE"; SSMLNodeType["ENGINE"] = "ENGINE"; })(SSMLNodeType = exports.SSMLNodeType || (exports.SSMLNodeType = {})); var SSMLStyleTagType; (function (SSMLStyleTagType) { SSMLStyleTagType["NEUTRAL"] = "NEUTRAL"; SSMLStyleTagType["SHEEPISH"] = "SHEEPISH"; SSMLStyleTagType["CONFUSED"] = "CONFUSED"; SSMLStyleTagType["CONFIDENT"] = "CONFIDENT"; SSMLStyleTagType["ENTHUSIASTIC"] = "ENTHUSIASTIC"; SSMLStyleTagType["NEWS"] = "NEWS"; })(SSMLStyleTagType = exports.SSMLStyleTagType || (exports.SSMLStyleTagType = {})); var AssetNodeType; (function (AssetNodeType) { AssetNodeType["ES"] = "ES"; AssetNodeType["ANIM"] = "ANIM"; AssetNodeType["SSA"] = "SSA"; AssetNodeType["SFX"] = "SFX"; AssetNodeType["AUDIO"] = "AUDIO"; })(AssetNodeType = exports.AssetNodeType || (exports.AssetNodeType = {})); var RequestType; (function (RequestType) { RequestType["NAME"] = "NAME"; RequestType["PATH"] = "PATH"; RequestType["QUERY"] = "QUERY"; RequestType["UNKNOWN"] = "UNKNOWN"; })(RequestType = exports.RequestType || (exports.RequestType = {})); var ESTreeType; (function (ESTreeType) { ESTreeType["TAG_TREE"] = "TAG_TREE"; ESTreeType["NLP_TREE"] = "NLP_TREE"; })(ESTreeType = exports.ESTreeType || (exports.ESTreeType = {})); var DispatchType; (function (DispatchType) { DispatchType["TTS"] = "TTS"; DispatchType["ALL"] = "ALL"; })(DispatchType = exports.DispatchType || (exports.DispatchType = {})); var TTSNodeType; (function (TTSNodeType) { TTSNodeType["TTS"] = "TTS"; })(TTSNodeType = exports.TTSNodeType || (exports.TTSNodeType = {})); var NLPNodeType; (function (NLPNodeType) { NLPNodeType["WORD"] = "WORD"; NLPNodeType["SENTENCE"] = "SENTENCE"; NLPNodeType["PART"] = "PART"; })(NLPNodeType = exports.NLPNodeType || (exports.NLPNodeType = {})); var WordType; (function (WordType) { WordType["NOUN"] = "NOUN"; WordType["VERB"] = "VERB"; WordType["END"] = "END"; WordType["CONN"] = "CONN"; WordType["PREP"] = "PREP"; WordType["UNKN"] = "UNKN"; })(WordType = exports.WordType || (exports.WordType = {})); var SentenceType; (function (SentenceType) { SentenceType["QUESTION"] = "QUESTION"; SentenceType["EXCLAMATION"] = "EXCLAMATION"; SentenceType["REGULAR"] = "REGULAR"; })(SentenceType = exports.SentenceType || (exports.SentenceType = {})); },{}],14:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); __export(require("./Types")); },{"./Types":13}],15:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); const timelines = require("./timelines"); exports.timelines = timelines; const processing = require("./processing"); exports.processing = processing; const common = require("./common"); exports.common = common; const utils = require("./utils"); exports.utils = utils; const states = require("./states"); exports.states = states; __export(require("./EmbodiedSpeech")); },{"./EmbodiedSpeech":12,"./common":14,"./processing":49,"./states":58,"./timelines":61,"./utils":64}],16:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); const Types_1 = require("../common/Types"); const log_1 = require("../../log"); const log = log_1.speechLog.createChild("AssetNode"); exports.MetaLayerMap = { body: 'Body', screen: 'Screen', audio: 'Audio' }; const ESAliasToFilter = { ANIM: '!ssa-only, !sfx-only', SSA: 'ssa-only', SFX: 'sfx-only' }; const ALL_LAYERS = 'all'; class AssetNode extends jibo_node_xml_1.default { constructor(type, att, timeSyncNode, playbackData) { super(type); this.timeSyncNode = timeSyncNode; this.playbackData = playbackData; let defaultFilterIntent; if (type !== Types_1.AssetNodeType.AUDIO) { this.type = Types_1.AssetNodeType.ES; defaultFilterIntent = ESAliasToFilter[type]; this.metaFilter = { allOf: [], someOf: [], noneOf: [], addAll: [] }; this.categoryFilter = { allOf: [], someOf: [], noneOf: [], addAll: [] }; AssetNode.addFilterAttributes(defaultFilterIntent, this.metaFilter); } if (att) { this.att = att; this.requestInfo = AssetNode.getRequestInfo(att); const requestedCategoryIntent = this.att.get('cat') || this.att.get('cats'); if (requestedCategoryIntent) { AssetNode.addCategories(requestedCategoryIntent, this.categoryFilter); } const requestedFilterIntent = this.att.get('filter') || this.att.get('filters') || this.att.get('meta') || this.att.get('metas'); if (requestedFilterIntent) { AssetNode.addMeta(requestedFilterIntent, this.metaFilter); } this.layers = {}; this.layersToAdd = {}; const reqLayers = this.att.get('layers') || this.att.get('layer'); if (reqLayers) { AssetNode.addLayers(reqLayers, this.layers, this.layersToAdd); } const reqLoop = this.att.get('loop') || this.att.get('loops'); if (reqLoop) { this.loops = parseInt(reqLoop); } } } static getRequestInfo(att) { let info = { type: Types_1.RequestType.UNKNOWN, description: Types_1.RequestType.UNKNOWN }; if (att.has('name')) { info.type = Types_1.RequestType.NAME; info.description = `${info.type}[ ${att.get('name')} ]`; } else if (att.has('cat') || att.has('cats') || att.has('filter') || att.has('filters') || att.has('meta') || att.has('metas')) { info.type = Types_1.RequestType.QUERY; const cat = `CAT[ ${att.get('cat') || att.get('cats')} ]`; const meta = `META[ ${att.get('filter') || att.get('filters') || att.get('meta') || att.get('metas')} ]`; info.description = `${info.type}[ ${cat} ${meta} ]`; } else if (att.has('path')) { info.type = Types_1.RequestType.PATH; info.description = `${info.type}[ ${att.get('path')} ]`; } return info; } static addCategories(reqestedCategoriesIntent, categoryFilter) { AssetNode.addFilterAttributes(reqestedCategoriesIntent, categoryFilter); } static addFilterAttributes(filterAttr, metaFilter) { if (filterAttr) { const terms = filterAttr.split(/[;,](?![^(]*\))/g); terms.forEach(term => { const [op, attrs] = AssetNode.parseTerm(term); if (attrs.length > 0) { if (!op || op === '&') { metaFilter.allOf.push(...attrs); } else if (op === '?') { metaFilter.someOf.push(...attrs); } else if (op === '!') { metaFilter.noneOf.push(...attrs); } else if (op === '+') { metaFilter.addAll.push(...attrs); } } }); } } static parseTerm(term) { const trimmed = term.trim().toLowerCase(); let op; let attrs = []; const groupedMatches = /^([!\?\+&]?)\(([\w\s-,;]+)\)/.exec(trimmed); if (groupedMatches) { op = groupedMatches[1]; const subterms = groupedMatches[2].split(','); attrs.push(...subterms.map(subterm => subterm.trim())); } else { const indMatches = /^([!\?\+&]?)([\w-]+)/.exec(trimmed); if (indMatches) { op = indMatches[1]; attrs.push(indMatches[2]); } else { log.warn('ESML Query contains un-parseable elements. No or unexpected results likely.'); } } return [op, attrs]; } static addMeta(reqFiltered, metaFilter) { AssetNode.addFilterAttributes(reqFiltered, metaFilter); if (metaFilter.allOf.length > 0) { metaFilter.allOf = Array.from(new Set(metaFilter.allOf)); } if (metaFilter.someOf.length > 0) { metaFilter.someOf = Array.from(new Set(metaFilter.someOf)); } if (metaFilter.noneOf.length > 0) { metaFilter.noneOf = Array.from(new Set(metaFilter.noneOf)); } } static addLayers(reqestedLayersIntent, layers, layersToAdd) { let layerFilter = { allOf: [], someOf: [], noneOf: [], addAll: [] }; AssetNode.addFilterAttributes(reqestedLayersIntent, layerFilter); if (layerFilter.addAll.length > 0) { if (layerFilter.allOf.length > 0 || layerFilter.noneOf.length > 0) { log.warn('Cannot both add layers and filter inclusively or exclusively at the same time.' + ' Modify layers filter to be one approach or the other.'); return; } if (layerFilter.addAll.length === 1 && layerFilter.addAll[0] === ALL_LAYERS) { layers.Body = true; layers.Screen = true; layers.Audio = true; layersToAdd.Body = true; layersToAdd.Screen = true; layersToAdd.Audio = true; } else { layerFilter.addAll.forEach(layer => { const metaLayerName = exports.MetaLayerMap[layer]; if (metaLayerName === undefined) { log.warn(`Layer ${layer} is not a valid layers filter.`); } else { layers[metaLayerName] = true; layersToAdd[metaLayerName] = true; } }); } } else { if (layerFilter.allOf.length > 0 && layerFilter.noneOf.length > 0) { log.warn('Cannot filter layers in both an inclusive and exclusive manner at the same time.' + ' Modify layers filter to be one or the other.'); return; } if (layerFilter.allOf.length > 0) { layers.Body = false; layers.Screen = false; layers.Audio = false; layerFilter.allOf.forEach(layer => { const metaLayerName = exports.MetaLayerMap[layer]; if (metaLayerName === undefined) { log.warn(`Layer ${layer} is not a valid layers filter.`); } else { layers[metaLayerName] = true; } }); } else { layerFilter.noneOf.forEach(layer => { const metaLayerName = exports.MetaLayerMap[layer]; if (metaLayerName === undefined) { log.warn(`Layer ${layer} is not a valid layers filter.`); } else { layers[metaLayerName] = false; } }); } } } } exports.AssetNode = AssetNode; },{"../../log":11,"../common/Types":13,"jibo-node-xml":undefined}],17:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const ContentLists = require('../../../resources/swears.json'); class ContentValidator { constructor() { const reWholeMatch = `\\b(${ContentLists.whole.join('|')})+\\b`; const rePartialMatch = `(\\w*(${ContentLists.partial.join('|')})\\w*)`; const reBlackList = `${reWholeMatch}|${rePartialMatch}`; const reWhitelist = `(\\b(${ContentLists.whitelist.join('|')})\\b)`; this.blacklist = new RegExp(reBlackList, 'ig'); this.whitelist = new RegExp(reWhitelist, 'ig'); } replaceRestricted(sentence, replacement) { this.blacklist.lastIndex = 0; this.whitelist.lastIndex = 0; let saved = {}; let count = 0; let ids = []; const hider = (match) => { const id = ``; ids.push(id); saved[id] = match; return id; }; const seeker = (match) => { return saved[match]; }; const hidden = sentence.replace(this.whitelist, hider); const replaced = hidden.replace(this.blacklist, replacement); if (ids.length) { const reSavelist = `(${ids.join('|')})`; const savelist = new RegExp(reSavelist, 'g'); return replaced.replace(savelist, seeker); } return replaced; } } exports.ContentValidator = ContentValidator; },{"../../../resources/swears.json":1}],18:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); class AudioAnimTemplate { constructor(duration, path) { this.duration = duration; this.version = '0.7.0'; this.framerate = 30; this.scale = 1; this.animdb = { name: 'audio_path_tag', metaTerms: ["audio-only"], scaleMax: 1, scaleMin: 1, speedMax: 1, speedMin: 1 }; this.pathless = true; let audioEvent = {}; audioEvent.AudioEvent = { file: path }; let keyframe = {}; keyframe.time = 0; keyframe.value = audioEvent; let layer = {}; layer.id = '96ca2c12e2f4459589c8dd99387ab59e'; layer.name = 'Audio Event'; layer.type = 'Audio Event'; layer.visible = true; layer.locked = false; layer.keyframes = [keyframe]; this.layers = [layer]; } } exports.AudioAnimTemplate = AudioAnimTemplate; },{}],19:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); const jibo_node_xml_2 = require("jibo-node-xml"); const AssetNode_1 = require("./AssetNode"); const ContentValidator_1 = require("./ContentValidator"); const Types_1 = require("../common/Types"); const SSMLNode_1 = require("./SSMLNode"); const NLPTools_1 = require("../utils/NLPTools"); const Lexer_1 = require("./parsers/Lexer"); const TimelineModUtils_1 = require("./auto_rules/timeline_mod/TimelineModUtils"); const log_1 = require("../../log"); const log = log_1.speechLog.createChild("InputProcessor"); exports.TagAttributes = { NON_BLOCKING: 'nonBlocking', END_NEUTRAL: 'endNeutral' }; class InputProcessor { constructor(jibo) { this.swearToTags = { ALL: ` e ` }; this._lexer = new Lexer_1.Lexer(jibo); this.contentValidator = new ContentValidator_1.ContentValidator(); } generatePreInputTree(sentence, censorSwears = true) { return __awaiter(this, void 0, void 0, function* () { if (censorSwears) { const replaced = this.contentValidator.replaceRestricted(sentence, this.swearToTags.ALL); if (sentence !== replaced) { log.warn('Requested TTS contains restricted content! It has been censored.'); } sentence = replaced; } let xmlTree = jibo_node_xml_2.Parser.parseXML(sentence); xmlTree.forEachDescendant(node => { node.type = node.type.toUpperCase(); }); xmlTree.forEachDescendantType(Types_1.SSMLNodeType.BREAK, node => { if (node.parents.length === 1) { const parent = node.parents[0]; const selfIndex = parent.children.indexOf(node); const size = node.getNumberAtt('size'); let properSSMLNode = new SSMLNode_1.SSMLNode(Types_1.SSMLNodeType.BREAK); properSSMLNode.setAttribute('size', size); parent.children.splice(selfIndex, 1, properSSMLNode); properSSMLNode.parents.push(parent); node.clearParents(); } else { throw new Error('Unexpected number of parents to SSML Break Node.'); } }); let textNodes = xmlTree.gatherDescendantsType(jibo_node_xml_2.NodeType.TEXT); xmlTree.expansions = []; xmlTree.contractions = []; for (let textNode of textNodes) { let sanitizedText = NLPTools_1.NLPTools.cleanInput(textNode.value); let preExpanded = yield this._lexer.lex(sanitizedText); let newNodes = []; preExpanded.forEach(token => { let [localExp, localContr] = NLPTools_1.NLPTools.getExpansionsAndContractions(token); xmlTree.expansions.push(...localExp); xmlTree.contractions.push(...localContr); localExp.forEach(exp => { let node = new jibo_node_xml_1.default(jibo_node_xml_2.NodeType.TEXT, exp); newNodes.push(node); }); }); textNode.parents.forEach(parent => { let positionIndex = parent.children.indexOf(textNode); if (positionIndex === -1) { throw new Error('Parent node missing reference to child.'); } else { parent.children.splice(positionIndex, 1); for (let newNode of newNodes) { newNode.parents = textNode.parents.slice(); parent.children.splice(positionIndex, 0, newNode); positionIndex++; } } }); } xmlTree.value = sentence; xmlTree.treeType = Types_1.ESTreeType.TAG_TREE; xmlTree.textValue = InputProcessor.getPlainTextFromTree(xmlTree); return xmlTree; }); } insertTimelineAutoRules(xmlTree, timelineAutoRuleData) { const timefulMatches = timelineAutoRuleData.filter(ruleData => { return ruleData.type === TimelineModUtils_1.TimelineAutoRuleType.TIMEFUL; }); const timelessMatches = timelineAutoRuleData.filter(ruleData => { return ruleData.type === TimelineModUtils_1.TimelineAutoRuleType.TIMELESS; }); let parentMerged = new Map(); timefulMatches.forEach(ruleData => { ruleData.matches.forEach((matchInfo, parent) => { let childrenToInsert = parentMerged.get(parent); if (!childrenToInsert) { childrenToInsert = []; parentMerged.set(parent, childrenToInsert); } childrenToInsert.push(...matchInfo); }); }); parentMerged.forEach((matchInfo, parent) => { matchInfo.sort((a, b) => { return a.offset - b.offset; }); let insertedNodeCount = 0; matchInfo.forEach(content => { content.nodes.forEach(node => { parent.children.splice((content.offset + insertedNodeCount), 0, node); insertedNodeCount++; }); }); }); timelessMatches.forEach(ruleData => { ruleData.matches.forEach((matchInfo, parent) => { matchInfo.forEach(content => { let workingChild; let shouldBeInsertedCount = 0; while (content.nodes.length > 0) { const innermost = content.nodes.pop(); if (!workingChild && shouldBeInsertedCount < 1) { const parentCheck = content.children.every(child => child.parents[0] === parent); const firstChildIdx = parent.children.indexOf(content.children[0]); if (parentCheck && (firstChildIdx > -1)) { parent.children.splice(firstChildIdx, 0, innermost); content.children.forEach(child => { child.clearParents(); innermost.addChild(child); }); innermost.parents.push(parent); parent.children.splice(firstChildIdx + 1, content.children.length); workingChild = innermost; } } else if (workingChild && shouldBeInsertedCount > 0) { const workingChildIdx = parent.children.indexOf(workingChild); parent.children.splice(workingChildIdx, 0, innermost); workingChild.clearParents(); innermost.addChild(workingChild); parent.children.splice(workingChildIdx + 1, 1); innermost.parents.push(parent); workingChild = innermost; } else { break; } shouldBeInsertedCount++; } }); }); }); return xmlTree; } static generateInputTree(inputTree, nlpTree) { InputProcessor.replaceTreeTextNodes(nlpTree, inputTree, true); let assetNodes = inputTree.gatherDescendants(node => { return InputProcessor.isAsset(node); }); assetNodes.forEach((assetNode) => { if (assetNode.metaFilter === undefined) { let newAssetNode = new AssetNode_1.AssetNode(assetNode.type, assetNode.att); InputProcessor._createSubTree(newAssetNode, assetNode.children); for (let parent of assetNode.parents) { let positionIndex = parent.children.indexOf(assetNode); if (positionIndex === -1) { throw new Error('Parent doesn\'t not recognize child.'); } parent.children.splice(positionIndex, 1, newAssetNode); } } }); const words = inputTree.gatherDescendantsType(Types_1.NLPNodeType.WORD); for (let idx = (words.length - 1); idx >= 0; idx--) { const word = words[idx]; let isContraction = word.getAttribute('contraction'); if (isContraction) { word.value = isContraction; } else if (word.contractionBase) { for (let parent of word.parents) { let positionIndex = parent.children.indexOf(word); if (positionIndex !== -1) { parent.children.splice(positionIndex, 1); } } word.clearParents(); } } return inputTree; } static _createSubTree(root, nodesToAdd) { nodesToAdd.forEach((node) => { node.clearParents(); root.addChild(node); }); return root; } static isAsset(node) { return Types_1.AssetNodeType.hasOwnProperty(node.type); } static isEmpty(node) { return (node.children.length === 0); } static isBlocking(node) { return (InputProcessor.isAsset(node)) && ((InputProcessor.isEmpty(node)) && ((node.getAttribute(exports.TagAttributes.NON_BLOCKING) !== 'true') || (node.getAttribute(exports.TagAttributes.NON_BLOCKING) === undefined))); } static isUnboundedNonBlocking(node) { return (InputProcessor.isAsset(node)) && (InputProcessor.isEmpty(node)) && (InputProcessor.isExplicitlyNonBlocking(node)); } static isExplicitlyNonBlocking(node) { return (node.getAttribute(exports.TagAttributes.NON_BLOCKING) === 'true'); } static isSSML(node) { return InputProcessor.isTimefulSSML(node) || InputProcessor.isTimelessSSML(node); } static isTimefulSSML(node) { return InputProcessor.isBreak(node) || InputProcessor.isSayAs(node); } static isTimelessSSML(node) { return InputProcessor.isPhoneme(node) || InputProcessor.isPitch(node) || InputProcessor.isDuration(node) || InputProcessor.isStyle(node) || InputProcessor.isEngine(node); } static isBreak(node) { return node.type === Types_1.SSMLNodeType.BREAK; } static isPhoneme(node) { return node.type === Types_1.SSMLNodeType.PHONEME; } static isPitch(node) { return node.type === Types_1.SSMLNodeType.PITCH; } static isDuration(node) { return node.type === Types_1.SSMLNodeType.DURATION; } static isSayAs(node) { return node.type === Types_1.SSMLNodeType.SAY_AS; } static isStyle(node) { return node.type === Types_1.SSMLNodeType.STYLE; } static isEngine(node) { return node.type === Types_1.SSMLNodeType.ENGINE; } static getPlainTextArrayFromTree(tree) { const textNodes = tree.gatherDescendantsType(jibo_node_xml_2.NodeType.TEXT); return textNodes.map(node => node.value); } static getPlainTextFromTree(tree) { const wordArray = InputProcessor.getPlainTextArrayFromTree(tree); return NLPTools_1.NLPTools.joinWordsIntoSentence(wordArray); } static getSSMLTextFromTree(tree) { let textNodes = tree.gatherDescendants(node => { return (node.type === jibo_node_xml_2.NodeType.TEXT) || (node.type === Types_1.NLPNodeType.WORD) || (InputProcessor.isSSML(node)); }); return InputProcessor.getSSMLTextFromArray(textNodes); } static getSSMLTextFromArray(textNodes, prune = false) { let stringifiedSSML = []; for (let idx = 0; idx < textNodes.length; idx++) { const terminal = InputProcessor._getSSMLTextRecursive(textNodes[idx], stringifiedSSML); const nextIdx = textNodes.indexOf(terminal); if (nextIdx < 0) { break; } else if (nextIdx > idx) { idx = nextIdx; } } return NLPTools_1.NLPTools.joinWordsIntoSentence(stringifiedSSML); } static _getSSMLTextRecursive(node, stringified) { let terminal; if (node.children.length) { let stringifiedRecur = []; if (!InputProcessor.isAsset(node)) { stringifiedRecur.push(node.toStringTagOnly()); } for (let idx = 0; idx < node.children.length; idx++) { terminal = InputProcessor._getSSMLTextRecursive(node.children[idx], stringifiedRecur); } if (!InputProcessor.isAsset(node)) { stringifiedRecur.push(node.toStringClosingTag()); } stringified.push(...stringifiedRecur); } else { if (InputProcessor.isAsset(node)) { if (InputProcessor.isBlocking(node) && node.timeSyncNode) { node = node.timeSyncNode; } else { return node; } } if (node.type === jibo_node_xml_2.NodeType.TEXT || node.type === Types_1.NLPNodeType.WORD) { stringified.push(node.value); } else { stringified.push(node.toStringTagOnly()); } terminal = node; } return terminal; } static replaceTreeTextNodes(sourceTree, destinationTree, optMatchWords) { let srcTxtNodes = sourceTree.gatherDescendants(n => { return n.type === Types_1.NLPNodeType.WORD || n.type === jibo_node_xml_2.NodeType.TEXT; }); let destTxtNodes = destinationTree.gatherDescendants(n => { return n.type === Types_1.NLPNodeType.WORD || n.type === jibo_node_xml_2.NodeType.TEXT; }); if (optMatchWords) { let srcValues = srcTxtNodes.map(n => n.value).join(' '); let destValues = destTxtNodes.map(n => n.value).join(' '); if (srcValues !== destValues) { throw new TypeError(`Both trees must have the same words. ` + `SrcValues: ${srcValues}, DstValues: ${destValues}`); } } if (srcTxtNodes.length !== destTxtNodes.length) { throw new TypeError(`Must have same # of nodes in both trees. ` + `SrcValues: ${srcTxtNodes}, DstValues: ${destTxtNodes}`); } destTxtNodes.forEach((destNode, index) => { for (let parent of destNode.parents) { let positionIndex = parent.children.indexOf(destNode); if (positionIndex === -1) { throw new Error('Parent doesn\'t not recognize child.'); } parent.children.splice(positionIndex, 1, srcTxtNodes[index]); let parentIndex = destNode.parents.indexOf(parent); if (parentIndex === -1) { throw new Error('Node does not recognize parent'); } destNode.parents.splice(parentIndex, 1); srcTxtNodes[index].parents.push(parent); } }); return destinationTree; } static generateWordList(annotationTree, ssml) { if (ssml) { annotationTree.forEachDescendant((node) => { if (InputProcessor.isSSML(node)) { node.value = node.toString(); node.type = Types_1.NLPNodeType.WORD; } }); } return annotationTree.gatherDescendants((node) => { return node.type === Types_1.NLPNodeType.WORD || node.type === jibo_node_xml_2.NodeType.TEXT; }); } } exports.InputProcessor = InputProcessor; },{"../../log":11,"../common/Types":13,"../utils/NLPTools":62,"./AssetNode":16,"./ContentValidator":17,"./SSMLNode":20,"./auto_rules/timeline_mod/TimelineModUtils":47,"./parsers/Lexer":50,"jibo-node-xml":undefined}],20:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); class SSMLNode extends jibo_node_xml_1.default { constructor(type) { super(type); } } exports.SSMLNode = SSMLNode; },{"jibo-node-xml":undefined}],21:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); class TagTreeNode extends jibo_node_xml_1.default { constructor(type, value) { super(type, value); } } exports.TagTreeNode = TagTreeNode; },{"jibo-node-xml":undefined}],22:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); const jibo_node_xml_2 = require("jibo-node-xml"); class AutoRule { constructor(ruleName, priority, probability = 1) { this.ruleName = ruleName; this.priority = priority; this.probability = probability; } _applyInternal(nlpTreeRoot, otherRoots) { throw new Error('Should be implemented by subclass'); } _createNode(children, subRule) { throw new Error('Should be implemented by subclass'); } applyRule(nlpTreeRoot, otherRoots) { let wordSequences = this._applyInternal(nlpTreeRoot, otherRoots); let root = new jibo_node_xml_1.default(jibo_node_xml_2.NodeType.ROOT); root.setAttribute('autoRule', this.ruleName); root.setAttribute('priority', this.priority); wordSequences.forEach(sequence => { if (Math.random() < this.probability) { const newNode = this._createNode(sequence); if (newNode) { sequence.forEach(wordNode => newNode.addChild(wordNode)); root.addChild(newNode); } } }); return root; } } exports.AutoRule = AutoRule; },{"jibo-node-xml":undefined}],23:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const path = require("path"); const fs = require("fs"); const QuestionRule_1 = require("./QuestionRule"); const CommaRule_1 = require("./CommaRule"); const InitiateRule_1 = require("./InitiateRule"); const BlinkRule_1 = require("./BlinkRule"); const DoubleBlinkRule_1 = require("./DoubleBlinkRule"); const BeatBlinkRule_1 = require("./BeatBlinkRule"); const HotWordsRule_1 = require("./HotWordsRule"); const OrRule_1 = require("./OrRule"); const ButRule_1 = require("./ButRule"); const NounRule_1 = require("./NounRule"); const ListRule_1 = require("./ListRule"); const InterSentenceTimingRule_1 = require("./timeline_mod/InterSentenceTimingRule"); const IntraSentenceTimingRule_1 = require("./timeline_mod/IntraSentenceTimingRule"); const EllipsisRule_1 = require("./timeline_mod/EllipsisRule"); const EmotionalVoiceRule_1 = require("./timeline_mod/EmotionalVoiceRule"); const log_1 = require("../../../log"); const log = log_1.speechLog.createChild("AutoRuleManager"); exports.AutoRuleNames = { Exclamation: 'Exclamation', Question: 'Question', Blink: 'Blink', DoubleBlink: 'DoubleBlink', BeatBlink: 'BeatBlink', Comma: 'Comma', Or: 'Or', But: 'But', List: 'List', Noun: 'Noun', BeatFirstWordPart: 'BeatFirstWordPart', BeatNewWord: 'BeatNewWord', Initiate: "Initiate", Settle: "Settle", HotWords: 'HotWords', ThemeRhemeGaze: 'ThemeRhemeGaze' }; exports.TimelineModifyingRuleNames = { InterSentenceTiming: 'InterSentenceTiming', IntraSentenceTiming: 'IntraSentenceTiming', Ellipsis: 'Ellipsis', ExclamationVoice: 'ExclamationVoice', EmotionalVoice: 'EmotionalVoice' }; class AutoRuleManager { constructor(_jibo) { this._jibo = _jibo; this.hotWordMap = new Map(); this.rejectWordMap = new Map(); this.descendingPhraseLengths = []; this.rules = [ new QuestionRule_1.QuestionRule(2), new CommaRule_1.CommaRule(this._jibo, 5), new BlinkRule_1.BlinkRule(3), new DoubleBlinkRule_1.DoubleBlinkRule(3), new BeatBlinkRule_1.BeatBlinkRule(4), new ListRule_1.ListRule(this._jibo, 10), new OrRule_1.OrRule(this._jibo, 10), new ButRule_1.ButRule(this._jibo, 10), new NounRule_1.NounRule(10), new HotWordsRule_1.HotWordsRule(0, 1, this.hotWordMap, this.rejectWordMap, this.descendingPhraseLengths), new InitiateRule_1.InitiateRule(this._jibo, 1), ]; this.timelineModRules = [ new InterSentenceTimingRule_1.InterSentenceTimingRule(1), new IntraSentenceTimingRule_1.IntraSentenceTimingRule(1), new EllipsisRule_1.EllipsisRule(1), new EmotionalVoiceRule_1.EmotionalVoiceRule(this._jibo, 1) ]; const hotWordsDir = this.retreiveHotWordsDir(); if (hotWordsDir) { const hotWordCategoryFiles = fs.readdirSync(hotWordsDir); let hotWordCategories = []; log.debug(`Generating HotWords Match and Reject Maps`); hotWordCategoryFiles.forEach(hotWordCategoryFile => { const hotWordCategoryFilePath = path.join(hotWordsDir, hotWordCategoryFile); if (!fs.statSync(hotWordCategoryFilePath).isDirectory()) { try { hotWordCategories.push(JSON.parse(fs.readFileSync(hotWordCategoryFilePath, 'utf8'))); } catch (err) { log.warn(`Unable to load/parse HotWord JSON '${hotWordCategoryFile}'; skipping...`, err); } } }); hotWordCategories.forEach(hotWordMappings => { hotWordMappings.forEach(([hotWordMatchReject, weightedMatches]) => { hotWordMatchReject.forEach(matchRejectPair => { const match = matchRejectPair.match; const reject = matchRejectPair.reject; const length = match.trim().split(' ').length; let subMap = this.hotWordMap.get(length); if (!subMap) { subMap = new Map(); this.hotWordMap.set(length, subMap); this.descendingPhraseLengths.push(length); } const keyName = match.toLowerCase(); subMap.set(keyName, weightedMatches); if (reject.length > 0) { this.rejectWordMap.set(keyName, reject.map(word => word.toLowerCase())); } }); }); }); this.descendingPhraseLengths.sort((a, b) => { return b - a; }); } } retreiveHotWordsDir() { const resolvedPath = this._jibo.animDB.resolveAnimDB(this._jibo); let hotWordsDir; if (resolvedPath) { const rootPath = path.dirname(resolvedPath); hotWordsDir = path.join(rootPath, 'hot'); if (!fs.existsSync(hotWordsDir)) { log.warn(`Module 'jibo-anim-db-animations' not found. HotWords will not work.`); } } return hotWordsDir; } applyRules(nlpTreeRoot, autoRuleConfig) { let out = []; this.rules.forEach(rule => { switch (rule.ruleName) { case exports.AutoRuleNames.Question: case exports.AutoRuleNames.Comma: if (autoRuleConfig.punctuation) { out.push(rule.applyRule(nlpTreeRoot)); } break; case exports.AutoRuleNames.Or: case exports.AutoRuleNames.But: case exports.AutoRuleNames.List: case exports.AutoRuleNames.Noun: if (autoRuleConfig.structure) { out.push(rule.applyRule(nlpTreeRoot)); } break; case exports.AutoRuleNames.Initiate: case exports.AutoRuleNames.Blink: case exports.AutoRuleNames.DoubleBlink: case exports.AutoRuleNames.BeatBlink: if (autoRuleConfig.beat) { out.push(rule.applyRule(nlpTreeRoot)); } break; case exports.AutoRuleNames.HotWords: if (autoRuleConfig.hotWords) { out.push(rule.applyRule(nlpTreeRoot)); } break; default: log.warn(`Unable to apply AutoRule: ${rule.ruleName}.`); } }); return out; } applyTimelineModifyingRules(xmlTree, autoRuleConfig) { let out = []; this.timelineModRules.forEach(rule => { switch (rule.ruleName) { case exports.TimelineModifyingRuleNames.InterSentenceTiming: if (autoRuleConfig.interSentenceTiming) { out.push(rule.applyRule(xmlTree, autoRuleConfig.punctuation)); } break; case exports.TimelineModifyingRuleNames.IntraSentenceTiming: if (autoRuleConfig.intraSentenceTiming) { out.push(rule.applyRule(xmlTree)); } break; case exports.TimelineModifyingRuleNames.Ellipsis: if (autoRuleConfig.interSentenceTiming && autoRuleConfig.punctuation) { out.push(rule.applyRule(xmlTree)); } break; case exports.TimelineModifyingRuleNames.EmotionalVoice: case exports.TimelineModifyingRuleNames.ExclamationVoice: if (autoRuleConfig.voice) { out.push(rule.applyRule(xmlTree)); } break; default: log.warn(`Unable to apply AutoRule: ${rule.ruleName}.`); } }); return out; } } exports.AutoRuleManager = AutoRuleManager; },{"../../../log":11,"./BeatBlinkRule":25,"./BlinkRule":28,"./ButRule":29,"./CommaRule":30,"./DoubleBlinkRule":31,"./HotWordsRule":33,"./InitiateRule":34,"./ListRule":35,"./NounRule":36,"./OrRule":37,"./QuestionRule":38,"./timeline_mod/EllipsisRule":41,"./timeline_mod/EmotionalVoiceRule":42,"./timeline_mod/InterSentenceTimingRule":44,"./timeline_mod/IntraSentenceTimingRule":45,"fs":undefined,"path":undefined}],24:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const Types_1 = require("../../common/Types"); const NLPTools_1 = require("../../utils/NLPTools"); exports.AR_SUBRULE = { PERIOD: '.', QUESTION: '?', EXCLAMATION: '!', COMMA: ',', ELLIPSIS: '...', OR: 'or', BUT: 'but', BLINK: 'single', DOUBLE_BLINK: 'double' }; class AutoRuleUtils { static getLastWordNodesInSentence(sentenceNode) { let parts = sentenceNode.gatherChildrenType(Types_1.NLPNodeType.PART); let lastPart = parts[parts.length - 1]; let out = []; if (!lastPart.getAttribute('verbCount')) { out = lastPart.gatherDescendantsType(Types_1.NLPNodeType.WORD); } else { let words = lastPart.gatherDescendantsType(Types_1.NLPNodeType.WORD); words.forEach((word) => { if (word.wordType === Types_1.WordType.VERB) { out.push(word); } else if (out.length > 0) { out.push(word); } }); } return AutoRuleUtils.removeExpandedContractionWords(out); } static getWordsFromSentenceSegmentedBy(segmenters, sentenceNode, keepPunct = false) { let segments = []; let words = sentenceNode.gatherDescendantsType(Types_1.NLPNodeType.WORD); words = AutoRuleUtils.removeExpandedContractionWords(words); let segmenterCount = 0; segments[segmenterCount] = []; words.forEach((word) => { if (segmenters.indexOf(word.value) !== -1) { if (segments[segmenterCount].length > 0) { segmenterCount++; segments[segmenterCount] = []; } } if (keepPunct || !NLPTools_1.NLPTools.isPunctuation(word.value)) { segments[segmenterCount].push(word); } }); return (segmenterCount > 0) ? segments : []; } static removeExpandedContractionWords(words) { for (let idx = 0; idx < words.length; idx++) { if (words[idx].contractionBase) { words.splice(idx, 1); } } return words; } } exports.AutoRuleUtils = AutoRuleUtils; },{"../../common/Types":13,"../../utils/NLPTools":62}],25:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AutoRuleManager_1 = require("./AutoRuleManager"); const TimelineModUtils_1 = require("./timeline_mod/TimelineModUtils"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const BEAT_BLINK_PERIOD = 7; class BeatBlinkRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Blink, priority, probability); } _createNode(children) { let n = TimelineModUtils_1.TimelineModUtils.blinkNodeBuilder(TimelineModUtils_1.BlinkNodeType.SINGLE); n.setAttribute('nonBlocking', 'true'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Blink); n.timeSyncNode = children[0]; return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, sentence => { const rawWords = sentence.gatherDescendantsType(Types_1.NLPNodeType.WORD); const words = AutoRuleUtils_1.AutoRuleUtils.removeExpandedContractionWords(rawWords); if (words.length > 0) { for (let idx = 0; idx < Math.max(1, words.length - BEAT_BLINK_PERIOD); idx += BEAT_BLINK_PERIOD) { sequences.push([words[idx]]); } } }); return sequences; } } exports.BeatBlinkRule = BeatBlinkRule; },{"../../common/Types":13,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"./timeline_mod/TimelineModUtils":47}],26:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleManager_1 = require("./AutoRuleManager"); class BeatFirstWordPartRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.BeatFirstWordPart, priority, probability); } _createNode() { let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM); n.setAttribute('cat', 'beat'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.BeatFirstWordPart); return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, sentence => { sentence.forEachChildType(Types_1.NLPNodeType.PART, part => { let firstWord = part.findFirstDescendantType(Types_1.NLPNodeType.WORD); if (firstWord) { if (firstWord.contractionBase) { if (!sequences || sequences.indexOf(firstWord.contractionBase) !== -1) { sequences.push([firstWord.contractionBase]); } } else { sequences.push([firstWord]); } } }); }); return sequences; } } exports.BeatFirstWordPartRule = BeatFirstWordPartRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23}],27:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleManager_1 = require("./AutoRuleManager"); class BeatNewWordRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.BeatNewWord, priority, probability); } _createNode() { let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM); n.setAttribute('cat', 'beat'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.BeatNewWord); return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachDescendantType(Types_1.NLPNodeType.WORD, _word => { const word = _word; if (word.getAttribute('new')) { if (word.contractionBase) { if (!sequences || sequences.indexOf(word.contractionBase) !== -1) { sequences.push([word.contractionBase]); } } else { sequences.push([word]); } } }); return sequences; } } exports.BeatNewWordRule = BeatNewWordRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23}],28:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AutoRuleManager_1 = require("./AutoRuleManager"); const TimelineModUtils_1 = require("./timeline_mod/TimelineModUtils"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); class BlinkRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Blink, priority, probability); } _createNode(children) { let n = TimelineModUtils_1.TimelineModUtils.blinkNodeBuilder(TimelineModUtils_1.BlinkNodeType.SINGLE); n.setAttribute('nonBlocking', 'true'); n.setAttribute('position', 'after'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Blink); n.timeSyncNode = children[0]; return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, sentence => { const sentenceType = sentence.getAttribute('sentenceType'); if (sentenceType === Types_1.SentenceType.REGULAR || sentenceType === Types_1.SentenceType.EXCLAMATION) { let words = sentence.gatherDescendantsType(Types_1.NLPNodeType.WORD); words = AutoRuleUtils_1.AutoRuleUtils.removeExpandedContractionWords(words); if (words) { sequences.push(words.slice(-1)); } } }); return sequences; } } exports.BlinkRule = BlinkRule; },{"../../common/Types":13,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"./timeline_mod/TimelineModUtils":47}],29:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const _ = require("lodash/lodash.min"); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const AutoRuleManager_1 = require("./AutoRuleManager"); const AR_BUT_SEGMENT_START_KEY = 'ar-but-segment-start'; const AR_BUT_SEGMENT_STOP_KEY = 'ar-but-segment-stop'; exports.BUT_KEYS = [ 'But_01', 'But_02' ]; const NUM_WORDS_TO_WRAP = 5; class ButRule extends AutoRule_1.AutoRule { constructor(jibo, priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.But, priority, probability); this.jibo = jibo; this.Orientation = this.jibo.animDB.Orientation; this.lastOrientation = (Math.random() > 0.5) ? this.Orientation.LEFT : this.Orientation.RIGHT; } _createNode(children) { const newSegment = children[0].getAttribute(AR_BUT_SEGMENT_START_KEY); const endSegment = children[0].getAttribute(AR_BUT_SEGMENT_STOP_KEY); if (newSegment) { this.currentSegmentAnim = _.sample(exports.BUT_KEYS); } this.lastOrientation = this.lastOrientation === this.Orientation.LEFT ? this.Orientation.RIGHT : this.Orientation.LEFT; let atts = new Map(); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('name', this.currentSegmentAnim); n.setAttribute('orientation', this.lastOrientation); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.But); n.setAttribute('endNeutral', 'false'); if (endSegment) { n.setAttribute('endNeutral', 'true'); } return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, (sentence) => { let wordSegments = AutoRuleUtils_1.AutoRuleUtils.getWordsFromSentenceSegmentedBy(['but'], sentence); if (wordSegments.length > 0) { let segmentIdx = 0; let endSegment; wordSegments.forEach(wordSegment => { let choppedWordSegment; if (wordSegment.length > 0) { if (segmentIdx === 0) { choppedWordSegment = wordSegment.slice(-NUM_WORDS_TO_WRAP); choppedWordSegment[0].setAttribute(AR_BUT_SEGMENT_START_KEY, true); } else { choppedWordSegment = wordSegment.slice(0, NUM_WORDS_TO_WRAP); endSegment = choppedWordSegment; } } sequences.push(choppedWordSegment); segmentIdx++; }); if (endSegment) { endSegment[0].setAttribute(AR_BUT_SEGMENT_STOP_KEY, true); } } }); return sequences; } } exports.ButRule = ButRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"lodash/lodash.min":undefined}],30:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const AutoRuleManager_1 = require("./AutoRuleManager"); class CommaRule extends AutoRule_1.AutoRule { constructor(jibo, priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Comma, priority, probability); this.jibo = jibo; this.Orientation = this.jibo.animDB.Orientation; this.lastOrientation = (Math.random() > 0.5) ? this.Orientation.LEFT : this.Orientation.RIGHT; } _createNode(children) { this.lastOrientation = this.lastOrientation === this.Orientation.LEFT ? this.Orientation.RIGHT : this.Orientation.LEFT; let atts = new Map(); atts.set('layers', 'body'); atts.set('cat', 'comma'); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('orientation', this.lastOrientation); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Comma); n.setAttribute('endNeutral', 'false'); if (children.length < 3) { n.setAttribute('nonBlocking', 'true'); n.timeSyncNode = children[0]; } return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, (sentence) => { const wordSegments = AutoRuleUtils_1.AutoRuleUtils.getWordsFromSentenceSegmentedBy([','], sentence); if (wordSegments.length < 3) { const leftSegments = wordSegments.slice(0, -1); sequences.push(...leftSegments); } }); return sequences; } } exports.CommaRule = CommaRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24}],31:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AutoRuleManager_1 = require("./AutoRuleManager"); const TimelineModUtils_1 = require("./timeline_mod/TimelineModUtils"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); class DoubleBlinkRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.DoubleBlink, priority, probability); } _createNode(children) { let n = TimelineModUtils_1.TimelineModUtils.blinkNodeBuilder(TimelineModUtils_1.BlinkNodeType.DOUBLE); n.setAttribute('nonBlocking', 'true'); n.setAttribute('position', 'after'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.DoubleBlink); n.timeSyncNode = children[0]; return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, sentence => { const sentenceType = sentence.getAttribute('sentenceType'); if (sentenceType === Types_1.SentenceType.QUESTION) { let words = sentence.gatherDescendantsType(Types_1.NLPNodeType.WORD); words = AutoRuleUtils_1.AutoRuleUtils.removeExpandedContractionWords(words); if (words) { sequences.push(words.slice(-1)); } } }); return sequences; } } exports.DoubleBlinkRule = DoubleBlinkRule; },{"../../common/Types":13,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"./timeline_mod/TimelineModUtils":47}],32:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const AutoRuleManager_1 = require("./AutoRuleManager"); class ExclamationRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Exclamation, priority, probability); } _createNode(children) { let atts = new Map(); atts.set('filter', '!exclamation-mark'); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('cat', 'exclamation'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Exclamation); return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, (sentence) => { if (sentence.sentenceType === Types_1.SentenceType.EXCLAMATION) { let chosenWords = AutoRuleUtils_1.AutoRuleUtils.getLastWordNodesInSentence(sentence); sequences.push(chosenWords); } }); return sequences; } } exports.ExclamationRule = ExclamationRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24}],33:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const _ = require("lodash/lodash.min"); const jibo_cai_utils_1 = require("jibo-cai-utils"); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleManager_1 = require("./AutoRuleManager"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const REJECT_FUZZY_SIZE = 2; class HotWordsRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1, hotWordMap, rejectWordMap, descendingPhraseLengths) { super(AutoRuleManager_1.AutoRuleNames.HotWords, priority, probability); this.hotWordMap = hotWordMap; this.rejectWordMap = rejectWordMap; this.descendingPhraseLengths = descendingPhraseLengths; } _createNode(children) { const words = children.map(word => { const contractedForm = word.getAttribute("contraction"); if (contractedForm) { return contractedForm; } else { return word.value; } }); const wordStr = words.join(' ').toLowerCase(); const match = this.hotWordMap.get(words.length).get(wordStr); if (!match) { return; } const hotWordAsset = jibo_cai_utils_1.RandomUtils.weightedRandomSample(match); let atts = new Map(); const mappedAsset = new Map(); Object.keys(hotWordAsset).forEach(asset => { mappedAsset.set(asset, hotWordAsset[asset]); }); mappedAsset.forEach((val, key) => { atts.set(key, val); }); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.HotWords); const isBounded = n.getAttribute('bounded'); if (!isBounded) { n.setAttribute('nonBlocking', 'true'); n.timeSyncNode = children[0]; } return n; } _applyInternal(nlpTreeRoot) { let sequences = []; if (this.descendingPhraseLengths.length === 0) { return sequences; } const wordNodes = nlpTreeRoot.gatherDescendantsType(Types_1.NLPNodeType.WORD); const noExpansions = AutoRuleUtils_1.AutoRuleUtils.removeExpandedContractionWords(wordNodes); noExpansions.forEach(wordNode => { wordNode.setAttribute('phraseLengthMember', 0); }); const nodesAndWords = noExpansions.map((word) => { let pair = []; pair.push(word); const contractedForm = word.getAttribute("contraction"); const value = contractedForm || word.value; pair.push(value.toLowerCase()); return pair; }); this.descendingPhraseLengths.forEach(phraseLength => { if (nodesAndWords.length >= phraseLength) { const hotWordMap = this.hotWordMap.get(phraseLength); for (let idx = 0; idx <= (nodesAndWords.length - phraseLength); idx++) { let slidingWindowMemberCheck = false; for (let walker = 0; walker < phraseLength; walker++) { if (nodesAndWords[idx + walker][0].getAttribute('phraseLengthMember') <= phraseLength) { slidingWindowMemberCheck = true; break; } } if (slidingWindowMemberCheck) { let slidingWindowNodes = []; let slidingWindowWords = []; for (let walker = 0; walker < phraseLength; walker++) { slidingWindowNodes.push(nodesAndWords[idx + walker][0]); slidingWindowWords.push(nodesAndWords[idx + walker][1]); } const slidingWindowStr = slidingWindowWords.join(' '); if (hotWordMap.get(slidingWindowStr)) { const rejectList = this.rejectWordMap.get(slidingWindowStr); let shouldReject = false; if (rejectList) { const rejectMin = Math.round(idx - REJECT_FUZZY_SIZE); const rejectMax = Math.round(idx + (phraseLength - 1) + REJECT_FUZZY_SIZE); const rejectWalkStart = Math.max(0, rejectMin); const rejectWalkStop = Math.min(nodesAndWords.length - 1, rejectMax); for (let walker = rejectWalkStart; walker <= rejectWalkStop; walker++) { for (let rejWalker = 0; rejWalker < rejectList.length; rejWalker++) { if (nodesAndWords[walker][1] === rejectList[rejWalker]) { shouldReject = true; break; } } } } if (!shouldReject) { slidingWindowNodes.forEach(node => { node.setAttribute('phraseLengthMember', phraseLength); }); sequences.push(slidingWindowNodes); } } } } } }); return _.shuffle(sequences); } } exports.HotWordsRule = HotWordsRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"jibo-cai-utils":undefined,"lodash/lodash.min":undefined}],34:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const AutoRuleManager_1 = require("./AutoRuleManager"); class InitiateRule extends AutoRule_1.AutoRule { constructor(jibo, priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Initiate, priority, probability); this.jibo = jibo; this.Orientation = this.jibo.animDB.Orientation; this.lastOrientation = (Math.random() > 0.5) ? this.Orientation.LEFT : this.Orientation.RIGHT; } _createNode(children) { this.lastOrientation = this.lastOrientation === this.Orientation.LEFT ? this.Orientation.RIGHT : this.Orientation.LEFT; let atts = new Map(); atts.set('layers', 'body'); atts.set('filter', 'initiation'); atts.set('cat', 'poses'); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Initiate); n.setAttribute('orientation', this.lastOrientation); return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, sentence => { const wordNodes = sentence.gatherDescendantsType(Types_1.NLPNodeType.WORD); const noExpansions = AutoRuleUtils_1.AutoRuleUtils.removeExpandedContractionWords(wordNodes); let firstWords = []; for (let idx = 0; idx < Math.min(3, noExpansions.length); idx++) { firstWords.push(noExpansions[idx]); } if (firstWords.length > 0) { sequences.push(firstWords); } }); return sequences; } } exports.InitiateRule = InitiateRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24}],35:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const _ = require("lodash/lodash.min"); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const AutoRuleManager_1 = require("./AutoRuleManager"); const AR_LIST_SUB_RULE_KEY = 'ar-list-subrule'; const AR_LIST_SUB_RULE_PART_KEY = 'ar-list-subrule-part'; exports.LIST_RULE_TYPES = { SUB_RULE_1: 'list-rule-sr1', SUB_RULE_2: 'list-rule-sr2' }; class ListRule extends AutoRule_1.AutoRule { constructor(jibo, priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.List, priority, probability); this.jibo = jibo; this.Orientation = this.jibo.animDB.Orientation; } _createNode(children) { const segmentSubRule = children[0].getAttribute(AR_LIST_SUB_RULE_KEY); const segmentSubRulePart = children[0].getAttribute(AR_LIST_SUB_RULE_PART_KEY); this.lastOrientation = (Math.random() > 0.5) ? this.Orientation.LEFT : this.Orientation.RIGHT; let atts = new Map(); if (segmentSubRulePart) { atts.set('filter', segmentSubRulePart); } let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('cat', segmentSubRule); n.setAttribute('orientation', this.lastOrientation); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.List); if (segmentSubRule === exports.LIST_RULE_TYPES.SUB_RULE_2 && segmentSubRulePart === 'a') { n.setAttribute('endNeutral', 'false'); } else { n.setAttribute('endNeutral', `true`); } return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, (sentence) => { const wordSegments = AutoRuleUtils_1.AutoRuleUtils.getWordsFromSentenceSegmentedBy([',', 'and'], sentence); if (wordSegments.length >= 3) { let possibleListSegments = []; let segmentCount = 0; wordSegments.forEach(segment => { if (segmentCount === 0) { let verbIdx = -1; for (let idx = 0; idx < segment.length; idx++) { if (segment[idx].wordType === Types_1.WordType.VERB) { verbIdx = idx; break; } } let slicedSegment; if (verbIdx !== -1) { slicedSegment = segment.slice(verbIdx + 1); } else { slicedSegment = segment.slice(-3); } if (slicedSegment.length > 0) { possibleListSegments.push(slicedSegment); segmentCount++; } } else if (segment.length <= 3) { segmentCount++; possibleListSegments.push(segment); } else { if (segmentCount < 3) { possibleListSegments = []; segmentCount = 0; } else { sequences.push(...this._addListSubRuleInfo(possibleListSegments)); possibleListSegments = []; segmentCount = 0; } } }); if (possibleListSegments && possibleListSegments.length >= 3) { sequences.push(...this._addListSubRuleInfo(possibleListSegments)); } } }); return sequences; } _addListSubRuleInfo(listSegments) { const choice = _.sample(_.values(exports.LIST_RULE_TYPES)); let mergedSegments = []; if (choice === exports.LIST_RULE_TYPES.SUB_RULE_1) { let mergingList = []; for (let idx = 0; idx < listSegments.length - 1; idx++) { if ((idx === 0) && (listSegments[idx].length > 0)) { listSegments[idx][0].setAttribute(AR_LIST_SUB_RULE_KEY, choice); } mergingList.push(...listSegments[idx]); } mergedSegments.push(mergingList); } else { let mergingList = []; for (let idx = 0; idx < listSegments.length; idx++) { if (idx === 0) { if (listSegments[idx].length > 0) { listSegments[idx][0].setAttribute(AR_LIST_SUB_RULE_KEY, choice); listSegments[idx][0].setAttribute(AR_LIST_SUB_RULE_PART_KEY, 'a'); } mergingList.push(...listSegments[idx]); } else if (idx === listSegments.length - 1) { mergedSegments.push(mergingList); mergingList = []; if (listSegments[idx].length > 0) { listSegments[idx][0].setAttribute(AR_LIST_SUB_RULE_KEY, choice); listSegments[idx][0].setAttribute(AR_LIST_SUB_RULE_PART_KEY, 'b'); } mergingList.push(...listSegments[idx]); mergedSegments.push(mergingList); } else { mergingList.push(...listSegments[idx]); } } } return mergedSegments; } } exports.ListRule = ListRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"lodash/lodash.min":undefined}],36:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleManager_1 = require("./AutoRuleManager"); const NLPTools_1 = require("../../utils/NLPTools"); exports.AR_EXEMPT_NOUNS = new Set(['i', 'you', 'we', 'me']); class NounRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Noun, priority, probability); } _createNode(children) { let atts = new Map(); atts.set('filter', '!down'); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('nonBlocking', 'true'); n.setAttribute('cat', 'glances'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Noun); n.timeSyncNode = children[0]; return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; let nounCount = 0; const wordAndPunctNodes = nlpTreeRoot.gatherDescendantsType(Types_1.NLPNodeType.WORD); const wordNodes = wordAndPunctNodes.filter(word => !NLPTools_1.NLPTools.isPunctuation(word.value)); wordNodes.forEach(word => { if (!word.contractionBase && (word.wordType === Types_1.WordType.NOUN) && !exports.AR_EXEMPT_NOUNS.has(word.value.toLowerCase())) { sequences.push([word]); nounCount++; } }); return sequences; } } exports.NounRule = NounRule; },{"../../common/Types":13,"../../utils/NLPTools":62,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23}],37:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const _ = require("lodash/lodash.min"); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const AutoRuleManager_1 = require("./AutoRuleManager"); const AR_OR_SEGMENT_START_KEY = 'ar-or-segment-start'; const AR_OR_SEGMENT_STOP_KEY = 'ar-or-segment-stop'; exports.OR_KEYS = [ 'Or_01', 'Or_02', 'Or_03' ]; const NUM_WORDS_TO_WRAP = 5; class OrRule extends AutoRule_1.AutoRule { constructor(jibo, priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Or, priority, probability); this.jibo = jibo; this.Orientation = this.jibo.animDB.Orientation; this.lastOrientation = (Math.random() > 0.5) ? this.Orientation.LEFT : this.Orientation.RIGHT; } _createNode(children) { const newSegment = children[0].getAttribute(AR_OR_SEGMENT_START_KEY); const endSegment = children[0].getAttribute(AR_OR_SEGMENT_STOP_KEY); if (newSegment) { this.currentSegmentAnim = _.sample(exports.OR_KEYS); } this.lastOrientation = this.lastOrientation === this.Orientation.LEFT ? this.Orientation.RIGHT : this.Orientation.LEFT; let atts = new Map(); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('name', this.currentSegmentAnim); n.setAttribute('orientation', this.lastOrientation); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Or); n.setAttribute('endNeutral', 'false'); if (endSegment) { n.setAttribute('endNeutral', 'true'); } return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, (sentence) => { let wordSegments = AutoRuleUtils_1.AutoRuleUtils.getWordsFromSentenceSegmentedBy(['or'], sentence, true); if (wordSegments.length > 0) { const zippedWordSegments = wordSegments.map(arr => [arr, arr.map(wordNode => wordNode.value)]); let segmentIdx = 0; let endSegment; zippedWordSegments.forEach(nodesAndStrings => { const [nodes, strings] = nodesAndStrings; let choppedWordSegment; if (segmentIdx === 0) { let commaIndex = strings.lastIndexOf(','); if (commaIndex !== -1) { nodes.splice(0, nodes.length - commaIndex - 1); } choppedWordSegment = nodes.slice(-NUM_WORDS_TO_WRAP); } else { let commaIndex = strings.indexOf(','); if (commaIndex !== -1) { nodes.splice(commaIndex, nodes.length - commaIndex); } choppedWordSegment = nodes.slice(0, NUM_WORDS_TO_WRAP); } if (choppedWordSegment.length > 0) { if (segmentIdx === 0) { choppedWordSegment[0].setAttribute(AR_OR_SEGMENT_START_KEY, true); } else { endSegment = choppedWordSegment; } sequences.push(choppedWordSegment); } segmentIdx++; }); if (endSegment) { endSegment[0].setAttribute(AR_OR_SEGMENT_STOP_KEY, true); } } }); return sequences; } } exports.OrRule = OrRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"lodash/lodash.min":undefined}],38:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const AssetNode_1 = require("../AssetNode"); const AutoRuleUtils_1 = require("./AutoRuleUtils"); const AutoRuleManager_1 = require("./AutoRuleManager"); class QuestionRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.Question, priority, probability); } _createNode(children) { let atts = new Map(); atts.set('layers', 'body'); atts.set('filter', '!eye-only'); let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); n.setAttribute('cat', 'question'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.Question); n.setAttribute('endNeutral', 'false'); if (children.length <= 3) { n.setAttribute('nonBlocking', 'true'); n.timeSyncNode = children[0]; } return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; nlpTreeRoot.forEachChildType(Types_1.NLPNodeType.SENTENCE, (sentence) => { if (sentence.sentenceType === Types_1.SentenceType.QUESTION) { let chosenWords = AutoRuleUtils_1.AutoRuleUtils.getLastWordNodesInSentence(sentence); sequences.push(chosenWords); } }); return sequences; } } exports.QuestionRule = QuestionRule; },{"../../common/Types":13,"../AssetNode":16,"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24}],39:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const AutoRule_1 = require("./AutoRule"); const Types_1 = require("../../common/Types"); const ThemeRhemeParser_1 = require("../parsers/ThemeRhemeParser"); const AssetNode_1 = require("../AssetNode"); const AutoRuleManager_1 = require("./AutoRuleManager"); class ThemeRhemeGazeRule extends AutoRule_1.AutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.AutoRuleNames.ThemeRhemeGaze, priority, probability); } _createNode() { let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM); n.setAttribute('cat', 'glances'); n.setAttribute('rule', AutoRuleManager_1.AutoRuleNames.ThemeRhemeGaze); return n; } _applyInternal(nlpTreeRoot, otherRoots) { let sequences = []; if (!otherRoots || otherRoots.length === 0) { throw new Error(`Need ThemeRheme tree root`); } otherRoots[0].forEachChildType(Types_1.NLPNodeType.SENTENCE, sentence => { sentence.forEachChildType(ThemeRhemeParser_1.TRPart.TR, tr => { let theme = tr.findFirstChildType(ThemeRhemeParser_1.TRPart.THEME); if (theme) { let words = theme.gatherDescendantsType(Types_1.NLPNodeType.WORD); for (let idx = 0; idx < words.length; idx++) { if (words[idx].contractionBase) { words.splice(idx, 1); } } if (words.length > 1) { sequences.push(words); } } }); }); return sequences; } } exports.ThemeRhemeGazeRule = ThemeRhemeGazeRule; },{"../../common/Types":13,"../AssetNode":16,"../parsers/ThemeRhemeParser":52,"./AutoRule":22,"./AutoRuleManager":23}],40:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); __export(require("./timeline_mod")); __export(require("./AutoRule")); __export(require("./AutoRuleManager")); __export(require("./AutoRuleUtils")); __export(require("./BeatFirstWordPartRule")); __export(require("./BeatNewWordRule")); __export(require("./BlinkRule")); __export(require("./BeatBlinkRule")); __export(require("./ButRule")); __export(require("./CommaRule")); __export(require("./DoubleBlinkRule")); __export(require("./ExclamationRule")); __export(require("./HotWordsRule")); __export(require("./ListRule")); __export(require("./NounRule")); __export(require("./OrRule")); __export(require("./QuestionRule")); __export(require("./ThemeRhemeGazeRule")); __export(require("./InitiateRule")); },{"./AutoRule":22,"./AutoRuleManager":23,"./AutoRuleUtils":24,"./BeatBlinkRule":25,"./BeatFirstWordPartRule":26,"./BeatNewWordRule":27,"./BlinkRule":28,"./ButRule":29,"./CommaRule":30,"./DoubleBlinkRule":31,"./ExclamationRule":32,"./HotWordsRule":33,"./InitiateRule":34,"./ListRule":35,"./NounRule":36,"./OrRule":37,"./QuestionRule":38,"./ThemeRhemeGazeRule":39,"./timeline_mod":48}],41:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const TimelineAutoRule_1 = require("./TimelineAutoRule"); const AssetNode_1 = require("../../AssetNode"); const Types_1 = require("../../../common/Types"); const InputProcessor_1 = require("../../InputProcessor"); const AutoRuleManager_1 = require("../AutoRuleManager"); const AutoRuleUtils_1 = require("../AutoRuleUtils"); const TimelineModUtils_1 = require("./TimelineModUtils"); class EllipsisRule extends TimelineAutoRule_1.TimelineAutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.TimelineModifyingRuleNames.Ellipsis, TimelineModUtils_1.TimelineAutoRuleType.TIMEFUL, priority, probability); } _createTimelineModifyingData(matchInfo) { const offset = 1; let n = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM); n.setAttribute('cat', 'ellipsis'); return TimelineModUtils_1.TimelineModUtils.getInsertionInfo(n, matchInfo.sibling, offset); } _applyInternal(xmlTree) { let matchesInfo = []; xmlTree.forEachDescendant(node => { if (node.value === AutoRuleUtils_1.AR_SUBRULE.ELLIPSIS) { let ancestorConflict = false; node.forEachAncestor(ancestor => { if (InputProcessor_1.InputProcessor.isAsset(ancestor)) { ancestorConflict = true; } }); if (!ancestorConflict) { matchesInfo.push({ sibling: node, subRule: AutoRuleUtils_1.AR_SUBRULE.ELLIPSIS }); } } }); return matchesInfo; } } exports.EllipsisRule = EllipsisRule; },{"../../../common/Types":13,"../../AssetNode":16,"../../InputProcessor":19,"../AutoRuleManager":23,"../AutoRuleUtils":24,"./TimelineAutoRule":46,"./TimelineModUtils":47}],42:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const InputProcessor_1 = require("../../InputProcessor"); const Types_1 = require("../../../common/Types"); const TimelineAutoRule_1 = require("./TimelineAutoRule"); const TimelineModUtils_1 = require("./TimelineModUtils"); const AutoRuleManager_1 = require("../AutoRuleManager"); class EmotionalVoiceRule extends TimelineAutoRule_1.TimelineAutoRule { constructor(jibo, priority = 1, probability = 1) { super(AutoRuleManager_1.TimelineModifyingRuleNames.EmotionalVoice, TimelineModUtils_1.TimelineAutoRuleType.TIMELESS, priority, probability); this.jibo = jibo; this.emotionToVoiceConfig = new Map(); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.NEUTRAL, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.NEUTRAL } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.INSECURE, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.CONFUSED } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.FRUSTRATED, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.CONFUSED } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.HOPEFUL, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.CONFIDENT } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.JOYFUL, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.ENTHUSIASTIC } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.PLEASED, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.ENTHUSIASTIC } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.DETERMINED, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.SHEEPISH } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.SAD, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.SHEEPISH } } ]); this.emotionToVoiceConfig.set(jibo.emotion.EmotionName.CONFIDENT, [ { type: Types_1.SSMLNodeType.STYLE, attrs: { set: Types_1.SSMLStyleTagType.CONFIDENT } } ]); } _createTimelineModifyingData(matchInfo) { let voiceConfig = this.emotionToVoiceConfig.get(this.emotion.name); if (!voiceConfig) { voiceConfig = this.emotionToVoiceConfig.get(this.jibo.emotion.EmotionName.NEUTRAL); } const ssmlNodesToInsert = voiceConfig.map(ssmlConfig => TimelineModUtils_1.TimelineModUtils.SSMLNodeBuilder(ssmlConfig.type, ssmlConfig.attrs)); const parent = matchInfo.children[0].parents[0]; const insertionInfo = { nodes: ssmlNodesToInsert, children: matchInfo.children }; return [parent, insertionInfo]; } _applyInternal(xmlTree) { let matchesInfo = []; this.emotion = this.jibo.emotion.getNearestEmotion(); const allNodes = xmlTree.gatherDescendants(); const containsTimelessSSML = allNodes.every(node => !InputProcessor_1.InputProcessor.isTimelessSSML(node)); if (containsTimelessSSML) { matchesInfo.push({ children: xmlTree.gatherChildren() }); } return matchesInfo; } } exports.EmotionalVoiceRule = EmotionalVoiceRule; },{"../../../common/Types":13,"../../InputProcessor":19,"../AutoRuleManager":23,"./TimelineAutoRule":46,"./TimelineModUtils":47}],43:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); const TimelineAutoRule_1 = require("./TimelineAutoRule"); const TimelineModUtils_1 = require("./TimelineModUtils"); const AutoRuleManager_1 = require("../AutoRuleManager"); const AutoRuleUtils_1 = require("../AutoRuleUtils"); const jibo_node_xml_2 = require("jibo-node-xml"); class ExclamationVoiceRule extends TimelineAutoRule_1.TimelineAutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.TimelineModifyingRuleNames.ExclamationVoice, TimelineModUtils_1.TimelineAutoRuleType.TIMELESS, priority, probability); } _createTimelineModifyingData(matchInfo) { let nodeToInsert = new jibo_node_xml_1.default('DEPRECATED'); nodeToInsert.setAttribute('config', 'happy'); const parent = matchInfo.children[0].parents[0]; const insertionInfo = { nodes: [nodeToInsert], children: matchInfo.children }; return [parent, insertionInfo]; } _applyInternal(xmlTree) { let matchesInfo = []; let sentenceWords = []; xmlTree.forEachDescendantType(jibo_node_xml_2.NodeType.TEXT, node => { switch (node.value) { case AutoRuleUtils_1.AR_SUBRULE.PERIOD: case AutoRuleUtils_1.AR_SUBRULE.QUESTION: case AutoRuleUtils_1.AR_SUBRULE.COMMA: case AutoRuleUtils_1.AR_SUBRULE.ELLIPSIS: sentenceWords = []; break; case AutoRuleUtils_1.AR_SUBRULE.EXCLAMATION: sentenceWords.push(node); matchesInfo.push({ children: sentenceWords }); sentenceWords = []; break; default: sentenceWords.push(node); } }); return matchesInfo; } } exports.ExclamationVoiceRule = ExclamationVoiceRule; },{"../AutoRuleManager":23,"../AutoRuleUtils":24,"./TimelineAutoRule":46,"./TimelineModUtils":47,"jibo-node-xml":undefined}],44:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const TimelineAutoRule_1 = require("./TimelineAutoRule"); const AutoRuleManager_1 = require("../AutoRuleManager"); const AutoRuleUtils_1 = require("../AutoRuleUtils"); const TimelineModUtils_1 = require("./TimelineModUtils"); exports.AR_MIN_PERIOD_PAU = 0.1; exports.AR_MAX_PERIOD_PAU = 0.3; exports.AR_QUESTION_PAU = 0.3; exports.AR_EXCLAMATION_PAU = 0.3; exports.AR_ELLIPSIS_PAU = 0.75; class InterSentenceTimingRule extends TimelineAutoRule_1.TimelineAutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.TimelineModifyingRuleNames.InterSentenceTiming, TimelineModUtils_1.TimelineAutoRuleType.TIMEFUL, priority, probability); } _createTimelineModifyingData(matchInfo) { let breakSize = 0; switch (matchInfo.subRule) { case AutoRuleUtils_1.AR_SUBRULE.PERIOD: breakSize = (matchInfo.sibling.getNumberAtt('wordCount') <= 3) ? exports.AR_MIN_PERIOD_PAU : exports.AR_MAX_PERIOD_PAU; break; case AutoRuleUtils_1.AR_SUBRULE.QUESTION: breakSize = exports.AR_QUESTION_PAU; break; case AutoRuleUtils_1.AR_SUBRULE.EXCLAMATION: breakSize = exports.AR_EXCLAMATION_PAU; break; case AutoRuleUtils_1.AR_SUBRULE.ELLIPSIS: breakSize = exports.AR_ELLIPSIS_PAU; break; } if (breakSize > 0) { const n = TimelineModUtils_1.TimelineModUtils.breakNodeBuilder(breakSize); return TimelineModUtils_1.TimelineModUtils.getInsertionInfo(n, matchInfo.sibling, 1); } return; } _applyInternal(xmlTree, doPunctuation) { let matchesInfo = []; let endedTextInPunctLoc = null; let wordCount = 0; xmlTree.forEachDescendant(node => { switch (node.value) { case AutoRuleUtils_1.AR_SUBRULE.PERIOD: case AutoRuleUtils_1.AR_SUBRULE.QUESTION: case AutoRuleUtils_1.AR_SUBRULE.EXCLAMATION: endedTextInPunctLoc = matchesInfo.length; node.setAttribute('wordCount', wordCount); matchesInfo.push({ sibling: node, subRule: node.value }); wordCount = 0; break; case AutoRuleUtils_1.AR_SUBRULE.ELLIPSIS: if (!doPunctuation) { endedTextInPunctLoc = matchesInfo.length; matchesInfo.push({ sibling: node, subRule: AutoRuleUtils_1.AR_SUBRULE.ELLIPSIS }); } break; default: endedTextInPunctLoc = null; if (/\w+/.test(node.value)) { wordCount++; } } }); if (endedTextInPunctLoc !== null) { matchesInfo.splice(endedTextInPunctLoc, 1); } return matchesInfo; } } exports.InterSentenceTimingRule = InterSentenceTimingRule; },{"../AutoRuleManager":23,"../AutoRuleUtils":24,"./TimelineAutoRule":46,"./TimelineModUtils":47}],45:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const TimelineAutoRule_1 = require("./TimelineAutoRule"); const AutoRuleManager_1 = require("../AutoRuleManager"); const AutoRuleUtils_1 = require("../AutoRuleUtils"); const TimelineModUtils_1 = require("./TimelineModUtils"); exports.AR_BUT_PAU = 0.2; exports.AR_OR_PAU = 0.2; class IntraSentenceTimingRule extends TimelineAutoRule_1.TimelineAutoRule { constructor(priority = 1, probability = 1) { super(AutoRuleManager_1.TimelineModifyingRuleNames.IntraSentenceTiming, TimelineModUtils_1.TimelineAutoRuleType.TIMEFUL, priority, probability); } _createTimelineModifyingData(matchInfo) { let breakSize = 0; let offset = 1; switch (matchInfo.subRule) { case AutoRuleUtils_1.AR_SUBRULE.OR: breakSize = exports.AR_OR_PAU; offset = 0; break; case AutoRuleUtils_1.AR_SUBRULE.BUT: breakSize = exports.AR_BUT_PAU; offset = 0; break; } let n = TimelineModUtils_1.TimelineModUtils.breakNodeBuilder(breakSize); return TimelineModUtils_1.TimelineModUtils.getInsertionInfo(n, matchInfo.sibling, offset); } _applyInternal(xmlTree) { let matchesInfo = []; xmlTree.forEachDescendant(node => { switch (node.value) { case AutoRuleUtils_1.AR_SUBRULE.BUT: matchesInfo.push({ sibling: node, subRule: node.value }); break; } }); return matchesInfo; } } exports.IntraSentenceTimingRule = IntraSentenceTimingRule; },{"../AutoRuleManager":23,"../AutoRuleUtils":24,"./TimelineAutoRule":46,"./TimelineModUtils":47}],46:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); class TimelineAutoRule { constructor(ruleName, ruleType, priority, probability = 1) { this.ruleName = ruleName; this.ruleType = ruleType; this.priority = priority; this.probability = probability; } _applyInternal(xmlTree, context) { throw new Error('Should be implemented by subclass'); } _createTimelineModifyingData(matchInfo) { throw new Error('Should be implemented by subclass'); } applyRule(xmlTree, context) { let matches = this._applyInternal(xmlTree, context); let ruleData = {}; ruleData.autoRule = this.ruleName; ruleData.type = this.ruleType; ruleData.priority = this.priority; ruleData.matches = new Map(); matches.forEach(matchInfo => { if (Math.random() < this.probability) { const [parent, nodeToInsertInfo] = this._createTimelineModifyingData(matchInfo); let childrenToInsert = ruleData.matches.get(parent); if (!childrenToInsert) { ruleData.matches.set(parent, childrenToInsert = []); } childrenToInsert.push(nodeToInsertInfo); } }); return ruleData; } } exports.TimelineAutoRule = TimelineAutoRule; },{}],47:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); const SSMLNode_1 = require("../../SSMLNode"); const AssetNode_1 = require("../../AssetNode"); const Types_1 = require("../../../common/Types"); var TimelineAutoRuleType; (function (TimelineAutoRuleType) { TimelineAutoRuleType["TIMELESS"] = "TIMELESS"; TimelineAutoRuleType["TIMEFUL"] = "TIMEFUL"; })(TimelineAutoRuleType = exports.TimelineAutoRuleType || (exports.TimelineAutoRuleType = {})); var BlinkNodeType; (function (BlinkNodeType) { BlinkNodeType["SINGLE"] = "SINGLE"; BlinkNodeType["DOUBLE"] = "DOUBLE"; })(BlinkNodeType = exports.BlinkNodeType || (exports.BlinkNodeType = {})); class TimelineModUtils { static breakNodeBuilder(size) { let nodeToInsert = new jibo_node_xml_1.default(Types_1.SSMLNodeType.BREAK); nodeToInsert.setAttribute('size', (Math.round(size * 1e2) / 1e2)); return nodeToInsert; } static blinkNodeBuilder(type) { let atts = new Map(); atts.set('filter', type); atts.set('layers', 'screen'); atts.set('cat', 'blinks'); let nodeToInsert = new AssetNode_1.AssetNode(Types_1.AssetNodeType.ANIM, atts); return nodeToInsert; } static SSMLNodeBuilder(type, attrs) { let nodeToInsert = new SSMLNode_1.SSMLNode(type); Object.keys(attrs).forEach(key => { nodeToInsert.setAttribute(key, attrs[key]); }); return nodeToInsert; } static getInsertionInfo(nodeToInsert, siblingNode, offset) { const parent = siblingNode.parents[0]; const insertIdx = parent.children.indexOf(siblingNode) + offset; const insertionInfo = { nodes: [nodeToInsert], offset: insertIdx }; return [parent, insertionInfo]; } } exports.TimelineModUtils = TimelineModUtils; },{"../../../common/Types":13,"../../AssetNode":16,"../../SSMLNode":20,"jibo-node-xml":undefined}],48:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); __export(require("./EllipsisRule")); __export(require("./ExclamationVoiceRule")); __export(require("./EmotionalVoiceRule")); __export(require("./InterSentenceTimingRule")); __export(require("./IntraSentenceTimingRule")); __export(require("./TimelineAutoRule")); __export(require("./TimelineModUtils")); },{"./EllipsisRule":41,"./EmotionalVoiceRule":42,"./ExclamationVoiceRule":43,"./InterSentenceTimingRule":44,"./IntraSentenceTimingRule":45,"./TimelineAutoRule":46,"./TimelineModUtils":47}],49:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); const auto_rules = require("./auto_rules"); exports.auto_rules = auto_rules; const parsers = require("./parsers"); exports.parsers = parsers; __export(require("./AssetNode")); __export(require("./Formats")); __export(require("./InputProcessor")); __export(require("./SSMLNode")); __export(require("./TagTreeNode")); __export(require("./ContentValidator")); },{"./AssetNode":16,"./ContentValidator":17,"./Formats":18,"./InputProcessor":19,"./SSMLNode":20,"./TagTreeNode":21,"./auto_rules":40,"./parsers":53}],50:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); class Lexer { constructor(jibo) { this.jibo = jibo; } lex(input) { return __awaiter(this, void 0, void 0, function* () { let data = yield this.jibo.tts.getPOSTokens(input); return data.tokens; }); } } exports.Lexer = Lexer; },{}],51:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); const jibo_node_xml_2 = require("jibo-node-xml"); const Types_1 = require("../../common/Types"); const jibo_cai_utils_1 = require("jibo-cai-utils"); const Lexer_1 = require("./Lexer"); let stemmer = require('stemmer'); let root = jibo_cai_utils_1.PathUtils.getProjectRoot(__filename); let common_words = require(root + '/resources/common_words_stemmed_500.json'); class PartNode extends jibo_node_xml_1.default { constructor(value) { super(Types_1.NLPNodeType.PART, value); this.tags = []; } } exports.PartNode = PartNode; class SentenceNode extends jibo_node_xml_1.default { constructor(value) { super(Types_1.NLPNodeType.SENTENCE, value); this.words = []; this.sentenceType = Types_1.SentenceType.REGULAR; } set sentenceType(type) { this.setAttribute('sentenceType', type); } get sentenceType() { return this.getAttribute('sentenceType'); } } exports.SentenceNode = SentenceNode; class WordNode extends jibo_node_xml_1.default { set wordType(type) { this.setAttribute('wordType', type); } get wordType() { return this.getAttribute('wordType'); } constructor(value) { super(Types_1.NLPNodeType.WORD, value); } } exports.WordNode = WordNode; class Tagger { constructor(jibo) { this.jibo = jibo; this.jibo = jibo; } tag(words) { return __awaiter(this, void 0, void 0, function* () { let data = yield this.jibo.tts.getPOSTags(words); return data.tokentags; }); } } exports.Tagger = Tagger; class NLParser { constructor(jibo) { this.neverNew = new Set(); this.globalInd = 0; this.contractionBase = null; this.wordCounts = new Map(); this.lexer = new Lexer_1.Lexer(jibo); this.tagger = new Tagger(jibo); common_words .map(w => stemmer(w)) .forEach(w => this.neverNew.add(w)); } _parseWord(word, pos) { let textNode = new WordNode(word); textNode.setAttribute('globalInd', this.globalInd++); textNode.setAttribute('stem', stemmer(word)); textNode.setAttribute('pos', pos); if (this.expansions && this.contractions) { let wordCount = this.wordCounts.get(word); if (wordCount !== undefined) { wordCount++; } else { wordCount = 0; } this.wordCounts.set(word, wordCount); let foundIndexes = this.expansions.reduce((arr, elm, idx) => { if (elm === word) { arr.push(idx); } return arr; }, []); if (foundIndexes.length > 0) { let contractedForm = this.contractions[foundIndexes[wordCount]]; if (contractedForm !== word) { if (contractedForm !== '') { textNode.setAttribute('contraction', contractedForm); this.contractionBase = textNode; } else { if (this.contractionBase) { textNode.contractionBase = this.contractionBase; } else { throw new Error('Cannot find base word of expanded contraction'); } } } } else { throw new Error('Error when attempting to expand contractions.'); } } if (pos.startsWith('VB')) { textNode.wordType = Types_1.WordType.VERB; } else if (pos.startsWith('N')) { textNode.wordType = Types_1.WordType.NOUN; } else if (pos.startsWith('P')) { textNode.wordType = Types_1.WordType.PREP; } else if (word[0] === ',' || pos.startsWith('CC')) { textNode.wordType = Types_1.WordType.CONN; } else { textNode.wordType = Types_1.WordType.UNKN; } let stemmed = textNode.getAttribute('stem'); if (!this.neverNew.has(stemmed) && !this.dialogContext.existing.has(stemmed)) { this.dialogContext.existing.add(stemmed); textNode.setAttribute('new', true); } return textNode; } _createWordNodes(tags) { return tags.map(tag => { const pos = tag[1]; const word = tag[0]; return this._parseWord(word, pos); }); } setDialogContext(dialogContext) { this.dialogContext = dialogContext; } parse(input, expansions, contractions) { return __awaiter(this, void 0, void 0, function* () { if (typeof input === 'string') { input = yield this.lexer.lex(input); } return this._parse(input, expansions, contractions); }); } _parse(words, expansions, contractions) { return __awaiter(this, void 0, void 0, function* () { this.globalInd = 0; this.wordCounts = new Map(); this.expansions = null; this.contractions = null; if (!this.dialogContext) { this.dialogContext = { existing: new Set() }; } if (expansions && contractions) { this.expansions = expansions; this.contractions = contractions; } let root = new jibo_node_xml_1.default(jibo_node_xml_2.NodeType.ROOT); root.setAttribute('treeType', 'NLP_TREE'); let sentence = new SentenceNode(); root.addChild(sentence); words.forEach(w => { if (w === '.' || w === '?' || w === '!' || w === ';' || w === ':') { if (w === '?') { sentence.sentenceType = Types_1.SentenceType.QUESTION; } else if (w === '!') { sentence.sentenceType = Types_1.SentenceType.EXCLAMATION; } let endNode = new WordNode(w); endNode.wordType = Types_1.WordType.END; root.addChild(endNode); sentence = new SentenceNode(); sentence.words = []; sentence.sentenceType = Types_1.SentenceType.REGULAR; root.addChild(sentence); } else { sentence.words.push(w); } }); let lastSentence = root.children[root.children.length - 1]; if (lastSentence.type === Types_1.NLPNodeType.SENTENCE && !lastSentence.words.length) { root.removeChild(lastSentence); } const sentences = root.gatherChildrenType(Types_1.NLPNodeType.SENTENCE); for (let sentence of sentences) { sentence.value = sentence.words.join(' '); let tags; try { tags = yield this.tagger.tag(sentence.words); } catch (err) { throw new Error('Error when attempting to retreive POS tags'); } sentence.words = null; let part = new PartNode(); part.setAttribute('verbCount', 0); sentence.addChild(part); tags.forEach(tag => { let word = tag[0]; let pos = tag[1]; if (word === ',' || pos.startsWith('CC')) { let endNode = new WordNode(word); endNode.wordType = Types_1.WordType.CONN; endNode.setAttribute('pos', pos); sentence.addChild(endNode); part = new PartNode(); part.setAttribute('verbCount', 0); sentence.addChild(part); } else { if (pos.startsWith('V')) { part.modifyAttribute('verbCount', (oldVal) => oldVal + 1); } part.tags.push(tag); } }); sentence.gatherChildrenType(Types_1.NLPNodeType.PART, (p) => !p.tags.length) .forEach(p => { sentence.removeChild(p); }); sentence.setAttribute('verbCount', 0); const parts = sentence.gatherChildrenType(Types_1.NLPNodeType.PART); parts.forEach(part => { const pvc = part.getAttribute('verbCount'); sentence.modifyAttribute('verbCount', (svc) => svc + pvc); part.value = part.tags.map(t => t[0]).join(' '); this._createWordNodes(part.tags) .forEach(w => part.addChild(w)); }); } root.gatherChildrenType(Types_1.NLPNodeType.SENTENCE, p => !p.children.length) .forEach(p => root.removeChild(p)); return root; }); } } exports.NLParser = NLParser; },{"../../common/Types":13,"./Lexer":50,"jibo-cai-utils":undefined,"jibo-node-xml":undefined,"stemmer":undefined}],52:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_node_xml_1 = require("jibo-node-xml"); const jibo_node_xml_2 = require("jibo-node-xml"); const Types_1 = require("../../common/Types"); function addAllWords(node, out) { if (node.type === Types_1.NLPNodeType.WORD) { out.addChild(node); } else { node.forEachDescendantType(Types_1.NLPNodeType.WORD, n => { out.addChild(n); }); } } function parseVP(vp, preV, V, postV) { let internalVP = vp.findFirstChildType('VP'); if (!internalVP) { let verbDone = false; for (let i = 0; i < vp.children.length; i++) { let node = vp.children[i]; if (!verbDone) { if (node.type !== Types_1.NLPNodeType.WORD) { verbDone = true; addAllWords(node, postV); } else { addAllWords(node, V); } } else { addAllWords(node, postV); } } } else { for (let i = 0; i < vp.children.length; i++) { let node = vp.children[i]; if (node.type === 'VP') { if (i !== vp.children.length - 1) { throw Error(`Don't think that this should happen`); } parseVP(node, preV, V, postV); } else { addAllWords(node, preV); } } } } exports.VerbPart = { COMPLEX: 'COMPLEX', PRE: 'PRE', V: 'V', POST: 'POST', }; exports.TRPart = { TR: 'TR', THEME: 'THEME', RHEME: 'RHEME', }; class ThemeRhemeParser { parse(nlpTreeRoot) { let tempRoot = new jibo_node_xml_1.default(jibo_node_xml_2.NodeType.ROOT); for (let j = 0; j < nlpTreeRoot.children.length; j++) { let node = nlpTreeRoot.children[j]; if (node.type === Types_1.NLPNodeType.WORD) { tempRoot.addChild(node); } else if (node.type === Types_1.NLPNodeType.SENTENCE) { let newSentence = node.cloneShallow(); tempRoot.addChild(newSentence); for (let k = 0; k < node.children.length; k++) { let node2 = node.children[k]; if (node2.type === Types_1.NLPNodeType.PART) { let complex = new jibo_node_xml_1.default(exports.VerbPart.COMPLEX); newSentence.addChild(complex); let preV = new jibo_node_xml_1.default(exports.VerbPart.PRE); let V = new jibo_node_xml_1.default(exports.VerbPart.V); let postV = new jibo_node_xml_1.default(exports.VerbPart.POST); complex.addChild(preV); complex.addChild(V); complex.addChild(postV); for (let i = 0; i < node2.children.length; i++) { let n = node2.children[i]; if (n.type === 'VP') { parseVP(n, preV, V, postV); break; } else { addAllWords(n, preV); } } if (!V.children.length) { complex.att.set('noVerb', true); } } else if (node2.type === Types_1.NLPNodeType.WORD) { newSentence.addChild(node2); } else { throw Error(`Shouldn't happen`); } } newSentence.forEachChildType(exports.VerbPart.COMPLEX, compl => { compl.att.set('newCount', 0); compl.forEachChild(part => { part.att.set('newCount', 0); part.forEachChildType(Types_1.NLPNodeType.WORD, c => { if (c.att.get('new')) { part.modifyAttribute('newCount', (old) => old + 1); } }); compl.modifyAttribute('newCount', (old) => { return old + part.getAttribute('newCount'); }); }); }); } else { throw Error(`Shouldn't happen`); } } let newRoot = new jibo_node_xml_1.default(jibo_node_xml_2.NodeType.ROOT); for (let i = 0; i < tempRoot.children.length; i++) { let c = tempRoot.children[i]; if (c.type === Types_1.NLPNodeType.SENTENCE) { let newSentence = c.cloneShallow(); newRoot.addChild(newSentence); c.forEachChild(n => { if (n.type === exports.VerbPart.COMPLEX) { let tr = new jibo_node_xml_1.default(exports.TRPart.TR); let theme = new jibo_node_xml_1.default(exports.TRPart.THEME); let rheme = new jibo_node_xml_1.default(exports.TRPart.RHEME); newSentence.addChild(tr); let pre = n.children[0]; let v = n.children[1]; let post = n.children[2]; let flipped = false; if (!this._hasNewCount(pre)) { if (!this._hasNewCount(post)) { theme.addChild(pre); rheme.addChild(v); rheme.addChild(post); } else { theme.addChild(pre); theme.addChild(v); rheme.addChild(post); } } else { if (!this._hasNewCount(post)) { if (!this._hasNewCount(v)) { rheme.addChild(pre); theme.addChild(v); theme.addChild(post); tr.addChild(rheme); tr.addChild(theme); flipped = true; } else { theme.addChild(pre); rheme.addChild(v); rheme.addChild(post); } } else { theme.addChild(pre); theme.addChild(v); rheme.addChild(post); } } if (!flipped) { tr.addChild(theme); tr.addChild(rheme); } } else if (n.type === Types_1.NLPNodeType.WORD) { newSentence.addChild(n); } else { throw Error(`Shouldn't happen`); } }); } else if (c.type === Types_1.NLPNodeType.WORD) { newRoot.addChild(c); } else { throw Error(`Shouldn't happen`); } } return newRoot; } _hasNewCount(node) { let n = node.att.get('newCount'); return !!n; } } exports.ThemeRhemeParser = ThemeRhemeParser; },{"../../common/Types":13,"jibo-node-xml":undefined}],53:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); __export(require("./Lexer")); __export(require("./NLParser")); __export(require("./ThemeRhemeParser")); },{"./Lexer":50,"./NLParser":51,"./ThemeRhemeParser":52}],54:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_state_machine_1 = require("jibo-state-machine"); const jibo_cai_utils_1 = require("jibo-cai-utils"); const InputProcessor_1 = require("../processing/InputProcessor"); const Utils_1 = require("../../utils/Utils"); class BuildDispatchableState extends jibo_state_machine_1.State { constructor(sm, es, name) { super(sm, name); this.es = es; this.onEntry = (transition, data) => __awaiter(this, void 0, void 0, function* () { this.es.log.debug("Building dispatchable Action Timelines"); this.ttsDispatchChainPr = Promise.resolve(); this.animDispatchChainPr = Promise.resolve(); this.animDispatchPrs = []; this.screenDispatchPrs = new jibo_cai_utils_1.PromiseQueue(); this.ttsDone = new Utils_1.ExtPromise(); this.animDone = new Utils_1.ExtPromise(); this.audioDone = new Utils_1.ExtPromise(); this.lastTTSFrame = null; this.lastAnimFrame = null; this.lastAudioStop = 0; let dispatchableTimeline = []; let consume = { ttsFrame: data.timeline.tts._root, animFrame: data.timeline.anim._root, }; while (consume.ttsFrame || consume.animFrame) { if (consume.ttsFrame) { this.lastTTSFrame = consume.ttsFrame; let currentTTSFrame = consume.ttsFrame; if (currentTTSFrame.data.speakingSSMLText) { dispatchableTimeline.push(this._generateTTSAction(currentTTSFrame)); } consume.ttsFrame = consume.ttsFrame.next; } if (consume.animFrame) { this.lastAnimFrame = consume.animFrame; const currentAnimFrame = consume.animFrame; const returnToIdle = currentAnimFrame.data.getAttribute(InputProcessor_1.TagAttributes.END_NEUTRAL) === 'true'; const playOptions = { forceReturnToIdle: returnToIdle, screenCenterOverride: true, ownerInformation: this.es.ownerInformation }; const playbackData = currentAnimFrame.data.playbackData; if (playbackData.playbackLayers.Audio && (currentAnimFrame.stop > this.lastAudioStop)) { this.lastAudioStop = currentAnimFrame.stop; } dispatchableTimeline.push(this._generateAnimAction(currentAnimFrame, playbackData, playOptions)); consume.animFrame = consume.animFrame.next; } } if (data.timeline.anim.length > 0 && this.lastAnimFrame) { if (this.lastAudioStop === 0) { this.audioDone.resolve(); } dispatchableTimeline.push(this._generateFinalAnimAction(data)); } else { this.audioDone.resolve(); this.animDone.resolve(data.speakOptions.dispatchComplete); } if (data.timeline.tts.length > 0 && this.lastTTSFrame) { dispatchableTimeline.push(this._generateFinalTTSAction(data)); } else { this.audioDone.promise.then(() => { this.ttsDone.resolve(data.speakOptions.dispatchComplete); }); } dispatchableTimeline.sort((actionA, actionB) => { return actionA.timeMs - actionB.timeMs; }); data.dispatchableTimeline = dispatchableTimeline; data.ttsDonePr = this.ttsDone.promise; data.animDonePr = this.animDone.promise; return data; }); } _generateTTSAction(ttsFrame) { return { timeMs: Utils_1.Utils.secToMillisec(ttsFrame.start), action: () => { this.ttsDispatchChainPr = this.ttsDispatchChainPr.then(() => { if (!ttsFrame.prev) { this.es.jibo.performance.log('EmbodiedSpeechTTSStartSkill', 'Before calling first TTS.speak'); this.es.jibo.performance.log('EmbodiedSpeech_currentTTSFrameData', ttsFrame.data.speakingSSMLText); } this.es.jibo.autobot.log('EmbodiedSpeech sending TTS', ttsFrame.data.speakingSSMLText, ttsFrame.data.params); return this.es.jibo.tts.speak(ttsFrame.data.speakingSSMLText, ttsFrame.data.params) .catch(e => { this.es.log.error(e); return this.es.stop(); }); }); } }; } _generateFinalTTSAction(data) { return { timeMs: Utils_1.Utils.secToMillisec(this.lastTTSFrame.start), action: () => { this.ttsDispatchChainPr = this.ttsDispatchChainPr .then(() => { this.es.jibo.performance.log('EmbodiedSpeechTTSStopSkill', 'After last TTS.speak is finished'); return this.audioDone; }) .then(() => { this.es.jibo.performance.log('EmbodiedSpeechTTSStopSkill', 'All speech (verbal and SSA) is finished'); this.ttsDone.resolve(data.speakOptions.dispatchComplete); }); } }; } _generateAnimAction(animFrame, playbackData, playOptions) { return { timeMs: Utils_1.Utils.secToMillisec(animFrame.start), action: () => { let playback; const _playedPrProvider = () => playbackData.playbackPr .then(_playback => { playback = _playback; return this.play(_playback, playOptions); }) .then(() => { if (animFrame.stop === this.lastAudioStop) { this.audioDone.resolve(); } const pbIdx = this.es.playingKeys.indexOf(playback); if (pbIdx !== -1) { this.es.playingKeys.splice(pbIdx, 1); } }); if (playbackData.playbackLayers.Screen) { this.screenDispatchPrs.add(_playedPrProvider); } else { this.animDispatchPrs.push(_playedPrProvider()); } } }; } _generateFinalAnimAction(data) { return { timeMs: Utils_1.Utils.secToMillisec(this.lastAnimFrame.start), action: () => { this.animDispatchPrs.push(this.screenDispatchPrs.waitUntilEmpty()); this.animDispatchChainPr = this.animDispatchChainPr .then(() => { return Promise.all(this.animDispatchPrs); }) .then(() => { this.es.jibo.performance.log('EmbodiedSpeechAnimStopSkill', 'After last animation is finished'); this.animDone.resolve(data.speakOptions.dispatchComplete); }); } }; } play(playback, options) { this.es.playingKeys.push(playback); return playback.play(options); } } exports.BuildDispatchableState = BuildDispatchableState; },{"../../utils/Utils":65,"../processing/InputProcessor":19,"jibo-cai-utils":undefined,"jibo-state-machine":undefined}],55:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_state_machine_1 = require("jibo-state-machine"); class CompleteState extends jibo_state_machine_1.State { constructor(sm, es, name) { super(sm, name); this.es = es; this.onEntry = () => __awaiter(this, void 0, void 0, function* () { this.es.jibo.performance.log('EmbodiedSpeechCenterRobotStopSkill', 'After end-speech clean-up is finished'); this.es.log.info("Cleaning up session"); yield this.sm.stop(); }); } } exports.CompleteState = CompleteState; },{"jibo-state-machine":undefined}],56:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_state_machine_1 = require("jibo-state-machine"); const InputProcessor_1 = require("../processing/InputProcessor"); class CreateTimelineState extends jibo_state_machine_1.State { constructor(sm, es, name) { super(sm, name); this.es = es; this.onEntry = (transition, data) => __awaiter(this, void 0, void 0, function* () { let preInputTree; let nlParseTree; let autoTagTrees; this.es.jibo.performance.log('EmbodiedSpeechBuildingStartSkill', 'When ES starts building structures for dispatch'); this.es.log.debug("Generating pre-Input Tree"); preInputTree = yield this.es.inputProcessor.generatePreInputTree(data.textToSpeak); if (!data.speakOptions.disableAutoRules) { this.es.log.debug("Applying Timeline-modifying Auto Rules"); const timelineAutoRuleMatches = this.es.autoRuleManager.applyTimelineModifyingRules(preInputTree, data.autoRuleConfig); preInputTree = this.es.inputProcessor.insertTimelineAutoRules(preInputTree, timelineAutoRuleMatches); } this.es.log.debug("Cleaning input and generating NL Parse Tree"); const cleanTextArray = InputProcessor_1.InputProcessor.getPlainTextArrayFromTree(preInputTree); nlParseTree = yield this.es.nlParser.parse(cleanTextArray, preInputTree.expansions, preInputTree.contractions); if (!data.speakOptions.disableAutoRules) { this.es.log.debug("Applying standard Auto Rules"); autoTagTrees = this.es.autoRuleManager.applyRules(nlParseTree, data.autoRuleConfig); } this.es.log.debug("Generating Input Tree"); const inputTree = InputProcessor_1.InputProcessor.generateInputTree(preInputTree, nlParseTree); this.es.jibo.performance.log('EmbodiedSpeechBuildingTreesCompleteSkill', 'When ES completes building trees for timelines'); this.es.log.debug("Generating Timeline of TTS and Animations"); const timeline = yield this.es.timelineManager.generateTimeline(inputTree, data.speakOptions, autoTagTrees); this.es.jibo.performance.log('EmbodiedSpeechBuildingTimelinesCompleteSkill', 'When ES completes building timelines for dispatch'); data.timeline = timeline; return data; }); } } exports.CreateTimelineState = CreateTimelineState; },{"../processing/InputProcessor":19,"jibo-state-machine":undefined}],57:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const jibo_state_machine_1 = require("jibo-state-machine"); const jibo_cai_utils_1 = require("jibo-cai-utils"); const jibo_common_types_1 = require("jibo-common-types"); const Types_1 = require("../common/Types"); const Utils_1 = require("../../utils/Utils"); const TIMEOUT = 3000; const STOP_FAILED = 'STOP_FAILED'; class DispatchState extends jibo_state_machine_1.State { constructor(sm, es, name) { super(sm, name); this.es = es; let dataflowCache; let actionTimeline; let dispatched; let dispatchedPr; let attnModeHandle; let dispatchInterruptPr; let failsafeTimeoutHandle; let failsafeTriggered; let stopped; this.onEntry = (transition, data) => __awaiter(this, void 0, void 0, function* () { dataflowCache = data; attnModeHandle = null; dispatched = false; dispatchInterruptPr = new Utils_1.ExtPromise(); failsafeTriggered = false; stopped = false; this.es.animationDispatchPr = dataflowCache.animDonePr; const resolvedDispatchType = data.timeline.tts._root.data.dispatchOverride || dataflowCache.speakOptions.dispatchComplete; dispatchedPr = (resolvedDispatchType === Types_1.DispatchType.TTS) ? dataflowCache.ttsDonePr : Promise.all([dataflowCache.ttsDonePr, dataflowCache.animDonePr]); actionTimeline = new jibo_cai_utils_1.ActionTimeline(); this.es.playingKeys = []; let currAttnMode; if (!stopped) { this.es.log.info("Pushing Speaking Attention Mode"); currAttnMode = yield Utils_1.timeout(this.es.jibo.expression.getAttentionMode(), TIMEOUT, { log: this.es.log, logMessage: 'expression.getAttentionMode', timeoutValue: jibo_common_types_1.AttentionMode.OFF }); } if (!stopped) { if (currAttnMode !== jibo_common_types_1.AttentionMode.OFF) { attnModeHandle = yield Utils_1.timeout(this.es.jibo.expression.pushAttentionMode(jibo_common_types_1.AttentionMode.SPEAKING), TIMEOUT, { log: this.es.log, logMessage: 'pushAttentionMode(AttentionMode.SPEAKING)', }); } } if (!stopped) { const activeMode = this.es.ed.listen.getActiveMode(); if (!activeMode) { this.es.log.info("Waiting for Embodied Listen to finish"); yield Utils_1.timeout(this.es.ed.listen.waitForIdle(), TIMEOUT, { log: this.es.log, logMessage: 'embodiedListen.waitForIdle' }); } } if (!stopped) { this.es.jibo.performance.log('EmbodiedSpeechDispatchStartSkill', 'Before starting all ES TTS/ANIM dispatch'); this.es.log.info("Dispatching Timeline: ", dataflowCache.session); actionTimeline.dispatchTimeline(dataflowCache.dispatchableTimeline); dispatched = true; if (dataflowCache.timeline.anim.length > 0) { const FAILSAFE_TIMEOUT_MS = ((3 * Math.max(dataflowCache.timeline.tts.stop, dataflowCache.timeline.anim.stop)) * 1000); failsafeTimeoutHandle = this.es.jibo.timer.setTimeout(() => { if (this.es.active && (this.es.active.id === dataflowCache.session)) { this.es.log.warn("Timeline dispatch timeout reached -- triggering failsafe stopping of dispatch: ", dataflowCache.session); failsafeTriggered = true; this.es._stop(); } }, FAILSAFE_TIMEOUT_MS); } } yield Promise.race([dispatchedPr, dispatchInterruptPr.promise]); }); this.onStop = () => __awaiter(this, void 0, void 0, function* () { this.es.log.info("Timeline Dispatch Stopped"); stopped = true; let reset = failsafeTriggered; if (failsafeTimeoutHandle) { this.es.jibo.timer.clearTimeout(failsafeTimeoutHandle); } if (actionTimeline) { this.es.log.debug("Fast forwarding action timeline"); actionTimeline.fastForward(); } const stopPromises = []; if (this.es.playingKeys.length > 0) { this.es.log.debug("Stopping all in-progress keys"); this.es.playingKeys.forEach(keysToStop => { stopPromises.push(keysToStop.stop()); }); this.es.playingKeys = []; } try { this.es.log.debug("Awaiting all in progress keys being stopped"); const stopAllResult = yield Utils_1.timeout(Promise.all(stopPromises), TIMEOUT, { log: this.es.log, logMessage: 'embodiedSpeech.stop awaiting in progress keys stop', timeoutValue: STOP_FAILED }); if (typeof stopAllResult === 'string') { if (stopAllResult === STOP_FAILED) { this.es.log.debug('Stopping timeout encountered, attempting hard reset'); reset = true; } } } catch (error) { this.es.log.error(error); this.es.log.debug('Bad stop encountered, attempting hard reset'); reset = true; } this.es.log.debug("Stopping any active TTS output"); try { yield this.es.jibo.tts.stop(); } catch (error) { this.es.log.error(error); } if (dataflowCache.centeringEnabled) { this.es.log.debug("Centering the robot after speech"); const centerRobotConfig = { requestor: this.es.ownerInformation, dofs: this.es.dofsToCenter, centerGlobally: false }; try { const centerResult = yield Utils_1.timeout(this.es.jibo.expression.centerRobot(centerRobotConfig), TIMEOUT, { log: this.es.log, logMessage: 'embodiedSpeech.stop awaiting center robot', timeoutValue: STOP_FAILED }); if (typeof centerResult === 'string') { if (centerResult === STOP_FAILED) { this.es.log.debug('Stopping timeout encountered, attempting hard reset'); reset = true; } } } catch (error) { this.es.log.error(error); this.es.log.debug('Bad stop encountered, attempting hard reset'); reset = true; } } if (!reset && dispatched) { this.es.log.debug("Awaiting all dispatch promises completion"); try { const dispatchResult = yield Utils_1.timeout(dispatchedPr, TIMEOUT, { log: this.es.log, logMessage: 'embodiedSpeech.stop awaiting dispatch completion', timeoutValue: STOP_FAILED }); if (typeof dispatchResult === 'string') { if (dispatchResult === STOP_FAILED) { this.es.log.debug('Stopping timeout encountered, attempting hard reset'); reset = true; } } } catch (error) { this.es.log.error(error); this.es.log.debug('Bad stop encountered, attempting hard reset'); reset = true; } } actionTimeline = null; dispatchInterruptPr.resolve(); }); this.onExit = () => __awaiter(this, void 0, void 0, function* () { this.es.jibo.performance.log('EmbodiedSpeechDispatchStopSkill', 'After completing all ES TTS/ANIM dispatch'); this.es.log.info("Timeline Dispatch Complete: ", dataflowCache.session); if (failsafeTimeoutHandle) { this.es.jibo.timer.clearTimeout(failsafeTimeoutHandle); } if (attnModeHandle) { this.es.log.debug("Releasing Speaking AttentionMode Handle"); try { yield Utils_1.timeout(attnModeHandle.release(), TIMEOUT, { log: this.es.log, logMessage: 'attnModeHandle.release()', }); } catch (error) { this.es.log.warn(error); } } if (actionTimeline) { this.es.log.debug("Fast forwarding action timeline"); actionTimeline.fastForward(); } if ((dataflowCache.timeline.anim.length > 0) && (dataflowCache.centeringEnabled)) { this.es.log.debug("Centering the robot after speech"); try { const centerRobotConfig = { requestor: this.es.ownerInformation, dofs: this.es.dofsToCenter, centerGlobally: false }; yield Utils_1.timeout(this.es.jibo.expression.centerRobot(centerRobotConfig), TIMEOUT, { log: this.es.log, logMessage: 'embodiedSpeech awaiting center robot' }); } catch (error) { this.es.log.warn(error); } this.es.jibo.performance.log('EmbodiedSpeechCenterRobotStopSkill', 'After automatic CenterRobot is finished'); } this.es.log.info("Speak Complete: ", dataflowCache.session); }); } } exports.DispatchState = DispatchState; },{"../../utils/Utils":65,"../common/Types":13,"jibo-cai-utils":undefined,"jibo-common-types":undefined,"jibo-state-machine":undefined}],58:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const CreateTimeline_1 = require("./CreateTimeline"); exports.CreateTimelineState = CreateTimeline_1.CreateTimelineState; const BuildDispatchable_1 = require("./BuildDispatchable"); exports.BuildDispatchableState = BuildDispatchable_1.BuildDispatchableState; const Dispatch_1 = require("./Dispatch"); exports.DispatchState = Dispatch_1.DispatchState; const Complete_1 = require("./Complete"); exports.CompleteState = Complete_1.CompleteState; },{"./BuildDispatchable":54,"./Complete":55,"./CreateTimeline":56,"./Dispatch":57}],59:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const Types_1 = require("../common/Types"); exports.TimelineLayerType = { ANIM: Types_1.AssetNodeType.ANIM, TTS: Types_1.TTSNodeType.TTS }; class TimelineFrame { constructor(start, stop, data = undefined, blocking = false, prev = null, next = null) { this.start = start; this.stop = stop; this.data = data; this.blocking = blocking; this.prev = prev; this.next = next; } } exports.TimelineFrame = TimelineFrame; class TimelineLayer { constructor(name, _timeline, _frameBuffer, _assetRelativeEnd, _enableMerging = false) { this.name = name; this._timeline = _timeline; this._frameBuffer = _frameBuffer; this._assetRelativeEnd = _assetRelativeEnd; this._enableMerging = _enableMerging; this._root = null; this.length = 0; } insert(pendingFrame, withBuffer = false, layerMergeOverride) { const pendingLayerData = pendingFrame.data.playbackData.playbackLayers; if ((this._assetRelativeEnd !== undefined) && ((pendingFrame.stop > (this._timeline.tts.stop + this._assetRelativeEnd))) && !pendingLayerData.Audio) { return false; } const pendingFrameLayersAsInt = TimelineLayer._metaLayerToInt(pendingLayerData); const conflicts = this._gatherPotentialConflicts(pendingFrame, withBuffer); const allowMerging = (layerMergeOverride !== undefined) ? layerMergeOverride : this._enableMerging; let insert = (conflicts.length === 0); if (conflicts.length !== 0 && allowMerging) { insert = conflicts.every(frame => { let conflictLayerData = frame.data.playbackData.playbackLayers; let layersAsInt = TimelineLayer._metaLayerToInt(conflictLayerData); let inConflict = layersAsInt & pendingFrameLayersAsInt; return (inConflict === 0); }); } if (insert) { if ((this.start === undefined) || (pendingFrame.start < this.start)) { this.start = pendingFrame.start; } if ((this.stop === undefined) || (pendingFrame.stop > this.stop)) { this.stop = pendingFrame.stop; } this._insert(pendingFrame); } return insert; } static _metaLayerToInt(metaLayerPresence) { const layerBits = [ Number(metaLayerPresence.Body), Number(metaLayerPresence.Screen), Number(metaLayerPresence.Audio) ]; return parseInt(layerBits.join(''), 2); } _gatherPotentialConflicts(pendingFrame, useBuffer) { let currentFrame; let potentialConflicts = []; if (!this._root) { return potentialConflicts; } else { currentFrame = this._root; while (currentFrame) { const dynamicBuffer = (useBuffer ? this._frameBuffer : 0); const currentFrameStart = Math.round((currentFrame.start - dynamicBuffer) * 1000) / 1000; const currentFrameStop = Math.round((currentFrame.stop + dynamicBuffer) * 1000) / 1000; const pendingFrameStart = Math.round((pendingFrame.start - dynamicBuffer) * 1000) / 1000; const pendingFrameStop = Math.round((pendingFrame.stop + dynamicBuffer) * 1000) / 1000; if (pendingFrameStart < currentFrameStop && pendingFrameStop > currentFrameStart) { potentialConflicts.push(currentFrame); } currentFrame = currentFrame.next; } } return potentialConflicts; } _insert(pendingFrame) { let currentFrame; let previousFrame; if (!this._root) { this._root = pendingFrame; } else { currentFrame = this._root; while (currentFrame) { if (pendingFrame.start <= currentFrame.start) { pendingFrame.next = currentFrame; pendingFrame.prev = currentFrame.prev; if (currentFrame.prev) { currentFrame.prev.next = pendingFrame; } currentFrame.prev = pendingFrame; if (currentFrame === this._root) { this._root = pendingFrame; } this.length++; return; } previousFrame = currentFrame; currentFrame = currentFrame.next; } pendingFrame.prev = previousFrame; previousFrame.next = pendingFrame; } this.length++; return; } } exports.TimelineLayer = TimelineLayer; class Timeline { constructor(frameBuffer, assetRelativeEnd, enableLayerMerging = false) { this.tts = new TimelineLayer(exports.TimelineLayerType.TTS, this, frameBuffer, assetRelativeEnd); this.anim = new TimelineLayer(exports.TimelineLayerType.ANIM, this, frameBuffer, assetRelativeEnd, enableLayerMerging); } _init(ttsFrames) { ttsFrames.forEach(frame => { if ((this.tts.start === undefined) || (frame.start < this.tts.start)) { this.tts.start = frame.start; } if ((this.tts.stop === undefined) || (frame.stop > this.tts.stop)) { this.tts.stop = frame.stop; } this.tts._insert(new TimelineFrame(frame.start, frame.stop, frame.data)); }); } } exports.Timeline = Timeline; },{"../common/Types":13}],60:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const _ = require("lodash/lodash.min"); const path = require("path"); const jibo_cai_utils_1 = require("jibo-cai-utils"); const Timeline_1 = require("./Timeline"); const InputProcessor_1 = require("../processing/InputProcessor"); const Timeline_2 = require("./Timeline"); const SSMLNode_1 = require("../processing/SSMLNode"); const Formats_1 = require("../processing/Formats"); const AutoRuleManager_1 = require("../processing/auto_rules/AutoRuleManager"); const Types_1 = require("../common/Types"); const log_1 = require("../../log"); const log = log_1.speechLog.createChild("TimelineManager"); const TTS_PAU = '[lpau]'; const TTS_BREAK = ''; const TTS_AUDIO_BREAK = ''; const TTS_SAY_AS = ''; const FRAME_RATE = 30.0; const AUTORULE_GATE_RATE = 1.0; const AUTORULE_ORIENTATION_PROBABILITY = 0.5; class AssetResolution { constructor(resolved = false, identifier = null, reason = null) { this.resolved = resolved; this.identifier = identifier; this.reason = reason; } } exports.AssetResolution = AssetResolution; class TimelineManager { constructor(jibo, _resourceRoot, _animResourceRoot, _audioResourceRoot) { this.jibo = jibo; this._resourceRoot = _resourceRoot; this._animResourceRoot = _animResourceRoot; this._audioResourceRoot = _audioResourceRoot; this._tempAnimationCollection = new this.jibo.animDB.AnimCollection(this.jibo.animDB._animDB); } generateTimeline(inputTree, options, autoTagTrees) { this.emojiCapReached = false; return this._preProcessPathBasedAssets(inputTree) .then(pathAssets => { this._pathAssets = pathAssets; return this._inputTreeToTimeline(inputTree, options); }) .then(timelineData => { const [timeline, wordSchedule] = timelineData; if (!autoTagTrees) { return Promise.resolve(timeline); } else { let pending = this._autoTagTreesToPrioritizedTimelines(autoTagTrees, wordSchedule, timeline); return this._mergePendingWithMaster(pending, timeline); } }); } _preProcessPathBasedAssets(inputTree) { return __awaiter(this, void 0, void 0, function* () { let pathAssetsPr = []; let pathAssetMap = new Map(); let cachedAssetMap = new Map(); let pathAssets = inputTree.gatherDescendants(node => { return InputProcessor_1.InputProcessor.isAsset(node) && node.att.has('path'); }); pathAssets.forEach(node => { let identifier = node.getAttribute('path'); if (node.type === Types_1.AssetNodeType.ES) { const animPath = path.join(this._animResourceRoot, identifier); pathAssetsPr.push(new Promise(resolve => { this.jibo.animDB.AnimDBParser.parseAnimation(this.jibo, animPath, this._resourceRoot) .then(resolve) .catch(e => { resolve(); }); })); } else if (node.type === Types_1.AssetNodeType.AUDIO) { const audioPath = this.jibo.utils.PathUtils.getAssetUri(identifier, undefined, this._audioResourceRoot); pathAssetsPr.push(new Promise(resolve => { identifier = path.join('audio', identifier); this.jibo.loader.load({ type: 'sound', id: identifier, src: audioPath, complete: (err, loadedSound) => { if (err) { resolve(); } else { resolve(this._getSoundMetadata(loadedSound, audioPath, node, cachedAssetMap)); } } }); })); } }); try { const metaDatum = yield Promise.all(pathAssetsPr); if (pathAssets.length === metaDatum.length) { let nodeAnimMetaZip = _.zip(pathAssets, metaDatum); nodeAnimMetaZip.forEach(([node, metadata]) => { if (metadata) { if (metadata.path) { metadata.path = path.relative(this._resourceRoot, metadata.path); } let cached = cachedAssetMap.get(node); pathAssetMap.set(node, new this.jibo.animDB.Animation(this._tempAnimationCollection, metadata, this._resourceRoot, cached)); } }); } else { log.warn('Missing data about local (path-based) animation(s) or audio; unable to utilize in output.'); } } catch (e) { log.warn(e); } return pathAssetMap; }); } _getSoundMetadata(loadedSound, audioPath, node, cachedAssetMap) { let duration = Math.ceil(loadedSound.buffer.duration * FRAME_RATE); if (duration > 0) { let relAudioPath = path.relative(this._audioResourceRoot, audioPath); let anim = new Formats_1.AudioAnimTemplate(duration, relAudioPath); cachedAssetMap.set(node, anim); try { return this.jibo.animDB.AnimDBParser.parseAnimationFromAnim(anim); } catch (e) { return; } } return; } _inputTreeToTimeline(inputTree, options) { return __awaiter(this, void 0, void 0, function* () { const blockingNodes = inputTree.gatherDescendants(node => { return InputProcessor_1.InputProcessor.isBlocking(node); }); blockingNodes.forEach((node) => { let resolution = this._resolveAssetToPlayback(node); if (!resolution.resolved) { log.debug(`Could not find match for requested Asset ${resolution.identifier} because ${resolution.reason}`); } }); let wordScheduleRelevantNodes = []; let nonBlockers = []; let noWords = true; inputTree.gatherDescendants().forEach(node => { if (InputProcessor_1.InputProcessor.isUnboundedNonBlocking(node)) { nonBlockers.push(node); } else if (InputProcessor_1.InputProcessor.isBlocking(node)) { const blockingNode = node; if (blockingNode.playbackData) { const newBreak = TimelineManager._buildBreak(blockingNode.playbackData.playbackDuration / FRAME_RATE); wordScheduleRelevantNodes.push(newBreak); blockingNode.timeSyncNode = newBreak; nonBlockers.forEach(nBNode => nBNode.timeSyncNode = newBreak); nonBlockers = []; } } else { if (!InputProcessor_1.InputProcessor.isAsset(node)) { if (!InputProcessor_1.InputProcessor.isTimelessSSML(node)) { nonBlockers.forEach(nBNode => nBNode.timeSyncNode = node); nonBlockers = []; } wordScheduleRelevantNodes.push(node); if (noWords && (node.type === Types_1.NLPNodeType.WORD || InputProcessor_1.InputProcessor.isSayAs(node))) { noWords = false; } } } }); let dispatchOverride; if (options.followedByListen) { dispatchOverride = noWords ? Types_1.DispatchType.ALL : Types_1.DispatchType.TTS; } const fullSSMLText = InputProcessor_1.InputProcessor.getSSMLTextFromArray(wordScheduleRelevantNodes); const shouldPrune = ((options.dispatchComplete === Types_1.DispatchType.TTS) && (dispatchOverride !== Types_1.DispatchType.ALL)); const timingSSMLText = `${fullSSMLText}`; const speakingSSMLText = noWords ? null : timingSSMLText; const cached = timingSSMLText === speakingSSMLText; if (options.debugMode) { console.log(`TIMING:\n ${timingSSMLText}`); console.log(`SPEAKING:\n ${speakingSSMLText}`); console.log('-----------'); } const params = { locale: this.jibo.tts.TTSLocale.EN_US, voice: this.jibo.tts.TTSVoice.GRIFFIN, mode: this.jibo.tts.TTSMode.SSML, skipWordEvents: this.jibo.tts.TTSBoolean.TRUE, cached: cached ? this.jibo.tts.TTSBoolean.TRUE : this.jibo.tts.TTSBoolean.FALSE }; const ttsSpeakData = { timingSSMLText: timingSSMLText, speakingSSMLText: speakingSSMLText, params: params, dispatchOverride: dispatchOverride }; const ttsTiming = yield this.jibo.tts.getWordTimings(timingSSMLText, params); let [timeline, mergedTokenTimes] = TimelineManager._initTimeline(ttsSpeakData, ttsTiming.tokentimes.tokens, options, shouldPrune); const wordSchedule = TimelineManager._generateWordSchedule(wordScheduleRelevantNodes, mergedTokenTimes, options); timeline = this._populateTimelineLayers(inputTree, timeline, wordSchedule); return [timeline, wordSchedule]; }); } static _buildBreak(size) { let newBreak = new SSMLNode_1.SSMLNode(Types_1.SSMLNodeType.BREAK); const roundedSize = Math.round(size * 1000) / 1000; newBreak.setAttribute('size', roundedSize); return newBreak; } _resolveAssetToPlayback(node, durationSeconds, randomizeOrient, cache) { const requestDescription = node.requestInfo.description; let resolution = new AssetResolution(); let identifier; let asset; let durationFramesToFill; let durationFramesLoopedPlayback; let durationFramesSinglePlayback; let playbackLoops = node.loops; if (durationSeconds !== undefined) { durationFramesToFill = durationSeconds * FRAME_RATE; if ((playbackLoops !== undefined) && (playbackLoops > 0)) { log.warn(`Requested invalid loops '${playbackLoops}' for time-bounded playback.`); playbackLoops = undefined; } } if (node.requestInfo.type === Types_1.RequestType.NAME) { identifier = node.att.get('name'); if (identifier) { this.jibo.performance.log('EmbodiedSpeech_resolveAssetToPlayback_namedAnimationAsset', requestDescription); resolution.identifier = requestDescription; asset = this.jibo.animDB.getAnimByName(identifier); } if (!asset) { resolution.reason = 'animation could not be found in the AnimationDB'; return resolution; } else if (durationFramesToFill) { if (playbackLoops === 0) { let choice; if (asset.meta.duration < durationFramesToFill) { choice = TimelineManager.findOptimalLoop([asset], durationFramesToFill); } else if (asset.durationRange.min < durationFramesToFill) { choice = TimelineManager.findOptimalLoop([asset], durationFramesToFill, true); } else { resolution.reason = 'animation cannot fit in the requested time bounds'; return resolution; } asset = choice.asset; durationFramesLoopedPlayback = choice.duration; playbackLoops = choice.loops; } else if (asset.durationRange.min <= durationFramesToFill) { durationFramesSinglePlayback = Math.max(asset.durationRange.min, Math.min(durationFramesToFill, asset.durationRange.max)); } else { resolution.reason = 'animation cannot fit in the requested time bounds'; return resolution; } } } else if (node.requestInfo.type === Types_1.RequestType.QUERY) { this.jibo.performance.log('EmbodiedSpeech_resolveAssetToPlayback_queryAnimationAsset', requestDescription); const query = { categories: (node.categoryFilter.someOf.length > 0) ? node.categoryFilter.someOf : node.categoryFilter.allOf, includeMeta: node.metaFilter.allOf, includeSomeMeta: node.metaFilter.someOf, excludeMeta: node.metaFilter.noneOf, }; const results = this.jibo.animDB.query(query); if (results.matching.length > 0) { if (durationFramesToFill) { if (playbackLoops === 0) { let choice; const loopableResults = results.matching.filter(result => { return result.meta.duration < durationFramesToFill; }); if (loopableResults.length > 0) { choice = TimelineManager.findOptimalLoop(loopableResults, durationFramesToFill); } else { const minResults = results.matching.filter(result => { return result.durationRange.min < durationFramesToFill; }); if (minResults.length > 0) { choice = TimelineManager.findOptimalLoop(loopableResults, durationFramesToFill, true); } } if (choice) { asset = choice.asset; durationFramesLoopedPlayback = choice.duration; playbackLoops = choice.loops; } } else if (playbackLoops === undefined) { const goodFitResults = results.matching.filter(result => { return (result.durationRange.distanceFrom(durationFramesToFill) === 0); }); if (goodFitResults.length > 0) { asset = _.sample(goodFitResults); durationFramesSinglePlayback = durationFramesToFill; } else { const validResults = results.matching.filter(result => { return (result.durationRange.min < durationFramesToFill); }); if (validResults.length > 0) { validResults.sort((a, b) => { return a.durationRange.max - b.durationRange.max; }); asset = results.matching[0]; durationFramesSinglePlayback = Math.min(durationFramesToFill, asset.durationRange.max); } } } } else { asset = _.sample(results.matching); } } if (!asset) { resolution.identifier = requestDescription; resolution.reason = 'no animation could be found in DB that matched requested attribute (i.e. CAT, META, DURATION) combination'; return resolution; } } else if (node.requestInfo.type === Types_1.RequestType.PATH) { identifier = node.att.get('path'); this.jibo.performance.log('EmbodiedSpeech_resolveAssetToPlayback_pathAnimationAsset', requestDescription); asset = this._pathAssets.get(node); if (!asset) { resolution.identifier = requestDescription; resolution.reason = 'requested path based asset could not be loaded'; return resolution; } } else { resolution.identifier = requestDescription; resolution.reason = 'no animation could be found in DB (asset tag was malformed)'; return resolution; } const [metaLayerPresence, muteConfig] = TimelineManager._createLayerPresences(asset, node); if (!cache) { cache = this.jibo.face.eye.CACHE_ID; } let animConfig = { cache: cache }; if (muteConfig) { animConfig.mutes = muteConfig; } const reqOrient = node.att.get('orientation'); if (reqOrient) { animConfig.orientation = this.jibo.animDB.Orientation[reqOrient]; } else if (randomizeOrient) { animConfig.orientation = (Math.random() > AUTORULE_ORIENTATION_PROBABILITY) ? this.jibo.animDB.Orientation.LEFT : this.jibo.animDB.Orientation.RIGHT; } if (durationFramesToFill) { if (playbackLoops) { animConfig.duration = durationFramesLoopedPlayback; animConfig.loops = playbackLoops; } else { animConfig.duration = durationFramesSinglePlayback; } } else { if (playbackLoops) { durationFramesToFill = asset.meta.duration * playbackLoops; animConfig.loops = playbackLoops; } else { durationFramesToFill = asset.meta.duration; } } node.playbackData = { playbackGenerator: () => asset.createFromConfig(animConfig), playbackDuration: durationFramesToFill, playbackName: asset.name, playbackLayers: metaLayerPresence }; resolution.resolved = true; resolution.identifier = identifier; return resolution; } static findOptimalLoop(candidates, durationFramesToFill, allowMin = false) { let optimalResult = { asset: undefined, duration: undefined, loops: undefined }; let smallestError = Infinity; candidates.forEach(candidate => { const candidateDuration = allowMin ? candidate.durationRange.min : candidate.meta.duration; const loops = Math.floor(durationFramesToFill / candidateDuration); const remainder = durationFramesToFill % candidateDuration; const adjustment = Math.floor(remainder / loops); const optimalDuration = Math.min(candidateDuration + adjustment, candidate.durationRange.max); const error = durationFramesToFill - (optimalDuration * loops); if (error < smallestError) { smallestError = error; optimalResult.asset = candidate; optimalResult.duration = optimalDuration; optimalResult.loops = loops; } }); return optimalResult; } static _createLayerPresences(anim, node) { const runtimeLayers = node.layers; const runtimeLayersToAdd = node.layersToAdd; let muteConfig; let metaLayerPresence = { Body: false, Screen: false, Audio: false }; if (anim.meta.layers.Body) { metaLayerPresence.Body = true; } if (anim.meta.layers.AudioEvent) { metaLayerPresence.Audio = true; } if (anim.meta.layers.Eye || anim.meta.layers.EyeTexture || anim.meta.layers.Overlay || anim.meta.layers.OverlayTexture || anim.meta.layers.Pixi || anim.meta.layers.BackgroundTexture) { metaLayerPresence.Screen = true; } if (runtimeLayers) { muteConfig = { BODY: runtimeLayers.Body, SCREEN: runtimeLayers.Screen, AUDIO: runtimeLayers.Audio }; if (runtimeLayers.Body !== undefined) { if (runtimeLayersToAdd.Body || ((runtimeLayers.Body && !metaLayerPresence.Body) || (!runtimeLayers.Body && metaLayerPresence.Body))) { metaLayerPresence.Body = runtimeLayers.Body; } } if (runtimeLayers.Audio !== undefined) { if (runtimeLayersToAdd.Audio || ((runtimeLayers.Audio && !metaLayerPresence.Audio) || (!runtimeLayers.Audio && metaLayerPresence.Audio))) { metaLayerPresence.Audio = runtimeLayers.Audio; } } if (runtimeLayers.Screen !== undefined) { if (runtimeLayersToAdd.Screen || ((runtimeLayers.Screen && !metaLayerPresence.Screen) || (!runtimeLayers.Screen && metaLayerPresence.Screen))) { metaLayerPresence.Screen = runtimeLayers.Screen; } } } return [metaLayerPresence, muteConfig]; } static _initTimeline(ttsSpeakData, tokenTimes, options, shouldPrune = false) { let timeline = new Timeline_1.Timeline(options.frameBuffer, options.assetRelativeEnd, options.enableLayerMerging); const start = tokenTimes[0].start; let stop = tokenTimes[tokenTimes.length - 1].end; if (shouldPrune) { stop = this.getPrunedStopTime(tokenTimes); ttsSpeakData.params.earlyStopTime = stop; } const ttsFrame = { start: start, stop: stop, data: ttsSpeakData }; timeline._init([ttsFrame]); return [timeline, tokenTimes]; } static getPrunedStopTime(tokenTimes) { let idx; let pauseTime = 0; for (idx = tokenTimes.length - 1; idx >= 0; idx--) { if (tokenTimes[idx].name === TTS_BREAK) { continue; } else if ((idx === tokenTimes.length - 1) && (tokenTimes[idx].name === TTS_PAU)) { pauseTime = (tokenTimes[idx].end - tokenTimes[idx].start); } else { break; } } if (idx < 0) { return tokenTimes[0].start + pauseTime; } return tokenTimes[idx].end + pauseTime; } static _generateWordSchedule(wordNodes, tokenTimes, options) { if (options.debugMode) { console.log('Word timing report (with buffer):'); console.log('--------------------------------'); } let ttsWordSchedule = new Map(); let lastNodeIdx = 0; for (let i = 0; i < tokenTimes.length; i++) { let wordNodeFound = false; const token = tokenTimes[i]; const tokenTime = { start: token.start, stop: token.end }; for (let j = lastNodeIdx; j < wordNodes.length; j++) { const word = wordNodes[j]; if (token.name === TTS_BREAK || token.name === TTS_SAY_AS || token.name === TTS_AUDIO_BREAK) { if (((token.name === TTS_BREAK) && (word.type === Types_1.SSMLNodeType.BREAK)) || ((token.name === TTS_AUDIO_BREAK) && (word.type === Types_1.SSMLNodeType.AUDIO_BREAK)) || ((token.name === TTS_SAY_AS) && (word.type === Types_1.SSMLNodeType.SAY_AS))) { ttsWordSchedule.set(word, tokenTime); lastNodeIdx = ++j; wordNodeFound = true; if (options.debugMode) { console.log(`${token.name}: ${token.start.toFixed(3)} - ${token.end.toFixed(3)} (${(token.end - token.start).toFixed(3)})`); } break; } } else if (word.value && (token.name.toLowerCase() === word.value.toLowerCase())) { ttsWordSchedule.set(word, tokenTime); if (options.debugMode) { console.log(`${word.value}: ${token.start.toFixed(3)} - ${token.end.toFixed(3)} (${(token.end - token.start).toFixed(3)})`); } lastNodeIdx = ++j; wordNodeFound = true; break; } } if ((token.name !== TTS_PAU) && (!wordNodeFound)) { if (options.debugMode) { console.log(`Unable to align TTS token: ${token.name} with requested words`); } } } if (options.debugMode) { console.log('-------------------'); } return ttsWordSchedule; } _populateTimelineLayers(inputTree, timeline, wordSchedule) { let assetNodes = inputTree.gatherDescendantsType(Types_1.AssetNodeType.ES); assetNodes = assetNodes.concat(inputTree.gatherDescendantsType(Types_1.AssetNodeType.AUDIO)); let blockingAssetNodes = assetNodes.filter((node) => InputProcessor_1.InputProcessor.isBlocking(node)); let emptyAssetNodes = blockingAssetNodes.concat(assetNodes.filter((node) => (InputProcessor_1.InputProcessor.isEmpty(node) && !InputProcessor_1.InputProcessor.isBlocking(node)))); let prioritizedAssetNodes = emptyAssetNodes.concat(assetNodes.filter((node) => !InputProcessor_1.InputProcessor.isEmpty(node))); log.debug('ESML resolution report:'); log.debug('-----------------------'); prioritizedAssetNodes.forEach((node) => { let bounds; let blocking = false; if (InputProcessor_1.InputProcessor.isBlocking(node)) { blocking = true; if ((node.playbackData) && (wordSchedule.has(node.timeSyncNode))) { const timeSyncNode = node.timeSyncNode; bounds = Object.assign({}, wordSchedule.get(timeSyncNode)); } else { log.debug(`Could not resolve time-sync information for blocking asset request: ${node.requestInfo.description}`); return; } } else { let resolution; if (InputProcessor_1.InputProcessor.isEmpty(node)) { if (wordSchedule.has(node.timeSyncNode)) { let timeSyncNode = node.timeSyncNode; bounds = Object.assign({}, wordSchedule.get(timeSyncNode)); } else { log.debug(`Could not resolve time-sync information for non-blocking asset request: ${node.requestInfo.description}`); return; } resolution = this._resolveAssetToPlayback(node); } else { try { bounds = TimelineManager._getBoundariesFromChildren(node, wordSchedule); } catch (e) { log.warn(e); return; } resolution = this._resolveAssetToPlayback(node, (bounds.stop - bounds.start)); } if (!resolution.resolved) { log.debug(`Could not find match for requested Asset ${resolution.identifier} because ${resolution.reason}`); return; } } const dynamicStopBoundary = bounds.start + (node.playbackData.playbackDuration / FRAME_RATE); const inserted = timeline.anim.insert(new Timeline_2.TimelineFrame(bounds.start, dynamicStopBoundary, node, blocking)); if (inserted) { node.playbackData.playbackPr = node.playbackData.playbackGenerator(); if (node.playbackData.playbackName.toLowerCase().indexOf('emoji') !== -1) { log.info(`Emoji ${node.playbackData.playbackName} triggered via ESML`); this.emojiCapReached = true; } log.debug(`Inserted asset ${node.playbackData.playbackName} in Timeline @ ${bounds.start.toFixed(3)} - ${dynamicStopBoundary.toFixed(3)}`); if (node.playbackData.playbackName.includes('SSA')) { this.jibo.performance.log('EmbodiedSpeech_ESML_ToBeDispatchedSSA', node.playbackData.playbackName); } else { this.jibo.performance.log('EmbodiedSpeech_ESML_ToBeDispatchedAnim', node.playbackData.playbackName); } } else { log.debug(`Could not insert asset ${node.playbackData.playbackName} in Timeline because it conflicts with other content @ ${bounds.start.toFixed(3)} - ${dynamicStopBoundary.toFixed(3)}`); } }); log.debug('-----------------------'); return timeline; } _autoTagTreesToPrioritizedTimelines(autoTagTrees, wordSchedule, timeline) { let priorityMap = new Map(); autoTagTrees.forEach(ruleTree => { let cache; const autoRule = ruleTree.getAttribute('autoRule'); switch (autoRule) { case AutoRuleManager_1.AutoRuleNames.Question: case AutoRuleManager_1.AutoRuleNames.Comma: case AutoRuleManager_1.AutoRuleNames.Initiate: case AutoRuleManager_1.AutoRuleNames.Blink: case AutoRuleManager_1.AutoRuleNames.DoubleBlink: case AutoRuleManager_1.AutoRuleNames.BeatBlink: cache = jibo_cai_utils_1.CacheUtils.GlobalCacheName; break; case AutoRuleManager_1.AutoRuleNames.HotWords: default: cache = this.jibo.face.eye.CACHE_ID; break; } if (ruleTree.children.length > 0) { const priority = ruleTree.getAttribute('priority'); let ruleFrames = []; ruleTree.forEachChild((subTree) => { let bounds; let resolution; if (InputProcessor_1.InputProcessor.isEmpty(subTree) || InputProcessor_1.InputProcessor.isExplicitlyNonBlocking(subTree)) { if (!wordSchedule.has(subTree.timeSyncNode)) { return; } const timeSyncNode = subTree.timeSyncNode; bounds = Object.assign({}, wordSchedule.get(timeSyncNode)); const position = subTree.getAttribute('position'); bounds.start = (position === 'after') ? bounds.stop : bounds.start; resolution = this._resolveAssetToPlayback(subTree, undefined, true, cache); } else { try { bounds = TimelineManager._getBoundariesFromChildren(subTree, wordSchedule); } catch (e) { log.warn(e); return; } resolution = this._resolveAssetToPlayback(subTree, (bounds.stop - bounds.start), true, cache); if (!resolution.resolved && (Math.random() < AUTORULE_GATE_RATE)) { resolution = this._resolveAssetToPlayback(subTree, undefined, true, cache); } } if (!resolution.resolved) { log.debug(`Could not find match for AutoRule ${resolution.identifier} because ${resolution.reason}`); return; } const dynamicStopBoundary = bounds.start + (subTree.playbackData.playbackDuration / FRAME_RATE); ruleFrames.push({ layer: timeline.anim, frame: new Timeline_2.TimelineFrame(bounds.start, dynamicStopBoundary, subTree) }); }); if (ruleFrames.length > 0) { let currMatches = priorityMap.get(priority); if (!currMatches) { priorityMap.set(priority, currMatches = []); } currMatches.push(ruleFrames); } } }); return priorityMap; } static _getBoundariesFromChildren(node, wordSchedule) { let firstWordBounds; let lastWordBounds; node.forEachChild(child => { let wordBounds = wordSchedule.get(child); if (!wordBounds && (InputProcessor_1.InputProcessor.isAsset(child) || InputProcessor_1.InputProcessor.isBreak(child))) { wordBounds = wordSchedule.get(child.timeSyncNode); } else if (!InputProcessor_1.InputProcessor.isEmpty(child)) { wordBounds = TimelineManager._getBoundariesFromChildren(child, wordSchedule); } if (wordBounds) { if (!firstWordBounds) { firstWordBounds = wordBounds; } else { lastWordBounds = wordBounds; } } }); if (firstWordBounds) { lastWordBounds = lastWordBounds || firstWordBounds; } else { throw new Error('Unable to find timing bounds for word length based Animation.'); } const timeBounds = { start: firstWordBounds.start, stop: lastWordBounds.stop }; return timeBounds; } _mergePendingWithMaster(pending, timeline) { let priorities = []; pending.forEach((_val, key) => priorities.push(key)); priorities.sort(); log.debug('AutoRules resolution report:'); log.debug('---------------------------------------'); priorities.forEach(priority => { _.shuffle(pending.get(priority)).forEach((autoRule) => { autoRule.forEach((ruleMatch) => { const isEmoji = (ruleMatch.frame.data.playbackData.playbackName.toLowerCase().indexOf('emoji') !== -1); if (isEmoji && this.emojiCapReached) { log.debug(`Could not insert AutoRule ${ruleMatch.frame.data.getAttribute('rule')}:${ruleMatch.frame.data.playbackData.playbackName} in Timeline because only 1 Emoji is allowed per input`); return; } const inserted = ruleMatch.layer.insert(ruleMatch.frame, true); if (inserted) { ruleMatch.frame.data.playbackData.playbackPr = ruleMatch.frame.data.playbackData.playbackGenerator(); if (isEmoji) { log.info(`Emoji ${ruleMatch.frame.data.playbackData.playbackName} triggered via AutoRule`); this.emojiCapReached = true; } log.debug(`AutoRule ${ruleMatch.frame.data.getAttribute('rule')}:${ruleMatch.frame.data.playbackData.playbackName} inserted @ ${ruleMatch.frame.start.toFixed(3)} - ${ruleMatch.frame.stop.toFixed(3)}`); this.jibo.performance.log('EmbodiedSpeech_AutoRule_ToBeDispatchedAnim', ruleMatch.frame.data.playbackData.playbackName); } else { log.debug(`Could not insert AutoRule ${ruleMatch.frame.data.getAttribute('rule')}:${ruleMatch.frame.data.playbackData.playbackName} in Timeline because it conflicts with other content @ ${ruleMatch.frame.start.toFixed(3)} - ${ruleMatch.frame.stop.toFixed(3)}`); } }); }); }); log.debug('---------------------------------------'); return timeline; } } exports.TimelineManager = TimelineManager; },{"../../log":11,"../common/Types":13,"../processing/Formats":18,"../processing/InputProcessor":19,"../processing/SSMLNode":20,"../processing/auto_rules/AutoRuleManager":23,"./Timeline":59,"jibo-cai-utils":undefined,"lodash/lodash.min":undefined,"path":undefined}],61:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); __export(require("./Timeline")); __export(require("./TimelineManager")); },{"./Timeline":59,"./TimelineManager":60}],62:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const TIME_PATTERN = /(?:[0-9]|0[0-9]|1[0-9]|2[0-3]):(?:[0-5][0-9])\s?(?:[aApP][mM])/ig; const uniqueContractionRules = []; uniqueContractionRules.push([/ma'am/gi, `madam`]); uniqueContractionRules.push([/o'clock/gi, `of the clock`]); const doubleContractionsRules = []; doubleContractionsRules.push([/(y)('all)/gi, `$1ou$2`]); doubleContractionsRules.push([/(sha)(n't)/gi, `shall$2`]); doubleContractionsRules.push([/(ai)(n't)/gi, `am$2`]); doubleContractionsRules.push([/(aim)(n't)/gi, `am$2`]); doubleContractionsRules.push([/(ca)(n't)/gi, `can$2`]); doubleContractionsRules.push([/(wo)(n't)/gi, `will$2`]); const contractionRules = []; contractionRules.push([/(let)'s/gi, `$1 us`]); contractionRules.push([/'s/gi, ` is`]); contractionRules.push([/n't/gi, ` not`]); contractionRules.push([/'ve/gi, ` have`]); contractionRules.push([/'re/gi, ` are`]); contractionRules.push([/'ll/gi, ` will`]); contractionRules.push([/'m/gi, ` am`]); contractionRules.push([/'d/gi, ` would`]); contractionRules.push([/'all/gi, ` all`]); class NLPTools { static isPunctuation(str) { if (!str) { return false; } return str.length === 1 && (str[0] === '.' || str[0] === '...' || str[0] === ',' || str[0] === ':' || str[0] === ';' || str[0] === '!' || str[0] === '-' || str[0] === '?'); } static getExpansionsAndContractions(text) { let splext = text.split(' '); let textExpansions = []; let textContractions = []; splext.forEach(token => { let contracted = NLPTools.applyContractionRules(token); let expanded = contracted.split(' '); let baseWordSet = false; expanded.forEach(word => { if (!baseWordSet) { textExpansions.push(word); if (word !== token) { textContractions.push(token.toLowerCase()); } else { textContractions.push(token); } baseWordSet = true; } else { textExpansions.push(word); textContractions.push(''); } }); }); return [textExpansions, textContractions]; } static applyContractionRules(text) { if (NLPTools.hasContractions(text)) { for (let i = 0; i < uniqueContractionRules.length; i++) { let contr = uniqueContractionRules[i]; text = text.replace(contr[0], contr[1]); } for (let i = 0; i < doubleContractionsRules.length; i++) { let contr = doubleContractionsRules[i]; text = text.replace(contr[0], contr[1]); } for (let i = 0; i < contractionRules.length; i++) { let contr = contractionRules[i]; text = text.replace(contr[0], contr[1]); } text = text.toLowerCase(); } return text; } static hasContractions(text) { return text.match(/[A-Za-z]+'[A-Za-z]+/g) !== null; } static cleanInput(str) { str = str.trim() .replace(/[‘’\`\´]/g, `'`) .replace(/[“”]/g, `"`) .replace(/\s'(\s)/g, `$1`) .replace(/(\.{3})\.*/ig, `$1`) .replace(/(^|[^\.])\.{2}([^\.]|$)/ig, `$1.$2`) .replace(TIME_PATTERN, timeStr => timeStr.replace(' ', '')) .replace(/\[/g, `(`) .replace(/\]/g, `)`) .replace(/[{}]/g, ``); return str; } static joinWordsIntoSentence(wordArray) { return wordArray.join(' '); } } exports.NLPTools = NLPTools; },{}],63:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const Utils_1 = require("../../utils/Utils"); const Types_1 = require("../common/Types"); const DEFAULT_OVERFLOW_LIMIT_MS = 0; class Validation { static validateSpeakOptions(validatedOptions, speakOptions) { let mergedValidatedOptions = Object.assign({}, validatedOptions); if (speakOptions) { if (speakOptions.disableAutoRules !== undefined) { if (typeof speakOptions.disableAutoRules === 'boolean') { mergedValidatedOptions.disableAutoRules = speakOptions.disableAutoRules; } else { throw new Error('Invalid DisableAutoRules value'); } } if (speakOptions.enableLayerMerging !== undefined) { if (typeof speakOptions.enableLayerMerging === 'boolean') { mergedValidatedOptions.enableLayerMerging = speakOptions.enableLayerMerging; } else { throw new Error('Invalid Layer Merging value'); } } if (speakOptions.frameBuffer !== undefined) { if (typeof speakOptions.frameBuffer === 'number' && speakOptions.frameBuffer >= 0) { mergedValidatedOptions.frameBuffer = speakOptions.frameBuffer; } else { throw new Error('Invalid Frame Buffer value'); } } if (speakOptions.followedByListen !== undefined) { if (typeof speakOptions.followedByListen === 'boolean') { mergedValidatedOptions.followedByListen = speakOptions.followedByListen; } else { throw new Error('Invalid FollowByListen value'); } } if (mergedValidatedOptions.followedByListen) { mergedValidatedOptions.dispatchComplete = Types_1.DispatchType.TTS; } else if (speakOptions.dispatchComplete) { let dispatchType = Types_1.DispatchType[speakOptions.dispatchComplete.toUpperCase()]; if (dispatchType) { mergedValidatedOptions.dispatchComplete = dispatchType; } else { throw new Error('Invalid Dispatch value'); } } if (speakOptions.assetRelativeEnd !== undefined) { if (typeof speakOptions.assetRelativeEnd === 'number') { mergedValidatedOptions.assetRelativeEnd = speakOptions.assetRelativeEnd; } else { throw new Error('Invalid AssetRelativeEnd value'); } } else if (mergedValidatedOptions.followedByListen) { mergedValidatedOptions.assetRelativeEnd = Utils_1.Utils.millisecToSec(DEFAULT_OVERFLOW_LIMIT_MS); } if (speakOptions.debugMode !== undefined) { if (typeof speakOptions.debugMode === 'boolean') { mergedValidatedOptions.debugMode = speakOptions.debugMode; } else { throw new Error('Invalid Debug Mode value'); } } } return mergedValidatedOptions; } static validateAutoRuleConfig(validatedConfig, autoRuleConfig) { let mergedValidatedConfig = Object.assign({}, validatedConfig); if (autoRuleConfig) { Object.keys(mergedValidatedConfig).forEach(key => { if (validatedConfig[key] && autoRuleConfig[key] !== undefined) { if (typeof autoRuleConfig[key] === 'boolean') { mergedValidatedConfig[key] = autoRuleConfig[key]; } } }); } return mergedValidatedConfig; } } exports.Validation = Validation; },{"../../utils/Utils":65,"../common/Types":13}],64:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); __export(require("./Validation")); __export(require("./NLPTools")); },{"./NLPTools":62,"./Validation":63}],65:[function(require,module,exports){ "use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); class Utils { static secToMillisec(seconds) { return seconds * 1000; } static millisecToSec(milliseconds) { return milliseconds / 1000; } static loadAnimationIntoCache(jibo, query, cacheName) { return __awaiter(this, void 0, void 0, function* () { let assets = []; if (typeof query === 'string') { const asset = jibo.animDB.getAnimByName(query); if (!asset) { return Promise.reject(`No animation of name '${query}' found in animDB`); } else { assets.push(asset); } } else { const results = jibo.animDB.query(query); if (!results.matching.length) { return Promise.reject(`No animation of ${query} found in animDB`); } else { assets.push(...results.matching); } } assets.forEach((asset) => __awaiter(this, void 0, void 0, function* () { yield asset.createFromConfig({ cache: cacheName }); })); }); } } exports.Utils = Utils; class ExtPromise { constructor() { this.promise = new Promise((res, rej) => { this.resolve = res; this.reject = rej; }); } } exports.ExtPromise = ExtPromise; function timeout(pr, timeoutTimeMs, options = {}) { return new Promise((resolve, reject) => { let timeout = setTimeout(() => { if (options.log) { let msg = `Timeout of ${timeoutTimeMs} ms occurred`; if (options.logMessage) { msg += `. ${options.logMessage}`; } options.log.warn(msg); } resolve(options.timeoutValue); }, timeoutTimeMs); pr.then((data) => { clearTimeout(timeout); resolve(data); }).catch(e => { clearTimeout(timeout); reject(e); }); }); } exports.timeout = timeout; },{}],66:[function(require,module,exports){ "use strict"; function __export(m) { for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; } Object.defineProperty(exports, "__esModule", { value: true }); const speech = require("./speech"); exports.speech = speech; const listen = require("./listen"); exports.listen = listen; const api = require("./api"); exports.api = api; __export(require("./EmbodiedDialog")); },{"./EmbodiedDialog":2,"./api":4,"./listen":10,"./speech":15}]},{},[66])(66) }); //# sourceMappingURL=jibo-embodied-dialog.js.map