using System.Text.Json; using Jibo.Cloud.Application.Services; using Jibo.Cloud.Domain.Models; using Jibo.Cloud.Infrastructure.Content; using Jibo.Cloud.Infrastructure.Persistence; using Jibo.Cloud.Tests.Fixtures; namespace Jibo.Cloud.Tests.WebSockets; public sealed class JiboWebSocketServiceTests { private readonly InMemoryCloudStateStore _store; private readonly JiboWebSocketService _service; public JiboWebSocketServiceTests() { _store = new InMemoryCloudStateStore(); var turnContextMapper = new ProtocolToTurnContextMapper(); var contentRepository = new InMemoryJiboExperienceContentRepository(); var contentCache = new JiboExperienceContentCache(contentRepository); var conversationBroker = new DemoConversationBroker(new JiboInteractionService(contentCache, new DefaultJiboRandomizer())); var replyMapper = new ResponsePlanToSocketMessagesMapper(); var sttSelector = new DefaultSttStrategySelector( [ new SyntheticBufferedAudioSttStrategy() ]); var sink = new NullTurnTelemetrySink(); _service = new JiboWebSocketService( _store, new NullWebSocketTelemetrySink(), new WebSocketTurnFinalizationService( turnContextMapper, conversationBroker, replyMapper, sttSelector, sink)); } [Fact] public async Task ListenMessage_ReturnsSyntheticListenEosAndSkillAction() { var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-test-token", Text = """{"type":"LISTEN","transID":"trans-hello","data":{"text":"hello jibo","rules":["wake-word"]}}""" }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); Assert.Equal(75, replies[2].DelayMs); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("hello jibo", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); using var eosPayload = JsonDocument.Parse(replies[1].Text!); Assert.True(eosPayload.RootElement.TryGetProperty("ts", out _)); Assert.StartsWith("mid-", eosPayload.RootElement.GetProperty("msgID").GetString()); Assert.Equal("trans-hello", eosPayload.RootElement.GetProperty("transID").GetString()); Assert.Equal(JsonValueKind.Object, eosPayload.RootElement.GetProperty("data").ValueKind); using var skillPayload = JsonDocument.Parse(replies[2].Text!); Assert.StartsWith("mid-", skillPayload.RootElement.GetProperty("msgID").GetString()); var meta = skillPayload.RootElement .GetProperty("data") .GetProperty("action") .GetProperty("config") .GetProperty("jcp") .GetProperty("config") .GetProperty("play") .GetProperty("meta"); Assert.False(meta.TryGetProperty("intent", out _)); Assert.False(meta.TryGetProperty("transcript", out _)); } [Fact] public async Task BinaryMessage_ReturnsAcknowledgementPayload() { var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-test-token", Binary = [1, 2, 3, 4] }); using var payload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("OPENJIBO_AUDIO_RECEIVED", payload.RootElement.GetProperty("type").GetString()); Assert.Equal(4, payload.RootElement.GetProperty("data").GetProperty("bytes").GetInt32()); Assert.Equal(4, payload.RootElement.GetProperty("data").GetProperty("bufferedBytes").GetInt32()); Assert.Equal(1, payload.RootElement.GetProperty("data").GetProperty("bufferedChunks").GetInt32()); } [Fact] public async Task BufferedAudio_WithContextAndTranscriptHint_AutoFinalizesAfterThreshold() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-finalize-token", Text = """{"type":"LISTEN","transID":"trans-auto","data":{"rules":["launch"]}}""" }); await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-finalize-token", Text = """{"type":"CONTEXT","transID":"trans-auto","data":{"audioTranscriptHint":"tell me a joke"}}""" }); IReadOnlyList replies; for (var index = 0; index < 4; index += 1) { replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-finalize-token", Binary = new byte[3000] }); Assert.Single(replies); Assert.Equal("OPENJIBO_AUDIO_RECEIVED", ReadReplyType(replies[0])); } var session = _store.FindSessionByToken("hub-auto-finalize-token"); Assert.NotNull(session); session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-finalize-token", Binary = new byte[3000] }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); Assert.Equal(75, replies[2].DelayMs); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("tell me a joke", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("joke", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); } [Fact] public async Task BufferedAudio_WithoutTranscriptHint_AutoFinalizesWithFallbackAndEos() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-fallback-token", Text = """{"type":"LISTEN","transID":"trans-auto-fallback","data":{"rules":["launch"]}}""" }); await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-fallback-token", Text = """{"type":"CONTEXT","transID":"trans-auto-fallback","data":{"topic":"conversation"}}""" }); IReadOnlyList replies; for (var index = 0; index < 4; index += 1) { replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-fallback-token", Binary = new byte[3000] }); Assert.Single(replies); Assert.Equal("OPENJIBO_AUDIO_RECEIVED", ReadReplyType(replies[0])); } var session = _store.FindSessionByToken("hub-auto-fallback-token"); Assert.NotNull(session); session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-auto-fallback-token", Binary = new byte[3000] }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); Assert.Equal(75, replies[2].DelayMs); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("heyJibo", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal(string.Empty, listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); } [Fact] public async Task MultiChunkAudio_AccumulatesBufferedStateAcrossMessages() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-multichunk-token", Text = """{"type":"LISTEN","transID":"trans-multi","data":{"rules":["wake-word"]}}""" }); var firstAudioReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-multichunk-token", Binary = [1, 2, 3] }); var secondAudioReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-multichunk-token", Binary = [4, 5, 6, 7] }); using var firstPayload = JsonDocument.Parse(firstAudioReplies[0].Text!); using var secondPayload = JsonDocument.Parse(secondAudioReplies[0].Text!); Assert.Equal(3, firstPayload.RootElement.GetProperty("data").GetProperty("bufferedBytes").GetInt32()); Assert.Equal(7, secondPayload.RootElement.GetProperty("data").GetProperty("bufferedBytes").GetInt32()); Assert.Equal(2, secondPayload.RootElement.GetProperty("data").GetProperty("bufferedChunks").GetInt32()); var session = _store.FindSessionByToken("hub-multichunk-token"); Assert.NotNull(session); Assert.Equal(7, session.TurnState.BufferedAudioBytes); Assert.Equal(2, session.TurnState.BufferedAudioChunkCount); } [Fact] public async Task ContextThenClientNlu_UsesFollowUpTurnStateAndSkipsSkillAction() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-follow-up-token", Text = """{"type":"LISTEN","transID":"trans-follow-up","data":{"text":"hello jibo","rules":["wake-word"]}}""" }); var contextReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-follow-up-token", Text = """{"type":"CONTEXT","transID":"trans-follow-up","data":{"topic":"conversation","screen":"home"}}""" }); Assert.Single(contextReplies); Assert.Equal("OPENJIBO_CONTEXT_ACK", ReadReplyType(contextReplies[0])); var nluReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-follow-up-token", Text = """{"type":"CLIENT_NLU","transID":"trans-follow-up","data":{"intent":"joke"}}""" }); Assert.Equal(2, nluReplies.Count); Assert.Equal("LISTEN", ReadReplyType(nluReplies[0])); Assert.Equal("EOS", ReadReplyType(nluReplies[1])); var session = _store.FindSessionByToken("hub-follow-up-token"); Assert.NotNull(session); Assert.True(session.FollowUpOpen); Assert.Equal("joke", session.LastIntent); Assert.Equal("trans-follow-up", session.LastTransId); } [Fact] public async Task ClientNlu_ClockAskForTime_PreservesObservedIntentRulesAndEntities() { var listenReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-clock-menu-token", Text = """{"type":"LISTEN","transID":"trans-clock-time","data":{"lang":"en-US","rules":["clock/clock_menu","globals/global_commands_launch"],"mode":"CLIENT_NLU"}}""" }); Assert.Single(listenReplies); Assert.Equal("OPENJIBO_TURN_PENDING", ReadReplyType(listenReplies[0])); var nluReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-clock-menu-token", Text = """{"type":"CLIENT_NLU","transID":"trans-clock-time","data":{"entities":{"domain":"clock"},"intent":"askForTime","rules":["clock/clock_menu"]}}""" }); Assert.Equal(2, nluReplies.Count); Assert.Equal("LISTEN", ReadReplyType(nluReplies[0])); Assert.Equal("EOS", ReadReplyType(nluReplies[1])); using var listenPayload = JsonDocument.Parse(nluReplies[0].Text!); Assert.Equal("askForTime", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("askForTime", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("clock", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString()); Assert.Equal("clock/clock_menu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules")[0].GetString()); Assert.Equal("clock/clock_menu", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } [Fact] public async Task ClientAsr_YesNoCreateFlow_PreservesCreateRuleAndDomain() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-yesno-token", Text = """{"type":"LISTEN","transID":"trans-yesno","data":{"rules":["create/is_it_a_keeper","$YESNO"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-yesno-token", Text = """{"type":"CLIENT_ASR","transID":"trans-yesno","data":{"text":"yeah"}}""" }); Assert.Equal(3, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("yeah", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("yes", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("create/is_it_a_keeper", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules")[0].GetString()); Assert.Equal("create", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString()); Assert.Equal("create/is_it_a_keeper", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } [Fact] public async Task ClientAsr_YesNoPromptFromAsrHints_MapsShortDenialToNoIntent() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-yesno-hints-token", Text = """{"type":"LISTEN","transID":"trans-yesno-hints","data":{"rules":["surprises-ota/want_to_download_now"],"asr":{"hints":["$YESNO"]}}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-yesno-hints-token", Text = """{"type":"CLIENT_ASR","transID":"trans-yesno-hints","data":{"text":"no"}}""" }); Assert.Equal(3, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("no", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("no", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("surprises-ota/want_to_download_now", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules")[0].GetString()); Assert.Equal("surprises-ota/want_to_download_now", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } [Fact] public async Task ClientAsr_OpenTheRadio_EmitsRadioRedirectAndSilentCompletion() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-radio-open-token", Text = """{"type":"LISTEN","transID":"trans-radio-open","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-radio-open-token", Text = """{"type":"CLIENT_ASR","transID":"trans-radio-open","data":{"text":"open the radio"}}""" }); Assert.Equal(4, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("menu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("@be/radio", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString()); Assert.Equal(0, listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules").GetArrayLength()); Assert.Equal(0, listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").EnumerateObject().Count()); using var redirectPayload = JsonDocument.Parse(replies[2].Text!); Assert.Equal("@be/radio", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString()); Assert.True(redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("launch").GetBoolean()); Assert.Equal("menu", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); using var completionPayload = JsonDocument.Parse(replies[3].Text!); Assert.Equal("@be/radio", completionPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString()); } [Fact] public async Task ClientAsr_PlayCountryMusic_EmitsRadioRedirectWithCountryStation() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-radio-country-token", Text = """{"type":"LISTEN","transID":"trans-radio-country","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-radio-country-token", Text = """{"type":"CLIENT_ASR","transID":"trans-radio-country","data":{"text":"play country music"}}""" }); Assert.Equal(4, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("menu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("Country", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("station").GetString()); using var redirectPayload = JsonDocument.Parse(replies[2].Text!); Assert.Equal("Country", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("station").GetString()); Assert.Equal("play country music", redirectPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); } [Fact] public async Task ClientNlu_WordOfDayGuess_UsesGuessEntityAsAsrTextAndCompletesTurn() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-guess-token", Text = """{"type":"LISTEN","transID":"trans-wod-guess","data":{"rules":["word-of-the-day/puzzle","globals/gui_nav"],"asr":{"hints":["pastoral","doodad","escarpment"],"earlyEOS":["pastoral","doodad","escarpment"]}}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-guess-token", Text = """{"type":"CLIENT_NLU","transID":"trans-wod-guess","data":{"entities":{"guess":"pastoral"},"intent":"guess","rules":["word-of-the-day/puzzle"]}}""" }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("guess", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("guess").GetString()); Assert.Equal("word-of-the-day/puzzle", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } [Fact] public async Task ClientAsr_WordOfDayGuess_UsesSpokenTranscriptDuringPuzzleTurn() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-spoken-guess-token", Text = """{"type":"LISTEN","transID":"trans-wod-spoken-guess","data":{"rules":["word-of-the-day/puzzle"],"asr":{"hints":["pastoral","doodad","escarpment"]}}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-spoken-guess-token", Text = """{"type":"CLIENT_ASR","transID":"trans-wod-spoken-guess","data":{"text":"pastoral"}}""" }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("guess", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("guess").GetString()); Assert.Equal("word-of-the-day/puzzle", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } [Fact] public async Task ClientAsr_WordOfDayGuess_LineNumberUsesHintOrder() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-line-guess-token", Text = """{"type":"LISTEN","transID":"trans-wod-line-guess","data":{"rules":["word-of-the-day/puzzle"],"asr":{"hints":["doodad","pastoral","escarpment"]}}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-line-guess-token", Text = """{"type":"CLIENT_ASR","transID":"trans-wod-line-guess","data":{"text":"Two."}}""" }); Assert.Equal(3, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("guess", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("guess").GetString()); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); } [Fact] public async Task ClientAsr_WordOfDayGuess_FuzzyMatchesClosestHint() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-fuzzy-guess-token", Text = """{"type":"LISTEN","transID":"trans-wod-fuzzy-guess","data":{"rules":["word-of-the-day/puzzle"],"asr":{"hints":["aglet","hovel","wisenheimer"]}}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-fuzzy-guess-token", Text = """{"type":"CLIENT_ASR","transID":"trans-wod-fuzzy-guess","data":{"text":"Haglet."}}""" }); Assert.Equal(3, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("aglet", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("aglet", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("guess").GetString()); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); } [Fact] public async Task ClientAsr_WordOfDayGuess_StripsGlobalRulesFromOutboundGuess() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-guess-rules-token", Text = """{"type":"LISTEN","transID":"trans-wod-guess-rules","data":{"rules":["word-of-the-day/puzzle","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"],"asr":{"hints":["aglet","hovel","wisenheimer"]}}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-guess-rules-token", Text = """{"type":"CLIENT_ASR","transID":"trans-wod-guess-rules","data":{"text":"aglet"}}""" }); Assert.Equal(3, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); var rules = listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules"); Assert.Single(rules.EnumerateArray()); Assert.Equal("word-of-the-day/puzzle", rules[0].GetString()); Assert.Equal("word-of-the-day/puzzle", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } [Fact] public async Task ClientAsr_SettingsDownloadNo_StripsGlobalRulesFromOutboundNo() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-settings-no-token", Text = """{"type":"LISTEN","transID":"trans-settings-no","data":{"rules":["settings/download_now_later","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"],"asr":{"hints":["$YESNO"]}}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-settings-no-token", Text = """{"type":"CLIENT_ASR","transID":"trans-settings-no","data":{"text":"No."}}""" }); Assert.Equal(3, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); var rules = listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules"); Assert.Single(rules.EnumerateArray()); Assert.Equal("settings/download_now_later", rules[0].GetString()); Assert.Equal("settings/download_now_later", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); Assert.Equal("no", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); } [Fact] public async Task ClientAsr_WordOfDayLaunch_EmitsMenuStyleLoadMenuAndRedirect() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-launch-token", Text = """{"type":"LISTEN","transID":"trans-wod-launch","data":{"rules":["launch","globals/global_commands_launch"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-launch-token", Text = """{"type":"CLIENT_ASR","transID":"trans-wod-launch","data":{"text":"Play word of the day."}}""" }); Assert.Equal(4, replies.Count); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("menu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal(string.Empty, listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("word-of-the-day", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString()); Assert.Equal("@be/word-of-the-day", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString()); Assert.Equal("word-of-the-day/menu", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3])); using var redirectPayload = JsonDocument.Parse(replies[2].Text!); Assert.Equal("@be/word-of-the-day", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString()); Assert.True(redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("onRobot").GetBoolean()); Assert.True(redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("launch").GetBoolean()); var session = _store.FindSessionByToken("hub-wod-launch-token"); Assert.NotNull(session); Assert.False(session.FollowUpOpen); } [Fact] public async Task AutoFinalizedWordOfDayLaunch_IgnoresLateSameTurnAudio() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-auto-token", Text = """{"type":"LISTEN","transID":"trans-wod-auto","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" }); await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-auto-token", Text = """{"type":"CONTEXT","transID":"trans-wod-auto","data":{"audioTranscriptHint":"play word of the day"}}""" }); for (var index = 0; index < 4; index += 1) { var interimReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-auto-token", Binary = new byte[3000] }); Assert.Single(interimReplies); Assert.Equal("OPENJIBO_AUDIO_RECEIVED", ReadReplyType(interimReplies[0])); } var session = _store.FindSessionByToken("hub-wod-auto-token"); Assert.NotNull(session); session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-auto-token", Binary = new byte[3000] }); Assert.Equal(4, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("menu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.Equal("word-of-the-day", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString()); Assert.Equal("@be/word-of-the-day", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString()); var lateReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-auto-token", Binary = new byte[3000] }); Assert.Empty(lateReplies); Assert.False(session.TurnState.AwaitingTurnCompletion); } [Fact] public async Task EmptyClientAsr_AfterCompletedWordOfDayTurn_IsIgnored() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-late-empty-token", Text = """{"type":"LISTEN","transID":"trans-wod-late-empty","data":{"rules":["word-of-the-day/puzzle"]}}""" }); var winReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-late-empty-token", Text = """{"type":"CLIENT_NLU","transID":"trans-wod-late-empty","data":{"entities":{"guess":"pastoral"},"intent":"guess","rules":["word-of-the-day/puzzle"]}}""" }); Assert.Equal(3, winReplies.Count); var lateReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-late-empty-token", Text = """{"type":"CLIENT_ASR","transID":"trans-wod-late-empty","data":{}}""" }); Assert.Empty(lateReplies); } [Fact] public async Task EmptyClientAsr_AfterWordOfDayRightWordListen_IsIgnored() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-right-word-token", Text = """{"type":"LISTEN","transID":"trans-wod-right-word","data":{"rules":["word-of-the-day/right_word","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-right-word-token", Text = """{"type":"CLIENT_ASR","transID":"trans-wod-right-word","data":{}}""" }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.False(listenPayload.RootElement.GetProperty("data").TryGetProperty("match", out _)); using var redirectPayload = JsonDocument.Parse(replies[2].Text!); Assert.Equal("@be/idle", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString()); } [Fact] public async Task ListenSetupWithoutTranscript_ReturnsPendingInsteadOfFinalizingTurn() { var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-listen-setup-token", Text = """{"type":"LISTEN","transID":"trans-listen-setup","data":{"rules":["main-menu/execute_fun_stuff","globals/global_commands_launch"],"mode":"CLIENT_NLU"}}""" }); Assert.Single(replies); Assert.Equal("OPENJIBO_TURN_PENDING", ReadReplyType(replies[0])); var session = _store.FindSessionByToken("hub-listen-setup-token"); Assert.NotNull(session); Assert.True(session.TurnState.AwaitingTurnCompletion); Assert.Null(session.LastIntent); } [Fact] public async Task BinaryAudio_AfterWordOfDayRightWordListen_IsIgnoredDuringCleanupWindow() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-right-word-audio-token", Text = """{"type":"LISTEN","transID":"trans-wod-right-word-audio","data":{"rules":["word-of-the-day/right_word","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-right-word-audio-token", Text = """{"type":"LISTEN","transID":"trans-wod-right-word-audio","data":{"rules":["word-of-the-day/right_word","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"]}}""" }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal(string.Empty, listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.False(listenPayload.RootElement.GetProperty("data").TryGetProperty("match", out _)); var binaryReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-wod-right-word-audio-token", Binary = new byte[4096] }); Assert.Empty(binaryReplies); var session = _store.FindSessionByToken("hub-wod-right-word-audio-token"); Assert.NotNull(session); Assert.False(session.TurnState.AwaitingTurnCompletion); Assert.True(session.TurnState.IgnoreAdditionalAudioUntilUtc.HasValue); } [Fact] public async Task BlankAudioHotphraseTurn_IsIgnored() { var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-blank-audio-token", Text = """{"type":"LISTEN","transID":"trans-blank-audio","data":{"text":"[BLANK_AUDIO]","rules":["launch","globals/global_commands_launch"]}}""" }); Assert.Empty(replies); } [Fact] public async Task InitialHotphraseListen_RemainsPendingInsteadOfGreeting() { var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-initial-hotphrase-token", Text = """{"type":"LISTEN","transID":"trans-initial-hotphrase","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" }); Assert.Single(replies); Assert.Equal("OPENJIBO_TURN_PENDING", ReadReplyType(replies[0])); var session = _store.FindSessionByToken("hub-initial-hotphrase-token"); Assert.NotNull(session); Assert.Null(session.LastIntent); Assert.Null(session.LastTranscript); } [Fact] public async Task SecondEmptyHotphraseTurn_BecomesGreetingAndKeepsFollowUpOpen() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-empty-hotphrase-token", Text = """{"type":"LISTEN","transID":"trans-empty-hotphrase","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" }); var firstReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-empty-hotphrase-token", Text = """{"type":"CLIENT_ASR","transID":"trans-empty-hotphrase","data":{}}""" }); Assert.Single(firstReplies); Assert.Equal("OPENJIBO_TURN_PENDING", ReadReplyType(firstReplies[0])); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-empty-hotphrase-token", Text = """{"type":"CLIENT_ASR","transID":"trans-empty-hotphrase","data":{}}""" }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); var session = _store.FindSessionByToken("hub-empty-hotphrase-token"); Assert.NotNull(session); Assert.True(session.FollowUpOpen); } [Fact] public async Task BufferedAudio_WithSyntheticTranscriptHint_FinalizesThroughSttSeam() { var listenReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-token", Text = """{"type":"LISTEN","transID":"trans-audio","data":{"rules":["wake-word"]}}""" }); Assert.Single(listenReplies); Assert.Equal("OPENJIBO_TURN_PENDING", ReadReplyType(listenReplies[0])); var contextReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-token", Text = """{"type":"CONTEXT","transID":"trans-audio","data":{"topic":"conversation","audioTranscriptHint":"tell me a joke"}}""" }); Assert.Single(contextReplies); Assert.Equal("OPENJIBO_CONTEXT_ACK", ReadReplyType(contextReplies[0])); var audioReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-token", Binary = [1, 2, 3, 4, 5, 6] }); Assert.Single(audioReplies); Assert.Equal("OPENJIBO_AUDIO_RECEIVED", ReadReplyType(audioReplies[0])); var finalizeReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-token", Text = """{"type":"CLIENT_ASR","transID":"trans-audio","data":{}}""" }); Assert.Equal(3, finalizeReplies.Count); Assert.Equal("LISTEN", ReadReplyType(finalizeReplies[0])); Assert.Equal("EOS", ReadReplyType(finalizeReplies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(finalizeReplies[2])); Assert.Equal(75, finalizeReplies[2].DelayMs); using var listenPayload = JsonDocument.Parse(finalizeReplies[0].Text!); Assert.Equal("tell me a joke", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("joke", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); var session = _store.FindSessionByToken("hub-audio-token"); Assert.NotNull(session); Assert.Equal(0, session.TurnState.BufferedAudioBytes); Assert.Equal(0, session.TurnState.BufferedAudioChunkCount); Assert.False(session.Metadata.ContainsKey("audioTranscriptHint")); } [Fact] public async Task BufferedAudio_WithoutTranscriptHint_RemainsPending() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-pending-token", Text = """{"type":"LISTEN","transID":"trans-pending","data":{"rules":["wake-word"]}}""" }); await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-pending-token", Binary = [1, 2, 3, 4] }); var finalizeReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-pending-token", Text = """{"type":"CLIENT_ASR","transID":"trans-pending","data":{}}""" }); Assert.Single(finalizeReplies); Assert.Equal("OPENJIBO_TURN_PENDING", ReadReplyType(finalizeReplies[0])); using var payload = JsonDocument.Parse(finalizeReplies[0].Text!); Assert.True(payload.RootElement.GetProperty("data").GetProperty("awaitingTranscriptHint").GetBoolean()); Assert.Equal(1, payload.RootElement.GetProperty("data").GetProperty("finalizeAttempts").GetInt32()); } [Fact] public async Task BufferedAudio_WithChatTranscriptHint_FinalizesAsChat() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-chat-token", Text = """{"type":"LISTEN","transID":"trans-audio-chat","data":{"rules":["wake-word"]}}""" }); await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-chat-token", Text = """{"type":"CONTEXT","transID":"trans-audio-chat","data":{"audioTranscriptHint":"hello from buffered audio"}}""" }); await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-chat-token", Binary = [1, 2, 3, 4, 5] }); var finalizeReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-audio-chat-token", Text = """{"type":"CLIENT_ASR","transID":"trans-audio-chat","data":{}}""" }); Assert.Equal(3, finalizeReplies.Count); using var listenPayload = JsonDocument.Parse(finalizeReplies[0].Text!); Assert.Equal("hello from buffered audio", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); using var skillPayload = JsonDocument.Parse(finalizeReplies[2].Text!); Assert.Equal("chitchat-skill", skillPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString()); } [Fact] public async Task BufferedHotphraseAudio_WithSttFailure_BecomesGreetingAndKeepsFollowUpOpen() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-hotphrase-greeting-token", Text = """{"type":"LISTEN","transID":"trans-hotphrase-greeting","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" }); await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-hotphrase-greeting-token", Text = """{"type":"CONTEXT","transID":"trans-hotphrase-greeting","data":{"topic":"conversation"}}""" }); for (var index = 0; index < 4; index += 1) { var interimReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-hotphrase-greeting-token", Binary = new byte[3000] }); Assert.Single(interimReplies); Assert.Equal("OPENJIBO_AUDIO_RECEIVED", ReadReplyType(interimReplies[0])); } var session = _store.FindSessionByToken("hub-hotphrase-greeting-token"); Assert.NotNull(session); session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); session.TurnState.LastSttError = "ffmpeg decode failed"; var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-hotphrase-greeting-token", Binary = new byte[3000] }); Assert.Equal(3, replies.Count); Assert.Equal("LISTEN", ReadReplyType(replies[0])); Assert.Equal("EOS", ReadReplyType(replies[1])); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); using var listenPayload = JsonDocument.Parse(replies[0].Text!); Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); Assert.True(session.FollowUpOpen); } [Fact] public async Task ClientAsrJokeFlow_MatchesNodePayloadShapeForEosAndSkillAction() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-client-asr-joke-token", Text = """{"type":"LISTEN","transID":"trans-joke-shape","data":{"rules":["wake-word"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-client-asr-joke-token", Text = """{"type":"CLIENT_ASR","transID":"trans-joke-shape","data":{"text":"tell me a joke"}}""" }); Assert.Equal(3, replies.Count); Assert.Equal(75, replies[2].DelayMs); using var eosPayload = JsonDocument.Parse(replies[1].Text!); Assert.Equal("EOS", eosPayload.RootElement.GetProperty("type").GetString()); Assert.Equal("trans-joke-shape", eosPayload.RootElement.GetProperty("transID").GetString()); Assert.True(eosPayload.RootElement.TryGetProperty("ts", out _)); Assert.StartsWith("mid-", eosPayload.RootElement.GetProperty("msgID").GetString()); Assert.Empty(eosPayload.RootElement.GetProperty("data").EnumerateObject()); using var skillPayload = JsonDocument.Parse(replies[2].Text!); Assert.Equal("SKILL_ACTION", skillPayload.RootElement.GetProperty("type").GetString()); Assert.Equal("trans-joke-shape", skillPayload.RootElement.GetProperty("transID").GetString()); Assert.StartsWith("mid-", skillPayload.RootElement.GetProperty("msgID").GetString()); Assert.Equal("@be/joke", skillPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString()); var meta = skillPayload.RootElement .GetProperty("data") .GetProperty("action") .GetProperty("config") .GetProperty("jcp") .GetProperty("config") .GetProperty("play") .GetProperty("meta"); Assert.Equal("RUNTIME_PROMPT", meta.GetProperty("prompt_id").GetString()); Assert.Equal("AN", meta.GetProperty("prompt_sub_category").GetString()); Assert.Equal("runtime-joke", meta.GetProperty("mim_id").GetString()); Assert.Equal("announcement", meta.GetProperty("mim_type").GetString()); Assert.False(meta.TryGetProperty("intent", out _)); Assert.False(meta.TryGetProperty("transcript", out _)); } [Fact] public async Task ClientAsrDanceFlow_EmitsAnimatedSkillAction() { await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-client-asr-dance-token", Text = """{"type":"LISTEN","transID":"trans-dance-shape","data":{"rules":["wake-word"]}}""" }); var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { HostName = "neo-hub.jibo.com", Path = "/listen", Kind = "neo-hub-listen", Token = "hub-client-asr-dance-token", Text = """{"type":"CLIENT_ASR","transID":"trans-dance-shape","data":{"text":"do a dance"}}""" }); Assert.Equal(3, replies.Count); Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); using var skillPayload = JsonDocument.Parse(replies[2].Text!); var esml = skillPayload.RootElement .GetProperty("data") .GetProperty("action") .GetProperty("config") .GetProperty("jcp") .GetProperty("config") .GetProperty("play") .GetProperty("esml") .GetString(); Assert.Contains("