From 3a150faf4b675580e8286c158c2e57f214b9dc0e Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Mon, 20 Apr 2026 22:03:17 -0500 Subject: [PATCH] backup yes/no path improvements --- .../WebSocketTurnFinalizationService.cs | 29 +++++++++ .../WebSockets/JiboWebSocketServiceTests.cs | 62 +++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs index 9cb67d4..0f95443 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs @@ -493,6 +493,24 @@ public sealed class WebSocketTurnFinalizationService( turnState.FinalizeAttemptCount += 1; } + if (allowFallbackOnMissingTranscript && + turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes && + IsYesNoTurn(finalizedTurn)) + { + turnState.AwaitingTurnCompletion = false; + session.LastTranscript = string.Empty; + session.LastIntent = null; + session.LastListenType = "no-input"; + var localRule = ReadPrimaryYesNoRule(finalizedTurn); + var noInputReplies = ResponsePlanToSocketMessagesMapper.MapNoInput( + turnState.TransId ?? session.LastTransId ?? string.Empty, + string.IsNullOrWhiteSpace(localRule) ? turnState.ListenRules : [localRule]) + .Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs }) + .ToArray(); + ResetBufferedAudio(session); + return noInputReplies; + } + if (allowFallbackOnMissingTranscript && turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes && string.IsNullOrWhiteSpace(turnState.LastSttError)) @@ -714,6 +732,17 @@ public sealed class WebSocketTurnFinalizationService( string.Equals(rule, "surprises-ota/want_to_download_now", StringComparison.OrdinalIgnoreCase)); } + private static string? ReadPrimaryYesNoRule(TurnContext turn) + { + return ReadRules(turn, "listenRules") + .Concat(ReadRules(turn, "clientRules")) + .FirstOrDefault(static rule => + string.Equals(rule, "create/is_it_a_keeper", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "settings/download_now_later", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "surprises-date/offer_date_fact", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "surprises-ota/want_to_download_now", StringComparison.OrdinalIgnoreCase)); + } + private static IEnumerable ReadRules(TurnContext turn, string key) { if (!turn.Attributes.TryGetValue(key, out var value) || value is null) diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 48e86d3..5e3cddc 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -404,6 +404,68 @@ public sealed class JiboWebSocketServiceTests Assert.Equal("surprises-ota/want_to_download_now", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } + [Fact] + public async Task BufferedAudio_YesNoPromptWithSttFailure_AutoFinalizesAsLocalNoInput() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-yesno-noinput-token", + Text = """{"type":"LISTEN","transID":"trans-yesno-noinput","data":{"rules":["surprises-ota/want_to_download_now","globals/gui_nav","globals/global_commands_launch"],"asr":{"hints":["$YESNO"]}}}""" + }); + + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-yesno-noinput-token", + Text = """{"type":"CONTEXT","transID":"trans-yesno-noinput","data":{"topic":"conversation"}}""" + }); + + for (var index = 0; index < 4; index += 1) + { + var interimReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-yesno-noinput-token", + Binary = new byte[3000] + }); + + Assert.Single(interimReplies); + Assert.Equal("OPENJIBO_AUDIO_RECEIVED", ReadReplyType(interimReplies[0])); + } + + var session = _store.FindSessionByToken("hub-yesno-noinput-token"); + Assert.NotNull(session); + session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); + session.TurnState.LastSttError = "whisper.cpp returned no transcript"; + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-yesno-noinput-token", + Binary = new byte[3000] + }); + + Assert.Equal(2, replies.Count); + Assert.Equal("LISTEN", ReadReplyType(replies[0])); + Assert.Equal("EOS", ReadReplyType(replies[1])); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + Assert.Equal(string.Empty, listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); + Assert.Equal(string.Empty, listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + var rules = listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules"); + Assert.Single(rules.EnumerateArray()); + Assert.Equal("surprises-ota/want_to_download_now", rules[0].GetString()); + } + [Fact] public async Task ClientAsr_SurprisesDateOfferPrompt_MapsYesWithoutGlobalRuleLeak() {