From 32d63584d62385f92e396bcad4ccd4d30c9e4980 Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Mon, 20 Apr 2026 20:55:49 -0500 Subject: [PATCH] mapping radio --- OpenJibo/docs/development-plan.md | 7 ++ OpenJibo/docs/feature-backlog.md | 1 + .../Services/DemoConversationBroker.cs | 2 + .../Services/JiboInteractionService.cs | 99 +++++++++++++++++++ .../ResponsePlanToSocketMessagesMapper.cs | 44 ++++++++- .../WebSocketTurnFinalizationService.cs | 2 + .../WebSockets/JiboInteractionServiceTests.cs | 32 ++++++ .../WebSockets/JiboWebSocketServiceTests.cs | 74 ++++++++++++++ 8 files changed, 258 insertions(+), 3 deletions(-) diff --git a/OpenJibo/docs/development-plan.md b/OpenJibo/docs/development-plan.md index e52ef16..57c1623 100644 --- a/OpenJibo/docs/development-plan.md +++ b/OpenJibo/docs/development-plan.md @@ -158,6 +158,13 @@ Latest stock-OS WOD findings: - Spoken WOD guesses should preferentially snap to the closest offered hint when Whisper lands very close to one of the menu words, since near-misses like `haglet` for `aglet` are common in live testing. - The stock robot still misroutes constrained local turns if the cloud echoes `globals/*` rules back on the reply. For spoken WOD guesses and settings/update `no`, we should only return the local rule (`word-of-the-day/puzzle`, `settings/download_now_later`, etc.) so Global Service does not relaunch Nimbus. +Latest radio discovery findings: + +- `@be/radio` is a true local skill, not a cloud placeholder. +- Its `open(result, refresh, previousSkillName)` path treats `result.nlu.intent === "menu"` as a `play` launch. +- `result.nlu.entities.station` is the genre selector, and `Country` is a real supported station key from the robot's `genres.json`. +- The smallest stock-shaped cloud handoff for voice launch is therefore a local `SKILL_REDIRECT` to `@be/radio` with `nlu.intent = "menu"`, optional `entities.station`, and a silent completion to settle the hotphrase cloud response. + ## Speech, Animation, And ESML The current joke flow is only a small foothold into Jibo expressiveness. diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index 1105922..974cb91 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -40,6 +40,7 @@ Parallel tags: - Current evidence: - [index.js](C:/Projects/JiboOs/V3.1/build/opt/jibo/Jibo/Skills/@be/be/node_modules/@be/radio/index.js) resumes from `lastStation` - the same file treats `menu` as a `play` launch and reads `result.nlu.entities.station` + - the same file confirms `menu + no station` is the clean resume path and `menu + station=Country` becomes a direct genre launch - Implementation notes: - add phrase routing for radio open/resume and genre launch - inspect radio genre and station metadata before locking the outbound entity values diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs index f262f96..0b44284 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs @@ -72,6 +72,8 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer { "word_of_the_day" => false, "word_of_the_day_guess" => false, + "radio" => false, + "radio_genre" => false, _ => true }; } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs index d350f99..8283dc2 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs @@ -29,6 +29,8 @@ public sealed class JiboInteractionService( "dance" => BuildDanceDecision(catalog), "time" => new JiboInteractionDecision("time", $"It is {DateTime.Now:h:mm tt}."), "date" => new JiboInteractionDecision("date", $"Today is {DateTime.Now:dddd, MMMM d}."), + "radio" => BuildRadioLaunchDecision(), + "radio_genre" => BuildRadioGenreLaunchDecision(lowered), "hello" => new JiboInteractionDecision("hello", randomizer.Choose(catalog.GreetingReplies)), "how_are_you" => new JiboInteractionDecision("how_are_you", randomizer.Choose(catalog.HowAreYouReplies)), "yes" => new JiboInteractionDecision("yes", "Yes."), @@ -151,6 +153,16 @@ public sealed class JiboInteractionService( return "joke"; } + if (TryResolveRadioGenre(loweredTranscript) is not null) + { + return "radio_genre"; + } + + if (MatchesAny(loweredTranscript, "open the radio", "play the radio", "turn on the radio", "radio")) + { + return "radio"; + } + if (MatchesAny(loweredTranscript, "dance", "boogie")) { return "dance"; @@ -235,6 +247,33 @@ public sealed class JiboInteractionService( }); } + private static JiboInteractionDecision BuildRadioLaunchDecision() + { + return new JiboInteractionDecision( + "radio", + "Opening the radio.", + "@be/radio", + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["skillId"] = "@be/radio" + }); + } + + private static JiboInteractionDecision BuildRadioGenreLaunchDecision(string loweredTranscript) + { + var station = TryResolveRadioGenre(loweredTranscript) ?? "Country"; + + return new JiboInteractionDecision( + "radio_genre", + $"Playing {FormatRadioGenreForSpeech(station)} on the radio.", + "@be/radio", + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["skillId"] = "@be/radio", + ["station"] = station + }); + } + private static JiboInteractionDecision BuildWordOfTheDayGuessDecision( IReadOnlyDictionary clientEntities, string transcript, @@ -417,6 +456,66 @@ public sealed class JiboInteractionService( { return candidates.Any(candidate => loweredTranscript.Contains(candidate, StringComparison.Ordinal)); } + + private static string? TryResolveRadioGenre(string loweredTranscript) + { + foreach (var (phrase, station) in RadioGenreAliases) + { + if (loweredTranscript.Contains(phrase, StringComparison.Ordinal)) + { + return station; + } + } + + return null; + } + + private static string FormatRadioGenreForSpeech(string station) + { + return station switch + { + "EightiesAndNinetiesHits" => "eighties and nineties hits", + "ChristianAndGospel" => "Christian and gospel", + "ClassicRock" => "classic rock", + "CollegeRadio" => "college radio", + "HipHop" => "hip hop", + "NewsAndTalk" => "news and talk", + "ReggaeAndIsland" => "reggae and island music", + "SoftRock" => "soft rock", + _ => station + }; + } + + private static readonly (string Phrase, string Station)[] RadioGenreAliases = + [ + ("country music", "Country"), + ("country radio", "Country"), + ("country", "Country"), + ("classic rock", "ClassicRock"), + ("soft rock", "SoftRock"), + ("hip hop", "HipHop"), + ("hip-hop", "HipHop"), + ("news and talk", "NewsAndTalk"), + ("news talk", "NewsAndTalk"), + ("news radio", "NewsAndTalk"), + ("sports radio", "Sports"), + ("christian music", "ChristianAndGospel"), + ("gospel music", "ChristianAndGospel"), + ("oldies", "Oldies"), + ("pop music", "Pop"), + ("jazz", "Jazz"), + ("latin music", "Latin"), + ("dance music", "Dance"), + ("reggae", "ReggaeAndIsland"), + ("island music", "ReggaeAndIsland"), + ("alternative", "Alternative"), + ("blues", "Blues"), + ("classical music", "Classical"), + ("classical", "Classical"), + ("college radio", "CollegeRadio"), + ("comedy radio", "Comedy"), + ("npr", "NPR") + ]; } public sealed record JiboInteractionDecision( diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs index cd03546..d877c8c 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs @@ -23,9 +23,14 @@ public sealed class ResponsePlanToSocketMessagesMapper string.Equals(plan.IntentName, "no", StringComparison.OrdinalIgnoreCase); var isWordOfDayLaunch = string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase); var isWordOfDayGuess = string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase); + var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) || + string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase); + var radioStation = ReadSkillPayloadString(skill, "station"); var nluGuess = ReadClientEntity(turn, "guess"); var wordOfDayGuess = ResolveWordOfDayGuess(turn, transcript, nluGuess); var outboundIntent = isWordOfDayLaunch + ? "menu" + : isRadioLaunch ? "menu" : isWordOfDayGuess ? "guess" @@ -36,6 +41,8 @@ public sealed class ResponsePlanToSocketMessagesMapper ? wordOfDayGuess : isWordOfDayLaunch ? string.Empty + : isRadioLaunch + ? transcript : string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(nluGuess) ? nluGuess : isYesNoTurn && isYesNoIntent @@ -45,10 +52,12 @@ public sealed class ResponsePlanToSocketMessagesMapper : transcript; var outboundRules = isWordOfDayLaunch ? ["word-of-the-day/menu"] + : isRadioLaunch + ? Array.Empty() : isWordOfDayGuess ? ["word-of-the-day/puzzle"] : isYesNoTurn && isYesNoIntent ? [yesNoRule!] : rules; - var entities = ReadEntities(turn, messageType, isYesNoTurn && isYesNoIntent, isWordOfDayLaunch, isWordOfDayGuess, wordOfDayGuess); + var entities = ReadEntities(turn, messageType, isYesNoTurn && isYesNoIntent, isWordOfDayLaunch, isRadioLaunch, isWordOfDayGuess, wordOfDayGuess, radioStation); var listenMessage = new { type = "LISTEN", @@ -61,7 +70,7 @@ public sealed class ResponsePlanToSocketMessagesMapper final = true, text = outboundAsrText }, - nlu = BuildNluPayload(outboundIntent, outboundRules, entities, isWordOfDayLaunch ? "@be/word-of-the-day" : null), + nlu = BuildNluPayload(outboundIntent, outboundRules, entities, isWordOfDayLaunch ? "@be/word-of-the-day" : isRadioLaunch ? "@be/radio" : null), match = new { intent = outboundIntent, @@ -100,6 +109,22 @@ public sealed class ResponsePlanToSocketMessagesMapper DelayMs: 125)); } + if (isRadioLaunch) + { + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildSkillRedirectPayload( + transId, + "@be/radio", + outboundIntent, + outboundAsrText, + outboundRules, + entities)), + DelayMs: 75)); + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/radio")), + DelayMs: 125)); + } + if (emitSkillActions && speak is not null) { messages.Add(new SocketReplyPlan( @@ -242,8 +267,10 @@ public sealed class ResponsePlanToSocketMessagesMapper string? messageType, bool yesNoCreateTurn, bool wordOfDayLaunch, + bool radioLaunch, bool wordOfDayGuess, - string? guess) + string? guess, + string? radioStation) { if (yesNoCreateTurn) { @@ -261,6 +288,17 @@ public sealed class ResponsePlanToSocketMessagesMapper }; } + if (radioLaunch) + { + var entities = new Dictionary(); + if (!string.IsNullOrWhiteSpace(radioStation)) + { + entities["station"] = radioStation; + } + + return entities; + } + if (wordOfDayGuess) { return new Dictionary diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs index 38dfbf1..b7bfac4 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs @@ -544,6 +544,8 @@ public sealed class WebSocketTurnFinalizationService( : DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow); var emitSkillActions = !string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase) && (messageType != "CLIENT_NLU" || string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase)); var replies = ResponsePlanToSocketMessagesMapper.Map(plan, finalizedTurn, session, emitSkillActions).Select(map => new WebSocketReply diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index d79b21d..1efd92a 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -129,6 +129,38 @@ public sealed class JiboInteractionServiceTests Assert.Equal("joke", decision.IntentName); } + [Fact] + public async Task BuildDecisionAsync_OpenTheRadio_MapsToRadioLaunchIntent() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "open the radio", + NormalizedTranscript = "open the radio" + }); + + Assert.Equal("radio", decision.IntentName); + Assert.Equal("@be/radio", decision.SkillName); + Assert.Equal("@be/radio", decision.SkillPayload!["skillId"]); + } + + [Fact] + public async Task BuildDecisionAsync_PlayCountryMusic_MapsToRadioGenreLaunchIntent() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "play country music", + NormalizedTranscript = "play country music" + }); + + Assert.Equal("radio_genre", decision.IntentName); + Assert.Equal("@be/radio", decision.SkillName); + Assert.Equal("Country", decision.SkillPayload!["station"]); + } + [Fact] public async Task BuildDecisionAsync_WordOfDayGuess_UsesStructuredClientNluGuess() { diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 3fcd6d9..db4323e 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -403,6 +403,80 @@ public sealed class JiboWebSocketServiceTests Assert.Equal("surprises-ota/want_to_download_now", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString()); } + [Fact] + public async Task ClientAsr_OpenTheRadio_EmitsRadioRedirectAndSilentCompletion() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-radio-open-token", + Text = """{"type":"LISTEN","transID":"trans-radio-open","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-radio-open-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-radio-open","data":{"text":"open the radio"}}""" + }); + + Assert.Equal(4, replies.Count); + Assert.Equal("LISTEN", ReadReplyType(replies[0])); + Assert.Equal("EOS", ReadReplyType(replies[1])); + Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); + Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3])); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + Assert.Equal("menu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + Assert.Equal("@be/radio", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString()); + Assert.Equal(0, listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules").GetArrayLength()); + Assert.Equal(0, listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").EnumerateObject().Count()); + + using var redirectPayload = JsonDocument.Parse(replies[2].Text!); + Assert.Equal("@be/radio", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString()); + Assert.True(redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("launch").GetBoolean()); + Assert.Equal("menu", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + + using var completionPayload = JsonDocument.Parse(replies[3].Text!); + Assert.Equal("@be/radio", completionPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString()); + } + + [Fact] + public async Task ClientAsr_PlayCountryMusic_EmitsRadioRedirectWithCountryStation() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-radio-country-token", + Text = """{"type":"LISTEN","transID":"trans-radio-country","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-radio-country-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-radio-country","data":{"text":"play country music"}}""" + }); + + Assert.Equal(4, replies.Count); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + Assert.Equal("menu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + Assert.Equal("Country", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("station").GetString()); + + using var redirectPayload = JsonDocument.Parse(replies[2].Text!); + Assert.Equal("Country", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("station").GetString()); + Assert.Equal("play country music", redirectPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); + } + [Fact] public async Task ClientNlu_WordOfDayGuess_UsesGuessEntityAsAsrTextAndCompletesTurn() {