From acbba413db2c7600062d2695634ec8729e97aca7 Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Sun, 26 Apr 2026 20:19:16 -0500 Subject: [PATCH] a couple more features for version 18 --- OpenJibo/docs/development-plan.md | 7 +- OpenJibo/docs/feature-backlog.md | 57 +++-- OpenJibo/docs/regression-test-plan.md | 30 ++- .../Services/DemoConversationBroker.cs | 5 + .../Services/JiboInteractionService.cs | 214 ++++++++++++++++++ .../ResponsePlanToSocketMessagesMapper.cs | 90 +++++++- .../WebSocketTurnFinalizationService.cs | 5 + .../WebSockets/JiboInteractionServiceTests.cs | 67 ++++++ .../WebSockets/JiboWebSocketServiceTests.cs | 135 +++++++++++ 9 files changed, 586 insertions(+), 24 deletions(-) diff --git a/OpenJibo/docs/development-plan.md b/OpenJibo/docs/development-plan.md index 6e90e13..cfefc70 100644 --- a/OpenJibo/docs/development-plan.md +++ b/OpenJibo/docs/development-plan.md @@ -91,6 +91,8 @@ The following behavior is present in source and covered by focused tests: - apostrophes are no longer escaped to `'` in spoken ESML, while `&`, `<`, `>`, and `"` remain escaped - radio voice launch supports `open the radio` and genre launch such as `play country music`, using local `@be/radio` `menu` payloads, `SKILL_REDIRECT`, and silent completion - news has a first Nimbus-shaped cloud path using `match.cloudSkill = news` and a `news` `SKILL_ACTION` with synthetic briefing content +- stop commands such as `stop that` and `never mind` emit stock `global_commands` `stop` NLU plus a local `@be/idle` redirect, without generic chat speech +- volume commands emit stock `global_commands` volume intents: `volumeUp`, `volumeDown`, and `volumeToValue` with `volumeLevel`; `show volume controls` redirects to `@be/settings` `volumeQuery` - stock-shaped clock handoffs cover time, date, day, clock open, timer/alarm menu, timer/alarm value, timer/alarm clarification, and timer/alarm delete - alarm parsing covers forms such as `7:30 am`, `830`, `8 30`, `7, 44`, `10-25`, `10:25 pm`, and `10 25 p m` - ambiguous alarm times can prefer the next local occurrence when the robot context includes `runtime.location.iso` @@ -147,6 +149,7 @@ Before calling `1.0.18` complete, prove or explicitly defer these: - Regression test photo/gallery flows again after the `jibo test 24` fixes: open gallery, answer the stock `shared/yes_no` prompt with a transcript-bearing `yes`, hand into create, take one photo, keep it, and avoid blue-ring or `I heard you` stale turns. - Live-test radio launch: `open the radio` passed in `jibo test 22`; re-run `play country music` if that exact phrase was not captured. - Treat basic news as live-proven by `jibo test 23`; defer provider-backed or category-expanded news unless it is chosen as an optional feature slice. +- Regression test the added stop and volume slices: `stop that`, `never mind`, `turn it up`, `turn it down`, `set volume to six`, and `show volume controls`. - Recheck constrained yes/no prompts for update/backup/share/gallery/alarm replacement without leaking global rules. - Recheck that stock OS no longer logs OpenJibo-only websocket events such as synthetic pending/context/ack packets from the current build. - Recheck backup/update behavior with explicit attention to robot-local `jibo.scheduler.backupStatus`, CPU/load, and whether the deployed cloud is involved at all. @@ -164,13 +167,13 @@ These are not blockers for calling `1.0.18` complete unless the live test shows - news content is synthetic; `jibo test 23` proved the path but not live provider-backed headlines - gallery `shared/yes_no` still needs a successful transcript-bearing live `yes` pass - weather, calendar, commute, personal report, identity, memory, and proactivity are still mostly discovery or placeholder content paths -- volume, stop, robot age, and command-versus-question personality routing are not implemented yet +- stop and volume are implemented but still need live stock-OS proof; robot age and command-versus-question personality routing are not implemented yet ## `1.0.19` Direction After `1.0.18` is tested and tagged, `1.0.19` should move back into feature work: -- one lightweight device-control feature, most likely stop or volume +- harden whichever stop/volume behavior is not fully proven by the `1.0.18` live pass, or pick the next lightweight device/persona slice - end-to-end update/backup/restore proof - STT reliability improvements, including noise screening and a managed STT comparison - provider-backed first content path, likely news or weather diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index 4ab8be6..5ef10ca 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -147,17 +147,26 @@ Current release theme: ### 5. Optional Small Feature Before `1.0.18` Freeze -- Status: `ready` +- Status: `implemented` - Tags: `protocol` - Why now: the user wants one or two features before `1.0.18` is called complete, but the release should not take on a risky subsystem. -- Preferred candidates: +- Selected slices: - Stop command - - Volume up / volume down voice control - - How old are you / robot age persona -- Guidance: - - pick only one if the live regression pass finds bugs - - pick at most two if the current bug-fix paths stay stable - - keep the implementation source-backed and easy to revert or defer + - Volume up / volume down / set-to-value voice control +- Current code: + - `stop`, `stop that`, and `never mind` map to stock `global_commands` `stop` NLU plus local `@be/idle` redirect/completion + - `turn it up` and `turn it down` emit stock `global_commands` `volumeUp` / `volumeDown` with `volumeLevel = null` and no cloud speech + - `set volume to six` emits stock `global_commands` `volumeToValue` with `volumeLevel = 6` and no cloud speech + - `show volume controls` redirects into `@be/settings` with `volumeQuery` +- Evidence: + - Pegasus `globals/global_commands_launch.rule` defines `stop`, `volumeUp`, `volumeDown`, and `volumeToValue` + - stock Jibo `VolumePlugin` subscribes to global volume events and uses the same intent/entity names + - stock `@be/settings` exposes `volumeQuery` and opens the volume panel +- Exit criteria: + - live stop settles the robot without a generic chat reply + - live volume up/down audibly changes volume or logs a local volume event + - live volume-to-value changes the setting to the requested value or logs the expected stock local handling + - live volume controls opens the settings volume panel ## Implemented In Current Source @@ -248,6 +257,19 @@ Current release theme: - Follow-up: - keep this in regression coverage because it shares turn-state machinery with gallery and alarm flows +### Stop And Volume First Pass + +- Status: `implemented` +- Tags: `protocol` +- Result: + - global stop commands emit stock `global_commands` `stop` and redirect to `@be/idle` + - relative volume commands emit stock `global_commands` `volumeUp` / `volumeDown` + - absolute volume commands emit `volumeToValue` with a `volumeLevel` entity + - volume controls launch redirects to `@be/settings` `volumeQuery` + - websocket responses avoid generic chat speech for these local/global command paths +- Follow-up: + - live validation remains in the immediate queue because volume depends on stock robot local global-command handling + ### Unknown OpenJibo Event Noise - Status: `implemented` @@ -272,7 +294,7 @@ Current release theme: ### 6. Stop Command -- Status: `ready` +- Status: `polish` - Tags: `protocol` - User goals: - `stop` @@ -280,15 +302,15 @@ Current release theme: - `never mind` - Evidence: - `@be/idle` exists and is already used as a cleanup redirect target + - current `1.0.18` source emits stock `global_commands` `stop` plus local `@be/idle` redirect - Questions: - - whether stock source has a dedicated stop/cancel intent beyond idle redirect - - whether stop should interrupt active local skills or only cloud speech paths in the first pass + - whether live stock OS treats the combined global stop plus idle redirect as cleanly as expected during active local skills - Exit criteria: - a spoken stop command settles the robot locally without a generic chat reply ### 7. Volume Up / Volume Down Voice Control -- Status: `ready` +- Status: `polish` - Tags: `protocol` - User goals: - `turn it up` @@ -296,10 +318,11 @@ Current release theme: - `increase the volume` - `decrease the volume` - Evidence: - - stock Jibo exposes volume control through robot UX, so there should be a local control or settings path to mirror + - Pegasus global commands define `volumeUp`, `volumeDown`, and `volumeToValue` + - stock Jibo `VolumePlugin` listens for those global intents and `volumeLevel` + - current `1.0.18` source emits those stock NLU shapes and opens `@be/settings` `volumeQuery` - Questions: - - exact local payload shape for relative volume changes - - whether first pass should support absolute values such as `set volume to 5` + - whether live stock OS applies the global volume event from the hosted cloud response without any additional local event payload - Exit criteria: - relative voice volume commands adjust volume without generic cloud speech @@ -505,13 +528,13 @@ Before closing `1.0.18`: 2. Basic news regression, with provider-backed expansion deferred 3. Backup / OTA / share yes-no regression 4. Alarm and photo/gallery regression -5. Optional small feature only if the regression pass stays calm +5. Stop and volume first-pass validation Use [regression-test-plan.md](regression-test-plan.md) as the detailed checklist for this sequence. For `1.0.19`: -1. Stop command or volume control +1. Harden stop or volume if the `1.0.18` live pass exposes stock-OS quirks; otherwise pick robot age/persona or another lightweight slice 2. Update, backup, and restore proof 3. STT upgrade and noise screening 4. Hosted capture/storage plan diff --git a/OpenJibo/docs/regression-test-plan.md b/OpenJibo/docs/regression-test-plan.md index 92df362..261e384 100644 --- a/OpenJibo/docs/regression-test-plan.md +++ b/OpenJibo/docs/regression-test-plan.md @@ -168,6 +168,34 @@ Expected: - no `ffmpeg` failure should become the dominant failure mode for non-Opus buffered audio - short replies such as `yes`, `no`, `cancel`, and short alarm times should either map correctly or be classified as STT misses with evidence +### Stop And Volume + +Goal: prove the added lightweight device-control slice before closing `1.0.18`. + +Test these phrases: + +- `stop` +- `stop that` +- `never mind` +- `turn it up` +- `turn it down` +- `set volume to six` +- `show volume controls` + +Expected: + +- stop commands settle the robot locally without generic chat speech +- `turn it up` and `turn it down` adjust volume or at least produce the stock local volume event/log +- `set volume to six` sets or attempts to set the local volume level to `6` +- `show volume controls` opens the settings volume panel + +Capture check: + +- stop emits `nlu.intent = stop`, `nlu.domain = global_commands`, then redirects to `@be/idle` +- relative volume emits `nlu.intent = volumeUp` or `volumeDown`, `nlu.domain = global_commands`, and `entities.volumeLevel = null`, with no `SKILL_ACTION` cloud speech +- absolute volume emits `nlu.intent = volumeToValue` and `entities.volumeLevel` matching the requested value, with no `SKILL_ACTION` cloud speech +- volume controls redirects to `@be/settings` with `nlu.intent = volumeQuery` + ## Optional Feature Slice Checks When a new feature is added before a release closes: @@ -178,8 +206,6 @@ When a new feature is added before a release closes: For the current candidate list, add cases here when implemented: -- stop command: `stop`, `stop that`, `never mind` -- volume: `turn it up`, `turn it down`, `increase the volume`, `decrease the volume` - robot age/persona: `how old are you` ## After The Run diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs index eb1f0f9..692ffac 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs @@ -74,6 +74,11 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer "word_of_the_day_guess" => false, "radio" => false, "radio_genre" => false, + "stop" => false, + "volume_up" => false, + "volume_down" => false, + "volume_to_value" => false, + "volume_query" => false, "time" => false, "date" => false, "day" => false, diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs index dd256c1..59bf09f 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs @@ -51,6 +51,11 @@ public sealed class JiboInteractionService( "cloud_version" => new JiboInteractionDecision("cloud_version", OpenJiboCloudBuildInfo.SpokenVersion), "radio" => BuildRadioLaunchDecision(), "radio_genre" => BuildRadioGenreLaunchDecision(lowered), + "stop" => BuildStopDecision(), + "volume_up" => BuildVolumeControlDecision("volume_up", "volumeUp", "null"), + "volume_down" => BuildVolumeControlDecision("volume_down", "volumeDown", "null"), + "volume_to_value" => BuildVolumeControlDecision("volume_to_value", "volumeToValue", ResolveVolumeLevel(lowered, clientEntities) ?? "7"), + "volume_query" => BuildSettingsVolumeDecision(), "clock_open" => BuildClockLaunchDecision("clock_open", "clock", "askForTime", "Opening the clock."), "clock_menu" => BuildClockLaunchDecision("clock_menu", "clock", "menu", "Opening the clock menu."), "timer_menu" => BuildClockLaunchDecision("timer", "Opening the timer."), @@ -309,6 +314,27 @@ public sealed class JiboInteractionService( return "radio_genre"; } + if (TryResolveVolumeLevel(loweredTranscript) is not null || + clientEntities.ContainsKey("volumeLevel")) + { + return "volume_to_value"; + } + + if (IsVolumeQueryRequest(loweredTranscript)) + { + return "volume_query"; + } + + if (IsVolumeUpRequest(loweredTranscript)) + { + return "volume_up"; + } + + if (IsVolumeDownRequest(loweredTranscript)) + { + return "volume_down"; + } + if (MatchesAny(loweredTranscript, "open the clock", "open clock", "show the clock", "show clock")) { return "clock_open"; @@ -346,6 +372,11 @@ public sealed class JiboInteractionService( return "timer_delete"; } + if (IsGlobalStopRequest(loweredTranscript, clientIntent, clientEntities)) + { + return "stop"; + } + if (TryParseAlarmValue(loweredTranscript, isAlarmValueTurn, referenceLocalTime) is not null) { return "alarm_value"; @@ -535,6 +566,47 @@ public sealed class JiboInteractionService( }); } + private static JiboInteractionDecision BuildStopDecision() + { + return new JiboInteractionDecision( + "stop", + "Stopping.", + "@be/idle", + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["skillId"] = "@be/idle", + ["globalIntent"] = "stop", + ["nluDomain"] = "global_commands" + }); + } + + private static JiboInteractionDecision BuildVolumeControlDecision(string intentName, string globalIntent, string volumeLevel) + { + return new JiboInteractionDecision( + intentName, + "Adjusting volume.", + "global_commands", + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["globalIntent"] = globalIntent, + ["nluDomain"] = "global_commands", + ["volumeLevel"] = volumeLevel + }); + } + + private static JiboInteractionDecision BuildSettingsVolumeDecision() + { + return new JiboInteractionDecision( + "volume_query", + "Opening volume controls.", + "@be/settings", + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["skillId"] = "@be/settings", + ["localIntent"] = "volumeQuery" + }); + } + private static JiboInteractionDecision BuildClockLaunchDecision(string intentName, string domain, string clockIntent, string replyText) { return new JiboInteractionDecision( @@ -1120,6 +1192,144 @@ public sealed class JiboInteractionService( loweredTranscript is "cancel" or "stop" or "never mind" or "nevermind"; } + private static bool IsGlobalStopRequest( + string loweredTranscript, + string? clientIntent, + IReadOnlyDictionary clientEntities) + { + if (string.Equals(clientIntent, "stop", StringComparison.OrdinalIgnoreCase) && + IsGlobalCommandsDomain(clientEntities)) + { + return true; + } + + return loweredTranscript is "stop" or "stop it" or "stop that" or "stop talking" or "be quiet" or "never mind" or "nevermind" or "forget it" || + MatchesAny(loweredTranscript, "that s enough", "that's enough", "that will do", "that ll do", "that'll do", "cut it out", "cut that out"); + } + + private static bool IsVolumeQueryRequest(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "volume controls", + "volume control", + "volume menu", + "volume level", + "show volume", + "show the volume", + "open volume", + "open the volume", + "what is your volume", + "what's your volume", + "how is your volume", + "how s your volume"); + } + + private static bool IsVolumeUpRequest(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "turn it up", + "turn this up", + "turn that up", + "turn up the volume", + "turn the volume up", + "turn volume up", + "turn your volume up", + "increase the volume", + "increase your volume", + "raise the volume", + "raise your volume", + "make it louder", + "make that louder", + "speak louder", + "talk louder", + "be louder", + "louder"); + } + + private static bool IsVolumeDownRequest(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "turn it down", + "turn this down", + "turn that down", + "turn down the volume", + "turn the volume down", + "turn volume down", + "turn your volume down", + "decrease the volume", + "decrease your volume", + "lower the volume", + "lower your volume", + "make it quieter", + "make that quieter", + "make it softer", + "speak quieter", + "talk quieter", + "be quieter", + "quieter", + "softer"); + } + + private static string? ResolveVolumeLevel(string loweredTranscript, IReadOnlyDictionary clientEntities) + { + if (clientEntities.TryGetValue("volumeLevel", out var entityValue) && + TryNormalizeVolumeLevel(entityValue) is { } structuredLevel) + { + return structuredLevel; + } + + return TryResolveVolumeLevel(loweredTranscript); + } + + private static string? TryResolveVolumeLevel(string loweredTranscript) + { + if (!loweredTranscript.Contains("volume", StringComparison.Ordinal) && + !loweredTranscript.Contains("loudness", StringComparison.Ordinal)) + { + return null; + } + + if (MatchesAny(loweredTranscript, "max volume", "maximum volume", "volume max", "volume maximum")) + { + return "10"; + } + + if (MatchesAny(loweredTranscript, "min volume", "minimum volume", "volume min", "volume minimum")) + { + return "1"; + } + + var match = VolumeLevelPattern.Match(loweredTranscript); + if (!match.Success) + { + return null; + } + + return TryNormalizeVolumeLevel(match.Groups["value"].Value); + } + + private static string? TryNormalizeVolumeLevel(string token) + { + if (string.Equals(token, "null", StringComparison.OrdinalIgnoreCase)) + { + return "null"; + } + + var parsed = ParseNumberToken(token); + return parsed is >= 1 and <= 10 + ? parsed.Value.ToString() + : null; + } + + private static bool IsGlobalCommandsDomain(IReadOnlyDictionary clientEntities) + { + return clientEntities.TryGetValue("domain", out var domain) && + string.Equals(domain, "global_commands", StringComparison.OrdinalIgnoreCase); + } + private static bool IsClockTimerValueTurn( IReadOnlyList clientRules, IReadOnlyList listenRules) @@ -1235,6 +1445,10 @@ public sealed class JiboInteractionService( @"\b(?\d{3,4})\s*(?a[\s\.]*m\.?|p[\s\.]*m\.?)?\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + private static readonly Regex VolumeLevelPattern = new( + @"\b(?:volume|loudness)\s*(?:to|at|level|is)?\s*(?10|\d|one|two|three|four|five|six|seven|eight|nine|ten)\b|\b(?:set|change|make|turn)\s+(?:the\s+|your\s+)?(?:volume|loudness)\s*(?:to|at)?\s*(?10|\d|one|two|three|four|five|six|seven|eight|nine|ten)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + private static readonly (string Phrase, string Station)[] RadioGenreAliases = [ ("country music", "Country"), diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs index 8bb1ad5..9ba3018 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs @@ -25,6 +25,12 @@ public sealed class ResponsePlanToSocketMessagesMapper var isWordOfDayGuess = string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase); var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) || string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase); + var isStopCommand = string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase); + var isVolumeControl = string.Equals(plan.IntentName, "volume_up", StringComparison.OrdinalIgnoreCase) || + string.Equals(plan.IntentName, "volume_down", StringComparison.OrdinalIgnoreCase) || + string.Equals(plan.IntentName, "volume_to_value", StringComparison.OrdinalIgnoreCase); + var isSettingsLaunch = string.Equals(skill?.SkillName, "@be/settings", StringComparison.OrdinalIgnoreCase); + var isGlobalCommand = isStopCommand || isVolumeControl; var isPhotoGalleryLaunch = string.Equals(plan.IntentName, "photo_gallery", StringComparison.OrdinalIgnoreCase); var isPhotoCreateLaunch = string.Equals(plan.IntentName, "snapshot", StringComparison.OrdinalIgnoreCase) || string.Equals(plan.IntentName, "photobooth", StringComparison.OrdinalIgnoreCase); @@ -39,12 +45,19 @@ public sealed class ResponsePlanToSocketMessagesMapper var alarmAmPm = ReadSkillPayloadString(skill, "ampm"); var radioStation = ReadSkillPayloadString(skill, "station"); var cloudSkill = ReadSkillPayloadString(skill, "cloudSkill"); + var globalIntent = ReadSkillPayloadString(skill, "globalIntent"); + var nluDomain = ReadSkillPayloadString(skill, "nluDomain"); + var volumeLevel = ReadSkillPayloadString(skill, "volumeLevel"); var nluGuess = ReadClientEntity(turn, "guess"); var wordOfDayGuess = ResolveWordOfDayGuess(turn, transcript, nluGuess); - var outboundIntent = isWordOfDayLaunch + var outboundIntent = isGlobalCommand && !string.IsNullOrWhiteSpace(globalIntent) + ? globalIntent + : isWordOfDayLaunch ? "menu" : isRadioLaunch ? "menu" + : isSettingsLaunch && !string.IsNullOrWhiteSpace(localIntent) + ? localIntent : (isPhotoGalleryLaunch || isPhotoCreateLaunch) && !string.IsNullOrWhiteSpace(localIntent) ? localIntent : isClockSkillLaunch && !string.IsNullOrWhiteSpace(clockIntent) @@ -58,8 +71,12 @@ public sealed class ResponsePlanToSocketMessagesMapper ? wordOfDayGuess : isWordOfDayLaunch ? string.Empty + : isGlobalCommand + ? transcript : isRadioLaunch ? transcript + : isSettingsLaunch + ? transcript : isPhotoGalleryLaunch || isPhotoCreateLaunch ? transcript : isClockSkillLaunch @@ -73,8 +90,12 @@ public sealed class ResponsePlanToSocketMessagesMapper : transcript; var outboundRules = isWordOfDayLaunch ? ["word-of-the-day/menu"] + : isGlobalCommand + ? BuildGlobalCommandRules(rules) : isRadioLaunch ? Array.Empty() + : isSettingsLaunch + ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty() : isPhotoGalleryLaunch || isPhotoCreateLaunch ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty() : isClockSkillLaunch @@ -88,6 +109,8 @@ public sealed class ResponsePlanToSocketMessagesMapper isYesNoTurn && isYesNoIntent, ShouldIncludeCreateDomain(yesNoRule), isWordOfDayLaunch, + isGlobalCommand, + volumeLevel, isRadioLaunch, isWordOfDayGuess, wordOfDayGuess, @@ -118,10 +141,12 @@ public sealed class ResponsePlanToSocketMessagesMapper entities, isWordOfDayLaunch ? "@be/word-of-the-day" : isRadioLaunch ? "@be/radio" : + isSettingsLaunch ? "@be/settings" : isPhotoGalleryLaunch ? "@be/gallery" : isPhotoCreateLaunch ? "@be/create" : isClockSkillLaunch ? "@be/clock" : - null), + null, + isGlobalCommand ? nluDomain ?? "global_commands" : null), match = new { intent = outboundIntent, @@ -177,6 +202,39 @@ public sealed class ResponsePlanToSocketMessagesMapper DelayMs: 125)); } + if (isStopCommand) + { + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildSkillRedirectPayload( + transId, + "@be/idle", + outboundIntent, + outboundAsrText, + outboundRules, + entities)), + DelayMs: 75)); + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/idle")), + DelayMs: 125)); + } + + if (isSettingsLaunch && + !string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)) + { + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildSkillRedirectPayload( + transId, + "@be/settings", + outboundIntent, + outboundAsrText, + outboundRules, + entities)), + DelayMs: 75)); + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/settings")), + DelayMs: 125)); + } + if (isClockSkillLaunch && !string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)) { @@ -355,6 +413,8 @@ public sealed class ResponsePlanToSocketMessagesMapper bool yesNoTurn, bool includeCreateDomain, bool wordOfDayLaunch, + bool globalCommand, + string? volumeLevel, bool radioLaunch, bool wordOfDayGuess, string? guess, @@ -389,6 +449,17 @@ public sealed class ResponsePlanToSocketMessagesMapper }; } + if (globalCommand) + { + var entities = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (!string.IsNullOrWhiteSpace(volumeLevel)) + { + entities["volumeLevel"] = volumeLevel; + } + + return entities; + } + if (radioLaunch) { var entities = new Dictionary(); @@ -702,7 +773,8 @@ public sealed class ResponsePlanToSocketMessagesMapper string outboundIntent, IReadOnlyList outboundRules, object entities, - string? skillId) + string? skillId, + string? domain = null) { var payload = new Dictionary(StringComparer.OrdinalIgnoreCase) { @@ -717,9 +789,21 @@ public sealed class ResponsePlanToSocketMessagesMapper payload["skill"] = skillId; } + if (!string.IsNullOrWhiteSpace(domain)) + { + payload["domain"] = domain; + } + return payload; } + private static IReadOnlyList BuildGlobalCommandRules(IReadOnlyList rules) + { + return rules.Any(static rule => string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase)) + ? ["globals/global_commands_launch"] + : Array.Empty(); + } + private static object BuildGenericFallbackSkillPayload(string transId) { return new diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs index 76c51a8..5014c10 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs @@ -504,6 +504,11 @@ public sealed class WebSocketTurnFinalizationService( var emitSkillActions = !string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) && !string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) && !string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "volume_up", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "volume_down", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "volume_to_value", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "volume_query", StringComparison.OrdinalIgnoreCase) && !string.Equals(plan.IntentName, "time", StringComparison.OrdinalIgnoreCase) && !string.Equals(plan.IntentName, "date", StringComparison.OrdinalIgnoreCase) && !string.Equals(plan.IntentName, "day", StringComparison.OrdinalIgnoreCase) && diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index 1bca54a..cf61e3b 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -258,6 +258,73 @@ public sealed class JiboInteractionServiceTests Assert.Equal("Country", decision.SkillPayload!["station"]); } + [Fact] + public async Task BuildDecisionAsync_StopThat_MapsToIdleStopCommand() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "stop that", + NormalizedTranscript = "stop that" + }); + + Assert.Equal("stop", decision.IntentName); + Assert.Equal("@be/idle", decision.SkillName); + Assert.Equal("stop", decision.SkillPayload!["globalIntent"]); + Assert.Equal("global_commands", decision.SkillPayload["nluDomain"]); + } + + [Fact] + public async Task BuildDecisionAsync_TurnItUp_MapsToGlobalVolumeUpCommand() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "turn it up", + NormalizedTranscript = "turn it up" + }); + + Assert.Equal("volume_up", decision.IntentName); + Assert.Equal("global_commands", decision.SkillName); + Assert.Equal("volumeUp", decision.SkillPayload!["globalIntent"]); + Assert.Equal("null", decision.SkillPayload["volumeLevel"]); + } + + [Fact] + public async Task BuildDecisionAsync_SetVolumeToSix_MapsToGlobalVolumeToValueCommand() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "set volume to six", + NormalizedTranscript = "set volume to six" + }); + + Assert.Equal("volume_to_value", decision.IntentName); + Assert.Equal("global_commands", decision.SkillName); + Assert.Equal("volumeToValue", decision.SkillPayload!["globalIntent"]); + Assert.Equal("6", decision.SkillPayload["volumeLevel"]); + } + + [Fact] + public async Task BuildDecisionAsync_ShowVolumeControls_MapsToSettingsVolumeQuery() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "show volume controls", + NormalizedTranscript = "show volume controls" + }); + + Assert.Equal("volume_query", decision.IntentName); + Assert.Equal("@be/settings", decision.SkillName); + Assert.Equal("volumeQuery", decision.SkillPayload!["localIntent"]); + } + [Fact] public async Task BuildDecisionAsync_OpenTimer_MapsToLocalClockTimerMenu() { diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 2636c3e..26cf4fb 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -1583,6 +1583,141 @@ public sealed class JiboWebSocketServiceTests Assert.Equal("play country music", redirectPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); } + [Fact] + public async Task ClientAsr_StopThat_EmitsGlobalStopAndIdleRedirect() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-stop-token", + Text = """{"type":"LISTEN","transID":"trans-stop","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-stop-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-stop","data":{"text":"stop that"}}""" + }); + + Assert.Equal(4, replies.Count); + Assert.Equal("LISTEN", ReadReplyType(replies[0])); + Assert.Equal("EOS", ReadReplyType(replies[1])); + Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); + Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3])); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + var nlu = listenPayload.RootElement.GetProperty("data").GetProperty("nlu"); + Assert.Equal("stop", nlu.GetProperty("intent").GetString()); + Assert.Equal("global_commands", nlu.GetProperty("domain").GetString()); + Assert.Equal("globals/global_commands_launch", nlu.GetProperty("rules")[0].GetString()); + + using var redirectPayload = JsonDocument.Parse(replies[2].Text!); + Assert.Equal("@be/idle", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString()); + Assert.Equal("stop", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + } + + [Fact] + public async Task ClientAsr_TurnItDown_EmitsGlobalVolumeDownWithoutCloudSpeech() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-volume-down-token", + Text = """{"type":"LISTEN","transID":"trans-volume-down","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-volume-down-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-volume-down","data":{"text":"turn it down"}}""" + }); + + Assert.Equal(2, replies.Count); + Assert.Equal("LISTEN", ReadReplyType(replies[0])); + Assert.Equal("EOS", ReadReplyType(replies[1])); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + var nlu = listenPayload.RootElement.GetProperty("data").GetProperty("nlu"); + Assert.Equal("volumeDown", nlu.GetProperty("intent").GetString()); + Assert.Equal("global_commands", nlu.GetProperty("domain").GetString()); + Assert.Equal("null", nlu.GetProperty("entities").GetProperty("volumeLevel").GetString()); + Assert.Equal("globals/global_commands_launch", nlu.GetProperty("rules")[0].GetString()); + } + + [Fact] + public async Task ClientAsr_SetVolumeToSix_EmitsGlobalVolumeToValue() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-volume-value-token", + Text = """{"type":"LISTEN","transID":"trans-volume-value","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-volume-value-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-volume-value","data":{"text":"set volume to six"}}""" + }); + + Assert.Equal(2, replies.Count); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + var nlu = listenPayload.RootElement.GetProperty("data").GetProperty("nlu"); + Assert.Equal("volumeToValue", nlu.GetProperty("intent").GetString()); + Assert.Equal("6", nlu.GetProperty("entities").GetProperty("volumeLevel").GetString()); + Assert.Equal("global_commands", nlu.GetProperty("domain").GetString()); + } + + [Fact] + public async Task ClientAsr_ShowVolumeControls_RedirectsIntoSettingsVolumeQuery() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-volume-query-token", + Text = """{"type":"LISTEN","transID":"trans-volume-query","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-volume-query-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-volume-query","data":{"text":"show volume controls"}}""" + }); + + Assert.Equal(4, replies.Count); + Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); + Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3])); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + Assert.Equal("volumeQuery", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + Assert.Equal("@be/settings", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString()); + + using var redirectPayload = JsonDocument.Parse(replies[2].Text!); + Assert.Equal("@be/settings", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString()); + Assert.Equal("volumeQuery", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + } + [Fact] public async Task ClientNlu_WordOfDayGuess_UsesGuessEntityAsAsrTextAndCompletesTurn() {