diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index bcba023..4f2c978 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -773,8 +773,8 @@ For `1.0.19`: 2. Expand memory-backed personal facts with tenant-scoped storage (beyond the first birthday/preferences foundation) - implemented 3. Proactivity selector baseline with source-backed first offers - implemented 4. Weather report-skill launch compatibility - implemented -5. Dialog parsing expansion and ambiguity guardrails - in progress (`2026-05-07` first guardrail slice implemented) -6. Presence-aware greetings and identity-triggered proactivity - ready +5. Dialog parsing expansion and ambiguity guardrails - in progress (`2026-05-09` third guardrail slice implemented; Pegasus affinity phrase families + continuation guardrails expanded) +6. Presence-aware greetings and identity-triggered proactivity - implemented (trigger path, identity-aware reactive/proactive replies, cooldown metadata wiring, focused websocket coverage) 7. Personal report parity track (weather visuals, live news path, commute path, calendar parity matrix) - ready 8. Holidays and seasonal personality behavior built on the new memory/proactivity foundation 9. Durable memory persistence path (multi-tenant backing store) diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index 1db9c6f..b8160a3 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -172,6 +172,16 @@ Second completed guardrail slice under this queue: - weather variants (`what's today's weather look like`, `will it be sunny tomorrow`) - listener continuation guardrail now differentiates incomplete preference fragments from complete shorthand preference sets +Third completed guardrail slice under this queue: + +- expanded Pegasus `userLikesThing` / `userDislikesThing` / `doesUserLikeThing` / `doesUserDislikeThing` phrase-family coverage + - includes additional dislike/negation variants (`loathe`, `did not like`, `didn't enjoy`, `don't really like`) + - includes group-preference variants (`we like`, `we love`, `we dislike`, `we can't stand`) + - includes lookup variants (`do you think i like ...`, `do you believe i don't like ...`) +- added affinity set/lookup attempt guardrails so partial captures route to affinity prompts instead of generic chat +- extended auto-finalize continuation deferral for the new Pegasus affinity stems (`we like`, `i loathe`, and related variants) +- added focused interaction + websocket tests for the new parser/guardrail behavior + Next queued implementation track after parser guardrails: - presence-aware greetings and identity-triggered proactivity (Pegasus `@be/greetings` parity slice) diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs index dfbb416..1671a47 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs @@ -100,6 +100,8 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer "snapshot" => false, "photobooth" => false, "news" => false, + "trigger_ignored" => false, + "proactive_greeting" => false, _ => true }; } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs index d0f7b74..40ff09a 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs @@ -18,6 +18,12 @@ public sealed class JiboInteractionService( var transcript = (turn.NormalizedTranscript ?? turn.RawTranscript ?? string.Empty).Trim(); var lowered = transcript.ToLowerInvariant(); var referenceLocalTime = TryResolveReferenceLocalTime(turn); + var messageType = turn.Attributes.TryGetValue("messageType", out var rawMessageType) + ? rawMessageType?.ToString() + : null; + var triggerSource = turn.Attributes.TryGetValue("triggerSource", out var rawTriggerSource) + ? rawTriggerSource?.ToString() + : null; var clientIntent = turn.Attributes.TryGetValue("clientIntent", out var rawClientIntent) ? rawClientIntent?.ToString() : null; @@ -32,6 +38,17 @@ public sealed class JiboInteractionService( ? rawPendingProactivityOffer?.ToString() : null; var isYesNoTurn = IsYesNoTurn(turn); + var greetingPresence = ResolveGreetingPresenceProfile(turn); + + if (string.Equals(messageType, "TRIGGER", StringComparison.OrdinalIgnoreCase)) + { + if (ShouldHandleProactiveGreetingTrigger(turn, triggerSource, greetingPresence)) + { + return BuildProactiveGreetingDecision(turn, greetingPresence, referenceLocalTime); + } + + return BuildTriggerIgnoredDecision(); + } var isTimerValueTurn = IsClockTimerValueTurn(clientRules, listenRules); var isAlarmValueTurn = IsClockAlarmValueTurn(clientRules, listenRules); @@ -110,6 +127,11 @@ public sealed class JiboInteractionService( "photobooth" => BuildPhotoCreateDecision("photobooth", "Starting photobooth.", "createSomePhotos"), "robot_age" => BuildRobotAgeDecision(referenceLocalTime), "robot_birthday" => BuildRobotBirthdayDecision(), + "good_morning" => BuildReactiveGreetingDecision(turn, "good_morning", referenceLocalTime), + "good_afternoon" => BuildReactiveGreetingDecision(turn, "good_afternoon", referenceLocalTime), + "good_evening" => BuildReactiveGreetingDecision(turn, "good_evening", referenceLocalTime), + "good_night" => BuildReactiveGreetingDecision(turn, "good_night", referenceLocalTime), + "welcome_back" => BuildReactiveGreetingDecision(turn, "welcome_back", referenceLocalTime), "memory_set_name" => BuildRememberNameDecision(turn, transcript), "memory_get_name" => BuildRecallNameDecision(turn), "memory_set_birthday" => BuildRememberBirthdayDecision(turn, transcript), @@ -163,6 +185,159 @@ public sealed class JiboInteractionService( $"My birthday is {OpenJiboCloudBuildInfo.PersonaBirthdayWords}."); } + private static JiboInteractionDecision BuildTriggerIgnoredDecision() + { + return new JiboInteractionDecision( + "trigger_ignored", + string.Empty, + "chitchat-skill", + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["skillId"] = "chitchat-skill", + ["cloudResponseMode"] = "completion_only" + }); + } + + private JiboInteractionDecision BuildReactiveGreetingDecision( + TurnContext turn, + string greetingIntent, + DateTimeOffset? referenceLocalTime) + { + var presence = ResolveGreetingPresenceProfile(turn); + var displayName = ResolvePreferredGreetingName(turn, presence); + var replyText = BuildReactiveGreetingReply(greetingIntent, displayName, referenceLocalTime); + return new JiboInteractionDecision( + greetingIntent, + replyText, + ContextUpdates: BuildGreetingContextUpdates("ReactiveGreeting", presence.PrimaryPersonId, proactive: false)); + } + + private JiboInteractionDecision BuildProactiveGreetingDecision( + TurnContext turn, + GreetingPresenceProfile presence, + DateTimeOffset? referenceLocalTime) + { + var displayName = ResolvePreferredGreetingName(turn, presence); + var greetingPrefix = ResolveTimeOfDayGreetingPrefix(referenceLocalTime); + var replyText = string.IsNullOrWhiteSpace(displayName) + ? $"{greetingPrefix}. I am glad to see you." + : $"{greetingPrefix}, {displayName}. Welcome back."; + return new JiboInteractionDecision( + "proactive_greeting", + replyText, + ContextUpdates: BuildGreetingContextUpdates("ProactiveGreeting", presence.PrimaryPersonId, proactive: true)); + } + + private static string BuildReactiveGreetingReply( + string greetingIntent, + string? displayName, + DateTimeOffset? referenceLocalTime) + { + var namePrefix = string.IsNullOrWhiteSpace(displayName) + ? string.Empty + : $", {displayName}"; + + return greetingIntent switch + { + "good_morning" => $"Good morning{namePrefix}. It is great to see you.", + "good_afternoon" => $"Good afternoon{namePrefix}. I am glad you are here.", + "good_evening" => $"Good evening{namePrefix}. It is nice to have you back.", + "good_night" => $"Good night{namePrefix}. Sleep well.", + "welcome_back" => string.IsNullOrWhiteSpace(displayName) + ? $"Welcome back. {ResolveTimeOfDayGreetingPrefix(referenceLocalTime)}." + : $"Welcome back, {displayName}. {ResolveTimeOfDayGreetingPrefix(referenceLocalTime)}.", + _ => $"Hello{namePrefix}. It is nice to see you." + }; + } + + private string? ResolvePreferredGreetingName(TurnContext turn, GreetingPresenceProfile presence) + { + var rememberedName = personalMemoryStore.GetName(ResolveTenantScope(turn)); + if (!string.IsNullOrWhiteSpace(rememberedName)) + { + return ToDisplayName(rememberedName); + } + + if (!string.IsNullOrWhiteSpace(presence.PrimaryPersonId) && + presence.LoopUserFirstNames.TryGetValue(presence.PrimaryPersonId, out var firstName) && + !string.IsNullOrWhiteSpace(firstName)) + { + return ToDisplayName(firstName); + } + + return null; + } + + private static string ToDisplayName(string value) + { + var trimmed = value.Trim(); + return string.IsNullOrWhiteSpace(trimmed) + ? string.Empty + : CultureInfo.InvariantCulture.TextInfo.ToTitleCase(trimmed); + } + + private static bool ShouldHandleProactiveGreetingTrigger( + TurnContext turn, + string? triggerSource, + GreetingPresenceProfile presence) + { + if (string.Equals(triggerSource, "SURPRISE", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (!presence.HasKnownIdentity) + { + return false; + } + + var lastGreetingUtc = ReadTimestampAttribute(turn, LastProactiveGreetingUtcMetadataKey); + return !lastGreetingUtc.HasValue || DateTimeOffset.UtcNow - lastGreetingUtc.Value >= ProactiveGreetingCooldown; + } + + private static DateTimeOffset? ReadTimestampAttribute(TurnContext turn, string key) + { + if (!turn.Attributes.TryGetValue(key, out var value) || value is null) + { + return null; + } + + return DateTimeOffset.TryParse( + value.ToString(), + CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind, + out var parsed) + ? parsed + : null; + } + + private static IDictionary BuildGreetingContextUpdates(string route, string? speakerId, bool proactive) + { + var updates = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [ChitchatStateMachine.StateMetadataKey] = "complete", + [ChitchatStateMachine.RouteMetadataKey] = "ScriptedResponse", + [ChitchatStateMachine.EmotionMetadataKey] = string.Empty, + [GreetingRouteMetadataKey] = route, + [GreetingSpeakerMetadataKey] = speakerId ?? string.Empty + }; + + updates[proactive ? LastProactiveGreetingUtcMetadataKey : LastReactiveGreetingUtcMetadataKey] = + DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture); + return updates; + } + + private static string ResolveTimeOfDayGreetingPrefix(DateTimeOffset? referenceLocalTime) + { + var hour = (referenceLocalTime ?? DateTimeOffset.UtcNow).Hour; + return hour switch + { + >= 5 and < 12 => "Good morning", + >= 12 and < 17 => "Good afternoon", + _ => "Good evening" + }; + } + private JiboInteractionDecision BuildRememberNameDecision(TurnContext turn, string transcript) { var name = TryExtractNameFact(transcript); @@ -1084,12 +1259,12 @@ public sealed class JiboInteractionService( return "memory_get_important_date"; } - if (IsAffinitySetStatement(loweredTranscript)) + if (IsAffinitySetStatement(loweredTranscript) || IsAffinitySetAttempt(loweredTranscript)) { return "memory_set_affinity"; } - if (IsAffinityRecallQuestion(loweredTranscript)) + if (IsAffinityRecallQuestion(loweredTranscript) || IsAffinityRecallAttempt(loweredTranscript)) { return "memory_get_affinity"; } @@ -1315,6 +1490,31 @@ public sealed class JiboInteractionService( return "news"; } + if (IsWelcomeBackGreeting(loweredTranscript)) + { + return "welcome_back"; + } + + if (IsGoodMorningGreeting(loweredTranscript)) + { + return "good_morning"; + } + + if (IsGoodAfternoonGreeting(loweredTranscript)) + { + return "good_afternoon"; + } + + if (IsGoodEveningGreeting(loweredTranscript)) + { + return "good_evening"; + } + + if (IsGoodNightGreeting(loweredTranscript)) + { + return "good_night"; + } + if (MatchesAny(loweredTranscript, "how are you", "what's up", "what s up", "what up")) { return "how_are_you"; @@ -2004,6 +2204,115 @@ public sealed class JiboInteractionService( } } + private static GreetingPresenceProfile ResolveGreetingPresenceProfile(TurnContext turn) + { + if (!turn.Attributes.TryGetValue("context", out var contextValue) || + contextValue is null || + string.IsNullOrWhiteSpace(contextValue.ToString())) + { + return GreetingPresenceProfile.Empty; + } + + try + { + using var document = JsonDocument.Parse(contextValue.ToString()!); + if (!document.RootElement.TryGetProperty("runtime", out var runtime) || + runtime.ValueKind != JsonValueKind.Object) + { + return GreetingPresenceProfile.Empty; + } + + var loopUsers = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (runtime.TryGetProperty("loop", out var loop) && + loop.ValueKind == JsonValueKind.Object && + loop.TryGetProperty("users", out var users) && + users.ValueKind == JsonValueKind.Array) + { + foreach (var user in users.EnumerateArray()) + { + var id = TryReadStringProperty(user, "id"); + var firstName = TryReadStringProperty(user, "firstName"); + if (!string.IsNullOrWhiteSpace(id) && !string.IsNullOrWhiteSpace(firstName)) + { + loopUsers[id] = firstName; + } + } + } + + var speakerId = string.Empty; + var peoplePresentIds = new List(); + if (runtime.TryGetProperty("perception", out var perception) && + perception.ValueKind == JsonValueKind.Object) + { + if (perception.TryGetProperty("speaker", out var speaker)) + { + if (speaker.ValueKind == JsonValueKind.String) + { + speakerId = speaker.GetString() ?? string.Empty; + } + else if (speaker.ValueKind == JsonValueKind.Object) + { + speakerId = TryReadStringProperty(speaker, "id", "looperID", "looperId") ?? string.Empty; + } + } + + if (perception.TryGetProperty("peoplePresent", out var peoplePresent) && + peoplePresent.ValueKind == JsonValueKind.Array) + { + foreach (var person in peoplePresent.EnumerateArray()) + { + var personId = person.ValueKind switch + { + JsonValueKind.String => person.GetString(), + JsonValueKind.Object => TryReadStringProperty(person, "id", "looperID", "looperId"), + _ => null + }; + + if (!string.IsNullOrWhiteSpace(personId) && + !string.Equals(personId, "NOT_TRAINED", StringComparison.OrdinalIgnoreCase)) + { + peoplePresentIds.Add(personId); + } + } + } + } + + var triggerLooperId = turn.Attributes.TryGetValue("triggerLooperId", out var rawTriggerLooperId) + ? rawTriggerLooperId?.ToString() + : null; + var primaryPersonId = !string.IsNullOrWhiteSpace(speakerId) + ? speakerId + : !string.IsNullOrWhiteSpace(triggerLooperId) + ? triggerLooperId + : peoplePresentIds.FirstOrDefault(); + + return new GreetingPresenceProfile( + primaryPersonId, + string.IsNullOrWhiteSpace(speakerId) ? null : speakerId, + peoplePresentIds, + loopUsers); + } + catch + { + return GreetingPresenceProfile.Empty; + } + } + + private static string? TryReadStringProperty(JsonElement source, params string[] propertyNames) + { + foreach (var propertyName in propertyNames) + { + if (source.TryGetProperty(propertyName, out var value) && + value.ValueKind == JsonValueKind.String && + !string.IsNullOrWhiteSpace(value.GetString())) + { + return value.GetString(); + } + } + + return null; + } + private static double? TryReadDoubleProperty(JsonElement source, params string[] propertyNames) { foreach (var propertyName in propertyNames) @@ -2315,6 +2624,57 @@ public sealed class JiboInteractionService( }; } + private static bool IsWelcomeBackGreeting(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "i am back", + "i m back", + "im back", + "i am home", + "i m home", + "im home", + "i'm back", + "i'm home", + "welcome back"); + } + + private static bool IsGoodMorningGreeting(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "good morning", + "morning jibo", + "morning, jibo"); + } + + private static bool IsGoodAfternoonGreeting(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "good afternoon", + "afternoon jibo", + "afternoon, jibo"); + } + + private static bool IsGoodEveningGreeting(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "good evening", + "evening jibo", + "evening, jibo"); + } + + private static bool IsGoodNightGreeting(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "good night", + "night jibo", + "night, jibo"); + } + private static bool IsDanceQuestion(string loweredTranscript) { return MatchesAny( @@ -2670,11 +3030,29 @@ public sealed class JiboInteractionService( return TryExtractAffinitySet(loweredTranscript) is not null; } + private static bool IsAffinitySetAttempt(string loweredTranscript) + { + var normalized = NormalizeCommandPhrase(loweredTranscript); + return PegasusUserAffinitySetPrefixes.Any(prefix => MatchesPrefixOrStem(normalized, prefix.Prefix)); + } + private static bool IsAffinityRecallQuestion(string loweredTranscript) { return TryExtractAffinityLookup(loweredTranscript) is not null; } + private static bool IsAffinityRecallAttempt(string loweredTranscript) + { + var normalized = NormalizeCommandPhrase(loweredTranscript); + return PegasusUserAffinityLookupPrefixes.Any(prefix => MatchesPrefixOrStem(normalized, prefix.Prefix)); + } + + private static bool MatchesPrefixOrStem(string normalized, string prefix) + { + return normalized.StartsWith(prefix, StringComparison.Ordinal) || + string.Equals(normalized, prefix.TrimEnd(), StringComparison.Ordinal); + } + private static (string Item, PersonalAffinity Affinity)? TryExtractAffinitySet(string transcript) { var normalized = NormalizeCommandPhrase(transcript); @@ -3331,6 +3709,21 @@ public sealed class JiboInteractionService( private sealed record PizzaSignal(PersonalAffinity? Affinity); + private sealed record GreetingPresenceProfile( + string? PrimaryPersonId, + string? SpeakerId, + IReadOnlyList PeoplePresentIds, + IReadOnlyDictionary LoopUserFirstNames) + { + public static GreetingPresenceProfile Empty { get; } = new( + null, + null, + Array.Empty(), + new Dictionary(StringComparer.OrdinalIgnoreCase)); + + public bool HasKnownIdentity => !string.IsNullOrWhiteSpace(PrimaryPersonId); + } + private sealed record WeatherDateEntity(string? DateEntity, int ForecastDayOffset, string? ForecastLeadIn) { public static WeatherDateEntity None { get; } = new(null, 0, null); @@ -3421,21 +3814,55 @@ public sealed class JiboInteractionService( [ ("i love ", PersonalAffinity.Love), ("i like ", PersonalAffinity.Like), + ("i like the ", PersonalAffinity.Like), ("i enjoy ", PersonalAffinity.Like), ("i do like ", PersonalAffinity.Like), + ("we love ", PersonalAffinity.Love), + ("we like ", PersonalAffinity.Like), + ("we enjoy ", PersonalAffinity.Like), ("i dislike ", PersonalAffinity.Dislike), ("i hate ", PersonalAffinity.Dislike), + ("i hate the ", PersonalAffinity.Dislike), + ("i loathe ", PersonalAffinity.Dislike), ("i don t like ", PersonalAffinity.Dislike), ("i dont like ", PersonalAffinity.Dislike), + ("i not like ", PersonalAffinity.Dislike), ("i do not like ", PersonalAffinity.Dislike), + ("i did not like ", PersonalAffinity.Dislike), + ("i did not like the ", PersonalAffinity.Dislike), + ("i didn t like ", PersonalAffinity.Dislike), + ("i didnt like ", PersonalAffinity.Dislike), + ("i didn t like the ", PersonalAffinity.Dislike), + ("i didnt like the ", PersonalAffinity.Dislike), + ("i didn t really like ", PersonalAffinity.Dislike), + ("i didnt really like ", PersonalAffinity.Dislike), + ("i don t really like ", PersonalAffinity.Dislike), + ("i dont really like ", PersonalAffinity.Dislike), ("i don t enjoy ", PersonalAffinity.Dislike), ("i dont enjoy ", PersonalAffinity.Dislike), ("i do not enjoy ", PersonalAffinity.Dislike), + ("i did not enjoy ", PersonalAffinity.Dislike), + ("i didn t enjoy ", PersonalAffinity.Dislike), + ("i didnt enjoy ", PersonalAffinity.Dislike), + ("i didn t really enjoy ", PersonalAffinity.Dislike), + ("i didnt really enjoy ", PersonalAffinity.Dislike), ("i don t love ", PersonalAffinity.Dislike), ("i dont love ", PersonalAffinity.Dislike), ("i do not love ", PersonalAffinity.Dislike), + ("i don t love to ", PersonalAffinity.Dislike), + ("i dont love to ", PersonalAffinity.Dislike), + ("i do not love to ", PersonalAffinity.Dislike), ("i can t stand ", PersonalAffinity.Dislike), ("i cant stand ", PersonalAffinity.Dislike), + ("i can t stand the ", PersonalAffinity.Dislike), + ("i cant stand the ", PersonalAffinity.Dislike), + ("we dislike ", PersonalAffinity.Dislike), + ("we hate ", PersonalAffinity.Dislike), + ("we despise ", PersonalAffinity.Dislike), + ("we detest ", PersonalAffinity.Dislike), + ("we loathe ", PersonalAffinity.Dislike), + ("we can t stand ", PersonalAffinity.Dislike), + ("we cant stand ", PersonalAffinity.Dislike), ("i despise ", PersonalAffinity.Dislike), ("i detest ", PersonalAffinity.Dislike) ]; @@ -3447,8 +3874,14 @@ public sealed class JiboInteractionService( ("do i enjoy ", PersonalAffinity.Like), ("do i dislike ", PersonalAffinity.Dislike), ("do i hate ", PersonalAffinity.Dislike), + ("do i loathe ", PersonalAffinity.Dislike), + ("do i not like ", PersonalAffinity.Dislike), ("do i despise ", PersonalAffinity.Dislike), ("do i detest ", PersonalAffinity.Dislike), + ("do you think i like ", PersonalAffinity.Like), + ("do you believe i like ", PersonalAffinity.Like), + ("do you think i don t like ", PersonalAffinity.Dislike), + ("do you believe i don t like ", PersonalAffinity.Dislike), ("how do i feel about ", null), ("what do i think about ", null) ]; @@ -3488,6 +3921,12 @@ public sealed class JiboInteractionService( "our neighbourhood" }; + private const string GreetingRouteMetadataKey = "greetingsRoute"; + private const string GreetingSpeakerMetadataKey = "greetingsSpeaker"; + private const string LastProactiveGreetingUtcMetadataKey = "greetingsLastProactiveUtc"; + private const string LastReactiveGreetingUtcMetadataKey = "greetingsLastReactiveUtc"; + private static readonly TimeSpan ProactiveGreetingCooldown = TimeSpan.FromMinutes(20); + private const int MaxWeatherForecastDayOffset = 5; private static readonly (string Phrase, string Station)[] RadioGenreAliases = diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs index 6a953a6..801ab3c 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs @@ -104,7 +104,7 @@ public sealed class JiboWebSocketService( }, cancellationToken); return replies; } - case "CLIENT_NLU" or "CLIENT_ASR": + case "CLIENT_NLU" or "CLIENT_ASR" or "TRIGGER": { var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken); await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs index 65ea1cd..ebd6ef8 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs @@ -60,7 +60,8 @@ public sealed class ProtocolToTurnContextMapper foreach (var pair in session.Metadata) { if ((!pair.Key.StartsWith("personalReport", StringComparison.OrdinalIgnoreCase) && - !pair.Key.StartsWith("chitchat", StringComparison.OrdinalIgnoreCase)) || + !pair.Key.StartsWith("chitchat", StringComparison.OrdinalIgnoreCase) && + !pair.Key.StartsWith("greetings", StringComparison.OrdinalIgnoreCase)) || pair.Value is null) { continue; @@ -154,6 +155,22 @@ public sealed class ProtocolToTurnContextMapper attributes["clientIntent"] = intent.GetString(); } + if (data.TryGetProperty("triggerSource", out var triggerSource) && + triggerSource.ValueKind == JsonValueKind.String && + !string.IsNullOrWhiteSpace(triggerSource.GetString())) + { + attributes["triggerSource"] = triggerSource.GetString(); + } + + if (data.TryGetProperty("triggerData", out var triggerData) && + triggerData.ValueKind == JsonValueKind.Object && + triggerData.TryGetProperty("looperID", out var triggerLooperId) && + triggerLooperId.ValueKind == JsonValueKind.String && + !string.IsNullOrWhiteSpace(triggerLooperId.GetString())) + { + attributes["triggerLooperId"] = triggerLooperId.GetString(); + } + if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array) { attributes["clientRules"] = rules.EnumerateArray() diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs index af2b9b2..fe4ff92 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs @@ -24,21 +24,52 @@ public sealed partial class WebSocketTurnFinalizationService( { "i love", "i like", + "i like the", "i enjoy", "i do like", + "we love", + "we like", + "we enjoy", "i dislike", "i hate", + "i hate the", + "i loathe", + "i not like", "i dont like", "i don t like", "i do not like", + "i did not like", + "i didn t like", + "i didnt like", + "i didn t really like", + "i didnt really like", + "i don t really like", + "i dont really like", "i dont enjoy", "i don t enjoy", "i do not enjoy", + "i did not enjoy", + "i didn t enjoy", + "i didnt enjoy", + "i didn t really enjoy", + "i didnt really enjoy", "i dont love", "i don t love", "i do not love", + "i don t love to", + "i dont love to", + "i do not love to", "i cant stand", "i can t stand", + "i cant stand the", + "i can t stand the", + "we dislike", + "we hate", + "we despise", + "we detest", + "we loathe", + "we cant stand", + "we can t stand", "i despise", "i detest" }; @@ -526,7 +557,9 @@ public sealed partial class WebSocketTurnFinalizationService( } var allowEmptyTranscriptForPersonalReport = IsActivePersonalReportTurn(finalizedTurn); + var allowEmptyTranscriptForTrigger = string.Equals(ReadMessageType(finalizedTurn), "TRIGGER", StringComparison.OrdinalIgnoreCase); if (!allowEmptyTranscriptForPersonalReport && + !allowEmptyTranscriptForTrigger && string.IsNullOrWhiteSpace(finalizedTurn.NormalizedTranscript) && string.IsNullOrWhiteSpace(finalizedTurn.RawTranscript)) { diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index 8db1278..dc4213e 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -20,6 +20,10 @@ public sealed class JiboInteractionServiceTests private const string ChitchatStateKey = "chitchatState"; private const string ChitchatRouteKey = "chitchatRoute"; private const string ChitchatEmotionKey = "chitchatEmotion"; + private const string GreetingRouteKey = "greetingsRoute"; + private const string GreetingSpeakerKey = "greetingsSpeaker"; + private const string GreetingLastProactiveUtcKey = "greetingsLastProactiveUtc"; + private const string GreetingLastReactiveUtcKey = "greetingsLastReactiveUtc"; [Fact] public async Task BuildDecisionAsync_Joke_UsesCatalogBackedRandomContent() @@ -150,6 +154,103 @@ public sealed class JiboInteractionServiceTests Assert.Equal("My birthday is March 22, 2026.", decision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_GoodMorning_UsesReactiveGreetingWithRememberedName() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + memoryStore.SetName(new PersonalMemoryTenantScope("acct-a", "loop-a", "device-a"), "jake"); + var service = CreateService(memoryStore); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "good morning", + NormalizedTranscript = "good morning", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("good_morning", decision.IntentName); + Assert.Equal("Good morning, Jake. It is great to see you.", decision.ReplyText); + Assert.NotNull(decision.ContextUpdates); + Assert.Equal("ReactiveGreeting", decision.ContextUpdates![GreetingRouteKey]); + Assert.Equal(string.Empty, decision.ContextUpdates[GreetingSpeakerKey]); + Assert.True(DateTimeOffset.TryParse(decision.ContextUpdates[GreetingLastReactiveUtcKey]?.ToString(), out _)); + } + + [Fact] + public async Task BuildDecisionAsync_TriggerWithKnownIdentity_BuildsProactiveGreetingAndContext() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = string.Empty, + NormalizedTranscript = string.Empty, + Attributes = new Dictionary + { + ["messageType"] = "TRIGGER", + ["triggerSource"] = "PRESENCE", + ["context"] = """{"runtime":{"perception":{"speaker":"person-1","peoplePresent":[{"id":"person-1"}]},"loop":{"users":[{"id":"person-1","firstName":"jake"}]}}}""" + } + }); + + Assert.Equal("proactive_greeting", decision.IntentName); + Assert.Contains("Jake", decision.ReplyText, StringComparison.Ordinal); + Assert.NotNull(decision.ContextUpdates); + Assert.Equal("ProactiveGreeting", decision.ContextUpdates![GreetingRouteKey]); + Assert.Equal("person-1", decision.ContextUpdates[GreetingSpeakerKey]); + Assert.True(DateTimeOffset.TryParse(decision.ContextUpdates[GreetingLastProactiveUtcKey]?.ToString(), out _)); + } + + [Fact] + public async Task BuildDecisionAsync_TriggerFromSurprise_ReturnsSilentTriggerIgnoredDecision() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = string.Empty, + NormalizedTranscript = string.Empty, + Attributes = new Dictionary + { + ["messageType"] = "TRIGGER", + ["triggerSource"] = "SURPRISE", + ["context"] = """{"runtime":{"perception":{"speaker":"person-1"},"loop":{"users":[{"id":"person-1","firstName":"jake"}]}}}""" + } + }); + + Assert.Equal("trigger_ignored", decision.IntentName); + Assert.Equal(string.Empty, decision.ReplyText); + Assert.Equal("chitchat-skill", decision.SkillName); + Assert.NotNull(decision.SkillPayload); + Assert.Equal("completion_only", decision.SkillPayload!["cloudResponseMode"]); + } + + [Fact] + public async Task BuildDecisionAsync_TriggerWithinCooldown_IsIgnored() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = string.Empty, + NormalizedTranscript = string.Empty, + Attributes = new Dictionary + { + ["messageType"] = "TRIGGER", + ["triggerSource"] = "PRESENCE", + ["context"] = """{"runtime":{"perception":{"speaker":"person-1"},"loop":{"users":[{"id":"person-1","firstName":"jake"}]}}}""", + [GreetingLastProactiveUtcKey] = DateTimeOffset.UtcNow.ToString("O") + } + }); + + Assert.Equal("trigger_ignored", decision.IntentName); + } + [Fact] public async Task BuildDecisionAsync_DoYouHaveAPersonality_UsesCatalogBackedPersonalityReply() { @@ -691,6 +792,144 @@ public sealed class JiboInteractionServiceTests Assert.Equal("Yes. You told me you like country music.", recallDecision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_AffinityMemory_PegasusWeLovePhrase_SetThenRecallWithinTenant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "we love pizza", + NormalizedTranscript = "we love pizza", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_affinity", setDecision.IntentName); + Assert.Equal("Got it. I will remember you love pizza.", setDecision.ReplyText); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "do i love pizza", + NormalizedTranscript = "do i love pizza", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_affinity", recallDecision.IntentName); + Assert.Equal("Yes. You told me you love pizza.", recallDecision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_AffinityMemory_PegasusLoathePhrase_SetThenRecallWithinTenant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "I loathe celery", + NormalizedTranscript = "I loathe celery", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_affinity", setDecision.IntentName); + Assert.Equal("Got it. I will remember you dislike celery.", setDecision.ReplyText); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "do i loathe celery", + NormalizedTranscript = "do i loathe celery", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_affinity", recallDecision.IntentName); + Assert.Equal("Yes. You told me you dislike celery.", recallDecision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_AffinityMemory_PegasusDoYouThinkLikeLookup_SetsAndRecalls() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "I enjoy country music", + NormalizedTranscript = "I enjoy country music", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "do you think i like country music", + NormalizedTranscript = "do you think i like country music", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_affinity", recallDecision.IntentName); + Assert.Equal("Yes. You told me you like country music.", recallDecision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_AffinitySetAttemptWithoutItem_RoutesToAffinityPrompt() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "we like", + NormalizedTranscript = "we like" + }); + + Assert.Equal("memory_set_affinity", decision.IntentName); + Assert.Equal("I can remember it if you say, I like pizza or I dislike mushrooms.", decision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_AffinityRecallAttemptWithoutItem_RoutesToRecallPrompt() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "do you think i like", + NormalizedTranscript = "do you think i like" + }); + + Assert.Equal("memory_get_affinity", decision.IntentName); + Assert.Equal("Ask me like this: do I like pizza?", decision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_PreferenceReversePhrase_ParsesFavoriteVariant() { diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 8355eef..1c4c19a 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -491,6 +491,75 @@ public sealed class JiboWebSocketServiceTests Assert.Equal("i do like pizza", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); } + [Fact] + public async Task BufferedAudio_WithIncompletePegasusWeAffinityHint_DefersThenFinalizesWhenContinuationArrives() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-affinity-we-continuation-token", + Text = """{"type":"LISTEN","transID":"trans-affinity-we-continuation","data":{"rules":["launch"]}}""" + }); + + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-affinity-we-continuation-token", + Text = """{"type":"CONTEXT","transID":"trans-affinity-we-continuation","data":{"audioTranscriptHint":"we like"}}""" + }); + + for (var index = 0; index < 4; index += 1) + { + var chunkReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-affinity-we-continuation-token", + Binary = new byte[3000] + }); + + Assert.Empty(chunkReplies); + } + + var session = _store.FindSessionByToken("hub-affinity-we-continuation-token"); + Assert.NotNull(session); + session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); + + var deferredReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-affinity-we-continuation-token", + Binary = new byte[3000] + }); + + Assert.Empty(deferredReplies); + + var finalizedReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-affinity-we-continuation-token", + Text = """{"type":"CONTEXT","transID":"trans-affinity-we-continuation","data":{"audioTranscriptHint":"we like pizza"}}""" + }); + + Assert.Equal(3, finalizedReplies.Count); + Assert.Equal("LISTEN", ReadReplyType(finalizedReplies[0])); + Assert.Equal("EOS", ReadReplyType(finalizedReplies[1])); + Assert.Equal("SKILL_ACTION", ReadReplyType(finalizedReplies[2])); + + using var listenPayload = JsonDocument.Parse(finalizedReplies[0].Text!); + Assert.Equal("memory_set_affinity", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + Assert.Equal("we like pizza", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); + } + [Fact] public async Task MultiChunkAudio_AccumulatesBufferedStateAcrossMessages() { @@ -3501,6 +3570,122 @@ public sealed class JiboWebSocketServiceTests Assert.False(session.Metadata.ContainsKey("pendingProactivityOffer")); } + [Fact] + public async Task TriggerPresence_WithIdentity_EmitsProactiveGreetingAndPersistsGreetingMetadata() + { + var token = _store.IssueRobotToken("trigger-greeting-device-a"); + + var listenSetupReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"LISTEN","transID":"trans-greeting-trigger","data":{"rules":["launch","globals/global_commands_launch"],"mode":"CLIENT_NLU"}}""" + }); + Assert.Empty(listenSetupReplies); + + var contextReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"CONTEXT","transID":"trans-greeting-trigger","data":{"runtime":{"perception":{"speaker":"person-1","peoplePresent":[{"id":"person-1"}]},"loop":{"users":[{"id":"person-1","firstName":"jake"}]}}}}""" + }); + Assert.Empty(contextReplies); + + var triggerReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"TRIGGER","transID":"trans-greeting-trigger","data":{"triggerSource":"PRESENCE","triggerData":{"looperID":"person-1"}}}""" + }); + + Assert.Equal(3, triggerReplies.Count); + Assert.Equal("LISTEN", ReadReplyType(triggerReplies[0])); + Assert.Equal("EOS", ReadReplyType(triggerReplies[1])); + Assert.Equal("SKILL_ACTION", ReadReplyType(triggerReplies[2])); + + using (var listenPayload = JsonDocument.Parse(triggerReplies[0].Text!)) + { + Assert.Equal("proactive_greeting", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + } + using (var skillPayload = JsonDocument.Parse(triggerReplies[2].Text!)) + { + var esml = skillPayload.RootElement + .GetProperty("data") + .GetProperty("action") + .GetProperty("config") + .GetProperty("jcp") + .GetProperty("config") + .GetProperty("play") + .GetProperty("esml") + .GetString(); + Assert.Contains("Jake", esml, StringComparison.Ordinal); + } + + var session = _store.FindSessionByToken(token); + Assert.NotNull(session); + Assert.False(session.FollowUpOpen); + Assert.True(session.Metadata.TryGetValue("greetingsRoute", out var route)); + Assert.Equal("ProactiveGreeting", route?.ToString()); + Assert.True(session.Metadata.TryGetValue("greetingsSpeaker", out var speaker)); + Assert.Equal("person-1", speaker?.ToString()); + Assert.True(session.Metadata.TryGetValue("greetingsLastProactiveUtc", out var lastUtc)); + Assert.True(DateTimeOffset.TryParse(lastUtc?.ToString(), out _)); + } + + [Fact] + public async Task TriggerSurprise_IsIgnoredWithoutLeavingMicOpen() + { + var token = _store.IssueRobotToken("trigger-greeting-device-b"); + + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"LISTEN","transID":"trans-greeting-trigger-ignore","data":{"rules":["launch"],"mode":"CLIENT_NLU"}}""" + }); + + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"CONTEXT","transID":"trans-greeting-trigger-ignore","data":{"runtime":{"perception":{"speaker":"person-1"},"loop":{"users":[{"id":"person-1","firstName":"jake"}]}}}}""" + }); + + var triggerReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"TRIGGER","transID":"trans-greeting-trigger-ignore","data":{"triggerSource":"SURPRISE","triggerData":{"looperID":"person-1"}}}""" + }); + + Assert.Equal(3, triggerReplies.Count); + Assert.Equal("LISTEN", ReadReplyType(triggerReplies[0])); + Assert.Equal("EOS", ReadReplyType(triggerReplies[1])); + Assert.Equal("SKILL_ACTION", ReadReplyType(triggerReplies[2])); + + using (var listenPayload = JsonDocument.Parse(triggerReplies[0].Text!)) + { + Assert.Equal("trigger_ignored", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + } + + var session = _store.FindSessionByToken(token); + Assert.NotNull(session); + Assert.False(session.FollowUpOpen); + Assert.False(session.Metadata.ContainsKey("greetingsLastProactiveUtc")); + } + [Fact] public async Task ClientAsrPersonalReport_StateMachinePersistsAcrossTurns() {