From b74ef3bfa2d7e18972b6e25f0d04799faba1bbf7 Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Wed, 6 May 2026 08:13:26 -0500 Subject: [PATCH] Add OpenWeather-backed weather reports --- OpenJibo/docs/feature-backlog.md | 57 +- OpenJibo/docs/release-1.0.19-plan.md | 53 +- .../src/Jibo.Cloud.Api/appsettings.json | 8 + .../Abstractions/IPersonalMemoryStore.cs | 14 + .../Abstractions/IWeatherReportProvider.cs | 24 + .../Services/JiboInteractionService.cs | 940 +++++++++++++++++- .../Services/ProtocolToTurnContextMapper.cs | 7 + .../ResponsePlanToSocketMessagesMapper.cs | 50 +- .../WebSocketTurnFinalizationService.cs | 35 +- .../ServiceCollectionExtensions.cs | 14 + .../InMemoryPersonalMemoryStore.cs | 59 ++ .../Weather/OpenWeatherOptions.cs | 12 + .../Weather/OpenWeatherReportProvider.cs | 364 +++++++ .../WebSockets/JiboInteractionServiceTests.cs | 359 ++++++- .../WebSockets/JiboWebSocketServiceTests.cs | 131 +++ 15 files changed, 2072 insertions(+), 55 deletions(-) create mode 100644 OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IWeatherReportProvider.cs create mode 100644 OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherOptions.cs create mode 100644 OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherReportProvider.cs diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index 8bf07ee..d1f5a60 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -610,9 +610,38 @@ Current release theme: - preference set/recall works (`my favorite X is Y` / `what is my favorite X`) - account/loop/device scoped lookup prevents cross-tenant leakage - Follow-up: - - extend phrase parsing beyond first rule-based patterns - add durable persistence path for personal facts - - broaden fact categories (names, important dates, household preferences) + - broaden fact categories further (multi-person household memory, relationship cues, and corrective updates) + +### 24. Memory-Triggered Proactivity Baseline + +- Status: `implemented` +- Tags: `content`, `storage`, `protocol` +- Result: + - `surprise me` now uses weighted candidate selection instead of only generic fallback text + - candidate weighting uses tenant-scoped memory signals and date triggers + - February 9 (`National Pizza Day`) can proactively launch the legacy pizza animation path + - proactive pizza fact offer flow stores pending offer state in session metadata and resolves direct short `yes/no` turns + - memory parsing now includes names, anniversary-style important dates, likes/dislikes variants, and reverse favorite phrasing (`pizza is my favorite food`) +- Follow-up: + - expand proactivity beyond pizza to additional Pegasus-backed categories + - add cooldown/throttle policy and observability around proactive offer frequency + - connect memory store to durable multi-tenant persistence + +### 25. Weather Report-Skill Launch Compatibility + +- Status: `implemented` +- Tags: `protocol`, `content` +- Result: + - weather requests now launch `report-skill` using Pegasus-aligned intent `requestWeatherPR` + - weather phrase coverage includes baseline forecast and condition-style questions (`will it rain`, `is it snowing`, tomorrow variants) + - weather launches emit `SKILL_REDIRECT` + completion and now also include cloud weather speech so weather turns remain useful even when local report providers are incomplete + - weather entity hints are carried in outbound NLU (`date = tomorrow`, `Weather = rain/snow/...`) for report-skill consumption + - OpenWeather provider integration is in place with configurable API key, default location, unit preference, and environment-variable fallback (`OPENWEATHER_API_KEY`) + - cloud weather speech now uses live provider summaries for current conditions and tomorrow high/low forecast when available +- Follow-up: + - connect weather units and location directly to user/report-skill settings parity instead of config defaults + - add richer condition-change commentary and view parity with original report-skill weather behaviors ## Suggested Order @@ -628,17 +657,19 @@ Use [regression-test-plan.md](regression-test-plan.md) as the detailed checklist For `1.0.19`: -1. Command-vs-question personality split (`dance` command vs `do you like to dance` question style; expand this pattern) -2. Expand memory-backed personal facts with tenant-scoped storage (beyond the first birthday/preferences foundation) -3. Proactivity selector baseline with source-backed first offers -4. Dialog parsing expansion and ambiguity guardrails -5. Holidays and seasonal personality behavior built on the new memory/proactivity foundation -6. Update, backup, and restore proof -7. STT upgrade and noise screening -8. Hosted capture/storage plan / indexing for group testing -9. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc. -10. Provider-backed news and weather -11. Lasso, identity, and onboarding as larger discovery-driven tracks +1. Command-vs-question personality split (`dance` command vs `do you like to dance` question style; expand this pattern) - implemented +2. Expand memory-backed personal facts with tenant-scoped storage (beyond the first birthday/preferences foundation) - implemented +3. Proactivity selector baseline with source-backed first offers - implemented +4. Weather report-skill launch compatibility - implemented +5. Dialog parsing expansion and ambiguity guardrails +6. Holidays and seasonal personality behavior built on the new memory/proactivity foundation +7. Durable memory persistence path (multi-tenant backing store) +8. Update, backup, and restore proof +9. STT upgrade and noise screening +10. Hosted capture/storage plan / indexing for group testing +11. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc. +12. Provider-backed news and weather parity polish +13. Lasso, identity, and onboarding as larger discovery-driven tracks For `1.0.20` and beyond: diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index f336da0..0e07c27 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -68,17 +68,52 @@ The second delivered slice is first tenant-scoped personal memory: Memory keys are scoped by account/loop/device tenant context so one tenant does not leak into another. +## Third Implemented Slice In `1.0.19` + +The third delivered slice starts memory-triggered proactivity and broadens memory parsing: + +- `surprise me` now runs a weighted proactivity selector +- selectors use tenant-scoped memory signals (favorites and likes/dislikes) plus date triggers +- February 9 (`National Pizza Day`) can proactively launch the pizza animation path +- proactive pizza fact offer flow now stores pending offer state and resolves direct `yes` / `no` follow-up answers +- memory parsing now covers: + - names (`my name is ...`, `what is my name`) + - important dates (`our anniversary is ...`, `when is our anniversary`) + - likes/dislikes (`i like ...`, `i love ...`, `i dislike ...`, `i don't like ...`) + - favorite phrase variants including reverse form (`pizza is my favorite food`) + +## Fourth Implemented Slice In `1.0.19` + +The fourth delivered slice starts weather compatibility using Pegasus-style report-skill routing: + +- weather phrases now route to `report-skill` instead of generic placeholder chat +- outbound NLU launch uses legacy reactive intent `requestWeatherPR` (source-aligned with Pegasus manifests/tests) +- weather entity hints are added for: + - `date = tomorrow` on tomorrow phrasing + - `Weather = rain|snow|...` on condition questions (for example `will it rain tomorrow`) +- websocket output now emits local skill redirect + silent completion for weather launch, matching existing local-skill launch patterns + +## Fifth Implemented Slice In `1.0.19` + +The fifth delivered slice adds provider-backed weather content while preserving Pegasus launch compatibility: + +- OpenWeather provider abstraction and infrastructure wiring are added to the hosted cloud +- weather requests still launch `report-skill` with `requestWeatherPR` and legacy weather/date entities +- weather replies now include cloud-generated spoken summaries from provider data: + - current conditions (`Right now in ...`) + - tomorrow forecast shape (`Tomorrow in ...`) with high/low temperatures when available +- simple location extraction is supported for phrasing like `what's the weather in Chicago tomorrow` +- provider config supports appsettings and `OPENWEATHER_API_KEY` environment fallback for deployment + ## Next Slices -1. Command-vs-question personality split (start with dance/twerk-style prompts, keep commands action-oriented and questions conversational) -2. Expand memory-backed personal facts (tenant-scoped birthday/preferences coverage, persistence depth, and parsing breadth) -3. Proactivity selector baseline (source-backed first proactive offers with safe throttling and stock-compatible payloads) -4. Dialog parsing expansion (more phrase variants, ambiguity handling, and transcript-to-intent guardrails) -5. Holidays and seasonal personality slice (time-scoped content backed by the new memory/proactivity path) -6. Update/backup/restore end-to-end proof (operator-run and documented) -7. STT noise-screening and short-utterance reliability pass -8. Provider-backed news/weather expansion using Pegasus-backed contracts -9. Capture indexing and retention boundary for group testing +1. Dialog parsing expansion (more phrase variants, ambiguity handling, and transcript-to-intent guardrails) +2. Holidays and seasonal personality slice beyond pizza day (time-scoped content backed by memory/proactivity path) +3. Durable memory persistence path (swap in provider-backed multi-tenant storage while preserving behavior contracts) +4. Update/backup/restore end-to-end proof (operator-run and documented) +5. STT noise-screening and short-utterance reliability pass +6. Provider-backed news expansion and deeper weather parity using Pegasus-backed contracts +7. Capture indexing and retention boundary for group testing For slices 1-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements. diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.json b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.json index fe64867..66f2329 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.json +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.json @@ -21,6 +21,14 @@ "WhisperLanguage": "en", "TempDirectory": "/tmp/openjibo-stt", "CleanupTempFiles": false + }, + "Weather": { + "OpenWeather": { + "BaseUrl": "https://api.openweathermap.org", + "ApiKey": "723667c9ab0318142227c5389900d087", + "DefaultLocation": "Boston,US", + "UseCelsius": false + } } } } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IPersonalMemoryStore.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IPersonalMemoryStore.cs index 66557bd..aac5c4b 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IPersonalMemoryStore.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IPersonalMemoryStore.cs @@ -6,6 +6,20 @@ public interface IPersonalMemoryStore string? GetBirthday(PersonalMemoryTenantScope tenantScope); void SetPreference(PersonalMemoryTenantScope tenantScope, string category, string value); string? GetPreference(PersonalMemoryTenantScope tenantScope, string category); + void SetName(PersonalMemoryTenantScope tenantScope, string name); + string? GetName(PersonalMemoryTenantScope tenantScope); + void SetImportantDate(PersonalMemoryTenantScope tenantScope, string label, string value); + string? GetImportantDate(PersonalMemoryTenantScope tenantScope, string label); + void SetAffinity(PersonalMemoryTenantScope tenantScope, string item, PersonalAffinity affinity); + PersonalAffinity? GetAffinity(PersonalMemoryTenantScope tenantScope, string item); + IReadOnlyDictionary GetAffinities(PersonalMemoryTenantScope tenantScope); } public sealed record PersonalMemoryTenantScope(string AccountId, string LoopId, string DeviceId); + +public enum PersonalAffinity +{ + Like, + Love, + Dislike +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IWeatherReportProvider.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IWeatherReportProvider.cs new file mode 100644 index 0000000..ad7a039 --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IWeatherReportProvider.cs @@ -0,0 +1,24 @@ +namespace Jibo.Cloud.Application.Abstractions; + +public interface IWeatherReportProvider +{ + Task GetReportAsync( + WeatherReportRequest request, + CancellationToken cancellationToken = default); +} + +public sealed record WeatherReportRequest( + string? LocationQuery, + double? Latitude, + double? Longitude, + bool IsTomorrow, + bool? UseCelsius); + +public sealed record WeatherReportSnapshot( + string LocationName, + string Summary, + int Temperature, + int? HighTemperature, + int? LowTemperature, + string? Condition, + bool UseCelsius); diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs index eb73d30..899f3fc 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs @@ -1,5 +1,6 @@ using Jibo.Cloud.Application.Abstractions; using Jibo.Runtime.Abstractions; +using System.Globalization; using System.Text.Json; using System.Text.RegularExpressions; @@ -8,7 +9,8 @@ namespace Jibo.Cloud.Application.Services; public sealed class JiboInteractionService( JiboExperienceContentCache contentCache, IJiboRandomizer randomizer, - IPersonalMemoryStore personalMemoryStore) + IPersonalMemoryStore personalMemoryStore, + IWeatherReportProvider? weatherReportProvider = null) { public async Task BuildDecisionAsync(TurnContext turn, CancellationToken cancellationToken = default) { @@ -26,6 +28,9 @@ public sealed class JiboInteractionService( var lastClockDomain = turn.Attributes.TryGetValue("lastClockDomain", out var rawLastClockDomain) ? rawLastClockDomain?.ToString() : null; + var pendingProactivityOffer = turn.Attributes.TryGetValue("pendingProactivityOffer", out var rawPendingProactivityOffer) + ? rawPendingProactivityOffer?.ToString() + : null; var isYesNoTurn = IsYesNoTurn(turn); var isTimerValueTurn = IsClockTimerValueTurn(clientRules, listenRules); @@ -38,6 +43,7 @@ public sealed class JiboInteractionService( listenRules, clientEntities, lastClockDomain, + pendingProactivityOffer, isYesNoTurn, isTimerValueTurn, isAlarmValueTurn); @@ -78,19 +84,30 @@ public sealed class JiboInteractionService( "robot_age" => BuildRobotAgeDecision(referenceLocalTime), "robot_birthday" => BuildRobotBirthdayDecision(), "robot_personality" => new JiboInteractionDecision("robot_personality", randomizer.Choose(catalog.PersonalityReplies)), + "memory_set_name" => BuildRememberNameDecision(turn, transcript), + "memory_get_name" => BuildRecallNameDecision(turn), "memory_set_birthday" => BuildRememberBirthdayDecision(turn, transcript), "memory_get_birthday" => BuildRecallBirthdayDecision(turn), + "memory_set_important_date" => BuildRememberImportantDateDecision(turn, transcript), + "memory_get_important_date" => BuildRecallImportantDateDecision(turn, transcript), "memory_set_preference" => BuildRememberPreferenceDecision(turn, transcript), "memory_get_preference" => BuildRecallPreferenceDecision(turn, transcript), + "memory_set_affinity" => BuildRememberAffinityDecision(turn, transcript), + "memory_get_affinity" => BuildRecallAffinityDecision(turn, transcript), "pizza" => BuildPizzaDecision(), "order_pizza" => BuildOrderPizzaDecision(), + "proactive_pizza_day" => BuildProactivePizzaDayDecision(referenceLocalTime), + "proactive_pizza_preference" => BuildProactivePizzaPreferenceDecision(), + "proactive_offer_pizza_fact" => BuildProactivePizzaFactOfferDecision(), + "proactive_pizza_fact" => BuildProactivePizzaFactDecision(), + "proactive_offer_declined" => BuildProactiveOfferDeclinedDecision(), + "weather" => await BuildWeatherReportDecisionAsync(turn, transcript, cancellationToken), "yes" => new JiboInteractionDecision("yes", "Yes."), "no" => new JiboInteractionDecision("no", "No."), "word_of_the_day" => BuildWordOfTheDayLaunchDecision(), "word_of_the_day_guess" => BuildWordOfTheDayGuessDecision(clientEntities, transcript, listenAsrHints), - "surprise" => new JiboInteractionDecision("surprise", randomizer.Choose(catalog.SurpriseReplies)), + "surprise" => BuildSurpriseDecision(catalog, turn, referenceLocalTime), "personal_report" => new JiboInteractionDecision("personal_report", randomizer.Choose(catalog.PersonalReportReplies)), - "weather" => new JiboInteractionDecision("weather", randomizer.Choose(catalog.WeatherReplies)), "calendar" => new JiboInteractionDecision("calendar", randomizer.Choose(catalog.CalendarReplies)), "commute" => new JiboInteractionDecision("commute", randomizer.Choose(catalog.CommuteReplies)), "news" => BuildNewsDecision(catalog), @@ -120,6 +137,34 @@ public sealed class JiboInteractionService( $"My birthday is {OpenJiboCloudBuildInfo.PersonaBirthdayWords}."); } + private JiboInteractionDecision BuildRememberNameDecision(TurnContext turn, string transcript) + { + var name = TryExtractNameFact(transcript); + if (string.IsNullOrWhiteSpace(name)) + { + return new JiboInteractionDecision( + "memory_set_name", + "I can remember it if you say, my name is Alex."); + } + + personalMemoryStore.SetName(ResolveTenantScope(turn), name); + return new JiboInteractionDecision( + "memory_set_name", + $"Nice to meet you, {name}. I will remember your name."); + } + + private JiboInteractionDecision BuildRecallNameDecision(TurnContext turn) + { + var name = personalMemoryStore.GetName(ResolveTenantScope(turn)); + return string.IsNullOrWhiteSpace(name) + ? new JiboInteractionDecision( + "memory_get_name", + "I do not know your name yet. You can say, my name is Alex.") + : new JiboInteractionDecision( + "memory_get_name", + $"You told me your name is {name}."); + } + private JiboInteractionDecision BuildRememberBirthdayDecision(TurnContext turn, string transcript) { var birthday = TryExtractBirthdayFact(transcript); @@ -148,6 +193,42 @@ public sealed class JiboInteractionService( $"You told me your birthday is {birthday}."); } + private JiboInteractionDecision BuildRememberImportantDateDecision(TurnContext turn, string transcript) + { + var importantDate = TryExtractImportantDateSet(transcript); + if (importantDate is null) + { + return new JiboInteractionDecision( + "memory_set_important_date", + "I can remember it if you say, our anniversary is June 10."); + } + + personalMemoryStore.SetImportantDate(ResolveTenantScope(turn), importantDate.Value.Label, importantDate.Value.Value); + return new JiboInteractionDecision( + "memory_set_important_date", + $"Got it. I will remember your {importantDate.Value.Label} is {importantDate.Value.Value}."); + } + + private JiboInteractionDecision BuildRecallImportantDateDecision(TurnContext turn, string transcript) + { + var label = TryExtractImportantDateLookupLabel(transcript); + if (string.IsNullOrWhiteSpace(label)) + { + return new JiboInteractionDecision( + "memory_get_important_date", + "Ask me like this: when is our anniversary?"); + } + + var storedDate = personalMemoryStore.GetImportantDate(ResolveTenantScope(turn), label); + return string.IsNullOrWhiteSpace(storedDate) + ? new JiboInteractionDecision( + "memory_get_important_date", + $"I do not know your {label} yet.") + : new JiboInteractionDecision( + "memory_get_important_date", + $"You told me your {label} is {storedDate}."); + } + private JiboInteractionDecision BuildRememberPreferenceDecision(TurnContext turn, string transcript) { var preference = TryExtractPreferenceSet(transcript); @@ -184,12 +265,71 @@ public sealed class JiboInteractionService( $"You told me your favorite {category} is {preference}."); } + private JiboInteractionDecision BuildRememberAffinityDecision(TurnContext turn, string transcript) + { + var affinitySet = TryExtractAffinitySet(transcript); + if (affinitySet is null) + { + return new JiboInteractionDecision( + "memory_set_affinity", + "I can remember it if you say, I like pizza or I dislike mushrooms."); + } + + personalMemoryStore.SetAffinity(ResolveTenantScope(turn), affinitySet.Value.Item, affinitySet.Value.Affinity); + return new JiboInteractionDecision( + "memory_set_affinity", + $"Got it. I will remember you {DescribeAffinityAsVerb(affinitySet.Value.Affinity)} {affinitySet.Value.Item}."); + } + + private JiboInteractionDecision BuildRecallAffinityDecision(TurnContext turn, string transcript) + { + var lookup = TryExtractAffinityLookup(transcript); + if (lookup is null) + { + return new JiboInteractionDecision( + "memory_get_affinity", + "Ask me like this: do I like pizza?"); + } + + var affinity = personalMemoryStore.GetAffinity(ResolveTenantScope(turn), lookup.Value.Item); + if (affinity is null) + { + return new JiboInteractionDecision( + "memory_get_affinity", + $"I do not remember how you feel about {lookup.Value.Item} yet."); + } + + if (lookup.Value.ExpectedAffinity is null) + { + return new JiboInteractionDecision( + "memory_get_affinity", + $"You told me you {DescribeAffinityAsVerb(affinity.Value)} {lookup.Value.Item}."); + } + + var matches = lookup.Value.ExpectedAffinity == PersonalAffinity.Dislike + ? affinity == PersonalAffinity.Dislike + : affinity is PersonalAffinity.Like or PersonalAffinity.Love; + + return matches + ? new JiboInteractionDecision( + "memory_get_affinity", + $"Yes. You told me you {DescribeAffinityAsVerb(affinity.Value)} {lookup.Value.Item}.") + : new JiboInteractionDecision( + "memory_get_affinity", + $"Not exactly. You told me you {DescribeAffinityAsVerb(affinity.Value)} {lookup.Value.Item}."); + } + private JiboInteractionDecision BuildPizzaDecision() + { + return BuildPizzaAnimationDecision("pizza", "One pizza, coming right up."); + } + + private JiboInteractionDecision BuildPizzaAnimationDecision(string intentName, string replyText) { var prompt = randomizer.Choose(PizzaMimPrompts); return new JiboInteractionDecision( - "pizza", - "One pizza, coming right up.", + intentName, + replyText, "chitchat-skill", new Dictionary(StringComparer.OrdinalIgnoreCase) { @@ -201,6 +341,158 @@ public sealed class JiboInteractionService( }); } + private JiboInteractionDecision BuildProactivePizzaDayDecision(DateTimeOffset? referenceLocalTime) + { + var referenceDate = (referenceLocalTime ?? DateTimeOffset.UtcNow).Date; + return BuildPizzaAnimationDecision( + "proactive_pizza_day", + $"Happy National Pizza Day for {referenceDate.ToString("MMMM d", CultureInfo.InvariantCulture)}. One pizza, coming right up."); + } + + private JiboInteractionDecision BuildProactivePizzaPreferenceDecision() + { + return BuildPizzaAnimationDecision( + "proactive_pizza_preference", + "You mentioned pizza is a favorite, so I thought we should make one."); + } + + private static JiboInteractionDecision BuildProactivePizzaFactOfferDecision() + { + return new JiboInteractionDecision( + "proactive_offer_pizza_fact", + "Do you want to hear a fun pizza fact?"); + } + + private static JiboInteractionDecision BuildProactivePizzaFactDecision() + { + return new JiboInteractionDecision( + "proactive_pizza_fact", + "Americans consume about 100 acres of pizza every day, roughly 350 slices per second. That's a lot of pizza."); + } + + private static JiboInteractionDecision BuildProactiveOfferDeclinedDecision() + { + return new JiboInteractionDecision( + "proactive_offer_declined", + "No problem. We can save the pizza fact for another time."); + } + + private async Task BuildWeatherReportDecisionAsync( + TurnContext turn, + string transcript, + CancellationToken cancellationToken) + { + var payload = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["skillId"] = "report-skill", + ["localIntent"] = "requestWeatherPR", + ["cloudSkill"] = "weather" + }; + var dateEntity = TryResolveWeatherDateEntity(transcript); + if (dateEntity is not null) + { + payload["date"] = dateEntity; + } + + var weatherConditionEntity = TryResolveWeatherConditionEntity(transcript); + if (weatherConditionEntity is not null) + { + payload["weatherCondition"] = weatherConditionEntity; + } + + var replyText = "Checking your weather report."; + if (weatherReportProvider is null) + { + return new JiboInteractionDecision( + "weather", + replyText, + "report-skill", + payload); + } + + var locationQuery = TryResolveWeatherLocationQuery(transcript); + if (!string.IsNullOrWhiteSpace(locationQuery)) + { + payload["locationQuery"] = locationQuery; + } + + var weatherCoordinates = TryResolveWeatherCoordinates(turn); + if (weatherCoordinates is not null) + { + payload["latitude"] = weatherCoordinates.Value.Latitude; + payload["longitude"] = weatherCoordinates.Value.Longitude; + } + + var useCelsius = ShouldUseCelsius(turn, transcript); + var snapshot = await weatherReportProvider.GetReportAsync( + new WeatherReportRequest( + locationQuery, + weatherCoordinates?.Latitude, + weatherCoordinates?.Longitude, + string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase), + useCelsius), + cancellationToken); + + if (snapshot is not null) + { + payload["provider"] = "openweather"; + payload["temperature"] = snapshot.Temperature; + if (snapshot.HighTemperature is not null) + { + payload["highTemperature"] = snapshot.HighTemperature.Value; + } + + if (snapshot.LowTemperature is not null) + { + payload["lowTemperature"] = snapshot.LowTemperature.Value; + } + + if (!string.IsNullOrWhiteSpace(snapshot.Condition)) + { + payload["weatherCondition"] = snapshot.Condition; + } + + replyText = BuildWeatherSpokenReply(snapshot, dateEntity); + } + + return new JiboInteractionDecision( + "weather", + replyText, + "report-skill", + payload); + } + + private static string BuildWeatherSpokenReply( + WeatherReportSnapshot snapshot, + string? dateEntity) + { + var unit = snapshot.UseCelsius ? "Celsius" : "Fahrenheit"; + var summary = string.IsNullOrWhiteSpace(snapshot.Summary) + ? "partly cloudy" + : snapshot.Summary.Trim().TrimEnd('.'); + var location = string.IsNullOrWhiteSpace(snapshot.LocationName) + ? "your area" + : snapshot.LocationName; + + if (string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase)) + { + var highText = snapshot.HighTemperature is null + ? null + : $"a high near {snapshot.HighTemperature.Value} degrees {unit}"; + var lowText = snapshot.LowTemperature is null + ? null + : $"a low around {snapshot.LowTemperature.Value} degrees {unit}"; + var tempRange = highText is null && lowText is null + ? string.Empty + : highText is not null && lowText is not null + ? $" with {highText} and {lowText}" + : $" with {highText ?? lowText}"; + return $"Tomorrow in {location}, expect {summary}{tempRange}."; + } + + return $"Right now in {location}, it is {summary} and {snapshot.Temperature} degrees {unit}."; + } + private static JiboInteractionDecision BuildOrderPizzaDecision() { return new JiboInteractionDecision( @@ -272,6 +564,101 @@ public sealed class JiboInteractionService( }); } + private JiboInteractionDecision BuildSurpriseDecision( + JiboExperienceCatalog catalog, + TurnContext turn, + DateTimeOffset? referenceLocalTime) + { + var tenantScope = ResolveTenantScope(turn); + var candidates = BuildProactivityCandidates(tenantScope, referenceLocalTime); + if (candidates.Count == 0) + { + return new JiboInteractionDecision("surprise", randomizer.Choose(catalog.SurpriseReplies)); + } + + var highestWeight = candidates.Max(static candidate => candidate.Weight); + var topCandidates = candidates + .Where(candidate => candidate.Weight == highestWeight) + .ToArray(); + var selected = topCandidates.Length == 1 + ? topCandidates[0] + : randomizer.Choose(topCandidates); + + return selected.IntentName switch + { + "proactive_pizza_day" => BuildProactivePizzaDayDecision(referenceLocalTime), + "proactive_pizza_preference" => BuildProactivePizzaPreferenceDecision(), + "proactive_offer_pizza_fact" => BuildProactivePizzaFactOfferDecision(), + _ => new JiboInteractionDecision("surprise", randomizer.Choose(catalog.SurpriseReplies)) + }; + } + + private List BuildProactivityCandidates( + PersonalMemoryTenantScope tenantScope, + DateTimeOffset? referenceLocalTime) + { + var candidates = new List(); + var referenceDate = (referenceLocalTime ?? DateTimeOffset.UtcNow).Date; + + var pizzaSignal = ResolvePizzaSignal(tenantScope); + if (pizzaSignal.Affinity == PersonalAffinity.Dislike) + { + return candidates; + } + + if (referenceDate.Month == 2 && referenceDate.Day == 9) + { + var holidayWeight = pizzaSignal.Affinity switch + { + PersonalAffinity.Love => 170, + PersonalAffinity.Like => 160, + _ => 150 + }; + candidates.Add(new ProactivityCandidate("proactive_pizza_day", holidayWeight)); + } + + if (pizzaSignal.Affinity is PersonalAffinity.Love or PersonalAffinity.Like) + { + var preferenceWeight = pizzaSignal.Affinity == PersonalAffinity.Love ? 140 : 120; + candidates.Add(new ProactivityCandidate("proactive_pizza_preference", preferenceWeight)); + candidates.Add(new ProactivityCandidate("proactive_offer_pizza_fact", preferenceWeight - 5)); + return candidates; + } + + candidates.Add(new ProactivityCandidate("proactive_offer_pizza_fact", 90)); + return candidates; + } + + private PizzaSignal ResolvePizzaSignal(PersonalMemoryTenantScope tenantScope) + { + var pizzaAffinity = personalMemoryStore.GetAffinity(tenantScope, "pizza"); + if (pizzaAffinity is not null) + { + return new PizzaSignal(pizzaAffinity); + } + + var affinityMatch = personalMemoryStore.GetAffinities(tenantScope) + .Where(pair => pair.Key.Contains("pizza", StringComparison.OrdinalIgnoreCase)) + .OrderByDescending(static pair => pair.Value == PersonalAffinity.Love ? 2 : pair.Value == PersonalAffinity.Like ? 1 : 0) + .FirstOrDefault(); + if (!string.IsNullOrWhiteSpace(affinityMatch.Key)) + { + return new PizzaSignal(affinityMatch.Value); + } + + foreach (var category in PizzaPreferenceCategories) + { + var preference = personalMemoryStore.GetPreference(tenantScope, category); + if (!string.IsNullOrWhiteSpace(preference) && + preference.Contains("pizza", StringComparison.OrdinalIgnoreCase)) + { + return new PizzaSignal(PersonalAffinity.Like); + } + } + + return new PizzaSignal(null); + } + private string BuildGenericReply(JiboExperienceCatalog catalog, string transcript, string lowered) { if (string.IsNullOrWhiteSpace(transcript)) @@ -303,6 +690,7 @@ public sealed class JiboInteractionService( IReadOnlyList listenRules, IReadOnlyDictionary clientEntities, string? lastClockDomain, + string? pendingProactivityOffer, bool isYesNoTurn, bool isTimerValueTurn, bool isAlarmValueTurn) @@ -335,6 +723,30 @@ public sealed class JiboInteractionService( }; } + if (!string.IsNullOrWhiteSpace(pendingProactivityOffer) && + string.Equals(pendingProactivityOffer, "pizza_fact", StringComparison.OrdinalIgnoreCase)) + { + if (IsAffirmativeReply(loweredTranscript)) + { + return "proactive_pizza_fact"; + } + + if (IsNegativeReply(loweredTranscript)) + { + return "proactive_offer_declined"; + } + } + + if (IsNameSetStatement(loweredTranscript)) + { + return "memory_set_name"; + } + + if (IsNameRecallQuestion(loweredTranscript)) + { + return "memory_get_name"; + } + if (IsUserBirthdaySetStatement(loweredTranscript)) { return "memory_set_birthday"; @@ -385,6 +797,12 @@ public sealed class JiboInteractionService( return "order_pizza"; } + if (string.Equals(clientIntent, "requestWeatherPR", StringComparison.OrdinalIgnoreCase) || + string.Equals(clientIntent, "requestWeather", StringComparison.OrdinalIgnoreCase)) + { + return "weather"; + } + if (IsCancelRequest(clientIntent, loweredTranscript)) { if (isAlarmValueTurn) @@ -479,6 +897,26 @@ public sealed class JiboInteractionService( return "memory_get_preference"; } + if (IsImportantDateSetStatement(loweredTranscript)) + { + return "memory_set_important_date"; + } + + if (IsImportantDateRecallQuestion(loweredTranscript)) + { + return "memory_get_important_date"; + } + + if (IsAffinitySetStatement(loweredTranscript)) + { + return "memory_set_affinity"; + } + + if (IsAffinityRecallQuestion(loweredTranscript)) + { + return "memory_get_affinity"; + } + if (TryResolveRadioGenre(loweredTranscript) is not null) { return "radio_genre"; @@ -674,7 +1112,7 @@ public sealed class JiboInteractionService( return "personal_report"; } - if (MatchesAny(loweredTranscript, "weather", "forecast", "weather report", "is it raining")) + if (IsWeatherRequest(loweredTranscript)) { return "weather"; } @@ -706,9 +1144,9 @@ public sealed class JiboInteractionService( switch (isYesNoTurn) { - case true when MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh"): + case true when IsAffirmativeReply(loweredTranscript): return "yes"; - case true when MatchesAny(loweredTranscript, "no", "nope", "nah"): + case true when IsNegativeReply(loweredTranscript): return "no"; } @@ -1177,6 +1615,200 @@ public sealed class JiboInteractionService( return candidates.Any(candidate => loweredTranscript.Contains(candidate, StringComparison.Ordinal)); } + private static bool IsAffirmativeReply(string loweredTranscript) + { + var normalized = NormalizeCommandPhrase(loweredTranscript); + return normalized is "yes" or "yeah" or "yep" or "yup" or "sure" or "ok" or "okay" or "absolutely" or "please do" or "why not" || + MatchesAny(normalized, "uh huh", "sounds good"); + } + + private static bool IsNegativeReply(string loweredTranscript) + { + var normalized = NormalizeCommandPhrase(loweredTranscript); + return normalized is "no" or "nope" or "nah" or "not now" or "no thanks" or "not today" || + MatchesAny(normalized, "no thank you", "maybe later"); + } + + private static bool IsWeatherRequest(string loweredTranscript) + { + if (MatchesAny( + loweredTranscript, + "weather", + "forecast", + "how is the weather", + "how s the weather", + "how's the weather", + "check the weather", + "weather report", + "what's today s weather", + "what's today's weather", + "what is the weather", + "what will the weather", + "what will tomorrow s weather", + "what will tomorrow's weather", + "look up the forecast", + "launch the weather skill", + "what is today s humidity", + "what is today's humidity", + "what's the humidity", + "what is the humidity")) + { + return true; + } + + return MatchesAny( + loweredTranscript, + "will it rain", + "will it snow", + "is it raining", + "is it snowing", + "is there going to be hail", + "does it look like rain", + "does it seem like snow", + "is it going to rain", + "is it going to snow", + "do you think it will rain", + "do you think it will snow"); + } + + private static string? TryResolveWeatherLocationQuery(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + var match = WeatherLocationPattern.Match(normalized); + if (!match.Success) + { + return null; + } + + var candidate = match.Groups["location"].Value.Trim(); + if (string.IsNullOrWhiteSpace(candidate)) + { + return null; + } + + candidate = WeatherLocationSuffixPattern.Replace(candidate, string.Empty).Trim(); + if (string.IsNullOrWhiteSpace(candidate) || + GenericWeatherLocationTerms.Contains(candidate)) + { + return null; + } + + return string.IsNullOrWhiteSpace(candidate) + ? null + : CultureInfo.InvariantCulture.TextInfo.ToTitleCase(candidate); + } + + private static (double Latitude, double Longitude)? TryResolveWeatherCoordinates(TurnContext turn) + { + if (!turn.Attributes.TryGetValue("context", out var contextValue) || + contextValue is null || + string.IsNullOrWhiteSpace(contextValue.ToString())) + { + return null; + } + + try + { + using var document = JsonDocument.Parse(contextValue.ToString()!); + if (!document.RootElement.TryGetProperty("runtime", out var runtime) || + runtime.ValueKind != JsonValueKind.Object || + !runtime.TryGetProperty("location", out var location) || + location.ValueKind != JsonValueKind.Object) + { + return null; + } + + var latitude = TryReadDoubleProperty(location, "lat", "latitude"); + var longitude = TryReadDoubleProperty(location, "lng", "lon", "longitude"); + return latitude is not null && longitude is not null + ? (latitude.Value, longitude.Value) + : null; + } + catch + { + return null; + } + } + + private static double? TryReadDoubleProperty(JsonElement source, params string[] propertyNames) + { + foreach (var propertyName in propertyNames) + { + if (source.TryGetProperty(propertyName, out var value) && + value.ValueKind == JsonValueKind.Number && + value.TryGetDouble(out var parsed)) + { + return parsed; + } + } + + return null; + } + + private static bool? ShouldUseCelsius(TurnContext turn, string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + if (normalized.Contains("celsius", StringComparison.Ordinal) || + normalized.Contains("centigrade", StringComparison.Ordinal)) + { + return true; + } + + if (normalized.Contains("fahrenheit", StringComparison.Ordinal)) + { + return false; + } + + var entities = ReadEntities(turn); + if (entities.TryGetValue("temperatureUnit", out var entityUnit)) + { + if (entityUnit.Contains("celsius", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (entityUnit.Contains("fahrenheit", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + } + + var locale = turn.Locale ?? string.Empty; + if (locale.EndsWith("-US", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return null; + } + + private static string? TryResolveWeatherDateEntity(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + if (MatchesAny(normalized, "tomorrow", "tomorrow s", "tomorrow's")) + { + return "tomorrow"; + } + + return null; + } + + private static string? TryResolveWeatherConditionEntity(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + return normalized switch + { + _ when normalized.Contains("rain", StringComparison.Ordinal) => "rain", + _ when normalized.Contains("snow", StringComparison.Ordinal) => "snow", + _ when normalized.Contains("hail", StringComparison.Ordinal) => "hail", + _ when normalized.Contains("sunny", StringComparison.Ordinal) || normalized.Contains("clear", StringComparison.Ordinal) => "sunny", + _ when normalized.Contains("cloud", StringComparison.Ordinal) => "cloudy", + _ when normalized.Contains("wind", StringComparison.Ordinal) => "windy", + _ when normalized.Contains("fog", StringComparison.Ordinal) => "fog", + _ => null + }; + } + private static bool IsDanceQuestion(string loweredTranscript) { return MatchesAny( @@ -1201,6 +1833,45 @@ public sealed class JiboInteractionService( "what day is your birthday"); } + private static bool IsNameSetStatement(string loweredTranscript) + { + return TryExtractNameFact(loweredTranscript) is not null; + } + + private static bool IsNameRecallQuestion(string loweredTranscript) + { + return MatchesAny( + loweredTranscript, + "what is my name", + "what s my name", + "what's my name", + "who am i", + "do you remember my name"); + } + + private static string? TryExtractNameFact(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + var prefixes = new[] + { + "my name is ", + "call me " + }; + + foreach (var prefix in prefixes) + { + if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) + { + continue; + } + + var name = normalized[prefix.Length..].Trim(); + return string.IsNullOrWhiteSpace(name) ? null : name; + } + + return null; + } + private static bool IsUserBirthdayRecallQuestion(string loweredTranscript) { return MatchesAny( @@ -1250,7 +1921,11 @@ public sealed class JiboInteractionService( "what is my favorite ", "what s my favorite ", "what's my favorite ", - "do you remember my favorite " + "do you remember my favorite ", + "what is my favourite ", + "what s my favourite ", + "what's my favourite ", + "do you remember my favourite " }; foreach (var prefix in prefixes) @@ -1270,29 +1945,191 @@ public sealed class JiboInteractionService( private static (string Category, string Value)? TryExtractPreferenceSet(string transcript) { var normalized = NormalizeCommandPhrase(transcript); - var marker = "my favorite "; - var markerIndex = normalized.IndexOf(marker, StringComparison.Ordinal); - if (markerIndex < 0) + foreach (var marker in PreferenceSetMarkers) + { + var markerIndex = normalized.IndexOf(marker, StringComparison.Ordinal); + if (markerIndex < 0) + { + continue; + } + + var preferencePhrase = normalized[(markerIndex + marker.Length)..]; + var splitMarker = " is "; + var splitIndex = preferencePhrase.IndexOf(splitMarker, StringComparison.Ordinal); + if (splitIndex <= 0 || splitIndex >= preferencePhrase.Length - splitMarker.Length) + { + continue; + } + + var category = preferencePhrase[..splitIndex].Trim(); + var value = preferencePhrase[(splitIndex + splitMarker.Length)..].Trim(); + if (!string.IsNullOrWhiteSpace(category) && !string.IsNullOrWhiteSpace(value)) + { + return (category, value); + } + } + + if (normalized.StartsWith("what ", StringComparison.Ordinal) || + normalized.StartsWith("do you remember ", StringComparison.Ordinal)) { return null; } - var preferencePhrase = normalized[(markerIndex + marker.Length)..]; - var splitMarker = " is "; - var splitIndex = preferencePhrase.IndexOf(splitMarker, StringComparison.Ordinal); - if (splitIndex <= 0 || splitIndex >= preferencePhrase.Length - splitMarker.Length) + foreach (var marker in PreferenceReverseMarkers) { - return null; + var markerIndex = normalized.IndexOf(marker, StringComparison.Ordinal); + if (markerIndex <= 0 || markerIndex >= normalized.Length - marker.Length) + { + continue; + } + + var value = normalized[..markerIndex].Trim(); + var category = normalized[(markerIndex + marker.Length)..].Trim(); + if (!string.IsNullOrWhiteSpace(category) && !string.IsNullOrWhiteSpace(value)) + { + return (category, value); + } } - var category = preferencePhrase[..splitIndex].Trim(); - var value = preferencePhrase[(splitIndex + splitMarker.Length)..].Trim(); - if (string.IsNullOrWhiteSpace(category) || string.IsNullOrWhiteSpace(value)) + return null; + } + + private static bool IsImportantDateSetStatement(string loweredTranscript) + { + return TryExtractImportantDateSet(loweredTranscript) is not null; + } + + private static bool IsImportantDateRecallQuestion(string loweredTranscript) + { + return TryExtractImportantDateLookupLabel(loweredTranscript) is not null; + } + + private static (string Label, string Value)? TryExtractImportantDateSet(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + var mapping = new (string Prefix, string Label)[] { - return null; + ("our anniversary is ", "anniversary"), + ("my anniversary is ", "anniversary"), + ("our wedding anniversary is ", "anniversary") + }; + + foreach (var (prefix, label) in mapping) + { + if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) + { + continue; + } + + var value = normalized[prefix.Length..].Trim(); + if (!string.IsNullOrWhiteSpace(value)) + { + return (label, value); + } } - return (category, value); + return null; + } + + private static string? TryExtractImportantDateLookupLabel(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + var candidates = new[] + { + "when is our anniversary", + "when s our anniversary", + "when's our anniversary", + "when is my anniversary", + "what is our anniversary", + "do you remember our anniversary" + }; + + return candidates.Any(candidate => string.Equals(normalized, candidate, StringComparison.Ordinal)) + ? "anniversary" + : null; + } + + private static bool IsAffinitySetStatement(string loweredTranscript) + { + return TryExtractAffinitySet(loweredTranscript) is not null; + } + + private static bool IsAffinityRecallQuestion(string loweredTranscript) + { + return TryExtractAffinityLookup(loweredTranscript) is not null; + } + + private static (string Item, PersonalAffinity Affinity)? TryExtractAffinitySet(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + + var directMappings = new (string Prefix, PersonalAffinity Affinity)[] + { + ("i love ", PersonalAffinity.Love), + ("i like ", PersonalAffinity.Like), + ("i dislike ", PersonalAffinity.Dislike), + ("i hate ", PersonalAffinity.Dislike), + ("i don t like ", PersonalAffinity.Dislike), + ("i dont like ", PersonalAffinity.Dislike), + ("i do not like ", PersonalAffinity.Dislike) + }; + + foreach (var (prefix, affinity) in directMappings) + { + if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) + { + continue; + } + + var item = normalized[prefix.Length..].Trim(); + if (!string.IsNullOrWhiteSpace(item)) + { + return (item, affinity); + } + } + + return null; + } + + private static (string Item, PersonalAffinity? ExpectedAffinity)? TryExtractAffinityLookup(string transcript) + { + var normalized = NormalizeCommandPhrase(transcript); + var expectationPrefixes = new (string Prefix, PersonalAffinity? ExpectedAffinity)[] + { + ("do i love ", PersonalAffinity.Love), + ("do i like ", PersonalAffinity.Like), + ("do i dislike ", PersonalAffinity.Dislike), + ("do i hate ", PersonalAffinity.Dislike), + ("how do i feel about ", null), + ("what do i think about ", null) + }; + + foreach (var (prefix, expectedAffinity) in expectationPrefixes) + { + if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) + { + continue; + } + + var item = normalized[prefix.Length..].Trim(); + if (!string.IsNullOrWhiteSpace(item)) + { + return (item, expectedAffinity); + } + } + + return null; + } + + private static string DescribeAffinityAsVerb(PersonalAffinity affinity) + { + return affinity switch + { + PersonalAffinity.Love => "love", + PersonalAffinity.Like => "like", + PersonalAffinity.Dislike => "dislike", + _ => "like" + }; } private static PersonalMemoryTenantScope ResolveTenantScope(TurnContext turn) @@ -1894,6 +2731,10 @@ public sealed class JiboInteractionService( private sealed record PizzaMimPrompt(string PromptId, string Esml); + private sealed record ProactivityCandidate(string IntentName, int Weight); + + private sealed record PizzaSignal(PersonalAffinity? Affinity); + private static readonly Regex SplitAlarmPattern = new( @"\b(?\d{1,2}|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)(?:[:\s,-]+(?\d{2}|[a-z\-]+(?:\s+[a-z\-]+)?))?\s*(?a[\s\.]*m\.?|p[\s\.]*m\.?)?\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); @@ -1922,6 +2763,14 @@ public sealed class JiboInteractionService( @"\b(?:cancel|delete|remove|stop|turn\s+off)\s+(?:the\s+)?(?:alarm|along|elo)\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + private static readonly Regex WeatherLocationPattern = new( + @"\bin\s+(?[a-z][a-z\s'\-]+)$", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex WeatherLocationSuffixPattern = new( + @"\b(?:today|tonight|tomorrow|outside|right now|please|thanks)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + private static readonly PizzaMimPrompt[] PizzaMimPrompts = [ new("RA_JBO_ShowPizzaMaking_AN_01", ""), @@ -1929,6 +2778,53 @@ public sealed class JiboInteractionService( new("RA_JBO_ShowPizzaMaking_AN_03", "My specialty .") ]; + private static readonly string[] PreferenceSetMarkers = + [ + "my favorite ", + "my favourite " + ]; + + private static readonly string[] PreferenceReverseMarkers = + [ + " is my favorite ", + " is my favourite " + ]; + + private static readonly string[] PizzaPreferenceCategories = + [ + "food", + "meal", + "dish", + "dinner", + "lunch", + "snack" + ]; + + private static readonly HashSet GenericWeatherLocationTerms = new(StringComparer.OrdinalIgnoreCase) + { + "my area", + "our area", + "this area", + "the area", + "the city", + "this city", + "our city", + "my city", + "the town", + "this town", + "our town", + "my town", + "our street", + "this street", + "my street", + "the neighborhood", + "the neighbourhood", + "this neighborhood", + "this neighbourhood", + "our neighborhood", + "our neighbourhood" + }; + private static readonly (string Phrase, string Station)[] RadioGenreAliases = [ ("country music", "Country"), diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs index f9ede49..6cd448e 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs @@ -50,6 +50,13 @@ public sealed class ProtocolToTurnContextMapper attributes["lastClockDomain"] = lastClockDomainText; } + if (session.Metadata.TryGetValue("pendingProactivityOffer", out var pendingProactivityOffer) && + pendingProactivityOffer is string pendingProactivityOfferText && + !string.IsNullOrWhiteSpace(pendingProactivityOfferText)) + { + attributes["pendingProactivityOffer"] = pendingProactivityOfferText; + } + attributes["listenHotphrase"] = turnState.ListenHotphrase; if (turnState.ListenRules.Count > 0) diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs index 9910961..7836512 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs @@ -37,6 +37,7 @@ public sealed class ResponsePlanToSocketMessagesMapper var isPhotoCreateLaunch = string.Equals(plan.IntentName, "snapshot", StringComparison.OrdinalIgnoreCase) || string.Equals(plan.IntentName, "photobooth", StringComparison.OrdinalIgnoreCase); var isClockSkillLaunch = string.Equals(skill?.SkillName, "@be/clock", StringComparison.OrdinalIgnoreCase); + var isReportSkillLaunch = string.Equals(skill?.SkillName, "report-skill", StringComparison.OrdinalIgnoreCase); var localIntent = ReadSkillPayloadString(skill, "localIntent"); var clockIntent = ReadSkillPayloadString(skill, "clockIntent"); var clockDomain = ReadSkillPayloadString(skill, "domain"); @@ -50,6 +51,8 @@ public sealed class ResponsePlanToSocketMessagesMapper var globalIntent = ReadSkillPayloadString(skill, "globalIntent"); var nluDomain = ReadSkillPayloadString(skill, "nluDomain"); var volumeLevel = ReadSkillPayloadString(skill, "volumeLevel"); + var reportDate = ReadSkillPayloadString(skill, "date"); + var reportWeatherCondition = ReadSkillPayloadString(skill, "weatherCondition"); var nluGuess = ReadClientEntity(turn, "guess"); var wordOfDayGuess = ResolveWordOfDayGuess(turn, transcript, nluGuess); var outboundIntent = isGlobalCommand && !string.IsNullOrWhiteSpace(globalIntent) @@ -64,6 +67,8 @@ public sealed class ResponsePlanToSocketMessagesMapper ? localIntent : isClockSkillLaunch && !string.IsNullOrWhiteSpace(clockIntent) ? clockIntent + : isReportSkillLaunch && !string.IsNullOrWhiteSpace(localIntent) + ? localIntent : isWordOfDayGuess ? "guess" : string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && @@ -112,6 +117,8 @@ public sealed class ResponsePlanToSocketMessagesMapper ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : [] + : isReportSkillLaunch + ? [] : isWordOfDayGuess ? ["word-of-the-day/puzzle"] : isYesNoTurn && isYesNoIntent @@ -136,7 +143,10 @@ public sealed class ResponsePlanToSocketMessagesMapper timerMinutes, timerSeconds, alarmTime, - alarmAmPm); + alarmAmPm, + isReportSkillLaunch, + reportDate, + reportWeatherCondition); var listenMessage = new { type = "LISTEN", @@ -159,6 +169,7 @@ public sealed class ResponsePlanToSocketMessagesMapper isPhotoGalleryLaunch ? "@be/gallery" : isPhotoCreateLaunch ? "@be/create" : isClockSkillLaunch ? "@be/clock" : + isReportSkillLaunch ? "report-skill" : null, isGlobalCommand ? nluDomain ?? "global_commands" : null), match = new @@ -286,6 +297,22 @@ public sealed class ResponsePlanToSocketMessagesMapper DelayMs: 125)); } + if (isReportSkillLaunch) + { + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildSkillRedirectPayload( + transId, + "report-skill", + outboundIntent, + outboundAsrText, + outboundRules, + entities)), + DelayMs: 75)); + messages.Add(new SocketReplyPlan( + JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "report-skill")), + DelayMs: 125)); + } + if (emitSkillActions && speak is not null) { messages.Add(new SocketReplyPlan( @@ -444,7 +471,10 @@ public sealed class ResponsePlanToSocketMessagesMapper string? timerMinutes, string? timerSeconds, string? alarmTime, - string? alarmAmPm) + string? alarmAmPm, + bool reportSkillLaunch, + string? reportDate, + string? reportWeatherCondition) { if (yesNoTurn) { @@ -514,6 +544,22 @@ public sealed class ResponsePlanToSocketMessagesMapper return entities; } + if (reportSkillLaunch) + { + var entities = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (!string.IsNullOrWhiteSpace(reportDate)) + { + entities["date"] = reportDate; + } + + if (!string.IsNullOrWhiteSpace(reportWeatherCondition)) + { + entities["Weather"] = reportWeatherCondition; + } + + return entities; + } + if (wordOfDayGuess) { return new Dictionary diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs index 87c1e74..f12a354 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs @@ -538,6 +538,9 @@ public sealed partial class WebSocketTurnFinalizationService( { session.Metadata["lastClockDomain"] = lastClockDomainValue.ToString(); } + + UpdatePendingProactivityOffer(session, plan.IntentName); + session.FollowUpExpiresUtc = plan.FollowUp.KeepMicOpen ? DateTimeOffset.UtcNow.Add(plan.FollowUp.Timeout) : null; @@ -567,13 +570,13 @@ public sealed partial class WebSocketTurnFinalizationService( !string.Equals(plan.IntentName, "alarm_cancel", StringComparison.OrdinalIgnoreCase) && !string.Equals(plan.IntentName, "timer_clarify", StringComparison.OrdinalIgnoreCase) && !string.Equals(plan.IntentName, "alarm_clarify", StringComparison.OrdinalIgnoreCase) && - !string.Equals(plan.IntentName, "timer_value", StringComparison.OrdinalIgnoreCase) && - !string.Equals(plan.IntentName, "alarm_value", StringComparison.OrdinalIgnoreCase) && - !string.Equals(plan.IntentName, "photo_gallery", StringComparison.OrdinalIgnoreCase) && - !string.Equals(plan.IntentName, "snapshot", StringComparison.OrdinalIgnoreCase) && - !string.Equals(plan.IntentName, "photobooth", StringComparison.OrdinalIgnoreCase) && - (messageType != "CLIENT_NLU" || - string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase)); + !string.Equals(plan.IntentName, "timer_value", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "alarm_value", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "photo_gallery", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "snapshot", StringComparison.OrdinalIgnoreCase) && + !string.Equals(plan.IntentName, "photobooth", StringComparison.OrdinalIgnoreCase) && + (messageType != "CLIENT_NLU" || + string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase)); var replies = ResponsePlanToSocketMessagesMapper.Map(plan, finalizedTurn, session, emitSkillActions).Select(map => new WebSocketReply { Text = map.Text, @@ -812,6 +815,7 @@ public sealed partial class WebSocketTurnFinalizationService( { var messageType = ReadMessageType(turn); var clientIntent = ReadAttribute(turn, "clientIntent"); + var pendingProactivityOffer = ReadAttribute(turn, "pendingProactivityOffer"); var transcript = NormalizeTranscript(turn.NormalizedTranscript ?? turn.RawTranscript); var listenRules = ReadRules(turn, "listenRules").Concat(ReadRules(turn, "clientRules")).ToArray(); @@ -846,6 +850,12 @@ public sealed partial class WebSocketTurnFinalizationService( return true; } + if (!string.IsNullOrWhiteSpace(pendingProactivityOffer) && + transcript is "yes" or "no" or "sure" or "nope" or "yup" or "uh huh" or "yeah" or "nah") + { + return true; + } + if (listenRules.Any(rule => string.Equals(rule, "word-of-the-day/puzzle", StringComparison.OrdinalIgnoreCase))) { return true; @@ -960,6 +970,17 @@ public sealed partial class WebSocketTurnFinalizationService( string.Equals(rule, "surprises-ota/want_to_download_now", StringComparison.OrdinalIgnoreCase); } + private static void UpdatePendingProactivityOffer(CloudSession session, string? intentName) + { + if (string.Equals(intentName, "proactive_offer_pizza_fact", StringComparison.OrdinalIgnoreCase)) + { + session.Metadata["pendingProactivityOffer"] = "pizza_fact"; + return; + } + + session.Metadata.Remove("pendingProactivityOffer"); + } + private static IEnumerable ReadRules(TurnContext turn, string key) { if (!turn.Attributes.TryGetValue(key, out var value) || value is null) diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs index 35eedda..0cee4b5 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs @@ -4,6 +4,7 @@ using Jibo.Cloud.Infrastructure.Audio; using Jibo.Cloud.Infrastructure.Content; using Jibo.Cloud.Infrastructure.Persistence; using Jibo.Cloud.Infrastructure.Telemetry; +using Jibo.Cloud.Infrastructure.Weather; using Jibo.Runtime.Abstractions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Configuration; @@ -23,7 +24,20 @@ public static class ServiceCollectionExtensions configuration.GetSection("OpenJibo:Stt").Bind(sttOptions); } + var openWeatherOptions = new OpenWeatherOptions(); + if (configuration is not null) + { + configuration.GetSection("OpenJibo:Weather:OpenWeather").Bind(openWeatherOptions); + } + + if (string.IsNullOrWhiteSpace(openWeatherOptions.ApiKey)) + { + openWeatherOptions.ApiKey = Environment.GetEnvironmentVariable("OPENWEATHER_API_KEY"); + } + services.AddSingleton(sttOptions); + services.AddSingleton(openWeatherOptions); + services.AddHttpClient(); var statePersistencePath = configuration?["OpenJibo:State:PersistencePath"] ?? Path.Combine(AppContext.BaseDirectory, "App_Data", "cloud-state.json"); services.AddSingleton(_ => new InMemoryCloudStateStore(statePersistencePath)); diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryPersonalMemoryStore.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryPersonalMemoryStore.cs index bc45bdf..077382f 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryPersonalMemoryStore.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryPersonalMemoryStore.cs @@ -36,6 +36,62 @@ public sealed class InMemoryPersonalMemoryStore : IPersonalMemoryStore : null; } + public void SetName(PersonalMemoryTenantScope tenantScope, string name) + { + var key = BuildTenantKey(tenantScope); + var record = _tenantMemory.GetOrAdd(key, static _ => new TenantMemoryRecord()); + record.Name = name; + } + + public string? GetName(PersonalMemoryTenantScope tenantScope) + { + var key = BuildTenantKey(tenantScope); + return _tenantMemory.TryGetValue(key, out var record) ? record.Name : null; + } + + public void SetImportantDate(PersonalMemoryTenantScope tenantScope, string label, string value) + { + var key = BuildTenantKey(tenantScope); + var record = _tenantMemory.GetOrAdd(key, static _ => new TenantMemoryRecord()); + record.ImportantDates[NormalizeCategory(label)] = value; + } + + public string? GetImportantDate(PersonalMemoryTenantScope tenantScope, string label) + { + var key = BuildTenantKey(tenantScope); + return _tenantMemory.TryGetValue(key, out var record) && + record.ImportantDates.TryGetValue(NormalizeCategory(label), out var value) + ? value + : null; + } + + public void SetAffinity(PersonalMemoryTenantScope tenantScope, string item, PersonalAffinity affinity) + { + var key = BuildTenantKey(tenantScope); + var record = _tenantMemory.GetOrAdd(key, static _ => new TenantMemoryRecord()); + record.Affinities[NormalizeCategory(item)] = affinity; + } + + public PersonalAffinity? GetAffinity(PersonalMemoryTenantScope tenantScope, string item) + { + var key = BuildTenantKey(tenantScope); + return _tenantMemory.TryGetValue(key, out var record) && + record.Affinities.TryGetValue(NormalizeCategory(item), out var affinity) + ? affinity + : null; + } + + public IReadOnlyDictionary GetAffinities(PersonalMemoryTenantScope tenantScope) + { + var key = BuildTenantKey(tenantScope); + if (!_tenantMemory.TryGetValue(key, out var record)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + return new Dictionary(record.Affinities, StringComparer.OrdinalIgnoreCase); + } + private static string BuildTenantKey(PersonalMemoryTenantScope tenantScope) { return $"{tenantScope.AccountId}|{tenantScope.LoopId}|{tenantScope.DeviceId}"; @@ -49,6 +105,9 @@ public sealed class InMemoryPersonalMemoryStore : IPersonalMemoryStore private sealed class TenantMemoryRecord { public string? Birthday { get; set; } + public string? Name { get; set; } public ConcurrentDictionary Preferences { get; } = new(StringComparer.OrdinalIgnoreCase); + public ConcurrentDictionary ImportantDates { get; } = new(StringComparer.OrdinalIgnoreCase); + public ConcurrentDictionary Affinities { get; } = new(StringComparer.OrdinalIgnoreCase); } } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherOptions.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherOptions.cs new file mode 100644 index 0000000..75d84f9 --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherOptions.cs @@ -0,0 +1,12 @@ +namespace Jibo.Cloud.Infrastructure.Weather; + +public sealed class OpenWeatherOptions +{ + public string BaseUrl { get; set; } = "https://api.openweathermap.org"; + + public string? ApiKey { get; set; } + + public string DefaultLocation { get; set; } = "Boston,US"; + + public bool UseCelsius { get; set; } +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherReportProvider.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherReportProvider.cs new file mode 100644 index 0000000..9b107a7 --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Weather/OpenWeatherReportProvider.cs @@ -0,0 +1,364 @@ +using System.Globalization; +using System.Text.Json; +using Jibo.Cloud.Application.Abstractions; +using Microsoft.Extensions.Logging; + +namespace Jibo.Cloud.Infrastructure.Weather; + +public sealed class OpenWeatherReportProvider( + HttpClient httpClient, + OpenWeatherOptions options, + ILogger logger) + : IWeatherReportProvider +{ + public async Task GetReportAsync( + WeatherReportRequest request, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(options.ApiKey)) + { + return null; + } + + try + { + var location = await ResolveLocationAsync(request, cancellationToken); + if (location is null) + { + return null; + } + + var useCelsius = request.UseCelsius ?? options.UseCelsius; + return request.IsTomorrow + ? await GetTomorrowForecastAsync(location.Value, useCelsius, cancellationToken) + : await GetCurrentWeatherAsync(location.Value, useCelsius, cancellationToken); + } + catch (Exception exception) + { + logger.LogWarning(exception, "OpenWeather lookup failed."); + return null; + } + } + + private async Task ResolveLocationAsync( + WeatherReportRequest request, + CancellationToken cancellationToken) + { + if (request is { Latitude: not null, Longitude: not null }) + { + return new LocationPoint(request.Latitude.Value, request.Longitude.Value, null); + } + + var query = string.IsNullOrWhiteSpace(request.LocationQuery) + ? options.DefaultLocation + : request.LocationQuery.Trim(); + if (string.IsNullOrWhiteSpace(query)) + { + return null; + } + + var geocodeUri = BuildRequestUri( + "/geo/1.0/direct", + ("q", query), + ("limit", "1"), + ("appid", options.ApiKey!)); + using var response = await httpClient.GetAsync(geocodeUri, cancellationToken); + if (!response.IsSuccessStatusCode) + { + return null; + } + + using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + if (document.RootElement.ValueKind != JsonValueKind.Array || + document.RootElement.GetArrayLength() == 0) + { + return null; + } + + var location = document.RootElement[0]; + if (!TryReadDouble(location, "lat", out var latitude) || + !TryReadDouble(location, "lon", out var longitude)) + { + return null; + } + + var displayName = BuildLocationDisplayName(location); + return new LocationPoint(latitude, longitude, displayName); + } + + private async Task GetCurrentWeatherAsync( + LocationPoint location, + bool useCelsius, + CancellationToken cancellationToken) + { + var weatherUri = BuildRequestUri( + "/data/2.5/weather", + ("lat", location.Latitude.ToString(CultureInfo.InvariantCulture)), + ("lon", location.Longitude.ToString(CultureInfo.InvariantCulture)), + ("units", useCelsius ? "metric" : "imperial"), + ("appid", options.ApiKey!)); + using var response = await httpClient.GetAsync(weatherUri, cancellationToken); + if (!response.IsSuccessStatusCode) + { + return null; + } + + using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + var root = document.RootElement; + if (!root.TryGetProperty("main", out var main)) + { + return null; + } + + var locationName = ReadNonEmptyString(root, "name") ?? location.DisplayName ?? options.DefaultLocation; + var summary = TryReadWeatherSummary(root); + var condition = TryReadWeatherCondition(root); + var temperature = TryReadInt(main, "temp"); + var high = TryReadInt(main, "temp_max"); + var low = TryReadInt(main, "temp_min"); + if (temperature is null && high is null && low is null) + { + return null; + } + + var resolvedTemperature = temperature ?? high ?? low ?? 0; + return new WeatherReportSnapshot( + locationName, + summary ?? "partly cloudy", + resolvedTemperature, + high, + low, + condition, + useCelsius); + } + + private async Task GetTomorrowForecastAsync( + LocationPoint location, + bool useCelsius, + CancellationToken cancellationToken) + { + var forecastUri = BuildRequestUri( + "/data/2.5/forecast", + ("lat", location.Latitude.ToString(CultureInfo.InvariantCulture)), + ("lon", location.Longitude.ToString(CultureInfo.InvariantCulture)), + ("units", useCelsius ? "metric" : "imperial"), + ("appid", options.ApiKey!)); + using var response = await httpClient.GetAsync(forecastUri, cancellationToken); + if (!response.IsSuccessStatusCode) + { + return null; + } + + using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + var root = document.RootElement; + if (!root.TryGetProperty("list", out var list) || list.ValueKind != JsonValueKind.Array) + { + return null; + } + + var offset = TryReadForecastOffset(root); + var tomorrow = DateOnly.FromDateTime(DateTimeOffset.UtcNow.ToOffset(offset).DateTime.AddDays(1)); + var entries = new List(); + foreach (var item in list.EnumerateArray()) + { + if (!TryReadLong(item, "dt", out var unixSeconds)) + { + continue; + } + + var localTimestamp = DateTimeOffset.FromUnixTimeSeconds(unixSeconds).ToOffset(offset); + if (DateOnly.FromDateTime(localTimestamp.DateTime) != tomorrow) + { + continue; + } + + if (!item.TryGetProperty("main", out var main)) + { + continue; + } + + entries.Add(new ForecastEntry( + localTimestamp, + TryReadInt(main, "temp"), + TryReadInt(main, "temp_max"), + TryReadInt(main, "temp_min"), + TryReadWeatherSummary(item), + TryReadWeatherCondition(item))); + } + + if (entries.Count == 0) + { + return null; + } + + var selectedEntry = entries + .OrderBy(entry => Math.Abs((entry.LocalTime.TimeOfDay - TimeSpan.FromHours(12)).TotalMinutes)) + .First(); + var highs = entries + .Where(entry => entry.HighTemperature is not null) + .Select(entry => entry.HighTemperature!.Value) + .ToArray(); + var lows = entries + .Where(entry => entry.LowTemperature is not null) + .Select(entry => entry.LowTemperature!.Value) + .ToArray(); + + var locationName = ReadForecastLocationName(root) ?? location.DisplayName ?? options.DefaultLocation; + var high = highs.Length > 0 ? highs.Max() : selectedEntry.HighTemperature; + var low = lows.Length > 0 ? lows.Min() : selectedEntry.LowTemperature; + var temperature = selectedEntry.Temperature ?? high ?? low ?? 0; + + return new WeatherReportSnapshot( + locationName, + selectedEntry.Summary ?? "partly cloudy", + temperature, + high, + low, + selectedEntry.Condition, + useCelsius); + } + + private Uri BuildRequestUri(string path, params (string Key, string Value)[] queryParts) + { + var baseUrl = options.BaseUrl.TrimEnd('/'); + var query = string.Join( + "&", + queryParts.Select(part => + $"{Uri.EscapeDataString(part.Key)}={Uri.EscapeDataString(part.Value)}")); + return new Uri($"{baseUrl}{path}?{query}"); + } + + private static TimeSpan TryReadForecastOffset(JsonElement root) + { + if (!root.TryGetProperty("city", out var city)) + { + return TimeSpan.Zero; + } + + var timezoneSeconds = TryReadInt(city, "timezone"); + if (timezoneSeconds is null) + { + return TimeSpan.Zero; + } + + var seconds = Math.Clamp(timezoneSeconds.Value, -50400, 50400); + return TimeSpan.FromSeconds(seconds); + } + + private static string? ReadForecastLocationName(JsonElement root) + { + if (!root.TryGetProperty("city", out var city)) + { + return null; + } + + var name = ReadNonEmptyString(city, "name"); + var country = ReadNonEmptyString(city, "country"); + return string.IsNullOrWhiteSpace(country) ? name : $"{name}, {country}"; + } + + private static string? BuildLocationDisplayName(JsonElement location) + { + var name = ReadNonEmptyString(location, "name"); + var state = ReadNonEmptyString(location, "state"); + var country = ReadNonEmptyString(location, "country"); + if (!string.IsNullOrWhiteSpace(name) && + !string.IsNullOrWhiteSpace(state) && + !string.IsNullOrWhiteSpace(country)) + { + return $"{name}, {state}, {country}"; + } + + if (!string.IsNullOrWhiteSpace(name) && !string.IsNullOrWhiteSpace(country)) + { + return $"{name}, {country}"; + } + + return name; + } + + private static string? TryReadWeatherSummary(JsonElement root) + { + return TryReadWeatherProperty(root, "description"); + } + + private static string? TryReadWeatherCondition(JsonElement root) + { + var main = TryReadWeatherProperty(root, "main"); + if (string.IsNullOrWhiteSpace(main)) + { + return null; + } + + var normalized = main.Trim().ToLowerInvariant(); + return normalized switch + { + "rain" or "drizzle" or "thunderstorm" => "rain", + "snow" => "snow", + "clear" => "sunny", + "clouds" => "cloudy", + "mist" or "smoke" or "haze" or "fog" => "fog", + _ => normalized + }; + } + + private static string? TryReadWeatherProperty(JsonElement root, string key) + { + if (!root.TryGetProperty("weather", out var weather) || + weather.ValueKind != JsonValueKind.Array || + weather.GetArrayLength() == 0) + { + return null; + } + + var first = weather[0]; + return ReadNonEmptyString(first, key); + } + + private static string? ReadNonEmptyString(JsonElement source, string key) + { + return source.TryGetProperty(key, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + } + + private static bool TryReadDouble(JsonElement source, string key, out double value) + { + value = 0; + return source.TryGetProperty(key, out var element) && element.TryGetDouble(out value); + } + + private static bool TryReadLong(JsonElement source, string key, out long value) + { + value = 0; + return source.TryGetProperty(key, out var element) && element.TryGetInt64(out value); + } + + private static int? TryReadInt(JsonElement source, string key) + { + if (!source.TryGetProperty(key, out var element)) + { + return null; + } + + if (element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var numeric)) + { + return (int)Math.Round(numeric, MidpointRounding.AwayFromZero); + } + + return null; + } + + private readonly record struct LocationPoint(double Latitude, double Longitude, string? DisplayName); + + private sealed record ForecastEntry( + DateTimeOffset LocalTime, + int? Temperature, + int? HighTemperature, + int? LowTemperature, + string? Summary, + string? Condition); +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index 5469307..41ca2ef 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -246,6 +246,232 @@ public sealed class JiboInteractionServiceTests Assert.Equal("I do not know your birthday yet. You can say, my birthday is March 14.", otherTenantRecall.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_NameMemory_SetThenRecallWithinTenant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "my name is Alex", + NormalizedTranscript = "my name is Alex", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_name", setDecision.IntentName); + Assert.Equal("Nice to meet you, alex. I will remember your name.", setDecision.ReplyText); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what is my name", + NormalizedTranscript = "what is my name", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_name", recallDecision.IntentName); + Assert.Equal("You told me your name is alex.", recallDecision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_ImportantDateMemory_SetThenRecallWithinTenant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "our anniversary is June 10", + NormalizedTranscript = "our anniversary is June 10", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_important_date", setDecision.IntentName); + Assert.Equal("Got it. I will remember your anniversary is june 10.", setDecision.ReplyText); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "when is our anniversary", + NormalizedTranscript = "when is our anniversary", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_important_date", recallDecision.IntentName); + Assert.Equal("You told me your anniversary is june 10.", recallDecision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_AffinityMemory_SetThenRecallWithinTenant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "I dislike mushrooms", + NormalizedTranscript = "I dislike mushrooms", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_affinity", setDecision.IntentName); + Assert.Equal("Got it. I will remember you dislike mushrooms.", setDecision.ReplyText); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "do i dislike mushrooms", + NormalizedTranscript = "do i dislike mushrooms", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_affinity", recallDecision.IntentName); + Assert.Equal("Yes. You told me you dislike mushrooms.", recallDecision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_PreferenceReversePhrase_ParsesFavoriteVariant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "pizza is my favorite food", + NormalizedTranscript = "pizza is my favorite food", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_preference", setDecision.IntentName); + Assert.Equal("Got it. I will remember your favorite food is pizza.", setDecision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_Surprise_WithPizzaPreference_UsesPizzaProactivity() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "my favorite food is pizza", + NormalizedTranscript = "my favorite food is pizza", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "surprise me", + NormalizedTranscript = "surprise me", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("proactive_pizza_preference", decision.IntentName); + Assert.Equal("chitchat-skill", decision.SkillName); + Assert.Equal("RA_JBO_MakePizza", decision.SkillPayload!["mim_id"]); + } + + [Fact] + public async Task BuildDecisionAsync_Surprise_OnNationalPizzaDay_UsesHolidayProactivity() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "surprise me", + NormalizedTranscript = "surprise me", + Attributes = new Dictionary + { + ["context"] = """{"runtime":{"location":{"iso":"2026-02-09T10:45:00-06:00"}}}""" + } + }); + + Assert.Equal("proactive_pizza_day", decision.IntentName); + Assert.Equal("chitchat-skill", decision.SkillName); + Assert.Contains("National Pizza Day", decision.ReplyText, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task BuildDecisionAsync_PendingPizzaFactOffer_YesMapsToFact() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "yes", + NormalizedTranscript = "yes", + Attributes = new Dictionary + { + ["pendingProactivityOffer"] = "pizza_fact" + } + }); + + Assert.Equal("proactive_pizza_fact", decision.IntentName); + Assert.Contains("350 slices per second", decision.ReplyText, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task BuildDecisionAsync_PendingPizzaFactOffer_NoMapsToDecline() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "no", + NormalizedTranscript = "no", + Attributes = new Dictionary + { + ["pendingProactivityOffer"] = "pizza_fact" + } + }); + + Assert.Equal("proactive_offer_declined", decision.IntentName); + Assert.Equal("No problem. We can save the pizza fact for another time.", decision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_MakePizza_UsesOriginalMimStylePayload() { @@ -339,6 +565,117 @@ public sealed class JiboInteractionServiceTests Assert.Equal("RA_JBO_OrderPizza", decision.SkillPayload!["mim_id"]); } + [Fact] + public async Task BuildDecisionAsync_WeatherQuery_LaunchesReportSkillWithPegasusIntent() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "how is the weather", + NormalizedTranscript = "how is the weather" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("report-skill", decision.SkillName); + Assert.Equal("requestWeatherPR", decision.SkillPayload!["localIntent"]); + Assert.Equal("weather", decision.SkillPayload["cloudSkill"]); + } + + [Fact] + public async Task BuildDecisionAsync_WeatherTomorrowQuery_SetsTomorrowEntity() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what's the weather tomorrow", + NormalizedTranscript = "what's the weather tomorrow" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("tomorrow", decision.SkillPayload!["date"]); + } + + [Fact] + public async Task BuildDecisionAsync_WeatherConditionQuery_SetsWeatherConditionEntity() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "will it rain tomorrow", + NormalizedTranscript = "will it rain tomorrow" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("rain", decision.SkillPayload!["weatherCondition"]); + Assert.Equal("tomorrow", decision.SkillPayload["date"]); + } + + [Fact] + public async Task BuildDecisionAsync_ClientNluRequestWeatherPR_LaunchesReportSkill() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "requestWeatherPR", + NormalizedTranscript = "requestWeatherPR", + Attributes = new Dictionary + { + ["clientIntent"] = "requestWeatherPR" + } + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("report-skill", decision.SkillName); + Assert.Equal("requestWeatherPR", decision.SkillPayload!["localIntent"]); + } + + [Fact] + public async Task BuildDecisionAsync_WeatherQuery_WithProvider_UsesProviderSummary() + { + var provider = new CapturingWeatherReportProvider + { + Snapshot = new WeatherReportSnapshot("Boston, US", "light rain", 61, 65, 54, "rain", false) + }; + var service = CreateService(weatherReportProvider: provider); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "how is the weather", + NormalizedTranscript = "how is the weather" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("Right now in Boston, US, it is light rain and 61 degrees Fahrenheit.", decision.ReplyText); + Assert.Equal("openweather", decision.SkillPayload!["provider"]); + Assert.Equal(61, decision.SkillPayload["temperature"]); + Assert.Equal("rain", decision.SkillPayload["weatherCondition"]); + } + + [Fact] + public async Task BuildDecisionAsync_WeatherLocationTomorrow_WithProvider_PassesLocationAndTomorrowRequest() + { + var provider = new CapturingWeatherReportProvider + { + Snapshot = new WeatherReportSnapshot("Chicago, US", "mostly cloudy", 72, 74, 60, "cloudy", false) + }; + var service = CreateService(weatherReportProvider: provider); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what's the weather in chicago tomorrow", + NormalizedTranscript = "what's the weather in chicago tomorrow" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("Chicago", provider.LastRequest?.LocationQuery); + Assert.True(provider.LastRequest?.IsTomorrow); + Assert.Equal("Tomorrow in Chicago, US, expect mostly cloudy with a high near 74 degrees Fahrenheit and a low around 60 degrees Fahrenheit.", decision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_ClientNluAskForDate_MapsToDateIntent() { @@ -1295,12 +1632,15 @@ public sealed class JiboInteractionServiceTests Assert.Equal("aglet", decision.SkillPayload!["guess"]); } - private static JiboInteractionService CreateService(IPersonalMemoryStore? personalMemoryStore = null) + private static JiboInteractionService CreateService( + IPersonalMemoryStore? personalMemoryStore = null, + IWeatherReportProvider? weatherReportProvider = null) { return new JiboInteractionService( new JiboExperienceContentCache(new InMemoryJiboExperienceContentRepository()), new FirstItemRandomizer(), - personalMemoryStore ?? new InMemoryPersonalMemoryStore()); + personalMemoryStore ?? new InMemoryPersonalMemoryStore(), + weatherReportProvider); } private sealed class FirstItemRandomizer : IJiboRandomizer @@ -1310,4 +1650,19 @@ public sealed class JiboInteractionServiceTests return items[0]; } } + + private sealed class CapturingWeatherReportProvider : IWeatherReportProvider + { + public WeatherReportRequest? LastRequest { get; private set; } + + public WeatherReportSnapshot? Snapshot { get; init; } + + public Task GetReportAsync( + WeatherReportRequest request, + CancellationToken cancellationToken = default) + { + LastRequest = request; + return Task.FromResult(Snapshot); + } + } } diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 15c9b89..521321d 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -1696,6 +1696,92 @@ public sealed class JiboWebSocketServiceTests Assert.Equal("announcement", meta.GetProperty("mim_type").GetString()); } + [Fact] + public async Task ClientAsr_HowIsTheWeather_EmitsReportSkillRedirectAndCompletion() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-weather-token", + Text = """{"type":"LISTEN","transID":"trans-weather","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-weather-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-weather","data":{"text":"how is the weather"}}""" + }); + + Assert.Equal(5, replies.Count); + Assert.Equal("LISTEN", ReadReplyType(replies[0])); + Assert.Equal("EOS", ReadReplyType(replies[1])); + Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2])); + Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3])); + Assert.Equal("SKILL_ACTION", ReadReplyType(replies[4])); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + Assert.Equal("requestWeatherPR", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + Assert.Equal("report-skill", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString()); + Assert.Equal("weather", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("cloudSkill").GetString()); + + using var redirectPayload = JsonDocument.Parse(replies[2].Text!); + Assert.Equal("report-skill", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString()); + Assert.Equal("requestWeatherPR", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + + using var completionPayload = JsonDocument.Parse(replies[3].Text!); + Assert.Equal("report-skill", completionPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString()); + + using var speakPayload = JsonDocument.Parse(replies[4].Text!); + var esml = speakPayload.RootElement + .GetProperty("data") + .GetProperty("action") + .GetProperty("config") + .GetProperty("jcp") + .GetProperty("config") + .GetProperty("play") + .GetProperty("esml") + .GetString(); + Assert.Contains("Checking your weather report", esml, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ClientAsr_WillItRainTomorrow_EmitsReportSkillWeatherEntities() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-weather-entities-token", + Text = """{"type":"LISTEN","transID":"trans-weather-entities","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-weather-entities-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-weather-entities","data":{"text":"will it rain tomorrow"}}""" + }); + + Assert.Equal(5, replies.Count); + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + var entities = listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities"); + Assert.Equal("tomorrow", entities.GetProperty("date").GetString()); + Assert.Equal("rain", entities.GetProperty("Weather").GetString()); + + using var redirectPayload = JsonDocument.Parse(replies[2].Text!); + var redirectEntities = redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities"); + Assert.Equal("tomorrow", redirectEntities.GetProperty("date").GetString()); + Assert.Equal("rain", redirectEntities.GetProperty("Weather").GetString()); + } + [Fact] public async Task ClientAsr_OpenTheRadio_EmitsRadioRedirectAndSilentCompletion() { @@ -2974,6 +3060,51 @@ public sealed class JiboWebSocketServiceTests Assert.Contains("I do not know your birthday yet", otherEsml, StringComparison.OrdinalIgnoreCase); } + [Fact] + public async Task ClientAsrSurpriseOffer_PersistsPendingOfferAndResolvesYesFollowUp() + { + var token = _store.IssueRobotToken("proactivity-device-a"); + + var offerReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"CLIENT_ASR","transID":"trans-proactive-offer","data":{"text":"surprise me"}}""" + }); + + Assert.Equal(3, offerReplies.Count); + using (var offerListenPayload = JsonDocument.Parse(offerReplies[0].Text!)) + { + Assert.Equal("proactive_offer_pizza_fact", offerListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + } + + var session = _store.FindSessionByToken(token); + Assert.NotNull(session); + Assert.True(session.Metadata.TryGetValue("pendingProactivityOffer", out var pendingOffer)); + Assert.Equal("pizza_fact", pendingOffer?.ToString()); + + var followUpReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"CLIENT_ASR","transID":"trans-proactive-offer-yes","data":{"text":"yes"}}""" + }); + + Assert.Equal(3, followUpReplies.Count); + using (var followUpListenPayload = JsonDocument.Parse(followUpReplies[0].Text!)) + { + Assert.Equal("proactive_pizza_fact", followUpListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + } + + session = _store.FindSessionByToken(token); + Assert.NotNull(session); + Assert.False(session.Metadata.ContainsKey("pendingProactivityOffer")); + } + [Fact] public async Task FollowUpTurn_UsesNewTurnStateWithoutLeakingBufferedAudio() {