From 92491adf85025e7308079c3f3d16227385ac6901 Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Thu, 7 May 2026 07:22:33 -0500 Subject: [PATCH] Add personal report parity planning and weather visuals --- OpenJibo/docs/feature-backlog.md | 100 ++++++- OpenJibo/docs/greetings-presence-plan.md | 173 ++++++++++++ OpenJibo/docs/personal-report-parity-plan.md | 105 +++++++ OpenJibo/docs/release-1.0.19-plan.md | 54 +++- OpenJibo/docs/system-diagram-alignment.md | 24 +- .../Services/JiboInteractionService.cs | 266 ++++++++++++++++-- .../WebSocketTurnFinalizationService.cs | 19 +- .../WebSockets/JiboInteractionServiceTests.cs | 166 ++++++++++- .../WebSockets/JiboWebSocketServiceTests.cs | 118 ++++++++ 9 files changed, 987 insertions(+), 38 deletions(-) create mode 100644 OpenJibo/docs/greetings-presence-plan.md create mode 100644 OpenJibo/docs/personal-report-parity-plan.md diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index 663f537..bcba023 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -476,7 +476,7 @@ Current release theme: ### Next Up (`2026-05-06`): Dialog Parsing Expansion And Ambiguity Guardrails -- Status: `ready` +- Status: `polish` - Tags: `protocol`, `content`, `stt`, `docs` - Why now: - this is the next queued `1.0.19` implementation slice after weather provider bring-up @@ -487,6 +487,13 @@ Current release theme: - add ambiguity guardrails for overlapping intents (date vs birthday, generic chat vs memory set/lookup, weather variants) - preserve command-vs-question personality behavior and stock skill launch compatibility - add focused tests for new phrase families and negative boundary cases +- Progress update (`2026-05-07`): + - implemented date/time guardrails so birthday phrasing is not misrouted to date + - expanded phrase coverage for: + - birthday alias set/recall (`bday` variants) + - shorthand favorites (`my favorite sport football`) + - weather phrasing (`what's today's weather look like`, `will it be sunny tomorrow`) + - updated continuation deferral so complete shorthand favorites finalize instead of waiting for missing continuation - Exit criteria: - ambiguous phrase handling is improved without regressions in existing `1.0.19` features - phrase imports are documented and traceable to Pegasus parser sources @@ -678,6 +685,76 @@ Current release theme: - connect weather units and location directly to user/report-skill settings parity instead of config defaults - add richer condition-change commentary and view parity with original report-skill weather behaviors +### 26. Presence-Aware Greetings And Identity Proactivity + +- Status: `ready` +- Tags: `protocol`, `content`, `storage`, `docs` +- Why now: + - this is the next personality-charm expansion after parser guardrail and weather bring-up + - Pegasus greetings behavior is strongly tied to presence/identity signals and proactive cooldown policy + - current OpenJibo has memory/proactivity foundations but no first-class presence extraction path yet +- Pegasus source anchors: + - `C:\Projects\jibo\pegasus\packages\hub\be-skills\greetings_manifest.json` + - `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSkill.ts` + - `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSM.ts` + - `C:\Projects\jibo\pegasus\packages\hub\src\proactive\ProactiveTransactionHandler.ts` + - `C:\Projects\jibo\pegasus\packages\hub\src\proactive\tools\ContextTools.ts` +- Scope: + - extract presence/identity context (`speaker`, `peoplePresent`, focused person) from runtime context payload + - add greeting intent families and state-machine split for reactive vs proactive greeting routes + - add cooldown and trigger-source guardrails for proactive greetings + - start person-aware greeting hooks (name-aware greeting, morning greeting policy, return greeting policy) +- Exit criteria: + - presence-aware greetings are routed deterministically with tests + - proactive greetings are frequency-bounded and do not trigger from surprise source when blocked by policy + - fallback behavior remains stable when identity is unknown or context is incomplete + - docs and release tracking are updated with shipped scope and residual gaps +- Tracking: + - [greetings-presence-plan.md](greetings-presence-plan.md) + - [release-1.0.19-plan.md](release-1.0.19-plan.md) + +### 27. Personal Report Parity Track (Weather/News/Commute/Calendar) + +- Status: `ready` +- Tags: `protocol`, `content`, `storage`, `docs` +- Why now: + - personal report is a core Jibo charm surface and currently split between implemented weather speech and placeholder calendar/commute/news content + - Pegasus weather used explicit condition animations and weather views; current OpenJibo weather is functional but visually lighter +- Scope: + - weather icon/animation parity and view support + - broader non-local weather query handling and short-range date coverage + - provider-backed news ingestion and filtering + - commute provider path and settings schema + - coverage matrix for personal report parity gaps and test/capture exit criteria +- Source anchors: + - `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\weather\WeatherMimLogic.ts` + - `C:\Projects\jibo\pegasus\packages\report-skill\resources\views\weatherHiLo.json` + - `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\news\NewsMimLogic.ts` + - `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\commute\CommuteMimLogic.ts` + - `C:\Projects\jibo\pegasus\packages\hub\pegasus-skills\report_skill_manifest.json` +- Tracking: + - [personal-report-parity-plan.md](personal-report-parity-plan.md) + - [release-1.0.19-plan.md](release-1.0.19-plan.md) + +### 28. Grocery List Capability (Requested Feature) + +- Status: `discovery` +- Tags: `content`, `docs`, `storage` +- Why now: + - directly requested by Jibo owners and fits memory + household utility roadmap +- Source findings: + - Pegasus has scripted responses for shopping/to-do list requests but no standalone grocery-list skill in this snapshot + - examples: + - `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ShoppingList.mim` + - `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ManageToDoList.mim` +- Candidate delivery paths: + - native lightweight list skill (fastest user value) + - integration-backed list orchestration (long-term richer ecosystem fit) +- Exit criteria: + - clear decision on MVP path + - first schema for list items + ownership scope + - initial voice flows and follow-up intent handling defined + ## Suggested Order Before closing `1.0.18`: @@ -696,15 +773,18 @@ For `1.0.19`: 2. Expand memory-backed personal facts with tenant-scoped storage (beyond the first birthday/preferences foundation) - implemented 3. Proactivity selector baseline with source-backed first offers - implemented 4. Weather report-skill launch compatibility - implemented -5. Dialog parsing expansion and ambiguity guardrails - queued next as of `2026-05-06` -6. Holidays and seasonal personality behavior built on the new memory/proactivity foundation -7. Durable memory persistence path (multi-tenant backing store) -8. Update, backup, and restore proof -9. STT upgrade and noise screening -10. Hosted capture/storage plan / indexing for group testing -11. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc. -12. Provider-backed news and weather parity polish -13. Lasso, identity, and onboarding as larger discovery-driven tracks +5. Dialog parsing expansion and ambiguity guardrails - in progress (`2026-05-07` first guardrail slice implemented) +6. Presence-aware greetings and identity-triggered proactivity - ready +7. Personal report parity track (weather visuals, live news path, commute path, calendar parity matrix) - ready +8. Holidays and seasonal personality behavior built on the new memory/proactivity foundation +9. Durable memory persistence path (multi-tenant backing store) +10. Update, backup, and restore proof +11. STT upgrade and noise screening +12. Hosted capture/storage plan / indexing for group testing +13. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc. +14. Provider-backed news and weather parity polish +15. Grocery list capability discovery and MVP selection +16. Lasso, identity, and onboarding as larger discovery-driven tracks For `1.0.20` and beyond: diff --git a/OpenJibo/docs/greetings-presence-plan.md b/OpenJibo/docs/greetings-presence-plan.md new file mode 100644 index 0000000..01a7e8d --- /dev/null +++ b/OpenJibo/docs/greetings-presence-plan.md @@ -0,0 +1,173 @@ +# Greetings And Presence Plan (`1.0.19`) + +## Purpose + +Recreate the original Jibo greeting charm with modern cloud architecture: + +- person-aware greetings when someone is detected +- proactive offers tied to presence, time of day, and memory +- safe cooldown rules so proactivity feels alive, not noisy + +This plan is source-anchored to Pegasus and scoped to shippable slices. + +## Pegasus Behavior Baseline + +Primary source artifacts: + +- `C:\Projects\jibo\pegasus\packages\hub\be-skills\greetings_manifest.json` +- `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSkill.ts` +- `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSM.ts` +- `C:\Projects\jibo\sdk\skills\greetings\src\states\IntentSplit.ts` +- `C:\Projects\jibo\sdk\skills\greetings\src\states\ProactiveGreetingState.ts` +- `C:\Projects\jibo\sdk\skills\greetings\src\states\ProactiveProbabilityState.ts` +- `C:\Projects\jibo\sdk\skills\greetings\src\states\ShouldDoMorningGreetingState.ts` +- `C:\Projects\jibo\sdk\skills\greetings\src\states\ShouldDoBirthdayState.ts` +- `C:\Projects\jibo\sdk\skills\greetings\src\states\ShouldDoHolidayState.ts` +- `C:\Projects\jibo\pegasus\packages\hub\src\proactive\ProactiveTransactionHandler.ts` +- `C:\Projects\jibo\pegasus\packages\hub\src\proactive\tools\ContextTools.ts` + +Key behaviors to port: + +- explicit reactive/proactive greeting split +- identity source split: + - reactive path uses active speaker + - proactive path uses present identified persons +- hub-level proactive gating: + - block greetings when trigger source is `SURPRISE` + - throttle by interaction history (`GreetingsLaunchLast2Hours < 1`) +- morning/birthday/holiday gates with per-user recency checks +- optional follow-up response flow after proactive greetings + +## Current OpenJibo Baseline + +Current implementation anchor: + +- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\JiboInteractionService.cs` +- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\ProtocolToTurnContextMapper.cs` +- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\WebSocketTurnFinalizationService.cs` +- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\ChitchatStateMachine.cs` +- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Infrastructure\Persistence\InMemoryPersonalMemoryStore.cs` + +What we already have: + +- tenant-scoped memory primitives (name, birthday, preferences, affinity) +- proactivity baseline with pending-offer follow-up handling +- state-machine style chitchat split (`ScriptedResponse`, `EmotionQuery`, `EmotionCommand`, `ErrorResponse`) +- GLSM-aware websocket lifecycle and stuck-listen recovery + +Main gap: + +- no first-class presence/identity perception extraction from runtime context for greeting policy decisions + +## Implementation Slices + +### Slice G1: Presence Context Extraction And Session Snapshot + +Goal: + +- extract presence/identity fields from websocket context payload into normalized metadata for routing + +Initial fields: + +- focused speaker id +- identified person ids present +- total people present +- trigger source if present +- time-of-day helper signals + +Notes: + +- no facial-recognition implementation is needed in cloud; cloud consumes robot perception signals + +### Slice G2: Greeting Intent Families And Parser Guardrails + +Goal: + +- add explicit greeting intent families with question/command guardrails + +Initial families: + +- `hello`, `hey jibo`, `what's up` +- `good morning`, `good afternoon`, `good evening`, `good night` +- `i'm home`, `i'm back` +- identity question (`who am i`) as a future-compatible hook + +Guardrails: + +- avoid stealing non-greeting domains +- keep existing date/time and birthday disambiguation intact + +### Slice G3: Greeting State-Machine Port (OpenJibo Style) + +Goal: + +- add a greeting state-machine module with explicit route metadata like chitchat + +Planned routes: + +- `ReactiveGreeting` +- `ProactiveGreeting` +- `MorningGreeting` +- `SpecialDayGreeting` +- `OptionalResponse` +- `ErrorResponse` + +Output shape: + +- keep stock-compatible skill payload patterns +- preserve MIM/ESML hook points for charm content + +### Slice G4: Proactive Gating And Cooldowns + +Goal: + +- port the critical Pegasus policy behavior to prevent spam + +Phase-1 rules: + +- skip proactive greetings when trigger source is surprise +- enforce per-tenant/person cooldown (target parity: 2-hour greeting window) +- suppress proactive launch when session is unstable (pending listen/follow-up conflict) + +### Slice G5: Person Queue And Memory Extensions + +Goal: + +- introduce lightweight person queue/history for greeting relevance + +Phase-1 storage additions: + +- last-seen timestamp per person key +- last-greeted timestamp per person key +- optional preferred-name alias for spoken greeting personalization + +### Slice G6: Rollout, Logging, And Live Validation + +Goal: + +- ship safely with observability and test confidence + +Required coverage: + +- unit tests for context extraction and intent routing +- websocket tests for presence-triggered greeting eligibility and cooldown behavior +- live captures validating: + - no stuck listening regressions + - no runaway proactive loops + - stable fallback when identity is unknown + +## Suggested Build Order + +1. G1 context extraction + diagnostics +2. G2 greeting parser families + guardrails +3. G3 greeting state machine (reactive first) +4. G4 proactive gating + cooldowns +5. G5 person queue memory extensions +6. G6 live validation and polish + +## Definition Of Done For This Track + +- presence-aware greeting behavior works with and without identified users +- proactive greeting frequency is policy-bounded and observable +- no regressions in existing `1.0.19` memory/weather/proactivity flows +- release docs and backlog are updated with shipped scope and next slice diff --git a/OpenJibo/docs/personal-report-parity-plan.md b/OpenJibo/docs/personal-report-parity-plan.md new file mode 100644 index 0000000..5d92577 --- /dev/null +++ b/OpenJibo/docs/personal-report-parity-plan.md @@ -0,0 +1,105 @@ +# Personal Report Parity Plan + +As-of: `2026-05-07` + +## Objective + +Bring OpenJibo personal report behavior closer to original Jibo charm while keeping cloud architecture modern and provider-agnostic. + +## Pegasus Findings (Source Anchors) + +- Weather personality and visuals were MIM-driven, not plain speech: + - `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\weather\WeatherMimLogic.ts` + - `C:\Projects\jibo\pegasus\packages\report-skill\mims\en-us\WeatherCommentRain.mim` + - `C:\Projects\jibo\pegasus\packages\report-skill\mims\en-us\WeatherTodayHighLow.mim` + - `C:\Projects\jibo\pegasus\packages\report-skill\resources\views\weatherHiLo.json` +- Weather icons were mapped to condition/time-of-day tokens (`clear-day`, `partly-cloudy-night`, etc.) and used in ``. +- Report-skill supported reactive entrypoints beyond full personal report: + - `requestWeatherPR`, `requestNews`, `requestCommute`, `requestCalendar` + - Source: `C:\Projects\jibo\pegasus\packages\hub\pegasus-skills\report_skill_manifest.json` +- Legacy data backends were Lasso-mediated: + - weather: Dark Sky + - commute: Google Maps directions/traffic + - news: AP News feeds + - calendar: Google/Outlook connectors +- Parser `main_agent` explicitly includes weather/news/personal-report intents; direct commute/calendar intents are not present in that same folder snapshot: + - `C:\Projects\jibo\pegasus\packages\parser\dialogflow\main_agent\intents` +- Grocery/list behavior found in Pegasus is scripted-response style, not a standalone list skill: + - `RA_JBO_ShoppingList.mim` and `RA_JBO_ManageToDoList.mim` are "not supported yet" style responses. + +## OpenJibo Current State + +- Personal report state machine exists and is test-backed. +- Weather provider integration exists (OpenWeather), including current and tomorrow. +- News and commute currently have baseline placeholder speech, not live provider-backed data orchestration. +- Calendar is currently reply-based and not yet provider-integrated. + +## Gap Summary + +1. Weather has factual speech but needs stronger visual/personality parity. +2. Non-local weather and broader date scopes need expansion beyond basic trailing `in ` and tomorrow handling. +3. Live news feed selection and filtering strategy is not yet implemented. +4. Commute data path and settings model are not yet mapped to an active provider integration. +5. Full personal report parity matrix (weather/commute/calendar/news behavior details) is not yet documented as a ship checklist. + +## Implementation Phases + +## Phase 1 (In Progress): Weather Personality Lift + +- Add weather-condition animation metadata and expressive weather MIM-style prompt metadata to cloud weather speech. +- Expand location phrase handling (`in/for/at`) and suffix stripping for common temporal tails. + +## Phase 2: Weather Visual Layer Parity + +- Add weather Hi/Lo view payload support (OpenJibo-side equivalent to `weatherHiLo.json` behavior). +- Carry mapped weather icon token + hi/lo values into outbound skill action config. +- Keep fallback behavior safe when view assets are unavailable. + +## Phase 3: Weather Scope Expansion + +- Add parser support for additional time requests (for example weekend/next-week phrasing). +- Extend weather request model to support short-range date windows. +- Decide whether range responses are summarized speech-only or include multi-card view behavior. + +## Phase 4: Live News Source + +- Introduce provider-backed headline ingestion with category toggles. +- Mirror core Pegasus constraints: + - de-duplicate headlines + - filter missing summaries/images + - child-safe filtering mode +- Preserve current speech fallback if provider is unavailable. + +## Phase 5: Commute Data Path + +- Implement commute provider abstraction and first provider integration. +- Recreate core commute decision logic: + - minutes-left + - normal vs delayed traffic commentary + - mode-aware phrasing (drive vs transit) +- Add settings contract for origin/destination/work-arrival/mode. + +## Phase 6: Personal Report Coverage Matrix + +- Build parity matrix across weather/news/commute/calendar: + - intent phrases + - required entities/settings + - provider dependencies + - expected MIM/view style outputs + - fallback behavior +- Attach tests and capture criteria for each row. + +## Phase 7 (Future Release): Grocery Lists + +- Track as a future release item (requested by users). +- Two candidate paths: + 1. Native lightweight list skill (fastest to ship). + 2. Integration-backed list orchestration (better long-term ecosystem fit). +- Recommendation: ship native MVP first, then add integration connectors. + +## Next Immediate Execution + +1. Validate weather personality-lift behavior in live runs. +2. Implement weather view payload support (Hi/Lo + condition icon). +3. Draft provider plan for live news source. +4. Draft commute provider interface + settings schema. diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index 7263019..1db9c6f 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -117,6 +117,33 @@ Reference: - [system-diagram-alignment.md](system-diagram-alignment.md) +## Greetings And Presence Planning Snapshot (`2026-05-07`) + +Pegasus greeting and presence behavior has now been captured into a source-anchored OpenJibo implementation plan. + +Reference: + +- [greetings-presence-plan.md](greetings-presence-plan.md) + +## Live Validation Snapshot (`2026-05-07`) + +User-confirmed end-to-end behavior now includes: + +- `Hey Jibo -> What's your cloud version?` (working) +- `Hey Jibo -> What's the time?` (working) +- `Hey Jibo -> Surprise me -> pizza fact -> $YESNO (Yes) -> fact` (working) +- `Hey Jibo -> Surprise me -> pizza fact -> $YESNO (No) -> decline reply` (working) + +This confirms the pizza-fact offer state now keeps the yes/no branch open through completion and does not require a second wake-word reset for the follow-up answer. + +## Personal Report Planning Snapshot (`2026-05-07`) + +Personal report parity planning is now captured with Pegasus source anchors for weather visuals/animations, live news, commute, and calendar gap coverage. + +Reference: + +- [personal-report-parity-plan.md](personal-report-parity-plan.md) + ## Next Queued Task (`2026-05-06`) Queued next `1.0.19` implementation task (now started): @@ -136,15 +163,30 @@ First completed guardrail slice under this queue: - GLSM listener flow capture + telemetry mapping - stale pending-listen recovery path for long-open no-context/no-audio listens +Second completed guardrail slice under this queue: + +- tightened date/time ambiguity handling (`what's your birthday`/`what's your bday` no longer falls into date intent) +- expanded Pegasus-inspired memory/weather phrase coverage: + - birthday alias parsing (`my bday is ...`, `when is my bday`) + - shorthand preference sets (`my favorite sport football`) + - weather variants (`what's today's weather look like`, `will it be sunny tomorrow`) +- listener continuation guardrail now differentiates incomplete preference fragments from complete shorthand preference sets + +Next queued implementation track after parser guardrails: + +- presence-aware greetings and identity-triggered proactivity (Pegasus `@be/greetings` parity slice) + ## Next Slices 1. Dialog parsing expansion (queued next as of `2026-05-06`; more phrase variants, ambiguity handling, and transcript-to-intent guardrails) -2. Holidays and seasonal personality slice beyond pizza day (time-scoped content backed by memory/proactivity path) -3. Durable memory persistence path (swap in provider-backed multi-tenant storage while preserving behavior contracts) -4. Update/backup/restore end-to-end proof (operator-run and documented) -5. STT noise-screening and short-utterance reliability pass -6. Provider-backed news expansion and deeper weather parity using Pegasus-backed contracts -7. Capture indexing and retention boundary for group testing +2. Presence-aware greetings and identity-triggered proactivity (reactive/proactive split, cooldowns, person-aware greeting hooks) +3. Personal report parity slices (weather visual layer, live news path, commute path, calendar parity matrix) +4. Holidays and seasonal personality slice beyond pizza day (time-scoped content backed by memory/proactivity path) +5. Durable memory persistence path (swap in provider-backed multi-tenant storage while preserving behavior contracts) +6. Update/backup/restore end-to-end proof (operator-run and documented) +7. STT noise-screening and short-utterance reliability pass +8. Provider-backed news expansion and deeper weather parity using Pegasus-backed contracts +9. Capture indexing and retention boundary for group testing For slices 1-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements. diff --git a/OpenJibo/docs/system-diagram-alignment.md b/OpenJibo/docs/system-diagram-alignment.md index 643c46c..582c36e 100644 --- a/OpenJibo/docs/system-diagram-alignment.md +++ b/OpenJibo/docs/system-diagram-alignment.md @@ -10,7 +10,7 @@ Use it to keep release planning grounded in three views: - where we are (current hosted `.NET` implementation) - where we are headed (next architecture slices) -As-of date: `2026-05-06` +As-of date: `2026-05-07` ## Diagram Inputs @@ -40,6 +40,7 @@ Conclusion: do not treat template-skill flow as a port target. Treat it as a sha | `Parser / Robust Parser` | rule-based intent resolution in [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) + focused state machines (personal report/chitchat) | deeper phrase import from Pegasus intents/entities plus ambiguity guardrails | | `Skill Router` | [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) decision switch and local skill payload shaping | external skill routing config and safer declarative intent mapping | | `Proactivity Selector` | weighted candidate selection in [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) + pending-offer session state in [WebSocketTurnFinalizationService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs) | externalized proactivity catalog, cooldown policy, and broader category coverage | +| `Presence / Identity Context` | runtime context passthrough in [ProtocolToTurnContextMapper.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs) and turn metadata handling in [WebSocketTurnFinalizationService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs) | normalize `runtime.perception` fields (`speaker`, `peoplePresent`, focused person) for greeting/proactivity policy decisions | | `Skill Registry` | implicit in current code/routing | formal registry abstraction for local/cloud capabilities and manifest metadata | | `History` | tenant-scoped memory store in [InMemoryPersonalMemoryStore.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryPersonalMemoryStore.cs) | durable multi-tenant persistence and history timeline/query support | | `Lasso` provider aggregation | partial provider integration via weather provider wiring in [ServiceCollectionExtensions.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs) | full aggregation service for weather/news/calendar/knowledge inputs | @@ -127,3 +128,24 @@ Tracking anchors: Primary objective: - import Pegasus parser intent phrases/entities to improve intent confidence while preserving command-vs-question personality behavior. + +## Greetings And Presence Track (`2026-05-07`) + +A dedicated presence-aware greetings plan is now captured for the next personality slice, grounded in Pegasus `@be/greetings` state, identity, and proactive policy behavior. + +Reference: + +- [greetings-presence-plan.md](greetings-presence-plan.md) + +## Personal Report Parity Track (`2026-05-07`) + +Personal report parity planning is now captured with a source-anchored implementation sequence for: + +- weather visual/personality parity +- live news provider path +- commute provider path +- calendar/report coverage matrix + +Reference: + +- [personal-report-parity-plan.md](personal-report-parity-plan.md) diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs index 011de76..91a449d 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs @@ -452,9 +452,12 @@ public sealed class JiboInteractionService( "I couldn't fetch the weather right now. Please try again."); } + var spokenReply = BuildWeatherSpokenReply(snapshot, dateEntity); + var weatherPayload = BuildWeatherSkillPayload(spokenReply, snapshot, TryResolveReferenceLocalTime(turn)); return new JiboInteractionDecision( "weather", - BuildWeatherSpokenReply(snapshot, dateEntity)); + spokenReply, + SkillPayload: weatherPayload); } private static string BuildWeatherSpokenReply( @@ -488,6 +491,114 @@ public sealed class JiboInteractionService( return $"Right now in {location}, it is {summary} and {snapshot.Temperature} degrees {unit}."; } + private static IDictionary BuildWeatherSkillPayload( + string spokenReply, + WeatherReportSnapshot snapshot, + DateTimeOffset? referenceLocalTime) + { + var weatherIcon = ResolveWeatherAnimationIcon(snapshot, referenceLocalTime); + var promptToken = ResolveWeatherPromptToken(weatherIcon); + + return new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["esml"] = + $"{EscapeForEsml(spokenReply)}", + ["mim_id"] = $"WeatherComment{promptToken}", + ["mim_type"] = "announcement", + ["prompt_id"] = $"WeatherComment{promptToken}_AN_13", + ["prompt_sub_category"] = "AN" + }; + } + + private static string ResolveWeatherAnimationIcon( + WeatherReportSnapshot snapshot, + DateTimeOffset? referenceLocalTime) + { + var isDaytime = (referenceLocalTime ?? DateTimeOffset.UtcNow).Hour is >= 6 and < 18; + var normalized = NormalizeCommandPhrase( + $"{snapshot.Condition ?? string.Empty} {snapshot.Summary ?? string.Empty}"); + + if (normalized.Contains("thunder", StringComparison.Ordinal) || + normalized.Contains("drizzle", StringComparison.Ordinal) || + normalized.Contains("rain", StringComparison.Ordinal)) + { + return "rain"; + } + + if (normalized.Contains("snow", StringComparison.Ordinal)) + { + return "snow"; + } + + if (normalized.Contains("sleet", StringComparison.Ordinal) || + normalized.Contains("freezing rain", StringComparison.Ordinal) || + normalized.Contains("ice", StringComparison.Ordinal)) + { + return "sleet"; + } + + if (normalized.Contains("fog", StringComparison.Ordinal) || + normalized.Contains("mist", StringComparison.Ordinal) || + normalized.Contains("haze", StringComparison.Ordinal) || + normalized.Contains("smoke", StringComparison.Ordinal)) + { + return "fog"; + } + + if (normalized.Contains("wind", StringComparison.Ordinal)) + { + return "wind"; + } + + if (normalized.Contains("partly cloudy", StringComparison.Ordinal) || + normalized.Contains("scattered clouds", StringComparison.Ordinal) || + normalized.Contains("few clouds", StringComparison.Ordinal)) + { + return isDaytime ? "partly-cloudy-day" : "partly-cloudy-night"; + } + + if (normalized.Contains("cloud", StringComparison.Ordinal) || + normalized.Contains("overcast", StringComparison.Ordinal)) + { + return "cloudy"; + } + + if (normalized.Contains("clear", StringComparison.Ordinal) || + normalized.Contains("sunny", StringComparison.Ordinal)) + { + return isDaytime ? "clear-day" : "clear-night"; + } + + return isDaytime ? "clear-day" : "clear-night"; + } + + private static string ResolveWeatherPromptToken(string weatherIcon) + { + return weatherIcon switch + { + "clear-day" => "ClearDay", + "clear-night" => "ClearNight", + "rain" => "Rain", + "snow" => "Snow", + "sleet" => "Sleet", + "fog" => "Fog", + "wind" => "Wind", + "cloudy" => "Cloudy", + "partly-cloudy-day" => "PartlyCloudyDay", + "partly-cloudy-night" => "PartlyCloudyNight", + _ => "Cloudy" + }; + } + + private static string EscapeForEsml(string value) + { + return value + .Replace("&", "&", StringComparison.Ordinal) + .Replace("<", "<", StringComparison.Ordinal) + .Replace(">", ">", StringComparison.Ordinal) + .Replace("\"", """, StringComparison.Ordinal); + } + private static JiboInteractionDecision BuildOrderPizzaDecision() { return new JiboInteractionDecision( @@ -1151,8 +1262,7 @@ public sealed class JiboInteractionService( return "no"; } - if (MatchesAny(loweredTranscript, "what time is it", "current time", "the time", "time is it") || - loweredTranscript.Contains("time", StringComparison.Ordinal)) + if (IsTimeRequest(loweredTranscript)) { return "time"; } @@ -1162,9 +1272,7 @@ public sealed class JiboInteractionService( return "day"; } - if (MatchesAny(loweredTranscript, "what day is it", "what is the date", "today s date", "today's date") || - loweredTranscript.Contains("date", StringComparison.Ordinal) || - loweredTranscript.Contains("day", StringComparison.Ordinal)) + if (IsDateRequest(loweredTranscript)) { return "date"; } @@ -1630,6 +1738,44 @@ public sealed class JiboInteractionService( MatchesAny(normalized, "no thank you", "maybe later"); } + private static bool IsTimeRequest(string loweredTranscript) + { + var normalized = NormalizeCommandPhrase(loweredTranscript); + if (string.IsNullOrWhiteSpace(normalized)) + { + return false; + } + + if (normalized is "time" or "the time" or "current time" or "what time is it" or "what s the time" or "what is the time") + { + return true; + } + + return normalized.StartsWith("what time", StringComparison.Ordinal) || + normalized.StartsWith("tell me the time", StringComparison.Ordinal) || + normalized.StartsWith("show me the time", StringComparison.Ordinal); + } + + private static bool IsDateRequest(string loweredTranscript) + { + var normalized = NormalizeCommandPhrase(loweredTranscript); + if (string.IsNullOrWhiteSpace(normalized)) + { + return false; + } + + return normalized is + "what is the date" or + "what s the date" or + "what date is it" or + "today s date" or + "today date" or + "what is today s date" or + "what s today s date" or + "what is todays date" or + "what s todays date"; + } + private static bool IsWeatherRequest(string loweredTranscript) { if (MatchesAny( @@ -1652,12 +1798,22 @@ public sealed class JiboInteractionService( "what is today s humidity", "what is today's humidity", "what's the humidity", - "what is the humidity")) + "what is the humidity", + "what's today's forecast", + "what s today's forecast", + "what s today s forecast", + "what is today s forecast", + "what is today's forecast", + "what's today's weather look like", + "what s today's weather look like", + "what s today s weather look like", + "what is today s weather look like", + "what is today's weather look like")) { return true; } - return MatchesAny( + if (MatchesAny( loweredTranscript, "will it rain", "will it snow", @@ -1669,7 +1825,12 @@ public sealed class JiboInteractionService( "is it going to rain", "is it going to snow", "do you think it will rain", - "do you think it will snow"); + "do you think it will snow")) + { + return true; + } + + return WeatherConditionForecastPattern.IsMatch(loweredTranscript); } private static string? TryResolveWeatherLocationQuery(string transcript) @@ -1830,6 +1991,10 @@ public sealed class JiboInteractionService( "when s your birthday", "what s your birthday", "what is your birthday", + "when is your bday", + "when s your bday", + "what s your bday", + "what is your bday", "when were you born", "what day is your birthday")) { @@ -1837,6 +2002,7 @@ public sealed class JiboInteractionService( } return (normalized.Contains("your birthday", StringComparison.Ordinal) || + normalized.Contains("your bday", StringComparison.Ordinal) || normalized.Contains("your birth date", StringComparison.Ordinal)) && !normalized.Contains("my birthday", StringComparison.Ordinal); } @@ -1889,6 +2055,11 @@ public sealed class JiboInteractionService( "what is my birthday", "what s my birthday", "what's my birthday", + "when is my bday", + "when s my bday", + "what is my bday", + "what s my bday", + "what's my bday", "do you remember my birthday"); } @@ -1900,13 +2071,15 @@ public sealed class JiboInteractionService( private static bool IsUserBirthdaySetAttempt(string loweredTranscript) { var normalized = NormalizeCommandPhrase(loweredTranscript); - return normalized.Contains("my birthday is", StringComparison.Ordinal); + return normalized.Contains("my birthday is", StringComparison.Ordinal) || + normalized.Contains("my bday is", StringComparison.Ordinal); } private static bool IsUserBirthdayRecallAttempt(string loweredTranscript) { var normalized = NormalizeCommandPhrase(loweredTranscript); - return normalized.Contains("my birthday", StringComparison.Ordinal) && + return (normalized.Contains("my birthday", StringComparison.Ordinal) || + normalized.Contains("my bday", StringComparison.Ordinal)) && (normalized.StartsWith("when", StringComparison.Ordinal) || normalized.StartsWith("what", StringComparison.Ordinal) || normalized.StartsWith("tell me", StringComparison.Ordinal) || @@ -1916,15 +2089,28 @@ public sealed class JiboInteractionService( private static string? TryExtractBirthdayFact(string transcript) { var normalized = NormalizeCommandPhrase(transcript); - var marker = "my birthday is "; - var markerIndex = normalized.IndexOf(marker, StringComparison.Ordinal); - if (markerIndex < 0) + var markers = new[] { - return null; + "my birthday is ", + "my bday is " + }; + + foreach (var marker in markers) + { + var markerIndex = normalized.IndexOf(marker, StringComparison.Ordinal); + if (markerIndex < 0) + { + continue; + } + + var value = normalized[(markerIndex + marker.Length)..].Trim(); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } } - var value = normalized[(markerIndex + marker.Length)..].Trim(); - return string.IsNullOrWhiteSpace(value) ? null : value; + return null; } private static bool IsPreferenceRecallQuestion(string loweredTranscript) @@ -2006,6 +2192,12 @@ public sealed class JiboInteractionService( var splitIndex = preferencePhrase.IndexOf(splitMarker, StringComparison.Ordinal); if (splitIndex <= 0 || splitIndex >= preferencePhrase.Length - splitMarker.Length) { + var fallbackPreference = TryExtractPreferenceSetWithoutCopula(preferencePhrase); + if (fallbackPreference is not null) + { + return fallbackPreference; + } + continue; } @@ -2042,6 +2234,38 @@ public sealed class JiboInteractionService( return null; } + private static (string Category, string Value)? TryExtractPreferenceSetWithoutCopula(string preferencePhrase) + { + if (string.IsNullOrWhiteSpace(preferencePhrase)) + { + return null; + } + + var normalized = preferencePhrase.Trim(); + if (normalized.Contains(" is ", StringComparison.Ordinal) || + normalized.Contains(" are ", StringComparison.Ordinal) || + normalized.EndsWith(" is", StringComparison.Ordinal) || + normalized.EndsWith(" are", StringComparison.Ordinal)) + { + return null; + } + + var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length < 2) + { + return null; + } + + var category = parts[0]; + var value = string.Join(' ', parts.Skip(1)).Trim(); + if (string.IsNullOrWhiteSpace(category) || string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return (category, value); + } + private static bool IsImportantDateSetStatement(string loweredTranscript) { return TryExtractImportantDateSet(loweredTranscript) is not null; @@ -2792,11 +3016,15 @@ public sealed class JiboInteractionService( RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); private static readonly Regex WeatherLocationPattern = new( - @"\bin\s+(?[a-z][a-z\s'\-]+)$", + @"\b(?:in|for|at)\s+(?[a-z][a-z\s'\-]+)$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); private static readonly Regex WeatherLocationSuffixPattern = new( - @"\b(?:today|tonight|tomorrow|outside|right now|please|thanks)\b", + @"\b(?:today|tonight|tomorrow|outside|right now|please|thanks|this weekend|next weekend|the weekend|weekend|this week|next week|on monday|on tuesday|on wednesday|on thursday|on friday|on saturday|on sunday|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex WeatherConditionForecastPattern = new( + @"\bwill it be\s+(sunny|cloudy|windy|foggy|stormy|rainy|snowy|hail|hailing)\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); private static readonly PizzaMimPrompt[] PizzaMimPrompts = diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs index ff20b57..af2b9b2 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs @@ -1556,9 +1556,15 @@ public sealed partial class WebSocketTurnFinalizationService( if (normalized.StartsWith("my favorite ", StringComparison.Ordinal) || normalized.StartsWith("my favourite ", StringComparison.Ordinal)) { + var preferenceTail = normalized.StartsWith("my favourite ", StringComparison.Ordinal) + ? normalized["my favourite ".Length..].Trim() + : normalized["my favorite ".Length..].Trim(); + var missingCopula = !normalized.Contains(" is ", StringComparison.Ordinal) && + !normalized.Contains(" are ", StringComparison.Ordinal); + if (normalized.EndsWith(" is", StringComparison.Ordinal) || normalized.EndsWith(" are", StringComparison.Ordinal) || - !normalized.Contains(" is ", StringComparison.Ordinal)) + (missingCopula && !LooksLikeBarePreferenceSet(preferenceTail))) { reason = "preference_set_incomplete"; return true; @@ -1591,6 +1597,17 @@ public sealed partial class WebSocketTurnFinalizationService( return PegasusAffinityContinuationStems.Contains(normalized); } + private static bool LooksLikeBarePreferenceSet(string preferenceTail) + { + if (string.IsNullOrWhiteSpace(preferenceTail)) + { + return false; + } + + var tokens = preferenceTail.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return tokens.Length >= 2; + } + private static void ClearListenTracking(WebSocketTurnState turnState) { turnState.SawListen = false; diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index 2f34940..00304c0 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -135,6 +135,21 @@ public sealed class JiboInteractionServiceTests Assert.Equal("My birthday is March 22, 2026.", decision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_WhatsYourBday_DoesNotFallThroughToDateIntent() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what's your bday", + NormalizedTranscript = "what's your bday" + }); + + Assert.Equal("robot_birthday", decision.IntentName); + Assert.Equal("My birthday is March 22, 2026.", decision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_DoYouHaveAPersonality_UsesCatalogBackedPersonalityReply() { @@ -338,6 +353,43 @@ public sealed class JiboInteractionServiceTests Assert.Equal("I can remember it if you say, my birthday is March 14.", decision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_BirthdayMemory_BdayAliasSetThenRecallWithinTenant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "my bday is April 12", + NormalizedTranscript = "my bday is April 12", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_birthday", setDecision.IntentName); + Assert.Equal("Got it. I will remember your birthday is april 12.", setDecision.ReplyText); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "when is my bday", + NormalizedTranscript = "when is my bday", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_birthday", recallDecision.IntentName); + Assert.Equal("You told me your birthday is april 12.", recallDecision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_PreferenceMemory_SetThenRecallWithinTenant() { @@ -375,6 +427,43 @@ public sealed class JiboInteractionServiceTests Assert.Equal("You told me your favorite music is jazz.", recallDecision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_PreferenceMemory_BareFavoriteSetThenRecallWithinTenant() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + + var setDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "my favorite sport football", + NormalizedTranscript = "my favorite sport football", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_set_preference", setDecision.IntentName); + Assert.Equal("Got it. I will remember your favorite sport is football.", setDecision.ReplyText); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what is my favorite sport", + NormalizedTranscript = "what is my favorite sport", + Attributes = new Dictionary + { + ["accountId"] = "acct-a", + ["loopId"] = "loop-a" + }, + DeviceId = "device-a" + }); + + Assert.Equal("memory_get_preference", recallDecision.IntentName); + Assert.Equal("You told me your favorite sport is football.", recallDecision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_PreferenceSetAttemptWithoutValue_RoutesToPreferencePrompt() { @@ -1027,6 +1116,36 @@ public sealed class JiboInteractionServiceTests Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_WeatherTodaysForecastQuery_WithoutProvider_StillReturnsFallback() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what's today's weather look like", + NormalizedTranscript = "what's today's weather look like" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_WeatherConditionForecastQuery_WithoutProvider_StillReturnsFallback() + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "will it be sunny tomorrow", + NormalizedTranscript = "will it be sunny tomorrow" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_ClientNluRequestWeatherPR_WithoutProvider_StillReturnsFallback() { @@ -1063,7 +1182,10 @@ public sealed class JiboInteractionServiceTests Assert.Equal("weather", decision.IntentName); Assert.Null(decision.SkillName); - Assert.Null(decision.SkillPayload); + Assert.NotNull(decision.SkillPayload); + Assert.Contains("cat='weather'", decision.SkillPayload!["esml"]?.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.Contains("meta='rain'", decision.SkillPayload["esml"]?.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.Equal("WeatherCommentRain", decision.SkillPayload["mim_id"]); Assert.Equal("Right now in Boston, US, it is light rain and 61 degrees Fahrenheit.", decision.ReplyText); Assert.NotNull(provider.LastRequest); Assert.False(provider.LastRequest!.IsTomorrow); @@ -1090,6 +1212,48 @@ public sealed class JiboInteractionServiceTests Assert.Equal("Tomorrow in Chicago, US, expect mostly cloudy with a high near 74 degrees Fahrenheit and a low around 60 degrees Fahrenheit.", decision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_WeatherLocationForToday_WithProvider_PassesLocation() + { + var provider = new CapturingWeatherReportProvider + { + Snapshot = new WeatherReportSnapshot("Seattle, US", "light rain", 58, 61, 52, "rain", false) + }; + var service = CreateService(weatherReportProvider: provider); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what's the weather for seattle today", + NormalizedTranscript = "what's the weather for seattle today" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("Seattle", provider.LastRequest?.LocationQuery); + Assert.False(provider.LastRequest?.IsTomorrow); + Assert.Equal("Right now in Seattle, US, it is light rain and 58 degrees Fahrenheit.", decision.ReplyText); + } + + [Fact] + public async Task BuildDecisionAsync_WeatherLocationWithWeekendSuffix_WithProvider_PassesLocation() + { + var provider = new CapturingWeatherReportProvider + { + Snapshot = new WeatherReportSnapshot("Paris, FR", "overcast clouds", 66, 70, 60, "cloudy", false) + }; + var service = CreateService(weatherReportProvider: provider); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what's the weather in paris this weekend", + NormalizedTranscript = "what's the weather in paris this weekend" + }); + + Assert.Equal("weather", decision.IntentName); + Assert.Equal("Paris", provider.LastRequest?.LocationQuery); + Assert.False(provider.LastRequest?.IsTomorrow); + Assert.Equal("Right now in Paris, FR, it is overcast clouds and 66 degrees Fahrenheit.", decision.ReplyText); + } + [Fact] public async Task BuildDecisionAsync_ClientNluAskForDate_MapsToDateIntent() { diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 5f30c6b..fdfac6d 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -363,6 +363,64 @@ public sealed class JiboWebSocketServiceTests Assert.Equal("my favorite sport is football", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); } + [Fact] + public async Task BufferedAudio_WithBarePreferenceSetHint_FinalizesWithoutDeferral() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-preference-bare-token", + Text = """{"type":"LISTEN","transID":"trans-preference-bare","data":{"rules":["launch"]}}""" + }); + + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-preference-bare-token", + Text = """{"type":"CONTEXT","transID":"trans-preference-bare","data":{"audioTranscriptHint":"my favorite sport football"}}""" + }); + + for (var index = 0; index < 4; index += 1) + { + var chunkReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-preference-bare-token", + Binary = new byte[3000] + }); + + Assert.Empty(chunkReplies); + } + + var session = _store.FindSessionByToken("hub-preference-bare-token"); + Assert.NotNull(session); + session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); + + var finalizedReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-preference-bare-token", + Binary = new byte[3000] + }); + + Assert.Equal(3, finalizedReplies.Count); + Assert.Equal("LISTEN", ReadReplyType(finalizedReplies[0])); + Assert.Equal("EOS", ReadReplyType(finalizedReplies[1])); + Assert.Equal("SKILL_ACTION", ReadReplyType(finalizedReplies[2])); + + using var listenPayload = JsonDocument.Parse(finalizedReplies[0].Text!); + Assert.Equal("memory_set_preference", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + Assert.Equal("my favorite sport football", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); + } + [Fact] public async Task BufferedAudio_WithIncompleteAffinityHint_DefersThenFinalizesWhenContinuationArrives() { @@ -3307,6 +3365,66 @@ public sealed class JiboWebSocketServiceTests Assert.False(session.Metadata.ContainsKey("pendingProactivityOffer")); } + [Fact] + public async Task ClientAsrSurpriseOffer_PersistsPendingOfferAndResolvesNoFollowUp() + { + var token = _store.IssueRobotToken("proactivity-device-b"); + + var offerReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"CLIENT_ASR","transID":"trans-proactive-offer-no","data":{"text":"surprise me"}}""" + }); + + Assert.Equal(3, offerReplies.Count); + using (var offerListenPayload = JsonDocument.Parse(offerReplies[0].Text!)) + { + Assert.Equal("proactive_offer_pizza_fact", offerListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + Assert.Equal("shared/yes_no", offerListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules")[0].GetString()); + } + + var session = _store.FindSessionByToken(token); + Assert.NotNull(session); + Assert.True(session.Metadata.TryGetValue("pendingProactivityOffer", out var pendingOffer)); + Assert.Equal("pizza_fact", pendingOffer?.ToString()); + + var followUpReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = token, + Text = """{"type":"CLIENT_ASR","transID":"trans-proactive-offer-no-followup","data":{"text":"no"}}""" + }); + + Assert.Equal(3, followUpReplies.Count); + using (var followUpListenPayload = JsonDocument.Parse(followUpReplies[0].Text!)) + { + Assert.Equal("proactive_offer_declined", followUpListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); + } + + using (var followUpSkillPayload = JsonDocument.Parse(followUpReplies[2].Text!)) + { + var esml = followUpSkillPayload.RootElement + .GetProperty("data") + .GetProperty("action") + .GetProperty("config") + .GetProperty("jcp") + .GetProperty("config") + .GetProperty("play") + .GetProperty("esml") + .GetString(); + Assert.Contains("No problem", esml, StringComparison.OrdinalIgnoreCase); + } + + session = _store.FindSessionByToken(token); + Assert.NotNull(session); + Assert.False(session.Metadata.ContainsKey("pendingProactivityOffer")); + } + [Fact] public async Task ClientAsrPersonalReport_StateMachinePersistsAcrossTurns() {