Compare commits
5 Commits
69707f32a7
...
ffb444e4f9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ffb444e4f9 | ||
|
|
7fd732ad17 | ||
|
|
3ad4a3e025 | ||
|
|
92491adf85 | ||
|
|
3e50fb9a49 |
@@ -88,6 +88,11 @@ Current websocket scope:
|
|||||||
- active local prompt preservation so `shared/yes_no`, clock, gallery, and settings prompts can still consume transcript-bearing short replies even when the stock skill reports a local context
|
- active local prompt preservation so `shared/yes_no`, clock, gallery, and settings prompts can still consume transcript-bearing short replies even when the stock skill reports a local context
|
||||||
- binary audio ignored for an existing transID until a fresh `LISTEN` has been seen, preventing context-only or post-speech tails from reopening an endless buffered turn
|
- binary audio ignored for an existing transID until a fresh `LISTEN` has been seen, preventing context-only or post-speech tails from reopening an endless buffered turn
|
||||||
- blank-audio hotphrase turns clear pending listen state and install a short late-audio ignore window
|
- blank-audio hotphrase turns clear pending listen state and install a short late-audio ignore window
|
||||||
|
- first GLSM-aligned listener telemetry and recovery slice is now in source:
|
||||||
|
- derived phase labels (`HJ_LISTENING`, `LISTENING`, `WAIT_LISTEN_FINISHED`, `DISPATCH_DIALOG`, `PROCESS_LISTENER_QUEUE`)
|
||||||
|
- `glsm_phase_transition` turn diagnostics
|
||||||
|
- websocket turn events with `glsmPhase` snapshots
|
||||||
|
- stale pending-listen recovery for long-open no-context/no-audio listens before processing a new hotphrase listen
|
||||||
- unknown inbound websocket types dropped silently instead of echoing stock-OS-unknown OpenJibo events
|
- unknown inbound websocket types dropped silently instead of echoing stock-OS-unknown OpenJibo events
|
||||||
- file telemetry and fixture export for HTTP, websocket, and turn captures
|
- file telemetry and fixture export for HTTP, websocket, and turn captures
|
||||||
|
|
||||||
@@ -145,6 +150,7 @@ Use these sources as evidence, not as code to copy blindly:
|
|||||||
- User-provided original source snapshot: `..\jibo` when extracted locally
|
- User-provided original source snapshot: `..\jibo` when extracted locally
|
||||||
- Original Pegasus cloud source inside that snapshot: `pegasus`
|
- Original Pegasus cloud source inside that snapshot: `pegasus`
|
||||||
- Original SDK and skill source inside that snapshot: `sdk`
|
- Original SDK and skill source inside that snapshot: `sdk`
|
||||||
|
- Legacy listener flow reference diagram: `..\jibo\sdk\packages\skills-service-manager\resources\state-diagrams\glsm.png`
|
||||||
- JiboOS reference tree: `..\JiboOS`
|
- JiboOS reference tree: `..\JiboOS`
|
||||||
- JiboOS skill snapshot: `..\JiboOS\opt\jibo\Jibo\Skills\@be`
|
- JiboOS skill snapshot: `..\JiboOS\opt\jibo\Jibo\Skills\@be`
|
||||||
|
|
||||||
|
|||||||
@@ -301,6 +301,20 @@ Current release theme:
|
|||||||
- Follow-up:
|
- Follow-up:
|
||||||
- live smoke should confirm `cloud version` speaks `1.0.18`, carries `match.skipSurprises = true`, does not stop itself on the word `Jibo`, and settles without a generic `I heard...` reply or a local surprise handoff
|
- live smoke should confirm `cloud version` speaks `1.0.18`, carries `match.skipSurprises = true`, does not stop itself on the word `Jibo`, and settles without a generic `I heard...` reply or a local surprise handoff
|
||||||
|
|
||||||
|
### GLSM Listener Flow Capture And Recovery
|
||||||
|
|
||||||
|
- Status: `implemented`
|
||||||
|
- Tags: `protocol`, `docs`
|
||||||
|
- Result:
|
||||||
|
- the legacy listener state machine source (`sdk ... glsm.png`) is now captured in current planning docs
|
||||||
|
- runtime now emits GLSM-aligned phase snapshots (`HJ_LISTENING`, `LISTENING`, `WAIT_LISTEN_FINISHED`, `DISPATCH_DIALOG`, `PROCESS_LISTENER_QUEUE`)
|
||||||
|
- turn diagnostics now include `glsm_phase_transition` for phase changes
|
||||||
|
- websocket telemetry now records `glsmPhase` on binary/context/turn events
|
||||||
|
- stale pending-listen recovery is now in source so a long-open no-context/no-audio listen can be cleared when the next hotphrase listen arrives
|
||||||
|
- Follow-up:
|
||||||
|
- live-capture proof is still required against the recurring blue-ring/stuck-listening sequence
|
||||||
|
- deeper GLSM parity (`Interrupt Listeners`, launch/global parse branches) should be tackled after this first capture slice is validated on-device
|
||||||
|
|
||||||
### End-Of-Skill Surprise Suppression
|
### End-Of-Skill Surprise Suppression
|
||||||
|
|
||||||
- Status: `implemented`
|
- Status: `implemented`
|
||||||
@@ -462,7 +476,7 @@ Current release theme:
|
|||||||
|
|
||||||
### Next Up (`2026-05-06`): Dialog Parsing Expansion And Ambiguity Guardrails
|
### Next Up (`2026-05-06`): Dialog Parsing Expansion And Ambiguity Guardrails
|
||||||
|
|
||||||
- Status: `ready`
|
- Status: `polish`
|
||||||
- Tags: `protocol`, `content`, `stt`, `docs`
|
- Tags: `protocol`, `content`, `stt`, `docs`
|
||||||
- Why now:
|
- Why now:
|
||||||
- this is the next queued `1.0.19` implementation slice after weather provider bring-up
|
- this is the next queued `1.0.19` implementation slice after weather provider bring-up
|
||||||
@@ -473,6 +487,13 @@ Current release theme:
|
|||||||
- add ambiguity guardrails for overlapping intents (date vs birthday, generic chat vs memory set/lookup, weather variants)
|
- add ambiguity guardrails for overlapping intents (date vs birthday, generic chat vs memory set/lookup, weather variants)
|
||||||
- preserve command-vs-question personality behavior and stock skill launch compatibility
|
- preserve command-vs-question personality behavior and stock skill launch compatibility
|
||||||
- add focused tests for new phrase families and negative boundary cases
|
- add focused tests for new phrase families and negative boundary cases
|
||||||
|
- Progress update (`2026-05-07`):
|
||||||
|
- implemented date/time guardrails so birthday phrasing is not misrouted to date
|
||||||
|
- expanded phrase coverage for:
|
||||||
|
- birthday alias set/recall (`bday` variants)
|
||||||
|
- shorthand favorites (`my favorite sport football`)
|
||||||
|
- weather phrasing (`what's today's weather look like`, `will it be sunny tomorrow`)
|
||||||
|
- updated continuation deferral so complete shorthand favorites finalize instead of waiting for missing continuation
|
||||||
- Exit criteria:
|
- Exit criteria:
|
||||||
- ambiguous phrase handling is improved without regressions in existing `1.0.19` features
|
- ambiguous phrase handling is improved without regressions in existing `1.0.19` features
|
||||||
- phrase imports are documented and traceable to Pegasus parser sources
|
- phrase imports are documented and traceable to Pegasus parser sources
|
||||||
@@ -664,6 +685,76 @@ Current release theme:
|
|||||||
- connect weather units and location directly to user/report-skill settings parity instead of config defaults
|
- connect weather units and location directly to user/report-skill settings parity instead of config defaults
|
||||||
- add richer condition-change commentary and view parity with original report-skill weather behaviors
|
- add richer condition-change commentary and view parity with original report-skill weather behaviors
|
||||||
|
|
||||||
|
### 26. Presence-Aware Greetings And Identity Proactivity
|
||||||
|
|
||||||
|
- Status: `ready`
|
||||||
|
- Tags: `protocol`, `content`, `storage`, `docs`
|
||||||
|
- Why now:
|
||||||
|
- this is the next personality-charm expansion after parser guardrail and weather bring-up
|
||||||
|
- Pegasus greetings behavior is strongly tied to presence/identity signals and proactive cooldown policy
|
||||||
|
- current OpenJibo has memory/proactivity foundations but no first-class presence extraction path yet
|
||||||
|
- Pegasus source anchors:
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\hub\be-skills\greetings_manifest.json`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSkill.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSM.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\hub\src\proactive\ProactiveTransactionHandler.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\hub\src\proactive\tools\ContextTools.ts`
|
||||||
|
- Scope:
|
||||||
|
- extract presence/identity context (`speaker`, `peoplePresent`, focused person) from runtime context payload
|
||||||
|
- add greeting intent families and state-machine split for reactive vs proactive greeting routes
|
||||||
|
- add cooldown and trigger-source guardrails for proactive greetings
|
||||||
|
- start person-aware greeting hooks (name-aware greeting, morning greeting policy, return greeting policy)
|
||||||
|
- Exit criteria:
|
||||||
|
- presence-aware greetings are routed deterministically with tests
|
||||||
|
- proactive greetings are frequency-bounded and do not trigger from surprise source when blocked by policy
|
||||||
|
- fallback behavior remains stable when identity is unknown or context is incomplete
|
||||||
|
- docs and release tracking are updated with shipped scope and residual gaps
|
||||||
|
- Tracking:
|
||||||
|
- [greetings-presence-plan.md](greetings-presence-plan.md)
|
||||||
|
- [release-1.0.19-plan.md](release-1.0.19-plan.md)
|
||||||
|
|
||||||
|
### 27. Personal Report Parity Track (Weather/News/Commute/Calendar)
|
||||||
|
|
||||||
|
- Status: `ready`
|
||||||
|
- Tags: `protocol`, `content`, `storage`, `docs`
|
||||||
|
- Why now:
|
||||||
|
- personal report is a core Jibo charm surface and currently split between implemented weather speech and placeholder calendar/commute/news content
|
||||||
|
- Pegasus weather used explicit condition animations and weather views; current OpenJibo weather is functional but visually lighter
|
||||||
|
- Scope:
|
||||||
|
- weather icon/animation parity and view support
|
||||||
|
- broader non-local weather query handling and short-range date coverage
|
||||||
|
- provider-backed news ingestion and filtering
|
||||||
|
- commute provider path and settings schema
|
||||||
|
- coverage matrix for personal report parity gaps and test/capture exit criteria
|
||||||
|
- Source anchors:
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\weather\WeatherMimLogic.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\resources\views\weatherHiLo.json`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\news\NewsMimLogic.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\commute\CommuteMimLogic.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\hub\pegasus-skills\report_skill_manifest.json`
|
||||||
|
- Tracking:
|
||||||
|
- [personal-report-parity-plan.md](personal-report-parity-plan.md)
|
||||||
|
- [release-1.0.19-plan.md](release-1.0.19-plan.md)
|
||||||
|
|
||||||
|
### 28. Grocery List Capability (Requested Feature)
|
||||||
|
|
||||||
|
- Status: `discovery`
|
||||||
|
- Tags: `content`, `docs`, `storage`
|
||||||
|
- Why now:
|
||||||
|
- directly requested by Jibo owners and fits memory + household utility roadmap
|
||||||
|
- Source findings:
|
||||||
|
- Pegasus has scripted responses for shopping/to-do list requests but no standalone grocery-list skill in this snapshot
|
||||||
|
- examples:
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ShoppingList.mim`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ManageToDoList.mim`
|
||||||
|
- Candidate delivery paths:
|
||||||
|
- native lightweight list skill (fastest user value)
|
||||||
|
- integration-backed list orchestration (long-term richer ecosystem fit)
|
||||||
|
- Exit criteria:
|
||||||
|
- clear decision on MVP path
|
||||||
|
- first schema for list items + ownership scope
|
||||||
|
- initial voice flows and follow-up intent handling defined
|
||||||
|
|
||||||
## Suggested Order
|
## Suggested Order
|
||||||
|
|
||||||
Before closing `1.0.18`:
|
Before closing `1.0.18`:
|
||||||
@@ -682,15 +773,18 @@ For `1.0.19`:
|
|||||||
2. Expand memory-backed personal facts with tenant-scoped storage (beyond the first birthday/preferences foundation) - implemented
|
2. Expand memory-backed personal facts with tenant-scoped storage (beyond the first birthday/preferences foundation) - implemented
|
||||||
3. Proactivity selector baseline with source-backed first offers - implemented
|
3. Proactivity selector baseline with source-backed first offers - implemented
|
||||||
4. Weather report-skill launch compatibility - implemented
|
4. Weather report-skill launch compatibility - implemented
|
||||||
5. Dialog parsing expansion and ambiguity guardrails - queued next as of `2026-05-06`
|
5. Dialog parsing expansion and ambiguity guardrails - in progress (`2026-05-07` first guardrail slice implemented)
|
||||||
6. Holidays and seasonal personality behavior built on the new memory/proactivity foundation
|
6. Presence-aware greetings and identity-triggered proactivity - ready
|
||||||
7. Durable memory persistence path (multi-tenant backing store)
|
7. Personal report parity track (weather visuals, live news path, commute path, calendar parity matrix) - ready
|
||||||
8. Update, backup, and restore proof
|
8. Holidays and seasonal personality behavior built on the new memory/proactivity foundation
|
||||||
9. STT upgrade and noise screening
|
9. Durable memory persistence path (multi-tenant backing store)
|
||||||
10. Hosted capture/storage plan / indexing for group testing
|
10. Update, backup, and restore proof
|
||||||
11. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc.
|
11. STT upgrade and noise screening
|
||||||
12. Provider-backed news and weather parity polish
|
12. Hosted capture/storage plan / indexing for group testing
|
||||||
13. Lasso, identity, and onboarding as larger discovery-driven tracks
|
13. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc.
|
||||||
|
14. Provider-backed news and weather parity polish
|
||||||
|
15. Grocery list capability discovery and MVP selection
|
||||||
|
16. Lasso, identity, and onboarding as larger discovery-driven tracks
|
||||||
|
|
||||||
For `1.0.20` and beyond:
|
For `1.0.20` and beyond:
|
||||||
|
|
||||||
|
|||||||
173
OpenJibo/docs/greetings-presence-plan.md
Normal file
173
OpenJibo/docs/greetings-presence-plan.md
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
# Greetings And Presence Plan (`1.0.19`)
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Recreate the original Jibo greeting charm with modern cloud architecture:
|
||||||
|
|
||||||
|
- person-aware greetings when someone is detected
|
||||||
|
- proactive offers tied to presence, time of day, and memory
|
||||||
|
- safe cooldown rules so proactivity feels alive, not noisy
|
||||||
|
|
||||||
|
This plan is source-anchored to Pegasus and scoped to shippable slices.
|
||||||
|
|
||||||
|
## Pegasus Behavior Baseline
|
||||||
|
|
||||||
|
Primary source artifacts:
|
||||||
|
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\hub\be-skills\greetings_manifest.json`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSkill.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\GreetingsSM.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\states\IntentSplit.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\states\ProactiveGreetingState.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\states\ProactiveProbabilityState.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\states\ShouldDoMorningGreetingState.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\states\ShouldDoBirthdayState.ts`
|
||||||
|
- `C:\Projects\jibo\sdk\skills\greetings\src\states\ShouldDoHolidayState.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\hub\src\proactive\ProactiveTransactionHandler.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\hub\src\proactive\tools\ContextTools.ts`
|
||||||
|
|
||||||
|
Key behaviors to port:
|
||||||
|
|
||||||
|
- explicit reactive/proactive greeting split
|
||||||
|
- identity source split:
|
||||||
|
- reactive path uses active speaker
|
||||||
|
- proactive path uses present identified persons
|
||||||
|
- hub-level proactive gating:
|
||||||
|
- block greetings when trigger source is `SURPRISE`
|
||||||
|
- throttle by interaction history (`GreetingsLaunchLast2Hours < 1`)
|
||||||
|
- morning/birthday/holiday gates with per-user recency checks
|
||||||
|
- optional follow-up response flow after proactive greetings
|
||||||
|
|
||||||
|
## Current OpenJibo Baseline
|
||||||
|
|
||||||
|
Current implementation anchor:
|
||||||
|
|
||||||
|
- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\JiboInteractionService.cs`
|
||||||
|
- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\ProtocolToTurnContextMapper.cs`
|
||||||
|
- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\WebSocketTurnFinalizationService.cs`
|
||||||
|
- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Services\ChitchatStateMachine.cs`
|
||||||
|
- `C:\Projects\JiboExperiments\OpenJibo\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Infrastructure\Persistence\InMemoryPersonalMemoryStore.cs`
|
||||||
|
|
||||||
|
What we already have:
|
||||||
|
|
||||||
|
- tenant-scoped memory primitives (name, birthday, preferences, affinity)
|
||||||
|
- proactivity baseline with pending-offer follow-up handling
|
||||||
|
- state-machine style chitchat split (`ScriptedResponse`, `EmotionQuery`, `EmotionCommand`, `ErrorResponse`)
|
||||||
|
- GLSM-aware websocket lifecycle and stuck-listen recovery
|
||||||
|
|
||||||
|
Main gap:
|
||||||
|
|
||||||
|
- no first-class presence/identity perception extraction from runtime context for greeting policy decisions
|
||||||
|
|
||||||
|
## Implementation Slices
|
||||||
|
|
||||||
|
### Slice G1: Presence Context Extraction And Session Snapshot
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
|
||||||
|
- extract presence/identity fields from websocket context payload into normalized metadata for routing
|
||||||
|
|
||||||
|
Initial fields:
|
||||||
|
|
||||||
|
- focused speaker id
|
||||||
|
- identified person ids present
|
||||||
|
- total people present
|
||||||
|
- trigger source if present
|
||||||
|
- time-of-day helper signals
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- no facial-recognition implementation is needed in cloud; cloud consumes robot perception signals
|
||||||
|
|
||||||
|
### Slice G2: Greeting Intent Families And Parser Guardrails
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
|
||||||
|
- add explicit greeting intent families with question/command guardrails
|
||||||
|
|
||||||
|
Initial families:
|
||||||
|
|
||||||
|
- `hello`, `hey jibo`, `what's up`
|
||||||
|
- `good morning`, `good afternoon`, `good evening`, `good night`
|
||||||
|
- `i'm home`, `i'm back`
|
||||||
|
- identity question (`who am i`) as a future-compatible hook
|
||||||
|
|
||||||
|
Guardrails:
|
||||||
|
|
||||||
|
- avoid stealing non-greeting domains
|
||||||
|
- keep existing date/time and birthday disambiguation intact
|
||||||
|
|
||||||
|
### Slice G3: Greeting State-Machine Port (OpenJibo Style)
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
|
||||||
|
- add a greeting state-machine module with explicit route metadata like chitchat
|
||||||
|
|
||||||
|
Planned routes:
|
||||||
|
|
||||||
|
- `ReactiveGreeting`
|
||||||
|
- `ProactiveGreeting`
|
||||||
|
- `MorningGreeting`
|
||||||
|
- `SpecialDayGreeting`
|
||||||
|
- `OptionalResponse`
|
||||||
|
- `ErrorResponse`
|
||||||
|
|
||||||
|
Output shape:
|
||||||
|
|
||||||
|
- keep stock-compatible skill payload patterns
|
||||||
|
- preserve MIM/ESML hook points for charm content
|
||||||
|
|
||||||
|
### Slice G4: Proactive Gating And Cooldowns
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
|
||||||
|
- port the critical Pegasus policy behavior to prevent spam
|
||||||
|
|
||||||
|
Phase-1 rules:
|
||||||
|
|
||||||
|
- skip proactive greetings when trigger source is surprise
|
||||||
|
- enforce per-tenant/person cooldown (target parity: 2-hour greeting window)
|
||||||
|
- suppress proactive launch when session is unstable (pending listen/follow-up conflict)
|
||||||
|
|
||||||
|
### Slice G5: Person Queue And Memory Extensions
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
|
||||||
|
- introduce lightweight person queue/history for greeting relevance
|
||||||
|
|
||||||
|
Phase-1 storage additions:
|
||||||
|
|
||||||
|
- last-seen timestamp per person key
|
||||||
|
- last-greeted timestamp per person key
|
||||||
|
- optional preferred-name alias for spoken greeting personalization
|
||||||
|
|
||||||
|
### Slice G6: Rollout, Logging, And Live Validation
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
|
||||||
|
- ship safely with observability and test confidence
|
||||||
|
|
||||||
|
Required coverage:
|
||||||
|
|
||||||
|
- unit tests for context extraction and intent routing
|
||||||
|
- websocket tests for presence-triggered greeting eligibility and cooldown behavior
|
||||||
|
- live captures validating:
|
||||||
|
- no stuck listening regressions
|
||||||
|
- no runaway proactive loops
|
||||||
|
- stable fallback when identity is unknown
|
||||||
|
|
||||||
|
## Suggested Build Order
|
||||||
|
|
||||||
|
1. G1 context extraction + diagnostics
|
||||||
|
2. G2 greeting parser families + guardrails
|
||||||
|
3. G3 greeting state machine (reactive first)
|
||||||
|
4. G4 proactive gating + cooldowns
|
||||||
|
5. G5 person queue memory extensions
|
||||||
|
6. G6 live validation and polish
|
||||||
|
|
||||||
|
## Definition Of Done For This Track
|
||||||
|
|
||||||
|
- presence-aware greeting behavior works with and without identified users
|
||||||
|
- proactive greeting frequency is policy-bounded and observable
|
||||||
|
- no regressions in existing `1.0.19` memory/weather/proactivity flows
|
||||||
|
- release docs and backlog are updated with shipped scope and next slice
|
||||||
105
OpenJibo/docs/personal-report-parity-plan.md
Normal file
105
OpenJibo/docs/personal-report-parity-plan.md
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
# Personal Report Parity Plan
|
||||||
|
|
||||||
|
As-of: `2026-05-07`
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Bring OpenJibo personal report behavior closer to original Jibo charm while keeping cloud architecture modern and provider-agnostic.
|
||||||
|
|
||||||
|
## Pegasus Findings (Source Anchors)
|
||||||
|
|
||||||
|
- Weather personality and visuals were MIM-driven, not plain speech:
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\weather\WeatherMimLogic.ts`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\mims\en-us\WeatherCommentRain.mim`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\mims\en-us\WeatherTodayHighLow.mim`
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\report-skill\resources\views\weatherHiLo.json`
|
||||||
|
- Weather icons were mapped to condition/time-of-day tokens (`clear-day`, `partly-cloudy-night`, etc.) and used in `<anim cat='weather' meta='...'>`.
|
||||||
|
- Report-skill supported reactive entrypoints beyond full personal report:
|
||||||
|
- `requestWeatherPR`, `requestNews`, `requestCommute`, `requestCalendar`
|
||||||
|
- Source: `C:\Projects\jibo\pegasus\packages\hub\pegasus-skills\report_skill_manifest.json`
|
||||||
|
- Legacy data backends were Lasso-mediated:
|
||||||
|
- weather: Dark Sky
|
||||||
|
- commute: Google Maps directions/traffic
|
||||||
|
- news: AP News feeds
|
||||||
|
- calendar: Google/Outlook connectors
|
||||||
|
- Parser `main_agent` explicitly includes weather/news/personal-report intents; direct commute/calendar intents are not present in that same folder snapshot:
|
||||||
|
- `C:\Projects\jibo\pegasus\packages\parser\dialogflow\main_agent\intents`
|
||||||
|
- Grocery/list behavior found in Pegasus is scripted-response style, not a standalone list skill:
|
||||||
|
- `RA_JBO_ShoppingList.mim` and `RA_JBO_ManageToDoList.mim` are "not supported yet" style responses.
|
||||||
|
|
||||||
|
## OpenJibo Current State
|
||||||
|
|
||||||
|
- Personal report state machine exists and is test-backed.
|
||||||
|
- Weather provider integration exists (OpenWeather), including current and tomorrow.
|
||||||
|
- News and commute currently have baseline placeholder speech, not live provider-backed data orchestration.
|
||||||
|
- Calendar is currently reply-based and not yet provider-integrated.
|
||||||
|
|
||||||
|
## Gap Summary
|
||||||
|
|
||||||
|
1. Weather has factual speech but needs stronger visual/personality parity.
|
||||||
|
2. Non-local weather and broader date scopes need expansion beyond basic trailing `in <location>` and tomorrow handling.
|
||||||
|
3. Live news feed selection and filtering strategy is not yet implemented.
|
||||||
|
4. Commute data path and settings model are not yet mapped to an active provider integration.
|
||||||
|
5. Full personal report parity matrix (weather/commute/calendar/news behavior details) is not yet documented as a ship checklist.
|
||||||
|
|
||||||
|
## Implementation Phases
|
||||||
|
|
||||||
|
## Phase 1 (In Progress): Weather Personality Lift
|
||||||
|
|
||||||
|
- Add weather-condition animation metadata and expressive weather MIM-style prompt metadata to cloud weather speech.
|
||||||
|
- Expand location phrase handling (`in/for/at`) and suffix stripping for common temporal tails.
|
||||||
|
|
||||||
|
## Phase 2: Weather Visual Layer Parity
|
||||||
|
|
||||||
|
- Add weather Hi/Lo view payload support (OpenJibo-side equivalent to `weatherHiLo.json` behavior).
|
||||||
|
- Carry mapped weather icon token + hi/lo values into outbound skill action config.
|
||||||
|
- Keep fallback behavior safe when view assets are unavailable.
|
||||||
|
|
||||||
|
## Phase 3: Weather Scope Expansion
|
||||||
|
|
||||||
|
- Add parser support for additional time requests (for example weekend/next-week phrasing).
|
||||||
|
- Extend weather request model to support short-range date windows.
|
||||||
|
- Decide whether range responses are summarized speech-only or include multi-card view behavior.
|
||||||
|
|
||||||
|
## Phase 4: Live News Source
|
||||||
|
|
||||||
|
- Introduce provider-backed headline ingestion with category toggles.
|
||||||
|
- Mirror core Pegasus constraints:
|
||||||
|
- de-duplicate headlines
|
||||||
|
- filter missing summaries/images
|
||||||
|
- child-safe filtering mode
|
||||||
|
- Preserve current speech fallback if provider is unavailable.
|
||||||
|
|
||||||
|
## Phase 5: Commute Data Path
|
||||||
|
|
||||||
|
- Implement commute provider abstraction and first provider integration.
|
||||||
|
- Recreate core commute decision logic:
|
||||||
|
- minutes-left
|
||||||
|
- normal vs delayed traffic commentary
|
||||||
|
- mode-aware phrasing (drive vs transit)
|
||||||
|
- Add settings contract for origin/destination/work-arrival/mode.
|
||||||
|
|
||||||
|
## Phase 6: Personal Report Coverage Matrix
|
||||||
|
|
||||||
|
- Build parity matrix across weather/news/commute/calendar:
|
||||||
|
- intent phrases
|
||||||
|
- required entities/settings
|
||||||
|
- provider dependencies
|
||||||
|
- expected MIM/view style outputs
|
||||||
|
- fallback behavior
|
||||||
|
- Attach tests and capture criteria for each row.
|
||||||
|
|
||||||
|
## Phase 7 (Future Release): Grocery Lists
|
||||||
|
|
||||||
|
- Track as a future release item (requested by users).
|
||||||
|
- Two candidate paths:
|
||||||
|
1. Native lightweight list skill (fastest to ship).
|
||||||
|
2. Integration-backed list orchestration (better long-term ecosystem fit).
|
||||||
|
- Recommendation: ship native MVP first, then add integration connectors.
|
||||||
|
|
||||||
|
## Next Immediate Execution
|
||||||
|
|
||||||
|
1. Validate weather personality-lift behavior in live runs.
|
||||||
|
2. Implement weather view payload support (Hi/Lo + condition icon).
|
||||||
|
3. Draft provider plan for live news source.
|
||||||
|
4. Draft commute provider interface + settings schema.
|
||||||
@@ -117,9 +117,36 @@ Reference:
|
|||||||
|
|
||||||
- [system-diagram-alignment.md](system-diagram-alignment.md)
|
- [system-diagram-alignment.md](system-diagram-alignment.md)
|
||||||
|
|
||||||
|
## Greetings And Presence Planning Snapshot (`2026-05-07`)
|
||||||
|
|
||||||
|
Pegasus greeting and presence behavior has now been captured into a source-anchored OpenJibo implementation plan.
|
||||||
|
|
||||||
|
Reference:
|
||||||
|
|
||||||
|
- [greetings-presence-plan.md](greetings-presence-plan.md)
|
||||||
|
|
||||||
|
## Live Validation Snapshot (`2026-05-07`)
|
||||||
|
|
||||||
|
User-confirmed end-to-end behavior now includes:
|
||||||
|
|
||||||
|
- `Hey Jibo -> What's your cloud version?` (working)
|
||||||
|
- `Hey Jibo -> What's the time?` (working)
|
||||||
|
- `Hey Jibo -> Surprise me -> pizza fact -> $YESNO (Yes) -> fact` (working)
|
||||||
|
- `Hey Jibo -> Surprise me -> pizza fact -> $YESNO (No) -> decline reply` (working)
|
||||||
|
|
||||||
|
This confirms the pizza-fact offer state now keeps the yes/no branch open through completion and does not require a second wake-word reset for the follow-up answer.
|
||||||
|
|
||||||
|
## Personal Report Planning Snapshot (`2026-05-07`)
|
||||||
|
|
||||||
|
Personal report parity planning is now captured with Pegasus source anchors for weather visuals/animations, live news, commute, and calendar gap coverage.
|
||||||
|
|
||||||
|
Reference:
|
||||||
|
|
||||||
|
- [personal-report-parity-plan.md](personal-report-parity-plan.md)
|
||||||
|
|
||||||
## Next Queued Task (`2026-05-06`)
|
## Next Queued Task (`2026-05-06`)
|
||||||
|
|
||||||
Queued next `1.0.19` implementation task:
|
Queued next `1.0.19` implementation task (now started):
|
||||||
|
|
||||||
- dialog parsing expansion and ambiguity guardrails
|
- dialog parsing expansion and ambiguity guardrails
|
||||||
|
|
||||||
@@ -129,16 +156,37 @@ Execution focus:
|
|||||||
- reduce trigger-only captures that drop the rest of the utterance
|
- reduce trigger-only captures that drop the rest of the utterance
|
||||||
- preserve command-vs-question personality split and local skill payload compatibility
|
- preserve command-vs-question personality split and local skill payload compatibility
|
||||||
- add focused tests for new phrase families and ambiguity boundaries
|
- add focused tests for new phrase families and ambiguity boundaries
|
||||||
|
- keep listener-state observability aligned with the legacy GLSM flow while phrase guardrails are added
|
||||||
|
|
||||||
|
First completed guardrail slice under this queue:
|
||||||
|
|
||||||
|
- GLSM listener flow capture + telemetry mapping
|
||||||
|
- stale pending-listen recovery path for long-open no-context/no-audio listens
|
||||||
|
|
||||||
|
Second completed guardrail slice under this queue:
|
||||||
|
|
||||||
|
- tightened date/time ambiguity handling (`what's your birthday`/`what's your bday` no longer falls into date intent)
|
||||||
|
- expanded Pegasus-inspired memory/weather phrase coverage:
|
||||||
|
- birthday alias parsing (`my bday is ...`, `when is my bday`)
|
||||||
|
- shorthand preference sets (`my favorite sport football`)
|
||||||
|
- weather variants (`what's today's weather look like`, `will it be sunny tomorrow`)
|
||||||
|
- listener continuation guardrail now differentiates incomplete preference fragments from complete shorthand preference sets
|
||||||
|
|
||||||
|
Next queued implementation track after parser guardrails:
|
||||||
|
|
||||||
|
- presence-aware greetings and identity-triggered proactivity (Pegasus `@be/greetings` parity slice)
|
||||||
|
|
||||||
## Next Slices
|
## Next Slices
|
||||||
|
|
||||||
1. Dialog parsing expansion (queued next as of `2026-05-06`; more phrase variants, ambiguity handling, and transcript-to-intent guardrails)
|
1. Dialog parsing expansion (queued next as of `2026-05-06`; more phrase variants, ambiguity handling, and transcript-to-intent guardrails)
|
||||||
2. Holidays and seasonal personality slice beyond pizza day (time-scoped content backed by memory/proactivity path)
|
2. Presence-aware greetings and identity-triggered proactivity (reactive/proactive split, cooldowns, person-aware greeting hooks)
|
||||||
3. Durable memory persistence path (swap in provider-backed multi-tenant storage while preserving behavior contracts)
|
3. Personal report parity slices (weather visual layer, live news path, commute path, calendar parity matrix)
|
||||||
4. Update/backup/restore end-to-end proof (operator-run and documented)
|
4. Holidays and seasonal personality slice beyond pizza day (time-scoped content backed by memory/proactivity path)
|
||||||
5. STT noise-screening and short-utterance reliability pass
|
5. Durable memory persistence path (swap in provider-backed multi-tenant storage while preserving behavior contracts)
|
||||||
6. Provider-backed news expansion and deeper weather parity using Pegasus-backed contracts
|
6. Update/backup/restore end-to-end proof (operator-run and documented)
|
||||||
7. Capture indexing and retention boundary for group testing
|
7. STT noise-screening and short-utterance reliability pass
|
||||||
|
8. Provider-backed news expansion and deeper weather parity using Pegasus-backed contracts
|
||||||
|
9. Capture indexing and retention boundary for group testing
|
||||||
|
|
||||||
For slices 1-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements.
|
For slices 1-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements.
|
||||||
|
|
||||||
|
|||||||
@@ -10,12 +10,13 @@ Use it to keep release planning grounded in three views:
|
|||||||
- where we are (current hosted `.NET` implementation)
|
- where we are (current hosted `.NET` implementation)
|
||||||
- where we are headed (next architecture slices)
|
- where we are headed (next architecture slices)
|
||||||
|
|
||||||
As-of date: `2026-05-06`
|
As-of date: `2026-05-07`
|
||||||
|
|
||||||
## Diagram Inputs
|
## Diagram Inputs
|
||||||
|
|
||||||
- Legacy system architecture: `C:\Projects\jibo\pegasus\resources\system_diagram.png`
|
- Legacy system architecture: `C:\Projects\jibo\pegasus\resources\system_diagram.png`
|
||||||
- Legacy generic skill scaffold: `C:\Projects\jibo\pegasus\packages\template-skill\docs\TemplateSkill.png`
|
- Legacy generic skill scaffold: `C:\Projects\jibo\pegasus\packages\template-skill\docs\TemplateSkill.png`
|
||||||
|
- Legacy listener state machine: `C:\Projects\jibo\sdk\packages\skills-service-manager\resources\state-diagrams\glsm.png`
|
||||||
|
|
||||||
## Template Skill Verdict
|
## Template Skill Verdict
|
||||||
|
|
||||||
@@ -39,12 +40,37 @@ Conclusion: do not treat template-skill flow as a port target. Treat it as a sha
|
|||||||
| `Parser / Robust Parser` | rule-based intent resolution in [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) + focused state machines (personal report/chitchat) | deeper phrase import from Pegasus intents/entities plus ambiguity guardrails |
|
| `Parser / Robust Parser` | rule-based intent resolution in [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) + focused state machines (personal report/chitchat) | deeper phrase import from Pegasus intents/entities plus ambiguity guardrails |
|
||||||
| `Skill Router` | [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) decision switch and local skill payload shaping | external skill routing config and safer declarative intent mapping |
|
| `Skill Router` | [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) decision switch and local skill payload shaping | external skill routing config and safer declarative intent mapping |
|
||||||
| `Proactivity Selector` | weighted candidate selection in [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) + pending-offer session state in [WebSocketTurnFinalizationService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs) | externalized proactivity catalog, cooldown policy, and broader category coverage |
|
| `Proactivity Selector` | weighted candidate selection in [JiboInteractionService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs) + pending-offer session state in [WebSocketTurnFinalizationService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs) | externalized proactivity catalog, cooldown policy, and broader category coverage |
|
||||||
|
| `Presence / Identity Context` | runtime context passthrough in [ProtocolToTurnContextMapper.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs) and turn metadata handling in [WebSocketTurnFinalizationService.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs) | normalize `runtime.perception` fields (`speaker`, `peoplePresent`, focused person) for greeting/proactivity policy decisions |
|
||||||
| `Skill Registry` | implicit in current code/routing | formal registry abstraction for local/cloud capabilities and manifest metadata |
|
| `Skill Registry` | implicit in current code/routing | formal registry abstraction for local/cloud capabilities and manifest metadata |
|
||||||
| `History` | tenant-scoped memory store in [InMemoryPersonalMemoryStore.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryPersonalMemoryStore.cs) | durable multi-tenant persistence and history timeline/query support |
|
| `History` | tenant-scoped memory store in [InMemoryPersonalMemoryStore.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryPersonalMemoryStore.cs) | durable multi-tenant persistence and history timeline/query support |
|
||||||
| `Lasso` provider aggregation | partial provider integration via weather provider wiring in [ServiceCollectionExtensions.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs) | full aggregation service for weather/news/calendar/knowledge inputs |
|
| `Lasso` provider aggregation | partial provider integration via weather provider wiring in [ServiceCollectionExtensions.cs](../src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs) | full aggregation service for weather/news/calendar/knowledge inputs |
|
||||||
| `Proactivity Catalog` | in-code candidate lists/weights | explicit catalog service with tuned weights and operator controls |
|
| `Proactivity Catalog` | in-code candidate lists/weights | explicit catalog service with tuned weights and operator controls |
|
||||||
| `Audio Logs` | file telemetry sinks in infrastructure telemetry | hosted indexed capture/retention for multi-operator analysis |
|
| `Audio Logs` | file telemetry sinks in infrastructure telemetry | hosted indexed capture/retention for multi-operator analysis |
|
||||||
|
|
||||||
|
## GLSM Listener Flow Alignment (`2026-05-06`)
|
||||||
|
|
||||||
|
Captured source:
|
||||||
|
|
||||||
|
- `C:\Projects\jibo\sdk\packages\skills-service-manager\resources\state-diagrams\glsm.png`
|
||||||
|
|
||||||
|
First OpenJibo support slice (implemented):
|
||||||
|
|
||||||
|
- explicit derived listener phases are now emitted in cloud diagnostics:
|
||||||
|
- `HJ_LISTENING`
|
||||||
|
- `LISTENING`
|
||||||
|
- `WAIT_LISTEN_FINISHED`
|
||||||
|
- `DISPATCH_DIALOG`
|
||||||
|
- `PROCESS_LISTENER_QUEUE`
|
||||||
|
- turn telemetry now records `glsm_phase_transition` with previous/next state and trigger
|
||||||
|
- websocket telemetry now includes `glsmPhase` on binary, context, and turn-processed events
|
||||||
|
- stale pending-listen recovery is now implemented:
|
||||||
|
- when a pending `LISTEN` stays open long enough with no context/audio, a new hotphrase listen can recover the stuck state before continuing
|
||||||
|
|
||||||
|
Current parity boundary:
|
||||||
|
|
||||||
|
- this slice focuses on listener lifecycle observability plus stuck-listen recovery
|
||||||
|
- deeper explicit parity states from GLSM (`Interrupt Listeners`, `Handle Launch Parse`, `Handle Global Parse`, `Dispatch Dialog` sub-branches) are next candidates once this capture-driven slice is validated live
|
||||||
|
|
||||||
## Where We Were
|
## Where We Were
|
||||||
|
|
||||||
Legacy cloud design was service-oriented around:
|
Legacy cloud design was service-oriented around:
|
||||||
@@ -102,3 +128,24 @@ Tracking anchors:
|
|||||||
Primary objective:
|
Primary objective:
|
||||||
|
|
||||||
- import Pegasus parser intent phrases/entities to improve intent confidence while preserving command-vs-question personality behavior.
|
- import Pegasus parser intent phrases/entities to improve intent confidence while preserving command-vs-question personality behavior.
|
||||||
|
|
||||||
|
## Greetings And Presence Track (`2026-05-07`)
|
||||||
|
|
||||||
|
A dedicated presence-aware greetings plan is now captured for the next personality slice, grounded in Pegasus `@be/greetings` state, identity, and proactive policy behavior.
|
||||||
|
|
||||||
|
Reference:
|
||||||
|
|
||||||
|
- [greetings-presence-plan.md](greetings-presence-plan.md)
|
||||||
|
|
||||||
|
## Personal Report Parity Track (`2026-05-07`)
|
||||||
|
|
||||||
|
Personal report parity planning is now captured with a source-anchored implementation sequence for:
|
||||||
|
|
||||||
|
- weather visual/personality parity
|
||||||
|
- live news provider path
|
||||||
|
- commute provider path
|
||||||
|
- calendar/report coverage matrix
|
||||||
|
|
||||||
|
Reference:
|
||||||
|
|
||||||
|
- [personal-report-parity-plan.md](personal-report-parity-plan.md)
|
||||||
|
|||||||
@@ -12,7 +12,8 @@ public sealed record WeatherReportRequest(
|
|||||||
double? Latitude,
|
double? Latitude,
|
||||||
double? Longitude,
|
double? Longitude,
|
||||||
bool IsTomorrow,
|
bool IsTomorrow,
|
||||||
bool? UseCelsius);
|
bool? UseCelsius,
|
||||||
|
int? ForecastDayOffset = null);
|
||||||
|
|
||||||
public sealed record WeatherReportSnapshot(
|
public sealed record WeatherReportSnapshot(
|
||||||
string LocationName,
|
string LocationName,
|
||||||
|
|||||||
@@ -417,7 +417,14 @@ public sealed class JiboInteractionService(
|
|||||||
string transcript,
|
string transcript,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var dateEntity = TryResolveWeatherDateEntity(transcript);
|
var referenceLocalTime = TryResolveReferenceLocalTime(turn);
|
||||||
|
var weatherDate = ResolveWeatherDateEntity(turn, transcript, referenceLocalTime);
|
||||||
|
var normalizedTranscript = NormalizeCommandPhrase(transcript);
|
||||||
|
if (ShouldDefaultForecastToTomorrow(normalizedTranscript, weatherDate))
|
||||||
|
{
|
||||||
|
weatherDate = new WeatherDateEntity("tomorrow", 1, "Tomorrow");
|
||||||
|
}
|
||||||
|
|
||||||
if (weatherReportProvider is null)
|
if (weatherReportProvider is null)
|
||||||
{
|
{
|
||||||
return new JiboInteractionDecision(
|
return new JiboInteractionDecision(
|
||||||
@@ -425,8 +432,17 @@ public sealed class JiboInteractionService(
|
|||||||
"I can check weather once my weather service is connected.");
|
"I can check weather once my weather service is connected.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (weatherDate.ForecastDayOffset > MaxWeatherForecastDayOffset)
|
||||||
|
{
|
||||||
|
return new JiboInteractionDecision(
|
||||||
|
"weather",
|
||||||
|
$"I can forecast up to {MaxWeatherForecastDayOffset} days ahead. Try tomorrow or another day this week.");
|
||||||
|
}
|
||||||
|
|
||||||
var locationQuery = TryResolveWeatherLocationQuery(transcript);
|
var locationQuery = TryResolveWeatherLocationQuery(transcript);
|
||||||
var weatherCoordinates = TryResolveWeatherCoordinates(turn);
|
var weatherCoordinates = string.IsNullOrWhiteSpace(locationQuery)
|
||||||
|
? TryResolveWeatherCoordinates(turn)
|
||||||
|
: null;
|
||||||
var useCelsius = ShouldUseCelsius(turn, transcript);
|
var useCelsius = ShouldUseCelsius(turn, transcript);
|
||||||
WeatherReportSnapshot? snapshot;
|
WeatherReportSnapshot? snapshot;
|
||||||
try
|
try
|
||||||
@@ -436,8 +452,9 @@ public sealed class JiboInteractionService(
|
|||||||
locationQuery,
|
locationQuery,
|
||||||
weatherCoordinates?.Latitude,
|
weatherCoordinates?.Latitude,
|
||||||
weatherCoordinates?.Longitude,
|
weatherCoordinates?.Longitude,
|
||||||
string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase),
|
string.Equals(weatherDate.DateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase),
|
||||||
useCelsius),
|
useCelsius,
|
||||||
|
weatherDate.ForecastDayOffset),
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
}
|
}
|
||||||
catch (Exception) when (!cancellationToken.IsCancellationRequested)
|
catch (Exception) when (!cancellationToken.IsCancellationRequested)
|
||||||
@@ -452,14 +469,18 @@ public sealed class JiboInteractionService(
|
|||||||
"I couldn't fetch the weather right now. Please try again.");
|
"I couldn't fetch the weather right now. Please try again.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var spokenReply = BuildWeatherSpokenReply(snapshot, weatherDate);
|
||||||
|
var weatherPayload = BuildWeatherSkillPayload(spokenReply, snapshot, referenceLocalTime);
|
||||||
return new JiboInteractionDecision(
|
return new JiboInteractionDecision(
|
||||||
"weather",
|
"weather",
|
||||||
BuildWeatherSpokenReply(snapshot, dateEntity));
|
spokenReply,
|
||||||
|
"chitchat-skill",
|
||||||
|
SkillPayload: weatherPayload);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string BuildWeatherSpokenReply(
|
private static string BuildWeatherSpokenReply(
|
||||||
WeatherReportSnapshot snapshot,
|
WeatherReportSnapshot snapshot,
|
||||||
string? dateEntity)
|
WeatherDateEntity weatherDate)
|
||||||
{
|
{
|
||||||
var unit = snapshot.UseCelsius ? "Celsius" : "Fahrenheit";
|
var unit = snapshot.UseCelsius ? "Celsius" : "Fahrenheit";
|
||||||
var summary = string.IsNullOrWhiteSpace(snapshot.Summary)
|
var summary = string.IsNullOrWhiteSpace(snapshot.Summary)
|
||||||
@@ -469,7 +490,7 @@ public sealed class JiboInteractionService(
|
|||||||
? "your area"
|
? "your area"
|
||||||
: snapshot.LocationName;
|
: snapshot.LocationName;
|
||||||
|
|
||||||
if (string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase))
|
if (weatherDate.ForecastDayOffset > 0)
|
||||||
{
|
{
|
||||||
var highText = snapshot.HighTemperature is null
|
var highText = snapshot.HighTemperature is null
|
||||||
? null
|
? null
|
||||||
@@ -482,12 +503,173 @@ public sealed class JiboInteractionService(
|
|||||||
: highText is not null && lowText is not null
|
: highText is not null && lowText is not null
|
||||||
? $" with {highText} and {lowText}"
|
? $" with {highText} and {lowText}"
|
||||||
: $" with {highText ?? lowText}";
|
: $" with {highText ?? lowText}";
|
||||||
return $"Tomorrow in {location}, expect {summary}{tempRange}.";
|
var forecastLeadIn = string.IsNullOrWhiteSpace(weatherDate.ForecastLeadIn)
|
||||||
|
? "Tomorrow"
|
||||||
|
: weatherDate.ForecastLeadIn;
|
||||||
|
return $"{forecastLeadIn} in {location}, expect {summary}{tempRange}.";
|
||||||
}
|
}
|
||||||
|
|
||||||
return $"Right now in {location}, it is {summary} and {snapshot.Temperature} degrees {unit}.";
|
return $"Right now in {location}, it is {summary} and {snapshot.Temperature} degrees {unit}.";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool ShouldDefaultForecastToTomorrow(string normalizedTranscript, WeatherDateEntity weatherDate)
|
||||||
|
{
|
||||||
|
if (weatherDate.ForecastDayOffset > 0 ||
|
||||||
|
string.IsNullOrWhiteSpace(normalizedTranscript) ||
|
||||||
|
!normalizedTranscript.Contains("forecast", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return !MatchesAny(
|
||||||
|
normalizedTranscript,
|
||||||
|
"today",
|
||||||
|
"today s",
|
||||||
|
"today's",
|
||||||
|
"tonight",
|
||||||
|
"right now",
|
||||||
|
"current weather",
|
||||||
|
"currently");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IDictionary<string, object?> BuildWeatherSkillPayload(
|
||||||
|
string spokenReply,
|
||||||
|
WeatherReportSnapshot snapshot,
|
||||||
|
DateTimeOffset? referenceLocalTime)
|
||||||
|
{
|
||||||
|
var weatherIcon = ResolveWeatherAnimationIcon(snapshot, referenceLocalTime);
|
||||||
|
var promptToken = ResolveWeatherPromptToken(weatherIcon);
|
||||||
|
var highTemperature = snapshot.HighTemperature ?? snapshot.Temperature;
|
||||||
|
var lowTemperature = snapshot.LowTemperature ?? snapshot.Temperature;
|
||||||
|
var temperatureUnit = snapshot.UseCelsius ? "C" : "F";
|
||||||
|
var temperatureBand = ResolveWeatherTemperatureBand(highTemperature, snapshot.UseCelsius);
|
||||||
|
|
||||||
|
return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["esml"] =
|
||||||
|
$"<speak><anim cat='weather' meta='{weatherIcon}' nonBlocking='true' /><break size='0.35'/><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeForEsml(spokenReply)}</es></speak>",
|
||||||
|
["mim_id"] = $"WeatherComment{promptToken}",
|
||||||
|
["mim_type"] = "announcement",
|
||||||
|
["prompt_id"] = $"WeatherComment{promptToken}_AN_13",
|
||||||
|
["prompt_sub_category"] = "AN",
|
||||||
|
["weather_view_enabled"] = true,
|
||||||
|
["weather_view_kind"] = "weatherHiLo",
|
||||||
|
["weather_icon"] = weatherIcon,
|
||||||
|
["weather_summary"] = snapshot.Summary,
|
||||||
|
["weather_location"] = snapshot.LocationName,
|
||||||
|
["weather_high"] = highTemperature,
|
||||||
|
["weather_low"] = lowTemperature,
|
||||||
|
["weather_unit"] = temperatureUnit,
|
||||||
|
["weather_theme"] = temperatureBand
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ResolveWeatherAnimationIcon(
|
||||||
|
WeatherReportSnapshot snapshot,
|
||||||
|
DateTimeOffset? referenceLocalTime)
|
||||||
|
{
|
||||||
|
var isDaytime = (referenceLocalTime ?? DateTimeOffset.UtcNow).Hour is >= 6 and < 18;
|
||||||
|
var normalized = NormalizeCommandPhrase(
|
||||||
|
$"{snapshot.Condition ?? string.Empty} {snapshot.Summary ?? string.Empty}");
|
||||||
|
|
||||||
|
if (normalized.Contains("thunder", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("drizzle", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("rain", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return "rain";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("snow", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return "snow";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("sleet", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("freezing rain", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("ice", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return "sleet";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("fog", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("mist", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("haze", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("smoke", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return "fog";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("wind", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return "wind";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("partly cloudy", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("scattered clouds", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("few clouds", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return isDaytime ? "partly-cloudy-day" : "partly-cloudy-night";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("cloud", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("overcast", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return "cloudy";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("clear", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("sunny", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return isDaytime ? "clear-day" : "clear-night";
|
||||||
|
}
|
||||||
|
|
||||||
|
return isDaytime ? "clear-day" : "clear-night";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ResolveWeatherPromptToken(string weatherIcon)
|
||||||
|
{
|
||||||
|
return weatherIcon switch
|
||||||
|
{
|
||||||
|
"clear-day" => "ClearDay",
|
||||||
|
"clear-night" => "ClearNight",
|
||||||
|
"rain" => "Rain",
|
||||||
|
"snow" => "Snow",
|
||||||
|
"sleet" => "Sleet",
|
||||||
|
"fog" => "Fog",
|
||||||
|
"wind" => "Wind",
|
||||||
|
"cloudy" => "Cloudy",
|
||||||
|
"partly-cloudy-day" => "PartlyCloudyDay",
|
||||||
|
"partly-cloudy-night" => "PartlyCloudyNight",
|
||||||
|
_ => "Cloudy"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ResolveWeatherTemperatureBand(int highTemperature, bool useCelsius)
|
||||||
|
{
|
||||||
|
var hotThreshold = useCelsius ? 29 : 85;
|
||||||
|
var coldThreshold = useCelsius ? 4 : 40;
|
||||||
|
if (highTemperature > hotThreshold)
|
||||||
|
{
|
||||||
|
return "Hot";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (highTemperature < coldThreshold)
|
||||||
|
{
|
||||||
|
return "Cold";
|
||||||
|
}
|
||||||
|
|
||||||
|
return "Normal";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string EscapeForEsml(string value)
|
||||||
|
{
|
||||||
|
return value
|
||||||
|
.Replace("&", "&", StringComparison.Ordinal)
|
||||||
|
.Replace("<", "<", StringComparison.Ordinal)
|
||||||
|
.Replace(">", ">", StringComparison.Ordinal)
|
||||||
|
.Replace("\"", """, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
private static JiboInteractionDecision BuildOrderPizzaDecision()
|
private static JiboInteractionDecision BuildOrderPizzaDecision()
|
||||||
{
|
{
|
||||||
return new JiboInteractionDecision(
|
return new JiboInteractionDecision(
|
||||||
@@ -1151,8 +1333,7 @@ public sealed class JiboInteractionService(
|
|||||||
return "no";
|
return "no";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (MatchesAny(loweredTranscript, "what time is it", "current time", "the time", "time is it") ||
|
if (IsTimeRequest(loweredTranscript))
|
||||||
loweredTranscript.Contains("time", StringComparison.Ordinal))
|
|
||||||
{
|
{
|
||||||
return "time";
|
return "time";
|
||||||
}
|
}
|
||||||
@@ -1162,9 +1343,7 @@ public sealed class JiboInteractionService(
|
|||||||
return "day";
|
return "day";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (MatchesAny(loweredTranscript, "what day is it", "what is the date", "today s date", "today's date") ||
|
if (IsDateRequest(loweredTranscript))
|
||||||
loweredTranscript.Contains("date", StringComparison.Ordinal) ||
|
|
||||||
loweredTranscript.Contains("day", StringComparison.Ordinal))
|
|
||||||
{
|
{
|
||||||
return "date";
|
return "date";
|
||||||
}
|
}
|
||||||
@@ -1630,8 +1809,52 @@ public sealed class JiboInteractionService(
|
|||||||
MatchesAny(normalized, "no thank you", "maybe later");
|
MatchesAny(normalized, "no thank you", "maybe later");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool IsTimeRequest(string loweredTranscript)
|
||||||
|
{
|
||||||
|
var normalized = NormalizeCommandPhrase(loweredTranscript);
|
||||||
|
if (string.IsNullOrWhiteSpace(normalized))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized is "time" or "the time" or "current time" or "what time is it" or "what s the time" or "what is the time")
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized.StartsWith("what time", StringComparison.Ordinal) ||
|
||||||
|
normalized.StartsWith("tell me the time", StringComparison.Ordinal) ||
|
||||||
|
normalized.StartsWith("show me the time", StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsDateRequest(string loweredTranscript)
|
||||||
|
{
|
||||||
|
var normalized = NormalizeCommandPhrase(loweredTranscript);
|
||||||
|
if (string.IsNullOrWhiteSpace(normalized))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized is
|
||||||
|
"what is the date" or
|
||||||
|
"what s the date" or
|
||||||
|
"what date is it" or
|
||||||
|
"today s date" or
|
||||||
|
"today date" or
|
||||||
|
"what is today s date" or
|
||||||
|
"what s today s date" or
|
||||||
|
"what is todays date" or
|
||||||
|
"what s todays date";
|
||||||
|
}
|
||||||
|
|
||||||
private static bool IsWeatherRequest(string loweredTranscript)
|
private static bool IsWeatherRequest(string loweredTranscript)
|
||||||
{
|
{
|
||||||
|
var normalized = NormalizeCommandPhrase(loweredTranscript);
|
||||||
|
if (IsWeatherTopicQuestion(normalized))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
if (MatchesAny(
|
if (MatchesAny(
|
||||||
loweredTranscript,
|
loweredTranscript,
|
||||||
"weather",
|
"weather",
|
||||||
@@ -1652,12 +1875,22 @@ public sealed class JiboInteractionService(
|
|||||||
"what is today s humidity",
|
"what is today s humidity",
|
||||||
"what is today's humidity",
|
"what is today's humidity",
|
||||||
"what's the humidity",
|
"what's the humidity",
|
||||||
"what is the humidity"))
|
"what is the humidity",
|
||||||
|
"what's today's forecast",
|
||||||
|
"what s today's forecast",
|
||||||
|
"what s today s forecast",
|
||||||
|
"what is today s forecast",
|
||||||
|
"what is today's forecast",
|
||||||
|
"what's today's weather look like",
|
||||||
|
"what s today's weather look like",
|
||||||
|
"what s today s weather look like",
|
||||||
|
"what is today s weather look like",
|
||||||
|
"what is today's weather look like"))
|
||||||
{
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return MatchesAny(
|
if (MatchesAny(
|
||||||
loweredTranscript,
|
loweredTranscript,
|
||||||
"will it rain",
|
"will it rain",
|
||||||
"will it snow",
|
"will it snow",
|
||||||
@@ -1669,7 +1902,47 @@ public sealed class JiboInteractionService(
|
|||||||
"is it going to rain",
|
"is it going to rain",
|
||||||
"is it going to snow",
|
"is it going to snow",
|
||||||
"do you think it will rain",
|
"do you think it will rain",
|
||||||
"do you think it will snow");
|
"do you think it will snow"))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return WeatherConditionForecastPattern.IsMatch(loweredTranscript);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsWeatherTopicQuestion(string normalizedTranscript)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(normalizedTranscript))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var mentionsWeatherTopic =
|
||||||
|
normalizedTranscript.Contains("weather", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.Contains("forecast", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.Contains("temperature", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.Contains("humidity", StringComparison.Ordinal);
|
||||||
|
if (!mentionsWeatherTopic)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedTranscript.StartsWith("what ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("how ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("check ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("show ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("tell ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("look up ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("launch ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("give me ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("temperature ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("forecast ", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.StartsWith("weather ", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return WeatherTopicLocationPattern.IsMatch(normalizedTranscript);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string? TryResolveWeatherLocationQuery(string transcript)
|
private static string? TryResolveWeatherLocationQuery(string transcript)
|
||||||
@@ -1783,15 +2056,247 @@ public sealed class JiboInteractionService(
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string? TryResolveWeatherDateEntity(string transcript)
|
private static WeatherDateEntity ResolveWeatherDateEntity(
|
||||||
|
TurnContext turn,
|
||||||
|
string transcript,
|
||||||
|
DateTimeOffset? referenceLocalTime)
|
||||||
{
|
{
|
||||||
var normalized = NormalizeCommandPhrase(transcript);
|
var entities = ReadEntities(turn);
|
||||||
if (MatchesAny(normalized, "tomorrow", "tomorrow s", "tomorrow's"))
|
if (TryResolveWeatherDateEntityFromClientEntities(entities, referenceLocalTime, out var entityFromClient))
|
||||||
{
|
{
|
||||||
return "tomorrow";
|
return entityFromClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
var normalized = NormalizeCommandPhrase(transcript);
|
||||||
|
if (string.IsNullOrWhiteSpace(normalized))
|
||||||
|
{
|
||||||
|
return WeatherDateEntity.None;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.Contains("day after tomorrow", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return new WeatherDateEntity("day_after_tomorrow", 2, "The day after tomorrow");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (MatchesAny(normalized, "tomorrow", "tomorrow s", "tomorrow's"))
|
||||||
|
{
|
||||||
|
return new WeatherDateEntity("tomorrow", 1, "Tomorrow");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (referenceLocalTime is not null &&
|
||||||
|
TryResolveWeatherTimeRangeOffset(normalized, referenceLocalTime.Value, out var rangeOffset, out var rangeLeadIn) &&
|
||||||
|
rangeOffset > 0)
|
||||||
|
{
|
||||||
|
return new WeatherDateEntity("range", rangeOffset, rangeLeadIn);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (referenceLocalTime is not null &&
|
||||||
|
TryResolveWeatherDayOfWeekOffset(normalized, referenceLocalTime.Value, out var dayOffset, out var dayName) &&
|
||||||
|
dayOffset > 0)
|
||||||
|
{
|
||||||
|
return new WeatherDateEntity("weekday", dayOffset, $"On {dayName}");
|
||||||
|
}
|
||||||
|
|
||||||
|
return WeatherDateEntity.None;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryResolveWeatherDateEntityFromClientEntities(
|
||||||
|
IReadOnlyDictionary<string, string> clientEntities,
|
||||||
|
DateTimeOffset? referenceLocalTime,
|
||||||
|
out WeatherDateEntity weatherDate)
|
||||||
|
{
|
||||||
|
weatherDate = WeatherDateEntity.None;
|
||||||
|
if (!TryReadClientWeatherDateValue(clientEntities, out var rawDateValue))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var normalizedDate = NormalizeCommandPhrase(rawDateValue);
|
||||||
|
if (normalizedDate.Contains("day after tomorrow", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
weatherDate = new WeatherDateEntity("day_after_tomorrow", 2, "The day after tomorrow");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (MatchesAny(normalizedDate, "tomorrow", "tomorrow s", "tomorrow's"))
|
||||||
|
{
|
||||||
|
weatherDate = new WeatherDateEntity("tomorrow", 1, "Tomorrow");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (referenceLocalTime is not null &&
|
||||||
|
TryResolveWeatherTimeRangeOffset(normalizedDate, referenceLocalTime.Value, out var rangeOffset, out var rangeLeadIn) &&
|
||||||
|
rangeOffset > 0)
|
||||||
|
{
|
||||||
|
weatherDate = new WeatherDateEntity("range", rangeOffset, rangeLeadIn);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
DateOnly targetDate;
|
||||||
|
if (DateOnly.TryParse(rawDateValue, out var parsedDate))
|
||||||
|
{
|
||||||
|
targetDate = parsedDate;
|
||||||
|
}
|
||||||
|
else if (DateTimeOffset.TryParse(rawDateValue, out var parsedDateTimeOffset))
|
||||||
|
{
|
||||||
|
targetDate = DateOnly.FromDateTime(parsedDateTimeOffset.DateTime);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var referenceDate = DateOnly.FromDateTime((referenceLocalTime ?? DateTimeOffset.UtcNow).DateTime);
|
||||||
|
var dayOffset = targetDate.DayNumber - referenceDate.DayNumber;
|
||||||
|
if (dayOffset <= 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
weatherDate = dayOffset == 1
|
||||||
|
? new WeatherDateEntity("tomorrow", 1, "Tomorrow")
|
||||||
|
: new WeatherDateEntity(
|
||||||
|
"date",
|
||||||
|
dayOffset,
|
||||||
|
$"On {targetDate.ToDateTime(TimeOnly.MinValue).ToString("dddd", CultureInfo.InvariantCulture)}");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryReadClientWeatherDateValue(
|
||||||
|
IReadOnlyDictionary<string, string> clientEntities,
|
||||||
|
out string dateValue)
|
||||||
|
{
|
||||||
|
foreach (var key in WeatherDateEntityKeys)
|
||||||
|
{
|
||||||
|
if (!clientEntities.TryGetValue(key, out var rawValue) ||
|
||||||
|
string.IsNullOrWhiteSpace(rawValue))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
dateValue = rawValue.Trim();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
dateValue = string.Empty;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryResolveWeatherDayOfWeekOffset(
|
||||||
|
string normalizedTranscript,
|
||||||
|
DateTimeOffset referenceLocalTime,
|
||||||
|
out int dayOffset,
|
||||||
|
out string dayName)
|
||||||
|
{
|
||||||
|
dayOffset = 0;
|
||||||
|
dayName = string.Empty;
|
||||||
|
|
||||||
|
var match = WeatherDayOfWeekPattern.Match(normalizedTranscript);
|
||||||
|
if (!match.Success)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var dayToken = match.Groups["day"].Value;
|
||||||
|
if (!TryParseDayOfWeek(dayToken, out var targetDay))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var currentDay = referenceLocalTime.DayOfWeek;
|
||||||
|
dayOffset = ((int)targetDay - (int)currentDay + 7) % 7;
|
||||||
|
if (match.Groups["next"].Success)
|
||||||
|
{
|
||||||
|
dayOffset = dayOffset == 0 ? 7 : dayOffset + 7;
|
||||||
|
}
|
||||||
|
else if (match.Groups["this"].Success && dayOffset == 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
dayName = CultureInfo.InvariantCulture.TextInfo.ToTitleCase(dayToken);
|
||||||
|
return dayOffset > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryResolveWeatherTimeRangeOffset(
|
||||||
|
string normalizedTranscript,
|
||||||
|
DateTimeOffset referenceLocalTime,
|
||||||
|
out int dayOffset,
|
||||||
|
out string leadIn)
|
||||||
|
{
|
||||||
|
dayOffset = 0;
|
||||||
|
leadIn = string.Empty;
|
||||||
|
if (string.IsNullOrWhiteSpace(normalizedTranscript))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var hasNextWeekend = normalizedTranscript.Contains("next weekend", StringComparison.Ordinal);
|
||||||
|
var hasThisWeekend =
|
||||||
|
normalizedTranscript.Contains("this weekend", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.Contains("the weekend", StringComparison.Ordinal) ||
|
||||||
|
normalizedTranscript.EndsWith("weekend", StringComparison.Ordinal);
|
||||||
|
if (hasNextWeekend || hasThisWeekend)
|
||||||
|
{
|
||||||
|
dayOffset = ((int)DayOfWeek.Saturday - (int)referenceLocalTime.DayOfWeek + 7) % 7;
|
||||||
|
if (hasNextWeekend)
|
||||||
|
{
|
||||||
|
dayOffset = dayOffset + 7;
|
||||||
|
leadIn = "Next weekend";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// If it's already Saturday, prefer forecasting Sunday for "this weekend".
|
||||||
|
if (dayOffset == 0 && referenceLocalTime.DayOfWeek == DayOfWeek.Saturday)
|
||||||
|
{
|
||||||
|
dayOffset = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
leadIn = "This weekend";
|
||||||
|
}
|
||||||
|
|
||||||
|
return dayOffset > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
var hasNextWeek = normalizedTranscript.Contains("next week", StringComparison.Ordinal);
|
||||||
|
if (hasNextWeek)
|
||||||
|
{
|
||||||
|
dayOffset = 7;
|
||||||
|
leadIn = "Next week";
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
var hasThisWeek = normalizedTranscript.Contains("this week", StringComparison.Ordinal);
|
||||||
|
if (hasThisWeek)
|
||||||
|
{
|
||||||
|
dayOffset = referenceLocalTime.DayOfWeek == DayOfWeek.Saturday ? 1 : 2;
|
||||||
|
leadIn = "Later this week";
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryParseDayOfWeek(string dayToken, out DayOfWeek dayOfWeek)
|
||||||
|
{
|
||||||
|
dayOfWeek = DayOfWeek.Sunday;
|
||||||
|
return dayToken switch
|
||||||
|
{
|
||||||
|
"monday" => AssignDayOfWeek(DayOfWeek.Monday, out dayOfWeek),
|
||||||
|
"tuesday" => AssignDayOfWeek(DayOfWeek.Tuesday, out dayOfWeek),
|
||||||
|
"wednesday" => AssignDayOfWeek(DayOfWeek.Wednesday, out dayOfWeek),
|
||||||
|
"thursday" => AssignDayOfWeek(DayOfWeek.Thursday, out dayOfWeek),
|
||||||
|
"friday" => AssignDayOfWeek(DayOfWeek.Friday, out dayOfWeek),
|
||||||
|
"saturday" => AssignDayOfWeek(DayOfWeek.Saturday, out dayOfWeek),
|
||||||
|
"sunday" => AssignDayOfWeek(DayOfWeek.Sunday, out dayOfWeek),
|
||||||
|
_ => false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool AssignDayOfWeek(DayOfWeek value, out DayOfWeek target)
|
||||||
|
{
|
||||||
|
target = value;
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string? TryResolveWeatherConditionEntity(string transcript)
|
private static string? TryResolveWeatherConditionEntity(string transcript)
|
||||||
@@ -1830,6 +2335,10 @@ public sealed class JiboInteractionService(
|
|||||||
"when s your birthday",
|
"when s your birthday",
|
||||||
"what s your birthday",
|
"what s your birthday",
|
||||||
"what is your birthday",
|
"what is your birthday",
|
||||||
|
"when is your bday",
|
||||||
|
"when s your bday",
|
||||||
|
"what s your bday",
|
||||||
|
"what is your bday",
|
||||||
"when were you born",
|
"when were you born",
|
||||||
"what day is your birthday"))
|
"what day is your birthday"))
|
||||||
{
|
{
|
||||||
@@ -1837,6 +2346,7 @@ public sealed class JiboInteractionService(
|
|||||||
}
|
}
|
||||||
|
|
||||||
return (normalized.Contains("your birthday", StringComparison.Ordinal) ||
|
return (normalized.Contains("your birthday", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("your bday", StringComparison.Ordinal) ||
|
||||||
normalized.Contains("your birth date", StringComparison.Ordinal))
|
normalized.Contains("your birth date", StringComparison.Ordinal))
|
||||||
&& !normalized.Contains("my birthday", StringComparison.Ordinal);
|
&& !normalized.Contains("my birthday", StringComparison.Ordinal);
|
||||||
}
|
}
|
||||||
@@ -1889,6 +2399,11 @@ public sealed class JiboInteractionService(
|
|||||||
"what is my birthday",
|
"what is my birthday",
|
||||||
"what s my birthday",
|
"what s my birthday",
|
||||||
"what's my birthday",
|
"what's my birthday",
|
||||||
|
"when is my bday",
|
||||||
|
"when s my bday",
|
||||||
|
"what is my bday",
|
||||||
|
"what s my bday",
|
||||||
|
"what's my bday",
|
||||||
"do you remember my birthday");
|
"do you remember my birthday");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1900,13 +2415,15 @@ public sealed class JiboInteractionService(
|
|||||||
private static bool IsUserBirthdaySetAttempt(string loweredTranscript)
|
private static bool IsUserBirthdaySetAttempt(string loweredTranscript)
|
||||||
{
|
{
|
||||||
var normalized = NormalizeCommandPhrase(loweredTranscript);
|
var normalized = NormalizeCommandPhrase(loweredTranscript);
|
||||||
return normalized.Contains("my birthday is", StringComparison.Ordinal);
|
return normalized.Contains("my birthday is", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("my bday is", StringComparison.Ordinal);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool IsUserBirthdayRecallAttempt(string loweredTranscript)
|
private static bool IsUserBirthdayRecallAttempt(string loweredTranscript)
|
||||||
{
|
{
|
||||||
var normalized = NormalizeCommandPhrase(loweredTranscript);
|
var normalized = NormalizeCommandPhrase(loweredTranscript);
|
||||||
return normalized.Contains("my birthday", StringComparison.Ordinal) &&
|
return (normalized.Contains("my birthday", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains("my bday", StringComparison.Ordinal)) &&
|
||||||
(normalized.StartsWith("when", StringComparison.Ordinal) ||
|
(normalized.StartsWith("when", StringComparison.Ordinal) ||
|
||||||
normalized.StartsWith("what", StringComparison.Ordinal) ||
|
normalized.StartsWith("what", StringComparison.Ordinal) ||
|
||||||
normalized.StartsWith("tell me", StringComparison.Ordinal) ||
|
normalized.StartsWith("tell me", StringComparison.Ordinal) ||
|
||||||
@@ -1916,15 +2433,28 @@ public sealed class JiboInteractionService(
|
|||||||
private static string? TryExtractBirthdayFact(string transcript)
|
private static string? TryExtractBirthdayFact(string transcript)
|
||||||
{
|
{
|
||||||
var normalized = NormalizeCommandPhrase(transcript);
|
var normalized = NormalizeCommandPhrase(transcript);
|
||||||
var marker = "my birthday is ";
|
var markers = new[]
|
||||||
var markerIndex = normalized.IndexOf(marker, StringComparison.Ordinal);
|
|
||||||
if (markerIndex < 0)
|
|
||||||
{
|
{
|
||||||
return null;
|
"my birthday is ",
|
||||||
|
"my bday is "
|
||||||
|
};
|
||||||
|
|
||||||
|
foreach (var marker in markers)
|
||||||
|
{
|
||||||
|
var markerIndex = normalized.IndexOf(marker, StringComparison.Ordinal);
|
||||||
|
if (markerIndex < 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var value = normalized[(markerIndex + marker.Length)..].Trim();
|
||||||
|
if (!string.IsNullOrWhiteSpace(value))
|
||||||
|
{
|
||||||
|
return value;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var value = normalized[(markerIndex + marker.Length)..].Trim();
|
return null;
|
||||||
return string.IsNullOrWhiteSpace(value) ? null : value;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool IsPreferenceRecallQuestion(string loweredTranscript)
|
private static bool IsPreferenceRecallQuestion(string loweredTranscript)
|
||||||
@@ -2006,6 +2536,12 @@ public sealed class JiboInteractionService(
|
|||||||
var splitIndex = preferencePhrase.IndexOf(splitMarker, StringComparison.Ordinal);
|
var splitIndex = preferencePhrase.IndexOf(splitMarker, StringComparison.Ordinal);
|
||||||
if (splitIndex <= 0 || splitIndex >= preferencePhrase.Length - splitMarker.Length)
|
if (splitIndex <= 0 || splitIndex >= preferencePhrase.Length - splitMarker.Length)
|
||||||
{
|
{
|
||||||
|
var fallbackPreference = TryExtractPreferenceSetWithoutCopula(preferencePhrase);
|
||||||
|
if (fallbackPreference is not null)
|
||||||
|
{
|
||||||
|
return fallbackPreference;
|
||||||
|
}
|
||||||
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2042,6 +2578,38 @@ public sealed class JiboInteractionService(
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static (string Category, string Value)? TryExtractPreferenceSetWithoutCopula(string preferencePhrase)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(preferencePhrase))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var normalized = preferencePhrase.Trim();
|
||||||
|
if (normalized.Contains(" is ", StringComparison.Ordinal) ||
|
||||||
|
normalized.Contains(" are ", StringComparison.Ordinal) ||
|
||||||
|
normalized.EndsWith(" is", StringComparison.Ordinal) ||
|
||||||
|
normalized.EndsWith(" are", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||||
|
if (parts.Length < 2)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var category = parts[0];
|
||||||
|
var value = string.Join(' ', parts.Skip(1)).Trim();
|
||||||
|
if (string.IsNullOrWhiteSpace(category) || string.IsNullOrWhiteSpace(value))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (category, value);
|
||||||
|
}
|
||||||
|
|
||||||
private static bool IsImportantDateSetStatement(string loweredTranscript)
|
private static bool IsImportantDateSetStatement(string loweredTranscript)
|
||||||
{
|
{
|
||||||
return TryExtractImportantDateSet(loweredTranscript) is not null;
|
return TryExtractImportantDateSet(loweredTranscript) is not null;
|
||||||
@@ -2763,6 +3331,11 @@ public sealed class JiboInteractionService(
|
|||||||
|
|
||||||
private sealed record PizzaSignal(PersonalAffinity? Affinity);
|
private sealed record PizzaSignal(PersonalAffinity? Affinity);
|
||||||
|
|
||||||
|
private sealed record WeatherDateEntity(string? DateEntity, int ForecastDayOffset, string? ForecastLeadIn)
|
||||||
|
{
|
||||||
|
public static WeatherDateEntity None { get; } = new(null, 0, null);
|
||||||
|
}
|
||||||
|
|
||||||
private static readonly Regex SplitAlarmPattern = new(
|
private static readonly Regex SplitAlarmPattern = new(
|
||||||
@"\b(?<hour>\d{1,2}|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)(?:[:\s,-]+(?<minute>\d{2}|[a-z\-]+(?:\s+[a-z\-]+)?))?\s*(?<ampm>a[\s\.]*m\.?|p[\s\.]*m\.?)?\b",
|
@"\b(?<hour>\d{1,2}|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)(?:[:\s,-]+(?<minute>\d{2}|[a-z\-]+(?:\s+[a-z\-]+)?))?\s*(?<ampm>a[\s\.]*m\.?|p[\s\.]*m\.?)?\b",
|
||||||
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||||
@@ -2792,11 +3365,23 @@ public sealed class JiboInteractionService(
|
|||||||
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||||
|
|
||||||
private static readonly Regex WeatherLocationPattern = new(
|
private static readonly Regex WeatherLocationPattern = new(
|
||||||
@"\bin\s+(?<location>[a-z][a-z\s'\-]+)$",
|
@"\b(?:in|for|at)\s+(?<location>[a-z][a-z\s'\-]+)$",
|
||||||
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||||
|
|
||||||
private static readonly Regex WeatherLocationSuffixPattern = new(
|
private static readonly Regex WeatherLocationSuffixPattern = new(
|
||||||
@"\b(?:today|tonight|tomorrow|outside|right now|please|thanks)\b",
|
@"\b(?:today|tonight|tomorrow|day after tomorrow|outside|right now|please|thanks|this weekend|next weekend|the weekend|weekend|this week|next week|on monday|on tuesday|on wednesday|on thursday|on friday|on saturday|on sunday|this monday|this tuesday|this wednesday|this thursday|this friday|this saturday|this sunday|next monday|next tuesday|next wednesday|next thursday|next friday|next saturday|next sunday|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b",
|
||||||
|
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||||
|
|
||||||
|
private static readonly Regex WeatherConditionForecastPattern = new(
|
||||||
|
@"\bwill it be\s+(sunny|cloudy|windy|foggy|stormy|rainy|snowy|hail|hailing)\b",
|
||||||
|
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||||
|
|
||||||
|
private static readonly Regex WeatherTopicLocationPattern = new(
|
||||||
|
@"\b(?:weather|forecast|temperature|humidity)\b.*\b(?:in|for|at)\s+[a-z]",
|
||||||
|
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||||
|
|
||||||
|
private static readonly Regex WeatherDayOfWeekPattern = new(
|
||||||
|
@"\b(?<next>next\s+)?(?<this>this\s+)?(?:on\s+)?(?<day>monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b",
|
||||||
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||||
|
|
||||||
private static readonly PizzaMimPrompt[] PizzaMimPrompts =
|
private static readonly PizzaMimPrompt[] PizzaMimPrompts =
|
||||||
@@ -2820,6 +3405,16 @@ public sealed class JiboInteractionService(
|
|||||||
" are my favourite "
|
" are my favourite "
|
||||||
];
|
];
|
||||||
|
|
||||||
|
private static readonly string[] WeatherDateEntityKeys =
|
||||||
|
[
|
||||||
|
"date",
|
||||||
|
"sys.date",
|
||||||
|
"datetime",
|
||||||
|
"dateTime",
|
||||||
|
"date_time",
|
||||||
|
"day"
|
||||||
|
];
|
||||||
|
|
||||||
// Directly imported from Pegasus parser intent phrase families:
|
// Directly imported from Pegasus parser intent phrase families:
|
||||||
// userLikesThing / userDislikesThing / doesUserLikeThing / doesUserDislikeThing.
|
// userLikesThing / userDislikesThing / doesUserLikeThing / doesUserDislikeThing.
|
||||||
private static readonly (string Prefix, PersonalAffinity Affinity)[] PegasusUserAffinitySetPrefixes =
|
private static readonly (string Prefix, PersonalAffinity Affinity)[] PegasusUserAffinitySetPrefixes =
|
||||||
@@ -2893,6 +3488,8 @@ public sealed class JiboInteractionService(
|
|||||||
"our neighbourhood"
|
"our neighbourhood"
|
||||||
};
|
};
|
||||||
|
|
||||||
|
private const int MaxWeatherForecastDayOffset = 5;
|
||||||
|
|
||||||
private static readonly (string Phrase, string Station)[] RadioGenreAliases =
|
private static readonly (string Phrase, string Station)[] RadioGenreAliases =
|
||||||
[
|
[
|
||||||
("country music", "Country"),
|
("country music", "Country"),
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ public sealed class JiboWebSocketService(
|
|||||||
var replies = await turnFinalizationService.HandleBinaryAudioAsync(session, envelope, cancellationToken);
|
var replies = await turnFinalizationService.HandleBinaryAudioAsync(session, envelope, cancellationToken);
|
||||||
await telemetrySink.RecordTurnEventAsync(envelope, session, "binary_audio_received", new Dictionary<string, object?>
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "binary_audio_received", new Dictionary<string, object?>
|
||||||
{
|
{
|
||||||
["bytes"] = envelope.Binary?.Length ?? 0
|
["bytes"] = envelope.Binary?.Length ?? 0,
|
||||||
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
return replies;
|
return replies;
|
||||||
}
|
}
|
||||||
@@ -33,6 +34,8 @@ public sealed class JiboWebSocketService(
|
|||||||
var parsedType = ReadMessageType(envelope.Text);
|
var parsedType = ReadMessageType(envelope.Text);
|
||||||
session.LastMessageType = parsedType;
|
session.LastMessageType = parsedType;
|
||||||
var containsInlineTurnPayload = parsedType == "LISTEN" && ContainsInlineTurnPayload(envelope.Text);
|
var containsInlineTurnPayload = parsedType == "LISTEN" && ContainsInlineTurnPayload(envelope.Text);
|
||||||
|
var staleListenRecovered = false;
|
||||||
|
var staleListenAgeMs = 0;
|
||||||
if (parsedType == "LISTEN" &&
|
if (parsedType == "LISTEN" &&
|
||||||
!containsInlineTurnPayload &&
|
!containsInlineTurnPayload &&
|
||||||
WebSocketTurnFinalizationService.ShouldIgnoreLateListenSetup(session, envelope.Text))
|
WebSocketTurnFinalizationService.ShouldIgnoreLateListenSetup(session, envelope.Text))
|
||||||
@@ -57,6 +60,19 @@ public sealed class JiboWebSocketService(
|
|||||||
return replies;
|
return replies;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (parsedType == "LISTEN" &&
|
||||||
|
!containsInlineTurnPayload &&
|
||||||
|
WebSocketTurnFinalizationService.TryRecoverStalePendingListen(session, out staleListenAgeMs))
|
||||||
|
{
|
||||||
|
staleListenRecovered = true;
|
||||||
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "glsm_stale_listen_recovered", new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["staleAgeMs"] = staleListenAgeMs,
|
||||||
|
["transID"] = session.TurnState.TransId,
|
||||||
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
||||||
|
}, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
|
WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
|
||||||
|
|
||||||
switch (parsedType)
|
switch (parsedType)
|
||||||
@@ -66,7 +82,8 @@ public sealed class JiboWebSocketService(
|
|||||||
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
|
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
|
||||||
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
|
||||||
{
|
{
|
||||||
["transID"] = session.TurnState.TransId
|
["transID"] = session.TurnState.TransId,
|
||||||
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
return replies;
|
return replies;
|
||||||
}
|
}
|
||||||
@@ -80,7 +97,10 @@ public sealed class JiboWebSocketService(
|
|||||||
["messageType"] = parsedType,
|
["messageType"] = parsedType,
|
||||||
["replyCount"] = replies.Count,
|
["replyCount"] = replies.Count,
|
||||||
["transcript"] = session.LastTranscript,
|
["transcript"] = session.LastTranscript,
|
||||||
["intent"] = session.LastIntent
|
["intent"] = session.LastIntent,
|
||||||
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session),
|
||||||
|
["staleListenRecovered"] = staleListenRecovered,
|
||||||
|
["staleListenAgeMs"] = staleListenAgeMs
|
||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
return replies;
|
return replies;
|
||||||
}
|
}
|
||||||
@@ -92,7 +112,8 @@ public sealed class JiboWebSocketService(
|
|||||||
["messageType"] = parsedType,
|
["messageType"] = parsedType,
|
||||||
["replyCount"] = replies.Count,
|
["replyCount"] = replies.Count,
|
||||||
["transcript"] = session.LastTranscript,
|
["transcript"] = session.LastTranscript,
|
||||||
["intent"] = session.LastIntent
|
["intent"] = session.LastIntent,
|
||||||
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
return replies;
|
return replies;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -795,19 +795,20 @@ public sealed class ResponsePlanToSocketMessagesMapper
|
|||||||
var promptId = ReadPayloadString(skillPayload, "prompt_id") ?? "RUNTIME_PROMPT";
|
var promptId = ReadPayloadString(skillPayload, "prompt_id") ?? "RUNTIME_PROMPT";
|
||||||
var promptSubCategory = ReadPayloadString(skillPayload, "prompt_sub_category") ?? "AN";
|
var promptSubCategory = ReadPayloadString(skillPayload, "prompt_sub_category") ?? "AN";
|
||||||
var listenContexts = ReadPayloadStringArray(skillPayload, "listen_contexts");
|
var listenContexts = ReadPayloadStringArray(skillPayload, "listen_contexts");
|
||||||
|
var playConfig = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["esml"] = esml,
|
||||||
|
["meta"] = new
|
||||||
|
{
|
||||||
|
prompt_id = promptId,
|
||||||
|
prompt_sub_category = promptSubCategory,
|
||||||
|
mim_id = mimId,
|
||||||
|
mim_type = mimType
|
||||||
|
}
|
||||||
|
};
|
||||||
var jcpConfig = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
var jcpConfig = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
{
|
{
|
||||||
["play"] = new
|
["play"] = playConfig
|
||||||
{
|
|
||||||
esml,
|
|
||||||
meta = new
|
|
||||||
{
|
|
||||||
prompt_id = promptId,
|
|
||||||
prompt_sub_category = promptSubCategory,
|
|
||||||
mim_id = mimId,
|
|
||||||
mim_type = mimType
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
if (listenContexts.Count > 0)
|
if (listenContexts.Count > 0)
|
||||||
@@ -820,6 +821,47 @@ public sealed class ResponsePlanToSocketMessagesMapper
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var weatherHiLoView = BuildWeatherHiLoView(skillPayload);
|
||||||
|
if (weatherHiLoView is not null)
|
||||||
|
{
|
||||||
|
var resolvedGuiConfig = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["type"] = "Javascript",
|
||||||
|
["data"] = weatherHiLoView,
|
||||||
|
["pause"] = true
|
||||||
|
};
|
||||||
|
|
||||||
|
var legacyGuiConfig = new
|
||||||
|
{
|
||||||
|
type = "Javascript",
|
||||||
|
data = "views.weatherHiLo",
|
||||||
|
pause = true
|
||||||
|
};
|
||||||
|
|
||||||
|
jcpConfig["gui"] = legacyGuiConfig;
|
||||||
|
jcpConfig["display"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["view"] = resolvedGuiConfig
|
||||||
|
};
|
||||||
|
|
||||||
|
playConfig["gui"] = resolvedGuiConfig;
|
||||||
|
playConfig["no_matches_for_gui"] = 0;
|
||||||
|
playConfig["no_inputs_for_gui"] = 0;
|
||||||
|
|
||||||
|
var weatherViews = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["weatherHiLo"] = weatherHiLoView
|
||||||
|
};
|
||||||
|
jcpConfig["views"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["weatherHiLo"] = weatherHiLoView
|
||||||
|
};
|
||||||
|
jcpConfig["local"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["views"] = weatherViews
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
return new
|
return new
|
||||||
{
|
{
|
||||||
type = "SKILL_ACTION",
|
type = "SKILL_ACTION",
|
||||||
@@ -1070,6 +1112,238 @@ public sealed class ResponsePlanToSocketMessagesMapper
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static object? BuildWeatherHiLoView(IDictionary<string, object?>? payload)
|
||||||
|
{
|
||||||
|
if (!TryReadPayloadBool(payload, "weather_view_enabled"))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.Equals(
|
||||||
|
ReadPayloadString(payload, "weather_view_kind"),
|
||||||
|
"weatherHiLo",
|
||||||
|
StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var icon = ReadPayloadString(payload, "weather_icon");
|
||||||
|
var unit = ReadPayloadString(payload, "weather_unit") ?? "F";
|
||||||
|
var theme = ReadPayloadString(payload, "weather_theme") ?? "Normal";
|
||||||
|
var high = TryReadPayloadInt(payload, "weather_high");
|
||||||
|
var low = TryReadPayloadInt(payload, "weather_low");
|
||||||
|
if (string.IsNullOrWhiteSpace(icon) || high is null || low is null)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var hiNumX = GetTemperatureLabelXPosition(370, high.Value);
|
||||||
|
var hiUnitX = GetTemperatureLabelXPosition(360, high.Value);
|
||||||
|
var loNumX = GetTemperatureLabelXPosition(1110, low.Value);
|
||||||
|
var loUnitX = GetTemperatureLabelXPosition(1100, low.Value);
|
||||||
|
|
||||||
|
return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["viewConfig"] = new
|
||||||
|
{
|
||||||
|
type = "View",
|
||||||
|
id = "weatherTempView",
|
||||||
|
category = "gui"
|
||||||
|
},
|
||||||
|
["open"] = new
|
||||||
|
{
|
||||||
|
transitionOpen = "trans_in",
|
||||||
|
removeAll = true
|
||||||
|
},
|
||||||
|
["defaultSelect"] = new
|
||||||
|
{
|
||||||
|
transitionClose = "trans_out",
|
||||||
|
removeAll = true,
|
||||||
|
leaveEmpty = false
|
||||||
|
},
|
||||||
|
["componentConfigs"] = new object[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "tempBGClip",
|
||||||
|
type = "Clip",
|
||||||
|
assets = new object[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "tempBG",
|
||||||
|
src = $"assets/personal-report-skill/weather/bg/temp{theme}_v01.crn",
|
||||||
|
type = "texture"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
position = new { x = 36, y = 0 }
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "iconClip",
|
||||||
|
type = "Clip",
|
||||||
|
assets = new object[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "icon",
|
||||||
|
src = $"assets/personal-report-skill/weather/icons/{icon}_v01.crn",
|
||||||
|
type = "texture"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
position = new { x = 475, y = 195 }
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "hiNumLabel",
|
||||||
|
type = "Label",
|
||||||
|
text = $"{high.Value}\u00B0",
|
||||||
|
style = new
|
||||||
|
{
|
||||||
|
fontSize = "160",
|
||||||
|
fontFamily = "Proxima Nova Soft",
|
||||||
|
fontWeight = "bold",
|
||||||
|
fill = "#FFFFFF",
|
||||||
|
align = "center"
|
||||||
|
},
|
||||||
|
position = new { x = hiNumX, y = 430 },
|
||||||
|
targetAnchor = new { x = 1, y = 1 }
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "hiUnitLabel",
|
||||||
|
type = "Label",
|
||||||
|
text = unit,
|
||||||
|
style = new
|
||||||
|
{
|
||||||
|
fontSize = "90",
|
||||||
|
fontFamily = "Proxima Nova Soft",
|
||||||
|
fontWeight = "bold",
|
||||||
|
fill = "#FFFFFF",
|
||||||
|
align = "center"
|
||||||
|
},
|
||||||
|
position = new { x = hiUnitX, y = 418 },
|
||||||
|
targetAnchor = new { x = 0, y = 1 }
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "loNumLabel",
|
||||||
|
type = "Label",
|
||||||
|
text = $"{low.Value}\u00B0",
|
||||||
|
style = new
|
||||||
|
{
|
||||||
|
fontSize = "160",
|
||||||
|
fontFamily = "Proxima Nova Soft",
|
||||||
|
fontWeight = "bold",
|
||||||
|
fill = "#FFFFFF",
|
||||||
|
align = "center"
|
||||||
|
},
|
||||||
|
position = new { x = loNumX, y = 430 },
|
||||||
|
targetAnchor = new { x = 1, y = 1 }
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "loUnitLabel",
|
||||||
|
type = "Label",
|
||||||
|
text = unit,
|
||||||
|
style = new
|
||||||
|
{
|
||||||
|
fontSize = "90",
|
||||||
|
fontFamily = "Proxima Nova Soft",
|
||||||
|
fontWeight = "bold",
|
||||||
|
fill = "#FFFFFF",
|
||||||
|
align = "center"
|
||||||
|
},
|
||||||
|
position = new { x = loUnitX, y = 418 },
|
||||||
|
targetAnchor = new { x = 0, y = 1 }
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "hiTextLabel",
|
||||||
|
type = "Label",
|
||||||
|
text = "Hi",
|
||||||
|
style = new
|
||||||
|
{
|
||||||
|
fontSize = "60",
|
||||||
|
fontFamily = "Proxima Nova Light",
|
||||||
|
fill = "#FFFFFF",
|
||||||
|
align = "center"
|
||||||
|
},
|
||||||
|
position = new { x = 280, y = 496 },
|
||||||
|
targetAnchor = new { x = 0.5, y = 1 }
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "loTextLabel",
|
||||||
|
type = "Label",
|
||||||
|
text = "Lo",
|
||||||
|
style = new
|
||||||
|
{
|
||||||
|
fontSize = "60",
|
||||||
|
fontFamily = "Proxima Nova Light",
|
||||||
|
fill = "#FFFFFF",
|
||||||
|
align = "center"
|
||||||
|
},
|
||||||
|
position = new { x = 990, y = 496 },
|
||||||
|
targetAnchor = new { x = 0.5, y = 1 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int GetTemperatureLabelXPosition(int baseX, int temperature)
|
||||||
|
{
|
||||||
|
const int xOffset = 70;
|
||||||
|
if (temperature < -9 || temperature > 99)
|
||||||
|
{
|
||||||
|
return baseX + xOffset;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (temperature is >= 0 and < 10)
|
||||||
|
{
|
||||||
|
return baseX - xOffset;
|
||||||
|
}
|
||||||
|
|
||||||
|
return baseX;
|
||||||
|
}
|
||||||
|
private static int? TryReadPayloadInt(IDictionary<string, object?>? payload, string key)
|
||||||
|
{
|
||||||
|
if (payload is null || !payload.TryGetValue(key, out var value) || value is null)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return value switch
|
||||||
|
{
|
||||||
|
int number => number,
|
||||||
|
long number when number <= int.MaxValue && number >= int.MinValue => (int)number,
|
||||||
|
double number => (int)Math.Round(number, MidpointRounding.AwayFromZero),
|
||||||
|
float number => (int)Math.Round(number, MidpointRounding.AwayFromZero),
|
||||||
|
string text when int.TryParse(text, out var parsed) => parsed,
|
||||||
|
JsonElement { ValueKind: JsonValueKind.Number } jsonNumber when jsonNumber.TryGetInt32(out var parsed) => parsed,
|
||||||
|
JsonElement jsonText when jsonText.ValueKind == JsonValueKind.String && int.TryParse(jsonText.GetString(), out var parsed) => parsed,
|
||||||
|
_ => null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool TryReadPayloadBool(IDictionary<string, object?>? payload, string key)
|
||||||
|
{
|
||||||
|
if (payload is null || !payload.TryGetValue(key, out var value) || value is null)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return value switch
|
||||||
|
{
|
||||||
|
bool flag => flag,
|
||||||
|
string text when bool.TryParse(text, out var parsed) => parsed,
|
||||||
|
JsonElement { ValueKind: JsonValueKind.True } => true,
|
||||||
|
JsonElement { ValueKind: JsonValueKind.False } => false,
|
||||||
|
JsonElement jsonText when jsonText.ValueKind == JsonValueKind.String && bool.TryParse(jsonText.GetString(), out var parsed) => parsed,
|
||||||
|
_ => false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private static string CreateHubMessageId()
|
private static string CreateHubMessageId()
|
||||||
{
|
{
|
||||||
return $"mid-{Guid.NewGuid()}";
|
return $"mid-{Guid.NewGuid()}";
|
||||||
@@ -1082,3 +1356,4 @@ public sealed class ResponsePlanToSocketMessagesMapper
|
|||||||
|
|
||||||
public sealed record SocketReplyPlan(string Text, int DelayMs = 0);
|
public sealed record SocketReplyPlan(string Text, int DelayMs = 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,9 +14,11 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
{
|
{
|
||||||
private const int AutoFinalizeMinBufferedAudioBytes = 15000;
|
private const int AutoFinalizeMinBufferedAudioBytes = 15000;
|
||||||
private const int AutoFinalizeMinBufferedAudioChunks = 5;
|
private const int AutoFinalizeMinBufferedAudioChunks = 5;
|
||||||
|
private const string GlsmPhaseMetadataKey = "glsmPhase";
|
||||||
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1800);
|
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1800);
|
||||||
private static readonly TimeSpan AutoFinalizeMissingTranscriptFallbackAge = TimeSpan.FromMilliseconds(4200);
|
private static readonly TimeSpan AutoFinalizeMissingTranscriptFallbackAge = TimeSpan.FromMilliseconds(4200);
|
||||||
private static readonly TimeSpan AutoFinalizeContinuationDeferralMaxAge = TimeSpan.FromMilliseconds(3600);
|
private static readonly TimeSpan AutoFinalizeContinuationDeferralMaxAge = TimeSpan.FromMilliseconds(3600);
|
||||||
|
private static readonly TimeSpan StaleListenSetupRecoveryAge = TimeSpan.FromSeconds(9);
|
||||||
private const int AutoFinalizeContinuationDeferralMaxAttempts = 2;
|
private const int AutoFinalizeContinuationDeferralMaxAttempts = 2;
|
||||||
private static readonly HashSet<string> PegasusAffinityContinuationStems = new(StringComparer.Ordinal)
|
private static readonly HashSet<string> PegasusAffinityContinuationStems = new(StringComparer.Ordinal)
|
||||||
{
|
{
|
||||||
@@ -61,54 +63,61 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
WebSocketMessageEnvelope envelope,
|
WebSocketMessageEnvelope envelope,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var turnState = session.TurnState;
|
try
|
||||||
var ignoreLateAudio = ShouldIgnoreLateAudio(session);
|
|
||||||
var ignoreAudioWithoutListen = ShouldIgnoreAudioWithoutListen(turnState);
|
|
||||||
if (ignoreLateAudio || ignoreAudioWithoutListen)
|
|
||||||
{
|
{
|
||||||
await sink.RecordTurnDiagnosticAsync("binary_audio_ignored", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
|
var turnState = session.TurnState;
|
||||||
|
var ignoreLateAudio = ShouldIgnoreLateAudio(session);
|
||||||
|
var ignoreAudioWithoutListen = ShouldIgnoreAudioWithoutListen(turnState);
|
||||||
|
if (ignoreLateAudio || ignoreAudioWithoutListen)
|
||||||
|
{
|
||||||
|
await sink.RecordTurnDiagnosticAsync("binary_audio_ignored", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["ignored"] = true,
|
||||||
|
["ignoreLateAudio"] = ignoreLateAudio,
|
||||||
|
["ignoreAudioWithoutListen"] = ignoreAudioWithoutListen,
|
||||||
|
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
|
||||||
|
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
|
||||||
|
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
|
||||||
|
["sawListen"] = turnState.SawListen,
|
||||||
|
["sawContext"] = turnState.SawContext
|
||||||
|
}), cancellationToken);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
session.LastMessageType = "BINARY_AUDIO";
|
||||||
|
turnState.FirstAudioReceivedUtc ??= DateTimeOffset.UtcNow;
|
||||||
|
turnState.BufferedAudioChunkCount += 1;
|
||||||
|
turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0;
|
||||||
|
if (envelope.Binary is { Length: > 0 })
|
||||||
|
{
|
||||||
|
turnState.BufferedAudioFrames.Add([.. envelope.Binary]);
|
||||||
|
}
|
||||||
|
turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow;
|
||||||
|
turnState.AwaitingTurnCompletion = true;
|
||||||
|
session.Metadata["lastAudioBytes"] = envelope.Binary?.Length ?? 0;
|
||||||
|
await sink.RecordTurnDiagnosticAsync("binary_audio_received", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
|
||||||
{
|
{
|
||||||
["ignored"] = true,
|
|
||||||
["ignoreLateAudio"] = ignoreLateAudio,
|
|
||||||
["ignoreAudioWithoutListen"] = ignoreAudioWithoutListen,
|
|
||||||
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
|
|
||||||
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
|
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
|
||||||
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
|
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
|
||||||
|
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
|
||||||
["sawListen"] = turnState.SawListen,
|
["sawListen"] = turnState.SawListen,
|
||||||
["sawContext"] = turnState.SawContext
|
["sawContext"] = turnState.SawContext,
|
||||||
|
["listenRules"] = turnState.ListenRules,
|
||||||
|
["listenAsrHints"] = turnState.ListenAsrHints,
|
||||||
|
["yesNoRule"] = turnState.ListenRules.FirstOrDefault(IsConstrainedYesNoRule)
|
||||||
}), cancellationToken);
|
}), cancellationToken);
|
||||||
|
|
||||||
|
if (ShouldAutoFinalize(session))
|
||||||
|
{
|
||||||
|
return await FinalizeTurnAsync(session, envelope, "AUTO_FINALIZE", allowFallbackOnMissingTranscript: true, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
session.LastMessageType = "BINARY_AUDIO";
|
|
||||||
turnState.FirstAudioReceivedUtc ??= DateTimeOffset.UtcNow;
|
|
||||||
turnState.BufferedAudioChunkCount += 1;
|
|
||||||
turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0;
|
|
||||||
if (envelope.Binary is { Length: > 0 })
|
|
||||||
{
|
{
|
||||||
turnState.BufferedAudioFrames.Add([.. envelope.Binary]);
|
await TrackGlsmPhaseAsync(session, envelope, "binary_audio", cancellationToken);
|
||||||
}
|
}
|
||||||
turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow;
|
|
||||||
turnState.AwaitingTurnCompletion = true;
|
|
||||||
session.Metadata["lastAudioBytes"] = envelope.Binary?.Length ?? 0;
|
|
||||||
await sink.RecordTurnDiagnosticAsync("binary_audio_received", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
|
|
||||||
{
|
|
||||||
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
|
|
||||||
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
|
|
||||||
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
|
|
||||||
["sawListen"] = turnState.SawListen,
|
|
||||||
["sawContext"] = turnState.SawContext,
|
|
||||||
["listenRules"] = turnState.ListenRules,
|
|
||||||
["listenAsrHints"] = turnState.ListenAsrHints,
|
|
||||||
["yesNoRule"] = turnState.ListenRules.FirstOrDefault(IsConstrainedYesNoRule)
|
|
||||||
}), cancellationToken);
|
|
||||||
|
|
||||||
if (ShouldAutoFinalize(session))
|
|
||||||
{
|
|
||||||
return await FinalizeTurnAsync(session, envelope, "AUTO_FINALIZE", allowFallbackOnMissingTranscript: true, cancellationToken);
|
|
||||||
}
|
|
||||||
|
|
||||||
return [];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<IReadOnlyList<WebSocketReply>> HandleContextAsync(
|
public async Task<IReadOnlyList<WebSocketReply>> HandleContextAsync(
|
||||||
@@ -116,34 +125,40 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
WebSocketMessageEnvelope envelope,
|
WebSocketMessageEnvelope envelope,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var turnState = session.TurnState;
|
try
|
||||||
turnState.SawContext = true;
|
|
||||||
turnState.ContextPayload = ExtractDataPayload(envelope.Text);
|
|
||||||
session.Metadata["context"] = turnState.ContextPayload;
|
|
||||||
|
|
||||||
if (TryReadContextProperty(envelope.Text, "audioTranscriptHint", out var transcriptHint) &&
|
|
||||||
!string.IsNullOrWhiteSpace(transcriptHint))
|
|
||||||
{
|
{
|
||||||
turnState.AudioTranscriptHint = transcriptHint;
|
var turnState = session.TurnState;
|
||||||
session.Metadata["audioTranscriptHint"] = transcriptHint;
|
turnState.SawContext = true;
|
||||||
}
|
turnState.ContextPayload = ExtractDataPayload(envelope.Text);
|
||||||
|
session.Metadata["context"] = turnState.ContextPayload;
|
||||||
|
|
||||||
|
if (TryReadContextProperty(envelope.Text, "audioTranscriptHint", out var transcriptHint) &&
|
||||||
|
!string.IsNullOrWhiteSpace(transcriptHint))
|
||||||
|
{
|
||||||
|
turnState.AudioTranscriptHint = transcriptHint;
|
||||||
|
session.Metadata["audioTranscriptHint"] = transcriptHint;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ShouldIgnorePassiveLocalSkillContext(session, envelope.Text))
|
||||||
|
{
|
||||||
|
turnState.AwaitingTurnCompletion = false;
|
||||||
|
turnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
||||||
|
ResetBufferedAudio(session);
|
||||||
|
ClearListenTracking(turnState);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ShouldAutoFinalize(session))
|
||||||
|
{
|
||||||
|
return await FinalizeTurnAsync(session, envelope, "AUTO_FINALIZE", allowFallbackOnMissingTranscript: true, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
if (ShouldIgnorePassiveLocalSkillContext(session, envelope.Text))
|
|
||||||
{
|
|
||||||
turnState.AwaitingTurnCompletion = false;
|
|
||||||
turnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
|
||||||
ResetBufferedAudio(session);
|
|
||||||
turnState.SawListen = false;
|
|
||||||
turnState.SawContext = false;
|
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
if (ShouldAutoFinalize(session))
|
|
||||||
{
|
{
|
||||||
return await FinalizeTurnAsync(session, envelope, "AUTO_FINALIZE", allowFallbackOnMissingTranscript: true, cancellationToken);
|
await TrackGlsmPhaseAsync(session, envelope, "context", cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
return [];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<IReadOnlyList<WebSocketReply>> HandleTurnAsync(
|
public async Task<IReadOnlyList<WebSocketReply>> HandleTurnAsync(
|
||||||
@@ -167,8 +182,8 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
session.TurnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
session.TurnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
||||||
session.FollowUpExpiresUtc = null;
|
session.FollowUpExpiresUtc = null;
|
||||||
ResetBufferedAudio(session);
|
ResetBufferedAudio(session);
|
||||||
session.TurnState.SawListen = false;
|
ClearListenTracking(session.TurnState);
|
||||||
session.TurnState.SawContext = false;
|
UpdateGlsmPhaseMarker(session);
|
||||||
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
|
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
|
||||||
session.TurnState.TransId ?? session.LastTransId ?? string.Empty,
|
session.TurnState.TransId ?? session.LastTransId ?? string.Empty,
|
||||||
session.TurnState.ListenRules,
|
session.TurnState.ListenRules,
|
||||||
@@ -181,6 +196,8 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
}
|
}
|
||||||
|
|
||||||
session.TurnState.AwaitingTurnCompletion = true;
|
session.TurnState.AwaitingTurnCompletion = true;
|
||||||
|
session.TurnState.ListenOpenedUtc ??= DateTimeOffset.UtcNow;
|
||||||
|
UpdateGlsmPhaseMarker(session);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,6 +292,7 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
string.Equals(type.GetString(), "LISTEN", StringComparison.OrdinalIgnoreCase))
|
string.Equals(type.GetString(), "LISTEN", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
turnState.SawListen = true;
|
turnState.SawListen = true;
|
||||||
|
turnState.ListenOpenedUtc ??= DateTimeOffset.UtcNow;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (root.TryGetProperty("transID", out var transId) && transId.ValueKind == JsonValueKind.String)
|
if (root.TryGetProperty("transID", out var transId) && transId.ValueKind == JsonValueKind.String)
|
||||||
@@ -351,6 +369,7 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
turnState.TransId = transId;
|
turnState.TransId = transId;
|
||||||
turnState.ContextPayload = null;
|
turnState.ContextPayload = null;
|
||||||
turnState.AudioTranscriptHint = null;
|
turnState.AudioTranscriptHint = null;
|
||||||
|
turnState.ListenOpenedUtc = null;
|
||||||
turnState.LastSttError = null;
|
turnState.LastSttError = null;
|
||||||
turnState.LastSttErrorUtc = null;
|
turnState.LastSttErrorUtc = null;
|
||||||
turnState.FirstAudioReceivedUtc = null;
|
turnState.FirstAudioReceivedUtc = null;
|
||||||
@@ -376,36 +395,37 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
bool allowFallbackOnMissingTranscript,
|
bool allowFallbackOnMissingTranscript,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var turn = ProtocolToTurnContextMapper.MapListenMessage(envelope, session, messageType);
|
try
|
||||||
var turnState = session.TurnState;
|
|
||||||
if (IsYesNoTurn(turn) || ReadPrimaryYesNoRule(turn) is not null)
|
|
||||||
{
|
{
|
||||||
await sink.RecordTurnDiagnosticAsync("yes_no_turn_received", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
|
var turn = ProtocolToTurnContextMapper.MapListenMessage(envelope, session, messageType);
|
||||||
|
var turnState = session.TurnState;
|
||||||
|
if (IsYesNoTurn(turn) || ReadPrimaryYesNoRule(turn) is not null)
|
||||||
{
|
{
|
||||||
["messageType"] = messageType,
|
await sink.RecordTurnDiagnosticAsync("yes_no_turn_received", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
|
||||||
["listenRules"] = ReadRules(turn, "listenRules").ToArray(),
|
{
|
||||||
["clientRules"] = ReadRules(turn, "clientRules").ToArray(),
|
["messageType"] = messageType,
|
||||||
["listenAsrHints"] = ReadRules(turn, "listenAsrHints").ToArray(),
|
["listenRules"] = ReadRules(turn, "listenRules").ToArray(),
|
||||||
["yesNoRule"] = ReadPrimaryYesNoRule(turn),
|
["clientRules"] = ReadRules(turn, "clientRules").ToArray(),
|
||||||
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
|
["listenAsrHints"] = ReadRules(turn, "listenAsrHints").ToArray(),
|
||||||
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
|
["yesNoRule"] = ReadPrimaryYesNoRule(turn),
|
||||||
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
|
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
|
||||||
["sawListen"] = turnState.SawListen,
|
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
|
||||||
["sawContext"] = turnState.SawContext,
|
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
|
||||||
["followUpOpen"] = session.FollowUpOpen,
|
["sawListen"] = turnState.SawListen,
|
||||||
["followUpExpiresUtc"] = session.FollowUpExpiresUtc
|
["sawContext"] = turnState.SawContext,
|
||||||
}), cancellationToken);
|
["followUpOpen"] = session.FollowUpOpen,
|
||||||
}
|
["followUpExpiresUtc"] = session.FollowUpExpiresUtc
|
||||||
if (ShouldIgnoreBlankAudioHotphraseTurn(turn))
|
}), cancellationToken);
|
||||||
{
|
}
|
||||||
session.TurnState.AwaitingTurnCompletion = false;
|
if (ShouldIgnoreBlankAudioHotphraseTurn(turn))
|
||||||
session.TurnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
{
|
||||||
session.FollowUpExpiresUtc = null;
|
session.TurnState.AwaitingTurnCompletion = false;
|
||||||
ResetBufferedAudio(session);
|
session.TurnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
||||||
session.TurnState.SawListen = false;
|
session.FollowUpExpiresUtc = null;
|
||||||
session.TurnState.SawContext = false;
|
ResetBufferedAudio(session);
|
||||||
return [];
|
ClearListenTracking(session.TurnState);
|
||||||
}
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
var finalizedTurn = await ResolveTranscriptAsync(turn, session, cancellationToken);
|
var finalizedTurn = await ResolveTranscriptAsync(turn, session, cancellationToken);
|
||||||
if (!IsTranscriptUsable(finalizedTurn))
|
if (!IsTranscriptUsable(finalizedTurn))
|
||||||
@@ -445,8 +465,7 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
turnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
turnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
|
||||||
session.FollowUpExpiresUtc = null;
|
session.FollowUpExpiresUtc = null;
|
||||||
ResetBufferedAudio(session);
|
ResetBufferedAudio(session);
|
||||||
turnState.SawListen = false;
|
ClearListenTracking(turnState);
|
||||||
turnState.SawContext = false;
|
|
||||||
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
|
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
|
||||||
turnState.TransId ?? session.LastTransId ?? string.Empty,
|
turnState.TransId ?? session.LastTransId ?? string.Empty,
|
||||||
turnState.ListenRules,
|
turnState.ListenRules,
|
||||||
@@ -483,8 +502,7 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
var localRule = ReadPrimaryNoInputRule(finalizedTurn);
|
var localRule = ReadPrimaryNoInputRule(finalizedTurn);
|
||||||
var noInputReplies = BuildLocalNoInputReplies(session, turnState, localRule);
|
var noInputReplies = BuildLocalNoInputReplies(session, turnState, localRule);
|
||||||
ResetBufferedAudio(session);
|
ResetBufferedAudio(session);
|
||||||
turnState.SawListen = false;
|
ClearListenTracking(turnState);
|
||||||
turnState.SawContext = false;
|
|
||||||
return noInputReplies;
|
return noInputReplies;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -545,8 +563,7 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
|
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
|
||||||
.ToArray();
|
.ToArray();
|
||||||
ResetBufferedAudio(session);
|
ResetBufferedAudio(session);
|
||||||
turnState.SawListen = false;
|
ClearListenTracking(turnState);
|
||||||
turnState.SawContext = false;
|
|
||||||
return fallbackReplies;
|
return fallbackReplies;
|
||||||
}
|
}
|
||||||
case true when
|
case true when
|
||||||
@@ -678,10 +695,14 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
}), cancellationToken);
|
}), cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
ResetBufferedAudio(session);
|
ResetBufferedAudio(session);
|
||||||
turnState.SawListen = false;
|
ClearListenTracking(turnState);
|
||||||
turnState.SawContext = false;
|
return replies;
|
||||||
return replies;
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await TrackGlsmPhaseAsync(session, envelope, $"finalize:{messageType}", cancellationToken);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool ShouldAutoFinalize(CloudSession session)
|
private static bool ShouldAutoFinalize(CloudSession session)
|
||||||
@@ -708,6 +729,58 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
return ShouldIgnoreLateAudio(session) && IsHotphraseLaunchListenSetup(text);
|
return ShouldIgnoreLateAudio(session) && IsHotphraseLaunchListenSetup(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static bool TryRecoverStalePendingListen(CloudSession session, out int staleAgeMs)
|
||||||
|
{
|
||||||
|
staleAgeMs = 0;
|
||||||
|
var turnState = session.TurnState;
|
||||||
|
if (!turnState.AwaitingTurnCompletion ||
|
||||||
|
!turnState.SawListen ||
|
||||||
|
turnState.SawContext ||
|
||||||
|
turnState.BufferedAudioBytes > 0 ||
|
||||||
|
!turnState.ListenOpenedUtc.HasValue)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var age = DateTimeOffset.UtcNow - turnState.ListenOpenedUtc.Value;
|
||||||
|
if (age < StaleListenSetupRecoveryAge)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
staleAgeMs = (int)age.TotalMilliseconds;
|
||||||
|
turnState.AwaitingTurnCompletion = false;
|
||||||
|
ResetBufferedAudio(session);
|
||||||
|
ClearListenTracking(turnState);
|
||||||
|
turnState.ListenHotphrase = false;
|
||||||
|
turnState.HotphraseEmptyTurnCount = 0;
|
||||||
|
UpdateGlsmPhaseMarker(session);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string ResolveGlsmPhase(CloudSession session)
|
||||||
|
{
|
||||||
|
var turnState = session.TurnState;
|
||||||
|
if (!turnState.AwaitingTurnCompletion)
|
||||||
|
{
|
||||||
|
return session.FollowUpOpen ? "DISPATCH_DIALOG" : "PROCESS_LISTENER_QUEUE";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (turnState.SawListen && !turnState.SawContext && turnState.BufferedAudioBytes == 0)
|
||||||
|
{
|
||||||
|
return "HJ_LISTENING";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (turnState.SawListen && turnState.SawContext && turnState.BufferedAudioBytes == 0)
|
||||||
|
{
|
||||||
|
return "LISTENING";
|
||||||
|
}
|
||||||
|
|
||||||
|
return turnState.BufferedAudioBytes > 0
|
||||||
|
? "WAIT_LISTEN_FINISHED"
|
||||||
|
: "LISTENING";
|
||||||
|
}
|
||||||
|
|
||||||
private static TimeSpan ResolveLateAudioIgnoreWindow(ResponsePlan plan)
|
private static TimeSpan ResolveLateAudioIgnoreWindow(ResponsePlan plan)
|
||||||
{
|
{
|
||||||
return string.Equals(plan.IntentName, "cloud_version", StringComparison.OrdinalIgnoreCase)
|
return string.Equals(plan.IntentName, "cloud_version", StringComparison.OrdinalIgnoreCase)
|
||||||
@@ -1483,9 +1556,15 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
if (normalized.StartsWith("my favorite ", StringComparison.Ordinal) ||
|
if (normalized.StartsWith("my favorite ", StringComparison.Ordinal) ||
|
||||||
normalized.StartsWith("my favourite ", StringComparison.Ordinal))
|
normalized.StartsWith("my favourite ", StringComparison.Ordinal))
|
||||||
{
|
{
|
||||||
|
var preferenceTail = normalized.StartsWith("my favourite ", StringComparison.Ordinal)
|
||||||
|
? normalized["my favourite ".Length..].Trim()
|
||||||
|
: normalized["my favorite ".Length..].Trim();
|
||||||
|
var missingCopula = !normalized.Contains(" is ", StringComparison.Ordinal) &&
|
||||||
|
!normalized.Contains(" are ", StringComparison.Ordinal);
|
||||||
|
|
||||||
if (normalized.EndsWith(" is", StringComparison.Ordinal) ||
|
if (normalized.EndsWith(" is", StringComparison.Ordinal) ||
|
||||||
normalized.EndsWith(" are", StringComparison.Ordinal) ||
|
normalized.EndsWith(" are", StringComparison.Ordinal) ||
|
||||||
!normalized.Contains(" is ", StringComparison.Ordinal))
|
(missingCopula && !LooksLikeBarePreferenceSet(preferenceTail)))
|
||||||
{
|
{
|
||||||
reason = "preference_set_incomplete";
|
reason = "preference_set_incomplete";
|
||||||
return true;
|
return true;
|
||||||
@@ -1518,6 +1597,64 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
return PegasusAffinityContinuationStems.Contains(normalized);
|
return PegasusAffinityContinuationStems.Contains(normalized);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool LooksLikeBarePreferenceSet(string preferenceTail)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(preferenceTail))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var tokens = preferenceTail.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||||
|
return tokens.Length >= 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ClearListenTracking(WebSocketTurnState turnState)
|
||||||
|
{
|
||||||
|
turnState.SawListen = false;
|
||||||
|
turnState.SawContext = false;
|
||||||
|
turnState.ListenOpenedUtc = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void UpdateGlsmPhaseMarker(CloudSession session)
|
||||||
|
{
|
||||||
|
session.Metadata[GlsmPhaseMetadataKey] = ResolveGlsmPhase(session);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task TrackGlsmPhaseAsync(
|
||||||
|
CloudSession session,
|
||||||
|
WebSocketMessageEnvelope envelope,
|
||||||
|
string trigger,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var nextPhase = ResolveGlsmPhase(session);
|
||||||
|
var previousPhase = session.Metadata.TryGetValue(GlsmPhaseMetadataKey, out var rawPhase)
|
||||||
|
? rawPhase?.ToString()
|
||||||
|
: null;
|
||||||
|
session.Metadata[GlsmPhaseMetadataKey] = nextPhase;
|
||||||
|
|
||||||
|
if (string.Equals(previousPhase, nextPhase, StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await sink.RecordTurnDiagnosticAsync("glsm_phase_transition", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["trigger"] = trigger,
|
||||||
|
["previousState"] = previousPhase,
|
||||||
|
["state"] = nextPhase,
|
||||||
|
["listenOpenedUtc"] = session.TurnState.ListenOpenedUtc,
|
||||||
|
["followUpOpen"] = session.FollowUpOpen,
|
||||||
|
["listenRules"] = session.TurnState.ListenRules
|
||||||
|
}), cancellationToken);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Diagnostics should not interrupt turn handling.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static Dictionary<string, object?> BuildTurnDiagnosticSnapshot(
|
private static Dictionary<string, object?> BuildTurnDiagnosticSnapshot(
|
||||||
CloudSession session,
|
CloudSession session,
|
||||||
WebSocketMessageEnvelope envelope,
|
WebSocketMessageEnvelope envelope,
|
||||||
@@ -1534,6 +1671,7 @@ public sealed partial class WebSocketTurnFinalizationService(
|
|||||||
details["bufferedAudioChunks"] = session.TurnState.BufferedAudioChunkCount;
|
details["bufferedAudioChunks"] = session.TurnState.BufferedAudioChunkCount;
|
||||||
details["sawListen"] = session.TurnState.SawListen;
|
details["sawListen"] = session.TurnState.SawListen;
|
||||||
details["sawContext"] = session.TurnState.SawContext;
|
details["sawContext"] = session.TurnState.SawContext;
|
||||||
|
details["glsmState"] = ResolveGlsmPhase(session);
|
||||||
return details;
|
return details;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ public sealed class WebSocketTurnState
|
|||||||
|
|
||||||
public string? TransId { get; set; }
|
public string? TransId { get; set; }
|
||||||
public string? ContextPayload { get; set; }
|
public string? ContextPayload { get; set; }
|
||||||
|
public DateTimeOffset? ListenOpenedUtc { get; set; }
|
||||||
public bool ListenHotphrase { get; set; }
|
public bool ListenHotphrase { get; set; }
|
||||||
public int HotphraseEmptyTurnCount { get; set; }
|
public int HotphraseEmptyTurnCount { get; set; }
|
||||||
public DateTimeOffset? IgnoreAdditionalAudioUntilUtc { get; set; }
|
public DateTimeOffset? IgnoreAdditionalAudioUntilUtc { get; set; }
|
||||||
|
|||||||
@@ -29,9 +29,18 @@ public sealed class OpenWeatherReportProvider(
|
|||||||
}
|
}
|
||||||
|
|
||||||
var useCelsius = request.UseCelsius ?? options.UseCelsius;
|
var useCelsius = request.UseCelsius ?? options.UseCelsius;
|
||||||
return request.IsTomorrow
|
var forecastDayOffset = request.ForecastDayOffset ?? (request.IsTomorrow ? 1 : 0);
|
||||||
? await GetTomorrowForecastAsync(location.Value, useCelsius, cancellationToken)
|
if (forecastDayOffset <= 0)
|
||||||
: await GetCurrentWeatherAsync(location.Value, useCelsius, cancellationToken);
|
{
|
||||||
|
return await GetCurrentWeatherAsync(location.Value, useCelsius, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (forecastDayOffset > MaxForecastDayOffset)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await GetForecastForDayOffsetAsync(location.Value, useCelsius, forecastDayOffset, cancellationToken);
|
||||||
}
|
}
|
||||||
catch (Exception exception)
|
catch (Exception exception)
|
||||||
{
|
{
|
||||||
@@ -44,14 +53,20 @@ public sealed class OpenWeatherReportProvider(
|
|||||||
WeatherReportRequest request,
|
WeatherReportRequest request,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (request is { Latitude: not null, Longitude: not null })
|
var query = string.IsNullOrWhiteSpace(request.LocationQuery)
|
||||||
|
? null
|
||||||
|
: request.LocationQuery.Trim();
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(query))
|
||||||
{
|
{
|
||||||
return new LocationPoint(request.Latitude.Value, request.Longitude.Value, null);
|
if (request is { Latitude: not null, Longitude: not null })
|
||||||
|
{
|
||||||
|
return new LocationPoint(request.Latitude.Value, request.Longitude.Value, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
query = options.DefaultLocation;
|
||||||
}
|
}
|
||||||
|
|
||||||
var query = string.IsNullOrWhiteSpace(request.LocationQuery)
|
|
||||||
? options.DefaultLocation
|
|
||||||
: request.LocationQuery.Trim();
|
|
||||||
if (string.IsNullOrWhiteSpace(query))
|
if (string.IsNullOrWhiteSpace(query))
|
||||||
{
|
{
|
||||||
return null;
|
return null;
|
||||||
@@ -134,9 +149,10 @@ public sealed class OpenWeatherReportProvider(
|
|||||||
useCelsius);
|
useCelsius);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<WeatherReportSnapshot?> GetTomorrowForecastAsync(
|
private async Task<WeatherReportSnapshot?> GetForecastForDayOffsetAsync(
|
||||||
LocationPoint location,
|
LocationPoint location,
|
||||||
bool useCelsius,
|
bool useCelsius,
|
||||||
|
int forecastDayOffset,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var forecastUri = BuildRequestUri(
|
var forecastUri = BuildRequestUri(
|
||||||
@@ -160,7 +176,7 @@ public sealed class OpenWeatherReportProvider(
|
|||||||
}
|
}
|
||||||
|
|
||||||
var offset = TryReadForecastOffset(root);
|
var offset = TryReadForecastOffset(root);
|
||||||
var tomorrow = DateOnly.FromDateTime(DateTimeOffset.UtcNow.ToOffset(offset).DateTime.AddDays(1));
|
var targetDate = DateOnly.FromDateTime(DateTimeOffset.UtcNow.ToOffset(offset).DateTime.AddDays(forecastDayOffset));
|
||||||
var entries = new List<ForecastEntry>();
|
var entries = new List<ForecastEntry>();
|
||||||
foreach (var item in list.EnumerateArray())
|
foreach (var item in list.EnumerateArray())
|
||||||
{
|
{
|
||||||
@@ -170,7 +186,7 @@ public sealed class OpenWeatherReportProvider(
|
|||||||
}
|
}
|
||||||
|
|
||||||
var localTimestamp = DateTimeOffset.FromUnixTimeSeconds(unixSeconds).ToOffset(offset);
|
var localTimestamp = DateTimeOffset.FromUnixTimeSeconds(unixSeconds).ToOffset(offset);
|
||||||
if (DateOnly.FromDateTime(localTimestamp.DateTime) != tomorrow)
|
if (DateOnly.FromDateTime(localTimestamp.DateTime) != targetDate)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -361,4 +377,6 @@ public sealed class OpenWeatherReportProvider(
|
|||||||
int? LowTemperature,
|
int? LowTemperature,
|
||||||
string? Summary,
|
string? Summary,
|
||||||
string? Condition);
|
string? Condition);
|
||||||
|
|
||||||
|
private const int MaxForecastDayOffset = 5;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -101,4 +101,49 @@ public sealed class FileTurnTelemetrySinkTests
|
|||||||
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||||
Times.Once());
|
Times.Once());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task HandleContext_EmitsGlsmPhaseTransitionDiagnostic()
|
||||||
|
{
|
||||||
|
var sink = new Mock<ITurnTelemetrySink>();
|
||||||
|
sink.Setup(s => s.RecordTurnDiagnosticAsync(It.IsAny<string>(), It.IsAny<IReadOnlyDictionary<string, object?>>(), It.IsAny<CancellationToken>()))
|
||||||
|
.Returns(Task.CompletedTask);
|
||||||
|
var turnService = new WebSocketTurnFinalizationService(
|
||||||
|
Mock.Of<IConversationBroker>(),
|
||||||
|
Mock.Of<ISttStrategySelector>(),
|
||||||
|
sink.Object);
|
||||||
|
|
||||||
|
var session = new CloudSession
|
||||||
|
{
|
||||||
|
Token = "glsm-phase-token",
|
||||||
|
TurnState =
|
||||||
|
{
|
||||||
|
TransId = "trans-glsm",
|
||||||
|
AwaitingTurnCompletion = true,
|
||||||
|
SawListen = true,
|
||||||
|
ListenOpenedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
session.Metadata["glsmPhase"] = "HJ_LISTENING";
|
||||||
|
|
||||||
|
await turnService.HandleContextAsync(
|
||||||
|
session,
|
||||||
|
new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Text = """{"type":"CONTEXT","transID":"trans-glsm","data":{"topic":"conversation"}}"""
|
||||||
|
},
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
sink.Verify(
|
||||||
|
s => s.RecordTurnDiagnosticAsync(
|
||||||
|
"glsm_phase_transition",
|
||||||
|
It.Is<IReadOnlyDictionary<string, object?>>(details =>
|
||||||
|
details.ContainsKey("state") &&
|
||||||
|
string.Equals(details["state"] == null ? null : details["state"]!.ToString(), "LISTENING", StringComparison.OrdinalIgnoreCase)),
|
||||||
|
It.IsAny<CancellationToken>()),
|
||||||
|
Times.AtLeastOnce());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -135,6 +135,21 @@ public sealed class JiboInteractionServiceTests
|
|||||||
Assert.Equal("My birthday is March 22, 2026.", decision.ReplyText);
|
Assert.Equal("My birthday is March 22, 2026.", decision.ReplyText);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WhatsYourBday_DoesNotFallThroughToDateIntent()
|
||||||
|
{
|
||||||
|
var service = CreateService();
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's your bday",
|
||||||
|
NormalizedTranscript = "what's your bday"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("robot_birthday", decision.IntentName);
|
||||||
|
Assert.Equal("My birthday is March 22, 2026.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BuildDecisionAsync_DoYouHaveAPersonality_UsesCatalogBackedPersonalityReply()
|
public async Task BuildDecisionAsync_DoYouHaveAPersonality_UsesCatalogBackedPersonalityReply()
|
||||||
{
|
{
|
||||||
@@ -338,6 +353,43 @@ public sealed class JiboInteractionServiceTests
|
|||||||
Assert.Equal("I can remember it if you say, my birthday is March 14.", decision.ReplyText);
|
Assert.Equal("I can remember it if you say, my birthday is March 14.", decision.ReplyText);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_BirthdayMemory_BdayAliasSetThenRecallWithinTenant()
|
||||||
|
{
|
||||||
|
var memoryStore = new InMemoryPersonalMemoryStore();
|
||||||
|
var service = CreateService(memoryStore);
|
||||||
|
|
||||||
|
var setDecision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "my bday is April 12",
|
||||||
|
NormalizedTranscript = "my bday is April 12",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["accountId"] = "acct-a",
|
||||||
|
["loopId"] = "loop-a"
|
||||||
|
},
|
||||||
|
DeviceId = "device-a"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("memory_set_birthday", setDecision.IntentName);
|
||||||
|
Assert.Equal("Got it. I will remember your birthday is april 12.", setDecision.ReplyText);
|
||||||
|
|
||||||
|
var recallDecision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "when is my bday",
|
||||||
|
NormalizedTranscript = "when is my bday",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["accountId"] = "acct-a",
|
||||||
|
["loopId"] = "loop-a"
|
||||||
|
},
|
||||||
|
DeviceId = "device-a"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("memory_get_birthday", recallDecision.IntentName);
|
||||||
|
Assert.Equal("You told me your birthday is april 12.", recallDecision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BuildDecisionAsync_PreferenceMemory_SetThenRecallWithinTenant()
|
public async Task BuildDecisionAsync_PreferenceMemory_SetThenRecallWithinTenant()
|
||||||
{
|
{
|
||||||
@@ -375,6 +427,43 @@ public sealed class JiboInteractionServiceTests
|
|||||||
Assert.Equal("You told me your favorite music is jazz.", recallDecision.ReplyText);
|
Assert.Equal("You told me your favorite music is jazz.", recallDecision.ReplyText);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_PreferenceMemory_BareFavoriteSetThenRecallWithinTenant()
|
||||||
|
{
|
||||||
|
var memoryStore = new InMemoryPersonalMemoryStore();
|
||||||
|
var service = CreateService(memoryStore);
|
||||||
|
|
||||||
|
var setDecision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "my favorite sport football",
|
||||||
|
NormalizedTranscript = "my favorite sport football",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["accountId"] = "acct-a",
|
||||||
|
["loopId"] = "loop-a"
|
||||||
|
},
|
||||||
|
DeviceId = "device-a"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("memory_set_preference", setDecision.IntentName);
|
||||||
|
Assert.Equal("Got it. I will remember your favorite sport is football.", setDecision.ReplyText);
|
||||||
|
|
||||||
|
var recallDecision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what is my favorite sport",
|
||||||
|
NormalizedTranscript = "what is my favorite sport",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["accountId"] = "acct-a",
|
||||||
|
["loopId"] = "loop-a"
|
||||||
|
},
|
||||||
|
DeviceId = "device-a"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("memory_get_preference", recallDecision.IntentName);
|
||||||
|
Assert.Equal("You told me your favorite sport is football.", recallDecision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BuildDecisionAsync_PreferenceSetAttemptWithoutValue_RoutesToPreferencePrompt()
|
public async Task BuildDecisionAsync_PreferenceSetAttemptWithoutValue_RoutesToPreferencePrompt()
|
||||||
{
|
{
|
||||||
@@ -1027,6 +1116,36 @@ public sealed class JiboInteractionServiceTests
|
|||||||
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
|
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherTodaysForecastQuery_WithoutProvider_StillReturnsFallback()
|
||||||
|
{
|
||||||
|
var service = CreateService();
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's today's weather look like",
|
||||||
|
NormalizedTranscript = "what's today's weather look like"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherConditionForecastQuery_WithoutProvider_StillReturnsFallback()
|
||||||
|
{
|
||||||
|
var service = CreateService();
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "will it be sunny tomorrow",
|
||||||
|
NormalizedTranscript = "will it be sunny tomorrow"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BuildDecisionAsync_ClientNluRequestWeatherPR_WithoutProvider_StillReturnsFallback()
|
public async Task BuildDecisionAsync_ClientNluRequestWeatherPR_WithoutProvider_StillReturnsFallback()
|
||||||
{
|
{
|
||||||
@@ -1062,11 +1181,22 @@ public sealed class JiboInteractionServiceTests
|
|||||||
});
|
});
|
||||||
|
|
||||||
Assert.Equal("weather", decision.IntentName);
|
Assert.Equal("weather", decision.IntentName);
|
||||||
Assert.Null(decision.SkillName);
|
Assert.Equal("chitchat-skill", decision.SkillName);
|
||||||
Assert.Null(decision.SkillPayload);
|
Assert.NotNull(decision.SkillPayload);
|
||||||
|
Assert.Contains("cat='weather'", decision.SkillPayload!["esml"]?.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||||
|
Assert.Contains("meta='rain'", decision.SkillPayload["esml"]?.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||||
|
Assert.Equal("WeatherCommentRain", decision.SkillPayload["mim_id"]);
|
||||||
|
Assert.Equal(true, decision.SkillPayload["weather_view_enabled"]);
|
||||||
|
Assert.Equal("weatherHiLo", decision.SkillPayload["weather_view_kind"]);
|
||||||
|
Assert.Equal("rain", decision.SkillPayload["weather_icon"]);
|
||||||
|
Assert.Equal(65, decision.SkillPayload["weather_high"]);
|
||||||
|
Assert.Equal(54, decision.SkillPayload["weather_low"]);
|
||||||
|
Assert.Equal("F", decision.SkillPayload["weather_unit"]);
|
||||||
|
Assert.Equal("Normal", decision.SkillPayload["weather_theme"]);
|
||||||
Assert.Equal("Right now in Boston, US, it is light rain and 61 degrees Fahrenheit.", decision.ReplyText);
|
Assert.Equal("Right now in Boston, US, it is light rain and 61 degrees Fahrenheit.", decision.ReplyText);
|
||||||
Assert.NotNull(provider.LastRequest);
|
Assert.NotNull(provider.LastRequest);
|
||||||
Assert.False(provider.LastRequest!.IsTomorrow);
|
Assert.False(provider.LastRequest!.IsTomorrow);
|
||||||
|
Assert.Equal(0, provider.LastRequest.ForecastDayOffset);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -1087,9 +1217,359 @@ public sealed class JiboInteractionServiceTests
|
|||||||
Assert.Equal("weather", decision.IntentName);
|
Assert.Equal("weather", decision.IntentName);
|
||||||
Assert.Equal("Chicago", provider.LastRequest?.LocationQuery);
|
Assert.Equal("Chicago", provider.LastRequest?.LocationQuery);
|
||||||
Assert.True(provider.LastRequest?.IsTomorrow);
|
Assert.True(provider.LastRequest?.IsTomorrow);
|
||||||
|
Assert.Equal(1, provider.LastRequest?.ForecastDayOffset);
|
||||||
Assert.Equal("Tomorrow in Chicago, US, expect mostly cloudy with a high near 74 degrees Fahrenheit and a low around 60 degrees Fahrenheit.", decision.ReplyText);
|
Assert.Equal("Tomorrow in Chicago, US, expect mostly cloudy with a high near 74 degrees Fahrenheit and a low around 60 degrees Fahrenheit.", decision.ReplyText);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherLocationForToday_WithProvider_PassesLocation()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Seattle, US", "light rain", 58, 61, 52, "rain", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather for seattle today",
|
||||||
|
NormalizedTranscript = "what's the weather for seattle today"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Seattle", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.False(provider.LastRequest?.IsTomorrow);
|
||||||
|
Assert.Equal(0, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("Right now in Seattle, US, it is light rain and 58 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherLocationWithWeekendSuffix_WithProvider_PassesLocation()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Paris, FR", "overcast clouds", 66, 70, 60, "cloudy", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather in paris this weekend",
|
||||||
|
NormalizedTranscript = "what's the weather in paris this weekend"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Paris", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.False(provider.LastRequest?.IsTomorrow);
|
||||||
|
Assert.Equal(0, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("Right now in Paris, FR, it is overcast clouds and 66 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_TemperatureLocationQuery_WithProvider_MapsToWeatherIntent()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Redmond, US", "clear sky", 63, 66, 52, "sunny", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what is the temperature in redmond oregon",
|
||||||
|
NormalizedTranscript = "what is the temperature in redmond oregon"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Redmond Oregon", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.False(provider.LastRequest?.IsTomorrow);
|
||||||
|
Assert.Equal(0, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("Right now in Redmond, US, it is clear sky and 63 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_ForecastLocationQuery_WithProvider_MapsToWeatherIntent()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("New York, US", "partly cloudy", 71, 76, 61, "cloudy", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "forecast for new york city",
|
||||||
|
NormalizedTranscript = "forecast for new york city"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("New York City", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.True(provider.LastRequest?.IsTomorrow);
|
||||||
|
Assert.Equal(1, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("Tomorrow in New York, US, expect partly cloudy with a high near 76 degrees Fahrenheit and a low around 61 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_ForecastWithoutDate_WithProvider_DefaultsToTomorrow()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Kansas City, US", "clear sky", 72, 79, 63, "sunny", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the forecast",
|
||||||
|
NormalizedTranscript = "what's the forecast"
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Null(provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.True(provider.LastRequest?.IsTomorrow);
|
||||||
|
Assert.Equal(1, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("Tomorrow in Kansas City, US, expect clear sky with a high near 79 degrees Fahrenheit and a low around 63 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherLocationQuery_IgnoresRuntimeCoordinates()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Chicago, US", "mostly cloudy", 70, 75, 62, "cloudy", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather in chicago",
|
||||||
|
NormalizedTranscript = "what's the weather in chicago",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["context"] = """{"runtime":{"location":{"lat":39.0997,"lng":-94.5786,"iso":"2026-05-09T09:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Chicago", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.Null(provider.LastRequest?.Latitude);
|
||||||
|
Assert.Null(provider.LastRequest?.Longitude);
|
||||||
|
Assert.Equal("Right now in Chicago, US, it is mostly cloudy and 70 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("how is the weather", null, 0, false)]
|
||||||
|
[InlineData("what's the forecast", null, 1, true)]
|
||||||
|
[InlineData("forecast for new york city", "New York City", 1, true)]
|
||||||
|
[InlineData("what's today's forecast", null, 0, false)]
|
||||||
|
[InlineData("what's the weather in chicago", "Chicago", 0, false)]
|
||||||
|
[InlineData("what's the weather in chicago tomorrow", "Chicago", 1, true)]
|
||||||
|
[InlineData("what is the temperature in redmond oregon", "Redmond Oregon", 0, false)]
|
||||||
|
[InlineData("will it rain tomorrow", null, 1, true)]
|
||||||
|
public async Task BuildDecisionAsync_WeatherPromptRegression_MatchesExpectedRouting(
|
||||||
|
string transcript,
|
||||||
|
string? expectedLocationQuery,
|
||||||
|
int expectedForecastOffset,
|
||||||
|
bool expectedIsTomorrow)
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Test City, US", "light rain", 62, 66, 55, "rain", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = transcript,
|
||||||
|
NormalizedTranscript = transcript
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.NotNull(provider.LastRequest);
|
||||||
|
Assert.Equal(expectedLocationQuery, provider.LastRequest!.LocationQuery);
|
||||||
|
Assert.Equal(expectedForecastOffset, provider.LastRequest.ForecastDayOffset);
|
||||||
|
Assert.Equal(expectedIsTomorrow, provider.LastRequest.IsTomorrow);
|
||||||
|
Assert.Equal("chitchat-skill", decision.SkillName);
|
||||||
|
Assert.Equal(true, decision.SkillPayload?["weather_view_enabled"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherQueryWithClientDateEntity_UsesForecastDayOffset()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Portland, US", "scattered clouds", 64, 68, 53, "cloudy", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather",
|
||||||
|
NormalizedTranscript = "what's the weather",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["clientEntities"] = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["date"] = "2026-05-11"
|
||||||
|
},
|
||||||
|
["context"] = """{"runtime":{"location":{"iso":"2026-05-09T09:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal(2, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.False(provider.LastRequest?.IsTomorrow);
|
||||||
|
Assert.Equal("On Monday in Portland, US, expect scattered clouds with a high near 68 degrees Fahrenheit and a low around 53 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherQueryWithWeekday_UsesForecastDayOffset()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Chicago, US", "light rain", 59, 63, 51, "rain", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather in chicago on tuesday",
|
||||||
|
NormalizedTranscript = "what's the weather in chicago on tuesday",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["context"] = """{"runtime":{"location":{"iso":"2026-04-20T08:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Chicago", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.Equal(1, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("On Tuesday in Chicago, US, expect light rain with a high near 63 degrees Fahrenheit and a low around 51 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherQueryBeyondSupportedForecastRange_ReturnsGuardrailMessage()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Chicago, US", "light rain", 59, 63, 51, "rain", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather next saturday",
|
||||||
|
NormalizedTranscript = "what's the weather next saturday",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["context"] = """{"runtime":{"location":{"iso":"2026-04-20T08:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("I can forecast up to 5 days ahead. Try tomorrow or another day this week.", decision.ReplyText);
|
||||||
|
Assert.Null(provider.LastRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherThisWeekend_WithContext_UsesWeekendOffset()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Paris, FR", "overcast clouds", 66, 70, 60, "cloudy", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather in paris this weekend",
|
||||||
|
NormalizedTranscript = "what's the weather in paris this weekend",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["context"] = """{"runtime":{"location":{"iso":"2026-04-20T08:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Paris", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.Equal(5, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("This weekend in Paris, FR, expect overcast clouds with a high near 70 degrees Fahrenheit and a low around 60 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherThisWeek_WithContext_UsesRangeOffset()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Seattle, US", "light rain", 58, 61, 52, "rain", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "forecast for seattle this week",
|
||||||
|
NormalizedTranscript = "forecast for seattle this week",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["context"] = """{"runtime":{"location":{"iso":"2026-04-20T08:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Seattle", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.Equal(2, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("Later this week in Seattle, US, expect light rain with a high near 61 degrees Fahrenheit and a low around 52 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherNextWeek_WithContext_ReturnsGuardrailMessage()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Seattle, US", "light rain", 58, 61, 52, "rain", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "forecast for seattle next week",
|
||||||
|
NormalizedTranscript = "forecast for seattle next week",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["context"] = """{"runtime":{"location":{"iso":"2026-04-20T08:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("I can forecast up to 5 days ahead. Try tomorrow or another day this week.", decision.ReplyText);
|
||||||
|
Assert.Null(provider.LastRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BuildDecisionAsync_WeatherDayAfterTomorrow_WithContext_PassesDayOffsetAndLocation()
|
||||||
|
{
|
||||||
|
var provider = new CapturingWeatherReportProvider
|
||||||
|
{
|
||||||
|
Snapshot = new WeatherReportSnapshot("Chicago, US", "mostly cloudy", 72, 74, 60, "cloudy", false)
|
||||||
|
};
|
||||||
|
var service = CreateService(weatherReportProvider: provider);
|
||||||
|
|
||||||
|
var decision = await service.BuildDecisionAsync(new TurnContext
|
||||||
|
{
|
||||||
|
RawTranscript = "what's the weather in chicago day after tomorrow",
|
||||||
|
NormalizedTranscript = "what's the weather in chicago day after tomorrow",
|
||||||
|
Attributes = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["context"] = """{"runtime":{"location":{"iso":"2026-04-20T08:00:00-05:00"}}}"""
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal("weather", decision.IntentName);
|
||||||
|
Assert.Equal("Chicago", provider.LastRequest?.LocationQuery);
|
||||||
|
Assert.Equal(2, provider.LastRequest?.ForecastDayOffset);
|
||||||
|
Assert.Equal("The day after tomorrow in Chicago, US, expect mostly cloudy with a high near 74 degrees Fahrenheit and a low around 60 degrees Fahrenheit.", decision.ReplyText);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BuildDecisionAsync_ClientNluAskForDate_MapsToDateIntent()
|
public async Task BuildDecisionAsync_ClientNluAskForDate_MapsToDateIntent()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
|
using Jibo.Cloud.Application.Abstractions;
|
||||||
using Jibo.Cloud.Application.Services;
|
using Jibo.Cloud.Application.Services;
|
||||||
using Jibo.Cloud.Domain.Models;
|
using Jibo.Cloud.Domain.Models;
|
||||||
using Jibo.Cloud.Infrastructure.Content;
|
using Jibo.Cloud.Infrastructure.Content;
|
||||||
@@ -363,6 +364,64 @@ public sealed class JiboWebSocketServiceTests
|
|||||||
Assert.Equal("my favorite sport is football", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
|
Assert.Equal("my favorite sport is football", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BufferedAudio_WithBarePreferenceSetHint_FinalizesWithoutDeferral()
|
||||||
|
{
|
||||||
|
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-preference-bare-token",
|
||||||
|
Text = """{"type":"LISTEN","transID":"trans-preference-bare","data":{"rules":["launch"]}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-preference-bare-token",
|
||||||
|
Text = """{"type":"CONTEXT","transID":"trans-preference-bare","data":{"audioTranscriptHint":"my favorite sport football"}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
for (var index = 0; index < 4; index += 1)
|
||||||
|
{
|
||||||
|
var chunkReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-preference-bare-token",
|
||||||
|
Binary = new byte[3000]
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Empty(chunkReplies);
|
||||||
|
}
|
||||||
|
|
||||||
|
var session = _store.FindSessionByToken("hub-preference-bare-token");
|
||||||
|
Assert.NotNull(session);
|
||||||
|
session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2);
|
||||||
|
|
||||||
|
var finalizedReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-preference-bare-token",
|
||||||
|
Binary = new byte[3000]
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal(3, finalizedReplies.Count);
|
||||||
|
Assert.Equal("LISTEN", ReadReplyType(finalizedReplies[0]));
|
||||||
|
Assert.Equal("EOS", ReadReplyType(finalizedReplies[1]));
|
||||||
|
Assert.Equal("SKILL_ACTION", ReadReplyType(finalizedReplies[2]));
|
||||||
|
|
||||||
|
using var listenPayload = JsonDocument.Parse(finalizedReplies[0].Text!);
|
||||||
|
Assert.Equal("memory_set_preference", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
|
||||||
|
Assert.Equal("my favorite sport football", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BufferedAudio_WithIncompleteAffinityHint_DefersThenFinalizesWhenContinuationArrives()
|
public async Task BufferedAudio_WithIncompleteAffinityHint_DefersThenFinalizesWhenContinuationArrives()
|
||||||
{
|
{
|
||||||
@@ -1928,6 +1987,81 @@ public sealed class JiboWebSocketServiceTests
|
|||||||
Assert.Contains("weather service is connected", esml, StringComparison.OrdinalIgnoreCase);
|
Assert.Contains("weather service is connected", esml, StringComparison.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ClientAsr_HowIsTheWeather_WithProvider_EmitsWeatherHiLoGuiCard()
|
||||||
|
{
|
||||||
|
var customStore = new InMemoryCloudStateStore();
|
||||||
|
var customService = CreateService(
|
||||||
|
customStore,
|
||||||
|
new StubWeatherReportProvider(
|
||||||
|
new WeatherReportSnapshot("Boston, US", "light rain", 61, 65, 54, "rain", false)));
|
||||||
|
|
||||||
|
await customService.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-weather-provider-token",
|
||||||
|
Text = """{"type":"LISTEN","transID":"trans-weather-provider","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
var replies = await customService.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-weather-provider-token",
|
||||||
|
Text = """{"type":"CLIENT_ASR","transID":"trans-weather-provider","data":{"text":"how is the weather"}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal(3, replies.Count);
|
||||||
|
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
|
||||||
|
Assert.Equal("EOS", ReadReplyType(replies[1]));
|
||||||
|
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
|
||||||
|
|
||||||
|
using var skillPayload = JsonDocument.Parse(replies[2].Text!);
|
||||||
|
var jcpConfig = skillPayload.RootElement
|
||||||
|
.GetProperty("data")
|
||||||
|
.GetProperty("action")
|
||||||
|
.GetProperty("config")
|
||||||
|
.GetProperty("jcp")
|
||||||
|
.GetProperty("config");
|
||||||
|
|
||||||
|
var esml = jcpConfig.GetProperty("play").GetProperty("esml").GetString();
|
||||||
|
Assert.Contains("cat='weather'", esml, StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
Assert.True(jcpConfig.TryGetProperty("gui", out var gui));
|
||||||
|
Assert.Equal("Javascript", gui.GetProperty("type").GetString());
|
||||||
|
Assert.Equal("views.weatherHiLo", gui.GetProperty("data").GetString());
|
||||||
|
Assert.True(gui.GetProperty("pause").GetBoolean());
|
||||||
|
|
||||||
|
var play = jcpConfig.GetProperty("play");
|
||||||
|
Assert.True(play.TryGetProperty("gui", out var playGui));
|
||||||
|
Assert.Equal("Javascript", playGui.GetProperty("type").GetString());
|
||||||
|
Assert.True(playGui.GetProperty("pause").GetBoolean());
|
||||||
|
Assert.Equal("weatherTempView", playGui.GetProperty("data").GetProperty("viewConfig").GetProperty("id").GetString());
|
||||||
|
Assert.Equal(0, play.GetProperty("no_matches_for_gui").GetInt32());
|
||||||
|
Assert.Equal(0, play.GetProperty("no_inputs_for_gui").GetInt32());
|
||||||
|
|
||||||
|
Assert.True(jcpConfig.TryGetProperty("display", out var display));
|
||||||
|
Assert.Equal(
|
||||||
|
"weatherTempView",
|
||||||
|
display.GetProperty("view").GetProperty("data").GetProperty("viewConfig").GetProperty("id").GetString());
|
||||||
|
|
||||||
|
Assert.True(jcpConfig.TryGetProperty("views", out var views));
|
||||||
|
var weatherHiLo = views.GetProperty("weatherHiLo");
|
||||||
|
Assert.Equal("weatherTempView", weatherHiLo.GetProperty("viewConfig").GetProperty("id").GetString());
|
||||||
|
|
||||||
|
Assert.True(jcpConfig.TryGetProperty("local", out var local));
|
||||||
|
Assert.Equal(
|
||||||
|
"weatherTempView",
|
||||||
|
local.GetProperty("views").GetProperty("weatherHiLo").GetProperty("viewConfig").GetProperty("id").GetString());
|
||||||
|
|
||||||
|
var payloadText = replies[2].Text!;
|
||||||
|
Assert.Contains("assets/personal-report-skill/weather/icons/rain_v01.crn", payloadText, StringComparison.Ordinal);
|
||||||
|
Assert.Contains("tempNormal_v01.crn", payloadText, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ClientAsr_OpenTheRadio_EmitsRadioRedirectAndSilentCompletion()
|
public async Task ClientAsr_OpenTheRadio_EmitsRadioRedirectAndSilentCompletion()
|
||||||
{
|
{
|
||||||
@@ -2523,6 +2657,47 @@ public sealed class JiboWebSocketServiceTests
|
|||||||
Assert.Null(session.LastIntent);
|
Assert.Null(session.LastIntent);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task StaleListenSetup_IsRecoveredWhenNextHotphraseListenArrives()
|
||||||
|
{
|
||||||
|
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-stale-listen-token",
|
||||||
|
Text = """{"type":"LISTEN","transID":"trans-stale-listen","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
var session = _store.FindSessionByToken("hub-stale-listen-token");
|
||||||
|
Assert.NotNull(session);
|
||||||
|
session.TurnState.ListenOpenedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(12);
|
||||||
|
session.TurnState.AwaitingTurnCompletion = true;
|
||||||
|
session.TurnState.SawListen = true;
|
||||||
|
session.TurnState.SawContext = false;
|
||||||
|
session.TurnState.BufferedAudioBytes = 0;
|
||||||
|
session.TurnState.BufferedAudioChunkCount = 0;
|
||||||
|
session.TurnState.HotphraseEmptyTurnCount = 2;
|
||||||
|
|
||||||
|
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-stale-listen-token",
|
||||||
|
Text = """{"type":"LISTEN","transID":"trans-stale-listen","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Empty(replies);
|
||||||
|
Assert.True(session.TurnState.AwaitingTurnCompletion);
|
||||||
|
Assert.True(session.TurnState.SawListen);
|
||||||
|
Assert.False(session.TurnState.SawContext);
|
||||||
|
Assert.Equal(0, session.TurnState.BufferedAudioBytes);
|
||||||
|
Assert.Equal(0, session.TurnState.BufferedAudioChunkCount);
|
||||||
|
Assert.Equal(0, session.TurnState.HotphraseEmptyTurnCount);
|
||||||
|
Assert.True(session.TurnState.ListenOpenedUtc > DateTimeOffset.UtcNow - TimeSpan.FromSeconds(3));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BinaryAudio_AfterWordOfDayRightWordListen_IsIgnoredDuringCleanupWindow()
|
public async Task BinaryAudio_AfterWordOfDayRightWordListen_IsIgnoredDuringCleanupWindow()
|
||||||
{
|
{
|
||||||
@@ -3266,6 +3441,66 @@ public sealed class JiboWebSocketServiceTests
|
|||||||
Assert.False(session.Metadata.ContainsKey("pendingProactivityOffer"));
|
Assert.False(session.Metadata.ContainsKey("pendingProactivityOffer"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ClientAsrSurpriseOffer_PersistsPendingOfferAndResolvesNoFollowUp()
|
||||||
|
{
|
||||||
|
var token = _store.IssueRobotToken("proactivity-device-b");
|
||||||
|
|
||||||
|
var offerReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = token,
|
||||||
|
Text = """{"type":"CLIENT_ASR","transID":"trans-proactive-offer-no","data":{"text":"surprise me"}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal(3, offerReplies.Count);
|
||||||
|
using (var offerListenPayload = JsonDocument.Parse(offerReplies[0].Text!))
|
||||||
|
{
|
||||||
|
Assert.Equal("proactive_offer_pizza_fact", offerListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
|
||||||
|
Assert.Equal("shared/yes_no", offerListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("rules")[0].GetString());
|
||||||
|
}
|
||||||
|
|
||||||
|
var session = _store.FindSessionByToken(token);
|
||||||
|
Assert.NotNull(session);
|
||||||
|
Assert.True(session.Metadata.TryGetValue("pendingProactivityOffer", out var pendingOffer));
|
||||||
|
Assert.Equal("pizza_fact", pendingOffer?.ToString());
|
||||||
|
|
||||||
|
var followUpReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = token,
|
||||||
|
Text = """{"type":"CLIENT_ASR","transID":"trans-proactive-offer-no-followup","data":{"text":"no"}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal(3, followUpReplies.Count);
|
||||||
|
using (var followUpListenPayload = JsonDocument.Parse(followUpReplies[0].Text!))
|
||||||
|
{
|
||||||
|
Assert.Equal("proactive_offer_declined", followUpListenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
|
||||||
|
}
|
||||||
|
|
||||||
|
using (var followUpSkillPayload = JsonDocument.Parse(followUpReplies[2].Text!))
|
||||||
|
{
|
||||||
|
var esml = followUpSkillPayload.RootElement
|
||||||
|
.GetProperty("data")
|
||||||
|
.GetProperty("action")
|
||||||
|
.GetProperty("config")
|
||||||
|
.GetProperty("jcp")
|
||||||
|
.GetProperty("config")
|
||||||
|
.GetProperty("play")
|
||||||
|
.GetProperty("esml")
|
||||||
|
.GetString();
|
||||||
|
Assert.Contains("No problem", esml, StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
session = _store.FindSessionByToken(token);
|
||||||
|
Assert.NotNull(session);
|
||||||
|
Assert.False(session.Metadata.ContainsKey("pendingProactivityOffer"));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ClientAsrPersonalReport_StateMachinePersistsAcrossTurns()
|
public async Task ClientAsrPersonalReport_StateMachinePersistsAcrossTurns()
|
||||||
{
|
{
|
||||||
@@ -3576,9 +3811,43 @@ public sealed class JiboWebSocketServiceTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static JiboWebSocketService CreateService(
|
||||||
|
InMemoryCloudStateStore stateStore,
|
||||||
|
IWeatherReportProvider? weatherReportProvider = null)
|
||||||
|
{
|
||||||
|
var contentRepository = new InMemoryJiboExperienceContentRepository();
|
||||||
|
var contentCache = new JiboExperienceContentCache(contentRepository);
|
||||||
|
var interactionService = new JiboInteractionService(
|
||||||
|
contentCache,
|
||||||
|
new DefaultJiboRandomizer(),
|
||||||
|
new InMemoryPersonalMemoryStore(),
|
||||||
|
weatherReportProvider);
|
||||||
|
var conversationBroker = new DemoConversationBroker(interactionService);
|
||||||
|
var sttSelector = new DefaultSttStrategySelector(
|
||||||
|
[
|
||||||
|
new SyntheticBufferedAudioSttStrategy()
|
||||||
|
]);
|
||||||
|
var sink = new NullTurnTelemetrySink();
|
||||||
|
|
||||||
|
return new JiboWebSocketService(
|
||||||
|
stateStore,
|
||||||
|
new NullWebSocketTelemetrySink(),
|
||||||
|
new WebSocketTurnFinalizationService(conversationBroker, sttSelector, sink));
|
||||||
|
}
|
||||||
|
|
||||||
private static string ReadReplyType(WebSocketReply reply)
|
private static string ReadReplyType(WebSocketReply reply)
|
||||||
{
|
{
|
||||||
using var payload = JsonDocument.Parse(reply.Text!);
|
using var payload = JsonDocument.Parse(reply.Text!);
|
||||||
return payload.RootElement.GetProperty("type").GetString() ?? string.Empty;
|
return payload.RootElement.GetProperty("type").GetString() ?? string.Empty;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sealed class StubWeatherReportProvider(WeatherReportSnapshot snapshot) : IWeatherReportProvider
|
||||||
|
{
|
||||||
|
public Task<WeatherReportSnapshot?> GetReportAsync(
|
||||||
|
WeatherReportRequest request,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
return Task.FromResult<WeatherReportSnapshot?>(snapshot);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user