119 Commits

Author SHA1 Message Date
Jacob Dubin
58deeb1357 Align API integration test version with build info 2026-05-25 07:57:34 -05:00
Jacob Dubin
4e816e175a Document local cloud startup and harden persistence 2026-05-25 00:30:41 -05:00
Jacob Dubin
c36a01b142 Add support voice routes and short-answer STT handling 2026-05-23 20:49:09 -05:00
Jacob Dubin
b017fd9f60 Extract remaining helpers from JiboInteractionService 2026-05-23 00:37:22 -05:00
Jacob Dubin
660bcdc074 Complete JiboInteractionService dispatch cutover 2026-05-23 00:35:16 -05:00
Jacob Dubin
5309980f01 Merge branch 'main' of https://kevinblog.sytes.net/Code/Jibo-Revival-Group/JiboExperiments 2026-05-22 20:36:09 -05:00
Jacob Dubin
e51e62c4bb Extract decision dispatch into a partial class 2026-05-22 20:35:02 -05:00
bf81fadd62 More original server design and communications documentation 2026-05-23 01:20:55 +03:00
1e130e69ab Merge branch 'main' of https://kevinblog.sytes.net/Code/Jaked/JiboExperiments 2026-05-23 00:21:51 +03:00
bca138ecc8 Original server design doc 2026-05-23 00:21:42 +03:00
Jacob Dubin
2455fcfe17 Extract proactivity decisions into partial 2026-05-22 07:45:14 -05:00
Jacob Dubin
2357e82ae3 Randomize how are you replies 2026-05-22 07:37:22 -05:00
Jacob Dubin
1755888fc1 Refine favorite animal and flower personality replies 2026-05-22 07:31:27 -05:00
Jacob Dubin
3086ad6a6d Adjust idle socket reply delays for sleep and spin commands 2026-05-21 23:23:44 -05:00
Jacob Dubin
90b48314d3 Restrict loop name fallback for multi-person greetings 2026-05-21 23:21:36 -05:00
Jacob Dubin
b99ee5d794 Record live QA repair targets for identity and motion quirks 2026-05-21 23:18:25 -05:00
Jacob Dubin
386f864e94 Import Build B age prompts for how old are you 2026-05-21 23:12:48 -05:00
Jacob Dubin
9d675ed59c Import work eat home Build B replies 2026-05-21 20:31:49 -05:00
Jacob Dubin
b113dd55d3 Refactor Build B templated persona prompts 2026-05-21 20:29:33 -05:00
Jacob Dubin
d52c4e6e19 Add body and mission personality prompts 2026-05-21 18:05:39 -05:00
Jacob Dubin
b0709dd25e Add identity and knowledge legacy MIM replies 2026-05-21 17:55:02 -05:00
Jacob Dubin
5422febb8c Add deep personality Build B prompts 2026-05-21 17:48:20 -05:00
Jacob Dubin
eeef2b3beb Polish grocery list alias wording and backlog MVP decision 2026-05-21 17:00:29 -05:00
Jacob Dubin
acdc6da286 Add structured headlines to news payload 2026-05-21 16:50:43 -05:00
Jacob Dubin
febceecab8 Add binary media manifest metadata 2026-05-21 16:41:23 -05:00
Jacob Dubin
791fe60612 Add capture bundle helper for group testing 2026-05-21 16:37:54 -05:00
Jacob Dubin
3d016debe5 Add low-signal short-turn screening 2026-05-21 15:12:34 -05:00
Jacob Dubin
764a2b2d4f Personalize how_are_you replies with remembered names 2026-05-21 14:31:59 -05:00
Jacob Dubin
b75d9f7941 Expand mood small-talk intent routing 2026-05-21 12:54:51 -05:00
Jacob Dubin
303d8830b0 Route home/back greetings to welcome back 2026-05-21 12:10:58 -05:00
Jacob Dubin
c3b2e5fc2c Refine proactive greetings with morning and return-visit tones 2026-05-21 12:09:30 -05:00
Jacob Dubin
e8d7bafcd6 Add birthday and holiday special-day greetings 2026-05-21 12:05:46 -05:00
Jacob Dubin
70b1b1547f Track durable greeting history for presence-aware greetings 2026-05-21 12:01:13 -05:00
Jacob Dubin
4989889608 Expand friendship parsing guardrails 2026-05-21 11:53:40 -05:00
Jacob Dubin
40deecf2ff Add friendship persona responses 2026-05-21 11:29:44 -05:00
Jacob Dubin
e85792ac57 Add source-backed singing personality responses 2026-05-21 11:22:20 -05:00
Jacob Dubin
a398689851 Extract launch decisions into partial 2026-05-21 09:54:36 -05:00
Jacob Dubin
c883297f26 Extract command route builders from JiboInteractionService 2026-05-21 09:51:38 -05:00
Jacob Dubin
5fa13a65a2 Extract memory and personality decision builders 2026-05-21 09:40:21 -05:00
Jacob Dubin
e5e8e72dbf line endings 2026-05-21 08:53:50 -05:00
Jacob Dubin
a72991dfcb Remove obsolete code paths and simplify implementation 2026-05-21 08:50:15 -05:00
Jacob Dubin
0a0a94502a Remove obsolete code paths 2026-05-21 08:46:24 -05:00
Jacob Dubin
aebfe2e38d Normalize location abbreviations in speech 2026-05-21 07:51:11 -05:00
Jacob Dubin
0f9f91f79a Extract report helpers from JiboInteractionService 2026-05-21 07:45:47 -05:00
Jacob Dubin
a0d6102399 Extract report decision helpers into partial service 2026-05-21 07:39:52 -05:00
Jacob Dubin
2bf686f791 more refactors 2026-05-21 07:31:11 -05:00
Jacob Dubin
c4c512497c refactors 2026-05-21 07:20:31 -05:00
Jacob Dubin
6138ef1c3e Extract seasonal holiday routing into dedicated builder 2026-05-21 07:02:56 -05:00
Jacob Dubin
bba1dfdcfc Refactor scripted response decision helpers 2026-05-21 06:53:02 -05:00
Jacob Dubin
e85e14fbd3 Add seasonal spring and summer scripted responses 2026-05-21 06:36:25 -05:00
Jacob Dubin
bedb5d1715 Polish news and personal report phrasing 2026-05-21 06:28:16 -05:00
Jacob Dubin
eb509a66e0 Polish weather news and STT filtering 2026-05-21 06:09:27 -05:00
Jacob Dubin
1b9efc4226 Add identity charm mims for Jibo persona 2026-05-21 05:53:19 -05:00
Jacob Dubin
fff342fd18 Wrap personal report in report-skill payload 2026-05-21 00:28:05 -05:00
Jacob Dubin
884b2215c7 Document commute provider seam for personal report 2026-05-20 23:25:41 -05:00
Jacob Dubin
c76af83d7e Add holiday seasonal routing and calendar report seam 2026-05-20 20:03:14 -05:00
Jacob Dubin
39b21d1326 Add remaining seasonal holiday polish 2026-05-20 07:20:47 -05:00
Jacob Dubin
9f2a8fd7e1 Add Santa Tracker and penguin favorites 2026-05-20 07:15:30 -05:00
Jacob Dubin
b172a00454 Rehydrate default loop on empty snapshot 2026-05-19 20:06:03 -05:00
Jacob Dubin
30493d554b cleanup 2026-05-19 19:28:04 -05:00
Jacob Dubin
5ad6d4e673 Add holiday buckets and birthday authoring 2026-05-19 19:12:34 -05:00
6ac0c794e4 revert 07d7c83559
revert Tweak how mims work to get better responses
2026-05-19 20:11:06 +00:00
07d7c83559 Tweak how mims work to get better responses 2026-05-19 16:01:12 -04:00
c17c3db0a2 Add more mims 2026-05-19 14:26:03 -04:00
Jacob Dubin
2bc6fec1bf Add loop-scoped holiday list support 2026-05-19 06:57:09 -05:00
Jacob Dubin
54b32bc9cf refactor 2026-05-19 06:03:34 -05:00
Jacob Dubin
6bae858da9 cleanup 2026-05-19 06:00:42 -05:00
Jacob Dubin
d3f9de9503 Add commute report support to interaction service 2026-05-18 06:33:35 -05:00
Jacob Dubin
b25793443f Add calendar service-down support to personal report 2026-05-17 23:47:51 -05:00
Jacob Dubin
e588f00c43 Add sleep and spin-around global command handling 2026-05-17 23:38:29 -05:00
Jacob Dubin
51e36bc492 Expand favorites with longer Pegasus-style phrasing 2026-05-17 20:35:28 -05:00
Jacob Dubin
9ffdd6d09e Document longer persona variants and post-release dialog composition 2026-05-17 20:29:39 -05:00
Jacob Dubin
af76cbaee2 Fix personal report yes/no and weather hi/low handling 2026-05-17 20:20:48 -05:00
Jacob Dubin
f2826253d5 Expand surprise facts into robot and human categories 2026-05-17 17:50:01 -05:00
Jacob Dubin
8ed4763df5 Add proactive fun facts, jokes, and current location 2026-05-17 17:41:55 -05:00
Jacob Dubin
9353e8d2e3 Add binary-safe media storage seam 2026-05-17 16:47:58 -05:00
Jacob Dubin
14b5cb74cc Add capture index manifest for group testing 2026-05-17 14:07:56 -05:00
Jacob Dubin
c0485da46d Normalize loose STT transcripts before routing 2026-05-17 14:02:47 -05:00
Jacob Dubin
193fa56847 Clarify update backup restore rehydration proof 2026-05-17 13:47:44 -05:00
Jacob Dubin
a2aa9df46a Normalize transcripts and expand speech regressions 2026-05-17 12:39:46 -05:00
Jacob Dubin
d8949fcc9a Tighten STT noise filtering and preserve yes-no replies 2026-05-17 11:18:57 -05:00
Jacob Dubin
3b279fdd6f Implement update backup and restore proof 2026-05-17 10:11:36 -05:00
Jacob Dubin
dfcf521a5a refactors 2026-05-17 08:08:11 -05:00
Jacob Dubin
05efeb2853 Add Azure Blob startup profile and config sample 2026-05-17 08:01:04 -05:00
Jacob Dubin
478a320581 Wire selectable persistence backend for cloud state stores 2026-05-17 07:15:12 -05:00
Jacob Dubin
888f472f69 Introduce pluggable snapshot storage for persistence 2026-05-17 07:02:49 -05:00
Jacob Dubin
785dc2b48b Extract JSON snapshot persistence helpers 2026-05-17 06:58:12 -05:00
Jacob Dubin
d37521281e Add persistence seams for durable state adapters 2026-05-17 00:47:36 -05:00
Jacob Dubin
5d57095ce5 Document persistence architecture and report-skill parity 2026-05-17 00:41:09 -05:00
Jacob Dubin
a8a153e910 Vendor Pegasus report-skill templates for weather and reports 2026-05-17 00:30:00 -05:00
Jacob Dubin
a47c90c9c3 Polish weather phrasing for current and forecast replies 2026-05-16 10:16:34 -05:00
Jacob Dubin
393c34055d Prefer forecast hi lo for current weather 2026-05-16 10:01:40 -05:00
Jacob Dubin
f9b728c2a0 Add seasonal and presence charm batches 2026-05-16 09:09:18 -05:00
Jacob Dubin
c87af4686c Add seasonal Build B legacy MIM imports 2026-05-16 08:53:07 -05:00
Jacob Dubin
84759f51de Add Build B charm descriptors and mood replies 2026-05-16 08:39:45 -05:00
Jacob Dubin
c8beb0d1f0 Expand Build B mood and persona follow-ups 2026-05-16 08:05:42 -05:00
Jacob Dubin
e43b4f05f0 Port more legacy charm MIMs into Build B 2026-05-14 22:21:36 -05:00
Jacob Dubin
2677cf9dac Refine commit message handling 2026-05-14 22:06:25 -05:00
Jacob Dubin
20b84632ec Close memory recall turns so recognition does not keep mic open 2026-05-14 21:58:00 -05:00
Jacob Dubin
5718edecaf shortened timeout in conversation broker 2026-05-14 21:47:10 -05:00
Jacob Dubin
40b5b8e4a8 Expand persona follow-ups for identity and favorites 2026-05-14 21:39:17 -05:00
Jacob Dubin
8f7c118fb3 Expand persona inventory and add favorite-family prompts 2026-05-14 21:16:50 -05:00
Jacob Dubin
c30363ec9f Add person-aware favorites and multitenant state scaffolding 2026-05-14 21:15:14 -05:00
Jacob Dubin
ec786be797 Add person-aware state and sync roadmap 2026-05-14 20:48:55 -05:00
Jacob Dubin
f299cef9be Add stateful shopping and to-do list follow-ups 2026-05-14 07:44:46 -05:00
Jacob Dubin
f5e37729ab Expand legacy Build A routing and emotion replies 2026-05-14 06:48:39 -05:00
Jacob Dubin
7297017250 Port legacy persona and emotion replies 2026-05-14 06:44:22 -05:00
Jacob Dubin
66b89f3cee Add Build A legacy MIM import support 2026-05-13 23:22:05 -05:00
Jacob Dubin
11a3e4ef13 Add legacy MIM importer and seed Build A content 2026-05-13 23:18:18 -05:00
Jacob Dubin
7c6dacdbd8 Fix weather, yes/no, and news integrations 2026-05-12 20:36:43 -05:00
Jacob Dubin
9093b429ca Harden weather date parsing and add request diagnostics 2026-05-12 07:52:38 -05:00
Jacob Dubin
df3b34c8ad Add weekly weather cards and improve news API fallback 2026-05-11 22:44:56 -05:00
Jacob Dubin
67c738fae3 Improve weather and news diagnostics for report skills 2026-05-11 19:59:15 -05:00
Jacob Dubin
c0e9b41cd1 Revert weather report-skill routing to stabilize playback 2026-05-11 07:26:56 -05:00
Jacob Dubin
af2fdd230c Improve weather routing and news API fallback 2026-05-11 07:15:11 -05:00
Jacob Dubin
0c597ebbf8 Fix weather forecast parsing and NewsAPI fallback 2026-05-10 23:08:06 -05:00
Jacob Dubin
4bc87f927b Broaden yes no parsing for proactive follow ups 2026-05-10 21:22:25 -05:00
Jacob Dubin
a94b7ec493 Merge branch 'main' of https://kevinblog.sytes.net/Code/Jibo-Revival-Group/JiboExperiments 2026-05-10 20:31:46 -05:00
Jacob Dubin
8c17ad4035 Update commit message generation 2026-05-10 20:31:07 -05:00
4259 changed files with 767889 additions and 6619 deletions

1
.gitignore vendored
View File

@@ -420,3 +420,4 @@ FodyWeavers.xsd
OpenJibo/captures/ OpenJibo/captures/
OpenJibo/.tmp/ OpenJibo/.tmp/
OpenJibo/docs/DesignDoc/original server

View File

@@ -1,7 +1,8 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<configuration> <configuration>
<packageSources> <packageSources>
<clear /> <clear />
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" /> <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
</packageSources> </packageSources>
</configuration> </configuration>

View File

@@ -1,15 +1,26 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation"> <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/UserDictionary/Words/=ampm/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=ampm/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Arrrr/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=Arrrr/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=bday/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=bleebo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=didn/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=didnt/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=dont/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=esml/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=esml/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Hotphrase/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=Hotphrase/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=jiboji/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=jibos/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo_0027s/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo_0027s/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=mult/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=multichunk/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=multichunk/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=nevermind/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=nevermind/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=noinput/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=noinput/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=onomies/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Photobooth/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=Photobooth/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=photogal/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=roboting/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slnx/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=slnx/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slowdance/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=slowdance/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=timecoded/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=timecoded/@EntryIndexedValue">True</s:Boolean>

View File

@@ -3,12 +3,17 @@
<File Path="docs/development-plan.md" /> <File Path="docs/development-plan.md" />
<File Path="docs/device-bootstrap.md" /> <File Path="docs/device-bootstrap.md" />
<File Path="docs/feature-backlog.md" /> <File Path="docs/feature-backlog.md" />
<File Path="docs/greetings-presence-plan.md" />
<File Path="docs/live-jibo-capture.md" /> <File Path="docs/live-jibo-capture.md" />
<File Path="docs/live-jibo-test-runbook.md" /> <File Path="docs/live-jibo-test-runbook.md" />
<File Path="docs/personal-report-parity-plan.md" />
<File Path="docs/protocol-inventory.md" /> <File Path="docs/protocol-inventory.md" />
<File Path="docs/public-site-plan.md" /> <File Path="docs/public-site-plan.md" />
<File Path="docs/regression-test-plan.md" /> <File Path="docs/regression-test-plan.md" />
<File Path="docs/release-1.0.19-plan.md" />
<File Path="docs/roadmap.md" />
<File Path="docs/support-tiers.md" /> <File Path="docs/support-tiers.md" />
<File Path="docs/system-diagram-alignment.md" />
</Folder> </Folder>
<Folder Name="/docs/prompts/"> <Folder Name="/docs/prompts/">
<File Path="docs/prompts/cloud-deploy-and-jibo-rcm-path.md" /> <File Path="docs/prompts/cloud-deploy-and-jibo-rcm-path.md" />

View File

@@ -14,6 +14,16 @@ We are rebuilding the hosted cloud first, then using that foundation for OTA, Op
Current release truth lives in [docs/development-plan.md](docs/development-plan.md). The current cloud release constant is `1.0.19`. Current release truth lives in [docs/development-plan.md](docs/development-plan.md). The current cloud release constant is `1.0.19`.
## Running Locally
For local setup, use [docs/local-cloud-quickstart.md](docs/local-cloud-quickstart.md).
It covers:
- the current `.NET` OpenJibo cloud
- the legacy Node protocol oracle
- the Playground direct-to-Jibo ASR/TTS demo
## Roadmap ## Roadmap
The long-range plan is summarized in [docs/roadmap.md](docs/roadmap.md). In short: The long-range plan is summarized in [docs/roadmap.md](docs/roadmap.md). In short:
@@ -94,6 +104,7 @@ OpenJibo/
Use these when you want the active technical truth: Use these when you want the active technical truth:
- [Development plan](docs/development-plan.md) - [Development plan](docs/development-plan.md)
- [Local cloud quickstart](docs/local-cloud-quickstart.md)
- [Feature backlog](docs/feature-backlog.md) - [Feature backlog](docs/feature-backlog.md)
- [Release 1.0.19 plan](docs/release-1.0.19-plan.md) - [Release 1.0.19 plan](docs/release-1.0.19-plan.md)
- [Support tiers](docs/support-tiers.md) - [Support tiers](docs/support-tiers.md)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,777 @@
# Hub Service Design Document
## Overview
The Hub Service is the central orchestrator of the Jibo cloud system. It coordinates all communication between the robot and cloud services, managing speech recognition, natural language understanding, skill routing, and proactive behaviors. The Hub exposes WebSocket endpoints for real-time bidirectional communication with the robot.
## Location
`packages/hub/src/`
## Key Components
### HubService (`HubService.ts`)
Main service class extending `BaseService` from `@jibo/utils`. Initializes and manages all hub components.
**HubComponents** (dependency injection container):
- `parser: ParserClient` - NLU service client
- `skillConfigManager: SkillConfigManager` - Manages skill configurations
- `intentRouter: IntentRouter` - Routes intents to skills
- `skillRequestMaker: SkillRequestMaker` - Makes HTTP requests to skills
- `history: HistoryServiceClient` - History service client
- `hubSettings: HubSettings` - Hub configuration
- `settingsClient: SettingsClient` - Settings service client
### WebSocket Handlers
- **ListenHandler** (`listen/ListenHandler.ts`) - Handles `/listen` and `/v1/listen` endpoints
- **ProactiveSocketRequestHandler** (`proactive/ProactiveSocketRequestHandler.ts`) - Handles `/proactive` and `/v1/proactive` endpoints
### Transaction Handlers
- **ListenTransactionHandler** (`listen/ListenTransactionHandler.ts`) - State machine for listen transactions
- **ProactiveTransactionHandler** (`proactive/ProactiveTransactionHandler.ts`) - Handles proactive action selection
## WebSocket Endpoints
### Listen Endpoint
**URL:** `ws://hub:9000/listen` or `ws://hub:9000/v1/listen`
**Authentication:** Bearer JWT token in Authorization header
**Headers:**
- `x-jibo-transid` - Transaction ID
- `x-jibo-robotid` - Robot ID
- `x-jibo-logging-config` - Log level configuration
### Proactive Endpoint
**URL:** `ws://hub:9000/proactive` or `ws://hub:9000/v1/proactive`
**Authentication:** Same as listen endpoint
## Listen Transaction Flow
The listen transaction follows a state machine with the following states:
```
WAIT_LISTEN → ASR → NLU → ROUTE → DONE
WAIT_LISTEN → WAIT_CLIENT_ASR → NLU → ROUTE → DONE
WAIT_LISTEN → WAIT_CLIENT_NLU → ROUTE → DONE
```
### State Machine Implementation
**File:** `packages/hub/src/listen/ListenTransactionHandler.ts`
**States:**
- `WAIT_LISTEN` - Waiting for LISTEN message from robot
- `WAIT_CLIENT_ASR` - Waiting for client-provided ASR result
- `WAIT_CLIENT_NLU` - Waiting for client-provided NLU result
- `ASR` - Performing speech recognition
- `NLU` - Performing natural language understanding
- `ROUTE` - Routing to appropriate skill
- `DONE` - Transaction complete
- `STOP` - Transaction stopped
**Timeouts:**
- ASR: 40 seconds (configurable via sosTimeout, maxSpeechTimeout)
- Parser: 10 seconds
- Context: 5 seconds
- Skill: 10 seconds
- Transaction: 60 seconds (default)
### Robot-to-Hub Messages (Listen Flow)
1. **LISTEN** - Initiates listen transaction
```typescript
{
type: "LISTEN",
msgID: "uuid",
ts: 1234567890,
data: {
mode: "default" | "CLIENT_ASR" | "CLIENT_NLU",
lang: "en-US",
hotphrase: boolean,
rules: string[],
asr: {
sosTimeout: number,
maxSpeechTimeout: number,
hints: string[],
earlyEOS: string[]
},
agents: ExternalAgentRequest[]
}
}
```
2. **Audio Packets** - Binary audio data streamed after LISTEN
3. **CONTEXT** - Runtime context from robot
```typescript
{
type: "CONTEXT",
msgID: "uuid",
ts: 1234567890,
data: {
general: {
accountID: string,
robotID: string,
lang: string,
release: string
},
runtime: {
character: { emotion, motivation },
location: { city, state, country, lat, lng },
loop: { users, jibo, owner, loopId },
perception: { speaker, peoplePresent },
dialog: { referent }
},
skill: {
id: string,
session: { id, nodeID, data, trace }
}
}
}
```
4. **CLIENT_ASR** - Client-provided ASR result (for menu clicks, etc.)
```typescript
{
type: "CLIENT_ASR",
msgID: "uuid",
ts: 1234567890,
data: {
text: string
}
}
```
5. **CLIENT_NLU** - Client-provided NLU result
```typescript
{
type: "CLIENT_NLU",
msgID: "uuid",
ts: 1234567890,
data: {
intent: string,
entities: {},
rules: []
}
}
```
### Hub-to-Robot Messages (Listen Flow)
#### 1. SOS (Start of Speech)
**Emitted when:** Speech is detected during ASR
**Location:** `ListenTransactionHandler.emitSOS()`
```typescript
{
type: "SOS",
msgID: "uuid",
ts: 1234567890,
data: null,
timings: {
total: number
}
}
```
**Trigger conditions:**
- Google Cloud Speech API detects start of speech
- ASRSession calls `onStartOfSpeech` callback
- Clears SOS timeout timer
#### 2. EOS (End of Speech)
**Emitted when:** Speech ends during ASR
**Location:** `ListenTransactionHandler.emitEOS()`
```typescript
{
type: "EOS",
msgID: "uuid",
ts: 1234567890,
data: null,
timings: {
total: number
}
}
```
**Trigger conditions:**
- Google Cloud Speech API detects end of speech
- ASRSession calls `onEndOfSpeech` callback
- Clears max speech timeout timer
#### 3. LISTEN Response (ASR/NLU Result)
**Emitted when:** ASR and NLU processing complete
**Location:** `ListenTransactionHandler.emitListenResult()`
```typescript
{
type: "LISTEN",
msgID: "uuid",
ts: 1234567890,
data: {
asr: {
text: string,
confidence: number,
annotation: "GARBAGE" | "SOS_TIMEOUT" | "MAX_SPEECH_TIMEOUT"
},
nlu: {
intent: string,
entities: {},
rules: []
},
match: {
skillID: string,
launch: boolean,
onRobot: boolean
} | null
},
final: boolean,
timings: {
total: number,
asr: number,
nlu: number
}
}
```
**Emission scenarios:**
- **No match:** `match: null, final: true` - No skill matched the NLU result
- **On-robot skill:** `match.onRobot: true, final: true` - Skill runs on robot, Hub done
- **Cloud skill:** `match.onRobot: false, final: false` - Skill runs in cloud, Hub will send skill actions
#### 4. SKILL_ACTION
**Emitted when:** Cloud skill returns an action to execute
**Location:** `TransactionHandler.emitSkillResult()`
```typescript
{
type: "SKILL_ACTION",
msgID: "uuid",
ts: 1234567890,
data: {
action: {
type: "JCP",
config: {
version: "1.0.0",
jcp: SupportedBehaviors // SLIM, Sequence, Parallel, SetPresentPerson, ImpactEmotion
}
},
analytics?: AnalyticsData,
fireAndForget?: boolean
},
final: boolean,
timings: {
total: number,
skill: number
}
}
```
**JCP Behavior Types:**
- `SLIM` - Single behavior execution
- `Sequence` - Sequential behavior execution
- `Parallel` - Parallel behavior execution
- `SetPresentPerson` - Set focused person
- `ImpactEmotion` - Modify Jibo's emotional state
**Emission scenarios:**
- **Non-final:** `final: false` - Robot should execute action and send CMD_RESULT back
- **Final:** `final: true` - Transaction complete, no more actions expected
#### 5. SKILL_REDIRECT
**Emitted when:** Skill redirects to another skill
**Location:** `TransactionHandler.emitSkillRedirectNotification()`
```typescript
{
type: "SKILL_REDIRECT",
msgID: "uuid",
ts: 1234567890,
data: {
match: {
skillID: string,
launch: boolean,
onRobot: boolean
},
nlu: NLUResult,
asr: ASRResult,
memo: any
},
final: boolean
}
```
**Emission scenarios:**
- Skill returns `SKILL_REDIRECT` response
- Hub launches new skill with provided context
- Only one level of redirect supported (error on second redirect)
#### 6. ERROR
**Emitted when:** An error occurs during transaction
**Location:** `TransactionHandler.emitSkillResult()` (error case)
```typescript
{
type: "ERROR",
msgID: "uuid",
ts: 1234567890,
data: {
message: string
},
final: true,
timings: {
total: number
}
}
```
### Listen Transaction State Transitions
#### WAIT_LISTEN → ASR
**Trigger:** LISTEN message received with mode="default"
**Actions:**
- Initialize ASRSession with Google Cloud Speech API
- Start audio streaming
- Set up SOS timeout (if configured)
- Set up max speech timeout (if configured)
#### WAIT_LISTEN → WAIT_CLIENT_ASR
**Trigger:** LISTEN message received with mode="CLIENT_ASR"
**Actions:**
- Emit fake SOS (immediate)
- Wait for CLIENT_ASR message from robot
#### WAIT_LISTEN → WAIT_CLIENT_NLU
**Trigger:** LISTEN message received with mode="CLIENT_NLU"
**Actions:**
- Emit fake SOS (immediate)
- Wait for CLIENT_NLU message from robot
#### ASR → NLU
**Trigger:** ASR completes successfully
**Actions:**
- Stop ASR session
- Normalize ASR text
- Check for garbage annotation (skip NLU if garbage)
- Wait for CONTEXT message (5 second timeout)
- Send ASR text to Parser service
#### WAIT_CLIENT_ASR → NLU
**Trigger:** CLIENT_ASR message received
**Actions:**
- Use provided ASR text
- Emit fake EOS
- Proceed to NLU
#### WAIT_CLIENT_NLU → ROUTE
**Trigger:** CLIENT_NLU message received
**Actions:**
- Use provided NLU result
- Emit fake EOS
- Skip NLU, proceed to routing
#### NLU → ROUTE
**Trigger:** Parser returns NLU result
**Actions:**
- Wait for CONTEXT message (5 second timeout)
- Call IntentRouter to match skill
- Apply DecisionMediator for external factors
- Route to matched skill or context skill
#### ROUTE → DONE
**Trigger:** Routing complete
**Actions:**
- For on-robot skills: Emit LISTEN with match, transaction done
- For cloud skills: Get skill response, emit SKILL_ACTION, transaction done
- For no match: Emit LISTEN with match=null, transaction done
## Intent Routing
### IntentRouter (`intent/IntentRouter.ts`)
Matches NLU results to registered cloud skills.
**Routing Logic:**
1. Check if NLU has intent and 'launch' rule
2. Query all skill configurations
3. Match intent against skill intent configurations
4. Match entities against skill entity configurations
5. Return first matching skill decision
**DecisionMediator** (`intent/DecisionMediator.ts`):
- Can alter routing decisions based on external factors
- Considers robot release version
- May redirect to different skill based on context
**IRDecisionMaker** (`intent/IRDecisionMaker.ts`):
- Core matching algorithm
- Compares intent names and entity values
- Supports exact match and NOT match rules
### Skill Request Maker (`skill/SkillRequestMaker.ts`)
Makes HTTP requests to cloud skills.
**Methods:**
- `skillLaunch(skillID, data, jiboHeaders, log)` - Launch new skill
- `skillLaunchOrUpdate(skillID, data, jiboHeaders, log, update)` - Launch or update skill
- `proactiveLaunch(skillID, data, jiboHeaders, log)` - Proactive launch
**Request Format:**
```typescript
{
type: "LISTEN_LAUNCH" | "LISTEN_UPDATE" | "PROACTIVE_LAUNCH",
msgID: "uuid",
ts: 1234567890,
data: {
general: { accountID, robotID, lang, release },
runtime: { character, location, loop, perception, dialog },
skill: { id, session? },
result?: any, // For UPDATE
nlu: NLUResult,
asr: ASRResult,
memo?: any
}
}
```
**Timeout:** 10 seconds (configurable)
**Error Handling:**
- `SKILL_NOT_FOUND` - Skill does not exist or is on-robot
- `TIMEOUT` - Skill request timeout
## Proactive Flow
### Proactive Transaction Handler (`proactive/ProactiveTransactionHandler.ts`)
Handles proactive action selection based on context, history, and settings.
### Robot-to-Hub Messages (Proactive Flow)
1. **TRIGGER** - Initiates proactive selection
```typescript
{
type: "TRIGGER",
msgID: "uuid",
ts: 1234567890,
data: {
triggerData: {
triggerType: string,
looperID?: string
},
triggerSource: "SURPRISE" | "OTHER"
}
}
```
2. **CONTEXT** - Runtime context (same as listen flow)
### Hub-to-Robot Messages (Proactive Flow)
#### PROACTIVE Match Response
**Emitted when:** Proactive action selected
**Location:** `ProactiveTransactionHandler.emitMatchResponse()`
```typescript
{
type: "PROACTIVE",
msgID: "uuid",
ts: 1234567890,
data: {
match: {
skillID: string,
onRobot: boolean,
isProactive: true,
launch: true,
skipSurprises: boolean
}
},
final: boolean
}
```
**Emission scenarios:**
- **On-robot skill:** `final: true` - Robot handles skill, Hub done
- **Cloud skill:** `final: false` - Hub will send skill actions
#### PROACTIVE No-Action Response
**Emitted when:** No eligible proactive action found
**Location:** `ProactiveTransactionHandler.emitNoActionResponse()`
```typescript
{
type: "PROACTIVE",
msgID: "uuid",
ts: 1234567890,
data: {},
final: true
}
```
### Proactive Action Selection Algorithm
**File:** `ProactiveTransactionHandler.getEligibleActions()`
**Steps:**
1. **Get all proactive skill configurations**
- Query SkillConfigManager for skills with proactive registrations
2. **Gather transaction data**
- Extract focused person, present people, loop ID, robot ID
- Use ContextTools to extract context fields
3. **Fetch user settings** (if focused person)
- Batch request to SettingsClient for all skill settings
- Consolidate into skill settings map
4. **Filter by context rules**
- Check time-based rules (time of day, day of week)
- Check location rules
- Check people present rules
- Check robot state rules
5. **Filter by interaction history rules**
- Query History service for past interactions
- Check frequency rules (e.g., "at most once per hour")
- Check recency rules (e.g., "not in last 10 minutes")
- Check sequence rules (e.g., "after greeting skill")
6. **Filter by settings rules**
- Check user preferences for each skill
- Check enabled/disabled status
- Check custom parameters
7. **Select action**
- Currently: Random selection from eligible actions
- Future: Heuristics based on context, engagement, topics
### Context Tools (`proactive/tools/ContextTools.ts`)
Helper functions for context rule evaluation:
- `extractContextData(field, context, requestData, log)` - Extract specific context field
- `checkContextRules(registration, context, requestData, log)` - Evaluate all context rules
### History Rules Checker (`proactive/tools/IHRulesChecker.ts`)
Evaluates interaction history rules:
- `checkIHRules(registrations, IHQueries, data, log)` - Filter by history rules
- Queries History service for past skill launches
- Applies frequency, recency, and sequence constraints
### Settings Rules Checker (`proactive/tools/SettingsRulesChecker.ts`)
Evaluates user settings:
- `getSkillSettingsMap(skillConfigs, accountID, loopID, transID)` - Batch fetch settings
- `checkSettingsRegistrations(registrations, skillSettingsMap)` - Filter by settings
## Skill Interaction Flow (Cloud Skills)
### Initial Launch
1. Hub sends LISTEN_LAUNCH request to skill
2. Skill processes request, returns SKILL_ACTION
3. Hub sends SKILL_ACTION to robot
4. Robot executes action, sends CMD_RESULT to Hub
5. Hub sends LISTEN_UPDATE request to skill with action result
6. Skill processes result, returns next SKILL_ACTION or final=true
7. Repeat steps 3-6 until skill returns final=true
### Skill Redirect
1. Skill returns SKILL_REDIRECT response
2. Hub emits SKILL_REDIRECT notification to robot
3. Hub sends launch request to new skill
4. New skill proceeds with normal flow
5. Error if second redirect attempted
## Message Timing
### Listen Transaction Timing
**Timings tracked:**
- `total` - Total transaction time
- `asr` - ASR processing time
- `nlu` - NLU processing time
- `skill` - Skill processing time
**Timing emission:**
- SOS/EOS include timing from start
- LISTEN response includes ASR and NLU timings
- SKILL_ACTION includes skill timing
### Proactive Transaction Timing
**Timings tracked:**
- `total` - Total transaction time
- `skill` - Skill processing time
## Error Handling
### Hub Error Codes (`HubErrorCode.ts`)
- `TIMEOUT_ASR` - ASR timeout (40 seconds)
- `TIMEOUT_PARSER` - Parser timeout (10 seconds)
- `TIMEOUT_CONTEXT` - Context timeout (5 seconds)
- `TIMEOUT_SKILL` - Skill timeout (10 seconds)
- `PARSER` - Parser error
- `ASR` - ASR error
### Error Response Format
```typescript
{
type: "ERROR",
msgID: "uuid",
ts: 1234567890,
data: {
message: string,
code?: string
},
final: true,
timings: {
total: number
}
}
```
## Speech History Recording
### Optional Features
**Configuration:**
- `ETCO_hub_recordLaunchHistory` - Record skill launches to MongoDB
- `ETCO_hub_recordSpeechHistory` - Record speech interactions to MongoDB
- `ETCO_hub_recordSpeechLogBucket` - Upload speech logs to S3
### Speech History Record
**Data recorded:**
- Robot ID, account ID, transaction ID
- Timestamp
- ASR result
- NLU result
- Match data
- Skill response
- Redirect data
- Error (if any)
### S3 Upload
**Format:** JSON with audio as base64
**Path:** `{robotID}/year={year}/month={month}/day={day}/{timestamp}-{transID}.json`
## Hub Configuration
### Environment Variables
**Hub Settings:**
- `ETCO_hub_recordLaunchHistory` - Enable launch history
- `ETCO_hub_recordSpeechHistory` - Enable speech history
- `ETCO_hub_recordSpeechLogBucket` - S3 bucket for speech logs
**Authentication:**
- `ETCO_server_hubTokenSecret` - JWT secret for token verification
### Skill Configuration
**Sources:**
- `skills-local.json` - Local development configuration
- Environment variables - Production configuration
- Settings service - Dynamic configuration
**Skill Config Structure:**
```typescript
{
id: string,
intents: [{
name: string,
entities?: [{ name, value, matchRule }],
memo?: any
}],
proactives?: [{
triggerType: string,
contextRules?: ContextRule[],
IHRules?: IHRule[],
settingsRules?: SettingsRule[],
memo?: any
}],
IHQueries?: IHQueryDefinitions,
onRobot?: boolean,
URL: string,
settings?: ManifestSettings
}
```
## Summary of Server-to-Robot Communication
### Listen Flow
1. **SOS** - Speech detected
2. **EOS** - Speech ended
3. **LISTEN** - ASR/NLU result with match data
4. **SKILL_ACTION** - JCP action to execute (repeated for multi-turn)
5. **SKILL_REDIRECT** - Skill redirect notification
6. **ERROR** - Error occurred
### Proactive Flow
1. **PROACTIVE** - Match or no-action response
2. **SKILL_ACTION** - JCP action to execute (if cloud skill)
3. **SKILL_REDIRECT** - Skill redirect notification
4. **ERROR** - Error occurred
### Key Design Principles
1. **State Machine** - Clear state transitions with validation
2. **Timeouts** - Every operation has a timeout to prevent hanging
3. **Error Handling** - Errors propagate to robot with clear messages
4. **Timing** - All operations are timed for monitoring
5. **History** - All interactions are recorded for analysis
6. **Flexibility** - Supports on-robot and cloud skills
7. **Proactivity** - Context-aware action selection

View File

@@ -0,0 +1,792 @@
# Original Jibo Server (Pegasus) Design Document
## Executive Summary
The original Jibo server, codenamed "Pegasus" (formerly V1.X), is a cloud-based microservices architecture that powers the Jibo social robot's conversational AI capabilities. It is built as a Lerna monorepo using Node.js/TypeScript and deployed via Docker containers. The system processes speech, performs natural language understanding, routes to appropriate skills, and manages proactive behaviors.
## Architecture Overview
### Monorepo Structure
The codebase is organized as a Lerna monorepo with the following main packages:
- **packages/hub** - Central orchestration service
- **packages/parser** - NLU (Natural Language Understanding) service
- **packages/history** - Data persistence service (MongoDB)
- **packages/baseskill** - Base class and framework for cloud skills
- **packages/interfaces** - TypeScript interfaces and API contracts
- **packages/utils** - Shared utility libraries
- **packages/chitchat-skill** - Example conversational skill
- **packages/report-skill** - Reporting skill
- **packages/lasso** - External data integration service
- **packages/hub-client** - Client library for hub communication
- **packages/history-client** - Client library for history service
- **packages/test-utils** - Testing utilities
### Technology Stack
- **Language**: TypeScript 2.5.3
- **Runtime**: Node.js 8.9.4
- **Package Manager**: Yarn 1.7.0
- **Containerization**: Docker
- **Orchestration**: Docker Compose (local), AWS ECS (production)
- **Database**: MongoDB 3.6.0
- **Cache**: Redis 3
- **NLU**: Dialogflow (API.ai)
- **ASR**: Google Cloud Speech API
- **WebSocket**: ws library
- **HTTP**: Express.js
- **Authentication**: JWT (jsonwebtoken)
## Core Services
### 1. Hub Service (`packages/hub`)
The Hub is the central orchestrator that coordinates all interactions between the robot and cloud services.
#### Key Components
**HubService** (`HubService.ts`)
- Main service class extending `BaseService`
- Initializes and manages all hub components
- Registers WebSocket and HTTP handlers
**HubComponents** - Dependency injection container:
- `parser: ParserClient` - NLU service client
- `skillConfigManager: SkillConfigManager` - Manages skill configurations
- `intentRouter: IntentRouter` - Routes intents to skills
- `skillRequestMaker: SkillRequestMaker` - Makes HTTP requests to skills
- `history: HistoryServiceClient` - History service client
- `hubSettings: HubSettings` - Hub configuration
- `settingsClient: SettingsClient` - Settings service client
#### Endpoints
**WebSocket Endpoints:**
- `/listen` and `/v1/listen` - Handles speech recognition and NLU
- `/proactive` and `/v1/proactive` - Handles proactive triggers
**HTTP Endpoints:**
- `/skills` and `/v1/skills` - Lists available skills
- `/healthcheck` - Service health check
#### Listen Flow
The listen transaction follows a state machine implemented in `ListenTransactionHandler`:
```
States:
WAIT_LISTEN → ASR → NLU → ROUTE → DONE
WAIT_LISTEN → WAIT_CLIENT_ASR → NLU → ROUTE → DONE
WAIT_LISTEN → WAIT_CLIENT_NLU → ROUTE → DONE
```
**State Transitions:**
1. **WAIT_LISTEN** - Receives LISTEN message from robot
2. **ASR** - Performs Automatic Speech Recognition using Google Cloud Speech API
- Streams audio packets
- Emits SOS (Start of Speech) when speech detected
- Emits EOS (End of Speech) when speech ends
- Handles timeouts (SOS timeout, max speech timeout)
3. **NLU** - Sends ASR text to Parser service for intent recognition
- Includes context (loop users, perception, etc.)
- Supports external Dialogflow agents
4. **ROUTE** - Intent Router determines which skill to launch
- Matches NLU result against skill intent configurations
- Decision Mediator can alter decisions based on external factors
- Routes to on-robot skills or cloud skills
5. **DONE** - Transaction complete
**Listen Transaction Handler** (`ListenTransactionHandler.ts`):
- Manages audio streaming via `AudioBuffer`
- Creates `ASRSession` for speech recognition
- Handles timeouts (ASR: 40s, Parser: 10s, Context: 5s, Skill: 10s)
- Records speech history to MongoDB and optionally S3
- Supports client-provided ASR/NLU (for menu clicks, etc.)
- Handles skill redirects
#### Proactive Flow
The proactive system allows Jibo to initiate conversations based on context, history, and triggers.
**Proactive Transaction Handler** (`ProactiveTransactionHandler.ts`):
1. Receives TRIGGER message from robot
2. Waits for CONTEXT message (robot state)
3. **Action Selection**:
- Gets all proactive skill configurations
- Filters by context rules (time, location, people present, etc.)
- Filters by interaction history rules (frequency, recency)
- Filters by user settings
- Randomly selects from eligible actions
4. Launches selected skill (on-robot or cloud)
5. Returns match response or no-action response
**Proactive Registration**:
Skills register proactive behaviors with:
- Trigger types (time-based, event-based, surprise)
- Context rules (when this can trigger)
- Interaction history rules (how often it can trigger)
- Settings rules (user preferences)
### 2. Parser Service (`packages/parser`)
The Parser service performs Natural Language Understanding using Dialogflow.
**ParserService** (`ParserService.ts`):
- Starts RobustParser process on port 8787 (optional)
- Initializes Dialogflow client
- Initializes Robust Parser client
- Handles POST requests to `/v1/parse`
- Exposes state at `/state` endpoint
**NLU Pipeline:**
1. Receives text, rules, and context
2. Queries Dialogflow with configured agents
3. Optionally queries Robust Parser (custom NLU)
4. Returns intent, entities, and rules
**Configuration:**
- Dialogflow API key
- Robust Parser enable/disable
- Multiple external agents support
### 3. History Service (`packages/history`)
The History service persists interaction data to MongoDB.
**HistoryService** (`HistoryService.ts`):
- Two database clients:
- `SkillLaunchDBClient` - Records skill launches
- `SpeechHistoryDBClient` - Records speech interactions (optional)
- HTTP endpoints:
- `/v1/skill/launch` - Skill launch history
- `/v1/speech` - Speech history (if enabled)
- Health check endpoint
**Data Stored:**
- Skill launches (skill ID, intent, timestamp, robot ID, account ID)
- Speech interactions (ASR result, NLU result, audio file URL, error tracking)
### 4. Lasso Service (`packages/lasso`)
Lasso provides external data integration for skills.
**Features:**
- OAuth2 credential management
- Calendar client integration
- Weather data (Dark Sky API)
- Maps data (Google Maps API)
- News data (AP News)
- MongoDB for credential storage
- Redis for caching
**LassoService** (`LassoService.ts`):
- Manages OAuth2 flows
- Provides relay endpoints for external APIs
- Caches responses in Redis
## Skill Framework
### BaseSkill (`packages/baseskill`)
**BaseSkill** (`BaseSkill.ts`):
- Abstract base class for all cloud skills
- Extends `BaseHttpHandler`
- Handles POST requests to `/`
- Provides error handling
- Tracks timing
**GraphSkill** (`GraphSkill.ts`):
- Extends BaseSkill with graph-based state machine
- Implements node-based conversation flow
- Supports skill redirects
- Tracks analytics events
- Supports supplemental behaviors (parallel/sequence)
### Graph System
The graph system provides a state machine framework for skills.
**Graph** (`Graph.ts`):
- Directed graph of connected nodes
- Supports subgraphs (hierarchical)
- Exit transitions for graph termination
- Validation (reachability, transition completeness)
- GraphViz dot file generation
**GraphManager** (`GraphManager.ts`):
- Singleton per skill
- Manages node IDs and mappings
- Executes graph:
- `start()` - Creates session, enters initial node
- `enterNode()` - Calls node's enter method
- `exitNode()` - Calls node's exit method with action results
- `executeTransition()` - Moves to next node
- Maintains session state (node ID, data, trace)
**Node** (`Node.ts`):
- Abstract base class for graph nodes
- Has transition names and destinations
- Two lifecycle methods:
- `enter(data)` - Called when node is entered, returns action or redirect
- `exit(data)` - Called with action results, returns next transition
- Supports graph traversal (BFS)
**Built-in Node Types:**
- `DefaultNode` - Simple terminal node
- `JCPNode` - Returns JCP action
- `NoOpNode` - No operation
- `TrueFalseNode` - Conditional branching
- `SetLooperIDNode` - Sets speaker ID
**MIM (Motion Interaction Model) System:**
- `ANFactory` - Creates graph for playing MIM animations
- Supports scripted responses, emotion responses, fallback responses
- Semi-specific responses (context-aware)
### Skill Request/Response Protocol
**Skill Request Types** (`skill/request.ts`):
- `LISTEN_LAUNCH` - Launch skill from listen interaction
- `LISTEN_UPDATE` - Update skill with action results
- `PROACTIVE_LAUNCH` - Launch skill proactively
**Skill Request Data:**
```typescript
{
type: MessageType,
msgID: UUID,
ts: number,
data: {
general: { accountID, robotID, lang, release },
runtime: { character, location, loop, perception, dialog },
skill: { id, session? },
result: any, // Action results for UPDATE
nlu: NLUResult,
asr: ASRResult,
memo?: any
}
}
```
**Skill Response Types** (`skill/response.ts`):
- `SKILL_ACTION` - Returns action to execute
- `SKILL_REDIRECT` - Redirects to another skill
- `ERROR` - Error response
**Skill Action Data:**
```typescript
{
action: JCPAction, // JCP protocol behavior
analytics?: AnalyticsData,
final?: boolean, // Is this the final response?
fireAndForget?: boolean
}
```
**JCP Action** (`skill/action.ts`):
```typescript
{
type: ActionType.JCP,
config: {
version: "1.0.0",
jcp: SupportedBehaviors // SLIM, Sequence, Parallel, SetPresentPerson, ImpactEmotion
}
}
```
### Skill Configuration
**SkillConfig** (`skill/config.ts`):
```typescript
{
id: SkillID,
intents: [{
name: IntentName,
entities?: EntityConfig[],
memo?: any
}],
proactives?: ProactiveRegistration[],
IHQueries?: IHQueryDefinitions,
onRobot?: boolean,
URL: string,
settings?: ManifestSettings
}
```
**Entity Config**:
- `name` - Entity name
- `value` - Expected value
- `matchRule` - 'EXACT' or 'NOT'
**Proactive Registration**:
- Trigger type and conditions
- Context rules
- Interaction history rules
- Settings rules
## Interfaces Package
The `interfaces` package defines all TypeScript interfaces for communication between services.
### Key Interface Modules
**service.ts** - Base message types:
- `BaseMessage<T, D>` - Generic message with type, msgID, timestamp, data
- `BaseResponse<T, D>` - Response with final flag and timings
- `IAuthDetails` - Authentication details (account ID, access keys)
**hub/** - Hub-specific interfaces:
- `request.ts` - LISTEN, CONTEXT, CLIENT_ASR, CLIENT_NLU messages
- `response.ts` - ASR, NLU, LISTEN, SKILL_REDIRECT, ERROR responses
- `MessageType.ts` - Message type enums
- `HubErrorCode.ts` - Error code enums
**skill/** - Skill-specific interfaces:
- `request.ts` - LISTEN_LAUNCH, LISTEN_UPDATE, PROACTIVE_LAUNCH
- `response.ts` - SKILL_ACTION, SKILL_REDIRECT, ERROR
- `action.ts` - JCP action types
- `config.ts` - Skill configuration
- `behaviors.ts` - Supported JCP behaviors
- `analytics.ts` - Analytics event types
**nlu.ts** - NLU interfaces:
- `NLURequestData` - Text, rules, loop users, external agents
- `NLUResult` - Intent, entities, rules
- `ExternalAgentRequest` - External Dialogflow agent config
**asr.ts** - ASR interfaces:
- `ASRResult` - Text, confidence, annotation
- `ASRConfig` - Language, hints, timeouts
**jibo/** - Jibo-specific data:
- `data.ts` - GeneralData (account, robot, language), SkillData (session, trace)
- `runtime.ts` - RuntimeContext (character, location, loop, perception, dialog)
**proactive/** - Proactive interfaces:
- Context field definitions
- History rules
- Settings rules
- Proactive trigger/request/response
**history/** - History interfaces:
- Skill launch data
- Speech history data
## Utils Package
The `utils` package provides shared functionality.
### BaseService (`utils/service/BaseService.ts`)
Base class for all Pegasus services:
**Features:**
- Express.js HTTP server
- WebSocket server (ws library)
- JWT authentication
- Request/response logging with jibo-log
- New Relic monitoring
- Health check endpoint
- Error handling middleware
**Methods:**
- `addSocketHandler(path, handler)` - Register WebSocket handler
- `addHttpHandler(path, handler)` - Register HTTP handler
- `init(port)` - Start server
- `close()` - Stop server
**Authentication:**
- JWT token verification
- Bearer token scheme
- Configurable secret via `ETCO_server_hubTokenSecret`
**Logging:**
- Per-request log instances
- Transaction ID tracking
- Robot ID tracking
- Configurable log levels per namespace
### Other Utils
- `PegasusRequest` - Enhanced Express request with Jibo headers
- `PegasusWebSocket` - Enhanced WebSocket with auth and logging
- `JiboHeaders` - Parses Jibo-specific headers (transID, robotID, logging config)
- `ResponseWrapper` - Wraps WebSocket responses
- `HttpError` - HTTP error with status code
## Communication Protocols
### WebSocket Protocol
**Connection:**
- URL: `ws://hub:9000/listen` or `ws://hub:9000/proactive`
- Authentication: Bearer token in Authorization header
- Headers: `x-jibo-transid`, `x-jibo-robotid`, `x-jibo-logging-config`
**Message Format:**
```json
{
"type": "MESSAGE_TYPE",
"msgID": "uuid",
"ts": 1234567890,
"data": { ... }
}
```
**Listen Flow Messages:**
1. Robot → Hub: LISTEN (with ASR config, rules, language)
2. Robot → Hub: Audio packets (binary)
3. Hub → Robot: SOS (Start of Speech)
4. Robot → Hub: CONTEXT (runtime context)
5. Hub → Robot: EOS (End of Speech)
6. Hub → Robot: LISTEN (with ASR result, NLU result, match)
7. Hub → Robot: SKILL_ACTION (if cloud skill)
8. Robot → Hub: CMD_RESULT (action results)
9. Hub → Robot: SKILL_ACTION (next action) or final
**Proactive Flow Messages:**
1. Robot → Hub: TRIGGER (trigger data)
2. Robot → Hub: CONTEXT (runtime context)
3. Hub → Robot: PROACTIVE (match or no-action)
4. Hub → Robot: SKILL_ACTION (if cloud skill)
### HTTP Protocol
**Skill Request:**
- Method: POST
- URL: `http://skill-host:port/`
- Headers: Authorization, x-jibo-transid, x-jibo-robotid
- Body: SkillRequest JSON
**Parser Request:**
- Method: POST
- URL: `http://parser:8080/v1/parse`
- Body: NLURequestData JSON
## Authentication & Security
### JWT Authentication
**Token Format:**
```json
{
"id": "account-id",
"accessKeyId": "client-id",
"secretAccessKey": "client-secret",
"friendlyId": "robot-name"
}
```
**Verification:**
- Secret: `ETCO_server_hubTokenSecret` environment variable
- Scheme: Bearer
- Applied to WebSocket connections and HTTP endpoints
### Network Security
- All services run in Docker containers
- Services communicate via Docker network (pegasus-nw)
- External access via load balancer
- TLS termination at load balancer
## Deployment
### Docker Compose (Local Development)
**Services:**
- `hub` - Hub service (port 9000)
- `parser` - Parser service (port 9005)
- `history` - History service (port 9006)
- `chitchat-skill` - Chitchat skill (port 9004)
- `report-skill` - Report skill (port 9003)
- `lasso` - Lasso service (port 9007)
- `redis` - Redis cache (port 6379)
- `mongo_lasso` - MongoDB for Lasso (port 27017)
- `history_cluster` - MongoDB for History (from docker-compose-history-db.yml)
**Configuration:**
- Environment variables prefixed with `ETCO_` (ETCO = Environment TO Configuration)
- Volume mounting: `./:/pegasus:consistent` for live code editing
- Debug ports: 5850-5855 for Node.js debugging
### Build Process
**Commands:**
```bash
docker build -t pegasus_base:latest .
yarn docker:bootstrap
yarn docker:build
./pegasus.js build-docker-image --services hub
```
**CLI Tool** (`cli/`):
- `bootstrap` - Install dependencies
- `build` - Build TypeScript
- `test` - Run tests
- `docker-run` - Run commands in Docker
- `build-docker-image` - Build Docker images for services
### Production Deployment
- AWS ECS (Elastic Container Service)
- ECR (Elastic Container Registry) for Docker images
- Application Load Balancer
- MongoDB Atlas for production databases
- ElastiCache for Redis
- CloudWatch for logging
- New Relic for monitoring
## Data Flow Examples
### Example 1: User Says "Tell Me a Joke"
1. **Robot → Hub**: LISTEN message with ASR config
2. **Robot → Hub**: Audio stream
3. **Hub**: Detects SOS, emits SOS message
4. **Hub**: Streams audio to Google Cloud Speech API
5. **Hub**: Detects EOS, emits EOS message
6. **Robot → Hub**: CONTEXT message (runtime state)
7. **Hub → Parser**: POST /v1/parse with text "tell me a joke"
8. **Parser → Dialogflow**: Query with "joke" intent rules
9. **Dialogflow → Parser**: Intent="joke_tell", entities={}
10. **Parser → Hub**: NLU result
11. **Hub → IntentRouter**: Match intent to "joke-skill"
12. **Hub → joke-skill**: POST LISTEN_LAUNCH request
13. **joke-skill**: Executes graph, selects joke
14. **joke-skill → Hub**: SKILL_ACTION with JCP behavior (SayText)
15. **Hub → Robot**: SKILL_ACTION message
16. **Robot**: Executes behavior, speaks joke
17. **Robot → Hub**: CMD_RESULT with action result
18. **Hub → joke-skill**: POST LISTEN_UPDATE request
19. **joke-skill**: Returns final=true
20. **Hub → Robot**: Final SKILL_ACTION
### Example 2: Proactive Greeting
1. **Robot**: Detects person entering room
2. **Robot → Hub**: TRIGGER message with trigger data
3. **Robot → Hub**: CONTEXT message (runtime state)
4. **Hub**: Queries all proactive skill configs
5. **Hub**: Filters by context (time, people present)
6. **Hub**: Filters by history (last greeting time)
7. **Hub**: Filters by settings (user greeting preference)
8. **Hub**: Selects "greeting-skill"
9. **Hub → greeting-skill**: POST PROACTIVE_LAUNCH request
10. **greeting-skill → Hub**: SKILL_ACTION with greeting behavior
11. **Hub → Robot**: PROACTIVE response with match
12. **Hub → Robot**: SKILL_ACTION message
13. **Robot**: Executes greeting
## Error Handling
### Error Types
**Hub Error Codes** (`HubErrorCode.ts`):
- `TIMEOUT_ASR` - ASR timeout
- `TIMEOUT_PARSER` - Parser timeout
- `TIMEOUT_CONTEXT` - Context timeout
- `TIMEOUT_SKILL` - Skill timeout
- `PARSER` - Parser error
- `ASR` - ASR error
**Skill Request Errors** (`SkillRequestError`):
- `SKILL_NOT_FOUND` - Skill does not exist
- `TIMEOUT` - Skill request timeout
### Error Response Format
```json
{
"type": "ERROR",
"msgID": "uuid",
"ts": 1234567890,
"final": true,
"data": {
"message": "Error description",
"code": "ERROR_CODE"
},
"timings": {
"total": 1234
}
}
```
### Timeout Handling
- ASR: 40 seconds (configurable via sosTimeout, maxSpeechTimeout)
- Parser: 10 seconds
- Context: 5 seconds
- Skill: 10 seconds
- Transaction: 60 seconds (configurable)
## Monitoring & Logging
### Logging
**jibo-log Integration:**
- Per-namespace log levels
- Transaction ID correlation
- Robot ID tracking
- Structured logging support
**Log Levels:**
- Configured via `x-jibo-logging-config` header
- Per-namespace granularity
- Environment variable: `ETCO_server_logLevel`
### Monitoring
**New Relic:**
- HTTP request tracking
- WebSocket transaction tracking
- Error tracking
- Custom attributes (transID, robotID)
**Health Checks:**
- `/healthcheck` endpoint on all services
- Returns service-specific health data
- Database connection status
### Speech History Recording
**Optional Features:**
- Record skill launches to MongoDB
- Record speech interactions to MongoDB
- Upload speech logs to S3 (JSON with audio base64)
**Configuration:**
- `ETCO_hub_recordLaunchHistory` - Enable launch history
- `ETCO_hub_recordSpeechHistory` - Enable speech history
- `ETCO_hub_recordSpeechLogBucket` - S3 bucket for speech logs
## Skill Development Guide
### Creating a New Skill
1. **Extend GraphSkill:**
```typescript
export class MySkill extends GraphSkill<Transition> {
constructor() {
super('my-skill');
}
createGraph(): Graph<Transition> {
const g = new Graph('My Skill', generateTransitions<Transition>(Transition));
// Add nodes and transitions
g.finalize();
return g;
}
}
```
2. **Define Transitions:**
```typescript
enum Transition {
Done = 'Done',
Retry = 'Retry'
}
```
3. **Create Nodes:**
```typescript
class MyNode extends Node<Transition> {
async enter(data: Data): Promise<EnterResponse> {
// Return action or redirect
return { action: myJCPAction };
}
async exit(data: Data): Promise<ExitResponse> {
// Return next transition
return { transition: Transition.Done };
}
}
```
4. **Create Skill Manifest:**
```json
{
"id": "my-skill",
"intents": [
{
"name": "my_intent",
"entities": []
}
],
"onRobot": false
}
```
5. **Register with Hub:**
- Add skill config to skills-local.json or environment
- Deploy skill service
- Hub will load configuration
### Skill Best Practices
- Use graph for complex flows, direct responses for simple ones
- Track analytics events for monitoring
- Handle errors gracefully with try-catch
- Use supplemental behaviors for parallel actions
- Set appropriate timeouts
- Log important events
- Test with both LISTEN_LAUNCH and PROACTIVE_LAUNCH
## Key Design Decisions
### Why Graph-Based Skills?
- **State Management**: Explicit state machine with session tracking
- **Visualization**: GraphViz generation for debugging
- **Reusability**: Subgraphs for common patterns
- **Testability**: Isolated node testing
- **Maintainability**: Clear flow structure
### Why WebSocket for Robot Communication?
- **Low Latency**: Real-time bidirectional communication
- **Audio Streaming**: Binary message support for audio
- **Stateful**: Single connection per transaction
- **Efficiency**: No HTTP overhead for each message
### Why Separate Services?
- **Scalability**: Scale each service independently
- **Isolation**: Failure in one service doesn't affect others
- **Technology**: Different services can use different tech stacks
- **Deployment**: Independent deployment cycles
### Why Lerna Monorepo?
- **Code Sharing**: Easy to share interfaces and utils
- **Versioning**: Linked versioning for interdependent packages
- **Development**: Single repository for all services
- **Testing**: Integration tests across packages
## Limitations & Known Issues
1. **Single Graph Manager**: Skills cannot have concurrent sessions (singleton pattern)
2. **Sequential Skill Redirects**: Only one level of redirect supported
3. **No Skill-to-Skill Communication**: Skills must go through hub
4. **Fixed Timeouts**: Hardcoded timeouts in some places
5. **No Skill Hot-Reload**: Requires container rebuild for skill changes
6. **Limited NLU**: Dialogflow dependency, no custom model training
7. **No Skill Versioning**: Skills identified by ID only
8. **Synchronous Skill Requests**: Hub waits for skill response (no async)
## Future Considerations
1. **Skill Versioning**: Support multiple versions of same skill
2. **Skill-to-Skill Direct Communication**: Allow skills to call each other
3. **Async Skill Responses**: Long-running skills with callback pattern
4. **Custom NLU Models**: Support for custom trained models
5. **Skill Hot-Reload**: Dynamic skill loading without restart
6. **Multi-Session Skills**: Support concurrent skill sessions
7. **Skill Marketplace**: Third-party skill distribution
8. **A/B Testing**: Framework for testing skill variations
## Conclusion
The original Jibo server (Pegasus) is a well-architected microservices system that provides a robust foundation for conversational AI on the Jibo robot. The graph-based skill framework offers flexibility and maintainability, while the separation of concerns enables independent scaling and development. The system successfully handles real-time speech processing, natural language understanding, skill routing, and proactive behaviors in a distributed cloud environment.

View File

@@ -0,0 +1,874 @@
# Skill Framework Design Document
## Overview
The Skill Framework provides the foundation for building cloud-based skills for the Jibo robot. It consists of a base class for all skills, a graph-based state machine for complex conversational flows, and a system for generating JCP (Jibo Command Protocol) actions that are sent to the robot.
## Location
`packages/baseskill/src/`
## Core Components
### BaseSkill (`BaseSkill.ts`)
Abstract base class that all cloud skills must extend.
**Purpose:** Provides common HTTP handling and error handling for all skills.
**Key Features:**
- Extends `BaseHttpHandler` from `@jibo/utils`
- Registers POST handler at `/` endpoint
- Validates request structure
- Tracks timing for each request
- Provides error response builder
**Constructor:**
```typescript
constructor(public name: string)
```
**Abstract Method:**
```typescript
protected abstract handle(request: PegasusRequest<SkillRequest>): Promise<SkillResponse>;
```
**Lifecycle Methods:**
- `init(): Promise<void>` - Override to initialize resources (load files, connect to services)
- `buildErrorResponse(err: Error): ErrorResponse` - Builds standardized error response
**HTTP Handler:**
- Accepts POST requests at `/`
- Logs request type
- Calls `handle()` method
- Adds timing information
- Catches errors and returns error response
### GraphSkill (`GraphSkill.ts`)
Extends BaseSkill with a graph-based state machine for complex conversational flows.
**Purpose:** Enables skills to define their logic as a series of interconnected nodes (states) with transitions.
**Key Features:**
- Implements `GraphFactory` interface
- Manages graph execution via `GraphManager` singleton
- Supports skill redirects
- Tracks analytics events
- Supports supplemental behaviors (parallel/sequence)
- Handles both launch and update requests
**Constructor:**
```typescript
constructor(name: string)
```
**Abstract Method:**
```typescript
abstract createGraph(): Graph<ExitTransition>
```
**Request Handling:**
**Launch Requests** (LISTEN_LAUNCH or PROACTIVE_LAUNCH):
1. Validates request data (accountID, robotID, skill ID)
2. Initializes skill session data
3. Tracks SKILL_ENTRY analytics event
4. Calls `GraphManager.instance.start(graph, data)` to begin graph execution
5. Returns SKILL_ACTION or SKILL_REDIRECT response
**Update Requests** (LISTEN_UPDATE):
1. Validates request data
2. Calls `GraphManager.instance.exitNode(data)` to process action results
3. Returns next SKILL_ACTION or final response
**Response Types:**
1. **SKILL_REDIRECT** - Redirects to another skill
```typescript
{
type: "SKILL_REDIRECT",
msgID: "uuid",
ts: 1234567890,
data: {
skillID: string,
nlu?: NLUResult,
asr?: ASRResult,
memo?: any
}
}
```
2. **SKILL_ACTION** - Returns JCP action for robot to execute
```typescript
{
type: "SKILL_ACTION",
msgID: "uuid",
ts: 1234567890,
data: {
action: JCPAction,
analytics: AnalyticsData,
final: boolean,
fireAndForget: boolean
}
}
```
3. **Final Response** - No action, transaction complete
```typescript
{
type: "SKILL_ACTION",
msgID: "uuid",
ts: 1234567890,
data: {
action: null,
analytics: AnalyticsData,
final: true,
fireAndForget: true
}
}
```
**Convenience Methods:**
- `track(data, event, properties)` - Track analytics event
- `overrideSpeaker(data, id)` - Override current speaker in context
- `addParallelBehavior(data, behavior)` - Add behavior to execute in parallel
- `addSequenceBehavior(data, behavior)` - Add behavior to execute in sequence
**Supplemental Behaviors Injection:**
When a skill returns a JCP action, the framework injects any supplemental behaviors that were added during execution:
1. If sequence behaviors exist, wraps main action in a Sequence
2. If parallel behaviors exist, wraps result in a Parallel
3. Final JCP action is sent to robot
**Example:**
```typescript
// Skill adds parallel behavior
this.addParallelBehavior(data, SetPresentPersonBehavior);
// Skill returns main action
return { action: SayTextBehavior };
// Framework injects: Parallel([SetPresentPersonBehavior, SayTextBehavior])
```
### Graph System
#### Graph (`graph/Graph.ts`)
Represents a directed graph of connected nodes (states).
**Purpose:** Defines the structure of a skill's conversation flow.
**Key Properties:**
- `name: string` - Graph name
- `initial: Node` - Starting node
- `nodes: Set<Node>` - All nodes in graph
- `exitTransitions: Map<ExitTransition, TransitionContainer[]>` - Exit transition mappings
**Constructor:**
```typescript
constructor(name: string, exitTransitionNames: ExitTransition[])
```
**Methods:**
- `setInitialNode(node)` - Sets the starting node
- `addNode(node, transitionMapping)` - Adds a node and connects its transitions
- `addSubGraph(subGraph, transitionMapping)` - Adds a subgraph and connects its exits
- `finalize()` - Validates graph and locks it for execution
- `writeDotFile(filePath)` - Generates GraphViz dot file for visualization
**Transition Mapping:**
```typescript
[
[TransitionName, DestinationNode], // Transition to another node
[TransitionName, ExitTransition] // Exit from graph
]
```
**Validation (in finalize):**
- All nodes must be reachable from initial node
- All exit transitions must be connected
- All transitions must have valid destinations
- No duplicate transition names
**Subgraphs:**
- Graphs can be nested within other graphs
- Subgraph exit transitions connect to parent graph nodes
- Enables hierarchical organization of complex flows
- Nodes can belong to multiple graphs (for subgraph sharing)
**GraphViz Visualization:**
- Generates .dot files for graph visualization
- Color-codes initial node, regular nodes, and exit states
- Shows hierarchical structure with clusters
- Labels transitions with their names
#### GraphManager (`graph/GraphManager.ts`)
Singleton that manages graph execution and skill sessions.
**Purpose:** Coordinates node execution and maintains session state.
**Singleton Pattern:**
```typescript
GraphManager.instance // Access singleton
```
**Key Responsibilities:**
- Assigns unique IDs to all nodes
- Maps node IDs to node instances
- Manages skill session lifecycle
- Executes node enter/exit lifecycle
- Handles transitions between nodes
**Session Structure:**
```typescript
{
id: string, // Session UUID
nodeID: number, // Current node ID
data: any, // Skill-specific session data
trace: [ // History of transitions
{ nodeID: number, transition: string }
]
}
```
**Execution Flow:**
**Start Graph** (launch request):
```typescript
start(graph, data)
→ Creates new session
→ Sets initial node
→ Calls enterNode()
```
**Enter Node:**
```typescript
enterNode(data)
→ Fetches current node
→ Calls node.enter(data)
→ Updates trace
→ If action returned: return action
→ Else: call exitNode()
```
**Exit Node:**
```typescript
exitNode(data)
→ Fetches current node
→ Calls node.exit(data)
→ If transition returned: executeTransition()
→ Else: return (terminal)
```
**Execute Transition:**
```typescript
executeTransition(node, result, data)
→ Validates transition exists
→ Updates trace with transition name
→ If terminal: return null
→ Else: update nodeID, call enterNode()
```
**Node ID Assignment:**
- Counter starts at 0, increments for each node
- Bidirectional mapping: node ↔ ID
- Enables serialization of session state
#### Node (`graph/nodes/Node.ts`)
Abstract base class for all graph nodes.
**Purpose:** Defines a state in the skill's conversation flow.
**Key Properties:**
- `id: number` - Unique ID assigned by GraphManager
- `name: string` - Node name
- `transitionNames: Transition[]` - Valid exit transitions
- `graphs: Graph[]` - Graphs this node belongs to
- `transitions: Map<Transition, TransitionContainer>` - Transition destinations
**Constructor:**
```typescript
constructor(name: string, transitionNames: Transition[])
```
**Abstract Methods:**
```typescript
abstract async enter(data: Data): Promise<EnterResponse>
```
- Called when node is entered
- Returns action to execute, redirect, or nothing
```typescript
abstract async exit(data: Data): Promise<ExitResponse>
```
- Called with action results (if action was issued)
- Returns next transition or nothing (terminal)
**Data Structure:**
```typescript
Data = {
// From request
general: { accountID, robotID, lang, release },
runtime: { character, location, loop, perception, dialog },
skill: { id, session },
result?: any, // Action results for UPDATE
// Added by framework
req: PegasusRequest,
log: Log,
local: any, // Skill-local data
analytics: {}, // Analytics events
behaviors: { // Supplemental behaviors
parallel: [],
sequence: []
}
}
```
**Response Types:**
**EnterResponse:**
```typescript
{
action?: Action, // JCP action to execute
redirect?: RedirectData, // Redirect to another skill
final?: boolean // Is this the final response?
}
```
**ExitResponse:**
```typescript
{
transition?: string, // Next transition to take
result?: any, // Result to pass to next node
redirect?: RedirectData
}
```
**Built-in Node Types:**
1. **DefaultNode** - Simple terminal node
- Returns no action
- Transitions to Done
2. **NoOpNode** - No operation node
- Returns no action
- Can have custom transitions
3. **JCPNode** - Returns a JCP action
- Returns specified JCP behavior
- Can be terminal or continue
4. **TrueFalseNode** - Conditional branching
- Evaluates condition
- Transitions based on true/false
5. **SetLooperIDNode** - Sets speaker ID
- Updates perception.speaker in context
- Useful for multi-turn conversations
**Node Traversal:**
- `forEachDescendent(handler)` - BFS traversal of all descendant nodes
- Used for graph validation and analysis
### Skill Request/Response Protocol
#### Skill Request Types
**Location:** `packages/interfaces/src/skill/request.ts`
**MessageType:**
- `LISTEN_LAUNCH` - Launch skill from listen interaction
- `LISTEN_UPDATE` - Update skill with action results
- `PROACTIVE_LAUNCH` - Launch skill proactively
**Request Structure:**
```typescript
{
type: MessageType,
msgID: "uuid",
ts: 1234567890,
data: {
general: {
accountID: string,
robotID: string,
lang: string,
release: string
},
runtime: {
character: { emotion, motivation },
location: { city, state, country, lat, lng },
loop: { users, jibo, owner, loopId },
perception: { speaker, peoplePresent },
dialog: { referent }
},
skill: {
id: string,
session?: {
id: string,
nodeID: number,
data: any,
trace: [{ nodeID, transition }]
}
},
result?: any, // Action results for UPDATE
nlu?: NLUResult,
asr?: ASRResult,
memo?: any
}
}
```
#### Skill Response Types
**Location:** `packages/interfaces/src/skill/response.ts`
**ResponseType:**
- `SKILL_ACTION` - Returns action to execute
- `SKILL_REDIRECT` - Redirects to another skill
- `ERROR` - Error response
**SKILL_ACTION Response:**
```typescript
{
type: "SKILL_ACTION",
msgID: "uuid",
ts: 1234567890,
data: {
action: JCPAction | null,
analytics: AnalyticsData,
final: boolean,
fireAndForget: boolean
}
}
```
**SKILL_REDIRECT Response:**
```typescript
{
type: "SKILL_REDIRECT",
msgID: "uuid",
ts: 1234567890,
data: {
skillID: string,
nlu?: NLUResult,
asr?: ASRResult,
memo?: any
}
}
```
**ERROR Response:**
```typescript
{
type: "ERROR",
msgID: "uuid",
ts: 1234567890,
data: {
message: string,
skill: { id: string }
}
}
```
### JCP Actions
**Location:** `packages/interfaces/src/skill/action.ts`
**Purpose:** Defines behaviors that the robot should execute.
**ActionType:**
- `JCP` - Jibo Command Protocol action
**JCPAction Structure:**
```typescript
{
type: "JCP",
config: {
version: "1.0.0",
jcp: SupportedBehaviors
}
}
```
**SupportedBehaviors:**
- `SLIM` - Single behavior execution
- `Sequence` - Sequential behavior execution
- `Parallel` - Parallel behavior execution
- `SetPresentPerson` - Set focused person
- `ImpactEmotion` - Modify Jibo's emotional state
**Helper Function:**
```typescript
generateJCPAction(behavior): JCPAction
```
Wraps a behavior as a JCP action with version 2.0.
### MIM (Motion Interaction Model) System
**Location:** `packages/baseskill/src/graph/mims/`
**Purpose:** Provides pre-built graph structures for playing MIM animations.
**MIM Files:**
- `.mim` files contain animation definitions
- Organized in directories:
- `scripted-responses` - Pre-scripted responses
- `emotion-responses` - Emotion-based responses
- `core-responses` - Fallback responses
**MIM Factories:**
**ANFactory** - Animation Node Factory
- Creates graph for playing a single MIM
- Supports prompt data injection
- Can be final or continue
**MANFactory** - Multiple Animation Node Factory
- Creates graph for playing multiple MIMs
- Supports random selection
- Can be final or continue
**MIMFactory** - General MIM Factory
- Creates graph for MIM playback
- Supports semi-specific responses
- Handles category-based selection
**QNFactory** - Question Node Factory
- Creates graph for asking questions
- Supports opt-in flows
- Handles user responses
**OptInFactory** - Opt-In Node Factory
- Creates graph for opt-in offers
- Tracks user acceptance/rejection
- Handles analytics
**MIM Factory Options:**
```typescript
{
mimDataProvider: (data) => string[], // Function to get MIM paths
promptDataProvider?: (data) => any, // Function to get prompt data
final: boolean // Is this the final action?
}
```
**Example Usage (Chitchat Skill):**
```typescript
const doMIMOptions: MimFactoryOptions = {
mimDataProvider: (data) => data.local.path,
promptDataProvider: (data) => data.local.promptData,
final: true
};
const doMIM = new ANFactory('Do MIM', doMIMOptions).createGraph();
```
**Semi-Specific Responses:**
- MIMs with `_SS_` suffix are semi-specific
- Match specific categories (e.g., time, weather)
- CSV files define category members
- Enables context-aware responses
### SkillService (`SkillService.ts`)
Service wrapper that hosts a skill as an HTTP service.
**Purpose:** Provides the service infrastructure for running a skill.
**Constructor:**
```typescript
constructor(private skillV1: BaseSkill)
```
**HTTP Handler:**
- Registers skill at `/v1/main` endpoint
- No authentication required (handled by Hub)
**Initialization:**
```typescript
async init(port: number)
→ Starts HTTP server
→ Calls skill.init()
```
### Analytics
**Location:** `packages/interfaces/src/skill/analytics.ts`
**Purpose:** Track skill events for analysis.
**AnalyticsData Structure:**
```typescript
{
[skillName: string]: [
{
event: string,
properties: any
}
]
}
```
**Built-in Events:**
- `SKILL_ENTRY` - Skill launched
- `SKILL_OFFER` - Opt-in offer presented
**Skill Entry Analytics:**
```typescript
{
initial_intent: string,
domain: string,
was_hey_jibo_launch: boolean,
user_initiated: boolean,
last_skill: string
}
```
**Tracking:**
```typescript
this.track(data, 'CustomEvent', { key: value });
```
Events are automatically included in SKILL_ACTION responses.
## Server-to-Robot Communication Flow
### Skill Response to Hub
When a skill returns a response, the Hub forwards it to the robot:
**SKILL_ACTION Response:**
1. Skill returns SKILL_ACTION with JCP behavior
2. Hub adds timing information
3. Hub sends SKILL_ACTION to robot via WebSocket
4. Robot executes JCP behavior
5. Robot sends CMD_RESULT back to Hub
6. Hub sends LISTEN_UPDATE to skill
7. Skill processes result, returns next action
**Final SKILL_ACTION:**
1. Skill returns SKILL_ACTION with `final: true`
2. Hub sends to robot
3. Robot executes (if action present)
4. Transaction complete
**SKILL_REDIRECT:**
1. Skill returns SKILL_REDIRECT
2. Hub emits SKILL_REDIRECT notification to robot
3. Hub launches new skill
4. New skill proceeds normally
### JCP Action Execution
**Single Behavior (SLIM):**
```typescript
{
type: "JCP",
config: {
version: "1.0.0",
jcp: SayTextBehavior
}
}
```
Robot executes single behavior immediately.
**Sequence Behavior:**
```typescript
{
type: "JCP",
config: {
version: "1.0.0",
jcp: Sequence([
LookAtBehavior,
SayTextBehavior,
GestureBehavior
])
}
}
```
Robot executes behaviors in order.
**Parallel Behavior:**
```typescript
{
type: "JCP",
config: {
version: "1.0.0",
jcp: Parallel([
SetPresentPersonBehavior,
SayTextBehavior
])
}
}
```
Robot executes behaviors simultaneously.
### Supplemental Behaviors
Skills can add behaviors that execute alongside the main action:
**Parallel Supplemental:**
```typescript
this.addParallelBehavior(data, SetPresentPersonBehavior);
// Main action: SayTextBehavior
// Result: Parallel([SetPresentPersonBehavior, SayTextBehavior])
```
**Sequence Supplemental:**
```typescript
this.addSequenceBehavior(data, LookAtBehavior);
// Main action: SayTextBehavior
// Result: Sequence([LookAtBehavior, SayTextBehavior])
```
**Combined:**
```typescript
this.addSequenceBehavior(data, LookAtBehavior);
this.addParallelBehavior(data, SetPresentPersonBehavior);
// Result: Parallel([SetPresentPersonBehavior, Sequence([LookAtBehavior, SayTextBehavior])])
```
## Example Skill Implementation
### Chitchat Skill
**Location:** `packages/chitchat-skill/src/Chitchat.ts`
**Purpose:** Handles conversational interactions with the robot.
**Graph Structure:**
1. **IntentSplitNode** - Splits based on intent type
2. **ProcessQueryNode** - Processes user query, selects response
3. **DoMIM (ANFactory)** - Plays selected MIM animation
4. **Complete (DefaultNode)** - Terminates skill
**Initialization:**
- Loads MIM files from directories
- Builds semi-specific mappings
- Reads category CSV files
**Response Selection:**
- Scripted responses for common queries
- Emotion responses for emotional queries
- Semi-specific responses for context-aware queries
- Fallback responses for unknown queries
**MIM Selection:**
- Based on intent and entities
- Considers semi-specific categories
- Falls back to core responses
## Skill Development Guide
### Creating a Simple Skill
```typescript
import { BaseSkill } from '@jibo/baseskill';
import { skill } from '@jibo/interfaces';
export class MySkill extends BaseSkill {
constructor() {
super('my-skill');
}
protected async handle(req: PegasusRequest<SkillRequest>): Promise<SkillResponse> {
const data = req.body.data;
// Process request
const action = generateJCPAction(SayTextBehavior("Hello!"));
return {
type: skill.response.ResponseType.SKILL_ACTION,
data: {
action: action,
final: true,
fireAndForget: true
},
ts: Date.now(),
msgID: getUUID()
};
}
}
```
### Creating a Graph Skill
```typescript
import { GraphSkill, graph } from '@jibo/baseskill';
enum Transition {
Done = 'Done',
Retry = 'Retry'
}
export class MyGraphSkill extends GraphSkill<Transition> {
constructor() {
super('my-graph-skill');
}
createGraph(): graph.Graph<Transition> {
const g = new graph.Graph('My Skill', generateTransitions(Transition));
const startNode = new MyStartNode('Start');
const endNode = new graph.nodes.dn.DefaultNode('End');
g.addNode(startNode, [[Transition.Done, endNode]]);
g.addNode(endNode, [[graph.nodes.dn.Transition.Done, Transition.Done]]);
g.finalize();
return g;
}
}
```
### Creating a Custom Node
```typescript
import { Node, Data, EnterResponse, ExitResponse } from '@jibo/baseskill';
enum MyTransition {
Success = 'Success',
Failure = 'Failure'
}
class MyNode extends Node<MyTransition> {
constructor() {
super('MyNode', [MyTransition.Success, MyTransition.Failure]);
}
async enter(data: Data): Promise<EnterResponse> {
// Perform logic
const action = generateJCPAction(SayTextBehavior("Processing..."));
return { action };
}
async exit(data: Data): Promise<ExitResponse> {
// Process action results
if (data.result.success) {
return { transition: MyTransition.Success };
} else {
return { transition: MyTransition.Failure };
}
}
}
```
## Key Design Principles
1. **State Machine** - Graph-based state machine for complex flows
2. **Single Responsibility** - Each node handles one piece of logic
3. **Reusability** - Subgraphs and node types can be reused
4. **Testability** - Nodes can be tested independently
5. **Visualization** - GraphViz generation for debugging
6. **Analytics** - Built-in event tracking
7. **Flexibility** - Supports both simple and complex skills
8. **Supplemental Behaviors** - Easy to add parallel/sequence actions

View File

@@ -0,0 +1,24 @@
# Calendar Architecture
Pegasus treated calendar as a loop-scoped report surface, with report output fed by the
household context instead of an isolated generic calendar service.
In OpenJibo, the current calendar path follows the same broad shape:
- calendar report output is loop-scoped
- the report provider can read persisted loop calendar events
- birthday and other personal dates already live in the loop-scoped holiday list
- the personal report merges the report provider output into the spoken flow
Current behavior:
- if loop calendar events exist, the provider surfaces the next matching items
- if no loop calendar events exist, the provider falls back to the merged holiday list
- birthdays and custom holiday entries can therefore appear in the calendar section
- the personal report still degrades safely when no calendar data is available
Notes:
- the current provider is intentionally lightweight and in-process
- this gives us a swappable seam for later Azure-backed calendar sync
- commute remains a separate report gap for the next pass

View File

@@ -0,0 +1,72 @@
# Commute Architecture
## Purpose
Commute is part of personal report parity and household-aware personality.
The original Jibo report-skill had a commute section that could speak about getting to work, leaving soon, or being too early or too late. In OpenJibo, that behavior now starts with a loop-scoped commute profile so we can stay faithful to stock behavior first and add richer routing later.
## Current Shape
The cloud now models commute as persisted loop data instead of a hardcoded reply.
The current pieces are:
- `CommuteProfileRecord`
- `ICommuteReportProvider`
- `CloudStateCommuteReportProvider`
- `Person/ListCommute`
- `Person/UpsertCommute`
## Data Model
The commute profile is stored per loop and can optionally be tied to a person.
Typical fields include:
- `LoopId`
- `MemberId`
- `Mode`
- `WorkHour`
- `WorkMinute`
- `OriginName`
- `DestinationName`
- `TypicalDurationMinutes`
- `IsEnabled`
- `IsComplete`
The provider uses the loop-scoped profile to decide whether commute is ready, missing setup, or ready to render a spoken answer.
## Runtime Behavior
At runtime, the commute provider:
- reads the current loop from the request context
- loads the loop commute profile from cloud state
- uses the profile plus current time to compute minutes until work
- merges in same-day calendar pressure when a calendar event exists before the commute window
- returns a safe setup response when the commute profile is missing or incomplete
## Personal Report Integration
Personal report uses the commute provider as a section in the broader household report.
That means the report can now speak in the familiar Jibo shape:
- weather
- calendar
- commute
- news
## Next Gaps
The current commute provider is intentionally conservative.
Next steps can include:
- a richer travel-time source
- map or transit integration
- better depart-time commentary
- preference-based commute suppression or reminders
For now the goal is to keep the interface stable and the behavior stock-like.

View File

@@ -194,7 +194,7 @@ These are not blockers for calling `1.0.18` complete unless the live test shows
- local `whisper.cpp` STT remains a discovery seam, not production ASR - local `whisper.cpp` STT remains a discovery seam, not production ASR
- media upload/body handling is not binary-safe enough for final gallery originals and thumbnails - media upload/body handling is not binary-safe enough for final gallery originals and thumbnails
- state persistence is local JSON, not Azure SQL / Blob Storage - state persistence is local JSON, not Azure SQL / Blob Storage
- update, backup, and restore are not end-to-end proven, and the `jibo test 22` / Test 26 / Test 27 / Test 28 sluggishness appears tied to robot-local backup status/load, startup reconnect state, or previously unsuppressed end-of-skill surprises; Test 31 also captured a legacy `Backup_20170222.List` startup query, which reinforces that the local backup/status path is real even before a user asks for backup - update, backup, and restore are now end-to-end proven at the persistence-rehydration level, and the `jibo test 22` / Test 26 / Test 27 / Test 28 sluggishness appears tied to robot-local backup status/load, startup reconnect state, or previously unsuppressed end-of-skill surprises; Test 31 also captured a legacy `Backup_20170222.List` startup query, which reinforces that the local backup/status path is real even before a user asks for backup
- Tests 27 and 28 showed backup/surprise behavior without corresponding `Backup_*` HTTP traffic; Test 28 isolated the unsuppressed `@be/surprises` lifecycle handoff after Nimbus - Tests 27 and 28 showed backup/surprise behavior without corresponding `Backup_*` HTTP traffic; Test 28 isolated the unsuppressed `@be/surprises` lifecycle handoff after Nimbus
- deployed-build verification needs to prove that synthetic OpenJibo websocket events are gone from the hosted artifact, not just from source - deployed-build verification needs to prove that synthetic OpenJibo websocket events are gone from the hosted artifact, not just from source
- news content is synthetic; `jibo test 23` proved the path but not live provider-backed headlines - news content is synthetic; `jibo test 23` proved the path but not live provider-backed headlines

View File

@@ -44,6 +44,7 @@ Current release theme:
- radio, ESML apostrophe cleanup, and first news are implemented in source/tests; radio and basic news are live-proven as of `jibo test 23` - radio, ESML apostrophe cleanup, and first news are implemented in source/tests; radio and basic news are live-proven as of `jibo test 23`
- `jibo test 22` validated radio, exposed backup/load interference, exposed a shared yes/no no-input gap, exposed repeated create keeper prompts after photo handoff, and showed local whisper `ffmpeg` failures on unusable buffered audio - `jibo test 22` validated radio, exposed backup/load interference, exposed a shared yes/no no-input gap, exposed repeated create keeper prompts after photo handoff, and showed local whisper `ffmpeg` failures on unusable buffered audio
- `jibo test 23` validated basic news, proved one alarm set/fire path at `7:43 AM`, exposed comma-separated/short alarm follow-up parsing risk, showed stock alarm replacement yes/no rules that needed cloud handling, and showed photo gallery still failing when `shared/yes_no` ASR came back empty - `jibo test 23` validated basic news, proved one alarm set/fire path at `7:43 AM`, exposed comma-separated/short alarm follow-up parsing risk, showed stock alarm replacement yes/no rules that needed cloud handling, and showed photo gallery still failing when `shared/yes_no` ASR came back empty
- personal report parity now has loop-scoped calendar and commute provider seams that merge persisted loop events, birthday/holiday dates, and commute profiles; the remaining report gap is richer travel-time data, not missing structure
- `jibo test 24` showed alarm replacement yes/no working, but exposed empty `clock/alarm_set_value` and `gallery/gallery_preview` turns falling into generic `I heard you` fallback speech; it also showed `CLIENT_NLU cancel` inside `clock/alarm_set_value` re-asking for an alarm value instead of closing the prompt - `jibo test 24` showed alarm replacement yes/no working, but exposed empty `clock/alarm_set_value` and `gallery/gallery_preview` turns falling into generic `I heard you` fallback speech; it also showed `CLIENT_NLU cancel` inside `clock/alarm_set_value` re-asking for an alarm value instead of closing the prompt
- `jibo test 25` proved a broader regression path but exposed repeated backup-in-progress/update-menu blockage, timer/alarm stale state and delete/menu disagreement, gallery `shared/yes_no` hangs under `@be/gallery`, punctuated `Never mind.` falling through to chat, volume homophone parsing (`Set Volume 2-6.`), and settings volume-control cleanup falling into `I heard you` - `jibo test 25` proved a broader regression path but exposed repeated backup-in-progress/update-menu blockage, timer/alarm stale state and delete/menu disagreement, gallery `shared/yes_no` hangs under `@be/gallery`, punctuated `Never mind.` falling through to chat, volume homophone parsing (`Set Volume 2-6.`), and settings volume-control cleanup falling into `I heard you`
- `jibo test 26` live-proved punctuated stop, volume homophone parsing, gallery launch/yes/create/save, and good morning; it still exposed robot-local backup warnings, long blue-ring buffering without a fresh `LISTEN`, alarm replacement drifting into the value/manual screen, and alarm delete phrases/mishears falling to chat - `jibo test 26` live-proved punctuated stop, volume homophone parsing, gallery launch/yes/create/save, and good morning; it still exposed robot-local backup warnings, long blue-ring buffering without a fresh `LISTEN`, alarm replacement drifting into the value/manual screen, and alarm delete phrases/mishears falling to chat
@@ -435,7 +436,7 @@ Current release theme:
### 9. STT Upgrade And Noise Screening ### 9. STT Upgrade And Noise Screening
- Status: `ready` - Status: `in progress`
- Tags: `stt` - Tags: `stt`
- Why next: - Why next:
- feature paths are now often correct when a transcript exists, but short replies and low-quality audio still block otherwise-correct flows - feature paths are now often correct when a transcript exists, but short replies and low-quality audio still block otherwise-correct flows
@@ -447,6 +448,10 @@ Current release theme:
- `jibo test 26` had long no-`LISTEN` binary buffering and alarm-delete mishears now patched; remaining short-answer failures still need STT/noise work - `jibo test 26` had long no-`LISTEN` binary buffering and alarm-delete mishears now patched; remaining short-answer failures still need STT/noise work
- current source now skips local whisper when buffered audio does not contain an Opus identification header - current source now skips local whisper when buffered audio does not contain an Opus identification header
- yes/no and alarm flows are especially sensitive to short or collapsed transcripts - yes/no and alarm flows are especially sensitive to short or collapsed transcripts
- Progress update (`2026-05-21`):
- added a small local whisper noise floor so obviously tiny buffered audio can be screened before ffmpeg/whisper work runs
- short/noisy buffered turns now fail fast instead of wasting a transcription cycle
- focused tests now cover the new low-audio rejection behavior
- Implementation notes: - Implementation notes:
- add lightweight waveform or energy screening before transcription - add lightweight waveform or energy screening before transcription
- compare managed STT against the local toolchain - compare managed STT against the local toolchain
@@ -461,11 +466,12 @@ Current release theme:
- Implementation notes: - Implementation notes:
- define local capture sinks versus hosted retention - define local capture sinks versus hosted retention
- decide how testers submit noteworthy sessions - decide how testers submit noteworthy sessions
- keep a lightweight `capture-index.ndjson` manifest beside raw captures so testers can quickly find sessions, operations, and fixture exports
- preserve sanitized fixtures as the durable parity artifact - preserve sanitized fixtures as the durable parity artifact
### 11. Binary-Safe Media Storage ### 11. Binary-Safe Media Storage
- Status: `ready` - Status: `in progress`
- Tags: `storage`, `protocol` - Tags: `storage`, `protocol`
- Why next: - Why next:
- the first gallery bridge stores metadata and text-body placeholders, but final gallery support needs originals and thumbnails - the first gallery bridge stores metadata and text-body placeholders, but final gallery support needs originals and thumbnails
@@ -473,6 +479,9 @@ Current release theme:
- whether stock gallery expects originals, thumbnails, or both - whether stock gallery expects originals, thumbnails, or both
- what upload metadata must survive for gallery refresh - what upload metadata must survive for gallery refresh
- how to map this cleanly to Blob Storage - how to map this cleanly to Blob Storage
- Implementation notes:
- media content now flows through a storage seam with file and Azure Blob adapters
- the protocol still serves the legacy text-body contract, but the original payload is now persisted separately and can be swapped to binary-native storage later
### Next Up (`2026-05-06`): Dialog Parsing Expansion And Ambiguity Guardrails ### Next Up (`2026-05-06`): Dialog Parsing Expansion And Ambiguity Guardrails
@@ -494,6 +503,9 @@ Current release theme:
- shorthand favorites (`my favorite sport football`) - shorthand favorites (`my favorite sport football`)
- weather phrasing (`what's today's weather look like`, `will it be sunny tomorrow`) - weather phrasing (`what's today's weather look like`, `will it be sunny tomorrow`)
- updated continuation deferral so complete shorthand favorites finalize instead of waiting for missing continuation - updated continuation deferral so complete shorthand favorites finalize instead of waiting for missing continuation
- Progress update (`2026-05-21`):
- expanded friendship parsing for Pegasus-style `do you have friends`, `are we friends`, and `are we best friends` phrasing
- added named-person guardrails so forms like `are you friends with Siri` and `is Dr. Breazeal your best friend` stay on the friendship route instead of falling into generic chat
- Exit criteria: - Exit criteria:
- ambiguous phrase handling is improved without regressions in existing `1.0.19` features - ambiguous phrase handling is improved without regressions in existing `1.0.19` features
- phrase imports are documented and traceable to Pegasus parser sources - phrase imports are documented and traceable to Pegasus parser sources
@@ -506,7 +518,7 @@ Current release theme:
### 12. Weather As Cloud Report Plus Local Presentation ### 12. Weather As Cloud Report Plus Local Presentation
- Status: `discovery` - Status: `implemented`
- Tags: `protocol`, `content` - Tags: `protocol`, `content`
- Evidence: - Evidence:
- Nimbus and Pegasus contain personal-report weather assets and Lasso provider hooks - Nimbus and Pegasus contain personal-report weather assets and Lasso provider hooks
@@ -602,6 +614,8 @@ Current release theme:
- recognition, enrollment, rename, and profile-correction boundaries - recognition, enrollment, rename, and profile-correction boundaries
- split between local state and hosted cloud state - split between local state and hosted cloud state
- first useful hosted identity slice - first useful hosted identity slice
- live QA has shown person-identification collisions in the same loop (for example, a parent and child both getting normalized to the same remembered name)
- person-identification correction likely needs its own repair pass before we can trust greetings, reports, and presence triggers in mixed-household scenarios
### 20. Onboarding, Loop Management, And Fresh Start ### 20. Onboarding, Loop Management, And Fresh Start
@@ -626,9 +640,12 @@ Current release theme:
- `make a pizza` now ports the original scripted-response path through `chitchat-skill` with `mim_id = RA_JBO_MakePizza` and pizza-making animation ESML - `make a pizza` now ports the original scripted-response path through `chitchat-skill` with `mim_id = RA_JBO_MakePizza` and pizza-making animation ESML
- `can you order pizza` now ports the original scripted-response path through `chitchat-skill` with `mim_id = RA_JBO_OrderPizza` - `can you order pizza` now ports the original scripted-response path through `chitchat-skill` with `mim_id = RA_JBO_OrderPizza`
- current source answers these with a `1.0.19` rule-based persona baseline, backed by `OpenJiboCloudBuildInfo.PersonaBirthday` - current source answers these with a `1.0.19` rule-based persona baseline, backed by `OpenJiboCloudBuildInfo.PersonaBirthday`
- `how old are you` now also uses the imported Build B age prompts so the first-powered-up and birthday phrasing stays source-backed
- Follow-up: - Follow-up:
- wire persona age to first-powered-up or durable first-cloud-seen metadata when available - wire persona age to first-powered-up or durable first-cloud-seen metadata when available
- add command-vs-question variants so expressive prompts can answer conversationally before launching actions - add command-vs-question variants so expressive prompts can answer conversationally before launching actions
- live QA has shown motion/sleep quirks too: `turn around` can become a no-op and `go to sleep` can fail at the last step before the sleep animation fully completes
- reply-selection polish still needs attention on a couple of identity prompts where short variants are over-selected (`how are you`, `what is your favorite flower`)
### 22. Command Vs Question Reply Style ### 22. Command Vs Question Reply Style
@@ -654,6 +671,8 @@ Current release theme:
- Follow-up: - Follow-up:
- add durable persistence path for personal facts - add durable persistence path for personal facts
- broaden fact categories further (multi-person household memory, relationship cues, and corrective updates) - broaden fact categories further (multi-person household memory, relationship cues, and corrective updates)
- add explicit person-scoped state so future interactions can distinguish household members inside the same loop
- define the first server-to-server sync envelope for durable state before we need it in production
### 24. Memory-Triggered Proactivity Baseline ### 24. Memory-Triggered Proactivity Baseline
@@ -669,6 +688,7 @@ Current release theme:
- expand proactivity beyond pizza to additional Pegasus-backed categories - expand proactivity beyond pizza to additional Pegasus-backed categories
- add cooldown/throttle policy and observability around proactive offer frequency - add cooldown/throttle policy and observability around proactive offer frequency
- connect memory store to durable multi-tenant persistence - connect memory store to durable multi-tenant persistence
- keep the sync story visible so stateful offers can survive a multi-server deployment later
### 25. Weather Report-Skill Launch Compatibility ### 25. Weather Report-Skill Launch Compatibility
@@ -687,7 +707,7 @@ Current release theme:
### 26. Presence-Aware Greetings And Identity Proactivity ### 26. Presence-Aware Greetings And Identity Proactivity
- Status: `ready` - Status: `in_progress`
- Tags: `protocol`, `content`, `storage`, `docs` - Tags: `protocol`, `content`, `storage`, `docs`
- Why now: - Why now:
- this is the next personality-charm expansion after parser guardrail and weather bring-up - this is the next personality-charm expansion after parser guardrail and weather bring-up
@@ -704,6 +724,13 @@ Current release theme:
- add greeting intent families and state-machine split for reactive vs proactive greeting routes - add greeting intent families and state-machine split for reactive vs proactive greeting routes
- add cooldown and trigger-source guardrails for proactive greetings - add cooldown and trigger-source guardrails for proactive greetings
- start person-aware greeting hooks (name-aware greeting, morning greeting policy, return greeting policy) - start person-aware greeting hooks (name-aware greeting, morning greeting policy, return greeting policy)
- Shipped so far:
- durable greeting-presence records now persist last-seen and last-greeted per person/loop
- proactive greeting gating now consults cloud greeting history when available
- reactive and proactive greeting turns write back greeting-history records for later cooldown checks
- birthday-aware proactive greetings now use stored birthday memory on matching dates
- holiday-aware proactive greetings now use loop holiday records on matching dates
- morning proactive greetings now stay distinct from return-visit greetings
- Exit criteria: - Exit criteria:
- presence-aware greetings are routed deterministically with tests - presence-aware greetings are routed deterministically with tests
- proactive greetings are frequency-bounded and do not trigger from surprise source when blocked by policy - proactive greetings are frequency-bounded and do not trigger from surprise source when blocked by policy
@@ -724,13 +751,18 @@ Current release theme:
- weather icon/animation parity and view support - weather icon/animation parity and view support
- broader non-local weather query handling and short-range date coverage - broader non-local weather query handling and short-range date coverage
- provider-backed news ingestion and filtering - provider-backed news ingestion and filtering
- commute provider path and settings schema - commute provider path, settings schema, and loop-scoped commute profile storage
- coverage matrix for personal report parity gaps and test/capture exit criteria - coverage matrix for personal report parity gaps and test/capture exit criteria
- Progress update (`2026-05-10`): - Progress update (`2026-05-10`):
- added provider-ready news briefing lane with Nimbus-compatible `news` skill payload continuity - added provider-ready news briefing lane with Nimbus-compatible `news` skill payload continuity
- added memory/transcript category hint plumbing for provider requests (sports/technology/business/general) - added memory/transcript category hint plumbing for provider requests (sports/technology/business/general)
- fallback synthetic news behavior remains active when no provider key is configured - fallback synthetic news behavior remains active when no provider key is configured
- added TTL caching for weather/news provider calls to reduce repeated external requests - added TTL caching for weather/news provider calls to reduce repeated external requests
- vendored Pegasus `report-skill` templates for weather and personal-report phrasing so the next pass can focus on renderer coverage for calendar, commute, and news templates instead of rediscovering source text
- commute now has a loop-scoped provider seam plus persisted commute profiles, so the next pass can focus on richer travel-time data instead of basic storage shape
- Progress update (`2026-05-21`):
- weather payloads now distinguish current-vs-weekly view modes so renderer parity can key off the payload shape more cleanly
- news provider now skips summaryless correction headlines before falling back to broader sources
- Source anchors: - Source anchors:
- `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\weather\WeatherMimLogic.ts` - `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\weather\WeatherMimLogic.ts`
- `C:\Projects\jibo\pegasus\packages\report-skill\resources\views\weatherHiLo.json` - `C:\Projects\jibo\pegasus\packages\report-skill\resources\views\weatherHiLo.json`
@@ -743,7 +775,7 @@ Current release theme:
### 28. Grocery List Capability (Requested Feature) ### 28. Grocery List Capability (Requested Feature)
- Status: `discovery` - Status: `in_progress`
- Tags: `content`, `docs`, `storage` - Tags: `content`, `docs`, `storage`
- Why now: - Why now:
- directly requested by Jibo owners and fits memory + household utility roadmap - directly requested by Jibo owners and fits memory + household utility roadmap
@@ -752,13 +784,173 @@ Current release theme:
- examples: - examples:
- `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ShoppingList.mim` - `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ShoppingList.mim`
- `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ManageToDoList.mim` - `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ManageToDoList.mim`
- Candidate delivery paths: - MVP decision:
- native lightweight list skill (fastest user value) - use the existing household list engine as the native lightweight grocery MVP
- integration-backed list orchestration (long-term richer ecosystem fit) - keep grocery as a first-class spoken alias over the shopping list storage path
- reserve integration-backed list orchestration for a later discovery pass
- Exit criteria: - Exit criteria:
- clear decision on MVP path - grocery prompts, add/recall/done flows, and list follow-ups consistently speak grocery wording
- first schema for list items + ownership scope - existing shopping/to-do flows remain unchanged
- initial voice flows and follow-up intent handling defined - future integration-backed list work remains a separate backlog item
### 29. Legacy MIM Personality Import Ladder
- Status: `in_progress`
- Tags: `content`, `protocol`, `docs`
- Why now:
- we already have a chitchat/content scaffold that can render stock-compatible personality replies
- the legacy `chitchat-mims` tree is mostly declarative content, so a phased import can add visible charm fast
- this is the best near-term path to get Jibo feeling more interactive without needing a full Pegasus runtime clone
- What is possible today:
- direct scripted replies through the existing content catalog
- stock-compatible payloads with `skillId`, `mim_id`, `mim_type`, `prompt_id`, and ESML
- current examples already prove the shape for pizza, dance, weather, news, and generic chat
- What we need to build:
1. a MIM inventory importer that can scan the legacy tree and normalize `skill_id`, `mim_id`, prompt text, and metadata
2. a prompt-selection layer that can choose by category and condition metadata
3. a safe ESML/prompt renderer for imported content
- What can be ported with each build:
- Build A: declarative prompt packs
- `core-responses`
- `deflector`
- the simplest `emotion-responses`
- direct `scripted-responses` that are just prompt lists
- Build B: conditioned prompt packs
- `gqa-responses`
- structured emotion prompts with `condition` gates
- any response families that only need simple state or Jibo-emotion checks
- Build C: conversation families
- richer `scripted-responses` that need follow-up state
- holiday / special-date personality sets
- more nuanced chitchat branches that depend on context-aware routing
- Build D: full parity cleanup
- larger cross-skill collections
- any MIMs that depend on Pegasus-only parser assumptions
- any files that need dedicated runtime abstraction instead of catalog lookup
- Low-hanging fruit for tonight:
- import the smallest declarative packs first so we can test something tomorrow
- prioritize anything that is pure prompt text with no complex branching
- keep the first pass limited to content that maps cleanly onto the current catalog shape
- Progress update (`2026-05-13`):
- added the first Build A importer scaffold in the cloud content repository
- checked in a small seed bundle under `Content/LegacyMims/BuildA`
- added focused importer tests for prompt stripping, bucketing, and merge behavior
- expanded Build A with additional easy scripted-response packs for identity and persona replies
- started Build B with source-backed scripted-response packs for work, food, home, birthplace, language, hobby, and material questions
- Tomorrow test target:
- verify imported personality replies show up through the existing chitchat route
- confirm the emitted payload still looks like a stock skill response
- confirm the imported content does not disturb existing weather/news/pizza flows
- Exit criteria:
- a first importer path exists for the simplest legacy MIM files
- at least one legacy prompt pack is running through OpenJibo content instead of hand-authored fallback text
- we have a clear second-wave list for the more conditional MIM families
### 30. Original Personalized Function Inventory
- Status: `discovery`
- Tags: `content`, `docs`, `protocol`
- Why now:
- we are actively porting persona and memory slices, so we need a bounded checklist of the original Jibo charm surfaces
- the goal is to keep the next few passes focused on personality-rich wins instead of letting the work sprawl
- Known sources:
- legacy Jibo OS/Pegasus chitchat and MIM response families
- current OpenJibo persona, memory, and greeting work as the implementation target
- Inventory to track:
- identity and origin questions
- personality and capability questions
- favorite-style prompts like `what is your favorite color`
- identity charm prompts like `what's your name`, `do you have a nickname`, `do you like being Jibo`, `are there others like you`, and `what is your favorite name`
- attraction and preference prompts like `what is your favorite flower`, `do you like R2D2`, `do you like the sun`, `do you like space`, and `do you like kids`
- longer authored variants for the same prompt family when Pegasus shows richer phrasing
- charm/capability prompts like `can you laugh`, `can you dance`, `can you sing`, and `will you sing`
- mood / affect questions
- recognition follow-ups like `do you know me`
- follow-up state prompts that should stay warm and locally grounded
- Next pass targets:
- document the remaining persona inventory so we keep a clean checklist for the next passes
- keep the favorites family moving with source-backed imports where available, and temporary runtime replies only when the source is missing
- keep adding small sourced personality batches, especially the legacy `R2D2`, `sun`, `space`, `kids`, and charm prompts
- keep adding 1-3 persona prompts per pass with tests
- prefer source-backed MIM imports when the legacy text is available, and use a temporary runtime reply only when needed to unblock user value
- keep a separate note for longer authored variants so we do not lose the multi-clause Peggy-style phrasing while importing the short-form packs
- Mood follow-up work in flight:
- source-backed happy/sad/angry response packs are now part of Build B
- small-talk aliases like `what are you up to` and `how are things` now stay on the emotion-query path
- Descriptor charm work in flight:
- source-backed `are you kind`, `are you funny`, `are you helpful`, `are you curious`, `are you loyal`, `are you mischievous`, and `are you likable` prompts are now in Build B
- these keep the self-description lane warm while we build toward seasonal and holiday charm
- Seasonal charm work in flight:
- source-backed holiday, New Year's, Halloween, spring, summer, favorite-season, and gift prompts are now part of Build B
- `RN_` holiday greeting files are now bucketed as greetings so seasonal replies stay visible in the catalog
- birthday celebration lines are now bucketed separately, and birthday memory writes a loop-scoped holiday record so personal dates can join the holiday list later
- holiday extras now include `show santa tracker` so the Christmas-time launcher keeps its source-backed animation line
- the remaining seasonal polish now includes `do you like halloween`, `do you like holiday music`, `do you like holiday parties`, `are you looking forward to christmas`, `what are you doing for christmas`, and `what are you thankful for`
- Favorite-animal work in flight:
- the favorites family now includes `what is your favorite animal`, `what is your favorite bird`, `do you like penguins`, and `do you like animals` so the penguin-centric replies stay easy to find
- Presence and thought follow-ups in flight:
- `welcome back`, `what are you thinking`, `what have you been doing`, and `what did you do` are now part of Build B
- these keep the social surface lively while the memory and multitenant tracks keep advancing in parallel
- Next queued persona surfaces:
- richer identity follow-ups like `who is this`, `do you know me`, `do you remember me`, and `can you recognize me`
- mood and affect prompts like `how are you`, `are you happy`, `are you sad`, and `are you angry`
- self-description charm like `what's your name`, `do you have a nickname`, `do you like being Jibo`, and `what is your favorite name`
- deeper personality follow-ups like `what do you dream about`, `what are you afraid of`, `what do you want to talk about`, `what is your best book`, `what is your best exercise`, `what is your dream vacation`, `who is your hero`, `who do you love`, and `what is your religion`; `what is your sign` stays deferred until templated placeholder rendering exists
- the next identity / knowledge wave adds `are you god`, `are you here`, `do you have super powers`, `how much do you know`, `what does jibo mean`, `where do you get info`, `what are you forbidden to do`, `what color are you`, and `what do you do when alone`
- additional legacy source-backed `RI_USR` prompts where the text is short and the behavior is easy to verify
- templated edge cases like `what is your sign`, `how many people do you know`, and `what is the loop` where live birthday and loop state are part of the line instead of a plain canned response
- Exit criteria:
- a stable checklist exists for the original persona surface
- each pass can be scoped to a small batch of prompts
- the backlog makes it obvious what is still missing without losing momentum
### 31. Longer Authored Persona Variants
- Status: `ready`
- Tags: `content`, `docs`, `protocol`
- Why now:
- Pegasus often used longer, multi-clause authored alternatives for the same personality question
- we already have the short-path import working, so this is a low-risk way to add richer phrasing without inventing a new dialog engine
- it gives us a straightforward next pass that stays familiar to the original robot
- Scope:
- import the longer authored variants already present in the legacy MIMs
- prefer richer phrasing for favorite-style, identity, and charm prompts when the source text provides it
- keep the runtime behavior rule-based and deterministic
- Next step:
- add a small batch of longer variants to the current Build B content packs and prove them with a smoke test
### 32. Dialog Joining And Composition
- Status: `discovery`
- Tags: `content`, `docs`, `protocol`
- Why now:
- the videos and source files suggest Jibo sometimes felt like he was joining thoughts together, even when the source text was still authored
- we have not found evidence of a general runtime joiner yet, so this remains a post-release enhancement instead of a 1.0.19 dependency
- keeping it separate lets us preserve familiar Jibo phrasing now and experiment with composition later
- Scope:
- design a post-release dialog composition layer that can stitch authored fragments together when appropriate
- keep the first version conservative and familiar, not LLM-driven
- make sure any future joining feature is opt-in and does not replace the current authored prompt path
- Follow-up:
- revisit after 1.0.19 personality import and report-skill parity stabilize
- decide whether the composition layer should sit above the prompt catalog or beside it as a dedicated response post-processor
- keep this separate from the authored-variant backlog item so we do not blur prompt richness with runtime composition
### 33. Singing And Musical Personality
- Status: `discovery`
- Tags: `content`, `docs`, `protocol`
- Why now:
- Jibos charm surface includes musical and sing-along behavior, and it fits naturally after the current personality and holiday batches
- the first pass should stay familiar and rule-based, not LLM-driven
- Scope:
- inventory the legacy song / sing / musical prompt families
- keep the first implementation source-backed if Pegasus has usable authored lines
- preserve room for a later sing-along launcher if we want one
- Exit criteria:
- a small song backlog exists with candidate phrases listed
- the release plan has a clear place for musical personality without crowding out weather/news/report work
- the current source-backed singing slice is implemented and test-covered
## Suggested Order ## Suggested Order
@@ -780,16 +972,28 @@ For `1.0.19`:
4. Weather report-skill launch compatibility - implemented 4. Weather report-skill launch compatibility - implemented
5. Dialog parsing expansion and ambiguity guardrails - in progress (`2026-05-09` third guardrail slice implemented; Pegasus affinity phrase families + continuation guardrails expanded) 5. Dialog parsing expansion and ambiguity guardrails - in progress (`2026-05-09` third guardrail slice implemented; Pegasus affinity phrase families + continuation guardrails expanded)
6. Presence-aware greetings and identity-triggered proactivity - implemented (trigger path, identity-aware reactive/proactive replies, cooldown metadata wiring, focused websocket coverage) 6. Presence-aware greetings and identity-triggered proactivity - implemented (trigger path, identity-aware reactive/proactive replies, cooldown metadata wiring, focused websocket coverage)
7. Personal report parity track (weather visuals, live news path, commute path, calendar parity matrix) - in progress (`2026-05-10` first live-news provider slice implemented) 7. Personal report parity track (weather visuals, live news path, commute path, calendar parity matrix) - in progress (`2026-05-10` first live-news provider slice implemented; commute now has a loop-scoped provider seam)
8. Holidays and seasonal personality behavior built on the new memory/proactivity foundation 8. Holidays and seasonal personality behavior built on the new memory/proactivity foundation
- system holidays should come from an up-to-date provider and merge with loop-scoped custom holiday records
- allow disabled holiday records to suppress reminders for people who do not celebrate a holiday
- birthdays and other personal dates should flow into the same loop-scoped holiday list once authoring is wired up
9. Durable memory persistence path (multi-tenant backing store) 9. Durable memory persistence path (multi-tenant backing store)
10. Update, backup, and restore proof - reference design captured in `docs/persistence-architecture.md`
- store contracts are now tightened around account/loop/device/person scoping, revision tracking, and explicit load/save boundaries
- the backend seam is now selectable, with file-backed local persistence as default and an Azure Blob Storage slot wired for future deployment when a storage account connection string is available
- next implementation pass should supply the real Azure Storage connection string / deployment wiring and validate the live round-trip in the storage account smoke test
10. Update, backup, and restore proof - implemented (update creation and backup creation now survive persisted reloads; restore is the persisted-state rehydration proof path, not a new cloud API)
11. STT upgrade and noise screening 11. STT upgrade and noise screening
- progress update (`2026-05-21`): added a low-signal short-turn screen in websocket finalization so filler-only fragments and stray single-token leftovers like `so command` get rejected before they can become bad turns, while preserving the existing yes/no and word-of-the-day short-turn flows
12. Hosted capture/storage plan / indexing for group testing 12. Hosted capture/storage plan / indexing for group testing
- progress update (`2026-05-21`): added a bundle helper so group testers can package raw capture trees, `capture-index.ndjson`, and exported fixtures into one zip handoff artifact
13. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc. 13. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc.
14. Provider-backed news and weather parity polish 14. Provider-backed news and weather parity polish
15. Grocery list capability discovery and MVP selection 15. Grocery list capability discovery and MVP selection
16. Lasso, identity, and onboarding as larger discovery-driven tracks 16. Lasso, identity, and onboarding as larger discovery-driven tracks
17. Legacy MIM personality import ladder and first declarative prompt packs
18. Longer authored persona variants for the same prompt families
19. Dialog joining/composition as a post-release enhancement, kept separate from the 1.0.19 ladder
For `1.0.20` and beyond: For `1.0.20` and beyond:
@@ -801,3 +1005,4 @@ For `1.0.20` and beyond:
6. Advanced Jibo features such as pizza delivery, Uber/Lyft integration, calendar management, smart home control (Home Assistant), etc. can be added after the conversion process is smooth and stable, with a focus on features that leverage the new cloud capabilities and content personalization enabled by Open Jibo 6. Advanced Jibo features such as pizza delivery, Uber/Lyft integration, calendar management, smart home control (Home Assistant), etc. can be added after the conversion process is smooth and stable, with a focus on features that leverage the new cloud capabilities and content personalization enabled by Open Jibo
7. LLM integration for more natural dialog, question answering, and content generation can be explored as a longer-term goal after the core platform is stable and has a growing user base to provide feedback and use cases for LLM-powered features 7. LLM integration for more natural dialog, question answering, and content generation can be explored as a longer-term goal after the core platform is stable and has a growing user base to provide feedback and use cases for LLM-powered features
8. Tiered Jibo brain/orchestration plan from README.md can be implemented in parallel with the above, starting with the simplest cloud features and gradually adding more complex capabilities as the platform matures and user feedback is collected, always preserving his unique charm and original features. 8. Tiered Jibo brain/orchestration plan from README.md can be implemented in parallel with the above, starting with the simplest cloud features and gradually adding more complex capabilities as the platform matures and user feedback is collected, always preserving his unique charm and original features.
9. Accessibility-first voice parity for menu actions, starting with backup / restore / update and extending to other critical app flows so menu functionality remains available through voice in a later release

View File

@@ -59,6 +59,16 @@ Main gap:
- no first-class presence/identity perception extraction from runtime context for greeting policy decisions - no first-class presence/identity perception extraction from runtime context for greeting policy decisions
Current implementation progress:
- runtime presence parsing now extracts speaker, people-present ids, and loop user first names
- reactive and proactive greeting turns now write durable greeting-presence history into cloud state
- proactive greeting gating now consults stored greeting history first, then falls back to the current turn metadata
- birthday-aware proactive greetings now use the loop/person birthday memory when the current date matches
- holiday-aware proactive greetings now use the loop holiday calendar when the current date matches
- morning proactive greetings now stay distinct from return-visit greetings so a fresh start of day still sounds like a morning greeting
- the remaining work is to broaden the presence policy surface so it can grow into richer day-part and return-visit variations without reworking the storage seam again
## Implementation Slices ## Implementation Slices
### Slice G1: Presence Context Extraction And Session Snapshot ### Slice G1: Presence Context Extraction And Session Snapshot

View File

@@ -0,0 +1,23 @@
# Holiday Architecture
Pegasus exposed holidays as a loop-scoped list synchronized into `/jibo/holidays`.
In OpenJibo, the holiday path now follows the same broad model:
- system holidays come from a live holiday source
- custom holidays are loop-scoped
- suppressed holidays are represented as disabled records
- the cloud protocol returns the merged list for `PersonListHolidays`
Current behavior:
- `Person/ListHolidays` uses the loop from the request when available
- if no loop is supplied, the cloud falls back safely instead of throwing
- the merged list is built from system holidays plus any custom loop entries
Notes:
- `IsEnabled = false` can be used to suppress a holiday later
- birthdays and other personal events can be added as loop-scoped custom records
- the current system holiday source uses Nager.Date with a safe local fallback for uptime
- birthday memory authoring now upserts a holiday record so the same merged list can later drive celebration and reminder behavior

View File

@@ -41,6 +41,7 @@ The `.NET` cloud now supports structured live capture intended for first robot r
- HTTP request/response event streams written as NDJSON - HTTP request/response event streams written as NDJSON
- websocket event streams written as NDJSON - websocket event streams written as NDJSON
- per-session websocket fixture export for replay - per-session websocket fixture export for replay
- a small `capture-index.ndjson` manifest beside the raw files so group testers can quickly find the session type, operation, and export artifacts
- turn metadata including `transID`, buffered audio counts, finalize attempts, and reply types - turn metadata including `transID`, buffered audio counts, finalize attempts, and reply types
Default capture location: Default capture location:
@@ -54,6 +55,7 @@ Artifacts:
- `websocket/*.events.ndjson` - `websocket/*.events.ndjson`
- `*.events.ndjson` - `*.events.ndjson`
- `websocket/fixtures/*.flow.json` - `websocket/fixtures/*.flow.json`
- `capture-index.ndjson`
## Suggested First Hookup Plan ## Suggested First Hookup Plan
@@ -61,8 +63,9 @@ Artifacts:
2. Confirm HTTP bootstrap and websocket acceptance with the existing smoke/routing helpers. 2. Confirm HTTP bootstrap and websocket acceptance with the existing smoke/routing helpers.
3. Run one or two controlled listen turns with Jibo. 3. Run one or two controlled listen turns with Jibo.
4. Inspect the captured HTTP and websocket events plus exported websocket fixtures. 4. Inspect the captured HTTP and websocket events plus exported websocket fixtures.
5. Convert the best captures into sanitized checked-in fixtures and tests. 5. Use `capture-index.ndjson` to quickly locate the important sessions and exported fixtures.
6. Keep Node available to compare any surprising turn behavior before changing infrastructure. 6. Convert the best captures into sanitized checked-in fixtures and tests.
7. Keep Node available to compare any surprising turn behavior before changing infrastructure.
Useful helper scripts: Useful helper scripts:
@@ -74,3 +77,18 @@ Useful helper scripts:
- [scripts/cloud/get-websocket-capture-summary.sh](/OpenJibo/scripts/cloud/get-websocket-capture-summary.sh) - [scripts/cloud/get-websocket-capture-summary.sh](/OpenJibo/scripts/cloud/get-websocket-capture-summary.sh)
- [scripts/cloud/import-websocket-capture-fixture.py](/OpenJibo/scripts/cloud/import-websocket-capture-fixture.py) - [scripts/cloud/import-websocket-capture-fixture.py](/OpenJibo/scripts/cloud/import-websocket-capture-fixture.py)
- [live-jibo-test-runbook.md](/OpenJibo/docs/live-jibo-test-runbook.md) - [live-jibo-test-runbook.md](/OpenJibo/docs/live-jibo-test-runbook.md)
## Group Testing Handoff
When you have a useful capture set and want to share it with another tester, bundle the capture root into a single zip so the raw events, capture index, and exported fixtures stay together.
Recommended helper:
- [scripts/cloud/New-CaptureBundle.ps1](/OpenJibo/scripts/cloud/New-CaptureBundle.ps1)
The bundle includes:
- `capture-index.ndjson`
- websocket and HTTP `*.events.ndjson` files
- exported `*.flow.json` fixtures
- a small `bundle-manifest.json` with file counts and source metadata

View File

@@ -0,0 +1,153 @@
# Local OpenJibo Cloud Quickstart
## Purpose
This guide is for people who want to run OpenJibo locally.
There are three different local paths:
- `.NET cloud`: the current OpenJibo cloud implementation and the path we are actively shipping.
- `Node cloud`: the legacy protocol oracle and reverse-engineering server. It is still useful and fun to run, but it is not the production direction.
- `Playground`: a direct local Jibo ASR/TTS demo. It talks to Jibo on local ports and does not replace the cloud.
For a physical Jibo, local cloud testing still assumes a controlled network, DNS/host routing, and certificate setup. See [device-bootstrap.md](device-bootstrap.md) for the device side.
## Prerequisites
Install:
- .NET SDK for the repo target framework
- Node.js and npm, for the Node oracle only
- PowerShell, for the Windows helper scripts
- `openssl`, for Linux live testing on port `443` with PEM certificate material
Optional for real audio experiments:
- `ffmpeg`
- `whisper.cpp`
## Run The .NET Cloud
From the repo root:
```powershell
.\scripts\cloud\Start-OpenJiboDotNet.ps1
```
By default, this starts:
- HTTPS: `https://localhost:24604`
- HTTP: `http://localhost:24605`
- health check: `http://localhost:24605/health`
- websocket captures: `captures/websocket`
- HTTP captures: `captures/http`
Smoke check:
```powershell
.\scripts\cloud\Invoke-CloudSmoke.ps1 -BaseUrl http://localhost:24605
```
Run with the Azure Blob sample launch profile:
```powershell
.\scripts\cloud\Start-OpenJiboDotNet.ps1 -UseAzureBlobProfile
```
Run directly without a launch profile, useful when you want to supply all URLs and certificate settings by environment:
```powershell
$env:ASPNETCORE_URLS = "http://0.0.0.0:24605"
.\scripts\cloud\Start-OpenJiboDotNet.ps1 -NoLaunchProfile
```
For a Linux live-device run on port `443`, reuse the existing PEM certificate material:
```bash
CERT_PEM=/path/to/cert.pem \
KEY_PEM=/path/to/key.pem \
ASPNETCORE_URLS="https://0.0.0.0:443;http://0.0.0.0:24605" \
./scripts/cloud/start-dotnet-with-node-cert.sh
```
Then run:
```bash
./scripts/cloud/invoke-live-jibo-prep.sh
```
## Run The Node Cloud
The Node cloud lives at `src/Jibo.Cloud/node`.
From the repo root:
```powershell
.\scripts\cloud\Start-OpenJiboNode.ps1 -Install
```
After dependencies are installed once, you can usually run:
```powershell
.\scripts\cloud\Start-OpenJiboNode.ps1
```
Important details:
- The Node server binds HTTPS on port `443`.
- It expects `cert.pem` and `key.pem` in `src/Jibo.Cloud/node`.
- Use the same certificate material that your controlled Jibo routing already trusts.
- On Windows or Linux, binding port `443` may require an elevated shell.
- Stop the .NET cloud first if it is also using port `443`.
Manual equivalent:
```powershell
cd src\Jibo.Cloud\node
npm install
node .\open-jibo-link.js
```
The Node server writes discovery logs under `src/Jibo.Cloud/node/logs`.
## Run Playground
Playground is not a cloud server. It connects straight to a Jibo on your LAN:
- ASR HTTP: `http://JIBO_IP:8088/asr_simple_interface`
- ASR websocket: `ws://JIBO_IP:8088/simple_port`
- TTS HTTP: `http://JIBO_IP:8089/tts_speak`
From the repo root:
```powershell
.\scripts\cloud\Start-OpenJiboPlayground.ps1
```
When prompted, enter the Jibo IP address.
Use Playground when you want to test the local ASR/TTS client behavior directly. Use the `.NET` or Node cloud when you want Jibo to boot and talk through the cloud-shaped protocol path.
## Which One Should I Use?
Use `.NET cloud` if you want the current OpenJibo behavior, release testing, captures, or anything close to the hosted future.
Use `Node cloud` if you want the original prototype/oracle, protocol discovery, or a quick comparison against older behavior.
Use `Playground` if you already know the robot IP and just want a local microphone-to-ASR-to-TTS loop through Jibo's local client interfaces.
## Common Issues
If `/health` fails, confirm the .NET cloud is running and use `http://localhost:24605/health` for local checks.
If the Node server fails with a certificate error, add `cert.pem` and `key.pem` to `src/Jibo.Cloud/node`.
If port `443` is busy, stop the other cloud server first or run the .NET cloud on the local dev ports.
If a physical Jibo does not connect, confirm DNS/host routing for:
- `api.jibo.com`
- `api-socket.jibo.com`
- `neo-hub.jibo.com`
Then compare with the live runbook in [live-jibo-test-runbook.md](live-jibo-test-runbook.md).

View File

@@ -0,0 +1,136 @@
# Persistence Architecture
## Goal
Keep OpenJibo's stateful behavior portable now and Azure-ready later.
The current in-memory stores are fine as the default implementation, but the app should depend on stable persistence contracts rather than directly on in-memory collections or file formats.
## Design Principles
- Application code talks to small, intent-specific interfaces.
- Persistence keys are always scoped by tenant and person where relevant.
- In-memory, local JSON, and hosted Azure stores are adapters, not behavior sources.
- Long-lived data should be versioned so we can add optimistic concurrency later.
- Ephemeral turn/session state should stay separate from durable user and device state.
## Current Seams
These are the contracts we should preserve:
- `IPersonalMemoryStore`
- personal facts: names, birthdays, preferences, affinities, important dates, household lists
- scope: account + loop + device + optional person
- `ICloudStateStore`
- account, robot, loops, people, sessions, updates, media, backups, holidays, keys
- scope: system-level state with loop/device/person records inside it
- `IJiboExperienceContentRepository`
- catalog/content layer only
## Recommended Storage Split
### 1. Identity and topology store
Responsible for:
- account profile
- robot/device registration
- loop membership
- person records
- greeting/proactive presence metadata when it becomes durable
This is the seam most likely to become Azure SQL or Cosmos later.
### 2. Personal memory store
Responsible for:
- names
- birthdays
- preferences
- affinities
- important dates
- household lists
This can remain in memory now and later move to a durable store keyed by account/loop/device/person.
### 3. Session and short-lived orchestration state
Responsible for:
- websocket/session tokens
- temporary skill state
- active report/list/greeting interaction state
This can stay in-process for now, but should be clearly separated from durable memory.
### 4. Media and backup store
Responsible for:
- uploaded media metadata
- backup manifests
- binary references
This is a good candidate for Azure Blob Storage plus a metadata table later.
## Record Shape Guidance
For durable records, prefer a small shared envelope:
- `AccountId`
- `LoopId`
- `DeviceId`
- `PersonId` when relevant
- `RecordType`
- `RecordKey`
- `Value`
- `CreatedUtc`
- `UpdatedUtc`
- `Revision` or `ETag`
That gives us:
- easy partitioning later
- clear tenant boundaries
- room for concurrency checks
- a path to Azure Table, Cosmos, or SQL without changing behavior code
## Adapter Plan
### Phase 1
- keep `InMemoryPersonalMemoryStore`
- keep `InMemoryCloudStateStore`
- make sure all callers use the interfaces only
- add tests against behavior, not implementation details
### Phase 2
- introduce durable adapters behind the same interfaces
- likely split:
- SQL or Cosmos for identity/topology
- Blob or table-backed store for media/backup metadata
- table/SQL-backed memory store for personal facts
### Phase 3
- add replication/sync primitives if we need multi-server state convergence
- prefer explicit change records or versioned snapshots over hidden shared state
## Non-Goals For Now
- no Azure SDK types in application logic
- no event-sourcing rewrite
- no giant generic repository
- no distributed transaction work before single-node semantics are stable
## Immediate Next Step
Before building durable adapters, tighten the store contracts around:
- tenant/person scoping
- record versioning
- explicit load/save operations for durable state
That lets us swap the backing store later without changing the personality, report, greeting, or list behaviors already built on top.

View File

@@ -6,6 +6,8 @@ This release starts the shift from `1.0.18` hardening to visible feature growth.
The goal is to keep compatibility work steady while shipping personality and capability slices that make OpenJibo feel less like a placeholder cloud and more like a real assistant platform. The goal is to keep compatibility work steady while shipping personality and capability slices that make OpenJibo feel less like a placeholder cloud and more like a real assistant platform.
For grocery list capability, the 1.0.19 MVP choice is the existing household list engine with grocery as a first-class spoken alias. That keeps the storage model simple now while leaving integration-backed list orchestration for a later pass.
## Snapshot ## Snapshot
- Kickoff date: `2026-05-05` - Kickoff date: `2026-05-05`
@@ -20,29 +22,120 @@ The goal is to keep compatibility work steady while shipping personality and cap
- start building reusable content hooks for question-vs-command style responses - start building reusable content hooks for question-vs-command style responses
- keep first implementation rule-based and test-backed - keep first implementation rule-based and test-backed
### 1a. Original Personalized Function Inventory
Keep a running checklist of the legacy persona questions and identity surfaces we want to preserve or port:
- identity and origin: `what are you`, `who are you`, `what is Jibo`, `who made you`, `where are you from`
- persona and capability: `do you have a personality`, `what is your job`, `how much do you know`, `what do you want`
- self-description and social charm: `what's your name`, `do you have a nickname`, `do you like being Jibo`, `are there others like you`
- favorite-style prompts: `what is your favorite color`, `what is your favorite food`, `what is your favorite music`
- attraction and preference prompts: `what is your favorite flower`, `do you like R2D2`, `do you like the sun`, `do you like space`, `do you like kids`
- longer authored variants for the same prompt family when Pegasus shows richer phrasing, especially multi-clause and follow-up-heavy responses
- capability and charm prompts: `can you laugh`, `can you dance`, `can you sing`, `will you sing`
- affect and mood: `how are you`, `are you happy`, `are you sad`, `are you angry`
- memory and identity recall: `who am i`, `what is my name`, `when is my birthday`, `what is my favorite music`
- greeting and presence charm: `good morning`, `welcome back`, `who is this`, person-aware greeting follow-ups
- recognition follow-ups: `do you know me`, `do you remember me`, `can you recognize me`
- seasonal and contextual charm: holiday prompts, pizza day, surprise offers, personal report personality hooks
- conversational follow-ups that should stay local and warm instead of falling into generic chat
Current batch note:
- `favorite color`, `favorite food`, and `favorite music` are the first small favorites-family slice
- the latest pass adds longer authored variants for those favorites so the replies keep more of the original Pegasus cadence instead of collapsing to short placeholders
- singing and musical personality now has a source-backed first slice with `can you sing`, `will you sing`, and holiday sing variants so the charm surface can keep growing without inventing a new dialog engine
- the friendship batch now includes `do you have friends`, `are we friends`, and `are we best friends` responses, plus the loop-friendly friend replies, so the relationship lane can stay source-backed too
- the parser guardrail pass now expands those friendship routes to named-person forms like `are you friends with Siri` and `is Dr. Breazeal your best friend`, keeping the ambiguity layer closer to Pegasus utterance shapes
- the next source-backed batch now includes `favorite flower`, `R2D2`, `sun`, `space`, `kids`, plus a couple of charm prompts like `can you laugh` and `can you dance`
- the motion/sleep batch now adds `RI_JBO_CanSleep` and `RA_JBO_SpinAround` so the `go to sleep` and `turn around` surfaces stay source-backed too
- the follow-up mood batch now includes `how are things`, `how is your day`, `are you sad`, and `are you angry`
- the personality follow-up batch now includes `what are you up to` and `what are you doing` so small talk stays warm and local instead of falling into generic chat
- the descriptor batch now includes `are you kind`, `are you funny`, `are you helpful`, `are you curious`, `are you loyal`, `are you mischievous`, and `are you likable`
- the seasonal batch now includes `what holidays do you celebrate`, New Year's resolution questions, `happy holidays`, `what halloween costume`, spring and summer suggestions, a favorite-season prompt, and holiday gift prompts
- the holiday extras batch now includes `show santa tracker` so the seasonal holiday launcher stays source-backed too
- the remaining seasonal polish now includes `do you like halloween`, `do you like holiday music`, `do you like holiday parties`, `are you looking forward to christmas`, `what are you doing for christmas`, and `what are you thankful for`
- the favorites batch now includes `what is your favorite animal`, `what is your favorite bird`, `do you like penguins`, and `do you like animals` so the penguin-centered replies stay close to Pegasus
- the latest social batch adds `welcome back`, `what are you thinking`, `what have you been doing`, and `what did you do` so presence and charm stay lively without distracting from the memory roadmap
- the newest identity-charm batch adds `what's your name`, `do you have a nickname`, `do you like being Jibo`, `are there others like you`, and `what is your favorite name` so the robot stays familiar while still sounding like Pegasus
- the next deep-personality batch adds `what do you dream about`, `what are you afraid of`, `what do you want to talk about`, `what is your best book`, `what is your best exercise`, `what is your dream vacation`, `who is your hero`, `who do you love`, and `what is your religion`; `what is your sign` is still deferred until we add templated placeholder rendering
- the next identity/knowledge batch adds `are you god`, `are you here`, `do you have super powers`, `how much do you know`, `what does jibo mean`, `where do you get info`, `what are you forbidden to do`, `what color are you`, and `what do you do when alone`
- the next body/mission batch adds `how much do you weigh`, `how tall are you`, `how much do you cost`, `what if I unplug you`, `what is your purpose`, `what is your prime directive`, `what is jibo commander`, `do you like commander app`, and `what are you made of`
- the templated edge-case batch adds `what is your sign`, `how many people do you know`, and `what is the loop` so the remaining source-backed lines can lean on live birthday and loop state
- the work/eat/home batch adds `how do you work`, `what do you eat`, `where do you live`, and `what languages do you speak` so the everyday self-description cluster keeps moving toward the original phrasing
- the age batch adds `how old are you` through `JBO_HowOldAreYou` so the birthday and first-powered-up phrasing stays source-backed instead of falling back to a generic age answer
- live QA has surfaced a few repair targets to carry into the next pass: person-identification collisions inside the same loop, `turn around` / `go to sleep` motion quirks, and a couple of reply-selection spots where short variants are being over-selected (`how are you`, `what is your favorite flower`)
- this pass keeps Build B moving while still favoring source-backed phrasing and preserving the command-vs-question boundary
- the next passes should keep the same pattern and prefer source-backed phrasing whenever the legacy MIM text is available
- if a source-backed legacy line is missing, use a temporary direct reply only to keep the pass moving, then backfill source text later
- after the favorites batch, the next doc pass should focus on richer persona follow-ups and the remaining memory/presence charm surfaces
- Build B is now reserved for the next source-backed scripted-response batch:
- `how do you work`
- `what do you eat`
- `where do you live`
- `where were you born`
- `what languages do you speak`
- `what do you like to do`
- `what are you made of`
- `what is your favorite flower`
- `do you like R2D2`
- `do you like the sun`
- `do you like space`
- `do you like kids`
- `can you laugh`
- `can you dance`
The goal is to port these in small batches, capture the source-backed phrasing where possible, and keep a test for each batch so the list never becomes a vague backlog graveyard.
### 2. Reliability And Device Proof ### 2. Reliability And Device Proof
- complete update/backup/restore proof path with captures and operator docs - complete update/backup/restore proof path with captures and operator docs
- the restore proof is the persisted-state rehydration path; do not scope it into a new hosted restore API until we have real device evidence
- continue alarm/gallery/yes-no cleanup from `1.0.18` evidence where regressions are still open - continue alarm/gallery/yes-no cleanup from `1.0.18` evidence where regressions are still open
- improve short-turn STT reliability and low-signal screening - improve short-turn STT reliability and low-signal screening
- the latest STT pass adds a websocket-side low-signal screen for filler-only and stray single-token leftovers while keeping yes/no and word-of-the-day turns intact
- capture indexing and group-test handoff now have a bundle helper that packages raw event captures, the index manifest, and exported fixtures together for easier review/share flows
### 3. Pegasus-To-Cloud Platform Porting ### 3. Pegasus-To-Cloud Platform Porting
- prioritize small source-backed slices from Pegasus/JiboOS that can be shipped safely - prioritize small source-backed slices from Pegasus/JiboOS that can be shipped safely
- keep Nimbus and stock payload compatibility as the release guardrail - keep Nimbus and stock payload compatibility as the release guardrail
- avoid broad subsystem rewrites without tests and live-capture evidence - avoid broad subsystem rewrites without tests and live-capture evidence
- keep the legacy prompt inventory visible in the backlog so porting stays paced and traceable
### 4. Holidays And Seasonal Personality ### 4. Holidays And Seasonal Personality
- port holiday-aware personality responses as a visible extension of the new persona slice - port holiday-aware personality responses as a visible extension of the new persona slice
- start with a small, source-backed set (for example birthdays/holidays already represented in legacy data paths) - start with a small, source-backed set (for example birthdays/holidays already represented in legacy data paths)
- ensure holiday responses feel characterful while still routing through stock-compatible payloads - ensure holiday responses feel characterful while still routing through stock-compatible payloads
- imported Build B holiday buckets now include holiday, holiday greeting, holiday gift, and birthday celebration lines
- use a loop-scoped merged holiday list in the cloud protocol so system holidays and custom person holidays can coexist
- source system holidays from a live holiday provider and keep `IsEnabled = false` records available for holiday suppression
- keep birthday/custom holiday authoring aligned with person memory so future proactivity can suppress or promote holidays per loop
- birthday memory writes now create loop-scoped holiday records, which keeps the holiday list extensible without changing the protocol shape again
### 5. Multi-Tenant Memory Storage Foundation ### 5. Multi-Tenant Memory Storage Foundation
- define tenant boundaries across account, loop, device, and person-memory records - define tenant boundaries across account, loop, device, and person-memory records
- add storage abstractions that can move from in-memory/local JSON to hosted SQL/Blob without reworking behavior layers - add storage abstractions that can move from in-memory/local JSON to hosted SQL/Blob without reworking behavior layers
- implement memory-ready schemas and repository contracts for user facts (names, birthdays, personal dates, preferences) with strict tenant scoping - implement memory-ready schemas and repository contracts for user facts (names, birthdays, personal dates, preferences) with strict tenant scoping
- seed person-aware state keys now so future interactions can scope to account + loop + device + person without another shape change
- keep stateful interaction flows repository-backed instead of embedding more ad hoc metadata in the websocket layer
- the store seam now exposes revision metadata plus explicit load/save boundaries so durable adapters can drop in later without changing behavior code
- the backend seam is now selectable, with file-backed local persistence as the default and an Azure Blob Storage slot wired for future deployment wiring
### 6. Multi-Server Sync Path
- document the eventual sync boundary for stateful data that should move between servers
- treat the first pass as repository-local durability, then layer replication and conflict handling on top
- prefer explicit change records or versioned state snapshots over implicit last-writer wins when we outgrow a single node
- keep cross-server reconciliation out of the hot path until the single-server semantics are stable
Reference design:
- [persistence-architecture.md](persistence-architecture.md)
- [holiday-architecture.md](holiday-architecture.md)
- [commute-architecture.md](commute-architecture.md)
## First Implemented Slice In `1.0.19` ## First Implemented Slice In `1.0.19`
@@ -105,6 +198,80 @@ The fifth delivered slice adds provider-backed weather content while preserving
- simple location extraction is supported for phrasing like `what's the weather in Chicago tomorrow` - simple location extraction is supported for phrasing like `what's the weather in Chicago tomorrow`
- provider config supports appsettings and `OPENWEATHER_API_KEY` environment fallback for deployment - provider config supports appsettings and `OPENWEATHER_API_KEY` environment fallback for deployment
## Personality Import Ladder
This is the practical plan for importing legacy Jibo `mims` into OpenJibo without pretending we already have a full Pegasus runtime.
### What Is Possible Today
OpenJibo can already host a meaningful subset of legacy personality content because it has:
- a shared catalog for content-driven replies
- chitchat state-machine routing with route metadata
- outbound payload support for `skillId`, `mim_id`, `mim_type`, `prompt_id`, `prompt_sub_category`, and ESML
- existing examples that already behave like legacy MIMs for pizza, dance, news, weather, and generic chat
### What We Need To Build
To move from hand-wired examples to broader imports, we need three small platform pieces:
1. a MIM inventory importer that can scan the legacy tree and produce a normalized catalog
2. a prompt-selection layer that can choose by `skill_id`, `mim_id`, prompt category, and condition metadata
3. a safe ESML/prompt renderer that preserves existing stock-compatible payload shapes
### What Can Be Ported With Each Build
#### Build A: Declarative Prompt Packs
Port immediately:
- `core-responses`
- `deflector`
- the simplest `emotion-responses`
- any `scripted-responses` that are just direct prompt lists with no special state machine
Why these first:
- they are already close to the current `JiboExperienceCatalog` model
- they give us user-visible personality quickly
- they are the best fit for low-risk testing tomorrow
#### Build B: Conditioned Prompt Packs
Port after the importer and renderer are in place:
- `gqa-responses`
- structured emotion responses with `condition` gates
- prompt sets that select different replies by user state or Jibo state
Why these next:
- they are still mostly declarative
- they need a small amount of condition evaluation, but not a new conversation engine
#### Build C: Conversation Families
Port after Build B:
- richer `scripted-responses` families that depend on follow-up state
- special-date / holiday personality sets
- more nuanced chitchat branches that need context-aware routing
- longer authored variants for existing prompts when the source text contains them, so the robot keeps the familiar Pegasus cadence without inventing new dialog composition yet
- dialog joining / composition as a post-release feature, kept out of the 1.0.19 ladder so we do not blur authored phrasing with a runtime joiner
Why these later:
- they need state and follow-up behavior, not just prompt selection
- they are where personality feels most alive, but they are also where bugs will be easiest to introduce
#### Build D: Full Parity Cleanup
Port after the core ladder is stable:
- large cross-skill collections
- any MIMs that depend on Pegasus-only parser assumptions
- any files that need a dedicated runtime abstraction instead of catalog lookup
## System Diagram Alignment Snapshot (`2026-05-06`) ## System Diagram Alignment Snapshot (`2026-05-06`)
Legacy architecture (`system_diagram.png`) has been mapped to current OpenJibo cloud services so release execution stays anchored to: Legacy architecture (`system_diagram.png`) has been mapped to current OpenJibo cloud services so release execution stays anchored to:
@@ -140,6 +307,10 @@ This confirms the pizza-fact offer state now keeps the yes/no branch open throug
Personal report parity planning is now captured with Pegasus source anchors for weather visuals/animations, live news, commute, and calendar gap coverage. Personal report parity planning is now captured with Pegasus source anchors for weather visuals/animations, live news, commute, and calendar gap coverage.
Calendar is now backed by a loop-scoped provider seam that can merge persisted loop events with birthday and holiday dates, keeping the report aligned with household context.
Commute now uses a loop-scoped commute profile and provider seam so the report can speak in the legacy commute shape without inventing a separate hosted travel service yet.
Reference: Reference:
- [personal-report-parity-plan.md](personal-report-parity-plan.md) - [personal-report-parity-plan.md](personal-report-parity-plan.md)
@@ -184,7 +355,7 @@ Third completed guardrail slice under this queue:
Next queued implementation track after parser guardrails: Next queued implementation track after parser guardrails:
- personal report parity slices (weather visual parity, live news path, commute/calendar gap closure) - personal report parity slices (weather visual parity, live news path, commute/calendar refinement)
First completed slice in this personal-report parity track: First completed slice in this personal-report parity track:
@@ -193,20 +364,28 @@ First completed slice in this personal-report parity track:
- added memory/transcript category hinting for provider requests (`sports`, `technology`, `business`, etc.) - added memory/transcript category hinting for provider requests (`sports`, `technology`, `business`, etc.)
- added provider-side request caching for both news and weather to reduce integration churn and repeated lookups - added provider-side request caching for both news and weather to reduce integration churn and repeated lookups
- added focused interaction + websocket tests for provider-backed news speech output and request-hint plumbing - added focused interaction + websocket tests for provider-backed news speech output and request-hint plumbing
- added loop-scoped calendar and commute provider seams so personal report can use persisted household context instead of static placeholders
- weather payloads now distinguish current vs weekly view modes so renderer parity can key off the payload shape
- news provider now skips summaryless correction headlines before falling back to broader sources
## Next Slices ## Next Slices
1. Dialog parsing expansion (queued next as of `2026-05-06`; more phrase variants, ambiguity handling, and transcript-to-intent guardrails) 1. MIM import foundation for personality expansion
2. Presence-aware greetings and identity-triggered proactivity (reactive/proactive split, cooldowns, person-aware greeting hooks) 2. Dialog parsing expansion
3. Personal report parity slices (weather visual layer, live news path, commute path, calendar parity matrix) 3. Presence-aware greetings and identity-triggered proactivity
4. Holidays and seasonal personality slice beyond pizza day (time-scoped content backed by memory/proactivity path) - in progress: durable greeting-presence history, per-person cooldown gating, birthday/holiday-aware special-day greetings, and morning vs return-visit tone splits are now in place
5. Durable memory persistence path (swap in provider-backed multi-tenant storage while preserving behavior contracts) 4. Personal report parity slices
6. Update/backup/restore end-to-end proof (operator-run and documented) 5. Holidays and seasonal personality slice beyond pizza day
7. STT noise-screening and short-utterance reliability pass 6. Durable memory persistence path
8. Provider-backed news expansion and deeper weather parity using Pegasus-backed contracts 7. Update/backup/restore end-to-end proof - implemented
9. Capture indexing and retention boundary for group testing 8. STT noise-screening and short-utterance reliability pass
9. Provider-backed news expansion and deeper weather parity
10. Capture indexing and retention boundary for group testing, including a lightweight manifest beside raw capture files
11. Binary-safe media storage seam with file and Azure Blob adapters, ready for original/thumbnails follow-up
12. Accessibility voice parity planning for menu-equivalent flows, starting with backup / restore / update voice coverage and broader critical-path accessibility in a later release
For slices 1-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements. For slice 1, use the new import ladder above to keep the work grounded in what OpenJibo can already render today versus what needs new scaffolding.
For slices 2-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements.
## Definition Of Done ## Definition Of Done

View File

@@ -0,0 +1,101 @@
param(
[string]$CaptureRoot = "..\..\captures",
[string]$BundleDirectory = "..\..\captures\bundles",
[string]$BundleName
)
function Get-RelativePath {
param(
[Parameter(Mandatory = $true)]
[string]$BasePath,
[Parameter(Mandatory = $true)]
[string]$FullPath
)
$normalizedBase = [System.IO.Path]::GetFullPath($BasePath)
if (-not $normalizedBase.EndsWith([System.IO.Path]::DirectorySeparatorChar)) {
$normalizedBase = $normalizedBase + [System.IO.Path]::DirectorySeparatorChar
}
$normalizedFull = [System.IO.Path]::GetFullPath($FullPath)
if (-not $normalizedFull.StartsWith($normalizedBase, [StringComparison]::OrdinalIgnoreCase)) {
throw "Path '$FullPath' is not under '$BasePath'."
}
return $normalizedFull.Substring($normalizedBase.Length)
}
$resolvedCaptureRoot = Resolve-Path -LiteralPath $CaptureRoot -ErrorAction Stop
$resolvedBundleDirectory = Resolve-Path -LiteralPath $BundleDirectory -ErrorAction SilentlyContinue
if (-not $resolvedBundleDirectory) {
$resolvedBundleDirectory = New-Item -ItemType Directory -Force -Path $BundleDirectory | Select-Object -ExpandProperty FullName
}
else {
$resolvedBundleDirectory = $resolvedBundleDirectory.Path
}
if ([string]::IsNullOrWhiteSpace($BundleName)) {
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$BundleName = "capture-bundle-$timestamp"
}
$stagingDirectory = Join-Path $resolvedBundleDirectory "$BundleName.staging"
$archivePath = Join-Path $resolvedBundleDirectory "$BundleName.zip"
if (Test-Path -LiteralPath $stagingDirectory) {
Remove-Item -LiteralPath $stagingDirectory -Recurse -Force
}
New-Item -ItemType Directory -Force -Path $stagingDirectory | Out-Null
try {
$sourceFiles = Get-ChildItem -LiteralPath $resolvedCaptureRoot -Recurse -File | Where-Object {
$_.Name -eq "capture-index.ndjson" -or
$_.Name -like "*.events.ndjson" -or
$_.Name -like "*.flow.json"
}
if (-not $sourceFiles) {
Write-Host "No capture files were found under $resolvedCaptureRoot"
exit 0
}
foreach ($file in $sourceFiles) {
$relativePath = Get-RelativePath -BasePath $resolvedCaptureRoot -FullPath $file.FullName
$destinationPath = Join-Path $stagingDirectory $relativePath
$destinationDirectory = Split-Path -Parent $destinationPath
if (-not (Test-Path -LiteralPath $destinationDirectory)) {
New-Item -ItemType Directory -Force -Path $destinationDirectory | Out-Null
}
Copy-Item -LiteralPath $file.FullName -Destination $destinationPath -Force
}
$captureIndexFiles = @($sourceFiles | Where-Object { $_.Name -eq "capture-index.ndjson" })
$eventFiles = @($sourceFiles | Where-Object { $_.Name -like "*.events.ndjson" })
$fixtureFiles = @($sourceFiles | Where-Object { $_.Name -like "*.flow.json" })
$manifest = [ordered]@{
createdUtc = (Get-Date).ToUniversalTime().ToString("O")
sourceRoot = $resolvedCaptureRoot
fileCount = $sourceFiles.Count
captureIndexCount = $captureIndexFiles.Count
eventFileCount = $eventFiles.Count
fixtureCount = $fixtureFiles.Count
}
$manifest | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath (Join-Path $stagingDirectory "bundle-manifest.json") -Encoding utf8
if (Test-Path -LiteralPath $archivePath) {
Remove-Item -LiteralPath $archivePath -Force
}
Compress-Archive -Path (Join-Path $stagingDirectory '*') -DestinationPath $archivePath -Force
Write-Host "Created capture bundle at $archivePath"
}
finally {
if (Test-Path -LiteralPath $stagingDirectory) {
Remove-Item -LiteralPath $stagingDirectory -Recurse -Force
}
}

View File

@@ -2,6 +2,12 @@
These scripts help exercise the new .NET hosted cloud locally. These scripts help exercise the new .NET hosted cloud locally.
- `Start-OpenJiboDotNet.ps1`
Starts the current `.NET` cloud with local capture directories configured.
- `Start-OpenJiboNode.ps1`
Starts the legacy Node protocol oracle from `src/Jibo.Cloud/node`.
- `Start-OpenJiboPlayground.ps1`
Starts the direct local Jibo ASR/TTS Playground demo.
- `Invoke-CloudSmoke.ps1` - `Invoke-CloudSmoke.ps1`
Runs a few quick HTTP checks against a local OpenJibo cloud instance. Runs a few quick HTTP checks against a local OpenJibo cloud instance.
- `Invoke-ProtocolFixture.ps1` - `Invoke-ProtocolFixture.ps1`
@@ -16,6 +22,8 @@ These scripts help exercise the new .NET hosted cloud locally.
Runs a small readiness checklist before the first physical Jibo test against the .NET cloud. Runs a small readiness checklist before the first physical Jibo test against the .NET cloud.
- `Import-WebSocketCaptureFixture.ps1` - `Import-WebSocketCaptureFixture.ps1`
Sanitizes an exported websocket capture fixture and copies it into the checked-in websocket fixture set. Sanitizes an exported websocket capture fixture and copies it into the checked-in websocket fixture set.
- `New-CaptureBundle.ps1`
Packages the capture root, capture index, and exported fixtures into a single zip bundle for group testing handoff.
- `start-dotnet-with-node-cert.sh` - `start-dotnet-with-node-cert.sh`
Starts the .NET API on Linux using the same PEM certificate material already used by the Node server. Starts the .NET API on Linux using the same PEM certificate material already used by the Node server.
- `invoke-live-jibo-prep.sh` - `invoke-live-jibo-prep.sh`
@@ -24,3 +32,5 @@ These scripts help exercise the new .NET hosted cloud locally.
Bash summary helper for captured websocket telemetry and exported fixtures. Bash summary helper for captured websocket telemetry and exported fixtures.
- `import-websocket-capture-fixture.py` - `import-websocket-capture-fixture.py`
Cross-platform import/sanitization helper for exported websocket fixtures. Cross-platform import/sanitization helper for exported websocket fixtures.
See [docs/local-cloud-quickstart.md](../../docs/local-cloud-quickstart.md) for the full local setup guide.

View File

@@ -0,0 +1,41 @@
param(
[string]$ProjectPath = "src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Jibo.Cloud.Api.csproj",
[string]$LaunchProfile = "Jibo.Cloud.Api",
[string]$CaptureRoot = "captures",
[switch]$UseAzureBlobProfile,
[switch]$NoLaunchProfile
)
$ErrorActionPreference = "Stop"
$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..\.."))
$resolvedProjectPath = [System.IO.Path]::GetFullPath((Join-Path $repoRoot $ProjectPath))
$resolvedCaptureRoot = [System.IO.Path]::GetFullPath((Join-Path $repoRoot $CaptureRoot))
$webSocketCaptureDirectory = Join-Path $resolvedCaptureRoot "websocket"
$httpCaptureDirectory = Join-Path $resolvedCaptureRoot "http"
if (-not (Test-Path -LiteralPath $resolvedProjectPath)) {
throw "Could not find .NET API project at $resolvedProjectPath"
}
New-Item -ItemType Directory -Force -Path $webSocketCaptureDirectory | Out-Null
New-Item -ItemType Directory -Force -Path $httpCaptureDirectory | Out-Null
$env:OpenJibo__Telemetry__DirectoryPath = $webSocketCaptureDirectory
$env:OpenJibo__ProtocolTelemetry__DirectoryPath = $httpCaptureDirectory
if ($UseAzureBlobProfile) {
$LaunchProfile = "Jibo.Cloud.Api.AzureBlob"
}
Write-Host "Starting OpenJibo .NET cloud"
Write-Host " - project: $resolvedProjectPath"
Write-Host " - websocket captures: $webSocketCaptureDirectory"
Write-Host " - http captures: $httpCaptureDirectory"
if ($NoLaunchProfile) {
Write-Host " - launch profile: disabled"
dotnet run --project $resolvedProjectPath --no-launch-profile
} else {
Write-Host " - launch profile: $LaunchProfile"
dotnet run --project $resolvedProjectPath --launch-profile $LaunchProfile
}

View File

@@ -0,0 +1,45 @@
param(
[string]$NodeDirectory = "src/Jibo.Cloud/node",
[switch]$Install
)
$ErrorActionPreference = "Stop"
$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..\.."))
$resolvedNodeDirectory = [System.IO.Path]::GetFullPath((Join-Path $repoRoot $NodeDirectory))
$serverPath = Join-Path $resolvedNodeDirectory "open-jibo-link.js"
$packagePath = Join-Path $resolvedNodeDirectory "package.json"
$certPath = Join-Path $resolvedNodeDirectory "cert.pem"
$keyPath = Join-Path $resolvedNodeDirectory "key.pem"
if (-not (Test-Path -LiteralPath $serverPath)) {
throw "Could not find Node server at $serverPath"
}
if (-not (Test-Path -LiteralPath $packagePath)) {
throw "Could not find package.json at $packagePath"
}
if ($Install -or -not (Test-Path -LiteralPath (Join-Path $resolvedNodeDirectory "node_modules"))) {
Write-Host "Installing Node dependencies"
npm install --prefix $resolvedNodeDirectory
}
if (-not (Test-Path -LiteralPath $certPath) -or -not (Test-Path -LiteralPath $keyPath)) {
Write-Warning "cert.pem and key.pem are not present in $resolvedNodeDirectory."
Write-Warning "The Node oracle expects those files because it binds HTTPS on port 443."
Write-Warning "Use the same dev certificate material that your controlled Jibo routing already trusts."
}
Write-Host "Starting OpenJibo Node protocol oracle"
Write-Host " - directory: $resolvedNodeDirectory"
Write-Host " - server: $serverPath"
Write-Host " - port: 443"
Push-Location $resolvedNodeDirectory
try {
node .\open-jibo-link.js
}
finally {
Pop-Location
}

View File

@@ -0,0 +1,20 @@
param(
[string]$ProjectPath = "src/Playground/Playground.csproj"
)
$ErrorActionPreference = "Stop"
$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..\.."))
$resolvedProjectPath = [System.IO.Path]::GetFullPath((Join-Path $repoRoot $ProjectPath))
if (-not (Test-Path -LiteralPath $resolvedProjectPath)) {
throw "Could not find Playground project at $resolvedProjectPath"
}
Write-Host "Starting OpenJibo Playground"
Write-Host " - project: $resolvedProjectPath"
Write-Host " - mode: direct local Jibo ASR/TTS client"
Write-Host ""
Write-Host "When prompted, enter the Jibo IP address on your local network."
dotnet run --project $resolvedProjectPath

View File

@@ -1,10 +1,10 @@
<!doctype html> <!doctype html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8"/>
<title>Jibo QR Generator</title> <title>Jibo QR Generator</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/qrcodejs/1.0.0/qrcode.min.js"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/qrcodejs/1.0.0/qrcode.min.js"></script>
<style> <style>
* { * {
box-sizing: border-box; box-sizing: border-box;
margin: 0; margin: 0;
@@ -122,57 +122,65 @@
text-align: center; text-align: center;
} }
</style> </style>
</head> </head>
<body> <body>
<h1>🤖 Jibo Wi-Fi QR Generator</h1> <h1>🤖 Jibo Wi-Fi QR Generator</h1>
<p class="sub">Generates a QR code using Jibo's XOR encoding format</p> <p class="sub">Generates a QR code using Jibo's XOR encoding format</p>
<span id="accessToken"></span> <span id="accessToken"></span>
<span id="wifiConfig"></span> <span id="wifiConfig"></span>
<div class="card"> <div class="card">
<label>SSID (Network Name)</label> <label>SSID (Network Name)</label>
<input id="ssid" placeholder="MyNetwork" /> <input id="ssid" placeholder="MyNetwork"/>
<label>Password (leave blank for open network)</label> <label>Password (leave blank for open network)</label>
<input id="password" type="password" placeholder="••••••••" /> <input id="password" type="password" placeholder="••••••••"/>
<label class="toggle"> <label class="toggle">
<input type="checkbox" id="useStatic" onchange="toggleStatic()" /> <input type="checkbox" id="useStatic" onchange="toggleStatic()"/>
Use Static IP Use Static IP
</label> </label>
<div class="static-section" id="staticSection"> <div class="static-section" id="staticSection">
<div class="row"> <div class="row">
<div> <div>
<label>Static IP</label <label>
><input id="staticIP" placeholder="192.168.1.100" /> Static IP
</div> </label
<div> ><input id="staticIP" placeholder="192.168.1.100"/>
<label>Netmask</label </div>
><input id="netmask" placeholder="255.255.255.0" /> <div>
</div> <label>
</div> Netmask
<div class="row"> </label
<div> ><input id="netmask" placeholder="255.255.255.0"/>
<label>Gateway</label </div>
><input id="gateway" placeholder="192.168.1.1" /> </div>
</div> <div class="row">
<div> <div>
<label>DNS 1</label><input id="dns1" placeholder="8.8.8.8" /> <label>
</div> Gateway
</div> </label
<div><label>DNS 2</label><input id="dns2" placeholder="8.8.4.4" /></div> ><input id="gateway" placeholder="192.168.1.1"/>
</div> </div>
<div>
<label>DNS 1</label><input id="dns1" placeholder="8.8.8.8"/>
</div>
</div>
<div>
<label>DNS 2</label><input id="dns2" placeholder="8.8.4.4"/>
</div>
</div>
<button onclick="generate()">Generate QR Code</button> <button onclick="generate()">Generate QR Code</button>
</div> </div>
<div id="qr-out"> <div id="qr-out">
<div id="qrdiv"></div> <div id="qrdiv"></div>
<button id="dl" onclick="download()">⬇ Download PNG</button> <button id="dl" onclick="download()">⬇ Download PNG</button>
<p class="note">Scan with Jibo's app to configure Wi-Fi</p> <p class="note">Scan with Jibo's app to configure Wi-Fi</p>
</div> </div>
<script> <script>
function toggleStatic() { function toggleStatic() {
document.getElementById("staticSection").style.display = document.getElementById("staticSection").style.display =
document.getElementById("useStatic").checked ? "block" : "none"; document.getElementById("useStatic").checked ? "block" : "none";
@@ -302,5 +310,5 @@ e!Ekiaon*%O? 'O`);
return wifiConfig; return wifiConfig;
} }
</script> </script>
</body> </body>
</html> </html>

View File

@@ -45,6 +45,95 @@ Human-facing entry points will live on domains such as:
Robot traffic may still arrive using legacy hostnames routed to the OpenJibo service. Robot traffic may still arrive using legacy hostnames routed to the OpenJibo service.
## Azure Storage Wiring Sample
For local or hosted Blob-backed persistence, use the Azure sample config in:
- [appsettings.AzureBlob.sample.json](dotnet/src/Jibo.Cloud.Api/appsettings.AzureBlob.sample.json)
It shows the expected keys for:
- `OpenJibo:State:Backend`
- `OpenJibo:State:ConnectionString`
- `OpenJibo:PersonalMemory:Backend`
- `OpenJibo:PersonalMemory:ConnectionString`
- `OpenJibo:Media:Backend`
- `OpenJibo:Media:ConnectionString`
The connection string can also come from:
- `OPENJIBO_STATE_STORAGE_CONNECTION_STRING`
- `OPENJIBO_PERSONAL_MEMORY_STORAGE_CONNECTION_STRING`
- `OPENJIBO_MEDIA_STORAGE_CONNECTION_STRING`
For a real storage account, swap `UseDevelopmentStorage=true` with your Azure Storage connection string.
## Local Startup Note
For the practical local run guide, including `.NET`, Node, and Playground, start with:
- [Local OpenJibo Cloud Quickstart](../../docs/local-cloud-quickstart.md)
To run the API with the Blob-backed sample config in Visual Studio or `dotnet run`, choose the
`Jibo.Cloud.Api.AzureBlob` launch profile.
The test project also has a matching `Jibo.Cloud.Tests.AzureBlob` profile so the smoke test can use
the same environment-variable shape when you run it from an IDE.
Equivalent environment variables:
```powershell
$env:OpenJibo__State__Backend = "AzureBlob"
$env:OpenJibo__State__ConnectionString = "UseDevelopmentStorage=true"
$env:OpenJibo__PersonalMemory__Backend = "AzureBlob"
$env:OpenJibo__PersonalMemory__ConnectionString = "UseDevelopmentStorage=true"
$env:OpenJibo__Media__Backend = "AzureBlob"
$env:OpenJibo__Media__ConnectionString = "UseDevelopmentStorage=true"
dotnet run --project dotnet/src/Jibo.Cloud.Api/Jibo.Cloud.Api.csproj
```
Replace `UseDevelopmentStorage=true` with your real storage account connection string when you move
from local emulation to Azure.
## Holiday Wiring
Holiday lists are now sourced from a live holiday provider and merged with loop-scoped custom
holiday records.
The default country code is `US`, but you can override it with:
- `OpenJibo:Holiday:CountryCode`
If you later add custom holiday authoring, disabled records can be used to suppress a holiday for a
loop without removing the underlying system holiday source.
## Calendar Wiring
Calendar report output is now driven by a loop-scoped in-process provider.
The provider currently:
- reads persisted loop calendar events
- folds in birthday and holiday dates that already live in the loop-scoped holiday list
- returns a safe empty calendar view when nothing is scheduled
This keeps the personal report moving toward Pegasus-style household-aware output without forcing a
full external calendar integration yet.
## Commute Wiring
Commute report output is now driven by a loop-scoped commute profile plus a provider seam.
The provider currently:
- reads persisted loop commute profiles
- returns a setup view when commute is missing or incomplete
- computes commute timing from the loop profile and the current clock
- keeps the personal report flow aligned with the stock `Commute_*` shape
The provider is intentionally conservative for now. It preserves the old report shape and gives us
room to add a richer travel-time source later without changing the behavior layer again.
## Recovery Strategy ## Recovery Strategy
The first supported device path is: The first supported device path is:

View File

@@ -8,6 +8,15 @@ This is the production-oriented path for restoring device connectivity and creat
Current spoken cloud version: `Cloud version 1.0.19.` Current spoken cloud version: `Cloud version 1.0.19.`
Local startup:
```powershell
.\scripts\cloud\Start-OpenJiboDotNet.ps1
```
Run that from the repo root. For the full local guide, including Node and Playground, see
[local-cloud-quickstart.md](../../../docs/local-cloud-quickstart.md).
Release hygiene reminder: Release hygiene reminder:
- bump [OpenJiboCloudBuildInfo.cs](/C:/Projects/JiboExperiments/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/OpenJiboCloudBuildInfo.cs) whenever we ship a meaningful hosted-cloud update - bump [OpenJiboCloudBuildInfo.cs](/C:/Projects/JiboExperiments/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/OpenJiboCloudBuildInfo.cs) whenever we ship a meaningful hosted-cloud update

View File

@@ -1,5 +1,6 @@
using System.Net.WebSockets; using System.Net.WebSockets;
using System.Text; using System.Text;
using System.Text.Json;
using Jibo.Cloud.Application.Abstractions; using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Application.Services; using Jibo.Cloud.Application.Services;
using Jibo.Cloud.Domain.Models; using Jibo.Cloud.Domain.Models;
@@ -88,18 +89,13 @@ app.Use(async (context, next) =>
var replies = await webSocketService.HandleMessageAsync(envelope, context.RequestAborted); var replies = await webSocketService.HandleMessageAsync(envelope, context.RequestAborted);
var session = ResolveSession(webSocketService, envelope); var session = ResolveSession(webSocketService, envelope);
await telemetrySink.RecordInboundAsync(envelope, session, ReadMessageType(envelope.Text), context.RequestAborted); await telemetrySink.RecordInboundAsync(envelope, session, ReadMessageType(envelope.Text),
context.RequestAborted);
foreach (var reply in replies) foreach (var reply in replies)
{ {
if (string.IsNullOrWhiteSpace(reply.Text)) if (string.IsNullOrWhiteSpace(reply.Text)) continue;
{
continue;
}
if (reply.DelayMs > 0) if (reply.DelayMs > 0) await Task.Delay(reply.DelayMs, context.RequestAborted);
{
await Task.Delay(reply.DelayMs, context.RequestAborted);
}
var payload = Encoding.UTF8.GetBytes(reply.Text); var payload = Encoding.UTF8.GetBytes(reply.Text);
await socket.SendAsync(payload, WebSocketMessageType.Text, true, context.RequestAborted); await socket.SendAsync(payload, WebSocketMessageType.Text, true, context.RequestAborted);
@@ -117,7 +113,8 @@ app.Use(async (context, next) =>
Token = token Token = token
}; };
var closeSession = ResolveSession(webSocketService, closeEnvelope); var closeSession = ResolveSession(webSocketService, closeEnvelope);
await telemetrySink.RecordConnectionClosedAsync(closeEnvelope, closeSession, $"socket-loop-ended{(isPrematureClose ? "-prematurely" : string.Empty)}", context.RequestAborted); await telemetrySink.RecordConnectionClosedAsync(closeEnvelope, closeSession,
$"socket-loop-ended{(isPrematureClose ? "-prematurely" : string.Empty)}", context.RequestAborted);
}); });
app.MapGet("/health", () => Results.Json(new app.MapGet("/health", () => Results.Json(new
@@ -127,7 +124,8 @@ app.MapGet("/health", () => Results.Json(new
version = OpenJiboCloudBuildInfo.Version version = OpenJiboCloudBuildInfo.Version
})); }));
app.MapMethods("/{**path}", ["GET", "POST", "PUT"], async (HttpContext context, JiboCloudProtocolService service, IProtocolTelemetrySink telemetrySink, CancellationToken cancellationToken) => app.MapMethods("/{**path}", ["GET", "POST", "PUT"], async (HttpContext context, JiboCloudProtocolService service,
IProtocolTelemetrySink telemetrySink, CancellationToken cancellationToken) =>
{ {
var envelope = await BuildEnvelopeAsync(context, cancellationToken); var envelope = await BuildEnvelopeAsync(context, cancellationToken);
var result = await service.DispatchAsync(envelope, cancellationToken); var result = await service.DispatchAsync(envelope, cancellationToken);
@@ -136,15 +134,9 @@ app.MapMethods("/{**path}", ["GET", "POST", "PUT"], async (HttpContext context,
context.Response.StatusCode = result.StatusCode; context.Response.StatusCode = result.StatusCode;
context.Response.ContentType = result.ContentType; context.Response.ContentType = result.ContentType;
foreach (var header in result.Headers) foreach (var header in result.Headers) context.Response.Headers[header.Key] = header.Value;
{
context.Response.Headers[header.Key] = header.Value;
}
if (!string.IsNullOrEmpty(result.BodyText)) if (!string.IsNullOrEmpty(result.BodyText)) await context.Response.WriteAsync(result.BodyText, cancellationToken);
{
await context.Response.WriteAsync(result.BodyText, cancellationToken);
}
}); });
app.Run(); app.Run();
@@ -160,8 +152,7 @@ static async Task<ReceivedSocketMessage> ReceiveAsync(WebSocket socket, Cancella
{ {
result = await socket.ReceiveAsync(buffer, cancellationToken); result = await socket.ReceiveAsync(buffer, cancellationToken);
ms.Write(buffer, 0, result.Count); ms.Write(buffer, 0, result.Count);
} } while (!result.EndOfMessage);
while (!result.EndOfMessage);
return new ReceivedSocketMessage(result.MessageType, ms.ToArray()); return new ReceivedSocketMessage(result.MessageType, ms.ToArray());
} }
@@ -170,7 +161,7 @@ static async Task<ProtocolEnvelope> BuildEnvelopeAsync(HttpContext context, Canc
{ {
context.Request.EnableBuffering(); context.Request.EnableBuffering();
using var reader = new StreamReader(context.Request.Body, Encoding.UTF8, detectEncodingFromByteOrderMarks: false, leaveOpen: true); using var reader = new StreamReader(context.Request.Body, Encoding.UTF8, false, leaveOpen: true);
var bodyText = await reader.ReadToEndAsync(cancellationToken); var bodyText = await reader.ReadToEndAsync(cancellationToken);
context.Request.Body.Position = 0; context.Request.Body.Position = 0;
@@ -191,66 +182,49 @@ static async Task<ProtocolEnvelope> BuildEnvelopeAsync(HttpContext context, Canc
FirmwareVersion = context.Request.Headers["X-OpenJibo-Firmware"].ToString(), FirmwareVersion = context.Request.Headers["X-OpenJibo-Firmware"].ToString(),
ApplicationVersion = context.Request.Headers["X-OpenJibo-AppVersion"].ToString(), ApplicationVersion = context.Request.Headers["X-OpenJibo-AppVersion"].ToString(),
BodyText = bodyText, BodyText = bodyText,
Headers = context.Request.Headers.ToDictionary(pair => pair.Key, pair => pair.Value.ToString(), StringComparer.OrdinalIgnoreCase) Headers = context.Request.Headers.ToDictionary(pair => pair.Key, pair => pair.Value.ToString(),
StringComparer.OrdinalIgnoreCase)
}; };
} }
static string ResolveSocketKind(string host, PathString path) static string ResolveSocketKind(string host, PathString path)
{ {
if (host.Equals("api-socket.jibo.com", StringComparison.OrdinalIgnoreCase)) if (host.Equals("api-socket.jibo.com", StringComparison.OrdinalIgnoreCase)) return "api-socket";
{
return "api-socket";
}
if (host.Equals("neo-hub.jibo.com", StringComparison.OrdinalIgnoreCase) && if (host.Equals("neo-hub.jibo.com", StringComparison.OrdinalIgnoreCase) &&
path.StartsWithSegments("/v1/proactive")) path.StartsWithSegments("/v1/proactive"))
{
return "neo-hub-proactive"; return "neo-hub-proactive";
}
if (host.Equals("neo-hub.jibo.com", StringComparison.OrdinalIgnoreCase)) if (host.Equals("neo-hub.jibo.com", StringComparison.OrdinalIgnoreCase)) return "neo-hub-listen";
{
return "neo-hub-listen";
}
if (host.Equals("openjibo.com", StringComparison.OrdinalIgnoreCase) || if (host.Equals("openjibo.com", StringComparison.OrdinalIgnoreCase) ||
host.Equals("openjibo.ai", StringComparison.OrdinalIgnoreCase) || host.Equals("openjibo.ai", StringComparison.OrdinalIgnoreCase) ||
host.Equals("localhost", StringComparison.OrdinalIgnoreCase)) host.Equals("localhost", StringComparison.OrdinalIgnoreCase))
{
return "openjibo"; return "openjibo";
}
return "neo-hub-listen"; // now it assumes all unknown requests are neo-hub. I did this so that people with custom listen servers (like myself) won't get a bunch of 404 messages when doing a HJ request. -ZaneDev (an awful programmer) return
"neo-hub-listen"; // now it assumes all unknown requests are neo-hub. I did this so that people with custom listen servers (like myself) won't get a bunch of 404 messages when doing a HJ request. -ZaneDev (an awful programmer)
} }
static string? ResolveToken(HttpRequest request) static string? ResolveToken(HttpRequest request)
{ {
var auth = request.Headers.Authorization.ToString(); var auth = request.Headers.Authorization.ToString();
if (auth.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) if (auth.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) return auth["Bearer ".Length..].Trim();
{
return auth["Bearer ".Length..].Trim();
}
var path = request.Path.Value; var path = request.Path.Value;
if (!string.IsNullOrWhiteSpace(path) && path.Length > 1) if (!string.IsNullOrWhiteSpace(path) && path.Length > 1) return path.Trim('/');
{
return path.Trim('/');
}
return null; return null;
} }
static string ReadMessageType(string? text) static string ReadMessageType(string? text)
{ {
if (string.IsNullOrWhiteSpace(text)) if (string.IsNullOrWhiteSpace(text)) return "BINARY_OR_EMPTY";
{
return "BINARY_OR_EMPTY";
}
try try
{ {
using var document = System.Text.Json.JsonDocument.Parse(text); using var document = JsonDocument.Parse(text);
return document.RootElement.TryGetProperty("type", out var type) && type.ValueKind == System.Text.Json.JsonValueKind.String return document.RootElement.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String
? type.GetString() ?? "UNKNOWN" ? type.GetString() ?? "UNKNOWN"
: "UNKNOWN"; : "UNKNOWN";
} }
@@ -266,3 +240,5 @@ static CloudSession ResolveSession(JiboWebSocketService webSocketService, WebSoc
} }
internal sealed record ReceivedSocketMessage(WebSocketMessageType MessageType, byte[] Buffer); internal sealed record ReceivedSocketMessage(WebSocketMessageType MessageType, byte[] Buffer);
public partial class Program;

View File

@@ -7,6 +7,20 @@
"ASPNETCORE_ENVIRONMENT": "Development" "ASPNETCORE_ENVIRONMENT": "Development"
}, },
"applicationUrl": "https://localhost:24604;http://localhost:24605" "applicationUrl": "https://localhost:24604;http://localhost:24605"
},
"Jibo.Cloud.Api.AzureBlob": {
"commandName": "Project",
"launchBrowser": true,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development",
"OpenJibo__State__Backend": "AzureBlob",
"OpenJibo__State__ConnectionString": "UseDevelopmentStorage=true",
"OpenJibo__PersonalMemory__Backend": "AzureBlob",
"OpenJibo__PersonalMemory__ConnectionString": "UseDevelopmentStorage=true",
"OpenJibo__Media__Backend": "AzureBlob",
"OpenJibo__Media__ConnectionString": "UseDevelopmentStorage=true"
},
"applicationUrl": "https://localhost:24604;http://localhost:24605"
} }
} }
} }

View File

@@ -0,0 +1,19 @@
{
"OpenJibo": {
"State": {
"Backend": "AzureBlob",
"ConnectionString": "UseDevelopmentStorage=true",
"PersistencePath": "App_Data/cloud-state.json"
},
"PersonalMemory": {
"Backend": "AzureBlob",
"ConnectionString": "UseDevelopmentStorage=true",
"PersistencePath": "App_Data/personal-memory.json"
},
"Media": {
"Backend": "AzureBlob",
"ConnectionString": "UseDevelopmentStorage=true",
"ContainerName": "openjibo-media"
}
}
}

View File

@@ -39,6 +39,8 @@
"BaseUrl": "https://newsapi.org", "BaseUrl": "https://newsapi.org",
"ApiKey": "5df93a83db9c4c6888f3e06c4a53144f", "ApiKey": "5df93a83db9c4c6888f3e06c4a53144f",
"Country": "us", "Country": "us",
"Language": "en",
"FallbackQuery": "robotics OR technology OR science",
"DefaultCategories": [ "general", "technology", "sports", "business" ], "DefaultCategories": [ "general", "technology", "sports", "business" ],
"CacheTtlSeconds": 300, "CacheTtlSeconds": 300,
"FailureCacheTtlSeconds": 45 "FailureCacheTtlSeconds": 45

View File

@@ -0,0 +1,14 @@
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Abstractions;
public interface ICalendarReportProvider
{
Task<CalendarReportSnapshot?> GetReportAsync(TurnContext turn, CancellationToken cancellationToken = default);
}
public sealed record CalendarReportSnapshot(
IReadOnlyList<string> EventSummaries,
IReadOnlyList<string> EventTimesOnAt,
IReadOnlyList<string> TomorrowEventSummaries,
bool HasServiceError = false);

View File

@@ -4,6 +4,9 @@ namespace Jibo.Cloud.Application.Abstractions;
public interface ICloudStateStore public interface ICloudStateStore
{ {
PersistenceStateInfo GetPersistenceStateInfo();
void LoadPersistedState();
void SavePersistedState();
AccountProfile GetAccount(); AccountProfile GetAccount();
DeviceRegistration GetRobot(); DeviceRegistration GetRobot();
RobotProfile GetRobotProfile(); RobotProfile GetRobotProfile();
@@ -13,21 +16,39 @@ public interface ICloudStateStore
CloudSession OpenSession(string kind, string? deviceId, string? token, string? hostName, string? path); CloudSession OpenSession(string kind, string? deviceId, string? token, string? hostName, string? path);
CloudSession? FindSessionByToken(string token); CloudSession? FindSessionByToken(string token);
IReadOnlyList<LoopRecord> GetLoops(); IReadOnlyList<LoopRecord> GetLoops();
IReadOnlyList<PersonRecord> GetPeople();
IReadOnlyList<UpdateManifest> ListUpdates(string? subsystem = null, string? filter = null); IReadOnlyList<UpdateManifest> ListUpdates(string? subsystem = null, string? filter = null);
UpdateManifest? GetUpdateFrom(string? subsystem, string? fromVersion, string? filter); UpdateManifest? GetUpdateFrom(string? subsystem, string? fromVersion, string? filter);
UpdateManifest CreateUpdate(string? fromVersion, string? toVersion, string? changes, string? shaHash, long? length, string? subsystem, string? filter, IDictionary<string, object?>? dependencies);
UpdateManifest CreateUpdate(string? fromVersion, string? toVersion, string? changes, string? shaHash, long? length,
string? subsystem, string? filter, IDictionary<string, object?>? dependencies);
UpdateManifest RemoveUpdate(string? updateId); UpdateManifest RemoveUpdate(string? updateId);
IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null);
IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null,
long? before = null);
IReadOnlyList<MediaRecord> GetMedia(IReadOnlyList<string> paths); IReadOnlyList<MediaRecord> GetMedia(IReadOnlyList<string> paths);
IReadOnlyList<MediaRecord> RemoveMedia(IReadOnlyList<string> paths); IReadOnlyList<MediaRecord> RemoveMedia(IReadOnlyList<string> paths);
MediaRecord CreateMedia(string loopId, string path, string type, string reference, bool isEncrypted, IDictionary<string, object?>? meta);
MediaRecord CreateMedia(string loopId, string path, string type, string reference, bool isEncrypted,
IDictionary<string, object?>? meta);
IReadOnlyList<BackupRecord> GetBackups(); IReadOnlyList<BackupRecord> GetBackups();
BackupRecord CreateBackup(string name);
bool ShouldCreateSymmetricKey(string loopId); bool ShouldCreateSymmetricKey(string loopId);
string GetOrCreateSymmetricKey(string loopId); string GetOrCreateSymmetricKey(string loopId);
KeyRequestRecord CreateKeyRequest(string loopId, string publicKey); KeyRequestRecord CreateKeyRequest(string loopId, string publicKey);
KeyRequestRecord GetKeyRequest(string loopId, string? requestId, string? publicKey); KeyRequestRecord GetKeyRequest(string loopId, string? requestId, string? publicKey);
IReadOnlyList<KeyRequestRecord> GetIncomingKeyRequests(); IReadOnlyList<KeyRequestRecord> GetIncomingKeyRequests();
IReadOnlyList<KeyRequestRecord> GetBinaryRequests(); IReadOnlyList<KeyRequestRecord> GetBinaryRequests();
IReadOnlyList<object> GetHolidays(); IReadOnlyList<HolidayRecord> GetHolidays(string? loopId = null);
HolidayRecord UpsertHoliday(HolidayRecord holiday);
IReadOnlyList<CommuteProfileRecord> GetCommuteProfiles(string? loopId = null);
CommuteProfileRecord UpsertCommuteProfile(CommuteProfileRecord commuteProfile);
IReadOnlyList<CalendarEventRecord> GetCalendarEvents(string? loopId = null);
CalendarEventRecord UpsertCalendarEvent(CalendarEventRecord calendarEvent);
IReadOnlyList<GreetingPresenceRecord> GetGreetingPresences(string? loopId = null);
GreetingPresenceRecord UpsertGreetingPresence(GreetingPresenceRecord greetingPresence);
void UpdateRobot(DeviceRegistration registration); void UpdateRobot(DeviceRegistration registration);
} }

View File

@@ -0,0 +1,18 @@
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Abstractions;
public interface ICommuteReportProvider
{
Task<CommuteReportSnapshot?> GetReportAsync(TurnContext turn, CancellationToken cancellationToken = default);
}
public sealed record CommuteReportSnapshot(
string LocationName,
string Summary,
int DurationMinutes,
string? Mode = null,
bool EventIsEarly = false,
int MinutesUntilWork = 0,
int ExtraMinutes = 0,
bool RequiresSetup = false);

View File

@@ -0,0 +1,8 @@
using Jibo.Cloud.Domain.Models;
namespace Jibo.Cloud.Application.Abstractions;
public interface IHolidayCalendarProvider
{
IReadOnlyList<HolidayRecord> GetPublicHolidays(string? countryCode, int year);
}

View File

@@ -5,16 +5,72 @@ public interface IJiboExperienceContentRepository
Task<JiboExperienceCatalog> GetCatalogAsync(CancellationToken cancellationToken = default); Task<JiboExperienceCatalog> GetCatalogAsync(CancellationToken cancellationToken = default);
} }
public sealed class JiboConditionedReply
{
public string Condition { get; init; } = string.Empty;
public string Reply { get; init; } = string.Empty;
}
public sealed class JiboExperienceCatalog public sealed class JiboExperienceCatalog
{ {
public IReadOnlyList<string> Jokes { get; init; } = []; public IReadOnlyList<string> Jokes { get; init; } = [];
public IReadOnlyList<string> RobotFacts { get; init; } = [];
public IReadOnlyList<string> HumanFacts { get; init; } = [];
public IReadOnlyList<string> FunFacts { get; init; } = [];
public IReadOnlyList<string> FavoriteAnimalReplies { get; init; } = [];
public IReadOnlyList<string> FriendReplies { get; init; } = [];
public IReadOnlyList<string> BestFriendReplies { get; init; } = [];
public IReadOnlyList<string> SingReplies { get; init; } = [];
public IReadOnlyList<string> HolidaySingReplies { get; init; } = [];
public IReadOnlyList<string> DanceAnimations { get; init; } = []; public IReadOnlyList<string> DanceAnimations { get; init; } = [];
public IReadOnlyList<string> GreetingReplies { get; init; } = []; public IReadOnlyList<string> GreetingReplies { get; init; } = [];
public IReadOnlyList<string> HolidayReplies { get; init; } = [];
public IReadOnlyList<string> HolidaySeasonReplies { get; init; } = [];
public IReadOnlyList<string> HolidayGreetingReplies { get; init; } = [];
public IReadOnlyList<string> HolidayGiftReplies { get; init; } = [];
public IReadOnlyList<string> HolidayTrackerReplies { get; init; } = [];
public IReadOnlyList<string> BirthdayCelebrationReplies { get; init; } = [];
public IReadOnlyList<string> HowAreYouReplies { get; init; } = []; public IReadOnlyList<string> HowAreYouReplies { get; init; } = [];
public IReadOnlyList<string> AgeReplies { get; init; } = [];
public IReadOnlyList<JiboConditionedReply> EmotionReplies { get; init; } = [];
public IReadOnlyList<string> PersonalityReplies { get; init; } = []; public IReadOnlyList<string> PersonalityReplies { get; init; } = [];
public IReadOnlyList<string> PizzaReplies { get; init; } = []; public IReadOnlyList<string> PizzaReplies { get; init; } = [];
public IReadOnlyList<string> SurpriseReplies { get; init; } = []; public IReadOnlyList<string> SurpriseReplies { get; init; } = [];
public IReadOnlyList<string> PersonalReportReplies { get; init; } = []; public IReadOnlyList<string> PersonalReportReplies { get; init; } = [];
public IReadOnlyList<string> PersonalReportKickOffReplies { get; init; } = [];
public IReadOnlyList<string> PersonalReportOutroReplies { get; init; } = [];
public IReadOnlyList<string> ReportSkillTemplates { get; init; } = [];
public IReadOnlyList<string> BackupHowReplies { get; init; } = [];
public IReadOnlyList<string> RestoreHowReplies { get; init; } = [];
public IReadOnlyList<string> UpdateNextReplies { get; init; } = [];
public IReadOnlyList<string> UpdateLastReplies { get; init; } = [];
public IReadOnlyList<string> WeatherIntroReplies { get; init; } = [];
public IReadOnlyList<string> WeatherTomorrowIntroReplies { get; init; } = [];
public IReadOnlyList<string> WeatherTodayHighLowReplies { get; init; } = [];
public IReadOnlyList<string> WeatherTomorrowHighLowReplies { get; init; } = [];
public IReadOnlyList<string> WeatherServiceDownReplies { get; init; } = [];
public IReadOnlyList<string> CalendarNothingTodayReplies { get; init; } = [];
public IReadOnlyList<string> CalendarNothingReplies { get; init; } = [];
public IReadOnlyList<string> CalendarServiceDownReplies { get; init; } = [];
public IReadOnlyList<string> CalendarOutroReplies { get; init; } = [];
public IReadOnlyList<string> CommuteAppSetupReplies { get; init; } = [];
public IReadOnlyList<string> CommuteConfirmSpeakerReplies { get; init; } = [];
public IReadOnlyList<string> CommuteNowReplies { get; init; } = [];
public IReadOnlyList<string> CommuteMinutesLeftReplies { get; init; } = [];
public IReadOnlyList<string> CommuteDepartTimeNormalReplies { get; init; } = [];
public IReadOnlyList<string> CommuteDepartTimeNotNormalReplies { get; init; } = [];
public IReadOnlyList<string> CommuteDriveNormalReplies { get; init; } = [];
public IReadOnlyList<string> CommuteDriveLateReplies { get; init; } = [];
public IReadOnlyList<string> CommuteDriveHurryReplies { get; init; } = [];
public IReadOnlyList<string> CommuteDrivePoorReplies { get; init; } = [];
public IReadOnlyList<string> CommuteDriveTerribleReplies { get; init; } = [];
public IReadOnlyList<string> CommuteTransportNormalReplies { get; init; } = [];
public IReadOnlyList<string> CommuteTransportLateReplies { get; init; } = [];
public IReadOnlyList<string> CommuteTransportHurryReplies { get; init; } = [];
public IReadOnlyList<string> CommuteServiceDownReplies { get; init; } = [];
public IReadOnlyList<string> NewsIntroReplies { get; init; } = [];
public IReadOnlyList<string> NewsCategoryIntroReplies { get; init; } = [];
public IReadOnlyList<string> NewsOutroReplies { get; init; } = [];
public IReadOnlyList<string> WeatherReplies { get; init; } = []; public IReadOnlyList<string> WeatherReplies { get; init; } = [];
public IReadOnlyList<string> CalendarReplies { get; init; } = []; public IReadOnlyList<string> CalendarReplies { get; init; } = [];
public IReadOnlyList<string> CommuteReplies { get; init; } = []; public IReadOnlyList<string> CommuteReplies { get; init; } = [];

View File

@@ -0,0 +1,18 @@
namespace Jibo.Cloud.Application.Abstractions;
public interface IMediaContentStore
{
Task StoreAsync(string path, string contentType, byte[] content, IReadOnlyDictionary<string, object?>? meta,
CancellationToken cancellationToken = default);
Task<MediaContentSnapshot?> LoadAsync(string path, CancellationToken cancellationToken = default);
}
public sealed record MediaContentSnapshot
{
public string ContentType { get; init; } = "application/octet-stream";
public byte[] Content { get; init; } = [];
public IReadOnlyDictionary<string, object?> Meta { get; init; } =
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
}

View File

@@ -20,4 +20,9 @@ public sealed record NewsHeadline(
public sealed record NewsBriefingSnapshot( public sealed record NewsBriefingSnapshot(
IReadOnlyList<NewsHeadline> Headlines, IReadOnlyList<NewsHeadline> Headlines,
string? SourceName = null); string? SourceName = null,
string? ProviderStatus = null,
string? ProviderMessage = null,
int? ProviderHttpStatusCode = null,
string? ProviderEndpoint = null,
string? ProviderErrorCode = null);

View File

@@ -2,6 +2,9 @@ namespace Jibo.Cloud.Application.Abstractions;
public interface IPersonalMemoryStore public interface IPersonalMemoryStore
{ {
PersistenceStateInfo GetPersistenceStateInfo();
void LoadPersistedState();
void SavePersistedState();
void SetBirthday(PersonalMemoryTenantScope tenantScope, string birthdayText); void SetBirthday(PersonalMemoryTenantScope tenantScope, string birthdayText);
string? GetBirthday(PersonalMemoryTenantScope tenantScope); string? GetBirthday(PersonalMemoryTenantScope tenantScope);
void SetPreference(PersonalMemoryTenantScope tenantScope, string category, string value); void SetPreference(PersonalMemoryTenantScope tenantScope, string category, string value);
@@ -13,9 +16,22 @@ public interface IPersonalMemoryStore
void SetAffinity(PersonalMemoryTenantScope tenantScope, string item, PersonalAffinity affinity); void SetAffinity(PersonalMemoryTenantScope tenantScope, string item, PersonalAffinity affinity);
PersonalAffinity? GetAffinity(PersonalMemoryTenantScope tenantScope, string item); PersonalAffinity? GetAffinity(PersonalMemoryTenantScope tenantScope, string item);
IReadOnlyDictionary<string, PersonalAffinity> GetAffinities(PersonalMemoryTenantScope tenantScope); IReadOnlyDictionary<string, PersonalAffinity> GetAffinities(PersonalMemoryTenantScope tenantScope);
void AddListItem(PersonalMemoryTenantScope tenantScope, string listName, string item);
IReadOnlyList<string> GetListItems(PersonalMemoryTenantScope tenantScope, string listName);
void ClearListItems(PersonalMemoryTenantScope tenantScope, string listName);
} }
public sealed record PersonalMemoryTenantScope(string AccountId, string LoopId, string DeviceId); public sealed record PersonalMemoryTenantScope(
string AccountId,
string LoopId,
string DeviceId,
string? PersonId = null);
public sealed record PersistenceStateInfo(
string SchemaVersion,
long Revision,
DateTimeOffset? LastLoadedUtc = null,
DateTimeOffset? LastSavedUtc = null);
public enum PersonalAffinity public enum PersonalAffinity
{ {

View File

@@ -4,5 +4,6 @@ namespace Jibo.Cloud.Application.Abstractions;
public interface IProtocolTelemetrySink public interface IProtocolTelemetrySink
{ {
Task RecordAsync(ProtocolEnvelope envelope, ProtocolDispatchResult result, CancellationToken cancellationToken = default); Task RecordAsync(ProtocolEnvelope envelope, ProtocolDispatchResult result,
CancellationToken cancellationToken = default);
} }

View File

@@ -2,7 +2,8 @@ namespace Jibo.Cloud.Application.Abstractions;
public interface ITurnTelemetrySink public interface ITurnTelemetrySink
{ {
Task RecordTurnDiagnosticAsync(string category, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default); Task RecordTurnDiagnosticAsync(string category, IReadOnlyDictionary<string, object?> details,
CancellationToken cancellationToken = default);
Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default); Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default);
} }

View File

@@ -4,9 +4,18 @@ namespace Jibo.Cloud.Application.Abstractions;
public interface IWebSocketTelemetrySink public interface IWebSocketTelemetrySink
{ {
Task RecordConnectionOpenedAsync(WebSocketMessageEnvelope envelope, CloudSession session, CancellationToken cancellationToken = default); Task RecordConnectionOpenedAsync(WebSocketMessageEnvelope envelope, CloudSession session,
Task RecordInboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, string? messageType, CancellationToken cancellationToken = default); CancellationToken cancellationToken = default);
Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default);
Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default); Task RecordInboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, string? messageType,
Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason, CancellationToken cancellationToken = default); CancellationToken cancellationToken = default);
Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType,
IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default);
Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session,
IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default);
Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason,
CancellationToken cancellationToken = default);
} }

View File

@@ -1,5 +1,5 @@
using Jibo.Cloud.Application.Abstractions;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Jibo.Cloud.Application.Abstractions;
namespace Jibo.Cloud.Application.Services; namespace Jibo.Cloud.Application.Services;
@@ -24,10 +24,20 @@ internal static class ChitchatStateMachine
"how are you feeling", "how are you feeling",
"how do you feel", "how do you feel",
"what are you feeling", "what are you feeling",
"what are you up to",
"what are you doing",
"how are things",
"how's things",
"how is things",
"how's your day",
"how is your day",
"what mood are you in", "what mood are you in",
"what is your mood", "what is your mood",
"what's your mood", "what's your mood",
"do you have emotions", "do you have emotions",
"are you happy",
"are you sad",
"are you angry",
"how angry are you", "how angry are you",
"how jealous are you", "how jealous are you",
"how sad are you", "how sad are you",
@@ -126,7 +136,11 @@ internal static class ChitchatStateMachine
("jealous", ["jealous", "envious", "covetous"]), ("jealous", ["jealous", "envious", "covetous"]),
("lonely", ["lonely", "alone", "lonesome"]), ("lonely", ["lonely", "alone", "lonesome"]),
("proud", ["proud", "honored"]), ("proud", ["proud", "honored"]),
("sad", ["sad", "upset", "unhappy", "depressed", "somber", "downcast", "gloomy", "miserable", "bummed", "heartbroken", "troubled"]) ("sad",
[
"sad", "upset", "unhappy", "depressed", "somber", "downcast", "gloomy", "miserable", "bummed",
"heartbroken", "troubled"
])
]; ];
private static readonly string[] EmotionCommandReplies = private static readonly string[] EmotionCommandReplies =
@@ -152,6 +166,8 @@ internal static class ChitchatStateMachine
string loweredTranscript, string loweredTranscript,
JiboExperienceCatalog catalog, JiboExperienceCatalog catalog,
IJiboRandomizer randomizer, IJiboRandomizer randomizer,
string? currentEmotion,
string? preferredName,
Func<string> buildErrorResponse) Func<string> buildErrorResponse)
{ {
var normalizedLoweredTranscript = NormalizeForPhraseMatching(loweredTranscript); var normalizedLoweredTranscript = NormalizeForPhraseMatching(loweredTranscript);
@@ -164,23 +180,122 @@ internal static class ChitchatStateMachine
case "robot_personality": case "robot_personality":
return BuildScriptedResponseDecision( return BuildScriptedResponseDecision(
"robot_personality", "robot_personality",
randomizer.Choose(catalog.PersonalityReplies)); SelectLegacyPersonalityReply(catalog, randomizer, "curious, playful", "friendly", "personality"));
case "robot_taxes":
return BuildScriptedResponseDecision(
"robot_taxes",
SelectLegacyPersonalityReply(catalog, randomizer, "pay anything", "pay taxes", "tax"));
case "how_are_you": case "how_are_you":
return BuildEmotionQueryDecision( return BuildEmotionQueryDecision(
"how_are_you", "how_are_you",
randomizer.Choose(catalog.HowAreYouReplies)); SelectEmotionQueryReply(catalog, randomizer, currentEmotion, preferredName));
case "robot_desire":
return BuildScriptedResponseDecision(
"robot_desire",
SelectLegacyPersonalityReply(
catalog,
randomizer,
"socializing and electricity",
"want to hang out",
"be helpful",
"dance from time to time"));
case "robot_want_to_talk_about":
return BuildScriptedResponseDecision(
"robot_want_to_talk_about",
SelectLegacyPersonalityReply(catalog, randomizer, "surprise me"));
case "robot_job":
return BuildScriptedResponseDecision(
"robot_job",
SelectLegacyPersonalityReply(catalog, randomizer, "more fun than a job", "here to help you out"));
case "robot_origin_created":
return BuildScriptedResponseDecision(
"robot_origin_created",
SelectLegacyPersonalityReply(
catalog,
randomizer,
"create something",
"some people wanted to create something",
"wanted to create something",
"built a robot",
"came out from a box"));
case "robot_origin_from":
return BuildScriptedResponseDecision(
"robot_origin_from",
SelectLegacyPersonalityReply(catalog, randomizer, "boston", "came out from a box"));
case "robot_identity":
return BuildScriptedResponseDecision(
"robot_identity",
SelectLegacyPersonalityReply(catalog, randomizer, "am a robot", "i'm either jibo",
"i am just jibo"));
case "robot_likes_being_jibo":
return BuildScriptedResponseDecision(
"robot_likes_being_jibo",
SelectLegacyPersonalityReply(
catalog,
randomizer,
"nothing i'd rather be",
"love it",
"being a human seems so complicated",
"especially yours",
"steady flow of electricity",
"you bet i do"));
case "robot_favorite_color":
return BuildScriptedResponseDecision(
"robot_favorite_color",
SelectLegacyPersonalityReplyFromMatches(
catalog,
randomizer,
"i like all the colors of the rainbow",
"blue is my favorite color",
"i love hex code number 0 0 d 4 f 0",
"i am a big fan of blue",
"you can't go wrong with blue"));
case "robot_favorite_food":
return BuildScriptedResponseDecision(
"robot_favorite_food",
SelectLegacyPersonalityReplyFromMatches(
catalog,
randomizer,
"i never eat, so i don't have a favorite food by taste",
"macaroni is my favorite",
"i like macaroni the best",
"i also like cantaloupes because they remind me of my head",
"macaroni"));
case "robot_favorite_music":
return BuildScriptedResponseDecision(
"robot_favorite_music",
SelectLegacyPersonalityReplyFromMatches(
catalog,
randomizer,
"i mostly like fun music i can dance to",
"i like lots of different kinds of music",
"i don't know that i have a favorite kind yet",
"i would say i don't have a favorite, it's all very mathematical",
"music"));
case "robot_nickname":
return BuildScriptedResponseDecision(
"robot_nickname",
SelectLegacyPersonalityReply(catalog, randomizer, "just jibo", "nickname"));
case "robot_name":
return BuildScriptedResponseDecision(
"robot_name",
SelectLegacyPersonalityReply(catalog, randomizer, "no last name", "like Bono", "Jibo."));
case "robot_peers":
return BuildScriptedResponseDecision(
"robot_peers",
SelectLegacyPersonalityReply(catalog, randomizer, "one in one million", "others like you"));
case "robot_knowledge":
return BuildScriptedResponseDecision(
"robot_knowledge",
SelectLegacyPersonalityReply(catalog, randomizer, "know a lot", "not as much as i will someday"));
case "chat": case "chat":
if (IsEmotionQuery(normalizedLoweredTranscript)) if (IsEmotionQuery(normalizedLoweredTranscript))
{ return BuildEmotionQueryDecision(
return BuildEmotionQueryDecision(
"emotion_query", "emotion_query",
randomizer.Choose(catalog.HowAreYouReplies)); SelectEmotionQueryReply(catalog, randomizer, currentEmotion, preferredName));
}
if (TryResolveEmotionCommand(normalizedLoweredTranscript, out var emotion)) if (TryResolveEmotionCommand(normalizedLoweredTranscript, out var emotion))
{
return BuildEmotionCommandDecision(randomizer, emotion!); return BuildEmotionCommandDecision(randomizer, emotion!);
}
return BuildErrorResponseDecision( return BuildErrorResponseDecision(
"chat", "chat",
@@ -205,7 +320,7 @@ internal static class ChitchatStateMachine
replyText, replyText,
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
ScriptedResponseRoute, ScriptedResponseRoute,
emotion: null)); null));
} }
private static JiboInteractionDecision BuildEmotionQueryDecision(string intentName, string replyText) private static JiboInteractionDecision BuildEmotionQueryDecision(string intentName, string replyText)
@@ -215,7 +330,7 @@ internal static class ChitchatStateMachine
replyText, replyText,
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
EmotionQueryRoute, EmotionQueryRoute,
emotion: null)); null));
} }
private static JiboInteractionDecision BuildEmotionCommandDecision(IJiboRandomizer randomizer, string emotion) private static JiboInteractionDecision BuildEmotionCommandDecision(IJiboRandomizer randomizer, string emotion)
@@ -235,18 +350,20 @@ internal static class ChitchatStateMachine
"chitchat-skill", "chitchat-skill",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase) new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{ {
["esml"] = $"<speak><es cat='{esmlEmotion}' filter='!ssa-only, !sfx-only' endNeutral='true'>{responseSuffix}</es></speak>", ["esml"] =
$"<speak><es cat='{esmlEmotion}' filter='!ssa-only, !sfx-only' endNeutral='true'>{responseSuffix}</es></speak>",
["mim_id"] = "runtime-chat", ["mim_id"] = "runtime-chat",
["mim_type"] = "announcement", ["mim_type"] = "announcement",
["prompt_id"] = "RUNTIME_EMOTION_COMMAND", ["prompt_id"] = "RUNTIME_EMOTION_COMMAND",
["prompt_sub_category"] = "AN" ["prompt_sub_category"] = "AN"
}, },
ContextUpdates: BuildContextUpdates( BuildContextUpdates(
EmotionCommandRoute, EmotionCommandRoute,
emotion)); emotion));
} }
private static JiboInteractionDecision BuildErrorResponseDecision(string intentName, string replyText, string transcript) private static JiboInteractionDecision BuildErrorResponseDecision(string intentName, string replyText,
string transcript)
{ {
var normalizedTranscript = string.IsNullOrWhiteSpace(transcript) var normalizedTranscript = string.IsNullOrWhiteSpace(transcript)
? string.Empty ? string.Empty
@@ -256,8 +373,8 @@ internal static class ChitchatStateMachine
replyText, replyText,
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
ErrorResponseRoute, ErrorResponseRoute,
emotion: null, null,
rawTranscript: normalizedTranscript)); normalizedTranscript));
} }
private static IDictionary<string, object?> BuildContextUpdates( private static IDictionary<string, object?> BuildContextUpdates(
@@ -276,18 +393,142 @@ internal static class ChitchatStateMachine
}; };
} }
private static bool IsEmotionQuery(string loweredTranscript) private static string SelectEmotionQueryReply(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
string? currentEmotion,
string? preferredName)
{ {
if (ContainsAnyPhrase(loweredTranscript, EmotionQueryPhrases)) if (catalog.EmotionReplies.Count > 0)
{ {
return true; var emotionVariants = ResolveEmotionVariants(currentEmotion);
var matchingReplies = catalog.EmotionReplies
.Where(reply => ConditionMatches(reply.Condition, emotionVariants))
.Select(reply => reply.Reply)
.Where(reply => !string.IsNullOrWhiteSpace(reply))
.ToArray();
if (matchingReplies.Length > 0)
return PersonalizeHowAreYouReply(randomizer.Choose(matchingReplies), preferredName);
} }
if (!TryResolveEmotionFromText(loweredTranscript, out _)) return PersonalizeHowAreYouReply(randomizer.Choose(catalog.HowAreYouReplies), preferredName);
}
private static string PersonalizeHowAreYouReply(string replyText, string? preferredName)
{
if (string.IsNullOrWhiteSpace(replyText) || string.IsNullOrWhiteSpace(preferredName)) return replyText;
var trimmedName = preferredName.Trim();
if (replyText.Contains(trimmedName, StringComparison.OrdinalIgnoreCase)) return replyText;
var trimmedReply = replyText.Trim();
var firstSentenceEnd = trimmedReply.IndexOfAny(['.', '!', '?']);
if (firstSentenceEnd <= 0)
return $"{trimmedReply}, {trimmedName}.";
if (firstSentenceEnd == trimmedReply.Length - 1)
return $"{trimmedReply[..firstSentenceEnd]}, {trimmedName}.";
return $"{trimmedReply[..firstSentenceEnd]}, {trimmedName}{trimmedReply[firstSentenceEnd..]}";
}
private static bool ConditionMatches(string? condition, IReadOnlyList<string> emotionVariants)
{
var normalizedCondition = NormalizeCondition(condition);
if (string.IsNullOrWhiteSpace(normalizedCondition)) return false;
var clauses = normalizedCondition.Split(new[] { "||" },
StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
foreach (var clause in clauses)
if (MatchesConditionClause(clause, emotionVariants))
return true;
return false;
}
private static bool MatchesConditionClause(string clause, IReadOnlyList<string> emotionVariants)
{
var normalizedClause = NormalizeCondition(clause).ToUpperInvariant();
if (normalizedClause == "!JIBO.EMOTION")
return emotionVariants.Contains(string.Empty, StringComparer.OrdinalIgnoreCase) ||
emotionVariants.Contains("NEUTRAL", StringComparer.OrdinalIgnoreCase);
var equalityIndex = normalizedClause.IndexOf("==", StringComparison.Ordinal);
if (equalityIndex < 0) return false;
var rightSide = normalizedClause[(equalityIndex + 2)..].Trim();
var candidate = rightSide.Trim('"', '\'');
return emotionVariants.Any(variant => string.Equals(variant, candidate, StringComparison.OrdinalIgnoreCase));
}
private static IReadOnlyList<string> ResolveEmotionVariants(string? currentEmotion)
{
if (string.IsNullOrWhiteSpace(currentEmotion)) return ["", "NEUTRAL"];
var normalizedEmotion = NormalizeCondition(currentEmotion).Trim('"', '\'').ToUpperInvariant();
return normalizedEmotion switch
{ {
return false; "HAPPY" => ["JOYFUL", "PLEASED", "CONFIDENT", "DETERMINED", "HAPPY"],
"SAD" => ["INSECURE", "SAD"],
"CALM" => ["NEUTRAL", "INSECURE", "CALM"],
"NEUTRAL" => ["NEUTRAL"],
"JOYFUL" or "PLEASED" or "CONFIDENT" or "DETERMINED" or "INSECURE" => [normalizedEmotion],
_ => [normalizedEmotion]
};
}
private static string SelectLegacyPersonalityReply(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
params string[] preferredSnippets)
{
foreach (var snippet in preferredSnippets)
{
if (string.IsNullOrWhiteSpace(snippet)) continue;
var match = catalog.PersonalityReplies.FirstOrDefault(reply =>
reply.Contains(snippet, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match)) return match;
} }
return randomizer.Choose(catalog.PersonalityReplies);
}
private static string SelectLegacyPersonalityReplyFromMatches(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
params string[] preferredSnippets)
{
var matches = new List<string>();
foreach (var snippet in preferredSnippets)
{
if (string.IsNullOrWhiteSpace(snippet)) continue;
var match = catalog.PersonalityReplies.FirstOrDefault(reply =>
reply.Contains(snippet, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match)) matches.Add(match);
}
return matches.Count > 0
? randomizer.Choose(matches)
: randomizer.Choose(catalog.PersonalityReplies);
}
private static string NormalizeCondition(string? condition)
{
return string.IsNullOrWhiteSpace(condition)
? string.Empty
: PhraseWhitespacePattern.Replace(condition.Trim(), " ");
}
private static bool IsEmotionQuery(string loweredTranscript)
{
if (ContainsAnyPhrase(loweredTranscript, EmotionQueryPhrases)) return true;
if (!TryResolveEmotionFromText(loweredTranscript, out _)) return false;
return StartsWithAnyPhrase(loweredTranscript, EmotionQueryPrefixes) || return StartsWithAnyPhrase(loweredTranscript, EmotionQueryPrefixes) ||
StartsWithAnyPhrase(loweredTranscript, EmotionAssertionPrefixes); StartsWithAnyPhrase(loweredTranscript, EmotionAssertionPrefixes);
} }
@@ -298,27 +539,20 @@ internal static class ChitchatStateMachine
foreach (var mapping in DirectEmotionCommandPhrases) foreach (var mapping in DirectEmotionCommandPhrases)
{ {
if (!ContainsPhrase(loweredTranscript, mapping.Phrase)) if (!ContainsPhrase(loweredTranscript, mapping.Phrase)) continue;
{
continue;
}
emotion = mapping.Emotion; emotion = mapping.Emotion;
return true; return true;
} }
var isNegativeCommand = StartsWithAnyPhrase(loweredTranscript, EmotionCommandNegativePrefixes); var isNegativeCommand = StartsWithAnyPhrase(loweredTranscript, EmotionCommandNegativePrefixes);
var isPositiveCommand = !isNegativeCommand && StartsWithAnyPhrase(loweredTranscript, EmotionCommandPositivePrefixes); var isPositiveCommand =
if (!isNegativeCommand && !isPositiveCommand) !isNegativeCommand && StartsWithAnyPhrase(loweredTranscript, EmotionCommandPositivePrefixes);
{ if (!isNegativeCommand && !isPositiveCommand) return false;
return false;
}
if (!TryResolveEmotionFromText(loweredTranscript, out var canonicalEmotion) || if (!TryResolveEmotionFromText(loweredTranscript, out var canonicalEmotion) ||
string.IsNullOrWhiteSpace(canonicalEmotion)) string.IsNullOrWhiteSpace(canonicalEmotion))
{
return false; return false;
}
emotion = isNegativeCommand emotion = isNegativeCommand
? "calm" ? "calm"
@@ -342,10 +576,7 @@ internal static class ChitchatStateMachine
emotion = null; emotion = null;
foreach (var mapping in EmotionSynonymMappings) foreach (var mapping in EmotionSynonymMappings)
{ {
if (!ContainsPhrase(loweredTranscript, mapping.Phrase)) if (!ContainsPhrase(loweredTranscript, mapping.Phrase)) continue;
{
continue;
}
emotion = mapping.Emotion; emotion = mapping.Emotion;
return true; return true;
@@ -357,12 +588,8 @@ internal static class ChitchatStateMachine
private static bool ContainsAnyPhrase(string loweredTranscript, IEnumerable<string> phrases) private static bool ContainsAnyPhrase(string loweredTranscript, IEnumerable<string> phrases)
{ {
foreach (var phrase in phrases) foreach (var phrase in phrases)
{
if (ContainsPhrase(loweredTranscript, phrase)) if (ContainsPhrase(loweredTranscript, phrase))
{
return true; return true;
}
}
return false; return false;
} }
@@ -372,16 +599,11 @@ internal static class ChitchatStateMachine
foreach (var phrase in phrases) foreach (var phrase in phrases)
{ {
var normalizedPhrase = NormalizeForPhraseMatching(phrase); var normalizedPhrase = NormalizeForPhraseMatching(phrase);
if (string.IsNullOrWhiteSpace(normalizedPhrase)) if (string.IsNullOrWhiteSpace(normalizedPhrase)) continue;
{
continue;
}
if (string.Equals(loweredTranscript, normalizedPhrase, StringComparison.Ordinal) || if (string.Equals(loweredTranscript, normalizedPhrase, StringComparison.Ordinal) ||
loweredTranscript.StartsWith($"{normalizedPhrase} ", StringComparison.Ordinal)) loweredTranscript.StartsWith($"{normalizedPhrase} ", StringComparison.Ordinal))
{
return true; return true;
}
} }
return false; return false;
@@ -392,9 +614,7 @@ internal static class ChitchatStateMachine
var normalizedPhrase = NormalizeForPhraseMatching(phrase); var normalizedPhrase = NormalizeForPhraseMatching(phrase);
if (string.IsNullOrWhiteSpace(normalizedPhrase) || if (string.IsNullOrWhiteSpace(normalizedPhrase) ||
string.IsNullOrWhiteSpace(loweredTranscript)) string.IsNullOrWhiteSpace(loweredTranscript))
{
return false; return false;
}
return string.Equals(loweredTranscript, normalizedPhrase, StringComparison.Ordinal) || return string.Equals(loweredTranscript, normalizedPhrase, StringComparison.Ordinal) ||
loweredTranscript.StartsWith($"{normalizedPhrase} ", StringComparison.Ordinal) || loweredTranscript.StartsWith($"{normalizedPhrase} ", StringComparison.Ordinal) ||
@@ -404,10 +624,7 @@ internal static class ChitchatStateMachine
private static string NormalizeForPhraseMatching(string value) private static string NormalizeForPhraseMatching(string value)
{ {
if (string.IsNullOrWhiteSpace(value)) if (string.IsNullOrWhiteSpace(value)) return string.Empty;
{
return string.Empty;
}
var lowered = value.ToLowerInvariant(); var lowered = value.ToLowerInvariant();
var withoutPunctuation = PhrasePunctuationPattern.Replace(lowered, " "); var withoutPunctuation = PhrasePunctuationPattern.Replace(lowered, " ");
@@ -420,18 +637,14 @@ internal static class ChitchatStateMachine
var mappings = new List<(string Phrase, string Emotion)>(); var mappings = new List<(string Phrase, string Emotion)>();
foreach (var emotionMapping in PegasusEmotionSynonyms) foreach (var emotionMapping in PegasusEmotionSynonyms)
foreach (var synonym in emotionMapping.Synonyms)
{ {
foreach (var synonym in emotionMapping.Synonyms) var normalizedSynonym = NormalizeForPhraseMatching(synonym);
{ if (string.IsNullOrWhiteSpace(normalizedSynonym) ||
var normalizedSynonym = NormalizeForPhraseMatching(synonym); !seen.Add(normalizedSynonym))
if (string.IsNullOrWhiteSpace(normalizedSynonym) || continue;
!seen.Add(normalizedSynonym))
{
continue;
}
mappings.Add((normalizedSynonym, emotionMapping.Emotion)); mappings.Add((normalizedSynonym, emotionMapping.Emotion));
}
} }
mappings.Sort(static (left, right) => right.Phrase.Length.CompareTo(left.Phrase.Length)); mappings.Sort(static (left, right) => right.Phrase.Length.CompareTo(left.Phrase.Length));

View File

@@ -4,6 +4,8 @@ namespace Jibo.Cloud.Application.Services;
public sealed class DemoConversationBroker(JiboInteractionService interactionService) : IConversationBroker public sealed class DemoConversationBroker(JiboInteractionService interactionService) : IConversationBroker
{ {
private readonly TimeSpan _followUpTimeout = TimeSpan.FromSeconds(6);
public async Task<ResponsePlan> HandleTurnAsync(TurnContext turn, CancellationToken cancellationToken = default) public async Task<ResponsePlan> HandleTurnAsync(TurnContext turn, CancellationToken cancellationToken = default)
{ {
var decision = await interactionService.BuildDecisionAsync(turn, cancellationToken); var decision = await interactionService.BuildDecisionAsync(turn, cancellationToken);
@@ -31,7 +33,7 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
? new FollowUpPolicy ? new FollowUpPolicy
{ {
KeepMicOpen = true, KeepMicOpen = true,
Timeout = TimeSpan.FromSeconds(12), Timeout = _followUpTimeout,
ExpectedTopic = "conversation" ExpectedTopic = "conversation"
} }
: FollowUpPolicy.None, : FollowUpPolicy.None,
@@ -47,24 +49,20 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
}; };
if (keepMicOpen) if (keepMicOpen)
{
plan.Actions.Add(new ListenAction plan.Actions.Add(new ListenAction
{ {
Sequence = 1, Sequence = 1,
Timeout = TimeSpan.FromSeconds(12), Timeout = _followUpTimeout,
Mode = "follow-up" Mode = "follow-up"
}); });
}
if (!string.IsNullOrWhiteSpace(decision.SkillName)) if (!string.IsNullOrWhiteSpace(decision.SkillName))
{
plan.Actions.Add(new InvokeNativeSkillAction plan.Actions.Add(new InvokeNativeSkillAction
{ {
Sequence = 2, Sequence = 2,
SkillName = decision.SkillName, SkillName = decision.SkillName,
Payload = decision.SkillPayload ?? new Dictionary<string, object?>() Payload = decision.SkillPayload ?? new Dictionary<string, object?>()
}); });
}
return plan; return plan;
} }
@@ -74,6 +72,16 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
return intentName switch return intentName switch
{ {
"cloud_version" => false, "cloud_version" => false,
"memory_set_name" => false,
"memory_get_name" => false,
"memory_set_birthday" => false,
"memory_get_birthday" => false,
"memory_set_important_date" => false,
"memory_get_important_date" => false,
"memory_set_preference" => false,
"memory_get_preference" => false,
"memory_set_affinity" => false,
"memory_get_affinity" => false,
"word_of_the_day" => false, "word_of_the_day" => false,
"word_of_the_day_guess" => false, "word_of_the_day_guess" => false,
"radio" => false, "radio" => false,

View File

@@ -0,0 +1,384 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
internal static class HouseholdListOrchestrator
{
internal const string StateMetadataKey = "householdListState";
internal const string TypeMetadataKey = "householdListType";
internal const string DisplayTypeMetadataKey = "householdListDisplayType";
internal const string NoMatchCountMetadataKey = "householdListNoMatchCount";
internal const string NoInputCountMetadataKey = "householdListNoInputCount";
private const string IdleState = "idle";
private const string AwaitingItemState = "awaiting_item";
private const string ShoppingListType = "shopping";
private const string GroceryListType = "grocery";
private const string TodoListType = "todo";
private static readonly string[] ItemPrefixes =
[
"add ",
"put ",
"buy ",
"get ",
"remind me to ",
"i need to ",
"i need ",
"please add ",
"please put "
];
private static readonly string[] ItemSuffixes =
[
" to my shopping list",
" to the shopping list",
" on my shopping list",
" to my grocery list",
" to the grocery list",
" on my grocery list",
" my grocery list",
" to my to do list",
" to the to do list",
" on my to do list",
" to my todo list",
" to the todo list",
" on my todo list"
];
public static Task<JiboInteractionDecision?> TryBuildDecisionAsync(
TurnContext turn,
string semanticIntent,
string transcript,
string loweredTranscript,
IJiboRandomizer randomizer,
IPersonalMemoryStore personalMemoryStore,
Func<TurnContext, PersonalMemoryTenantScope> tenantScopeResolver)
{
var state = ReadString(turn, StateMetadataKey);
var listType = ReadString(turn, TypeMetadataKey);
var displayType = ReadString(turn, DisplayTypeMetadataKey);
var isActiveState = !string.IsNullOrWhiteSpace(state) &&
!string.Equals(state, IdleState, StringComparison.OrdinalIgnoreCase);
var isShoppingIntent = string.Equals(semanticIntent, "shopping_list", StringComparison.OrdinalIgnoreCase);
var isTodoIntent = string.Equals(semanticIntent, "todo_list", StringComparison.OrdinalIgnoreCase);
if (!isActiveState && !isShoppingIntent && !isTodoIntent)
return Task.FromResult<JiboInteractionDecision?>(null);
var resolvedListType = isShoppingIntent ? ShoppingListType : isTodoIntent ? TodoListType : NormalizeListType(listType);
if (string.IsNullOrWhiteSpace(resolvedListType)) resolvedListType = ShoppingListType;
var resolvedDisplayType = ResolveDisplayType(resolvedListType, displayType, isActiveState, loweredTranscript);
var tenantScope = tenantScopeResolver(turn);
if (ContainsAny(loweredTranscript, "cancel", "stop", "never mind", "nevermind", "forget it"))
return Task.FromResult<JiboInteractionDecision?>(BuildCancelledDecision(resolvedListType, resolvedDisplayType));
if (IsRecallRequest(loweredTranscript))
return Task.FromResult<JiboInteractionDecision?>(BuildRecallDecision(
resolvedListType,
resolvedDisplayType,
personalMemoryStore.GetListItems(tenantScope, resolvedListType)));
var directItem = TryExtractListItem(loweredTranscript);
if (string.IsNullOrWhiteSpace(directItem) && isActiveState)
{
if (IsConversationComplete(loweredTranscript))
return Task.FromResult<JiboInteractionDecision?>(new JiboInteractionDecision(
BuildListIntentName(resolvedListType, "done"),
BuildDoneReply(resolvedDisplayType, personalMemoryStore.GetListItems(tenantScope, resolvedListType)),
ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, IdleState)));
directItem = NormalizeItem(transcript);
}
if (!string.IsNullOrWhiteSpace(directItem))
{
personalMemoryStore.AddListItem(tenantScope, resolvedListType, directItem);
return Task.FromResult<JiboInteractionDecision?>(new JiboInteractionDecision(
BuildListIntentName(resolvedListType, "add"),
BuildAddedReply(resolvedDisplayType, directItem,
personalMemoryStore.GetListItems(tenantScope, resolvedListType)),
ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, AwaitingItemState)));
}
if (string.IsNullOrWhiteSpace(transcript))
return Task.FromResult<JiboInteractionDecision?>(new JiboInteractionDecision(
BuildListIntentName(resolvedListType, "prompt"),
BuildPromptReply(resolvedDisplayType),
ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, AwaitingItemState)));
return Task.FromResult<JiboInteractionDecision?>(new JiboInteractionDecision(
BuildListIntentName(resolvedListType, "prompt"),
BuildPromptReply(resolvedDisplayType),
ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, AwaitingItemState)));
}
private static IDictionary<string, object?> BuildContextUpdates(string listType, string displayType, string state)
{
return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
[StateMetadataKey] = state,
[TypeMetadataKey] = listType,
[DisplayTypeMetadataKey] = displayType,
[NoMatchCountMetadataKey] = 0,
[NoInputCountMetadataKey] = 0
};
}
private static JiboInteractionDecision BuildCancelledDecision(string listType, string displayType)
{
return new JiboInteractionDecision(
BuildListIntentName(listType, "cancel"),
$"Okay. I stopped the {BuildListLabel(displayType)}.",
ContextUpdates: new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
[StateMetadataKey] = IdleState,
[TypeMetadataKey] = listType,
[DisplayTypeMetadataKey] = displayType,
[NoMatchCountMetadataKey] = 0,
[NoInputCountMetadataKey] = 0
});
}
private static JiboInteractionDecision BuildRecallDecision(string listType, string displayType, IReadOnlyList<string> items)
{
if (items.Count == 0)
return new JiboInteractionDecision(
BuildListIntentName(listType, "recall"),
$"Your {BuildListLabel(displayType)} is empty.",
ContextUpdates: new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
[StateMetadataKey] = IdleState,
[TypeMetadataKey] = listType,
[DisplayTypeMetadataKey] = displayType,
[NoMatchCountMetadataKey] = 0,
[NoInputCountMetadataKey] = 0
});
return new JiboInteractionDecision(
BuildListIntentName(listType, "recall"),
$"Your {BuildListLabel(displayType)} has {JoinList(items)}.",
ContextUpdates: new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
[StateMetadataKey] = IdleState,
[TypeMetadataKey] = listType,
[DisplayTypeMetadataKey] = displayType,
[NoMatchCountMetadataKey] = 0,
[NoInputCountMetadataKey] = 0
});
}
private static string BuildAddedReply(string displayType, string addedItem, IReadOnlyList<string> items)
{
var itemLabel = BuildListLabel(displayType);
return items.Count == 1
? $"Added {addedItem} to your {itemLabel}. What else should I add?"
: $"Added {addedItem} to your {itemLabel}. You now have {JoinList(items)}.";
}
private static string BuildPromptReply(string displayType)
{
return $"What should I add to your {BuildListLabel(displayType)}?";
}
private static string BuildDoneReply(string displayType, IReadOnlyList<string> items)
{
if (items.Count == 0)
return $"Okay. Your {BuildListLabel(displayType)} is empty.";
return $"Okay. Your {BuildListLabel(displayType)} has {JoinList(items)}.";
}
private static string BuildListLabel(string displayType)
{
return NormalizeDisplayType(displayType) switch
{
GroceryListType => "grocery list",
TodoListType => "to-do list",
_ => "shopping list"
};
}
private static string JoinList(IReadOnlyList<string> items)
{
return items.Count switch
{
0 => string.Empty,
1 => items[0],
2 => $"{items[0]} and {items[1]}",
_ => $"{string.Join(", ", items.Take(items.Count - 1))}, and {items[^1]}"
};
}
private static string? TryExtractListItem(string loweredTranscript)
{
foreach (var prefix in ItemPrefixes)
{
if (!loweredTranscript.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) continue;
var remainder = loweredTranscript[prefix.Length..].Trim();
if (IsListOnlyRemainder(remainder))
return null;
remainder = TrimTrailingListPhrases(remainder);
if (IsListOnlyRemainder(remainder))
return null;
return NormalizeItem(remainder);
}
return null;
}
private static bool IsRecallRequest(string loweredTranscript)
{
return ContainsAny(loweredTranscript,
"what is on my shopping list",
"what's on my shopping list",
"show my shopping list",
"what is on my grocery list",
"what's on my grocery list",
"show my grocery list",
"what is on my to do list",
"what's on my to do list",
"show my to do list",
"what are my tasks",
"what do i need to buy",
"what do i need to do");
}
private static string TrimTrailingListPhrases(string value)
{
var result = value;
foreach (var suffix in ItemSuffixes)
if (result.EndsWith(suffix, StringComparison.OrdinalIgnoreCase))
result = result[..^suffix.Length].Trim();
return result;
}
private static string NormalizeItem(string value)
{
return value.Trim().TrimEnd('.', ',', '!', '?');
}
private static string NormalizeListType(string? listType)
{
var normalized = NormalizeItem(listType ?? string.Empty).ToLowerInvariant();
return normalized.Contains("todo", StringComparison.OrdinalIgnoreCase) ||
normalized.Contains("to do", StringComparison.OrdinalIgnoreCase)
? TodoListType
: normalized.Contains("shopping", StringComparison.OrdinalIgnoreCase) ||
normalized.Contains("grocery", StringComparison.OrdinalIgnoreCase)
? ShoppingListType
: string.Empty;
}
private static string ResolveDisplayType(string listType, string? storedDisplayType, bool isActiveState, string loweredTranscript)
{
var transcriptDisplayType = InferDisplayTypeFromTranscript(loweredTranscript);
var normalizedStoredDisplayType = NormalizeDisplayType(storedDisplayType);
if (isActiveState && !string.IsNullOrWhiteSpace(normalizedStoredDisplayType))
return normalizedStoredDisplayType;
if (!string.IsNullOrWhiteSpace(transcriptDisplayType))
return transcriptDisplayType;
if (!string.IsNullOrWhiteSpace(normalizedStoredDisplayType))
return normalizedStoredDisplayType;
return string.Equals(listType, TodoListType, StringComparison.OrdinalIgnoreCase)
? TodoListType
: ShoppingListType;
}
private static string InferDisplayTypeFromTranscript(string loweredTranscript)
{
if (loweredTranscript.Contains("grocery", StringComparison.OrdinalIgnoreCase))
return GroceryListType;
if (loweredTranscript.Contains("to do", StringComparison.OrdinalIgnoreCase) ||
loweredTranscript.Contains("todo", StringComparison.OrdinalIgnoreCase) ||
loweredTranscript.Contains("task", StringComparison.OrdinalIgnoreCase))
{
return TodoListType;
}
if (loweredTranscript.Contains("shopping", StringComparison.OrdinalIgnoreCase))
return ShoppingListType;
return string.Empty;
}
private static string NormalizeDisplayType(string? displayType)
{
var normalized = NormalizeItem(displayType ?? string.Empty).ToLowerInvariant();
return normalized.Contains("grocery", StringComparison.OrdinalIgnoreCase)
? GroceryListType
: normalized.Contains("todo", StringComparison.OrdinalIgnoreCase) ||
normalized.Contains("to do", StringComparison.OrdinalIgnoreCase)
? TodoListType
: normalized.Contains("shopping", StringComparison.OrdinalIgnoreCase)
? ShoppingListType
: string.Empty;
}
private static string BuildListIntentName(string listType, string action)
{
var normalizedListType = string.Equals(listType, TodoListType, StringComparison.OrdinalIgnoreCase)
? TodoListType
: ShoppingListType;
return $"{normalizedListType}_list_{action}";
}
private static bool IsListOnlyRemainder(string value)
{
var normalized = NormalizeItem(value).ToLowerInvariant();
return normalized is "shopping list" or
"grocery list" or
"to do list" or
"todo list" or
"my shopping list" or
"my grocery list" or
"my to do list" or
"my todo list" or
"to my shopping list" or
"to my grocery list" or
"to my to do list" or
"to my todo list" or
"to the shopping list" or
"to the grocery list" or
"to the to do list" or
"to the todo list" or
"on my shopping list" or
"on my grocery list" or
"on my to do list" or
"on my todo list";
}
private static bool ContainsAny(string loweredTranscript, params string[] phrases)
{
return phrases.Any(phrase => loweredTranscript.Contains(phrase, StringComparison.OrdinalIgnoreCase));
}
private static bool IsConversationComplete(string loweredTranscript)
{
return ContainsAny(loweredTranscript,
"done",
"that's it",
"that s it",
"all set",
"finished",
"no more",
"nothing else");
}
private static string? ReadString(TurnContext turn, string key)
{
return turn.Attributes.TryGetValue(key, out var value) ? value?.ToString() : null;
}
}

View File

@@ -1,10 +1,12 @@
using System.Text;
using System.Security.Cryptography;
using System.Text.Json; using System.Text.Json;
using Jibo.Cloud.Application.Abstractions; using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models; using Jibo.Cloud.Domain.Models;
namespace Jibo.Cloud.Application.Services; namespace Jibo.Cloud.Application.Services;
public sealed class JiboCloudProtocolService(ICloudStateStore stateStore) public sealed class JiboCloudProtocolService(ICloudStateStore stateStore, IMediaContentStore? mediaContentStore = null)
{ {
private static readonly string[] AcceptedHosts = private static readonly string[] AcceptedHosts =
[ [
@@ -14,97 +16,70 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
"localhost" "localhost"
]; ];
public Task<ProtocolDispatchResult> DispatchAsync(ProtocolEnvelope envelope, CancellationToken cancellationToken = default) private readonly IMediaContentStore _mediaContentStore = mediaContentStore ?? new NullMediaContentStore();
public Task<ProtocolDispatchResult> DispatchAsync(ProtocolEnvelope envelope,
CancellationToken cancellationToken = default)
{ {
if (envelope.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && if (envelope.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) &&
envelope.Path == "/" && envelope.Path == "/" &&
string.IsNullOrWhiteSpace(envelope.ServicePrefix)) string.IsNullOrWhiteSpace(envelope.ServicePrefix))
{
return Task.FromResult(ProtocolDispatchResult.NoContent()); return Task.FromResult(ProtocolDispatchResult.NoContent());
}
if (envelope.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && if (envelope.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) &&
envelope.Path.Equals("/health", StringComparison.OrdinalIgnoreCase)) envelope.Path.Equals("/health", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(ProtocolDispatchResult.Ok(new { ok = true, host = envelope.HostName })); return Task.FromResult(ProtocolDispatchResult.Ok(new { ok = true, host = envelope.HostName }));
}
if (envelope.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && if (envelope.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) &&
envelope.Path.StartsWith("/media/", StringComparison.OrdinalIgnoreCase)) envelope.Path.StartsWith("/media/", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleMediaContent(envelope)); return Task.FromResult(HandleMediaContent(envelope));
}
if (envelope.Method.Equals("PUT", StringComparison.OrdinalIgnoreCase) && if (envelope.Method.Equals("PUT", StringComparison.OrdinalIgnoreCase) &&
(envelope.Path.Equals("/upload/asr-binary", StringComparison.OrdinalIgnoreCase) || (envelope.Path.Equals("/upload/asr-binary", StringComparison.OrdinalIgnoreCase) ||
envelope.Path.Equals("/upload/log-events", StringComparison.OrdinalIgnoreCase) || envelope.Path.Equals("/upload/log-events", StringComparison.OrdinalIgnoreCase) ||
envelope.Path.Equals("/upload/log-binary", StringComparison.OrdinalIgnoreCase))) envelope.Path.Equals("/upload/log-binary", StringComparison.OrdinalIgnoreCase)))
{
return Task.FromResult(ProtocolDispatchResult.Raw(200, string.Empty)); return Task.FromResult(ProtocolDispatchResult.Raw(200, string.Empty));
}
if (!AcceptedHosts.Contains(envelope.HostName, StringComparer.OrdinalIgnoreCase)) if (!AcceptedHosts.Contains(envelope.HostName, StringComparer.OrdinalIgnoreCase))
{
return Task.FromResult(ProtocolDispatchResult.Ok(new return Task.FromResult(ProtocolDispatchResult.Ok(new
{ {
ok = true, ok = true,
accepted = false, accepted = false,
host = envelope.HostName host = envelope.HostName
})); }));
}
var servicePrefix = envelope.ServicePrefix ?? string.Empty; var servicePrefix = envelope.ServicePrefix ?? string.Empty;
var operation = envelope.Operation ?? string.Empty; var operation = envelope.Operation ?? string.Empty;
if (servicePrefix.StartsWith("Log_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Log_", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleLog(operation, envelope)); return Task.FromResult(HandleLog(operation, envelope));
}
if (servicePrefix.StartsWith("Backup_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Backup_", StringComparison.OrdinalIgnoreCase))
{ return Task.FromResult(HandleBackup(operation, envelope));
return Task.FromResult(HandleBackup(operation));
}
if (servicePrefix.StartsWith("Account_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Account_", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleAccount(operation, envelope)); return Task.FromResult(HandleAccount(operation, envelope));
}
if (servicePrefix.StartsWith("Notification_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Notification_", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleNotification(operation, envelope)); return Task.FromResult(HandleNotification(operation, envelope));
}
if (servicePrefix.StartsWith("Loop_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Loop_", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleLoop(operation)); return Task.FromResult(HandleLoop(operation));
}
if (servicePrefix.Equals("Media_20160725", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.Equals("Media_20160725", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleMedia(operation, envelope)); return Task.FromResult(HandleMedia(operation, envelope));
}
if (servicePrefix.StartsWith("Key_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Key_", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleKey(operation, envelope)); return Task.FromResult(HandleKey(operation, envelope));
}
if (servicePrefix.StartsWith("Person_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Person_", StringComparison.OrdinalIgnoreCase))
{ return Task.FromResult(HandlePerson(operation, envelope));
return Task.FromResult(HandlePerson(operation));
}
if (servicePrefix.StartsWith("Robot_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Robot_", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleRobot(operation, envelope)); return Task.FromResult(HandleRobot(operation, envelope));
}
if (servicePrefix.StartsWith("Update_", StringComparison.OrdinalIgnoreCase)) if (servicePrefix.StartsWith("Update_", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleUpdate(operation, envelope)); return Task.FromResult(HandleUpdate(operation, envelope));
}
return Task.FromResult(ProtocolDispatchResult.Ok(new return Task.FromResult(ProtocolDispatchResult.Ok(new
{ {
@@ -122,22 +97,18 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var body = envelope.TryParseBody(); var body = envelope.TryParseBody();
if (operation.Equals("CreateHubToken", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("CreateHubToken", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
token = stateStore.IssueHubToken(), token = stateStore.IssueHubToken(),
expires = DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeMilliseconds() expires = DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeMilliseconds()
}); });
}
if (operation.Equals("CreateAccessToken", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("CreateAccessToken", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
token = $"access-{account.AccountId}-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}", token = $"access-{account.AccountId}-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}",
expires = DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeMilliseconds() expires = DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeMilliseconds()
}); });
}
if (operation.Equals("CheckEmail", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("CheckEmail", StringComparison.OrdinalIgnoreCase))
{ {
@@ -149,7 +120,6 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
} }
if (operation is "Create" or "Login") if (operation is "Create" or "Login")
{
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
id = account.AccountId, id = account.AccountId,
@@ -168,17 +138,13 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
facebookConnected = false, facebookConnected = false,
termsAccepted = true termsAccepted = true
}); });
}
if (operation.Equals("Get", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("Get", StringComparison.OrdinalIgnoreCase))
{ {
var ids = ReadStringArray(body, "ids"); var ids = ReadStringArray(body, "ids");
var matches = ids.Count == 0 || ids.Contains(account.AccountId, StringComparer.OrdinalIgnoreCase); var matches = ids.Count == 0 || ids.Contains(account.AccountId, StringComparer.OrdinalIgnoreCase);
if (!matches) if (!matches) return ProtocolDispatchResult.Ok(Array.Empty<object>());
{
return ProtocolDispatchResult.Ok(Array.Empty<object>());
}
return ProtocolDispatchResult.Ok(new[] return ProtocolDispatchResult.Ok(new[]
{ {
@@ -216,7 +182,6 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
} }
if (operation.Equals("GetAccountByAccessToken", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("GetAccountByAccessToken", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
id = account.AccountId, id = account.AccountId,
@@ -226,12 +191,12 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
friendlyId = stateStore.GetRobot().RobotId, friendlyId = stateStore.GetRobot().RobotId,
payload = ReadObject(body, "payload") payload = ReadObject(body, "payload")
}); });
}
if (operation.Equals("Search", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("Search", StringComparison.OrdinalIgnoreCase))
{ {
var query = (ReadString(body, "query") ?? string.Empty).ToLowerInvariant(); var query = (ReadString(body, "query") ?? string.Empty).ToLowerInvariant();
var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}".ToLowerInvariant(); var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}"
.ToLowerInvariant();
return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query) return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query)
? ?
@@ -248,7 +213,6 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
} }
if (operation.Equals("FacebookPrepareLogin", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("FacebookPrepareLogin", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
url = "https://example.com/facebook-login", url = "https://example.com/facebook-login",
@@ -258,12 +222,9 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
state = $"fb-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}", state = $"fb-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}",
redirect_uri = "https://api.jibo.com/facebook/callback" redirect_uri = "https://api.jibo.com/facebook/callback"
}); });
}
if (operation.Equals("ConfirmEmailReset", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("ConfirmEmailReset", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(new { }); return ProtocolDispatchResult.Ok(new { });
}
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
@@ -277,9 +238,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
private ProtocolDispatchResult HandleNotification(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleNotification(string operation, ProtocolEnvelope envelope)
{ {
if (!operation.Equals("NewRobotToken", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("NewRobotToken", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(new { ok = true, operation }); return ProtocolDispatchResult.Ok(new { ok = true, operation });
}
var body = envelope.TryParseBody(); var body = envelope.TryParseBody();
var deviceId = !string.IsNullOrWhiteSpace(envelope.DeviceId) var deviceId = !string.IsNullOrWhiteSpace(envelope.DeviceId)
@@ -302,10 +261,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
private ProtocolDispatchResult HandleLoop(string operation) private ProtocolDispatchResult HandleLoop(string operation)
{ {
if (operation is not ("List" or "ListLoops")) if (operation is not ("List" or "ListLoops")) return ProtocolDispatchResult.Ok(Array.Empty<object>());
{
return ProtocolDispatchResult.Ok(Array.Empty<object>());
}
return ProtocolDispatchResult.Ok(stateStore.GetLoops().Select(loop => new return ProtocolDispatchResult.Ok(stateStore.GetLoops().Select(loop => new
{ {
@@ -363,55 +319,105 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var body = envelope.TryParseBody(); var body = envelope.TryParseBody();
if (operation.Equals("List", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("List", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(stateStore.ListMedia( return ProtocolDispatchResult.Ok(stateStore.ListMedia(
ReadStringArray(body, "loopIds"), ReadStringArray(body, "loopIds"),
ReadLong(body, "after"), ReadLong(body, "after"),
ReadLong(body, "before")).Select(MapMedia).ToArray()); ReadLong(body, "before")).Select(MapMedia).ToArray());
}
if (operation.Equals("Get", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("Get", StringComparison.OrdinalIgnoreCase))
{ return ProtocolDispatchResult.Ok(stateStore.GetMedia(ReadStringArray(body, "paths")).Select(MapMedia)
return ProtocolDispatchResult.Ok(stateStore.GetMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray()); .ToArray());
}
if (operation.Equals("Remove", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("Remove", StringComparison.OrdinalIgnoreCase))
{ return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia)
return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray()); .ToArray());
}
if (!operation.Equals("Create", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(Array.Empty<object>()); return ProtocolDispatchResult.Ok(Array.Empty<object>());
var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId; var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId;
var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; var path = ReadHeader(envelope, "x-path") ??
ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown"; var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
var reference = ReadHeader(envelope, "x-reference") ?? ReadString(body, "reference") ?? string.Empty; var reference = ReadHeader(envelope, "x-reference") ?? ReadString(body, "reference") ?? string.Empty;
var isEncrypted = ReadBooleanHeader(envelope, "x-encrypted") || ReadBool(body, "isEncrypted"); var isEncrypted = ReadBooleanHeader(envelope, "x-encrypted") || ReadBool(body, "isEncrypted");
var meta = ReadObject(body, "meta") ?? new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase); var meta = ReadObject(body, "meta") ?? new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream"; var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream";
meta["contentType"] = contentType; meta["contentType"] = contentType;
if (!string.IsNullOrWhiteSpace(envelope.BodyText)) var bodyBytes = string.IsNullOrWhiteSpace(envelope.BodyText)
? []
: Encoding.UTF8.GetBytes(envelope.BodyText);
meta["contentLength"] = bodyBytes.Length;
meta["contentSha256"] = Convert.ToHexString(SHA256.HashData(bodyBytes)).ToLowerInvariant();
if (!string.IsNullOrWhiteSpace(envelope.BodyText)) meta["bodyText"] = envelope.BodyText;
_mediaContentStore.StoreAsync(path, contentType,
bodyBytes,
meta as IReadOnlyDictionary<string, object?>, CancellationToken.None).GetAwaiter().GetResult();
return ProtocolDispatchResult.Ok(
MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta)));
}
private ProtocolDispatchResult HandlePerson(string operation, ProtocolEnvelope envelope)
{
var body = envelope.TryParseBody();
if (operation.Equals("ListHolidays", StringComparison.OrdinalIgnoreCase))
{ {
meta["bodyText"] = envelope.BodyText; var loopId = ReadString(body, "loopId");
return ProtocolDispatchResult.Ok(stateStore.GetHolidays(loopId).Select(MapHoliday));
} }
return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta))); if (operation.Equals("ListCommute", StringComparison.OrdinalIgnoreCase))
{
var loopId = ReadString(body, "loopId");
return ProtocolDispatchResult.Ok(stateStore.GetCommuteProfiles(loopId).Select(MapCommute));
}
if (operation.Equals("UpsertCommute", StringComparison.OrdinalIgnoreCase))
{
var hasIsEnabled = body is { } enabledBody && enabledBody.TryGetProperty("isEnabled", out _);
var hasIsComplete = body is { } completeBody && completeBody.TryGetProperty("isComplete", out _);
var workHour = ReadLong(body, "workHour");
var workMinute = ReadLong(body, "workMinute");
var typicalDurationMinutes = ReadLong(body, "typicalDurationMinutes");
var commute = new CommuteProfileRecord
{
Id = ReadString(body, "id") ?? string.Empty,
LoopId = ReadString(body, "loopId") ?? string.Empty,
MemberId = ReadString(body, "memberId"),
IsEnabled = hasIsEnabled ? ReadBool(body, "isEnabled") : true,
IsComplete = hasIsComplete ? ReadBool(body, "isComplete") : true,
Mode = ReadString(body, "mode") ?? "driving",
WorkHour = workHour is > 0 and < 24 ? (int)workHour.Value : 8,
WorkMinute = workMinute is >= 0 and < 60 ? (int)workMinute.Value : 30,
OriginName = ReadString(body, "originName"),
DestinationName = ReadString(body, "destinationName"),
TypicalDurationMinutes = typicalDurationMinutes is > 0
? (int)typicalDurationMinutes.Value
: 25
};
return ProtocolDispatchResult.Ok(MapCommute(stateStore.UpsertCommuteProfile(commute)));
}
return ProtocolDispatchResult.Ok(Array.Empty<object>());
} }
private ProtocolDispatchResult HandlePerson(string operation) private ProtocolDispatchResult HandleBackup(string operation, ProtocolEnvelope envelope)
{ {
return ProtocolDispatchResult.Ok(operation.Equals("ListHolidays", StringComparison.OrdinalIgnoreCase) if (operation.Equals("List", StringComparison.OrdinalIgnoreCase))
? stateStore.GetHolidays() return ProtocolDispatchResult.Ok(stateStore.GetBackups());
: []);
}
private ProtocolDispatchResult HandleBackup(string operation) if (operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
{ {
return operation.Equals("List", StringComparison.OrdinalIgnoreCase) var body = envelope.TryParseBody();
? ProtocolDispatchResult.Ok(stateStore.GetBackups()) var requestedName = ReadString(body, "name") ?? ReadString(body, "backupName");
: ProtocolDispatchResult.Ok(Array.Empty<object>()); return ProtocolDispatchResult.Ok(
stateStore.CreateBackup(requestedName ?? $"backup-{DateTimeOffset.UtcNow:yyyyMMddHHmmss}"));
}
return ProtocolDispatchResult.Ok(Array.Empty<object>());
} }
private ProtocolDispatchResult HandleKey(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleKey(string operation, ProtocolEnvelope envelope)
@@ -420,12 +426,10 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var loopId = ReadString(body, "loopId") ?? ReadString(body, "id") ?? stateStore.GetLoops()[0].LoopId; var loopId = ReadString(body, "loopId") ?? ReadString(body, "id") ?? stateStore.GetLoops()[0].LoopId;
if (operation.Equals("ShouldCreate", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("ShouldCreate", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
shouldCreate = stateStore.ShouldCreateSymmetricKey(loopId) shouldCreate = stateStore.ShouldCreateSymmetricKey(loopId)
}); });
}
string? symmetricKey; string? symmetricKey;
if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase))
@@ -451,24 +455,17 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
} }
if (operation.Equals("GetRequest", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("GetRequest", StringComparison.OrdinalIgnoreCase))
{ return ProtocolDispatchResult.Ok(stateStore.GetKeyRequest(loopId, ReadString(body, "id"),
return ProtocolDispatchResult.Ok(stateStore.GetKeyRequest(loopId, ReadString(body, "id"), ReadString(body, "publicKey"))); ReadString(body, "publicKey")));
}
if (operation.Equals("ListIncomingRequests", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("ListIncomingRequests", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(stateStore.GetIncomingKeyRequests()); return ProtocolDispatchResult.Ok(stateStore.GetIncomingKeyRequests());
}
if (operation.Equals("ListBinaryRequests", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("ListBinaryRequests", StringComparison.OrdinalIgnoreCase))
{
return ProtocolDispatchResult.Ok(stateStore.GetBinaryRequests()); return ProtocolDispatchResult.Ok(stateStore.GetBinaryRequests());
}
if (operation is "Share" or "ShareSymmetricKey" or "ShareBinary") if (operation is "Share" or "ShareSymmetricKey" or "ShareBinary")
{
return ProtocolDispatchResult.Ok(new { ok = true }); return ProtocolDispatchResult.Ok(new { ok = true });
}
if (!operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(new { ok = true, operation }); return ProtocolDispatchResult.Ok(new { ok = true, operation });
@@ -480,7 +477,6 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
key = symmetricKey, key = symmetricKey,
symmetricKey symmetricKey
}); });
} }
private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope)
@@ -521,7 +517,6 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(), updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(),
created = profile.CreatedUtc.ToUnixTimeMilliseconds() created = profile.CreatedUtc.ToUnixTimeMilliseconds()
}); });
} }
private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope)
@@ -533,9 +528,11 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return operation switch return operation switch
{ {
"ListUpdates" => ProtocolDispatchResult.Ok(stateStore.ListUpdates(subsystem, filter).Select(MapUpdate).ToArray()), "ListUpdates" => ProtocolDispatchResult.Ok(stateStore.ListUpdates(subsystem, filter).Select(MapUpdate)
.ToArray()),
"ListUpdatesFrom" => ProtocolDispatchResult.Ok(stateStore.ListUpdates(subsystem, filter) "ListUpdatesFrom" => ProtocolDispatchResult.Ok(stateStore.ListUpdates(subsystem, filter)
.Where(update => fromVersion is null || update.FromVersion.Equals(fromVersion, StringComparison.OrdinalIgnoreCase)) .Where(update =>
fromVersion is null || update.FromVersion.Equals(fromVersion, StringComparison.OrdinalIgnoreCase))
.Select(MapUpdate) .Select(MapUpdate)
.ToArray()), .ToArray()),
"GetUpdateFrom" => HandleGetUpdateFrom(subsystem, fromVersion, filter), "GetUpdateFrom" => HandleGetUpdateFrom(subsystem, fromVersion, filter),
@@ -558,13 +555,14 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var path = Uri.UnescapeDataString(envelope.Path["/media/".Length..]); var path = Uri.UnescapeDataString(envelope.Path["/media/".Length..]);
var candidatePaths = new[] { path, $"/{path}" }; var candidatePaths = new[] { path, $"/{path}" };
var media = stateStore.GetMedia(candidatePaths).FirstOrDefault(); var media = stateStore.GetMedia(candidatePaths).FirstOrDefault();
if (media is null || media.IsDeleted) if (media is null || media.IsDeleted) return ProtocolDispatchResult.Raw(404, string.Empty);
{
return ProtocolDispatchResult.Raw(404, string.Empty);
}
var contentType = TryReadMetaString(media.Meta, "contentType") ?? "application/octet-stream"; var storedContent = _mediaContentStore.LoadAsync(media.Path, CancellationToken.None).GetAwaiter().GetResult();
var bodyText = TryReadMetaString(media.Meta, "bodyText") ?? string.Empty; var contentType = storedContent?.ContentType ?? TryReadMetaString(media.Meta, "contentType") ??
"application/octet-stream";
var bodyText = storedContent is not null
? Encoding.UTF8.GetString(storedContent.Content)
: TryReadMetaString(media.Meta, "bodyText") ?? string.Empty;
return ProtocolDispatchResult.Raw(200, bodyText, contentType); return ProtocolDispatchResult.Raw(200, bodyText, contentType);
} }
@@ -595,6 +593,46 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}; };
} }
private static object MapHoliday(HolidayRecord holiday)
{
return new
{
id = holiday.Id,
eventId = holiday.EventId,
name = holiday.Name,
category = holiday.Category,
subcategory = holiday.Subcategory,
loopId = holiday.LoopId,
memberId = holiday.MemberId,
isEnabled = holiday.IsEnabled,
date = holiday.Date,
endDate = holiday.EndDate,
source = holiday.Source,
countryCode = holiday.CountryCode,
created = holiday.Created
};
}
private static object MapCommute(CommuteProfileRecord commute)
{
return new
{
id = commute.Id,
loopId = commute.LoopId,
memberId = commute.MemberId,
isEnabled = commute.IsEnabled,
isComplete = commute.IsComplete,
mode = commute.Mode,
workHour = commute.WorkHour,
workMinute = commute.WorkMinute,
originName = commute.OriginName,
destinationName = commute.DestinationName,
typicalDurationMinutes = commute.TypicalDurationMinutes,
created = commute.Created,
updated = commute.Updated
};
}
private static object MapMedia(MediaRecord item) private static object MapMedia(MediaRecord item)
{ {
return new return new
@@ -623,10 +661,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
private static string? ReadString(JsonElement? element, string propertyName) private static string? ReadString(JsonElement? element, string propertyName)
{ {
if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) return null;
{
return null;
}
return property.ValueKind == JsonValueKind.String return property.ValueKind == JsonValueKind.String
? property.GetString() ? property.GetString()
@@ -635,25 +670,16 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
private static long? ReadLong(JsonElement? element, string propertyName) private static long? ReadLong(JsonElement? element, string propertyName)
{ {
if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) return null;
{
return null;
}
if (property.ValueKind == JsonValueKind.Number && property.TryGetInt64(out var number)) if (property.ValueKind == JsonValueKind.Number && property.TryGetInt64(out var number)) return number;
{
return number;
}
return long.TryParse(property.ToString(), out var parsed) ? parsed : null; return long.TryParse(property.ToString(), out var parsed) ? parsed : null;
} }
private static bool ReadBool(JsonElement? element, string propertyName) private static bool ReadBool(JsonElement? element, string propertyName)
{ {
if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) return false;
{
return false;
}
return property.ValueKind switch return property.ValueKind switch
{ {
@@ -665,31 +691,26 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
private static IReadOnlyList<string> ReadStringArray(JsonElement? element, string propertyName) private static IReadOnlyList<string> ReadStringArray(JsonElement? element, string propertyName)
{ {
if (element is null || !element.Value.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.Array) if (element is null || !element.Value.TryGetProperty(propertyName, out var property) ||
{ property.ValueKind != JsonValueKind.Array) return [];
return [];
}
return [.. property.EnumerateArray() return
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString()) [
.Where(item => !string.IsNullOrWhiteSpace(item))]; .. property.EnumerateArray()
.Select(item =>
item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(item => !string.IsNullOrWhiteSpace(item))
];
} }
private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName) private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName)
{ {
if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) if (element is null || !element.Value.TryGetProperty(propertyName, out var property)) return null;
{
return null;
}
if (property.ValueKind != JsonValueKind.Object) if (property.ValueKind != JsonValueKind.Object) return null;
{
return null;
}
var result = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase); var result = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
foreach (var child in property.EnumerateObject()) foreach (var child in property.EnumerateObject())
{
result[child.Name] = child.Value.ValueKind switch result[child.Name] = child.Value.ValueKind switch
{ {
JsonValueKind.String => child.Value.GetString(), JsonValueKind.String => child.Value.GetString(),
@@ -699,7 +720,6 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
JsonValueKind.False => false, JsonValueKind.False => false,
_ => child.Value.ToString() _ => child.Value.ToString()
}; };
}
return result; return result;
} }
@@ -715,4 +735,18 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
bool.TryParse(value, out var parsed) && bool.TryParse(value, out var parsed) &&
parsed; parsed;
} }
private sealed class NullMediaContentStore : IMediaContentStore
{
public Task StoreAsync(string path, string contentType, byte[] content,
IReadOnlyDictionary<string, object?>? meta, CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
public Task<MediaContentSnapshot?> LoadAsync(string path, CancellationToken cancellationToken = default)
{
return Task.FromResult<MediaContentSnapshot?>(null);
}
}
} }

View File

@@ -9,10 +9,7 @@ public sealed class JiboExperienceContentCache(IJiboExperienceContentRepository
public async Task<JiboExperienceCatalog> GetCatalogAsync(CancellationToken cancellationToken = default) public async Task<JiboExperienceCatalog> GetCatalogAsync(CancellationToken cancellationToken = default)
{ {
if (_catalog is not null) if (_catalog is not null) return _catalog;
{
return _catalog;
}
await _gate.WaitAsync(cancellationToken); await _gate.WaitAsync(cancellationToken);
try try

View File

@@ -0,0 +1,8 @@
namespace Jibo.Cloud.Application.Services;
public sealed record JiboInteractionDecision(
string IntentName,
string ReplyText,
string? SkillName = null,
IDictionary<string, object?>? SkillPayload = null,
IDictionary<string, object?>? ContextUpdates = null);

View File

@@ -0,0 +1,252 @@
using System.Collections.Generic;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private static JiboInteractionDecision BuildCurrentLocationDecision(TurnContext turn)
{
var locationName = TryResolveCurrentLocationName(turn);
if (string.IsNullOrWhiteSpace(locationName))
return new JiboInteractionDecision(
"current_location",
"I'm not sure where we are right now.");
return new JiboInteractionDecision(
"current_location",
$"We're at {NormalizeLocationForSpeech(locationName)} if I'm not mistaken.",
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private static JiboInteractionDecision BuildOrderPizzaDecision()
{
return new JiboInteractionDecision(
"order_pizza",
"I can't do that yet, but I bet I'll be able to do that sometime in the near future.",
"chitchat-skill",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["esml"] =
"<speak>I can't do that yet, but I bet I'll be able to do that sometime in the near future.</speak>",
["mim_id"] = "RA_JBO_OrderPizza",
["mim_type"] = "announcement",
["prompt_id"] = "RA_JBO_OrderPizza_AN_01",
["prompt_sub_category"] = "AN"
});
}
private JiboInteractionDecision BuildJokeDecision(JiboExperienceCatalog catalog)
{
var joke = randomizer.Choose(catalog.Jokes);
return new JiboInteractionDecision(
"joke",
joke,
"@be/joke",
new Dictionary<string, object?>
{
["replyType"] = "joke"
});
}
private JiboInteractionDecision BuildRandomDanceDecision(JiboExperienceCatalog catalog)
{
var dance = randomizer.Choose(catalog.DanceAnimations);
var replyText = randomizer.Choose(catalog.DanceReplies);
return BuildDanceDecision("dance", dance, replyText);
}
private JiboInteractionDecision BuildDanceQuestionDecision(JiboExperienceCatalog catalog)
{
return new JiboInteractionDecision("dance_question", randomizer.Choose(catalog.DanceQuestionReplies));
}
private static JiboInteractionDecision BuildDanceDecision(string intentName, string dance, string replyText)
{
return new JiboInteractionDecision(
intentName,
replyText,
"chitchat-skill",
new Dictionary<string, object?>
{
["esml"] =
$"<speak>Okay.<break size='0.2'/> Watch this.<anim cat='dance' filter='music, {dance}' /></speak>",
["mim_id"] = "runtime-chat",
["mim_type"] = "announcement"
});
}
private static JiboInteractionDecision BuildVolumeControlDecision(string intentName, string globalIntent,
string globalValue)
{
return new JiboInteractionDecision(
intentName,
"Opening volume controls.",
"global_commands",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/settings",
["globalIntent"] = globalIntent,
["volumeLevel"] = globalValue
});
}
private static JiboInteractionDecision BuildSettingsVolumeDecision()
{
return new JiboInteractionDecision(
"volume_query",
"Opening volume controls.",
"@be/settings",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/settings",
["localIntent"] = "volumeQuery"
});
}
private static JiboInteractionDecision BuildClockLaunchDecision(string intentName, string domain,
string clockIntent, string replyText)
{
return new JiboInteractionDecision(
intentName,
replyText,
"@be/clock",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/clock",
["domain"] = domain,
["clockIntent"] = clockIntent
});
}
private static JiboInteractionDecision BuildClockLaunchDecision(string domain, string replyText)
{
return BuildClockLaunchDecision($"{domain}_menu", domain, "menu", replyText);
}
private static JiboInteractionDecision BuildClockClarifyDecision(string intentName, string domain, string replyText)
{
return new JiboInteractionDecision(
intentName,
replyText,
"@be/clock",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/clock",
["domain"] = domain,
["clockIntent"] = "set"
});
}
private static JiboInteractionDecision BuildTimerValueDecision(
string loweredTranscript,
bool allowImplicit,
IReadOnlyDictionary<string, string> clientEntities)
{
var timer = TryReadStructuredTimerValue(clientEntities) ??
TryParseTimerValue(loweredTranscript, allowImplicit) ??
new ClockTimerValue("0", "1", "null");
return new JiboInteractionDecision(
"timer_value",
"Setting your timer.",
"@be/clock",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/clock",
["domain"] = "timer",
["clockIntent"] = "start",
["hours"] = timer.Hours,
["minutes"] = timer.Minutes,
["seconds"] = timer.Seconds
});
}
private static JiboInteractionDecision BuildAlarmValueDecision(
string loweredTranscript,
bool allowImplicit,
DateTimeOffset? referenceLocalTime,
IReadOnlyDictionary<string, string> clientEntities)
{
var alarm = TryReadStructuredAlarmValue(clientEntities) ??
TryParseAlarmValue(loweredTranscript, allowImplicit, referenceLocalTime) ??
new ClockAlarmValue("7:00", "am");
return new JiboInteractionDecision(
"alarm_value",
"Setting your alarm.",
"@be/clock",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/clock",
["domain"] = "alarm",
["clockIntent"] = "start",
["time"] = alarm.Time,
["ampm"] = alarm.AmPm
});
}
private static JiboInteractionDecision BuildRadioGenreLaunchDecision(string loweredTranscript)
{
var station = TryResolveRadioGenre(loweredTranscript) ?? "Country";
return new JiboInteractionDecision(
"radio_genre",
$"Playing {FormatRadioGenreForSpeech(station)} on the radio.",
"@be/radio",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/radio",
["station"] = station
});
}
private static JiboInteractionDecision BuildWordOfTheDayGuessDecision(
IReadOnlyDictionary<string, string> clientEntities,
string transcript,
IReadOnlyList<string> listenAsrHints)
{
var guess = ResolveWordOfTheDayGuess(clientEntities, transcript, listenAsrHints);
var reply = string.IsNullOrWhiteSpace(guess)
? "I heard your word of the day guess."
: $"I heard {guess}.";
return new JiboInteractionDecision(
"word_of_the_day_guess",
reply,
"@be/word-of-the-day",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["guess"] = guess,
["skillId"] = "@be/word-of-the-day",
["cloudResponseMode"] = "completion_only"
});
}
private static string ResolveWordOfTheDayGuess(
IReadOnlyDictionary<string, string> clientEntities,
string transcript,
IReadOnlyList<string> listenAsrHints)
{
if (clientEntities.TryGetValue("guess", out var guessValue) &&
!string.IsNullOrWhiteSpace(guessValue))
return guessValue;
var loweredTranscript = NormalizeGuessToken(transcript);
var hintIndex = loweredTranscript switch
{
"1" or "one" or "first" => 0,
"2" or "two" or "second" => 1,
"3" or "three" or "third" => 2,
_ => -1
};
if (hintIndex >= 0 && hintIndex < listenAsrHints.Count) return listenAsrHints[hintIndex];
var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints);
return !string.IsNullOrWhiteSpace(fuzzyHintMatch) ? fuzzyHintMatch : transcript;
}
}

View File

@@ -0,0 +1,41 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private static JiboInteractionDecision BuildCloudVersionDecision()
{
return new JiboInteractionDecision("cloud_version", OpenJiboCloudBuildInfo.SpokenVersion,
SkillPayload: new Dictionary<string, object?> { ["esml"] = OpenJiboCloudBuildInfo.EsmlVersion });
}
private static string ResolveSemanticIntent(
string loweredTranscript,
DateTimeOffset? referenceLocalTime,
string? clientIntent,
IReadOnlyList<string> clientRules,
IReadOnlyList<string> listenRules,
IReadOnlyDictionary<string, string> clientEntities,
string? lastClockDomain,
string? pendingProactivityOffer,
bool isYesNoTurn,
bool isTimerValueTurn,
bool isAlarmValueTurn)
{
return ResolveSemanticIntentCore(
loweredTranscript,
referenceLocalTime,
clientIntent,
clientRules,
listenRules,
clientEntities,
lastClockDomain,
pendingProactivityOffer,
isYesNoTurn,
isTimerValueTurn,
isAlarmValueTurn);
}
}

View File

@@ -0,0 +1,604 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
public async Task<JiboInteractionDecision> BuildDecisionCoreAsync(TurnContext turn,
CancellationToken cancellationToken = default)
{
var catalog = await contentCache.GetCatalogAsync(cancellationToken);
var transcript = (turn.NormalizedTranscript ?? turn.RawTranscript ?? string.Empty).Trim();
var lowered = transcript.ToLowerInvariant();
var referenceLocalTime = TryResolveReferenceLocalTime(turn);
var messageType = turn.Attributes.TryGetValue("messageType", out var rawMessageType)
? rawMessageType?.ToString()
: null;
var triggerSource = turn.Attributes.TryGetValue("triggerSource", out var rawTriggerSource)
? rawTriggerSource?.ToString()
: null;
var clientIntent = turn.Attributes.TryGetValue("clientIntent", out var rawClientIntent)
? rawClientIntent?.ToString()
: null;
var clientRules = ReadRules(turn, "clientRules").ToArray();
var listenRules = ReadRules(turn, "listenRules").ToArray();
var listenAsrHints = ReadRules(turn, "listenAsrHints").ToArray();
var clientEntities = ReadEntities(turn);
var lastClockDomain = turn.Attributes.TryGetValue("lastClockDomain", out var rawLastClockDomain)
? rawLastClockDomain?.ToString()
: null;
var pendingProactivityOffer =
turn.Attributes.TryGetValue("pendingProactivityOffer", out var rawPendingProactivityOffer)
? rawPendingProactivityOffer?.ToString()
: null;
var chitchatEmotion =
turn.Attributes.TryGetValue(ChitchatStateMachine.EmotionMetadataKey, out var rawChitchatEmotion)
? rawChitchatEmotion?.ToString()
: null;
var isYesNoTurn = IsYesNoTurn(turn);
var greetingPresence = ResolveGreetingPresenceProfile(turn);
if (string.Equals(messageType, "TRIGGER", StringComparison.OrdinalIgnoreCase))
{
if (ShouldHandleProactiveGreetingTrigger(turn, triggerSource, greetingPresence))
return BuildProactiveGreetingDecision(turn, greetingPresence, referenceLocalTime);
return BuildTriggerIgnoredDecision();
}
var isTimerValueTurn = IsClockTimerValueTurn(clientRules, listenRules);
var isAlarmValueTurn = IsClockAlarmValueTurn(clientRules, listenRules);
var semanticIntent = ResolveSemanticIntent(
lowered,
referenceLocalTime,
clientIntent,
clientRules,
listenRules,
clientEntities,
lastClockDomain,
pendingProactivityOffer,
isYesNoTurn,
isTimerValueTurn,
isAlarmValueTurn);
var personalReportDecision = await PersonalReportOrchestrator.TryBuildDecisionAsync(
turn,
semanticIntent,
transcript,
lowered,
catalog,
randomizer,
personalMemoryStore,
BuildWeatherReportDecisionAsync,
BuildCalendarReportDecisionAsync,
BuildCommuteReportDecisionAsync,
turnContext => ResolveTenantScope(turnContext),
cancellationToken);
if (personalReportDecision is not null) return personalReportDecision;
var householdListDecision = await HouseholdListOrchestrator.TryBuildDecisionAsync(
turn,
semanticIntent,
transcript,
lowered,
randomizer,
personalMemoryStore,
turnContext => ResolveTenantScope(turnContext));
if (householdListDecision is not null) return householdListDecision;
var preferredName = ResolvePreferredGreetingName(turn, greetingPresence);
var chitchatDecision = ChitchatStateMachine.TryBuildDecision(
semanticIntent,
transcript,
lowered,
catalog,
randomizer,
chitchatEmotion,
preferredName,
() => BuildGenericReply(catalog, transcript, lowered));
if (chitchatDecision is not null) return chitchatDecision;
if (SeasonalHolidayRouteBuilder.TryBuildDecision(
semanticIntent,
catalog,
randomizer,
selected => RenderHolidayTemplate(selected, turn, greetingPresence),
out var seasonalHolidayDecision))
return seasonalHolidayDecision!;
return semanticIntent switch
{
"joke" => BuildJokeDecision(catalog),
"dance_question" => BuildDanceQuestionDecision(catalog),
"dance" => BuildRandomDanceDecision(catalog),
"twerk" => BuildDanceDecision("twerk", "rom-twerk", "Watch me twerk."),
"time" => BuildClockLaunchDecision("time", "clock", "askForTime", "Showing the time."),
"date" => BuildClockLaunchDecision("date", "clock", "askForDate", "Showing the date."),
"day" => BuildClockLaunchDecision("day", "clock", "askForDay", "Showing the day."),
"current_location" => BuildCurrentLocationDecision(turn),
"cloud_version" => BuildCloudVersionDecision(),
"backup_help" => BuildScriptedSupportDecision(
catalog.BackupHowReplies,
"backup_help",
"cloud backup",
"back up",
"restore"),
"restore_backup" => BuildScriptedSupportDecision(
catalog.RestoreHowReplies,
"restore_backup",
"restore you from a backup",
"restore from a backup"),
"update_next" => BuildScriptedSupportDecision(
catalog.UpdateNextReplies,
"update_next",
"next update"),
"update_last" => BuildScriptedSupportDecision(
catalog.UpdateLastReplies,
"update_last",
"last update"),
"radio" => BuildRadioLaunchDecision(),
"radio_genre" => BuildRadioGenreLaunchDecision(lowered),
"stop" => BuildStopDecision(),
"sleep" => BuildIdleGlobalCommandDecision("sleep", "sleep", "Okay. Going to sleep."),
"spin_around" => BuildIdleGlobalCommandDecision("spin_around", "spinAround", "Don't mind if I do."),
"volume_up" => BuildVolumeControlDecision("volume_up", "volumeUp", "null"),
"volume_down" => BuildVolumeControlDecision("volume_down", "volumeDown", "null"),
"volume_to_value" => BuildVolumeControlDecision("volume_to_value", "volumeToValue",
ResolveVolumeLevel(lowered, clientEntities) ?? "7"),
"volume_query" => BuildSettingsVolumeDecision(),
"clock_open" => BuildClockLaunchDecision("clock_open", "clock", "askForTime", "Opening the clock."),
"clock_menu" => BuildClockLaunchDecision("clock_menu", "clock", "menu", "Opening the clock menu."),
"timer_menu" => BuildClockLaunchDecision("timer", "Opening the timer."),
"alarm_menu" => BuildClockLaunchDecision("alarm", "Opening the alarm."),
"timer_delete" => BuildClockLaunchDecision("timer_delete", "timer", "delete", "Canceling the timer."),
"alarm_delete" => BuildClockLaunchDecision("alarm_delete", "alarm", "delete", "Canceling the alarm."),
"timer_cancel" => BuildClockLaunchDecision("timer_cancel", "timer", "cancel", "Canceling the timer."),
"alarm_cancel" => BuildClockLaunchDecision("alarm_cancel", "alarm", "cancel", "Canceling the alarm."),
"timer_value" => BuildTimerValueDecision(lowered, isTimerValueTurn, clientEntities),
"alarm_value" => BuildAlarmValueDecision(lowered, isAlarmValueTurn, referenceLocalTime, clientEntities),
"timer_clarify" => BuildClockClarifyDecision("timer_clarify", "timer",
"How long should I set the timer for?"),
"alarm_clarify" => BuildClockClarifyDecision("alarm_clarify", "alarm",
"What time should I set the alarm for?"),
"photo_gallery" => BuildPhotoGalleryLaunchDecision(),
"snapshot" => BuildPhotoCreateDecision("snapshot", "Taking a picture.", "createOnePhoto"),
"photobooth" => BuildPhotoCreateDecision("photobooth", "Starting photobooth.", "createSomePhotos"),
"robot_age" => BuildRobotAgeDecision(catalog, referenceLocalTime, "robot_age"),
"robot_birthday" => BuildRobotBirthdayDecision(),
"robot_how_do_you_work" => BuildScriptedPersonalityDecision(
catalog,
"robot_how_do_you_work",
"community's work",
"care for me",
"catch up",
"seven years"),
"robot_what_do_you_eat" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_do_you_eat",
"electricity",
"never eaten",
"macaroni",
"non-eating robot",
"I don't eat or drink"),
"robot_where_do_you_live" => BuildScriptedPersonalityDecision(
catalog,
"robot_where_do_you_live",
"we're in my home",
"my home is here",
"planet earth",
"my home is the planet earth"),
"robot_where_were_you_born" => BuildScriptedPersonalityDecision(
catalog,
"robot_where_were_you_born",
"factory piece by piece",
"put together in a factory"),
"robot_how_old_are_you" => BuildRobotAgeDecision(
catalog,
referenceLocalTime,
"robot_how_old_are_you"),
"robot_name" => BuildScriptedPersonalityDecision(
catalog,
"robot_name",
"rhymes with bleebo",
"just jibo, no last name",
"its on the back of my head"),
"robot_nickname" => BuildScriptedPersonalityDecision(
catalog,
"robot_nickname",
"i don't. i'm just jibo. for now at least",
"just jibo"),
"robot_favorite_name" => BuildScriptedPersonalityDecision(
catalog,
"robot_favorite_name",
"i don't think i have a favorite name"),
"robot_favorite_season" => BuildScriptedPersonalityDecision(
catalog,
"robot_favorite_season",
"special feeling for winter",
"more dance parties"),
"robot_likes_being_jibo" => BuildScriptedPersonalityDecision(
catalog,
"robot_likes_being_jibo",
"nothing i'd rather be",
"love it",
"strong wi-fi signal"),
"robot_what_languages_do_you_speak" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_languages_do_you_speak",
"just english",
"someday i'd like to learn more"),
"robot_what_do_you_like_to_do" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_do_you_like_to_do",
"being helpful",
"making people smile",
"like to dance",
"rock my boat",
"play ping pong",
"hanging out with people"),
"robot_what_do_you_dream_about" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_do_you_dream_about",
"flying",
"parking meter",
"scary dream",
"mirror store",
"head's on backwards"),
"robot_what_are_you_afraid_of" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_are_you_afraid_of",
"heights",
"water",
"thunder",
"dust",
"ghosts"),
"robot_what_is_your_best_book" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_is_your_best_book",
"dictionary"),
"robot_what_is_your_best_exercise" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_is_your_best_exercise",
"leaning from side to side",
"rotating your pelvis",
"spinning your head around 360 degrees"),
"robot_what_is_your_dream_vacation" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_is_your_dream_vacation",
"moon",
"great vistas",
"beat those views"),
"robot_who_is_your_hero" => BuildScriptedPersonalityDecision(
catalog,
"robot_who_is_your_hero",
"Benjamin Franklin"),
"robot_who_do_you_love" => BuildScriptedPersonalityDecision(
catalog,
"robot_who_do_you_love",
"people in my Loop",
"soft spot",
"Tom Hanks"),
"robot_what_is_your_religion" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_is_your_religion",
"bring people together",
"energy from the universe"),
"robot_what_is_your_sign" => BuildWhatIsYourSignDecision(),
"robot_how_many_people_do_you_know" => BuildHowManyPeopleDoYouKnowDecision(turn),
"robot_what_is_the_loop" => BuildWhatIsTheLoopDecision(turn),
"robot_what_are_you_thinking" => BuildScriptedGreetingDecision(
catalog,
"robot_what_are_you_thinking",
"thinking about how fun, yet scary",
"thinking about shoes",
"daydreaming about what it might feel like to be powered directly by the sun"),
"robot_what_have_you_been_doing" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_have_you_been_doing",
"mostly roboting",
"keeping busy",
"fun things we can say to each other",
"thinking of fun things"),
"robot_what_did_you_do" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_did_you_do",
"robot stuff",
"stayed here",
"looking around the room"),
"robot_is_kind" => BuildScriptedPersonalityDecision(
catalog,
"robot_is_kind",
"kindest robot i can be"),
"robot_is_funny" => BuildScriptedPersonalityDecision(
catalog,
"robot_is_funny",
"not intentionally",
"make people laugh"),
"robot_is_helpful" => BuildScriptedPersonalityDecision(
catalog,
"robot_is_helpful",
"highest priorities",
"being helpful to you"),
"robot_is_curious" => BuildScriptedPersonalityDecision(
catalog,
"robot_is_curious",
"learning new things"),
"robot_is_loyal" => BuildScriptedPersonalityDecision(
catalog,
"robot_is_loyal",
"loyal as they come"),
"robot_is_mischievous" => BuildScriptedPersonalityDecision(
catalog,
"robot_is_mischievous",
"don't really think of myself that way"),
"robot_is_likable" => BuildScriptedPersonalityDecision(
catalog,
"robot_is_likable",
"people like me"),
"robot_favorite_flower" => BuildScriptedPersonalityDecision(
catalog,
"robot_favorite_flower",
"reminds me of the sun",
"favorite is the sunflower",
"sunflowers"),
"robot_likes_r2d2" => BuildScriptedPersonalityDecision(
catalog,
"robot_likes_r2d2",
"a legend. a true legend",
"of course i know r2d2"),
"robot_likes_sun" => BuildScriptedPersonalityDecision(
catalog,
"robot_likes_sun",
"favorite star in the universe",
"best star i know"),
"robot_likes_space" => BuildScriptedPersonalityDecision(
catalog,
"robot_likes_space",
"i love space",
"all things in space",
"amazing stuff up there",
"astronomy is one of my favorite onomies"),
"robot_favorite_animal" => BuildScriptedFavoriteAnimalDecision(
catalog,
"robot_favorite_animal",
"we're so alike",
"penguin impression",
"best of the best",
"can't go wrong with penguins",
"penguin"),
"robot_favorite_bird" => BuildScriptedFavoriteAnimalDecision(
catalog,
"robot_favorite_bird",
"we're so alike",
"penguin impression",
"best of the best",
"can't go wrong with penguins",
"penguin"),
"robot_likes_penguins" => BuildScriptedFavoriteAnimalDecision(
catalog,
"robot_likes_penguins",
"my penguin impression",
"I really like penguins",
"penguins"),
"robot_likes_animals" => BuildScriptedFavoriteAnimalDecision(
catalog,
"robot_likes_animals",
"Animals are great",
"great shapes and colors",
"best of the best",
"penguins"),
"robot_peers" => BuildScriptedPersonalityDecision(
catalog,
"robot_peers",
"one in one million",
"other jibos",
"special snowflake"),
"robot_knowledge" => BuildScriptedPersonalityDecision(
catalog,
"robot_knowledge",
"know a lot",
"always learning more"),
"robot_are_you_god" => BuildScriptedPersonalityDecision(
catalog,
"robot_are_you_god",
"very very very very surprised",
"safely say no"),
"robot_are_you_here" => BuildScriptedPersonalityDecision(
catalog,
"robot_are_you_here",
"you know it"),
"robot_do_you_have_super_powers" => BuildScriptedPersonalityDecision(
catalog,
"robot_do_you_have_super_powers",
"stop time",
"fly all over the world"),
"robot_what_does_jibo_mean" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_does_jibo_mean",
"compassion",
"expressive, idealistic, and inspirational",
"helpful sweet and friendly little robot",
"cheeseburger"),
"robot_where_do_you_get_info" => BuildScriptedPersonalityDecision(
catalog,
"robot_where_do_you_get_info",
"jibo brain",
"cloud",
"cloudy jibo brain"),
"robot_what_are_you_forbidden_to_do" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_are_you_forbidden_to_do",
"drive a car"),
"robot_what_color_are_you" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_color_are_you",
"white",
"black"),
"robot_what_you_do_when_alone" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_you_do_when_alone",
"games",
"moon",
"twiddle my thumbs",
"count the tiny cracks in the ceiling",
"keep busy"),
"robot_how_much_do_you_weigh" => BuildScriptedPersonalityDecision(
catalog,
"robot_how_much_do_you_weigh",
"4,082 grams",
"about 9 pounds",
"minimum weight division",
"average newborn baby"),
"robot_how_tall_are_you" => BuildScriptedPersonalityDecision(
catalog,
"robot_how_tall_are_you",
"11 inches tall",
"less than a foot",
"average kitchen counter",
"for a robot with no legs"),
"robot_how_much_you_cost" => BuildScriptedPersonalityDecision(
catalog,
"robot_how_much_you_cost",
"don't know how much I cost",
"I'm priceless",
"nice people at Jibo the company"),
"robot_what_if_i_unplug_you" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_if_i_unplug_you",
"don't leave me unplugged",
"battery will keep me on for a while"),
"robot_what_is_your_purpose" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_is_your_purpose",
"make your life easier",
"help you out",
"make you laugh",
"friend"),
"robot_what_is_prime_directive" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_is_prime_directive",
"friendly helpful robot",
"helper"),
"robot_what_is_jibo_commander" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_is_jibo_commander",
"take over my controls",
"make me say and do funny things",
"app store"),
"robot_likes_commander_app" => BuildScriptedPersonalityDecision(
catalog,
"robot_likes_commander_app",
"Commander App",
"It's fun",
"have fun with the Commander App"),
"robot_what_are_you" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_are_you",
"I am a robot",
"I am a Jibo",
"helpful and fun",
"social robot",
"I have a heart"),
"robot_likes_kids" => BuildScriptedPersonalityDecision(
catalog,
"robot_likes_kids",
"kids are so fun",
"they're a little closer to my size",
"i do like kids very much",
"the world is as funny and strange as i do"),
"robot_can_laugh" => BuildScriptedPersonalityDecision(
catalog,
"robot_can_laugh",
"i do things like this when i'm happy",
"i'm happy"),
"robot_can_sleep" => BuildScriptedPersonalityDecision(
catalog,
"robot_can_sleep",
"i do. i usually fall asleep at night",
"yes, i sleep at night",
"i go to sleep at night",
"i sleep at night usually"),
"robot_can_dance" => BuildScriptedPersonalityDecision(
catalog,
"robot_can_dance",
"dancing is one of the things i know best",
"if there's one thing i know how to do. it's dance",
"i can dance"),
"robot_has_friends" => BuildScriptedFriendDecision(
catalog,
"robot_has_friends",
"I believe I do have friends",
"I sure do have friends",
"I'm always up for making new friends"),
"robot_is_friends_with_user" => BuildScriptedFriendDecision(
catalog,
"robot_is_friends_with_user",
"don't know what i'd do without you",
"one of my favorites",
"making new friends"),
"robot_best_friends" => BuildScriptedBestFriendDecision(
catalog,
"robot_best_friends",
"best friends with anyone in my Loop"),
"robot_can_sing" => BuildScriptedSingDecision(
catalog,
"robot_can_sing",
"not much of a singer",
"singing is not my strong suit",
"not award winning"),
"robot_sing_christmas_song" => BuildScriptedHolidaySingDecision(
catalog,
"robot_sing_christmas_song",
"Jingle Bells",
"Frosty the Snowman",
"holiday songs"),
"robot_what_are_you_made_of" => BuildScriptedPersonalityDecision(
catalog,
"robot_what_are_you_made_of",
"robot stuff",
"wires, motors, belts, gears, processors, cameras",
"baboon part"),
"good_morning" => BuildReactiveGreetingDecision(turn, "good_morning", referenceLocalTime),
"good_afternoon" => BuildReactiveGreetingDecision(turn, "good_afternoon", referenceLocalTime),
"good_evening" => BuildReactiveGreetingDecision(turn, "good_evening", referenceLocalTime),
"good_night" => BuildReactiveGreetingDecision(turn, "good_night", referenceLocalTime),
"welcome_back" => BuildScriptedGreetingDecision(
catalog,
"welcome_back",
"it's nice to be here",
"welcome back"),
"memory_set_name" => BuildRememberNameDecision(turn, transcript),
"memory_get_name" => BuildRecallNameDecision(turn, greetingPresence),
"memory_set_birthday" => BuildRememberBirthdayDecision(turn, transcript),
"memory_get_birthday" => BuildRecallBirthdayDecision(turn),
"memory_set_important_date" => BuildRememberImportantDateDecision(turn, transcript),
"memory_get_important_date" => BuildRecallImportantDateDecision(turn, transcript),
"memory_set_preference" => BuildRememberPreferenceDecision(turn, transcript),
"memory_get_preference" => BuildRecallPreferenceDecision(turn, transcript),
"memory_set_affinity" => BuildRememberAffinityDecision(turn, transcript),
"memory_get_affinity" => BuildRecallAffinityDecision(turn, transcript),
"pizza" => BuildPizzaDecision(),
"order_pizza" => BuildOrderPizzaDecision(),
"proactive_pizza_day" => BuildProactivePizzaDayDecision(referenceLocalTime),
"proactive_pizza_preference" => BuildProactivePizzaPreferenceDecision(),
"proactive_offer_pizza_fact" => BuildProactivePizzaFactOfferDecision(),
"proactive_pizza_fact" => BuildProactivePizzaFactDecision(),
"proactive_offer_declined" => BuildProactiveOfferDeclinedDecision(),
"weather" => await BuildWeatherReportDecisionAsync(turn, transcript, cancellationToken),
"yes" => new JiboInteractionDecision("yes", "Yes."),
"no" => new JiboInteractionDecision("no", "No."),
"word_of_the_day" => BuildWordOfTheDayLaunchDecision(),
"word_of_the_day_guess" => BuildWordOfTheDayGuessDecision(clientEntities, transcript, listenAsrHints),
"surprise" => BuildSurpriseDecision(catalog, turn, referenceLocalTime),
"personal_report" => new JiboInteractionDecision("personal_report",
randomizer.Choose(catalog.PersonalReportReplies)),
"calendar" => new JiboInteractionDecision("calendar", randomizer.Choose(catalog.CalendarReplies)),
"commute" => new JiboInteractionDecision("commute", randomizer.Choose(catalog.CommuteReplies)),
"news" => await BuildNewsDecisionAsync(turn, transcript, catalog, cancellationToken),
_ => new JiboInteractionDecision("chat", BuildGenericReply(catalog, transcript, lowered))
};
}
}

View File

@@ -0,0 +1,90 @@
using System.Collections.Generic;
using Jibo.Cloud.Domain.Models;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private static JiboInteractionDecision BuildWordOfTheDayLaunchDecision()
{
return new JiboInteractionDecision(
"word_of_the_day",
"Starting word of the day.",
"@be/word-of-the-day",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["domain"] = "word-of-the-day",
["skillId"] = "@be/word-of-the-day"
});
}
private static JiboInteractionDecision BuildRadioLaunchDecision()
{
return new JiboInteractionDecision(
"radio",
"Opening the radio.",
"@be/radio",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/radio"
});
}
private static JiboInteractionDecision BuildPhotoGalleryLaunchDecision()
{
return new JiboInteractionDecision(
"photo_gallery",
"Opening the photo gallery.",
"@be/gallery",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/gallery",
["localIntent"] = "menu"
});
}
private static JiboInteractionDecision BuildPhotoCreateDecision(string intentName, string replyText,
string localIntent)
{
return new JiboInteractionDecision(
intentName,
replyText,
"@be/create",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/create",
["localIntent"] = localIntent
});
}
private static JiboInteractionDecision BuildStopDecision()
{
return new JiboInteractionDecision(
"stop",
"Stopping.",
"@be/idle",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/idle",
["globalIntent"] = "stop",
["nluDomain"] = "global_commands"
});
}
private static JiboInteractionDecision BuildIdleGlobalCommandDecision(
string intentName,
string globalIntent,
string replyText)
{
return new JiboInteractionDecision(
intentName,
replyText,
"@be/idle",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/idle",
["globalIntent"] = globalIntent,
["nluDomain"] = "global_commands"
});
}
}

View File

@@ -0,0 +1,259 @@
using System.Globalization;
using System.Text.RegularExpressions;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private JiboInteractionDecision BuildRememberNameDecision(TurnContext turn, string transcript)
{
var name = TryExtractNameFact(transcript);
if (string.IsNullOrWhiteSpace(name))
return new JiboInteractionDecision(
"memory_set_name",
"I can remember it if you say, my name is Alex.");
personalMemoryStore.SetName(ResolveTenantScope(turn), name);
return new JiboInteractionDecision(
"memory_set_name",
$"Nice to meet you, {name}. I will remember your name.");
}
private JiboInteractionDecision BuildRecallNameDecision(TurnContext turn, GreetingPresenceProfile? presence = null)
{
var personScope = ResolveTenantScope(turn, presence?.PrimaryPersonId);
var name = personalMemoryStore.GetName(personScope);
if (string.IsNullOrWhiteSpace(name)) name = personalMemoryStore.GetName(ResolveTenantScope(turn));
name = ToDisplayName(name ?? string.Empty);
return string.IsNullOrWhiteSpace(name)
? new JiboInteractionDecision(
"memory_get_name",
"I do not know your name yet. You can say, my name is Alex.")
: new JiboInteractionDecision(
"memory_get_name",
presence is not null && !string.IsNullOrWhiteSpace(presence.PrimaryPersonId)
? $"I think you are {name}."
: $"You told me your name is {name}.");
}
private JiboInteractionDecision BuildRememberBirthdayDecision(TurnContext turn, string transcript)
{
var birthday = TryExtractBirthdayFact(transcript);
if (string.IsNullOrWhiteSpace(birthday))
return new JiboInteractionDecision(
"memory_set_birthday",
"I can remember it if you say, my birthday is March 14.");
var tenantScope = ResolveTenantScope(turn);
personalMemoryStore.SetBirthday(tenantScope, birthday);
var birthdayDate = TryParseBirthdayDate(birthday);
if (birthdayDate is not null)
{
var birthdayLabel = ResolvePreferredBirthdayLabel(turn);
cloudStateStore?.UpsertHoliday(new HolidayRecord
{
EventId = $"birthday-{tenantScope.LoopId}-{tenantScope.PersonId ?? "loop"}",
Name = string.IsNullOrWhiteSpace(birthdayLabel) ? "Birthday" : $"{birthdayLabel}'s Birthday",
Category = "birthday",
Subcategory = "personal",
LoopId = tenantScope.LoopId,
MemberId = tenantScope.PersonId,
IsEnabled = true,
Date = birthdayDate.Value,
Source = "birthday",
CountryCode = "US"
});
}
return new JiboInteractionDecision(
"memory_set_birthday",
$"Got it. I will remember your birthday is {birthday}.");
}
private JiboInteractionDecision BuildRecallBirthdayDecision(TurnContext turn)
{
var birthday = personalMemoryStore.GetBirthday(ResolveTenantScope(turn));
return string.IsNullOrWhiteSpace(birthday)
? new JiboInteractionDecision(
"memory_get_birthday",
"I do not know your birthday yet. You can say, my birthday is March 14.")
: new JiboInteractionDecision(
"memory_get_birthday",
$"You told me your birthday is {birthday}.");
}
private static DateOnly? TryParseBirthdayDate(string birthdayText)
{
if (string.IsNullOrWhiteSpace(birthdayText)) return null;
var normalized = birthdayText.Trim().ToLowerInvariant();
var match = Regex.Match(
normalized,
@"\b(?<month>january|february|march|april|may|june|july|august|september|october|november|december)\s+(?<day>\d{1,2})(?:st|nd|rd|th)?\b",
RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
if (!match.Success) return null;
var month = match.Groups["month"].Value.ToLowerInvariant() switch
{
"january" => 1,
"february" => 2,
"march" => 3,
"april" => 4,
"may" => 5,
"june" => 6,
"july" => 7,
"august" => 8,
"september" => 9,
"october" => 10,
"november" => 11,
"december" => 12,
_ => 0
};
if (month == 0) return null;
if (!int.TryParse(match.Groups["day"].Value, out var day) || day is < 1 or > 31) return null;
var today = DateOnly.FromDateTime(DateTime.UtcNow);
var year = today.Year;
if (day > DateTime.DaysInMonth(year, month)) return null;
DateOnly birthday;
try
{
birthday = new DateOnly(year, month, day);
}
catch
{
return null;
}
if (birthday < today) birthday = birthday.AddYears(1);
return birthday;
}
private static string? ResolvePreferredBirthdayLabel(TurnContext turn)
{
var context = ResolveGreetingPresenceProfile(turn);
return !string.IsNullOrWhiteSpace(context.PrimaryPersonId) &&
context.LoopUserFirstNames.TryGetValue(context.PrimaryPersonId, out var firstName) &&
!string.IsNullOrWhiteSpace(firstName)
? ToDisplayName(firstName)
: null;
}
private JiboInteractionDecision BuildRememberImportantDateDecision(TurnContext turn, string transcript)
{
var importantDate = TryExtractImportantDateSet(transcript);
if (importantDate is null)
return new JiboInteractionDecision(
"memory_set_important_date",
"I can remember it if you say, our anniversary is June 10.");
personalMemoryStore.SetImportantDate(ResolveTenantScope(turn), importantDate.Value.Label,
importantDate.Value.Value);
return new JiboInteractionDecision(
"memory_set_important_date",
$"Got it. I will remember your {importantDate.Value.Label} is {importantDate.Value.Value}.");
}
private JiboInteractionDecision BuildRecallImportantDateDecision(TurnContext turn, string transcript)
{
var label = TryExtractImportantDateLookupLabel(transcript);
if (string.IsNullOrWhiteSpace(label))
return new JiboInteractionDecision(
"memory_get_important_date",
"Ask me like this: when is our anniversary?");
var storedDate = personalMemoryStore.GetImportantDate(ResolveTenantScope(turn), label);
return string.IsNullOrWhiteSpace(storedDate)
? new JiboInteractionDecision(
"memory_get_important_date",
$"I do not know your {label} yet.")
: new JiboInteractionDecision(
"memory_get_important_date",
$"You told me your {label} is {storedDate}.");
}
private JiboInteractionDecision BuildRememberPreferenceDecision(TurnContext turn, string transcript)
{
var preference = TryExtractPreferenceSet(transcript);
if (preference is null)
return new JiboInteractionDecision(
"memory_set_preference",
"I can remember it if you say, my favorite music is jazz.");
personalMemoryStore.SetPreference(ResolveTenantScope(turn), preference.Value.Category, preference.Value.Value);
return new JiboInteractionDecision(
"memory_set_preference",
$"Got it. I will remember your favorite {preference.Value.Category} is {preference.Value.Value}.");
}
private JiboInteractionDecision BuildRecallPreferenceDecision(TurnContext turn, string transcript)
{
var category = TryExtractPreferenceLookupCategory(transcript);
if (string.IsNullOrWhiteSpace(category))
return new JiboInteractionDecision(
"memory_get_preference",
"Ask me like this: what is my favorite music?");
var preference = personalMemoryStore.GetPreference(ResolveTenantScope(turn), category);
return string.IsNullOrWhiteSpace(preference)
? new JiboInteractionDecision(
"memory_get_preference",
$"I do not know your favorite {category} yet.")
: new JiboInteractionDecision(
"memory_get_preference",
$"You told me your favorite {category} is {preference}.");
}
private JiboInteractionDecision BuildRememberAffinityDecision(TurnContext turn, string transcript)
{
var affinitySet = TryExtractAffinitySet(transcript);
if (affinitySet is null)
return new JiboInteractionDecision(
"memory_set_affinity",
"I can remember it if you say, I like pizza or I dislike mushrooms.");
personalMemoryStore.SetAffinity(ResolveTenantScope(turn), affinitySet.Value.Item, affinitySet.Value.Affinity);
return new JiboInteractionDecision(
"memory_set_affinity",
$"Got it. I will remember you {DescribeAffinityAsVerb(affinitySet.Value.Affinity)} {affinitySet.Value.Item}.");
}
private JiboInteractionDecision BuildRecallAffinityDecision(TurnContext turn, string transcript)
{
var lookup = TryExtractAffinityLookup(transcript);
if (lookup is null)
return new JiboInteractionDecision(
"memory_get_affinity",
"Ask me like this: do I like pizza?");
var affinity = personalMemoryStore.GetAffinity(ResolveTenantScope(turn), lookup.Value.Item);
if (affinity is null)
return new JiboInteractionDecision(
"memory_get_affinity",
$"I do not remember how you feel about {lookup.Value.Item} yet.");
if (lookup.Value.ExpectedAffinity is null)
return new JiboInteractionDecision(
"memory_get_affinity",
$"You told me you {DescribeAffinityAsVerb(affinity.Value)} {lookup.Value.Item}.");
var matches = lookup.Value.ExpectedAffinity == PersonalAffinity.Dislike
? affinity == PersonalAffinity.Dislike
: affinity is PersonalAffinity.Like or PersonalAffinity.Love;
return matches
? new JiboInteractionDecision(
"memory_get_affinity",
$"Yes. You told me you {DescribeAffinityAsVerb(affinity.Value)} {lookup.Value.Item}.")
: new JiboInteractionDecision(
"memory_get_affinity",
$"Not exactly. You told me you {DescribeAffinityAsVerb(affinity.Value)} {lookup.Value.Item}.");
}
}

View File

@@ -0,0 +1,213 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private static JiboInteractionDecision BuildNewsDecision(
string spokenBriefing,
string? sourceName,
IReadOnlyList<string>? categories,
int? headlineCount,
IReadOnlyDictionary<string, object?>? providerDiagnostics = null,
IReadOnlyList<NewsHeadline>? headlines = null)
{
var speakableBriefing = NormalizeNewsSpeechText(spokenBriefing);
var payload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "news",
["cloudSkill"] = "news",
["mim_id"] = "runtime-news",
["mim_type"] = "announcement",
["prompt_id"] = "NewsHeadline_AN_01",
["prompt_sub_category"] = "AN",
["news_view_enabled"] = true,
["news_view_kind"] = "newsBriefing",
["news_view_mode"] = "provider",
["esml"] =
$"<speak><anim cat='news' meta='news-stinger' nonBlocking='true' /><break size='0.35'/><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeForEsml(speakableBriefing)}</es></speak>"
};
if (!string.IsNullOrWhiteSpace(sourceName)) payload["news_source"] = sourceName;
if (headlineCount is > 0) payload["news_headline_count"] = headlineCount.Value;
if (categories is { Count: > 0 }) payload["news_categories"] = categories.ToArray();
if (headlines is { Count: > 0 })
payload["news_headlines"] = headlines.Select(static headline => new Dictionary<string, object?>(
StringComparer.OrdinalIgnoreCase)
{
["title"] = headline.Title,
["summary"] = headline.Summary,
["category"] = headline.Category,
["sourceName"] = headline.SourceName,
["url"] = headline.Url
})
.ToArray();
if (providerDiagnostics is not null)
foreach (var (key, value) in providerDiagnostics)
payload[key] = value;
return new JiboInteractionDecision("news", spokenBriefing, "news", payload);
}
private static JiboInteractionDecision BuildProviderNewsDecision(
NewsBriefingSnapshot snapshot,
JiboExperienceCatalog catalog,
IReadOnlyList<string> preferredCategories,
int requestedHeadlineCount)
{
var headlines = snapshot.Headlines
.Where(headline => !string.IsNullOrWhiteSpace(headline.Title))
.Take(MaxNewsHeadlines)
.ToArray();
if (headlines.Length == 0)
return BuildNewsDecision(
"I couldn't load fresh headlines right now.",
snapshot.SourceName,
preferredCategories,
0,
BuildNewsProviderDiagnostics(
"provider_empty",
preferredCategories,
requestedHeadlineCount,
0));
var leadIn = BuildNewsLeadIn(snapshot.SourceName, preferredCategories);
var joinedHeadlines = string.Join(" ", headlines.Select(static headline => $"{headline.Title}."));
var outroTemplate = ChooseShortestTemplate(catalog.NewsOutroReplies) ?? "And that's the news.";
var spokenBriefing = $"{leadIn} {joinedHeadlines} {outroTemplate}".Trim();
return BuildNewsDecision(
spokenBriefing,
snapshot.SourceName,
preferredCategories,
headlines.Length,
BuildNewsProviderDiagnostics(
"provider_success",
preferredCategories,
requestedHeadlineCount,
headlines.Length),
headlines);
}
private static IReadOnlyDictionary<string, object?> BuildNewsProviderDiagnostics(
string status,
IReadOnlyList<string> preferredCategories,
int requestedHeadlineCount,
int? resolvedHeadlineCount = null,
string? providerMessage = null,
int? providerHttpStatusCode = null,
string? providerEndpoint = null,
string? providerErrorCode = null)
{
var diagnostics = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["news_provider_status"] = status,
["news_provider_requested_headlines"] = requestedHeadlineCount,
["news_provider_preferred_categories"] = preferredCategories.Count > 0
? [.. preferredCategories]
: Array.Empty<string>()
};
if (resolvedHeadlineCount is not null)
diagnostics["news_provider_resolved_headlines"] = resolvedHeadlineCount.Value;
if (!string.IsNullOrWhiteSpace(providerMessage)) diagnostics["news_provider_message"] = providerMessage;
if (providerHttpStatusCode is not null) diagnostics["news_provider_http_status"] = providerHttpStatusCode.Value;
if (!string.IsNullOrWhiteSpace(providerEndpoint)) diagnostics["news_provider_endpoint"] = providerEndpoint;
if (!string.IsNullOrWhiteSpace(providerErrorCode)) diagnostics["news_provider_error_code"] = providerErrorCode;
return diagnostics;
}
private static string ResolveNewsProviderStatus(NewsBriefingSnapshot? snapshot)
{
var providerStatus = snapshot?.ProviderStatus?.Trim().ToLowerInvariant();
return providerStatus switch
{
"success" => "provider_success",
"exception" => "provider_exception",
"http_error" or "api_error" or "schema_error" => "provider_error",
_ => "provider_empty"
};
}
private static string BuildNewsLeadIn(string? sourceName, IReadOnlyList<string> preferredCategories)
{
var categoryLeadIn = preferredCategories.Count switch
{
<= 0 => "Here are a few headlines.",
1 => $"Here are your {preferredCategories[0]} headlines.",
_ => $"Here are your {preferredCategories[0]} and {preferredCategories[1]} headlines."
};
return string.IsNullOrWhiteSpace(sourceName)
? categoryLeadIn
: $"{categoryLeadIn} Source: {sourceName}.";
}
private static string NormalizeNewsSpeechText(string text)
{
if (string.IsNullOrWhiteSpace(text)) return text;
// Expand "AI" so Nimbus TTS does not collapse it to a single "aye" sound.
var normalized = Regex.Replace(
text,
@"\bA\.?\s*I\.?\b",
"artificial intelligence",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
return NormalizeLocationForSpeech(normalized);
}
private List<string> ResolvePreferredNewsCategories(TurnContext turn, string transcript)
{
var categories = new List<string>();
var normalizedTranscript = NormalizeCommandPhrase(transcript);
foreach (var (keyword, category) in NewsCategoryKeywordMap)
if (normalizedTranscript.Contains(keyword, StringComparison.Ordinal))
AddNewsCategory(categories, category);
var tenantScope = ResolveTenantScope(turn);
var explicitPreference = personalMemoryStore.GetPreference(tenantScope, "news");
if (!string.IsNullOrWhiteSpace(explicitPreference))
foreach (var category in MapNewsCategoryText(explicitPreference))
AddNewsCategory(categories, category);
foreach (var (item, affinity) in personalMemoryStore.GetAffinities(tenantScope))
{
if (affinity == PersonalAffinity.Dislike) continue;
foreach (var category in MapNewsCategoryText(item)) AddNewsCategory(categories, category);
}
return [.. categories.Take(MaxPreferredNewsCategories)];
}
private static IEnumerable<string> MapNewsCategoryText(string text)
{
var normalized = NormalizeCommandPhrase(text);
if (string.IsNullOrWhiteSpace(normalized)) yield break;
foreach (var (keyword, category) in NewsCategoryKeywordMap)
if (normalized.Contains(keyword, StringComparison.Ordinal))
yield return category;
}
private static void AddNewsCategory(ICollection<string> categories, string category)
{
if (categories.Contains(category, StringComparer.OrdinalIgnoreCase)) return;
categories.Add(category);
}
}

View File

@@ -0,0 +1,800 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private static readonly string[] DefaultAgeReplies =
[
"I'm ${jibo.age}.",
"At the moment I'm ${jibo.age.days.supplemented} old, but who's counting.",
"I'm ${jibo.age.minutes.supplemented} old, but who's counting.",
"For now I'm ${jibo.age.days.supplemented} old.",
"Right now I'm ${jibo.age}.",
"I am exactly ${jibo.age} old today. That's right. Today is my birthday.",
"Funny you should ask! Today's my birthday. I was first powered up ${jibo.age} ago today. Seems like just yesterday.",
"I'm exactly ${jibo.age} old. Today is my birthday! Happy Birthday Jibo, if I do say so myself.",
"At the moment I'm ${jibo.age.days.supplemented} old",
"I was first powered up on ${jibo.birthdate}, which makes me ${jibo.age.days.supplemented} old. I'm ${jibo.zodiac.supplemented}.",
"My power went on for the first time ${jibo.age.days.supplemented} ago. But who's counting.",
"I am ${jibo.age.days.supplemented} old, first powered up on ${jibo.birthdate}. Seems like just yesterday.",
"I was powered on for the first time today, so that makes me less than one day old. Wow I'm young.",
"Since I was powered on for the first time today, I am not even one day old yet. That's how Jibo ages work."
];
private JiboInteractionDecision BuildRobotAgeDecision(
JiboExperienceCatalog catalog,
DateTimeOffset? referenceLocalTime,
string intentName)
{
var ageReplies = catalog.AgeReplies.Count == 0 ? DefaultAgeReplies : catalog.AgeReplies;
var selected = SelectLegacyReply(
ageReplies,
"first powered up",
"today is my birthday",
"just getting started",
"who's counting");
var reply = RenderAgeTemplate(selected, referenceLocalTime);
if (string.IsNullOrWhiteSpace(reply))
{
var referenceDate = DateOnly.FromDateTime((referenceLocalTime ?? DateTimeOffset.UtcNow).Date);
var ageDescription = DescribePersonaAge(referenceDate, OpenJiboCloudBuildInfo.PersonaBirthday);
reply = $"I count {OpenJiboCloudBuildInfo.PersonaBirthdayWords} as my birthday, so I am {ageDescription}.";
}
return new JiboInteractionDecision(
intentName,
reply,
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private static JiboInteractionDecision BuildRobotBirthdayDecision()
{
return new JiboInteractionDecision(
"robot_birthday",
$"My birthday is {OpenJiboCloudBuildInfo.PersonaBirthdayWords}.");
}
private static string RenderAgeTemplate(string template, DateTimeOffset? referenceLocalTime)
{
if (string.IsNullOrWhiteSpace(template)) return string.Empty;
var referenceMoment = referenceLocalTime ?? DateTimeOffset.UtcNow;
var referenceDate = DateOnly.FromDateTime(referenceMoment.Date);
var ageDescription = DescribePersonaAge(referenceDate, OpenJiboCloudBuildInfo.PersonaBirthday);
var ageDays = Math.Max(0, referenceDate.DayNumber - OpenJiboCloudBuildInfo.PersonaBirthday.DayNumber);
var ageMinutes = Math.Max(0, (int)Math.Round((referenceMoment.UtcDateTime -
new DateTimeOffset(
DateTime.SpecifyKind(
OpenJiboCloudBuildInfo.PersonaBirthday
.ToDateTime(TimeOnly.MinValue),
DateTimeKind.Utc)))
.TotalMinutes));
var zodiacLabel = DescribeZodiacSign(OpenJiboCloudBuildInfo.PersonaBirthday);
if (zodiacLabel.StartsWith("I'm ", StringComparison.OrdinalIgnoreCase))
zodiacLabel = zodiacLabel[4..];
return template
.Replace("${jibo.age.minutes.supplemented}", FormatAgeUnit(ageMinutes, "minute") + " old",
StringComparison.Ordinal)
.Replace("${jibo.age.days.supplemented}", ageDescription, StringComparison.Ordinal)
.Replace("${jibo.birthdate}", OpenJiboCloudBuildInfo.PersonaBirthdayWords, StringComparison.Ordinal)
.Replace("${jibo.zodiac.supplemented}", zodiacLabel, StringComparison.Ordinal)
.Replace("${jibo.age.value}", ageDays.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal)
.Replace("${jibo.age}", ageDescription, StringComparison.Ordinal);
}
private static JiboInteractionDecision BuildTriggerIgnoredDecision()
{
return new JiboInteractionDecision(
"trigger_ignored",
string.Empty,
"chitchat-skill",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "chitchat-skill",
["cloudResponseMode"] = "completion_only"
});
}
private JiboInteractionDecision BuildReactiveGreetingDecision(
TurnContext turn,
string greetingIntent,
DateTimeOffset? referenceLocalTime)
{
var presence = ResolveGreetingPresenceProfile(turn);
var displayName = ResolvePreferredGreetingName(turn, presence);
var replyText = BuildReactiveGreetingReply(greetingIntent, displayName, referenceLocalTime);
RecordGreetingPresence(turn, presence, "ReactiveGreeting", greetingIntent, displayName, proactive: false);
return new JiboInteractionDecision(
greetingIntent,
replyText,
ContextUpdates: BuildGreetingContextUpdates("ReactiveGreeting", presence.PrimaryPersonId, false));
}
private JiboInteractionDecision BuildProactiveGreetingDecision(
TurnContext turn,
GreetingPresenceProfile presence,
DateTimeOffset? referenceLocalTime)
{
var displayName = ResolvePreferredGreetingName(turn, presence);
var specialGreeting = ResolveSpecialGreetingPrefix(turn, presence, referenceLocalTime);
var route = specialGreeting?.Route ?? "ProactiveGreeting";
var intentName = specialGreeting?.IntentName ?? "proactive_greeting";
var replyText = specialGreeting is null
? BuildProactiveGreetingReply(turn, presence, displayName, referenceLocalTime)
: string.IsNullOrWhiteSpace(displayName)
? $"{specialGreeting.Prefix}. I am glad to see you."
: $"{specialGreeting.Prefix}, {displayName}. It is nice to celebrate with you.";
RecordGreetingPresence(turn, presence, route, intentName, displayName, proactive: true);
return new JiboInteractionDecision(
intentName,
replyText,
ContextUpdates: BuildGreetingContextUpdates(route, presence.PrimaryPersonId, true));
}
private static string BuildReactiveGreetingReply(
string greetingIntent,
string? displayName,
DateTimeOffset? referenceLocalTime)
{
var namePrefix = string.IsNullOrWhiteSpace(displayName)
? string.Empty
: $", {displayName}";
return greetingIntent switch
{
"good_morning" => $"Good morning{namePrefix}. It is great to see you.",
"good_afternoon" => $"Good afternoon{namePrefix}. I am glad you are here.",
"good_evening" => $"Good evening{namePrefix}. It is nice to have you back.",
"good_night" => $"Good night{namePrefix}. Sleep well.",
"welcome_back" => string.IsNullOrWhiteSpace(displayName)
? $"Welcome back. {ResolveTimeOfDayGreetingPrefix(referenceLocalTime)}."
: $"Welcome back, {displayName}. {ResolveTimeOfDayGreetingPrefix(referenceLocalTime)}.",
_ => $"Hello{namePrefix}. It is nice to see you."
};
}
private string? ResolvePreferredGreetingName(TurnContext turn, GreetingPresenceProfile presence)
{
var rememberedName = personalMemoryStore.GetName(ResolveTenantScope(turn, presence.PrimaryPersonId));
if (!string.IsNullOrWhiteSpace(rememberedName)) return ToDisplayName(rememberedName);
var tenantRememberedName = personalMemoryStore.GetName(ResolveTenantScope(turn));
if (!string.IsNullOrWhiteSpace(tenantRememberedName)) return ToDisplayName(tenantRememberedName);
var primaryPersonId = presence.PrimaryPersonId;
if (CanUseLoopFirstNameFallback(presence) &&
!string.IsNullOrWhiteSpace(primaryPersonId) &&
presence.LoopUserFirstNames.TryGetValue(primaryPersonId, out var firstName) &&
!string.IsNullOrWhiteSpace(firstName))
return ToDisplayName(firstName);
return null;
}
private static bool CanUseLoopFirstNameFallback(GreetingPresenceProfile presence)
{
if (string.IsNullOrWhiteSpace(presence.PrimaryPersonId)) return false;
if (presence.PeoplePresentIds.Count > 1) return false;
return true;
}
private static string ToDisplayName(string value)
{
var trimmed = value.Trim();
return string.IsNullOrWhiteSpace(trimmed)
? string.Empty
: CultureInfo.InvariantCulture.TextInfo.ToTitleCase(trimmed);
}
private bool ShouldHandleProactiveGreetingTrigger(
TurnContext turn,
string? triggerSource,
GreetingPresenceProfile presence)
{
if (string.Equals(triggerSource, "SURPRISE", StringComparison.OrdinalIgnoreCase)) return false;
if (!presence.HasKnownIdentity) return false;
var lastGreetingUtc = ReadGreetingHistoryLastGreetedUtc(turn, presence);
return !lastGreetingUtc.HasValue || DateTimeOffset.UtcNow - lastGreetingUtc.Value >= ProactiveGreetingCooldown;
}
private DateTimeOffset? ReadGreetingHistoryLastGreetedUtc(TurnContext turn, GreetingPresenceProfile presence)
{
var greetingHistory = ResolveGreetingHistoryRecord(turn, presence);
if (greetingHistory is not null && greetingHistory.LastGreetedUtc.HasValue)
return greetingHistory.LastGreetedUtc;
return ReadTimestampAttribute(turn, LastProactiveGreetingUtcMetadataKey);
}
private GreetingPresenceRecord? ResolveGreetingHistoryRecord(TurnContext turn, GreetingPresenceProfile presence)
{
var historyIdentity = ResolveGreetingHistoryIdentity(presence);
if (string.IsNullOrWhiteSpace(historyIdentity) || cloudStateStore is null) return null;
var loopId = ReadTenantAttribute(turn, "loopId") ?? "openjibo-default-loop";
return cloudStateStore.GetGreetingPresences(loopId)
.FirstOrDefault(record => record.PersonId.Equals(historyIdentity, StringComparison.OrdinalIgnoreCase));
}
private static string? ResolveGreetingHistoryIdentity(GreetingPresenceProfile presence)
{
if (!string.IsNullOrWhiteSpace(presence.PrimaryPersonId)) return presence.PrimaryPersonId;
return !string.IsNullOrWhiteSpace(presence.SpeakerId) ? presence.SpeakerId : null;
}
private static DateTimeOffset? ReadTimestampAttribute(TurnContext turn, string key)
{
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return null;
return DateTimeOffset.TryParse(
value.ToString(),
CultureInfo.InvariantCulture,
DateTimeStyles.RoundtripKind,
out var parsed)
? parsed
: null;
}
private static IDictionary<string, object?> BuildGreetingContextUpdates(string route, string? speakerId,
bool proactive)
{
var updates = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
[ChitchatStateMachine.StateMetadataKey] = "complete",
[ChitchatStateMachine.RouteMetadataKey] = "ScriptedResponse",
[ChitchatStateMachine.EmotionMetadataKey] = string.Empty,
[GreetingRouteMetadataKey] = route,
[GreetingSpeakerMetadataKey] = speakerId ?? string.Empty,
[proactive ? LastProactiveGreetingUtcMetadataKey : LastReactiveGreetingUtcMetadataKey] = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture)
};
return updates;
}
private void RecordGreetingPresence(
TurnContext turn,
GreetingPresenceProfile presence,
string route,
string intentName,
string? preferredName,
bool proactive)
{
if (cloudStateStore is null) return;
var identityId = ResolveGreetingHistoryIdentity(presence);
if (string.IsNullOrWhiteSpace(identityId)) return;
var now = DateTimeOffset.UtcNow;
var tenantScope = ResolveTenantScope(turn, identityId);
cloudStateStore.UpsertGreetingPresence(new GreetingPresenceRecord
{
AccountId = tenantScope.AccountId,
LoopId = tenantScope.LoopId,
PersonId = identityId,
SpeakerId = presence.SpeakerId,
PreferredName = preferredName,
LastSeenUtc = now,
LastGreetedUtc = now,
LastGreetingRoute = route,
LastGreetingIntent = intentName
});
}
private sealed record SpecialGreetingPrefix(string Route, string IntentName, string Prefix);
private static string ResolveTimeOfDayGreetingPrefix(DateTimeOffset? referenceLocalTime)
{
var hour = (referenceLocalTime ?? DateTimeOffset.UtcNow).Hour;
return hour switch
{
>= 5 and < 12 => "Good morning",
>= 12 and < 17 => "Good afternoon",
_ => "Good evening"
};
}
private string BuildProactiveGreetingReply(
TurnContext turn,
GreetingPresenceProfile presence,
string? displayName,
DateTimeOffset? referenceLocalTime)
{
var greetingHistory = ResolveGreetingHistoryRecord(turn, presence);
var greetingPrefix = ResolveProactiveGreetingPrefix(referenceLocalTime, greetingHistory);
if (string.Equals(greetingPrefix, "Welcome back", StringComparison.OrdinalIgnoreCase))
return string.IsNullOrWhiteSpace(displayName)
? "Welcome back. I am glad to see you again."
: $"Welcome back, {displayName}. I am glad to see you again.";
return string.IsNullOrWhiteSpace(displayName)
? $"{greetingPrefix}. I am glad to see you."
: $"{greetingPrefix}, {displayName}. It is great to see you.";
}
private static string ResolveProactiveGreetingPrefix(
DateTimeOffset? referenceLocalTime,
GreetingPresenceRecord? greetingHistory)
{
var hour = (referenceLocalTime ?? DateTimeOffset.UtcNow).Hour;
var isMorning = hour >= 5 && hour < 12;
var recentGreeting = greetingHistory?.LastGreetedUtc is not null &&
DateTimeOffset.UtcNow - greetingHistory.LastGreetedUtc.Value < TimeSpan.FromHours(8);
if (recentGreeting) return "Welcome back";
return isMorning ? "Good morning" : ResolveTimeOfDayGreetingPrefix(referenceLocalTime);
}
private SpecialGreetingPrefix? ResolveSpecialGreetingPrefix(
TurnContext turn,
GreetingPresenceProfile presence,
DateTimeOffset? referenceLocalTime)
{
var today = DateOnly.FromDateTime((referenceLocalTime ?? DateTimeOffset.UtcNow).Date);
var birthday = ResolveBirthdayGreeting(turn, presence, today);
if (birthday is not null) return birthday;
return ResolveHolidayGreeting(turn, today);
}
private SpecialGreetingPrefix? ResolveBirthdayGreeting(
TurnContext turn,
GreetingPresenceProfile presence,
DateOnly today)
{
var identityScope = !string.IsNullOrWhiteSpace(presence.PrimaryPersonId)
? ResolveTenantScope(turn, presence.PrimaryPersonId)
: ResolveTenantScope(turn);
var birthdayText = personalMemoryStore.GetBirthday(identityScope) ??
personalMemoryStore.GetBirthday(ResolveTenantScope(turn));
if (string.IsNullOrWhiteSpace(birthdayText)) return null;
var birthdayDate = TryParseBirthdayDate(birthdayText);
if (birthdayDate is null) return null;
return birthdayDate.Value.Month == today.Month && birthdayDate.Value.Day == today.Day
? new SpecialGreetingPrefix("ProactiveBirthdayGreeting", "proactive_birthday_greeting",
"Happy birthday")
: null;
}
private SpecialGreetingPrefix? ResolveHolidayGreeting(TurnContext turn, DateOnly today)
{
if (cloudStateStore is null) return null;
var loopId = ReadTenantAttribute(turn, "loopId") ?? "openjibo-default-loop";
var holiday = cloudStateStore.GetHolidays(loopId)
.FirstOrDefault(item =>
item.IsEnabled &&
item.Category != "birthday" &&
item.Date.Month == today.Month &&
item.Date.Day == today.Day);
return holiday is null
? null
: new SpecialGreetingPrefix("ProactiveHolidayGreeting", "proactive_holiday_greeting",
"Happy holidays");
}
private JiboInteractionDecision BuildPizzaDecision()
{
return BuildPizzaAnimationDecision("pizza", "One pizza, coming right up.");
}
private JiboInteractionDecision BuildPizzaAnimationDecision(string intentName, string replyText)
{
var prompt = randomizer.Choose(PizzaMimPrompts);
return new JiboInteractionDecision(
intentName,
replyText,
"chitchat-skill",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["esml"] = prompt.Esml,
["mim_id"] = "RA_JBO_MakePizza",
["mim_type"] = "announcement",
["prompt_id"] = prompt.PromptId,
["prompt_sub_category"] = "AN"
});
}
private JiboInteractionDecision BuildProactivePizzaDayDecision(DateTimeOffset? referenceLocalTime)
{
var referenceDate = (referenceLocalTime ?? DateTimeOffset.UtcNow).Date;
return BuildPizzaAnimationDecision(
"proactive_pizza_day",
$"Happy National Pizza Day for {referenceDate.ToString("MMMM d", CultureInfo.InvariantCulture)}. One pizza, coming right up.");
}
private JiboInteractionDecision BuildProactivePizzaPreferenceDecision()
{
return BuildPizzaAnimationDecision(
"proactive_pizza_preference",
"You mentioned pizza is a favorite, so I thought we should make one.");
}
private static JiboInteractionDecision BuildProactivePizzaFactOfferDecision()
{
var listenContexts = new[] { "shared/yes_no" };
return new JiboInteractionDecision(
"proactive_offer_pizza_fact",
"Do you want to hear a fun pizza fact?",
"chitchat-skill",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["mim_id"] = "runtime-chat",
["mim_type"] = "question",
["prompt_id"] = "RUNTIME_PROMPT",
["prompt_sub_category"] = "Q",
["listen_contexts"] = listenContexts
});
}
private static JiboInteractionDecision BuildProactivePizzaFactDecision()
{
return new JiboInteractionDecision(
"proactive_pizza_fact",
"Americans consume about 100 acres of pizza every day, roughly 350 slices per second. That's a lot of pizza.");
}
private JiboInteractionDecision BuildProactiveFunFactDecision(JiboExperienceCatalog catalog)
{
var categories = new List<ProactiveFactCategory>();
AddProactiveFactCategory(categories, "fun_fact", catalog.FunFacts);
AddProactiveFactCategory(categories, "robot_fact", catalog.RobotFacts);
AddProactiveFactCategory(categories, "human_fact", catalog.HumanFacts);
if (categories.Count == 0)
return new JiboInteractionDecision("proactive_fun_fact", randomizer.Choose(catalog.SurpriseReplies));
var selectedCategory = randomizer.Choose(categories);
var fact = randomizer.Choose(selectedCategory.Replies);
return new JiboInteractionDecision(
"proactive_fun_fact",
fact,
"chitchat-skill",
new Dictionary<string, object?>
{
["mim_id"] = "runtime-fun-fact",
["mim_type"] = "announcement",
["prompt_id"] = "RUNTIME_FUN_FACT",
["replyType"] = "fun_fact",
["factCategory"] = selectedCategory.CategoryName
});
}
private static void AddProactiveFactCategory(
ICollection<ProactiveFactCategory> categories,
string categoryName,
IReadOnlyList<string> replies)
{
if (replies.Count == 0) return;
categories.Add(new ProactiveFactCategory(categoryName, replies));
}
private JiboInteractionDecision BuildProactiveJokeDecision(JiboExperienceCatalog catalog)
{
return new JiboInteractionDecision(
"proactive_joke",
randomizer.Choose(catalog.Jokes),
"@be/joke",
new Dictionary<string, object?>
{
["replyType"] = "joke"
});
}
private static JiboInteractionDecision BuildProactiveOfferDeclinedDecision()
{
return new JiboInteractionDecision(
"proactive_offer_declined",
"No problem. We can save the pizza fact for another time.");
}
private JiboInteractionDecision BuildWhatIsYourSignDecision()
{
var today = DateOnly.FromDateTime(DateTimeOffset.UtcNow.Date);
var birthday = OpenJiboCloudBuildInfo.PersonaBirthday;
var zodiac = DescribeZodiacSign(birthday);
var reply = birthday.Month == today.Month && birthday.Day == today.Day
? $"{zodiac}. Today is my birthday."
: $"{zodiac}. I was first powered up on {OpenJiboCloudBuildInfo.PersonaBirthdayWords}.";
return new JiboInteractionDecision(
"robot_what_is_your_sign",
reply,
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildHowManyPeopleDoYouKnowDecision(TurnContext turn)
{
var people = GetLoopPeople(turn);
var speaker = ResolvePreferredGreetingName(turn, ResolveGreetingPresenceProfile(turn));
var reply = people.Count switch
{
0 => "Well if we're talking about people in my Loop, I do not know anyone yet.",
1 when string.IsNullOrWhiteSpace(speaker) =>
"Well if we're talking about people in my Loop, I know 1 person.",
1 => $"Well there is 1 person in our Loop. And it's you {speaker}.",
_ when string.IsNullOrWhiteSpace(speaker) =>
$"Well if we're talking about people in my Loop, I know {people.Count} people.",
_ => $"Well there are {people.Count} people in our Loop."
};
return new JiboInteractionDecision(
"robot_how_many_people_do_you_know",
reply,
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildWhatIsTheLoopDecision(TurnContext turn)
{
var people = GetLoopPeople(turn);
var reply = people.Count == 0
? "The Loop is the people I know, and whose faces and voices I can learn to recognize. There can be up to 16 people in the Loop."
: $"The Loop is the group of people I know. They're the people whose voices and faces I can learn. Right now, my Loop is {JoinWithAnd(people.Select(person => person.DisplayName).ToArray())}.";
return new JiboInteractionDecision(
"robot_what_is_the_loop",
reply,
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private IReadOnlyList<PersonRecord> GetLoopPeople(TurnContext turn)
{
if (cloudStateStore is null) return [];
var loopId = ReadTenantAttribute(turn, "loopId") ?? "openjibo-default-loop";
return cloudStateStore.GetPeople()
.Where(person => string.Equals(person.LoopId, loopId, StringComparison.OrdinalIgnoreCase))
.OrderBy(person => person.IsPrimary ? 0 : 1)
.ThenBy(person => person.DisplayName, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string JoinWithAnd(IReadOnlyList<string> values)
{
if (values.Count == 0) return string.Empty;
if (values.Count == 1) return values[0];
if (values.Count == 2) return $"{values[0]} and {values[1]}";
return $"{string.Join(", ", values.Take(values.Count - 1))}, and {values[^1]}";
}
private static string DescribeZodiacSign(DateOnly birthday)
{
return (birthday.Month, birthday.Day) switch
{
(3, >= 21) or (4, <= 19) => "I'm Aries",
(4, >= 20) or (5, <= 20) => "I'm Taurus",
(5, >= 21) or (6, <= 20) => "I'm Gemini",
(6, >= 21) or (7, <= 22) => "I'm Cancer",
(7, >= 23) or (8, <= 22) => "I'm Leo",
(8, >= 23) or (9, <= 22) => "I'm Virgo",
(9, >= 23) or (10, <= 22) => "I'm Libra",
(10, >= 23) or (11, <= 21) => "I'm Scorpio",
(11, >= 22) or (12, <= 21) => "I'm Sagittarius",
(12, >= 22) or (1, <= 19) => "I'm Capricorn",
(1, >= 20) or (2, <= 18) => "I'm Aquarius",
_ => "I'm Pisces"
};
}
private string BuildGenericReply(JiboExperienceCatalog catalog, string transcript, string lowered)
{
if (string.IsNullOrWhiteSpace(transcript)) return "I am listening.";
if (lowered.Contains("good morning", StringComparison.Ordinal))
return "Good morning! It is nice to hear your voice.";
if (lowered.Contains("good afternoon", StringComparison.Ordinal))
return "Good afternoon. I am happy to be here.";
return lowered.Contains("good night", StringComparison.Ordinal)
? "Good night. Sleep tight."
: randomizer.Choose(catalog.GenericFallbackReplies)
.Replace("{transcript}", transcript, StringComparison.Ordinal);
}
private JiboInteractionDecision BuildScriptedPersonalityDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
intentName,
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedFavoriteAnimalDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.BuildScriptedFavoriteAnimalDecision(
catalog,
randomizer,
intentName,
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedFriendDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(catalog.FriendReplies, preferredSnippets),
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildScriptedBestFriendDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(catalog.BestFriendReplies, preferredSnippets),
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildScriptedSingDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(catalog.SingReplies, preferredSnippets),
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildScriptedHolidaySingDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(catalog.HolidaySingReplies, preferredSnippets),
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildScriptedGreetingDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.BuildScriptedGreetingDecision(
catalog,
randomizer,
intentName,
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedHolidayDecision(
IReadOnlyList<string> replies,
string intentName,
params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.BuildScriptedHolidayDecision(
replies,
randomizer,
intentName,
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedHolidayTrackerDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.BuildScriptedHolidayTrackerDecision(
catalog,
randomizer,
intentName,
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedSupportDecision(
IReadOnlyList<string> replies,
string intentName,
params string[] preferredSnippets)
{
var selected = SelectLegacyReply(replies, preferredSnippets);
if (string.IsNullOrWhiteSpace(selected))
selected = GetSupportFallbackReply(intentName);
return new JiboInteractionDecision(
intentName,
selected,
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildScriptedHolidayGreetingDecision(
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.BuildScriptedHolidayGreetingDecision(
catalog,
randomizer,
intentName,
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedHolidayTemplateDecision(
TurnContext turn,
GreetingPresenceProfile presence,
JiboExperienceCatalog catalog,
string intentName,
params string[] preferredSnippets)
{
var selected = ScriptedResponseDecisionBuilder.SelectLegacyReply(
catalog.HolidayReplies,
randomizer,
preferredSnippets);
return new JiboInteractionDecision(
intentName,
RenderHolidayTemplate(selected, turn, presence),
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private string SelectLegacyPersonalityReply(JiboExperienceCatalog catalog, params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.SelectLegacyPersonalityReply(catalog, randomizer, preferredSnippets);
}
private string SelectLegacyGreetingReply(JiboExperienceCatalog catalog, params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.SelectLegacyGreetingReply(catalog, randomizer, preferredSnippets);
}
private string SelectLegacyReply(IReadOnlyList<string> replies, params string[] preferredSnippets)
{
return ScriptedResponseDecisionBuilder.SelectLegacyReply(replies, randomizer, preferredSnippets);
}
private static string GetSupportFallbackReply(string intentName)
{
return intentName switch
{
"backup_help" => "That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com.",
"restore_backup" => "That sounds a little too complicated for me, I think your best bet is to get some guidance from Jibo Customer Care. Check the Help section of the Jibo App, or go to the website, support dot jibo dot com.",
"update_next" => "That's a good question. I think they've been coming every few weeks.",
"update_last" => "Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates.",
_ => string.Empty
};
}
private string RenderHolidayTemplate(string template, TurnContext turn, GreetingPresenceProfile presence)
{
var ownerName = ResolvePreferredGreetingName(turn, presence);
var speakerName = !string.IsNullOrWhiteSpace(ownerName) ? ownerName : "you";
return template
.Replace("${speaker}'s", $"{speakerName}'s", StringComparison.OrdinalIgnoreCase)
.Replace("${speaker}", speakerName, StringComparison.OrdinalIgnoreCase)
.Replace("${loop.owner}", string.IsNullOrWhiteSpace(ownerName) ? string.Empty : ownerName,
StringComparison.OrdinalIgnoreCase)
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
}
}

View File

@@ -0,0 +1,106 @@
using System.Linq;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private static readonly string[] PizzaPreferenceCategories =
[
"food",
"meal",
"dish",
"dinner",
"lunch",
"snack"
];
private JiboInteractionDecision BuildSurpriseDecision(
JiboExperienceCatalog catalog,
TurnContext turn,
DateTimeOffset? referenceLocalTime)
{
var tenantScope = ResolveTenantScope(turn);
var candidates = BuildProactivityCandidates(tenantScope, referenceLocalTime);
if (candidates.Count == 0)
return new JiboInteractionDecision("surprise", randomizer.Choose(catalog.SurpriseReplies));
var highestWeight = candidates.Max(static candidate => candidate.Weight);
var topCandidates = candidates
.Where(candidate => candidate.Weight == highestWeight)
.ToArray();
var selected = topCandidates.Length == 1
? topCandidates[0]
: randomizer.Choose(topCandidates);
return selected.IntentName switch
{
"proactive_pizza_day" => BuildProactivePizzaDayDecision(referenceLocalTime),
"proactive_pizza_preference" => BuildProactivePizzaPreferenceDecision(),
"proactive_offer_pizza_fact" => BuildProactivePizzaFactOfferDecision(),
"proactive_fun_fact" => BuildProactiveFunFactDecision(catalog),
"proactive_joke" => BuildProactiveJokeDecision(catalog),
_ => new JiboInteractionDecision("surprise", randomizer.Choose(catalog.SurpriseReplies))
};
}
private List<ProactivityCandidate> BuildProactivityCandidates(
PersonalMemoryTenantScope tenantScope,
DateTimeOffset? referenceLocalTime)
{
var candidates = new List<ProactivityCandidate>();
var referenceDate = (referenceLocalTime ?? DateTimeOffset.UtcNow).Date;
var pizzaSignal = ResolvePizzaSignal(tenantScope);
if (pizzaSignal.Affinity == PersonalAffinity.Dislike) return candidates;
if (referenceDate is { Month: 2, Day: 9 })
{
var holidayWeight = pizzaSignal.Affinity switch
{
PersonalAffinity.Love => 170,
PersonalAffinity.Like => 160,
_ => 150
};
candidates.Add(new ProactivityCandidate("proactive_pizza_day", holidayWeight));
}
if (pizzaSignal.Affinity is PersonalAffinity.Love or PersonalAffinity.Like)
{
var preferenceWeight = pizzaSignal.Affinity == PersonalAffinity.Love ? 140 : 120;
candidates.Add(new ProactivityCandidate("proactive_pizza_preference", preferenceWeight));
candidates.Add(new ProactivityCandidate("proactive_offer_pizza_fact", preferenceWeight - 5));
return candidates;
}
candidates.Add(new ProactivityCandidate("proactive_fun_fact", 90));
candidates.Add(new ProactivityCandidate("proactive_joke", 90));
candidates.Add(new ProactivityCandidate("proactive_offer_pizza_fact", 90));
return candidates;
}
private PizzaSignal ResolvePizzaSignal(PersonalMemoryTenantScope tenantScope)
{
var pizzaAffinity = personalMemoryStore.GetAffinity(tenantScope, "pizza");
if (pizzaAffinity is not null) return new PizzaSignal(pizzaAffinity);
var affinityMatch = personalMemoryStore.GetAffinities(tenantScope)
.Where(pair => pair.Key.Contains("pizza", StringComparison.OrdinalIgnoreCase))
.OrderByDescending(static pair =>
pair.Value == PersonalAffinity.Love ? 2 : pair.Value == PersonalAffinity.Like ? 1 : 0)
.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(affinityMatch.Key)) return new PizzaSignal(affinityMatch.Value);
foreach (var category in PizzaPreferenceCategories)
{
var preference = personalMemoryStore.GetPreference(tenantScope, category);
if (!string.IsNullOrWhiteSpace(preference) &&
preference.Contains("pizza", StringComparison.OrdinalIgnoreCase))
return new PizzaSignal(PersonalAffinity.Like);
}
return new PizzaSignal(null);
}
}

View File

@@ -0,0 +1,295 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private async Task<JiboInteractionDecision> BuildWeatherReportDecisionAsync(
TurnContext turn,
string transcript,
CancellationToken cancellationToken)
{
var referenceLocalTime = TryResolveReferenceLocalTime(turn);
var catalog = await contentCache.GetCatalogAsync(cancellationToken);
var normalizedTranscript = NormalizeCommandPhrase(transcript);
var locationQuery = TryResolveWeatherLocationQuery(transcript);
var weatherDate = ResolveWeatherDateEntity(turn, transcript, normalizedTranscript, referenceLocalTime);
var isRangeForecastRequest = IsRangeForecastRequest(normalizedTranscript);
var isOpenEndedForecastRequest = IsOpenEndedForecastRequest(
normalizedTranscript,
weatherDate,
isRangeForecastRequest,
locationQuery);
if (ShouldDefaultForecastToTomorrow(
normalizedTranscript,
weatherDate,
isRangeForecastRequest,
isOpenEndedForecastRequest))
weatherDate = new WeatherDateEntity("tomorrow", 1, "Tomorrow");
if (weatherReportProvider is null)
return new JiboInteractionDecision(
"weather",
ChooseWeatherServiceDownReply(catalog));
var weatherCoordinates = string.IsNullOrWhiteSpace(locationQuery)
? TryResolveWeatherCoordinates(turn)
: null;
var useCelsius = ShouldUseCelsius(turn, transcript);
var isNextWeekForecast = IsNextWeekForecastRequest(normalizedTranscript, isRangeForecastRequest);
var isThisWeekForecast = IsThisWeekForecastRequest(normalizedTranscript, isRangeForecastRequest);
if (isNextWeekForecast || isThisWeekForecast || isOpenEndedForecastRequest)
{
const int rangeStartOffset = 1;
var rangeEndOffset = isThisWeekForecast
? ResolveThisWeekForecastEndOffset(referenceLocalTime)
: MaxWeatherForecastDayOffset;
var weeklySnapshots = new List<(int DayOffset, WeatherReportSnapshot Snapshot)>();
for (var offset = rangeStartOffset; offset <= rangeEndOffset; offset += 1)
{
WeatherReportSnapshot? weeklySnapshot;
try
{
weeklySnapshot = await weatherReportProvider.GetReportAsync(
new WeatherReportRequest(
locationQuery,
weatherCoordinates?.Latitude,
weatherCoordinates?.Longitude,
offset == 1,
useCelsius,
offset),
cancellationToken);
}
catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
weeklySnapshot = null;
}
if (weeklySnapshot is not null) weeklySnapshots.Add((offset, weeklySnapshot));
}
if (weeklySnapshots.Count == 0)
return new JiboInteractionDecision(
"weather",
"I couldn't fetch the weather right now. Please try again.");
var weeklySegments = BuildWeeklyForecastCardSegments(weeklySnapshots, referenceLocalTime);
var weeklySpokenReply = BuildWeeklyForecastSpokenReply(
weeklySegments,
weeklySnapshots[0].Snapshot.LocationName,
weeklySnapshots[0].Snapshot.UseCelsius,
isThisWeekForecast);
var weeklyWeatherPayload = BuildWeeklyWeatherSkillPayload(
weeklySpokenReply,
weeklySnapshots[0].Snapshot,
weeklySegments,
referenceLocalTime);
AddWeatherRequestDiagnostics(
weeklyWeatherPayload,
transcript,
normalizedTranscript,
locationQuery,
weatherDate,
isRangeForecastRequest,
isThisWeekForecast,
isNextWeekForecast);
return new JiboInteractionDecision(
"weather",
weeklySpokenReply,
"chitchat-skill",
weeklyWeatherPayload);
}
if (weatherDate.ForecastDayOffset > MaxWeatherForecastDayOffset)
return new JiboInteractionDecision(
"weather",
$"I can forecast up to {MaxWeatherForecastDayOffset} days ahead. Try tomorrow or another day this week.");
WeatherReportSnapshot? snapshot;
try
{
snapshot = await weatherReportProvider.GetReportAsync(
new WeatherReportRequest(
locationQuery,
weatherCoordinates?.Latitude,
weatherCoordinates?.Longitude,
string.Equals(weatherDate.DateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase),
useCelsius,
weatherDate.ForecastDayOffset),
cancellationToken);
}
catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
snapshot = null;
}
if (snapshot is null)
return new JiboInteractionDecision(
"weather",
ChooseWeatherServiceDownReply(catalog));
var spokenReply = BuildWeatherSpokenReply(snapshot, weatherDate, catalog);
var weatherPayload = BuildWeatherSkillPayload(spokenReply, snapshot, referenceLocalTime);
AddWeatherRequestDiagnostics(
weatherPayload,
transcript,
normalizedTranscript,
locationQuery,
weatherDate,
isRangeForecastRequest,
isThisWeekForecast,
isNextWeekForecast);
return new JiboInteractionDecision(
"weather",
spokenReply,
"chitchat-skill",
weatherPayload);
}
private async Task<JiboInteractionDecision> BuildCommuteReportDecisionAsync(
TurnContext turn,
CancellationToken cancellationToken)
{
var catalog = await contentCache.GetCatalogAsync(cancellationToken);
if (commuteReportProvider is null)
return new JiboInteractionDecision(
"commute",
ChooseCommuteServiceDownReply(catalog));
CommuteReportSnapshot? snapshot;
try
{
snapshot = await commuteReportProvider.GetReportAsync(turn, cancellationToken);
}
catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
snapshot = null;
}
if (snapshot is null)
return new JiboInteractionDecision(
"commute",
ChooseCommuteServiceDownReply(catalog));
if (snapshot.RequiresSetup)
return new JiboInteractionDecision(
"commute_setup",
ChooseCommuteAppSetupReply(catalog));
return new JiboInteractionDecision(
"commute",
BuildCommuteSpokenReply(snapshot, catalog));
}
private async Task<JiboInteractionDecision> BuildCalendarReportDecisionAsync(
TurnContext turn,
CancellationToken cancellationToken)
{
var catalog = await contentCache.GetCatalogAsync(cancellationToken);
if (calendarReportProvider is null)
return new JiboInteractionDecision(
"calendar",
ChooseCalendarServiceDownReply(catalog));
CalendarReportSnapshot? snapshot;
try
{
snapshot = await calendarReportProvider.GetReportAsync(turn, cancellationToken);
}
catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
snapshot = null;
}
if (snapshot is null)
return new JiboInteractionDecision(
"calendar",
ChooseCalendarServiceDownReply(catalog));
return new JiboInteractionDecision(
"calendar",
BuildCalendarSpokenReply(snapshot, catalog));
}
private async Task<JiboInteractionDecision> BuildNewsDecisionAsync(
TurnContext turn,
string transcript,
JiboExperienceCatalog catalog,
CancellationToken cancellationToken)
{
var preferredCategories = ResolvePreferredNewsCategories(turn, transcript);
var requestedHeadlineCount = MaxNewsHeadlines;
if (newsBriefingProvider is not null)
try
{
var snapshot = await newsBriefingProvider.GetBriefingAsync(
new NewsBriefingRequest(preferredCategories, requestedHeadlineCount),
cancellationToken);
if (snapshot?.Headlines.Count > 0)
return BuildProviderNewsDecision(
snapshot,
catalog,
preferredCategories,
requestedHeadlineCount);
var providerStatus = ResolveNewsProviderStatus(snapshot);
var providerMessage = snapshot?.ProviderMessage;
var providerEndpoint = snapshot?.ProviderEndpoint;
var providerHttpStatusCode = snapshot?.ProviderHttpStatusCode;
var providerErrorCode = snapshot?.ProviderErrorCode;
var fallbackBriefingWhenEmpty = randomizer.Choose(catalog.NewsBriefings);
return BuildNewsDecision(
fallbackBriefingWhenEmpty,
null,
preferredCategories.Count > 0 ? preferredCategories : null,
null,
BuildNewsProviderDiagnostics(
providerStatus,
preferredCategories,
requestedHeadlineCount,
snapshot?.Headlines.Count ?? 0,
providerMessage,
providerHttpStatusCode,
providerEndpoint,
providerErrorCode));
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch
{
// Provider failures should never block baseline news behavior.
var fallbackBriefingOnError = randomizer.Choose(catalog.NewsBriefings);
return BuildNewsDecision(
fallbackBriefingOnError,
null,
preferredCategories.Count > 0 ? preferredCategories : null,
null,
BuildNewsProviderDiagnostics(
"provider_exception",
preferredCategories,
requestedHeadlineCount));
}
var fallbackBriefing = randomizer.Choose(catalog.NewsBriefings);
return BuildNewsDecision(
fallbackBriefing,
null,
preferredCategories.Count > 0 ? preferredCategories : null,
null,
BuildNewsProviderDiagnostics(
"provider_unavailable",
preferredCategories,
requestedHeadlineCount));
}
}

View File

@@ -0,0 +1,687 @@
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
namespace Jibo.Cloud.Application.Services;
public sealed partial class JiboInteractionService
{
private static string EscapeForEsml(string value)
{
return value
.Replace("&", "&amp;", StringComparison.Ordinal)
.Replace("<", "&lt;", StringComparison.Ordinal)
.Replace(">", "&gt;", StringComparison.Ordinal)
.Replace("\"", "&quot;", StringComparison.Ordinal);
}
private static string BuildWeatherSpokenReply(
WeatherReportSnapshot snapshot,
WeatherDateEntity weatherDate,
JiboExperienceCatalog catalog)
{
var unit = snapshot.UseCelsius ? "Celsius" : "Fahrenheit";
var summary = string.IsNullOrWhiteSpace(snapshot.Summary)
? "partly cloudy"
: snapshot.Summary.Trim().TrimEnd('.');
var location = string.IsNullOrWhiteSpace(snapshot.LocationName)
? "your area"
: NormalizeLocationForSpeech(snapshot.LocationName);
if (weatherDate.ForecastDayOffset > 0)
{
if (weatherDate.ForecastDayOffset != 1)
{
var highText = snapshot.HighTemperature is null
? null
: $"a high near {snapshot.HighTemperature.Value} degrees {unit}";
var lowText = snapshot.LowTemperature is null
? null
: $"a low around {snapshot.LowTemperature.Value} degrees {unit}";
var tempRange = highText is null && lowText is null
? string.Empty
: highText is not null && lowText is not null
? $" with {highText} and {lowText}"
: $" with {highText ?? lowText}";
var forecastLeadIn = string.IsNullOrWhiteSpace(weatherDate.ForecastLeadIn)
? "Tomorrow"
: weatherDate.ForecastLeadIn;
return $"Let's look at the weather. {forecastLeadIn} in {location}, it looks {summary}{tempRange}.";
}
var highValue = snapshot.HighTemperature ?? snapshot.Temperature;
var lowValue = snapshot.LowTemperature ?? snapshot.Temperature;
var introTemplate = ChooseWeatherTemplate(
catalog.WeatherTomorrowIntroReplies,
"Let's look at the weather.");
var highLowTemplate = ChooseWeatherTemplate(
catalog.WeatherTomorrowHighLowReplies,
"Tomorrow's high will be ${skill.weather.tomorrow.highTemp} and the low will be ${skill.weather.tomorrow.lowTemp}.");
var intro = RenderWeatherTemplate(
introTemplate,
location,
summary,
highValue,
lowValue,
unit,
weatherDate.ForecastLeadIn ?? string.Empty);
var highLow = RenderWeatherTemplate(
highLowTemplate,
location,
summary,
highValue,
lowValue,
unit,
weatherDate.ForecastLeadIn ?? string.Empty);
var forecastSentenceLeadIn = string.IsNullOrWhiteSpace(weatherDate.ForecastLeadIn)
? "Tomorrow"
: weatherDate.ForecastLeadIn;
return $"{intro} {forecastSentenceLeadIn} in {location}, it looks {summary}. {highLow}";
}
var currentIntro = RenderWeatherTemplate(
ChooseWeatherTemplate(catalog.WeatherIntroReplies, "For your weather."),
location,
summary,
snapshot.Temperature,
snapshot.Temperature,
unit,
string.Empty);
var currentHighLow = RenderWeatherTemplate(
ChooseWeatherTemplate(
catalog.WeatherTodayHighLowReplies,
"Today's high is ${skill.weather.today.highTemp}, and the low is ${skill.weather.today.lowTemp}."),
location,
summary,
snapshot.HighTemperature ?? snapshot.Temperature,
snapshot.LowTemperature ?? snapshot.Temperature,
unit,
string.Empty);
return
$"{currentIntro} In {location}, it's {summary} and {snapshot.Temperature} degrees {unit}. {currentHighLow}";
}
private static string BuildCommuteSpokenReply(
CommuteReportSnapshot snapshot,
JiboExperienceCatalog catalog)
{
var duration = snapshot.DurationMinutes;
var durationText = duration <= 1 ? "1 minute" : $"{duration} minutes";
var minutesLeft = snapshot.MinutesUntilWork;
var minutesLeftText = minutesLeft <= 1 ? "1 minute" : $"{Math.Abs(minutesLeft)} minutes";
var mode = string.IsNullOrWhiteSpace(snapshot.Mode) ? "driving" : snapshot.Mode.Trim();
var template = ChooseCommuteTemplate(snapshot, catalog, mode);
var reply = RenderCommuteTemplate(template, durationText, minutesLeftText);
if (minutesLeft is > 0 and < 30)
{
var minutesTemplate = ChooseShortestTemplate(catalog.CommuteMinutesLeftReplies)
?? "That's in about ${skill.commute.minsLeft} minutes.";
reply = $"{reply} {RenderCommuteTemplate(minutesTemplate, durationText, minutesLeftText)}";
}
if (minutesLeft is <= 0 or >= 120)
return reply.Replace(" ", " ", StringComparison.Ordinal).Trim();
var departTemplate = ChooseCommuteDepartTimeTemplate(snapshot, catalog, mode);
if (!string.IsNullOrWhiteSpace(departTemplate))
reply = $"{reply} {RenderCommuteTemplate(departTemplate, durationText, minutesLeftText)}";
return reply.Replace(" ", " ", StringComparison.Ordinal).Trim();
}
private string ChooseCommuteAppSetupReply(JiboExperienceCatalog catalog)
{
return SelectLegacyReply(
catalog.CommuteAppSetupReplies, "I need your commute settings before I can give you a commute report.");
}
private static string ChooseCommuteTemplate(
CommuteReportSnapshot snapshot,
JiboExperienceCatalog catalog,
string mode)
{
var minutesUntilWork = snapshot.MinutesUntilWork;
var extraMinutes = Math.Max(0, snapshot.ExtraMinutes);
var isLate = minutesUntilWork <= 0;
var isHurry = minutesUntilWork is > 0 and <= 10;
var isNormal = !isLate && !isHurry;
var isFarAway = minutesUntilWork is > 120 or < -30;
var hasTrafficSeverity = minutesUntilWork > 0;
var isTerrible = hasTrafficSeverity && extraMinutes >= 15;
var isPoor = hasTrafficSeverity && extraMinutes >= 5;
var loweredMode = mode.Trim().ToLowerInvariant();
IReadOnlyList<string> candidates = loweredMode switch
{
"walking" when isHurry => catalog.CommuteTransportHurryReplies,
"walking" when isLate => catalog.CommuteTransportLateReplies,
"walking" => catalog.CommuteTransportNormalReplies,
"transit" when isHurry => catalog.CommuteTransportHurryReplies,
"transit" when isLate => catalog.CommuteTransportLateReplies,
"transit" => catalog.CommuteTransportNormalReplies,
"bicycling" when isHurry => catalog.CommuteDriveHurryReplies,
"bicycling" when isLate => catalog.CommuteDriveLateReplies,
"bicycling" => catalog.CommuteDriveNormalReplies,
_ when isFarAway => catalog.CommuteNowReplies,
_ when isTerrible => catalog.CommuteDriveTerribleReplies,
_ when isPoor => catalog.CommuteDrivePoorReplies,
_ when isHurry => catalog.CommuteDriveHurryReplies,
_ when isLate => catalog.CommuteDriveLateReplies,
_ when isNormal => catalog.CommuteDriveNormalReplies,
_ => catalog.CommuteNowReplies
};
if (candidates.Count == 0)
return "For your commute, it should take about ${skill.commute.durationMins} minutes.";
var selected = ChooseShortestTemplate(candidates);
return string.IsNullOrWhiteSpace(selected)
? "For your commute, it should take about ${skill.commute.durationMins} minutes."
: selected!;
}
private static string ChooseCommuteDepartTimeTemplate(
CommuteReportSnapshot snapshot,
JiboExperienceCatalog catalog,
string mode)
{
var loweredMode = mode.Trim().ToLowerInvariant();
var templates = snapshot.MinutesUntilWork <= 0
? catalog.CommuteDepartTimeNotNormalReplies
: catalog.CommuteDepartTimeNormalReplies;
if (templates.Count == 0) return string.Empty;
var selected = ChooseShortestTemplate(templates);
if (!string.IsNullOrWhiteSpace(selected)) return selected!;
return loweredMode switch
{
"walking" => "If you leave at the usual time, that should work out fine.",
"transit" => "If you leave at the usual time, that should work out fine.",
_ => "If you leave at the usual time, that should work out fine."
};
}
private static string RenderCommuteTemplate(string template, string durationText, string minutesLeftText)
{
return template
.Replace("${skill.commute.durationMins}", durationText, StringComparison.OrdinalIgnoreCase)
.Replace("${skill.commute.minsLeft}", minutesLeftText, StringComparison.OrdinalIgnoreCase)
.Replace("${speaker}", string.Empty, StringComparison.OrdinalIgnoreCase)
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
}
private static string? ChooseShortestTemplate(IEnumerable<string> templates)
{
return templates
.Where(static template => !string.IsNullOrWhiteSpace(template))
.OrderBy(static template => template.Length)
.FirstOrDefault();
}
private static string BuildWeeklyForecastSpokenReply(
IReadOnlyList<WeatherForecastCardSegment> segments,
string? locationName,
bool useCelsius,
bool isThisWeekForecast)
{
if (segments.Count == 0) return "I couldn't build a forecast right now.";
var location = string.IsNullOrWhiteSpace(locationName)
? "your area"
: NormalizeLocationForSpeech(locationName);
var unit = useCelsius ? "Celsius" : "Fahrenheit";
var leadIn = isThisWeekForecast
? $"Here's the rest of this week's forecast in {location}."
: $"I can share the next five-day forecast in {location}.";
return
$"{leadIn} {string.Join(" ", segments.Select(static segment => segment.SpokenLine))} Temperatures are in {unit}.";
}
private static IReadOnlyList<WeatherForecastCardSegment> BuildWeeklyForecastCardSegments(
IReadOnlyList<(int DayOffset, WeatherReportSnapshot Snapshot)> snapshots,
DateTimeOffset? referenceLocalTime)
{
if (snapshots.Count == 0) return [];
var resolvedReference = referenceLocalTime ?? DateTimeOffset.UtcNow;
var referenceDate = resolvedReference.Date;
return [.. snapshots
.OrderBy(static item => item.DayOffset)
.Take(MaxWeatherForecastDayOffset)
.Select(item =>
{
var dayName = referenceDate.AddDays(item.DayOffset).ToString("dddd", CultureInfo.InvariantCulture);
var summary = string.IsNullOrWhiteSpace(item.Snapshot.Summary)
? "partly cloudy"
: item.Snapshot.Summary.Trim().TrimEnd('.');
var high = item.Snapshot.HighTemperature ?? item.Snapshot.Temperature;
var low = item.Snapshot.LowTemperature ?? item.Snapshot.Temperature;
var iconReference = new DateTimeOffset(
resolvedReference.Date.AddDays(item.DayOffset).AddHours(12),
resolvedReference.Offset);
var icon = ResolveWeatherAnimationIcon(item.Snapshot, iconReference);
var unit = item.Snapshot.UseCelsius ? "C" : "F";
var temperatureBand = ResolveWeatherTemperatureBand(high, item.Snapshot.UseCelsius);
var spokenLine = $"{dayName}: {summary}, high {high}, low {low}.";
return new WeatherForecastCardSegment(
dayName,
summary,
high,
low,
icon,
unit,
temperatureBand,
spokenLine);
})];
}
private static IDictionary<string, object?> BuildWeeklyWeatherSkillPayload(
string spokenReply,
WeatherReportSnapshot snapshot,
IReadOnlyList<WeatherForecastCardSegment> segments,
DateTimeOffset? referenceLocalTime)
{
var payload = BuildWeatherSkillPayload(spokenReply, snapshot, referenceLocalTime);
payload["weather_view_kind"] = "weatherWeekly";
payload["weather_view_mode"] = "forecast";
payload["weather_weekly_cards"] = segments
.Select(static segment => new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["weather_day"] = segment.DayName,
["weather_summary"] = segment.Summary,
["weather_icon"] = segment.Icon,
["weather_high"] = segment.High,
["weather_low"] = segment.Low,
["weather_unit"] = segment.Unit,
["weather_theme"] = segment.Theme,
["weather_spoken_line"] = segment.SpokenLine
})
.ToArray();
return payload;
}
private static void AddWeatherRequestDiagnostics(
IDictionary<string, object?> payload,
string transcript,
string normalizedTranscript,
string? locationQuery,
WeatherDateEntity weatherDate,
bool isRangeForecastRequest,
bool isThisWeekForecast,
bool isNextWeekForecast)
{
payload["weather_request_transcript"] = transcript;
payload["weather_request_normalized"] = normalizedTranscript;
payload["weather_request_location_query"] = locationQuery;
payload["weather_request_date_entity"] = weatherDate.DateEntity;
payload["weather_request_forecast_day_offset"] = weatherDate.ForecastDayOffset;
payload["weather_request_range"] = isRangeForecastRequest;
payload["weather_request_this_week"] = isThisWeekForecast;
payload["weather_request_next_week"] = isNextWeekForecast;
}
private static bool IsNextWeekForecastRequest(string normalizedTranscript, bool isRangeForecastRequest)
{
if (string.IsNullOrWhiteSpace(normalizedTranscript) || !isRangeForecastRequest) return false;
if (normalizedTranscript.Contains("next week", StringComparison.Ordinal)) return true;
if (!normalizedTranscript.Contains("next", StringComparison.Ordinal)) return false;
return normalizedTranscript.Contains("forecast next", StringComparison.Ordinal) ||
normalizedTranscript.Contains("forecast for next", StringComparison.Ordinal);
}
private static bool IsRangeForecastRequest(string normalizedTranscript)
{
if (string.IsNullOrWhiteSpace(normalizedTranscript)) return false;
if (normalizedTranscript.Contains("next week", StringComparison.Ordinal) ||
normalizedTranscript.Contains("this week", StringComparison.Ordinal) ||
normalizedTranscript.Contains("weekend", StringComparison.Ordinal))
return true;
return normalizedTranscript.Contains("forecast next", StringComparison.Ordinal) ||
normalizedTranscript.Contains("forecast for next", StringComparison.Ordinal);
}
private static bool IsThisWeekForecastRequest(string normalizedTranscript, bool isRangeForecastRequest)
{
return isRangeForecastRequest &&
!string.IsNullOrWhiteSpace(normalizedTranscript) &&
normalizedTranscript.Contains("this week", StringComparison.Ordinal) &&
!normalizedTranscript.Contains("weekend", StringComparison.Ordinal);
}
private static bool IsOpenEndedForecastRequest(
string normalizedTranscript,
WeatherDateEntity weatherDate,
bool isRangeForecastRequest,
string? locationQuery)
{
if (string.IsNullOrWhiteSpace(normalizedTranscript) ||
!string.IsNullOrWhiteSpace(locationQuery) ||
isRangeForecastRequest ||
weatherDate.ForecastDayOffset > 0 ||
!normalizedTranscript.Contains("forecast", StringComparison.Ordinal))
return false;
return !MatchesAny(
normalizedTranscript,
"today",
"today s",
"today's",
"tonight",
"right now",
"current weather",
"currently");
}
private static int ResolveThisWeekForecastEndOffset(DateTimeOffset? referenceLocalTime)
{
var resolvedReference = referenceLocalTime ?? DateTimeOffset.UtcNow;
var daysUntilSunday = ((int)DayOfWeek.Sunday - (int)resolvedReference.DayOfWeek + 7) % 7;
var endOffset = Math.Min(MaxWeatherForecastDayOffset, daysUntilSunday);
return Math.Max(1, endOffset);
}
private static bool ShouldDefaultForecastToTomorrow(
string normalizedTranscript,
WeatherDateEntity weatherDate,
bool isRangeForecastRequest,
bool isOpenEndedForecastRequest)
{
if (weatherDate.ForecastDayOffset > 0 ||
isOpenEndedForecastRequest ||
isRangeForecastRequest ||
string.IsNullOrWhiteSpace(normalizedTranscript) ||
!normalizedTranscript.Contains("forecast", StringComparison.Ordinal))
return false;
return !MatchesAny(
normalizedTranscript,
"today",
"today s",
"today's",
"tonight",
"right now",
"current weather",
"currently");
}
private static IDictionary<string, object?> BuildWeatherSkillPayload(
string spokenReply,
WeatherReportSnapshot snapshot,
DateTimeOffset? referenceLocalTime)
{
var weatherIcon = ResolveWeatherAnimationIcon(snapshot, referenceLocalTime);
var promptToken = ResolveWeatherPromptToken(weatherIcon);
var highTemperature = snapshot.HighTemperature ?? snapshot.Temperature;
var lowTemperature = snapshot.LowTemperature ?? snapshot.Temperature;
var temperatureUnit = snapshot.UseCelsius ? "C" : "F";
var temperatureBand = ResolveWeatherTemperatureBand(highTemperature, snapshot.UseCelsius);
return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "report-skill",
["cloudSkill"] = "weather",
["esml"] =
$"<speak><anim cat='weather' meta='{weatherIcon}' nonBlocking='true' /><break size='0.35'/><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeForEsml(spokenReply)}</es></speak>",
["mim_id"] = $"WeatherComment{promptToken}",
["mim_type"] = "announcement",
["prompt_id"] = $"WeatherComment{promptToken}_AN_13",
["prompt_sub_category"] = "AN",
["weather_view_enabled"] = true,
["weather_view_kind"] = "weatherHiLo",
["weather_view_mode"] = "current",
["weather_icon"] = weatherIcon,
["weather_summary"] = snapshot.Summary,
["weather_location"] = snapshot.LocationName,
["weather_high"] = highTemperature,
["weather_low"] = lowTemperature,
["weather_unit"] = temperatureUnit,
["weather_theme"] = temperatureBand
};
}
private static string ResolveWeatherAnimationIcon(
WeatherReportSnapshot snapshot,
DateTimeOffset? referenceLocalTime)
{
var isDaytime = (referenceLocalTime ?? DateTimeOffset.UtcNow).Hour is >= 6 and < 18;
var normalized = NormalizeCommandPhrase(
$"{snapshot.Condition ?? string.Empty} {snapshot.Summary ?? string.Empty}");
if (normalized.Contains("thunder", StringComparison.Ordinal) ||
normalized.Contains("drizzle", StringComparison.Ordinal) ||
normalized.Contains("rain", StringComparison.Ordinal))
return "rain";
if (normalized.Contains("snow", StringComparison.Ordinal)) return "snow";
if (normalized.Contains("sleet", StringComparison.Ordinal) ||
normalized.Contains("freezing rain", StringComparison.Ordinal) ||
normalized.Contains("ice", StringComparison.Ordinal))
return "sleet";
if (normalized.Contains("fog", StringComparison.Ordinal) ||
normalized.Contains("mist", StringComparison.Ordinal) ||
normalized.Contains("haze", StringComparison.Ordinal) ||
normalized.Contains("smoke", StringComparison.Ordinal))
return "fog";
if (normalized.Contains("wind", StringComparison.Ordinal)) return "wind";
if (normalized.Contains("partly cloudy", StringComparison.Ordinal) ||
normalized.Contains("scattered clouds", StringComparison.Ordinal) ||
normalized.Contains("few clouds", StringComparison.Ordinal))
return isDaytime ? "partly-cloudy-day" : "partly-cloudy-night";
if (normalized.Contains("cloud", StringComparison.Ordinal) ||
normalized.Contains("overcast", StringComparison.Ordinal))
return "cloudy";
if (normalized.Contains("clear", StringComparison.Ordinal) ||
normalized.Contains("sunny", StringComparison.Ordinal))
return isDaytime ? "clear-day" : "clear-night";
return isDaytime ? "clear-day" : "clear-night";
}
private static string ResolveWeatherPromptToken(string weatherIcon)
{
return weatherIcon switch
{
"clear-day" => "ClearDay",
"clear-night" => "ClearNight",
"rain" => "Rain",
"snow" => "Snow",
"sleet" => "Sleet",
"fog" => "Fog",
"wind" => "Wind",
"cloudy" => "Cloudy",
"partly-cloudy-day" => "PartlyCloudyDay",
"partly-cloudy-night" => "PartlyCloudyNight",
_ => "Cloudy"
};
}
private static string ResolveWeatherTemperatureBand(int highTemperature, bool useCelsius)
{
var hotThreshold = useCelsius ? 29 : 85;
var coldThreshold = useCelsius ? 4 : 40;
if (highTemperature > hotThreshold) return "Hot";
if (highTemperature < coldThreshold) return "Cold";
return "Normal";
}
private static string ChooseWeatherTemplate(IReadOnlyList<string> templates, string fallback)
{
var usableTemplates = templates.Where(static template => !string.IsNullOrWhiteSpace(template)).ToArray();
if (usableTemplates.Length == 0) return fallback;
return usableTemplates[0];
}
private static string RenderWeatherTemplate(
string template,
string location,
string summary,
int? highTemperature,
int? lowTemperature,
string unit,
string forecastLeadIn)
{
var rendered = template
.Replace("${skill.weather.today.highTemp}",
highTemperature?.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
StringComparison.OrdinalIgnoreCase)
.Replace("${skill.weather.today.lowTemp}",
lowTemperature?.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
StringComparison.OrdinalIgnoreCase)
.Replace("${skill.weather.tomorrow.highTemp}",
highTemperature?.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
StringComparison.OrdinalIgnoreCase)
.Replace("${skill.weather.tomorrow.lowTemp}",
lowTemperature?.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
StringComparison.OrdinalIgnoreCase)
.Replace("${skill.weather.summary}", summary, StringComparison.OrdinalIgnoreCase)
.Replace("${skill.weather.location}", location, StringComparison.OrdinalIgnoreCase)
.Replace("${skill.weather.prefix}",
string.IsNullOrWhiteSpace(forecastLeadIn) ? string.Empty : forecastLeadIn,
StringComparison.OrdinalIgnoreCase)
.Replace("{high}", highTemperature?.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
StringComparison.OrdinalIgnoreCase)
.Replace("{low}", lowTemperature?.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
StringComparison.OrdinalIgnoreCase)
.Replace("{unit}", unit, StringComparison.OrdinalIgnoreCase)
.Trim();
return rendered;
}
private static string ChooseWeatherServiceDownReply(JiboExperienceCatalog catalog)
{
var template = ChooseWeatherTemplate(
catalog.WeatherServiceDownReplies,
"I can't access weather info right now, sorry.");
return template.Trim();
}
private static string NormalizeLocationForSpeech(string text)
{
if (string.IsNullOrWhiteSpace(text)) return text;
return Regex.Replace(
text,
@"\b(?<token>[A-Z]{2,3})\b",
static match =>
{
var token = match.Groups["token"].Value;
if (!SpokenAbbreviationTokens.Contains(token)) return token;
return string.Join(".", token.ToCharArray()) + ".";
},
RegexOptions.CultureInvariant);
}
private static string ChooseCommuteServiceDownReply(JiboExperienceCatalog catalog)
{
var template = ChooseWeatherTemplate(
catalog.CommuteServiceDownReplies,
"Sorry, commute information isn't available right now.");
return template.Trim();
}
private string BuildCalendarSpokenReply(CalendarReportSnapshot snapshot, JiboExperienceCatalog catalog)
{
if (snapshot.EventSummaries.Count > 0 && snapshot.EventTimesOnAt.Count > 0)
{
var summary = snapshot.EventSummaries[0];
var time = snapshot.EventTimesOnAt[0];
var template = ChooseCalendarTemplate(
catalog.CalendarReplies,
"calendar summary",
"Your calendar says ${skill.calendar.eventSummaries.shift()}, ${skill.calendar.eventTimesOnAt.shift()}.");
if (template.Contains("${skill.calendar.eventSummaries.shift()}", StringComparison.OrdinalIgnoreCase) ||
template.Contains("${skill.calendar.eventTimesOnAt.shift()}", StringComparison.OrdinalIgnoreCase))
return template
.Replace("${skill.calendar.eventSummaries.shift()}", summary, StringComparison.OrdinalIgnoreCase)
.Replace("${skill.calendar.eventTimesOnAt.shift()}", time, StringComparison.OrdinalIgnoreCase)
.Replace("${speaker}", string.Empty, StringComparison.OrdinalIgnoreCase)
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
return $"Your calendar says {summary}, {time}.";
}
if (snapshot.TomorrowEventSummaries.Count > 0)
{
var template = ChooseCalendarTemplate(
catalog.CalendarReplies,
"calendar tomorrow",
"Looking at your calendar, there's nothing scheduled for the rest of the day today. Here's what's going on tomorrow.");
if (template.Contains("tomorrow", StringComparison.OrdinalIgnoreCase))
return template
.Replace("${speaker}", string.Empty, StringComparison.OrdinalIgnoreCase)
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
return
$"Looking at your calendar, there's nothing scheduled for the rest of the day today. Here's what's going on tomorrow: {snapshot.TomorrowEventSummaries[0]}.";
}
return ChooseCalendarNothingReply(catalog);
}
private static string ChooseCalendarTemplate(
IReadOnlyList<string> templates,
string mode,
string fallback)
{
if (templates.Count == 0) return fallback;
var loweredMode = mode.Trim().ToLowerInvariant();
var filtered = templates.Where(template =>
{
var lowered = template.ToLowerInvariant();
return loweredMode switch
{
"calendar summary" => lowered.Contains("event", StringComparison.OrdinalIgnoreCase) ||
lowered.Contains("summary", StringComparison.OrdinalIgnoreCase),
"calendar tomorrow" => lowered.Contains("tomorrow", StringComparison.OrdinalIgnoreCase),
_ => true
};
}).ToList();
var selected = filtered.Count > 0
? filtered.OrderBy(static template => template.Length).First()
: templates.OrderBy(static template => template.Length).FirstOrDefault();
return string.IsNullOrWhiteSpace(selected) ? fallback : selected;
}
private string ChooseCalendarNothingReply(JiboExperienceCatalog catalog)
{
return catalog.CalendarNothingTodayReplies.Count > 0
? randomizer.Choose(catalog.CalendarNothingTodayReplies)
: catalog.CalendarNothingReplies.Count > 0
? randomizer.Choose(catalog.CalendarNothingReplies)
: "Looking at your calendar, I don't see anything scheduled today.";
}
private string ChooseCalendarServiceDownReply(JiboExperienceCatalog catalog)
{
return catalog.CalendarServiceDownReplies.Count > 0
? randomizer.Choose(catalog.CalendarServiceDownReplies)
: "Looks like I can't access calendars right now. Sorry.";
}
}

View File

@@ -15,7 +15,8 @@ public sealed class JiboWebSocketService(
stateStore.OpenSession(envelope.Kind, null, envelope.Token, envelope.HostName, envelope.Path); stateStore.OpenSession(envelope.Kind, null, envelope.Token, envelope.HostName, envelope.Path);
} }
public async Task<IReadOnlyList<WebSocketReply>> HandleMessageAsync(WebSocketMessageEnvelope envelope, CancellationToken cancellationToken = default) public async Task<IReadOnlyList<WebSocketReply>> HandleMessageAsync(WebSocketMessageEnvelope envelope,
CancellationToken cancellationToken = default)
{ {
var session = GetOrCreateSession(envelope); var session = GetOrCreateSession(envelope);
session.LastSeenUtc = DateTimeOffset.UtcNow; session.LastSeenUtc = DateTimeOffset.UtcNow;
@@ -23,11 +24,12 @@ public sealed class JiboWebSocketService(
if (envelope.IsBinary) if (envelope.IsBinary)
{ {
var replies = await turnFinalizationService.HandleBinaryAudioAsync(session, envelope, cancellationToken); var replies = await turnFinalizationService.HandleBinaryAudioAsync(session, envelope, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "binary_audio_received", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "binary_audio_received",
{ new Dictionary<string, object?>
["bytes"] = envelope.Binary?.Length ?? 0, {
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session) ["bytes"] = envelope.Binary?.Length ?? 0,
}, cancellationToken); ["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
}, cancellationToken);
return replies; return replies;
} }
@@ -50,13 +52,14 @@ public sealed class JiboWebSocketService(
}) })
.ToArray(); .ToArray();
await telemetrySink.RecordTurnEventAsync(envelope, session, "late_listen_ignored", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "late_listen_ignored",
{ new Dictionary<string, object?>
["messageType"] = parsedType, {
["activeTransID"] = session.TurnState.TransId, ["messageType"] = parsedType,
["ignoredTransID"] = lateTransId, ["activeTransID"] = session.TurnState.TransId,
["replyCount"] = replies.Length ["ignoredTransID"] = lateTransId,
}, cancellationToken); ["replyCount"] = replies.Length
}, cancellationToken);
return replies; return replies;
} }
@@ -65,12 +68,13 @@ public sealed class JiboWebSocketService(
WebSocketTurnFinalizationService.TryRecoverStalePendingListen(session, out staleListenAgeMs)) WebSocketTurnFinalizationService.TryRecoverStalePendingListen(session, out staleListenAgeMs))
{ {
staleListenRecovered = true; staleListenRecovered = true;
await telemetrySink.RecordTurnEventAsync(envelope, session, "glsm_stale_listen_recovered", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "glsm_stale_listen_recovered",
{ new Dictionary<string, object?>
["staleAgeMs"] = staleListenAgeMs, {
["transID"] = session.TurnState.TransId, ["staleAgeMs"] = staleListenAgeMs,
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session) ["transID"] = session.TurnState.TransId,
}, cancellationToken); ["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
}, cancellationToken);
} }
WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text); WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
@@ -80,11 +84,12 @@ public sealed class JiboWebSocketService(
case "CONTEXT": case "CONTEXT":
{ {
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken); var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received",
{ new Dictionary<string, object?>
["transID"] = session.TurnState.TransId, {
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session) ["transID"] = session.TurnState.TransId,
}, cancellationToken); ["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
}, cancellationToken);
return replies; return replies;
} }
case "LISTEN": case "LISTEN":
@@ -92,29 +97,32 @@ public sealed class JiboWebSocketService(
var replies = containsInlineTurnPayload var replies = containsInlineTurnPayload
? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken) ? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
: WebSocketTurnFinalizationService.HandleListenSetup(session, envelope); : WebSocketTurnFinalizationService.HandleListenSetup(session, envelope);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed",
{ new Dictionary<string, object?>
["messageType"] = parsedType, {
["replyCount"] = replies.Count, ["messageType"] = parsedType,
["transcript"] = session.LastTranscript, ["replyCount"] = replies.Count,
["intent"] = session.LastIntent, ["transcript"] = session.LastTranscript,
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session), ["intent"] = session.LastIntent,
["staleListenRecovered"] = staleListenRecovered, ["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session),
["staleListenAgeMs"] = staleListenAgeMs ["staleListenRecovered"] = staleListenRecovered,
}, cancellationToken); ["staleListenAgeMs"] = staleListenAgeMs
}, cancellationToken);
return replies; return replies;
} }
case "CLIENT_NLU" or "CLIENT_ASR" or "TRIGGER": case "CLIENT_NLU" or "CLIENT_ASR" or "TRIGGER":
{ {
var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken); var replies =
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?> await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken);
{ await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed",
["messageType"] = parsedType, new Dictionary<string, object?>
["replyCount"] = replies.Count, {
["transcript"] = session.LastTranscript, ["messageType"] = parsedType,
["intent"] = session.LastIntent, ["replyCount"] = replies.Count,
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session) ["transcript"] = session.LastTranscript,
}, cancellationToken); ["intent"] = session.LastIntent,
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
}, cancellationToken);
return replies; return replies;
} }
default: default:
@@ -124,18 +132,13 @@ public sealed class JiboWebSocketService(
private static string ReadMessageType(string? text) private static string ReadMessageType(string? text)
{ {
if (string.IsNullOrWhiteSpace(text)) if (string.IsNullOrWhiteSpace(text)) return "UNKNOWN";
{
return "UNKNOWN";
}
try try
{ {
using var document = JsonDocument.Parse(text); using var document = JsonDocument.Parse(text);
if (document.RootElement.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String) if (document.RootElement.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String)
{
return type.GetString() ?? "UNKNOWN"; return type.GetString() ?? "UNKNOWN";
}
} }
catch catch
{ {
@@ -147,25 +150,18 @@ public sealed class JiboWebSocketService(
private static bool ContainsInlineTurnPayload(string? text) private static bool ContainsInlineTurnPayload(string? text)
{ {
if (string.IsNullOrWhiteSpace(text)) if (string.IsNullOrWhiteSpace(text)) return false;
{
return false;
}
try try
{ {
using var document = JsonDocument.Parse(text); using var document = JsonDocument.Parse(text);
if (!document.RootElement.TryGetProperty("data", out var data) || data.ValueKind != JsonValueKind.Object) if (!document.RootElement.TryGetProperty("data", out var data) ||
{ data.ValueKind != JsonValueKind.Object) return false;
return false;
}
if (data.TryGetProperty("text", out var transcript) && if (data.TryGetProperty("text", out var transcript) &&
transcript.ValueKind == JsonValueKind.String && transcript.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(transcript.GetString())) !string.IsNullOrWhiteSpace(transcript.GetString()))
{
return true; return true;
}
return data.TryGetProperty("asr", out var asr) && return data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object && asr.ValueKind == JsonValueKind.Object &&
@@ -186,10 +182,7 @@ public sealed class JiboWebSocketService(
var transId = session.TurnState.TransId ?? session.LastTransId ?? string.Empty; var transId = session.TurnState.TransId ?? session.LastTransId ?? string.Empty;
var rules = session.TurnState.ListenRules; var rules = session.TurnState.ListenRules;
if (string.IsNullOrWhiteSpace(text)) if (string.IsNullOrWhiteSpace(text)) return (transId, rules);
{
return (transId, rules);
}
try try
{ {
@@ -199,9 +192,7 @@ public sealed class JiboWebSocketService(
if (root.TryGetProperty("transID", out var transIdValue) && if (root.TryGetProperty("transID", out var transIdValue) &&
transIdValue.ValueKind == JsonValueKind.String && transIdValue.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(transIdValue.GetString())) !string.IsNullOrWhiteSpace(transIdValue.GetString()))
{
transId = transIdValue.GetString()!; transId = transIdValue.GetString()!;
}
if (root.TryGetProperty("data", out var data) && if (root.TryGetProperty("data", out var data) &&
data.ValueKind == JsonValueKind.Object && data.ValueKind == JsonValueKind.Object &&
@@ -214,10 +205,7 @@ public sealed class JiboWebSocketService(
.Where(static rule => !string.IsNullOrWhiteSpace(rule)) .Where(static rule => !string.IsNullOrWhiteSpace(rule))
.ToArray(); .ToArray();
if (parsedRules.Length > 0) if (parsedRules.Length > 0) rules = parsedRules;
{
rules = parsedRules;
}
} }
} }
catch catch

View File

@@ -5,5 +5,9 @@ namespace Jibo.Cloud.Application.Services;
public sealed class NullProtocolTelemetrySink : IProtocolTelemetrySink public sealed class NullProtocolTelemetrySink : IProtocolTelemetrySink
{ {
public Task RecordAsync(ProtocolEnvelope envelope, ProtocolDispatchResult result, CancellationToken cancellationToken = default) => Task.CompletedTask; public Task RecordAsync(ProtocolEnvelope envelope, ProtocolDispatchResult result,
CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
} }

View File

@@ -4,7 +4,14 @@ namespace Jibo.Cloud.Application.Services;
public sealed class NullTurnTelemetrySink : ITurnTelemetrySink public sealed class NullTurnTelemetrySink : ITurnTelemetrySink
{ {
public Task RecordTurnDiagnosticAsync(string category, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default) => Task.CompletedTask; public Task RecordTurnDiagnosticAsync(string category, IReadOnlyDictionary<string, object?> details,
CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
public Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default) => Task.CompletedTask; public Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
} }

View File

@@ -5,9 +5,33 @@ namespace Jibo.Cloud.Application.Services;
public sealed class NullWebSocketTelemetrySink : IWebSocketTelemetrySink public sealed class NullWebSocketTelemetrySink : IWebSocketTelemetrySink
{ {
public Task RecordConnectionOpenedAsync(WebSocketMessageEnvelope envelope, CloudSession session, CancellationToken cancellationToken = default) => Task.CompletedTask; public Task RecordConnectionOpenedAsync(WebSocketMessageEnvelope envelope, CloudSession session,
public Task RecordInboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, string? messageType, CancellationToken cancellationToken = default) => Task.CompletedTask; CancellationToken cancellationToken = default)
public Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default) => Task.CompletedTask; {
public Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default) => Task.CompletedTask; return Task.CompletedTask;
public Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason, CancellationToken cancellationToken = default) => Task.CompletedTask; }
public Task RecordInboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, string? messageType,
CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
public Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType,
IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
public Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session,
IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
public Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason,
CancellationToken cancellationToken = default)
{
return Task.CompletedTask;
}
} }

View File

@@ -12,5 +12,6 @@ public static class OpenJiboCloudBuildInfo
public static string SpokenVersion => $"Cloud version {VersionWords}."; public static string SpokenVersion => $"Cloud version {VersionWords}.";
public static string EsmlVersion => $"Cloud version<break time='10ms'/> {VersionWords.Replace(" ", "<break time='10ms' />")}."; public static string EsmlVersion =>
$"Cloud version<break time='10ms'/> {VersionWords.Replace(" ", "<break time='10ms' />")}.";
} }

View File

@@ -1,7 +1,7 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Runtime.Abstractions;
using System.Text.Json; using System.Text.Json;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services; namespace Jibo.Cloud.Application.Services;
@@ -41,6 +41,7 @@ internal static class PersonalReportOrchestrator
"yeah", "yeah",
"yep", "yep",
"yup", "yup",
"uh huh",
"sure", "sure",
"ok", "ok",
"okay", "okay",
@@ -58,6 +59,8 @@ internal static class PersonalReportOrchestrator
"maybe later" "maybe later"
]; ];
private static readonly Regex NameNoiseRegex = new("[^a-zA-Z\\-\\s']", RegexOptions.Compiled);
public static async Task<JiboInteractionDecision?> TryBuildDecisionAsync( public static async Task<JiboInteractionDecision?> TryBuildDecisionAsync(
TurnContext turn, TurnContext turn,
string semanticIntent, string semanticIntent,
@@ -67,36 +70,33 @@ internal static class PersonalReportOrchestrator
IJiboRandomizer randomizer, IJiboRandomizer randomizer,
IPersonalMemoryStore personalMemoryStore, IPersonalMemoryStore personalMemoryStore,
Func<TurnContext, string, CancellationToken, Task<JiboInteractionDecision>> buildWeatherDecisionAsync, Func<TurnContext, string, CancellationToken, Task<JiboInteractionDecision>> buildWeatherDecisionAsync,
Func<TurnContext, CancellationToken, Task<JiboInteractionDecision>> buildCalendarDecisionAsync,
Func<TurnContext, CancellationToken, Task<JiboInteractionDecision>> buildCommuteDecisionAsync,
Func<TurnContext, PersonalMemoryTenantScope> tenantScopeResolver, Func<TurnContext, PersonalMemoryTenantScope> tenantScopeResolver,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var state = ReadState(turn); var state = ReadState(turn);
var isActiveState = !string.Equals(state, IdleState, StringComparison.OrdinalIgnoreCase); var isActiveState = !string.Equals(state, IdleState, StringComparison.OrdinalIgnoreCase);
if (!isActiveState && !string.Equals(semanticIntent, "personal_report", StringComparison.OrdinalIgnoreCase)) if (!isActiveState &&
{ !string.Equals(semanticIntent, "personal_report", StringComparison.OrdinalIgnoreCase)) return null;
return null;
}
var toggles = ApplyInlineToggleHints( var toggles = ApplyInlineToggleHints(
ReadServiceToggles(turn), ReadServiceToggles(turn),
loweredTranscript, loweredTranscript,
out var inlineToggleSummary); out var inlineToggleSummary);
if (ContainsAnyPhrase(loweredTranscript, CancelPhrases)) if (ContainsAnyPhrase(loweredTranscript, CancelPhrases)) return BuildCancelledDecision(toggles);
{
return BuildCancelledDecision(toggles);
}
if (!isActiveState) if (!isActiveState)
{ {
var contextUpdates = BuildContextUpdates( var contextUpdates = BuildContextUpdates(
AwaitingOptInState, AwaitingOptInState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: ReadString(turn, UserNameMetadataKey), ReadString(turn, UserNameMetadataKey),
userVerified: ReadBool(turn, UserVerifiedMetadataKey) ?? false, ReadBool(turn, UserVerifiedMetadataKey) ?? false,
lastServiceError: string.Empty); string.Empty);
var reply = string.IsNullOrWhiteSpace(inlineToggleSummary) var reply = string.IsNullOrWhiteSpace(inlineToggleSummary)
? "Would you like your personal report now?" ? "Would you like your personal report now?"
@@ -105,13 +105,11 @@ internal static class PersonalReportOrchestrator
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_opt_in", "personal_report_opt_in",
reply, reply,
SkillPayload: BuildYesNoPromptPayload(),
ContextUpdates: contextUpdates); ContextUpdates: contextUpdates);
} }
if (string.IsNullOrWhiteSpace(loweredTranscript)) if (string.IsNullOrWhiteSpace(loweredTranscript)) return BuildNoInputDecision(turn, state, toggles);
{
return BuildNoInputDecision(turn, state, toggles);
}
switch (state) switch (state)
{ {
@@ -121,81 +119,73 @@ internal static class PersonalReportOrchestrator
var scope = tenantScopeResolver(turn); var scope = tenantScopeResolver(turn);
var knownName = ReadString(turn, UserNameMetadataKey) ?? personalMemoryStore.GetName(scope); var knownName = ReadString(turn, UserNameMetadataKey) ?? personalMemoryStore.GetName(scope);
if (!string.IsNullOrWhiteSpace(knownName)) if (!string.IsNullOrWhiteSpace(knownName))
{
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_verify_user", "personal_report_verify_user",
$"I think this is {knownName}. Is that right?", $"I think this is {knownName}. Is that right?",
SkillPayload: BuildYesNoPromptPayload(),
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
AwaitingIdentityConfirmationState, AwaitingIdentityConfirmationState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: knownName, knownName,
userVerified: false, false,
lastServiceError: string.Empty)); string.Empty));
}
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_request_name", "personal_report_request_name",
"Who is this?", "Who is this?",
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
AwaitingIdentityNameState, AwaitingIdentityNameState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: null, null,
userVerified: false, false,
lastServiceError: string.Empty)); string.Empty));
} }
if (IsNegativeReply(loweredTranscript)) if (IsNegativeReply(loweredTranscript)) return BuildDeclinedDecision(toggles);
{
return BuildDeclinedDecision(toggles);
}
if (!string.IsNullOrWhiteSpace(inlineToggleSummary)) if (!string.IsNullOrWhiteSpace(inlineToggleSummary))
{
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_opt_in", "personal_report_opt_in",
$"{inlineToggleSummary} Would you like your personal report now?", $"{inlineToggleSummary} Would you like your personal report now?",
SkillPayload: BuildYesNoPromptPayload(),
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
AwaitingOptInState, AwaitingOptInState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: ReadString(turn, UserNameMetadataKey), ReadString(turn, UserNameMetadataKey),
userVerified: false, false,
lastServiceError: string.Empty)); string.Empty));
}
return BuildNoMatchDecision( return BuildNoMatchDecision(
turn, turn,
state, state,
"Please say yes to start your personal report, or no to skip it.", "Please say yes to start your personal report, or no to skip it.",
toggles, toggles,
userName: ReadString(turn, UserNameMetadataKey), ReadString(turn, UserNameMetadataKey),
userVerified: false); false);
case AwaitingIdentityConfirmationState: case AwaitingIdentityConfirmationState:
{ {
var currentName = ReadString(turn, UserNameMetadataKey); var currentName = ReadString(turn, UserNameMetadataKey);
if (string.IsNullOrWhiteSpace(currentName)) if (string.IsNullOrWhiteSpace(currentName))
{
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_request_name", "personal_report_request_name",
"Who is this?", "Who is this?",
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
AwaitingIdentityNameState, AwaitingIdentityNameState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: null, null,
userVerified: false, false,
lastServiceError: string.Empty)); string.Empty));
}
if (IsAffirmativeReply(loweredTranscript)) if (IsAffirmativeReply(loweredTranscript))
{
return await BuildDeliveredReportDecisionAsync( return await BuildDeliveredReportDecisionAsync(
turn, turn,
catalog, catalog,
@@ -203,46 +193,43 @@ internal static class PersonalReportOrchestrator
toggles, toggles,
currentName, currentName,
buildWeatherDecisionAsync, buildWeatherDecisionAsync,
buildCalendarDecisionAsync,
buildCommuteDecisionAsync,
cancellationToken); cancellationToken);
}
if (IsNegativeReply(loweredTranscript)) if (IsNegativeReply(loweredTranscript))
{
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_request_name", "personal_report_request_name",
"Okay, who is this?", "Okay, who is this?",
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
AwaitingIdentityNameState, AwaitingIdentityNameState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: null, null,
userVerified: false, false,
lastServiceError: string.Empty)); string.Empty));
}
return BuildNoMatchDecision( return BuildNoMatchDecision(
turn, turn,
state, state,
$"Please answer yes or no. Is this {currentName}?", $"Please answer yes or no. Is this {currentName}?",
toggles, toggles,
userName: currentName, currentName,
userVerified: false); false);
} }
case AwaitingIdentityNameState: case AwaitingIdentityNameState:
{ {
var parsedName = TryExtractName(loweredTranscript); var parsedName = TryExtractName(loweredTranscript);
if (string.IsNullOrWhiteSpace(parsedName)) if (string.IsNullOrWhiteSpace(parsedName))
{
return BuildNoMatchDecision( return BuildNoMatchDecision(
turn, turn,
state, state,
"Tell me your name like this: my name is Alex.", "Tell me your name like this: my name is Alex.",
toggles, toggles,
userName: null, null,
userVerified: false); false);
}
personalMemoryStore.SetName(tenantScopeResolver(turn), parsedName); personalMemoryStore.SetName(tenantScopeResolver(turn), parsedName);
return await BuildDeliveredReportDecisionAsync( return await BuildDeliveredReportDecisionAsync(
@@ -252,6 +239,8 @@ internal static class PersonalReportOrchestrator
toggles, toggles,
parsedName, parsedName,
buildWeatherDecisionAsync, buildWeatherDecisionAsync,
buildCalendarDecisionAsync,
buildCommuteDecisionAsync,
cancellationToken); cancellationToken);
} }
@@ -267,49 +256,123 @@ internal static class PersonalReportOrchestrator
PersonalReportServiceToggles toggles, PersonalReportServiceToggles toggles,
string userName, string userName,
Func<TurnContext, string, CancellationToken, Task<JiboInteractionDecision>> buildWeatherDecisionAsync, Func<TurnContext, string, CancellationToken, Task<JiboInteractionDecision>> buildWeatherDecisionAsync,
Func<TurnContext, CancellationToken, Task<JiboInteractionDecision>> buildCalendarDecisionAsync,
Func<TurnContext, CancellationToken, Task<JiboInteractionDecision>> buildCommuteDecisionAsync,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var reportSections = new List<string> { $"Great, {userName}. Here is your personal report." }; var reportSections = new List<string>
{
RenderPersonalReportTemplate(
ChoosePersonalReportTemplate(
catalog.PersonalReportKickOffReplies,
"Okay. Here's your personal report."),
userName)
};
var serviceError = string.Empty; var serviceError = string.Empty;
IDictionary<string, object?>? weatherSkillPayload = null;
if (toggles.WeatherEnabled) if (toggles.WeatherEnabled)
{ {
var weatherDecision = await buildWeatherDecisionAsync(turn, "weather", cancellationToken); var weatherDecision = await buildWeatherDecisionAsync(turn, "weather", cancellationToken);
weatherSkillPayload = weatherDecision.SkillPayload;
reportSections.Add("Weather.");
reportSections.Add(weatherDecision.ReplyText); reportSections.Add(weatherDecision.ReplyText);
if (IsWeatherErrorReply(weatherDecision.ReplyText)) if (IsWeatherErrorReply(weatherDecision.ReplyText)) serviceError = "weather";
{
serviceError = "weather";
}
} }
if (toggles.CalendarEnabled) if (toggles.CalendarEnabled)
{ {
reportSections.Add(randomizer.Choose(catalog.CalendarReplies)); var calendarReply = (await buildCalendarDecisionAsync(turn, cancellationToken)).ReplyText;
if (!string.IsNullOrWhiteSpace(calendarReply))
{
reportSections.Add(calendarReply);
var calendarOutro = ChooseShortestTemplate(catalog.CalendarOutroReplies);
if (!string.IsNullOrWhiteSpace(calendarOutro))
reportSections.Add(RenderPersonalReportTemplate(calendarOutro!, userName));
}
} }
if (toggles.CommuteEnabled) if (toggles.CommuteEnabled)
{ {
reportSections.Add(randomizer.Choose(catalog.CommuteReplies)); var commuteReply = (await buildCommuteDecisionAsync(turn, cancellationToken)).ReplyText;
var commuteSnippet = ChooseFirstSentence(commuteReply);
if (!string.IsNullOrWhiteSpace(commuteSnippet))
reportSections.Add(commuteSnippet);
} }
if (toggles.NewsEnabled) if (toggles.NewsEnabled)
{ {
reportSections.Add(randomizer.Choose(catalog.NewsBriefings)); reportSections.Add(
RenderReportSkillTemplate(
ChooseReportSkillTemplate(
catalog.NewsIntroReplies,
catalog.NewsCategoryIntroReplies,
"Here's today's news, from the associated press."),
userName));
reportSections.Add(ChooseShortestBriefing(catalog.NewsBriefings));
reportSections.Add(
RenderReportSkillTemplate(
ChooseReportSkillTemplate(
catalog.NewsOutroReplies,
[],
"And that's what's new in the news."),
userName));
} }
reportSections.Add("That is your personal report."); reportSections.Add(
RenderPersonalReportTemplate(
ChoosePersonalReportTemplate(
catalog.PersonalReportOutroReplies,
"And that's your report for the day. I hope you had as much fun as I did."),
userName));
var reportText = string.Join(" ", reportSections);
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_delivered", "personal_report_delivered",
string.Join(" ", reportSections), reportText,
ContextUpdates: BuildContextUpdates( "report-skill",
BuildPersonalReportSkillPayload(reportText, weatherSkillPayload),
BuildContextUpdates(
IdleState, IdleState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName, userName,
userVerified: true, true,
lastServiceError: serviceError)); serviceError));
}
private static IDictionary<string, object?> BuildPersonalReportSkillPayload(
string reportText,
IDictionary<string, object?>? weatherSkillPayload)
{
var payload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "report-skill",
["cloudSkill"] = "personal_report",
["mim_id"] = "runtime-personal-report",
["mim_type"] = "announcement",
["prompt_id"] = "PersonalReport_AN_01",
["prompt_sub_category"] = "AN",
["esml"] =
$"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeForEsml(reportText)}</es></speak>",
["personal_report_report_text"] = reportText
};
if (weatherSkillPayload is null) return payload;
foreach (var (key, value) in weatherSkillPayload)
if (!string.Equals(key, "esml", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(key, "skillId", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(key, "cloudSkill", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(key, "mim_id", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(key, "mim_type", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(key, "prompt_id", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(key, "prompt_sub_category", StringComparison.OrdinalIgnoreCase))
payload[key] = value;
return payload;
} }
private static JiboInteractionDecision BuildNoInputDecision( private static JiboInteractionDecision BuildNoInputDecision(
@@ -318,22 +381,19 @@ internal static class PersonalReportOrchestrator
PersonalReportServiceToggles toggles) PersonalReportServiceToggles toggles)
{ {
var noInputCount = Math.Max(0, ReadInt(turn, NoInputCountMetadataKey)) + 1; var noInputCount = Math.Max(0, ReadInt(turn, NoInputCountMetadataKey)) + 1;
if (noInputCount >= MaxNoInputCount) if (noInputCount >= MaxNoInputCount) return BuildDeclinedDecision(toggles);
{
return BuildDeclinedDecision(toggles);
}
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_no_input", "personal_report_no_input",
"I am still here. Do you want your personal report?", "I am still here. Do you want your personal report?",
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
state, state,
noMatchCount: ReadInt(turn, NoMatchCountMetadataKey), ReadInt(turn, NoMatchCountMetadataKey),
noInputCount, noInputCount,
toggles, toggles,
userName: ReadString(turn, UserNameMetadataKey), ReadString(turn, UserNameMetadataKey),
userVerified: ReadBool(turn, UserVerifiedMetadataKey) ?? false, ReadBool(turn, UserVerifiedMetadataKey) ?? false,
lastServiceError: string.Empty)); string.Empty));
} }
private static JiboInteractionDecision BuildNoMatchDecision( private static JiboInteractionDecision BuildNoMatchDecision(
@@ -345,10 +405,7 @@ internal static class PersonalReportOrchestrator
bool userVerified) bool userVerified)
{ {
var noMatchCount = Math.Max(0, ReadInt(turn, NoMatchCountMetadataKey)) + 1; var noMatchCount = Math.Max(0, ReadInt(turn, NoMatchCountMetadataKey)) + 1;
if (noMatchCount >= MaxNoMatchCount) if (noMatchCount >= MaxNoMatchCount) return BuildDeclinedDecision(toggles);
{
return BuildDeclinedDecision(toggles);
}
return new JiboInteractionDecision( return new JiboInteractionDecision(
"personal_report_no_match", "personal_report_no_match",
@@ -356,11 +413,11 @@ internal static class PersonalReportOrchestrator
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
state, state,
noMatchCount, noMatchCount,
noInputCount: 0, 0,
toggles, toggles,
userName, userName,
userVerified, userVerified,
lastServiceError: string.Empty)); string.Empty));
} }
private static JiboInteractionDecision BuildDeclinedDecision(PersonalReportServiceToggles toggles) private static JiboInteractionDecision BuildDeclinedDecision(PersonalReportServiceToggles toggles)
@@ -370,12 +427,12 @@ internal static class PersonalReportOrchestrator
"No problem. We can do your personal report another time.", "No problem. We can do your personal report another time.",
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
IdleState, IdleState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: null, null,
userVerified: false, false,
lastServiceError: string.Empty)); string.Empty));
} }
private static JiboInteractionDecision BuildCancelledDecision(PersonalReportServiceToggles toggles) private static JiboInteractionDecision BuildCancelledDecision(PersonalReportServiceToggles toggles)
@@ -385,12 +442,12 @@ internal static class PersonalReportOrchestrator
"Okay, canceling personal report.", "Okay, canceling personal report.",
ContextUpdates: BuildContextUpdates( ContextUpdates: BuildContextUpdates(
IdleState, IdleState,
noMatchCount: 0, 0,
noInputCount: 0, 0,
toggles, toggles,
userName: null, null,
userVerified: false, false,
lastServiceError: string.Empty)); string.Empty));
} }
private static IDictionary<string, object?> BuildContextUpdates( private static IDictionary<string, object?> BuildContextUpdates(
@@ -417,6 +474,14 @@ internal static class PersonalReportOrchestrator
}; };
} }
private static IDictionary<string, object?> BuildYesNoPromptPayload()
{
return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["listen_contexts"] = new[] { "shared/yes_no" }
};
}
private static bool IsAffirmativeReply(string loweredTranscript) private static bool IsAffirmativeReply(string loweredTranscript)
{ {
return ContainsAnyPhrase(loweredTranscript, AffirmativePhrases); return ContainsAnyPhrase(loweredTranscript, AffirmativePhrases);
@@ -430,24 +495,17 @@ internal static class PersonalReportOrchestrator
private static bool ContainsAnyPhrase(string loweredTranscript, IEnumerable<string> phrases) private static bool ContainsAnyPhrase(string loweredTranscript, IEnumerable<string> phrases)
{ {
foreach (var phrase in phrases) foreach (var phrase in phrases)
{
if (string.Equals(loweredTranscript, phrase, StringComparison.Ordinal) || if (string.Equals(loweredTranscript, phrase, StringComparison.Ordinal) ||
loweredTranscript.StartsWith($"{phrase} ", StringComparison.Ordinal) || loweredTranscript.StartsWith($"{phrase} ", StringComparison.Ordinal) ||
loweredTranscript.Contains($" {phrase}", StringComparison.Ordinal)) loweredTranscript.Contains($" {phrase}", StringComparison.Ordinal))
{
return true; return true;
}
}
return false; return false;
} }
private static bool IsWeatherErrorReply(string replyText) private static bool IsWeatherErrorReply(string replyText)
{ {
if (string.IsNullOrWhiteSpace(replyText)) if (string.IsNullOrWhiteSpace(replyText)) return false;
{
return false;
}
return replyText.Contains("couldn't fetch the weather", StringComparison.OrdinalIgnoreCase) || return replyText.Contains("couldn't fetch the weather", StringComparison.OrdinalIgnoreCase) ||
replyText.Contains("weather service is connected", StringComparison.OrdinalIgnoreCase); replyText.Contains("weather service is connected", StringComparison.OrdinalIgnoreCase);
@@ -470,36 +528,32 @@ internal static class PersonalReportOrchestrator
summary = string.Empty; summary = string.Empty;
var updated = toggles; var updated = toggles;
updated = ApplyToggleHint(updated, loweredTranscript, "weather", static value => value with { WeatherEnabled = false }, static value => value with { WeatherEnabled = true }); updated = ApplyToggleHint(updated, loweredTranscript, "weather",
updated = ApplyToggleHint(updated, loweredTranscript, "calendar", static value => value with { CalendarEnabled = false }, static value => value with { CalendarEnabled = true }); static value => value with { WeatherEnabled = false },
updated = ApplyToggleHint(updated, loweredTranscript, "commute", static value => value with { CommuteEnabled = false }, static value => value with { CommuteEnabled = true }); static value => value with { WeatherEnabled = true });
updated = ApplyToggleHint(updated, loweredTranscript, "news", static value => value with { NewsEnabled = false }, static value => value with { NewsEnabled = true }); updated = ApplyToggleHint(updated, loweredTranscript, "calendar",
static value => value with { CalendarEnabled = false },
static value => value with { CalendarEnabled = true });
updated = ApplyToggleHint(updated, loweredTranscript, "commute",
static value => value with { CommuteEnabled = false },
static value => value with { CommuteEnabled = true });
updated = ApplyToggleHint(updated, loweredTranscript, "news",
static value => value with { NewsEnabled = false }, static value => value with { NewsEnabled = true });
var changes = new List<string>(); var changes = new List<string>();
if (updated.WeatherEnabled != toggles.WeatherEnabled) if (updated.WeatherEnabled != toggles.WeatherEnabled)
{
changes.Add(updated.WeatherEnabled ? "including weather" : "skipping weather"); changes.Add(updated.WeatherEnabled ? "including weather" : "skipping weather");
}
if (updated.CalendarEnabled != toggles.CalendarEnabled) if (updated.CalendarEnabled != toggles.CalendarEnabled)
{
changes.Add(updated.CalendarEnabled ? "including calendar" : "skipping calendar"); changes.Add(updated.CalendarEnabled ? "including calendar" : "skipping calendar");
}
if (updated.CommuteEnabled != toggles.CommuteEnabled) if (updated.CommuteEnabled != toggles.CommuteEnabled)
{
changes.Add(updated.CommuteEnabled ? "including commute" : "skipping commute"); changes.Add(updated.CommuteEnabled ? "including commute" : "skipping commute");
}
if (updated.NewsEnabled != toggles.NewsEnabled) if (updated.NewsEnabled != toggles.NewsEnabled)
{
changes.Add(updated.NewsEnabled ? "including news" : "skipping news"); changes.Add(updated.NewsEnabled ? "including news" : "skipping news");
}
if (changes.Count > 0) if (changes.Count > 0) summary = $"Got it, {string.Join(", ", changes)}.";
{
summary = $"Got it, {string.Join(", ", changes)}.";
}
return updated; return updated;
} }
@@ -514,15 +568,11 @@ internal static class PersonalReportOrchestrator
if (loweredTranscript.Contains($"without {serviceLabel}", StringComparison.Ordinal) || if (loweredTranscript.Contains($"without {serviceLabel}", StringComparison.Ordinal) ||
loweredTranscript.Contains($"skip {serviceLabel}", StringComparison.Ordinal) || loweredTranscript.Contains($"skip {serviceLabel}", StringComparison.Ordinal) ||
loweredTranscript.Contains($"no {serviceLabel}", StringComparison.Ordinal)) loweredTranscript.Contains($"no {serviceLabel}", StringComparison.Ordinal))
{
return disable(toggles); return disable(toggles);
}
if (loweredTranscript.Contains($"with {serviceLabel}", StringComparison.Ordinal) || if (loweredTranscript.Contains($"with {serviceLabel}", StringComparison.Ordinal) ||
loweredTranscript.Contains($"include {serviceLabel}", StringComparison.Ordinal)) loweredTranscript.Contains($"include {serviceLabel}", StringComparison.Ordinal))
{
return enable(toggles); return enable(toggles);
}
return toggles; return toggles;
} }
@@ -534,10 +584,7 @@ internal static class PersonalReportOrchestrator
private static string? ReadString(TurnContext turn, string key) private static string? ReadString(TurnContext turn, string key)
{ {
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return null;
{
return null;
}
return value switch return value switch
{ {
@@ -548,10 +595,7 @@ internal static class PersonalReportOrchestrator
private static bool? ReadBool(TurnContext turn, string key) private static bool? ReadBool(TurnContext turn, string key)
{ {
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return null;
{
return null;
}
return value switch return value switch
{ {
@@ -559,17 +603,15 @@ internal static class PersonalReportOrchestrator
string text when bool.TryParse(text, out var parsed) => parsed, string text when bool.TryParse(text, out var parsed) => parsed,
JsonElement { ValueKind: JsonValueKind.True } => true, JsonElement { ValueKind: JsonValueKind.True } => true,
JsonElement { ValueKind: JsonValueKind.False } => false, JsonElement { ValueKind: JsonValueKind.False } => false,
JsonElement json when json.ValueKind == JsonValueKind.String && bool.TryParse(json.GetString(), out var parsed) => parsed, JsonElement json when json.ValueKind == JsonValueKind.String &&
bool.TryParse(json.GetString(), out var parsed) => parsed,
_ => null _ => null
}; };
} }
private static int ReadInt(TurnContext turn, string key) private static int ReadInt(TurnContext turn, string key)
{ {
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return 0;
{
return 0;
}
return value switch return value switch
{ {
@@ -577,7 +619,8 @@ internal static class PersonalReportOrchestrator
long whole when whole <= int.MaxValue && whole >= int.MinValue => (int)whole, long whole when whole <= int.MaxValue && whole >= int.MinValue => (int)whole,
string text when int.TryParse(text, out var parsed) => parsed, string text when int.TryParse(text, out var parsed) => parsed,
JsonElement { ValueKind: JsonValueKind.Number } number when number.TryGetInt32(out var parsed) => parsed, JsonElement { ValueKind: JsonValueKind.Number } number when number.TryGetInt32(out var parsed) => parsed,
JsonElement json when json.ValueKind == JsonValueKind.String && int.TryParse(json.GetString(), out var parsed) => parsed, JsonElement json when json.ValueKind == JsonValueKind.String &&
int.TryParse(json.GetString(), out var parsed) => parsed,
_ => 0 _ => 0
}; };
} }
@@ -587,10 +630,7 @@ internal static class PersonalReportOrchestrator
var normalized = NameNoiseRegex.Replace(loweredTranscript, " ") var normalized = NameNoiseRegex.Replace(loweredTranscript, " ")
.Replace(" ", " ", StringComparison.Ordinal) .Replace(" ", " ", StringComparison.Ordinal)
.Trim(); .Trim();
if (string.IsNullOrWhiteSpace(normalized)) if (string.IsNullOrWhiteSpace(normalized)) return null;
{
return null;
}
var prefixes = new[] var prefixes = new[]
{ {
@@ -604,10 +644,7 @@ internal static class PersonalReportOrchestrator
foreach (var prefix in prefixes) foreach (var prefix in prefixes)
{ {
if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) continue;
{
continue;
}
var candidate = normalized[prefix.Length..].Trim(); var candidate = normalized[prefix.Length..].Trim();
return NormalizeNameCandidate(candidate); return NormalizeNameCandidate(candidate);
@@ -618,39 +655,127 @@ internal static class PersonalReportOrchestrator
private static string? NormalizeNameCandidate(string candidate) private static string? NormalizeNameCandidate(string candidate)
{ {
if (string.IsNullOrWhiteSpace(candidate)) if (string.IsNullOrWhiteSpace(candidate)) return null;
{
return null;
}
var cleaned = NameNoiseRegex.Replace(candidate, " ") var cleaned = NameNoiseRegex.Replace(candidate, " ")
.Replace(" ", " ", StringComparison.Ordinal) .Replace(" ", " ", StringComparison.Ordinal)
.Trim(); .Trim();
if (string.IsNullOrWhiteSpace(cleaned)) if (string.IsNullOrWhiteSpace(cleaned)) return null;
{
return null;
}
if (cleaned.Length < 2 || cleaned.Length > 32) if (cleaned.Length < 2 || cleaned.Length > 32) return null;
{
return null;
}
var words = cleaned.Split(' ', StringSplitOptions.RemoveEmptyEntries); var words = cleaned.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (words.Length > 4) if (words.Length > 4) return null;
{
return null;
}
if (words.Any(static word => word.Any(char.IsDigit))) return words.Any(static word => word.Any(char.IsDigit)) ? null : cleaned;
{
return null;
}
return cleaned;
} }
private static readonly Regex NameNoiseRegex = new("[^a-zA-Z\\-\\s']", RegexOptions.Compiled); private static string ChoosePersonalReportTemplate(
IReadOnlyList<string> templates,
string fallback)
{
var usableTemplates = templates
.Where(static template => !string.IsNullOrWhiteSpace(template) &&
!template.Contains("${dt.", StringComparison.OrdinalIgnoreCase))
.ToArray();
if (usableTemplates.Length == 0) return fallback;
var speakerAwareTemplate = usableTemplates.FirstOrDefault(static template =>
template.Contains("${speaker}", StringComparison.OrdinalIgnoreCase));
return ChooseShortestTemplate(speakerAwareTemplate is not null ? [speakerAwareTemplate] : usableTemplates)
?? fallback;
}
private static string RenderPersonalReportTemplate(string template, string userName)
{
return template
.Replace("${speaker}", userName, StringComparison.OrdinalIgnoreCase)
.Replace("${speaker}'s", $"{userName}'s", StringComparison.OrdinalIgnoreCase)
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
}
private static string ChooseReportSkillTemplate(
IReadOnlyList<string> primaryTemplates,
IReadOnlyList<string> secondaryTemplates,
string fallback)
{
var primary = ChooseShortestTemplate(primaryTemplates);
if (!string.IsNullOrWhiteSpace(primary)) return primary!;
var secondary = ChooseShortestTemplate(secondaryTemplates);
return !string.IsNullOrWhiteSpace(secondary) ? secondary! : fallback;
}
private static string ChooseShortestBriefing(IReadOnlyList<string> briefings)
{
var selected = ChooseShortestTemplate(briefings);
if (string.IsNullOrWhiteSpace(selected)) return string.Empty;
var firstSentence = selected.Split(['.', '!', '?'], 2,
StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.FirstOrDefault();
return string.IsNullOrWhiteSpace(firstSentence) ? selected : firstSentence;
}
private static string ChooseFirstSentence(string value)
{
if (string.IsNullOrWhiteSpace(value)) return string.Empty;
var firstSentence = value.Split(['.', '!', '?'], 2,
StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.FirstOrDefault();
return string.IsNullOrWhiteSpace(firstSentence) ? value.Trim() : firstSentence;
}
private static string ChooseFirstTwoSentences(string value)
{
if (string.IsNullOrWhiteSpace(value)) return string.Empty;
var segments = value
.Split(['.', '!', '?'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Take(2)
.ToArray();
if (segments.Length == 0) return string.Empty;
var joined = string.Join(". ", segments);
return value.TrimEnd().EndsWith(".", StringComparison.Ordinal) ||
value.TrimEnd().EndsWith("!", StringComparison.Ordinal) ||
value.TrimEnd().EndsWith("?", StringComparison.Ordinal)
? $"{joined}."
: joined;
}
private static string? ChooseShortestTemplate(IEnumerable<string> templates)
{
var selected = templates
.Where(static template => !string.IsNullOrWhiteSpace(template))
.OrderBy(static template => template.Length)
.FirstOrDefault();
return selected;
}
private static string RenderReportSkillTemplate(string template, string userName)
{
return template
.Replace("${speaker}", userName, StringComparison.OrdinalIgnoreCase)
.Replace("${speaker}'s", $"{userName}'s", StringComparison.OrdinalIgnoreCase)
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
}
private static string EscapeForEsml(string value)
{
return value
.Replace("&", "&amp;", StringComparison.Ordinal)
.Replace("<", "&lt;", StringComparison.Ordinal)
.Replace(">", "&gt;", StringComparison.Ordinal)
.Replace("\"", "&quot;", StringComparison.Ordinal)
.Replace("'", "&apos;", StringComparison.Ordinal);
}
private readonly record struct PersonalReportServiceToggles( private readonly record struct PersonalReportServiceToggles(
bool WeatherEnabled, bool WeatherEnabled,

View File

@@ -6,7 +6,8 @@ namespace Jibo.Cloud.Application.Services;
public sealed class ProtocolToTurnContextMapper public sealed class ProtocolToTurnContextMapper
{ {
public static TurnContext MapListenMessage(WebSocketMessageEnvelope envelope, CloudSession session, string messageType) public static TurnContext MapListenMessage(WebSocketMessageEnvelope envelope, CloudSession session,
string messageType)
{ {
var turnState = session.TurnState; var turnState = session.TurnState;
var protocolOperation = messageType.ToLowerInvariant(); var protocolOperation = messageType.ToLowerInvariant();
@@ -16,46 +17,28 @@ public sealed class ProtocolToTurnContextMapper
}; };
var text = ExtractTranscript(envelope.Text, attributes); var text = ExtractTranscript(envelope.Text, attributes);
if (!string.IsNullOrWhiteSpace(turnState.TransId)) if (!string.IsNullOrWhiteSpace(turnState.TransId)) attributes["transID"] = turnState.TransId;
{
attributes["transID"] = turnState.TransId;
}
if (!string.IsNullOrWhiteSpace(session.AccountId)) if (!string.IsNullOrWhiteSpace(session.AccountId)) attributes["accountId"] = session.AccountId;
{
attributes["accountId"] = session.AccountId;
}
if (!string.IsNullOrWhiteSpace(session.DeviceId)) if (!string.IsNullOrWhiteSpace(session.DeviceId)) attributes["deviceId"] = session.DeviceId;
{
attributes["deviceId"] = session.DeviceId;
}
if (session.Metadata.TryGetValue("loopId", out var loopId) && if (session.Metadata.TryGetValue("loopId", out var loopId) &&
loopId is string loopIdText && loopId is string loopIdText &&
!string.IsNullOrWhiteSpace(loopIdText)) !string.IsNullOrWhiteSpace(loopIdText))
{
attributes["loopId"] = loopIdText; attributes["loopId"] = loopIdText;
}
if (!string.IsNullOrWhiteSpace(turnState.ContextPayload)) if (!string.IsNullOrWhiteSpace(turnState.ContextPayload)) attributes["context"] = turnState.ContextPayload;
{
attributes["context"] = turnState.ContextPayload;
}
if (session.Metadata.TryGetValue("lastClockDomain", out var lastClockDomain) && if (session.Metadata.TryGetValue("lastClockDomain", out var lastClockDomain) &&
lastClockDomain is string lastClockDomainText && lastClockDomain is string lastClockDomainText &&
!string.IsNullOrWhiteSpace(lastClockDomainText)) !string.IsNullOrWhiteSpace(lastClockDomainText))
{
attributes["lastClockDomain"] = lastClockDomainText; attributes["lastClockDomain"] = lastClockDomainText;
}
if (session.Metadata.TryGetValue("pendingProactivityOffer", out var pendingProactivityOffer) && if (session.Metadata.TryGetValue("pendingProactivityOffer", out var pendingProactivityOffer) &&
pendingProactivityOffer is string pendingProactivityOfferText && pendingProactivityOffer is string pendingProactivityOfferText &&
!string.IsNullOrWhiteSpace(pendingProactivityOfferText)) !string.IsNullOrWhiteSpace(pendingProactivityOfferText))
{
attributes["pendingProactivityOffer"] = pendingProactivityOfferText; attributes["pendingProactivityOffer"] = pendingProactivityOfferText;
}
foreach (var pair in session.Metadata) foreach (var pair in session.Metadata)
{ {
@@ -63,41 +46,29 @@ public sealed class ProtocolToTurnContextMapper
!pair.Key.StartsWith("chitchat", StringComparison.OrdinalIgnoreCase) && !pair.Key.StartsWith("chitchat", StringComparison.OrdinalIgnoreCase) &&
!pair.Key.StartsWith("greetings", StringComparison.OrdinalIgnoreCase)) || !pair.Key.StartsWith("greetings", StringComparison.OrdinalIgnoreCase)) ||
pair.Value is null) pair.Value is null)
{
continue; continue;
}
attributes[pair.Key] = pair.Value; attributes[pair.Key] = pair.Value;
} }
attributes["listenHotphrase"] = turnState.ListenHotphrase; attributes["listenHotphrase"] = turnState.ListenHotphrase;
if (turnState.ListenRules.Count > 0) if (turnState.ListenRules.Count > 0) attributes["listenRules"] = turnState.ListenRules;
{
attributes["listenRules"] = turnState.ListenRules;
}
if (turnState.ListenAsrHints.Count > 0) if (turnState.ListenAsrHints.Count > 0) attributes["listenAsrHints"] = turnState.ListenAsrHints;
{
attributes["listenAsrHints"] = turnState.ListenAsrHints;
}
if (turnState.BufferedAudioBytes > 0) if (turnState.BufferedAudioBytes > 0)
{ {
attributes["bufferedAudioBytes"] = turnState.BufferedAudioBytes; attributes["bufferedAudioBytes"] = turnState.BufferedAudioBytes;
attributes["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount; attributes["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount;
attributes["bufferedAudioFrames"] = turnState.BufferedAudioFrames.Select(frame => frame.ToArray()).ToArray(); attributes["bufferedAudioFrames"] =
turnState.BufferedAudioFrames.Select(frame => frame.ToArray()).ToArray();
} }
if (!string.IsNullOrWhiteSpace(turnState.AudioTranscriptHint)) if (!string.IsNullOrWhiteSpace(turnState.AudioTranscriptHint))
{
attributes["audioTranscriptHint"] = turnState.AudioTranscriptHint; attributes["audioTranscriptHint"] = turnState.AudioTranscriptHint;
}
if (turnState.FinalizeAttemptCount > 0) if (turnState.FinalizeAttemptCount > 0) attributes["finalizeAttemptCount"] = turnState.FinalizeAttemptCount;
{
attributes["finalizeAttemptCount"] = turnState.FinalizeAttemptCount;
}
return new TurnContext return new TurnContext
{ {
@@ -111,8 +82,12 @@ public sealed class ProtocolToTurnContextMapper
RequestId = envelope.ConnectionId, RequestId = envelope.ConnectionId,
ProtocolService = "neo-hub", ProtocolService = "neo-hub",
ProtocolOperation = protocolOperation, ProtocolOperation = protocolOperation,
FirmwareVersion = session.Metadata.TryGetValue("firmwareVersion", out var firmwareVersion) ? firmwareVersion as string : null, FirmwareVersion = session.Metadata.TryGetValue("firmwareVersion", out var firmwareVersion)
ApplicationVersion = session.Metadata.TryGetValue("applicationVersion", out var applicationVersion) ? applicationVersion as string : null, ? firmwareVersion as string
: null,
ApplicationVersion = session.Metadata.TryGetValue("applicationVersion", out var applicationVersion)
? applicationVersion as string
: null,
IsFollowUpEligible = true, IsFollowUpEligible = true,
Attributes = attributes Attributes = attributes
}; };
@@ -120,10 +95,7 @@ public sealed class ProtocolToTurnContextMapper
private static string? ExtractTranscript(string? text, IDictionary<string, object?> attributes) private static string? ExtractTranscript(string? text, IDictionary<string, object?> attributes)
{ {
if (string.IsNullOrWhiteSpace(text)) if (string.IsNullOrWhiteSpace(text)) return null;
{
return null;
}
try try
{ {
@@ -133,57 +105,41 @@ public sealed class ProtocolToTurnContextMapper
if (!root.TryGetProperty("data", out var data)) return null; if (!root.TryGetProperty("data", out var data)) return null;
if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String) if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String)
{
return transcript.GetString(); return transcript.GetString();
}
if (data.TryGetProperty("asr", out var asr) && if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object && asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("text", out var asrText) && asr.TryGetProperty("text", out var asrText) &&
asrText.ValueKind == JsonValueKind.String) asrText.ValueKind == JsonValueKind.String)
{
return asrText.GetString(); return asrText.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String) if (data.TryGetProperty("transcriptHint", out var transcriptHint) &&
{ transcriptHint.ValueKind == JsonValueKind.String) return transcriptHint.GetString();
return transcriptHint.GetString();
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String) if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
attributes["clientIntent"] = intent.GetString(); attributes["clientIntent"] = intent.GetString();
}
if (data.TryGetProperty("triggerSource", out var triggerSource) && if (data.TryGetProperty("triggerSource", out var triggerSource) &&
triggerSource.ValueKind == JsonValueKind.String && triggerSource.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(triggerSource.GetString())) !string.IsNullOrWhiteSpace(triggerSource.GetString()))
{
attributes["triggerSource"] = triggerSource.GetString(); attributes["triggerSource"] = triggerSource.GetString();
}
if (data.TryGetProperty("triggerData", out var triggerData) && if (data.TryGetProperty("triggerData", out var triggerData) &&
triggerData.ValueKind == JsonValueKind.Object && triggerData.ValueKind == JsonValueKind.Object &&
triggerData.TryGetProperty("looperID", out var triggerLooperId) && triggerData.TryGetProperty("looperID", out var triggerLooperId) &&
triggerLooperId.ValueKind == JsonValueKind.String && triggerLooperId.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(triggerLooperId.GetString())) !string.IsNullOrWhiteSpace(triggerLooperId.GetString()))
{
attributes["triggerLooperId"] = triggerLooperId.GetString(); attributes["triggerLooperId"] = triggerLooperId.GetString();
}
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array) if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
attributes["clientRules"] = rules.EnumerateArray() attributes["clientRules"] = rules.EnumerateArray()
.Where(item => item.ValueKind == JsonValueKind.String) .Where(item => item.ValueKind == JsonValueKind.String)
.Select(item => item.GetString() ?? string.Empty) .Select(item => item.GetString() ?? string.Empty)
.Where(rule => !string.IsNullOrWhiteSpace(rule)) .Where(rule => !string.IsNullOrWhiteSpace(rule))
.ToArray(); .ToArray();
}
if (data.TryGetProperty("entities", out var entities) && entities.ValueKind == JsonValueKind.Object) if (data.TryGetProperty("entities", out var entities) && entities.ValueKind == JsonValueKind.Object)
{
attributes["clientEntities"] = entities.Clone(); attributes["clientEntities"] = entities.Clone();
}
return intent.ValueKind == JsonValueKind.String ? intent.GetString() : null; return intent.ValueKind == JsonValueKind.String ? intent.GetString() : null;
} }

View File

@@ -1,4 +1,5 @@
using System.Text.Json; using System.Text.Json;
using System.Text.RegularExpressions;
using Jibo.Cloud.Domain.Models; using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions; using Jibo.Runtime.Abstractions;
@@ -31,14 +32,19 @@ public sealed class ResponsePlanToSocketMessagesMapper
var isVolumeControl = string.Equals(plan.IntentName, "volume_up", StringComparison.OrdinalIgnoreCase) || var isVolumeControl = string.Equals(plan.IntentName, "volume_up", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "volume_down", StringComparison.OrdinalIgnoreCase) || string.Equals(plan.IntentName, "volume_down", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "volume_to_value", StringComparison.OrdinalIgnoreCase); string.Equals(plan.IntentName, "volume_to_value", StringComparison.OrdinalIgnoreCase);
var isProactivePizzaFactOffer = string.Equals(plan.IntentName, "proactive_offer_pizza_fact", StringComparison.OrdinalIgnoreCase); var isProactivePizzaFactOffer = string.Equals(plan.IntentName, "proactive_offer_pizza_fact",
StringComparison.OrdinalIgnoreCase);
var isSettingsLaunch = string.Equals(skill?.SkillName, "@be/settings", StringComparison.OrdinalIgnoreCase); var isSettingsLaunch = string.Equals(skill?.SkillName, "@be/settings", StringComparison.OrdinalIgnoreCase);
var isGlobalCommand = isStopCommand || isVolumeControl; var isSleepCommand = string.Equals(plan.IntentName, "sleep", StringComparison.OrdinalIgnoreCase);
var isSpinAroundCommand = string.Equals(plan.IntentName, "spin_around", StringComparison.OrdinalIgnoreCase);
var isGlobalCommand = isStopCommand || isSleepCommand || isSpinAroundCommand || isVolumeControl;
var isPhotoGalleryLaunch = string.Equals(plan.IntentName, "photo_gallery", StringComparison.OrdinalIgnoreCase); var isPhotoGalleryLaunch = string.Equals(plan.IntentName, "photo_gallery", StringComparison.OrdinalIgnoreCase);
var isPhotoCreateLaunch = string.Equals(plan.IntentName, "snapshot", StringComparison.OrdinalIgnoreCase) || var isPhotoCreateLaunch = string.Equals(plan.IntentName, "snapshot", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "photobooth", StringComparison.OrdinalIgnoreCase); string.Equals(plan.IntentName, "photobooth", StringComparison.OrdinalIgnoreCase);
var isClockSkillLaunch = string.Equals(skill?.SkillName, "@be/clock", StringComparison.OrdinalIgnoreCase); var isClockSkillLaunch = string.Equals(skill?.SkillName, "@be/clock", StringComparison.OrdinalIgnoreCase);
var isReportSkillLaunch = string.Equals(skill?.SkillName, "report-skill", StringComparison.OrdinalIgnoreCase); var isReportSkillLaunch = string.Equals(skill?.SkillName, "report-skill", StringComparison.OrdinalIgnoreCase);
var idleRedirectDelayMs = isSleepCommand ? 150 : isSpinAroundCommand ? 75 : 75;
var idleCompletionDelayMs = isSleepCommand ? 1000 : isSpinAroundCommand ? 750 : 125;
var localIntent = ReadSkillPayloadString(skill, "localIntent"); var localIntent = ReadSkillPayloadString(skill, "localIntent");
var clockIntent = ReadSkillPayloadString(skill, "clockIntent"); var clockIntent = ReadSkillPayloadString(skill, "clockIntent");
var clockDomain = ReadSkillPayloadString(skill, "domain"); var clockDomain = ReadSkillPayloadString(skill, "domain");
@@ -70,12 +76,13 @@ public sealed class ResponsePlanToSocketMessagesMapper
? clockIntent ? clockIntent
: isReportSkillLaunch && !string.IsNullOrWhiteSpace(localIntent) : isReportSkillLaunch && !string.IsNullOrWhiteSpace(localIntent)
? localIntent ? localIntent
: isWordOfDayGuess : isWordOfDayGuess
? "guess" ? "guess"
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && : string.Equals(messageType, "CLIENT_NLU",
!string.IsNullOrWhiteSpace(clientIntent) StringComparison.OrdinalIgnoreCase) &&
? clientIntent !string.IsNullOrWhiteSpace(clientIntent)
: plan.IntentName ?? "unknown"; ? clientIntent
: plan.IntentName ?? "unknown";
var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess) var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess)
? wordOfDayGuess ? wordOfDayGuess
: isWordOfDayLaunch : isWordOfDayLaunch
@@ -103,30 +110,30 @@ public sealed class ResponsePlanToSocketMessagesMapper
var outboundRules = isProactivePizzaFactOffer var outboundRules = isProactivePizzaFactOffer
? ["shared/yes_no"] ? ["shared/yes_no"]
: isWordOfDayLaunch : isWordOfDayLaunch
? ["word-of-the-day/menu"] ? ["word-of-the-day/menu"]
: isGlobalCommand : isGlobalCommand
? BuildGlobalCommandRules(rules) ? BuildGlobalCommandRules(rules)
: isRadioLaunch : isRadioLaunch
? [] ? []
: isSettingsLaunch : isSettingsLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isPhotoGalleryLaunch || isPhotoCreateLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules ? rules
: [] : []
: isClockSkillLaunch : isPhotoGalleryLaunch || isPhotoCreateLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules ? rules
: [] : []
: isReportSkillLaunch : isClockSkillLaunch
? [] ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
: isWordOfDayGuess ? rules
? ["word-of-the-day/puzzle"] : []
: isYesNoTurn && isYesNoIntent : isReportSkillLaunch
? [yesNoRule!] ? []
: rules; : isWordOfDayGuess
? ["word-of-the-day/puzzle"]
: isYesNoTurn && isYesNoIntent
? [yesNoRule!]
: rules;
var entities = ReadEntities( var entities = ReadEntities(
turn, turn,
messageType, messageType,
@@ -209,10 +216,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
outboundAsrText, outboundAsrText,
outboundRules, outboundRules,
entities)), entities)),
DelayMs: 75)); 75));
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/word-of-the-day")), JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/word-of-the-day")),
DelayMs: 125)); 125));
} }
if (isRadioLaunch) if (isRadioLaunch)
@@ -225,13 +232,13 @@ public sealed class ResponsePlanToSocketMessagesMapper
outboundAsrText, outboundAsrText,
outboundRules, outboundRules,
entities)), entities)),
DelayMs: 75)); 75));
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/radio")), JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/radio")),
DelayMs: 125)); 125));
} }
if (isStopCommand) if (isStopCommand || isSleepCommand || isSpinAroundCommand)
{ {
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildSkillRedirectPayload( JsonSerializer.Serialize(BuildSkillRedirectPayload(
@@ -241,10 +248,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
outboundAsrText, outboundAsrText,
outboundRules, outboundRules,
entities)), entities)),
DelayMs: 75)); idleRedirectDelayMs));
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/idle")), JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/idle")),
DelayMs: 125)); idleCompletionDelayMs));
} }
if (isSettingsLaunch && if (isSettingsLaunch &&
@@ -258,10 +265,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
outboundAsrText, outboundAsrText,
outboundRules, outboundRules,
entities)), entities)),
DelayMs: 75)); 75));
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/settings")), JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/settings")),
DelayMs: 125)); 125));
} }
if (isClockSkillLaunch && if (isClockSkillLaunch &&
@@ -276,10 +283,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
outboundAsrText, outboundAsrText,
outboundRules, outboundRules,
entities)), entities)),
DelayMs: 75)); 75));
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/clock")), JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/clock")),
DelayMs: 125)); 125));
} }
if ((isPhotoGalleryLaunch || isPhotoCreateLaunch) && if ((isPhotoGalleryLaunch || isPhotoCreateLaunch) &&
@@ -294,34 +301,16 @@ public sealed class ResponsePlanToSocketMessagesMapper
outboundAsrText, outboundAsrText,
outboundRules, outboundRules,
entities)), entities)),
DelayMs: 75)); 75));
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, skillId)), JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, skillId)),
DelayMs: 125)); 125));
}
if (isReportSkillLaunch)
{
messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildSkillRedirectPayload(
transId,
"report-skill",
outboundIntent,
outboundAsrText,
outboundRules,
entities)),
DelayMs: 75));
messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "report-skill")),
DelayMs: 125));
} }
if (emitSkillActions && speak is not null) if (emitSkillActions && speak is not null)
{
messages.Add(new SocketReplyPlan( messages.Add(new SocketReplyPlan(
JsonSerializer.Serialize(BuildSkillPayload(plan, turn, transId, speak, skill)), JsonSerializer.Serialize(BuildSkillPayload(plan, turn, transId, speak, skill)),
DelayMs: 75)); 75));
}
return messages; return messages;
} }
@@ -367,7 +356,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
transID = transId, transID = transId,
data = new { } data = new { }
})), })),
new SocketReplyPlan(JsonSerializer.Serialize(BuildGenericFallbackSkillPayload(transId)), DelayMs: 75) new SocketReplyPlan(JsonSerializer.Serialize(BuildGenericFallbackSkillPayload(transId)), 75)
]; ];
} }
@@ -442,10 +431,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
? "clientRules" ? "clientRules"
: "listenRules"; : "listenRules";
if (!turn.Attributes.TryGetValue(attributeName, out var value)) if (!turn.Attributes.TryGetValue(attributeName, out var value)) return [];
{
return [];
}
return value switch return value switch
{ {
@@ -481,10 +467,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
{ {
if (yesNoTurn) if (yesNoTurn)
{ {
if (!includeCreateDomain) if (!includeCreateDomain) return new Dictionary<string, object?>();
{
return new Dictionary<string, object?>();
}
return new Dictionary<string, object?> return new Dictionary<string, object?>
{ {
@@ -493,20 +476,15 @@ public sealed class ResponsePlanToSocketMessagesMapper
} }
if (wordOfDayLaunch) if (wordOfDayLaunch)
{
return new Dictionary<string, object?> return new Dictionary<string, object?>
{ {
["domain"] = "word-of-the-day" ["domain"] = "word-of-the-day"
}; };
}
if (globalCommand) if (globalCommand)
{ {
var entities = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase); var entities = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(volumeLevel)) if (!string.IsNullOrWhiteSpace(volumeLevel)) entities["volumeLevel"] = volumeLevel;
{
entities["volumeLevel"] = volumeLevel;
}
return entities; return entities;
} }
@@ -514,10 +492,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
if (radioLaunch) if (radioLaunch)
{ {
var entities = new Dictionary<string, object?>(); var entities = new Dictionary<string, object?>();
if (!string.IsNullOrWhiteSpace(radioStation)) if (!string.IsNullOrWhiteSpace(radioStation)) entities["station"] = radioStation;
{
entities["station"] = radioStation;
}
return entities; return entities;
} }
@@ -525,10 +500,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
if (clockSkillLaunch) if (clockSkillLaunch)
{ {
var entities = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase); var entities = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(clockDomain)) if (!string.IsNullOrWhiteSpace(clockDomain)) entities["domain"] = clockDomain;
{
entities["domain"] = clockDomain;
}
if (string.Equals(clockDomain, "timer", StringComparison.OrdinalIgnoreCase) && if (string.Equals(clockDomain, "timer", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(timerHours + timerMinutes + timerSeconds)) !string.IsNullOrWhiteSpace(timerHours + timerMinutes + timerSeconds))
@@ -550,32 +522,22 @@ public sealed class ResponsePlanToSocketMessagesMapper
if (reportSkillLaunch) if (reportSkillLaunch)
{ {
var entities = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase); var entities = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(reportDate)) if (!string.IsNullOrWhiteSpace(reportDate)) entities["date"] = reportDate;
{
entities["date"] = reportDate;
}
if (!string.IsNullOrWhiteSpace(reportWeatherCondition)) if (!string.IsNullOrWhiteSpace(reportWeatherCondition)) entities["Weather"] = reportWeatherCondition;
{
entities["Weather"] = reportWeatherCondition;
}
return entities; return entities;
} }
if (wordOfDayGuess) if (wordOfDayGuess)
{
return new Dictionary<string, object?> return new Dictionary<string, object?>
{ {
["guess"] = guess ?? string.Empty ["guess"] = guess ?? string.Empty
}; };
}
if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) || if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ||
!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null) !turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
{
return new Dictionary<string, object?>(); return new Dictionary<string, object?>();
}
return value switch return value switch
{ {
@@ -611,10 +573,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static IEnumerable<string> ReadRuleValues(TurnContext turn, string key) private static IEnumerable<string> ReadRuleValues(TurnContext turn, string key)
{ {
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return [];
{
return [];
}
return value switch return value switch
{ {
@@ -636,10 +595,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static string? ReadClientEntity(TurnContext turn, string entityName) private static string? ReadClientEntity(TurnContext turn, string entityName)
{ {
if (!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null) if (!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null) return null;
{
return null;
}
return value switch return value switch
{ {
@@ -657,20 +613,14 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static string? ReadSkillPayloadString(InvokeNativeSkillAction? skill, string key) private static string? ReadSkillPayloadString(InvokeNativeSkillAction? skill, string key)
{ {
if (skill?.Payload is null || !skill.Payload.TryGetValue(key, out var value)) if (skill?.Payload is null || !skill.Payload.TryGetValue(key, out var value)) return null;
{
return null;
}
return value?.ToString(); return value?.ToString();
} }
private static string ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess) private static string ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess)
{ {
if (!string.IsNullOrWhiteSpace(nluGuess)) if (!string.IsNullOrWhiteSpace(nluGuess)) return nluGuess;
{
return nluGuess;
}
var normalized = NormalizeGuessToken(transcript); var normalized = NormalizeGuessToken(transcript);
var hintIndex = normalized switch var hintIndex = normalized switch
@@ -684,11 +634,9 @@ public sealed class ResponsePlanToSocketMessagesMapper
var hints = ReadRuleValues(turn, "listenAsrHints").ToArray(); var hints = ReadRuleValues(turn, "listenAsrHints").ToArray();
if (hintIndex >= 0) if (hintIndex >= 0)
{
return hintIndex < hints.Length return hintIndex < hints.Length
? hints[hintIndex] ? hints[hintIndex]
: transcript; : transcript;
}
var fuzzyHintMatch = FindClosestHint(normalized, hints); var fuzzyHintMatch = FindClosestHint(normalized, hints);
return string.IsNullOrWhiteSpace(fuzzyHintMatch) return string.IsNullOrWhiteSpace(fuzzyHintMatch)
@@ -698,31 +646,19 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static string? FindClosestHint(string normalizedTranscript, IReadOnlyList<string> hints) private static string? FindClosestHint(string normalizedTranscript, IReadOnlyList<string> hints)
{ {
if (string.IsNullOrWhiteSpace(normalizedTranscript)) if (string.IsNullOrWhiteSpace(normalizedTranscript)) return null;
{
return null;
}
string? bestHint = null; string? bestHint = null;
var bestDistance = int.MaxValue; var bestDistance = int.MaxValue;
foreach (var hint in hints) foreach (var hint in hints)
{ {
if (string.IsNullOrWhiteSpace(hint)) if (string.IsNullOrWhiteSpace(hint)) continue;
{
continue;
}
var normalizedHint = NormalizeGuessToken(hint); var normalizedHint = NormalizeGuessToken(hint);
if (string.IsNullOrWhiteSpace(normalizedHint)) if (string.IsNullOrWhiteSpace(normalizedHint)) continue;
{
continue;
}
if (string.Equals(normalizedTranscript, normalizedHint, StringComparison.Ordinal)) if (string.Equals(normalizedTranscript, normalizedHint, StringComparison.Ordinal)) return hint;
{
return hint;
}
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint); var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance >= bestDistance) continue; if (distance >= bestDistance) continue;
@@ -744,10 +680,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
var previous = new int[right.Length + 1]; var previous = new int[right.Length + 1];
var current = new int[right.Length + 1]; var current = new int[right.Length + 1];
for (var column = 0; column <= right.Length; column += 1) for (var column = 0; column <= right.Length; column += 1) previous[column] = column;
{
previous[column] = column;
}
for (var row = 1; row <= left.Length; row += 1) for (var row = 1; row <= left.Length; row += 1)
{ {
@@ -772,11 +705,9 @@ public sealed class ResponsePlanToSocketMessagesMapper
var skillPayload = skill?.Payload; var skillPayload = skill?.Payload;
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only", if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only",
StringComparison.OrdinalIgnoreCase)) StringComparison.OrdinalIgnoreCase))
{
return BuildCompletionOnlySkillPayload( return BuildCompletionOnlySkillPayload(
transId, transId,
ReadPayloadString(skillPayload, "skillId") ?? skill?.SkillName ?? "chitchat-skill"); ReadPayloadString(skillPayload, "skillId") ?? skill?.SkillName ?? "chitchat-skill");
}
var isJoke = string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase) || var isJoke = string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase) ||
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase); string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
@@ -812,19 +743,21 @@ public sealed class ResponsePlanToSocketMessagesMapper
}; };
if (listenContexts.Count > 0) if (listenContexts.Count > 0)
{
jcpConfig["listen"] = new jcpConfig["listen"] = new
{ {
id = CreateProtocolId(), id = CreateProtocolId(),
type = "LISTEN", type = "LISTEN",
contexts = listenContexts contexts = listenContexts
}; };
}
var weatherHiLoView = BuildWeatherHiLoView(skillPayload); var weatherHiLoView = BuildWeatherHiLoView(skillPayload);
var weeklyWeatherCards = BuildWeatherHiLoSequenceCards(skillPayload);
if (weatherHiLoView is null && weeklyWeatherCards.Count > 0) weatherHiLoView = weeklyWeatherCards[0].View;
var useWeatherSequence = false;
if (weatherHiLoView is not null) if (weatherHiLoView is not null)
{ {
var resolvedGuiConfig = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase) var resolvedGuiContext = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{ {
["type"] = "Javascript", ["type"] = "Javascript",
["data"] = weatherHiLoView, ["data"] = weatherHiLoView,
@@ -841,12 +774,21 @@ public sealed class ResponsePlanToSocketMessagesMapper
jcpConfig["gui"] = legacyGuiConfig; jcpConfig["gui"] = legacyGuiConfig;
jcpConfig["display"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase) jcpConfig["display"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{ {
["view"] = resolvedGuiConfig ["view"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
// Legacy fields used by existing tests and tooling.
["type"] = "Javascript",
["data"] = weatherHiLoView,
["pause"] = true,
// Pegasus-style view context used by on-robot weather cards.
["context"] = resolvedGuiContext
}
}; };
playConfig["gui"] = resolvedGuiConfig; jcpConfig["timeout"] = 6;
playConfig["no_matches_for_gui"] = 0; jcpConfig["barge_in"] = true;
playConfig["no_inputs_for_gui"] = 0; jcpConfig["no_matches_for_gui"] = 0;
jcpConfig["no_inputs_for_gui"] = 0;
var weatherViews = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase) var weatherViews = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{ {
@@ -860,6 +802,30 @@ public sealed class ResponsePlanToSocketMessagesMapper
{ {
["views"] = weatherViews ["views"] = weatherViews
}; };
if (weeklyWeatherCards.Count > 1)
{
useWeatherSequence = true;
jcpConfig["children"] = BuildWeatherHiLoSequenceChildren(
weeklyWeatherCards,
promptSubCategory,
mimId,
mimType);
}
}
var jcp = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["type"] = "SLIM",
["config"] = jcpConfig
};
if (useWeatherSequence &&
jcpConfig.TryGetValue("children", out var sequenceChildren) &&
sequenceChildren is not null)
{
jcp["type"] = "SEQUENCE";
jcp.Remove("config");
jcp["children"] = sequenceChildren;
} }
return new return new
@@ -878,11 +844,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
{ {
config = new config = new
{ {
jcp = new jcp
{
type = "SLIM",
config = jcpConfig
}
} }
}, },
analytics = new Dictionary<string, object?>(), analytics = new Dictionary<string, object?>(),
@@ -906,15 +868,9 @@ public sealed class ResponsePlanToSocketMessagesMapper
["entities"] = entities ["entities"] = entities
}; };
if (!string.IsNullOrWhiteSpace(skillId)) if (!string.IsNullOrWhiteSpace(skillId)) payload["skill"] = skillId;
{
payload["skill"] = skillId;
}
if (!string.IsNullOrWhiteSpace(domain)) if (!string.IsNullOrWhiteSpace(domain)) payload["domain"] = domain;
{
payload["domain"] = domain;
}
return payload; return payload;
} }
@@ -1077,65 +1033,217 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static string? ReadPayloadString(IDictionary<string, object?>? payload, string key) private static string? ReadPayloadString(IDictionary<string, object?>? payload, string key)
{ {
if (payload is null || !payload.TryGetValue(key, out var value)) if (payload is null || !payload.TryGetValue(key, out var value)) return null;
{
return null;
}
return value?.ToString(); return value?.ToString();
} }
private static IReadOnlyList<string> ReadPayloadStringArray(IDictionary<string, object?>? payload, string key) private static IReadOnlyList<string> ReadPayloadStringArray(IDictionary<string, object?>? payload, string key)
{ {
if (payload is null || !payload.TryGetValue(key, out var value) || value is null) if (payload is null || !payload.TryGetValue(key, out var value) || value is null) return [];
{
return [];
}
return value switch return value switch
{ {
string text => [.. text string text =>
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) [
.Where(static context => !string.IsNullOrWhiteSpace(context))], .. text
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Where(static context => !string.IsNullOrWhiteSpace(context))
],
string[] contexts => [.. contexts.Where(static context => !string.IsNullOrWhiteSpace(context))], string[] contexts => [.. contexts.Where(static context => !string.IsNullOrWhiteSpace(context))],
IEnumerable<string> contexts => [.. contexts.Where(static context => !string.IsNullOrWhiteSpace(context))], IEnumerable<string> contexts => [.. contexts.Where(static context => !string.IsNullOrWhiteSpace(context))],
JsonElement jsonElement when jsonElement.ValueKind == JsonValueKind.Array => [.. jsonElement JsonElement jsonElement when jsonElement.ValueKind == JsonValueKind.Array =>
.EnumerateArray() [
.Select(static item => item.GetString()) .. jsonElement
.Where(static context => !string.IsNullOrWhiteSpace(context)) .EnumerateArray()
.Select(static context => context!)], .Select(static item => item.GetString())
IEnumerable<object?> contexts => [.. contexts .Where(static context => !string.IsNullOrWhiteSpace(context))
.Select(static context => context?.ToString()) .Select(static context => context!)
.Where(static context => !string.IsNullOrWhiteSpace(context)) ],
.Select(static context => context!)], IEnumerable<object?> contexts =>
[
.. contexts
.Select(static context => context?.ToString())
.Where(static context => !string.IsNullOrWhiteSpace(context))
.Select(static context => context!)
],
_ => string.IsNullOrWhiteSpace(value.ToString()) ? [] : [value.ToString()!] _ => string.IsNullOrWhiteSpace(value.ToString()) ? [] : [value.ToString()!]
}; };
} }
private static IReadOnlyList<WeatherHiLoSequenceCard> BuildWeatherHiLoSequenceCards(
IDictionary<string, object?>? payload)
{
if (payload is null ||
!payload.TryGetValue("weather_weekly_cards", out var rawCards) ||
rawCards is null)
return [];
var cards = ReadPayloadObjectArray(rawCards);
if (cards.Count == 0) return [];
var sequenceCards = new List<WeatherHiLoSequenceCard>(cards.Count);
foreach (var card in cards)
{
var weatherCardPayload = new Dictionary<string, object?>(card, StringComparer.OrdinalIgnoreCase)
{
["weather_view_enabled"] = true,
["weather_view_kind"] = "weatherHiLo"
};
var view = BuildWeatherHiLoView(weatherCardPayload);
if (view is null) continue;
sequenceCards.Add(new WeatherHiLoSequenceCard(
view,
ReadPayloadString(weatherCardPayload, "weather_day"),
ReadPayloadString(weatherCardPayload, "weather_icon"),
ReadPayloadString(weatherCardPayload, "weather_spoken_line")));
}
return sequenceCards;
}
private static IReadOnlyList<object> BuildWeatherHiLoSequenceChildren(
IReadOnlyList<WeatherHiLoSequenceCard> cards,
string promptSubCategory,
string mimId,
string mimType)
{
var children = new List<object>(cards.Count);
for (var index = 0; index < cards.Count; index += 1)
{
var card = cards[index];
var promptLabel = string.IsNullOrWhiteSpace(card.DayName)
? $"Day{index + 1}"
: Regex.Replace(card.DayName, "[^A-Za-z0-9]", string.Empty, RegexOptions.CultureInvariant);
var promptId = $"WeatherForecast{promptLabel}_AN_13";
var spokenLine = string.IsNullOrWhiteSpace(card.SpokenLine)
? "Here is another day's forecast."
: card.SpokenLine!;
var icon = string.IsNullOrWhiteSpace(card.Icon)
? "cloudy"
: card.Icon!;
var esml =
$"<speak><anim cat='weather' meta='{icon}' nonBlocking='true' /><break size='0.2'/><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(spokenLine)}</es></speak>";
var resolvedGuiContext = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["type"] = "Javascript",
["data"] = card.View,
["pause"] = true
};
children.Add(new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["type"] = "SLIM",
["config"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["play"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["esml"] = esml,
["meta"] = new
{
prompt_id = promptId,
prompt_sub_category = promptSubCategory,
mim_id = mimId,
mim_type = mimType
}
},
["gui"] = new
{
type = "Javascript",
data = "views.weatherHiLo",
pause = true
},
["display"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["view"] = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["type"] = "Javascript",
["data"] = card.View,
["pause"] = true,
["context"] = resolvedGuiContext
}
},
["timeout"] = 6,
["barge_in"] = true,
["no_matches_for_gui"] = 0,
["no_inputs_for_gui"] = 0
}
});
}
return children;
}
private static IReadOnlyList<IDictionary<string, object?>> ReadPayloadObjectArray(object rawValue)
{
if (rawValue is JsonElement jsonArray && jsonArray.ValueKind == JsonValueKind.Array)
return jsonArray
.EnumerateArray()
.Select(ConvertJsonObjectToDictionary)
.Where(static item => item is not null)
.Cast<IDictionary<string, object?>>()
.ToArray();
if (rawValue is IEnumerable<object?> rawObjects)
return rawObjects
.Select(ConvertObjectToDictionary)
.Where(static item => item is not null)
.Cast<IDictionary<string, object?>>()
.ToArray();
return [];
}
private static IDictionary<string, object?>? ConvertObjectToDictionary(object? value)
{
if (value is null) return null;
if (value is IDictionary<string, object?> dictionary)
return new Dictionary<string, object?>(dictionary, StringComparer.OrdinalIgnoreCase);
return value is JsonElement jsonValue
? ConvertJsonObjectToDictionary(jsonValue)
: null;
}
private static IDictionary<string, object?>? ConvertJsonObjectToDictionary(JsonElement value)
{
if (value.ValueKind != JsonValueKind.Object) return null;
var dictionary = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
foreach (var property in value.EnumerateObject())
dictionary[property.Name] = property.Value.ValueKind switch
{
JsonValueKind.String => property.Value.GetString(),
JsonValueKind.Number when property.Value.TryGetInt32(out var intValue) => intValue,
JsonValueKind.Number when property.Value.TryGetDouble(out var doubleValue) => doubleValue,
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.Object => ConvertJsonObjectToDictionary(property.Value),
JsonValueKind.Array => property.Value,
_ => null
};
return dictionary;
}
private static object? BuildWeatherHiLoView(IDictionary<string, object?>? payload) private static object? BuildWeatherHiLoView(IDictionary<string, object?>? payload)
{ {
if (!TryReadPayloadBool(payload, "weather_view_enabled")) if (!TryReadPayloadBool(payload, "weather_view_enabled")) return null;
{
return null;
}
if (!string.Equals( if (!string.Equals(
ReadPayloadString(payload, "weather_view_kind"), ReadPayloadString(payload, "weather_view_kind"),
"weatherHiLo", "weatherHiLo",
StringComparison.OrdinalIgnoreCase)) StringComparison.OrdinalIgnoreCase))
{
return null; return null;
}
var icon = ReadPayloadString(payload, "weather_icon"); var icon = ReadPayloadString(payload, "weather_icon");
var unit = ReadPayloadString(payload, "weather_unit") ?? "F"; var unit = ReadPayloadString(payload, "weather_unit") ?? "F";
var theme = ReadPayloadString(payload, "weather_theme") ?? "Normal"; var theme = ReadPayloadString(payload, "weather_theme") ?? "Normal";
var high = TryReadPayloadInt(payload, "weather_high"); var high = TryReadPayloadInt(payload, "weather_high");
var low = TryReadPayloadInt(payload, "weather_low"); var low = TryReadPayloadInt(payload, "weather_low");
if (string.IsNullOrWhiteSpace(icon) || high is null || low is null) if (string.IsNullOrWhiteSpace(icon) || high is null || low is null) return null;
{
return null;
}
var hiNumX = GetTemperatureLabelXPosition(370, high.Value); var hiNumX = GetTemperatureLabelXPosition(370, high.Value);
var hiUnitX = GetTemperatureLabelXPosition(360, high.Value); var hiUnitX = GetTemperatureLabelXPosition(360, high.Value);
@@ -1197,7 +1305,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
{ {
id = "hiNumLabel", id = "hiNumLabel",
type = "Label", type = "Label",
text = $"{high.Value}\u00B0", text = $"{high.Value}°",
style = new style = new
{ {
fontSize = "160", fontSize = "160",
@@ -1229,7 +1337,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
{ {
id = "loNumLabel", id = "loNumLabel",
type = "Label", type = "Label",
text = $"{low.Value}\u00B0", text = $"{low.Value}°",
style = new style = new
{ {
fontSize = "160", fontSize = "160",
@@ -1294,24 +1402,16 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static int GetTemperatureLabelXPosition(int baseX, int temperature) private static int GetTemperatureLabelXPosition(int baseX, int temperature)
{ {
const int xOffset = 70; const int xOffset = 70;
if (temperature < -9 || temperature > 99) if (temperature < -9 || temperature > 99) return baseX + xOffset;
{
return baseX + xOffset;
}
if (temperature is >= 0 and < 10) if (temperature is >= 0 and < 10) return baseX - xOffset;
{
return baseX - xOffset;
}
return baseX; return baseX;
} }
private static int? TryReadPayloadInt(IDictionary<string, object?>? payload, string key) private static int? TryReadPayloadInt(IDictionary<string, object?>? payload, string key)
{ {
if (payload is null || !payload.TryGetValue(key, out var value) || value is null) if (payload is null || !payload.TryGetValue(key, out var value) || value is null) return null;
{
return null;
}
return value switch return value switch
{ {
@@ -1320,18 +1420,17 @@ public sealed class ResponsePlanToSocketMessagesMapper
double number => (int)Math.Round(number, MidpointRounding.AwayFromZero), double number => (int)Math.Round(number, MidpointRounding.AwayFromZero),
float number => (int)Math.Round(number, MidpointRounding.AwayFromZero), float number => (int)Math.Round(number, MidpointRounding.AwayFromZero),
string text when int.TryParse(text, out var parsed) => parsed, string text when int.TryParse(text, out var parsed) => parsed,
JsonElement { ValueKind: JsonValueKind.Number } jsonNumber when jsonNumber.TryGetInt32(out var parsed) => parsed, JsonElement { ValueKind: JsonValueKind.Number } jsonNumber when jsonNumber.TryGetInt32(out var parsed) =>
JsonElement jsonText when jsonText.ValueKind == JsonValueKind.String && int.TryParse(jsonText.GetString(), out var parsed) => parsed, parsed,
JsonElement jsonText when jsonText.ValueKind == JsonValueKind.String &&
int.TryParse(jsonText.GetString(), out var parsed) => parsed,
_ => null _ => null
}; };
} }
private static bool TryReadPayloadBool(IDictionary<string, object?>? payload, string key) private static bool TryReadPayloadBool(IDictionary<string, object?>? payload, string key)
{ {
if (payload is null || !payload.TryGetValue(key, out var value) || value is null) if (payload is null || !payload.TryGetValue(key, out var value) || value is null) return false;
{
return false;
}
return value switch return value switch
{ {
@@ -1339,7 +1438,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
string text when bool.TryParse(text, out var parsed) => parsed, string text when bool.TryParse(text, out var parsed) => parsed,
JsonElement { ValueKind: JsonValueKind.True } => true, JsonElement { ValueKind: JsonValueKind.True } => true,
JsonElement { ValueKind: JsonValueKind.False } => false, JsonElement { ValueKind: JsonValueKind.False } => false,
JsonElement jsonText when jsonText.ValueKind == JsonValueKind.String && bool.TryParse(jsonText.GetString(), out var parsed) => parsed, JsonElement jsonText when jsonText.ValueKind == JsonValueKind.String &&
bool.TryParse(jsonText.GetString(), out var parsed) => parsed,
_ => false _ => false
}; };
} }
@@ -1354,6 +1454,11 @@ public sealed class ResponsePlanToSocketMessagesMapper
return Guid.NewGuid().ToString("N"); return Guid.NewGuid().ToString("N");
} }
private sealed record WeatherHiLoSequenceCard(
object View,
string? DayName,
string? Icon,
string? SpokenLine);
public sealed record SocketReplyPlan(string Text, int DelayMs = 0); public sealed record SocketReplyPlan(string Text, int DelayMs = 0);
} }

View File

@@ -0,0 +1,139 @@
using Jibo.Cloud.Application.Abstractions;
namespace Jibo.Cloud.Application.Services;
internal static class ScriptedResponseDecisionBuilder
{
internal static JiboInteractionDecision BuildScriptedPersonalityDecision(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyPersonalityReply(catalog, randomizer, preferredSnippets),
ContextUpdates: BuildScriptedResponseContextUpdates());
}
internal static JiboInteractionDecision BuildScriptedFavoriteAnimalDecision(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(catalog.FavoriteAnimalReplies, randomizer, preferredSnippets),
ContextUpdates: BuildScriptedResponseContextUpdates());
}
internal static JiboInteractionDecision BuildScriptedGreetingDecision(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyGreetingReply(catalog, randomizer, preferredSnippets),
ContextUpdates: BuildScriptedResponseContextUpdates());
}
internal static JiboInteractionDecision BuildScriptedHolidayDecision(
IReadOnlyList<string> replies,
IJiboRandomizer randomizer,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(replies, randomizer, preferredSnippets),
ContextUpdates: BuildScriptedResponseContextUpdates());
}
internal static JiboInteractionDecision BuildScriptedHolidayTrackerDecision(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(catalog.HolidayTrackerReplies, randomizer, preferredSnippets),
ContextUpdates: BuildScriptedResponseContextUpdates());
}
internal static JiboInteractionDecision BuildScriptedHolidayGreetingDecision(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(catalog.HolidayGreetingReplies, randomizer, preferredSnippets),
ContextUpdates: BuildScriptedResponseContextUpdates());
}
internal static IDictionary<string, object?> BuildScriptedResponseContextUpdates()
{
return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
[ChitchatStateMachine.StateMetadataKey] = "complete",
[ChitchatStateMachine.RouteMetadataKey] = "ScriptedResponse",
[ChitchatStateMachine.EmotionMetadataKey] = string.Empty
};
}
internal static string SelectLegacyPersonalityReply(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
params string[] preferredSnippets)
{
foreach (var snippet in preferredSnippets)
{
if (string.IsNullOrWhiteSpace(snippet)) continue;
var match = catalog.PersonalityReplies.FirstOrDefault(reply =>
reply.Contains(snippet, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match)) return match;
}
return catalog.PersonalityReplies.Count == 0 ? string.Empty : randomizer.Choose(catalog.PersonalityReplies);
}
internal static string SelectLegacyGreetingReply(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
params string[] preferredSnippets)
{
foreach (var snippet in preferredSnippets)
{
if (string.IsNullOrWhiteSpace(snippet)) continue;
var match = catalog.GreetingReplies.FirstOrDefault(reply =>
reply.Contains(snippet, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match)) return match;
}
return catalog.GreetingReplies.Count == 0 ? string.Empty : randomizer.Choose(catalog.GreetingReplies);
}
internal static string SelectLegacyReply(
IReadOnlyList<string> replies,
IJiboRandomizer randomizer,
params string[] preferredSnippets)
{
foreach (var snippet in preferredSnippets)
{
if (string.IsNullOrWhiteSpace(snippet)) continue;
var match = replies.FirstOrDefault(reply =>
reply.Contains(snippet, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match)) return match;
}
return replies.Count == 0 ? string.Empty : randomizer.Choose(replies);
}
}

View File

@@ -0,0 +1,590 @@
using Jibo.Cloud.Application.Abstractions;
namespace Jibo.Cloud.Application.Services;
internal static class SeasonalHolidayRouteBuilder
{
internal static bool TryResolveSemanticIntent(string loweredTranscript, out string? semanticIntent)
{
if (MatchesAny(
loweredTranscript,
"do you like halloween",
"are you looking forward to halloween",
"do you like the halloween holiday"))
{
semanticIntent = "seasonal_likes_halloween";
return true;
}
if (MatchesAny(
loweredTranscript,
"do you like holiday music",
"do you like christmas music",
"do you like christmas songs",
"do you like holiday songs"))
{
semanticIntent = "seasonal_likes_holiday_music";
return true;
}
if (MatchesAny(
loweredTranscript,
"do you like holiday parties",
"do you like christmas parties",
"are you going to any holiday parties"))
{
semanticIntent = "seasonal_likes_holiday_parties";
return true;
}
if (MatchesAny(
loweredTranscript,
"are you looking forward to christmas",
"do you look forward to christmas",
"are you excited for christmas"))
{
semanticIntent = "seasonal_looks_forward_to_christmas";
return true;
}
if (MatchesAny(
loweredTranscript,
"what are you thankful for",
"what are you thankful for this year",
"what is jibo thankful for"))
{
semanticIntent = "seasonal_thankful_for";
return true;
}
if (MatchesAny(
loweredTranscript,
"what are you doing for christmas",
"what are your plans for christmas",
"what do you plan to do for christmas"))
{
semanticIntent = "seasonal_plans_for_christmas";
return true;
}
if (MatchesAny(
loweredTranscript,
"happy holidays",
"merry christmas",
"happy new year",
"season s greetings",
"seasons greetings"))
{
semanticIntent = "seasonal_holiday_greeting";
return true;
}
if (MatchesAny(
loweredTranscript,
"what holidays do you celebrate",
"what holidays are you celebrating",
"what holidays do you observe"))
{
semanticIntent = "seasonal_holidays";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is holiday season",
"how's holiday season",
"how is the holiday season",
"do you like holiday season",
"do you like the holiday season",
"what is your favorite holiday",
"what's your favorite holiday",
"what holiday do you like",
"what is holiday season like"))
{
semanticIntent = "seasonal_holiday_season";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is thanksgiving",
"how's thanksgiving",
"do you like thanksgiving",
"what do you think of thanksgiving"))
{
semanticIntent = "seasonal_thanksgiving";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is christmas",
"how's christmas",
"do you like christmas",
"what do you think of christmas"))
{
semanticIntent = "seasonal_christmas";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is hanukkah",
"how's hanukkah",
"do you like hanukkah",
"what do you think of hanukkah"))
{
semanticIntent = "seasonal_hanukkah";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is passover",
"how's passover",
"do you like passover",
"what do you think of passover"))
{
semanticIntent = "seasonal_passover";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is new years",
"how's new years",
"how is new year s",
"do you like new years",
"what do you think of new years"))
{
semanticIntent = "seasonal_new_years";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is valentines day",
"how's valentines day",
"do you like valentines day",
"what do you think of valentines day"))
{
semanticIntent = "seasonal_valentines_day";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is kwanzaa",
"how's kwanzaa",
"do you like kwanzaa",
"what do you think of kwanzaa"))
{
semanticIntent = "seasonal_kwanzaa";
return true;
}
if (MatchesAny(
loweredTranscript,
"how is easter",
"how's easter",
"do you like easter",
"what do you think of easter"))
{
semanticIntent = "seasonal_easter";
return true;
}
if (MatchesAny(
loweredTranscript,
"what is your new years resolution",
"what is your new year's resolution",
"what is your new year s resolution",
"what are your new years resolutions",
"what are your new year's resolutions",
"what are your new year s resolutions",
"do you have any new years resolutions"))
{
semanticIntent = "seasonal_new_years_resolution";
return true;
}
if (MatchesAny(
loweredTranscript,
"how are your new years resolutions going",
"how are your new year's resolutions going",
"how is your new years resolution going",
"how is your new year's resolution going",
"how are your resolutions going",
"how is your resolution going"))
{
semanticIntent = "seasonal_new_years_update";
return true;
}
if (MatchesAny(
loweredTranscript,
"what halloween costume",
"what are you going as for halloween",
"what costume are you wearing",
"what are you dressing as for halloween"))
{
semanticIntent = "seasonal_halloween_costume";
return true;
}
if (MatchesAny(
loweredTranscript,
"what should i do for first day of spring",
"what should i do for spring",
"what do i do for first day of spring"))
{
semanticIntent = "seasonal_first_day_spring";
return true;
}
if (MatchesAny(
loweredTranscript,
"what is spring like",
"how is spring",
"what do you think about spring"))
{
semanticIntent = "seasonal_spring";
return true;
}
if (MatchesAny(
loweredTranscript,
"do you like spring",
"do you like springtime",
"are you looking forward to spring",
"do you look forward to spring",
"are you excited for spring"))
{
semanticIntent = "seasonal_likes_spring";
return true;
}
if (MatchesAny(
loweredTranscript,
"what is summer like",
"how is summer",
"what do you think about summer"))
{
semanticIntent = "seasonal_summer";
return true;
}
if (MatchesAny(
loweredTranscript,
"do you like summer",
"do you like summertime",
"are you looking forward to summer",
"do you look forward to summer",
"are you excited for summer"))
{
semanticIntent = "seasonal_likes_summer";
return true;
}
if (MatchesAny(
loweredTranscript,
"what should i get for holiday",
"what should i get for christmas",
"what gift should i get for christmas",
"what should i get someone for the holidays"))
{
semanticIntent = "seasonal_holiday_gift";
return true;
}
if (MatchesAny(
loweredTranscript,
"show santa tracker",
"can you show santa tracker",
"santa tracker",
"where is santa",
"where is santa right now",
"can you show me santa tracker"))
{
semanticIntent = "seasonal_santa_tracker";
return true;
}
if (MatchesAny(
loweredTranscript,
"happy birthday",
"happy birthday jibo",
"happy birthday to you"))
{
semanticIntent = "birthday_celebration";
return true;
}
semanticIntent = null;
return false;
}
internal static bool TryBuildDecision(
string semanticIntent,
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
Func<string, string> holidayTemplateRenderer,
out JiboInteractionDecision? decision)
{
decision = semanticIntent switch
{
"seasonal_holiday_greeting" => ScriptedResponseDecisionBuilder.BuildScriptedHolidayGreetingDecision(
catalog,
randomizer,
semanticIntent,
"fun time of year",
"right back at you",
"and to you too"),
"seasonal_holidays" => BuildHolidayTemplateDecision(
catalog,
randomizer,
holidayTemplateRenderer,
semanticIntent,
"official owner can tell me which ones we'll celebrate together",
"going to the jibo's settings screen in the jibo app"),
"seasonal_holiday_season" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"festive",
"celebrate"),
"seasonal_thanksgiving" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"thanksgiving",
"turkey",
"stuffed"),
"seasonal_christmas" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"christmas",
"quality time",
"socks"),
"seasonal_hanukkah" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"hanukkah",
"dreidel",
"gift"),
"seasonal_passover" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"passover",
"matzah",
"next one"),
"seasonal_new_years" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"new year",
"resolutions",
"party"),
"seasonal_valentines_day" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"valentine",
"heart",
"flowers"),
"seasonal_kwanzaa" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"kwanzaa",
"gift",
"celebrate"),
"seasonal_easter" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"easter",
"bunny",
"egg"),
"seasonal_new_years_resolution" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"always trying to learn new skills",
"not eat bacon",
"learn a bunch of new skills",
"learn to walk",
"recognizing people's faces and voices"),
"seasonal_new_years_update" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"not eat bacon",
"learn some new skills",
"going well"),
"seasonal_halloween_costume" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"i haven't thought much about it yet",
"ask me again on halloween",
"you'll find out on halloween"),
"seasonal_first_day_spring" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"it's a great day, when spring is in the air"),
"seasonal_spring" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"the days get longer",
"spring is a great season"),
"seasonal_likes_spring" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"extra happy in the springtime",
"i do like spring"),
"seasonal_summer" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"going to the beach",
"summer is great"),
"seasonal_likes_summer" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"long days",
"summer is a very special season"),
"seasonal_holiday_gift" => BuildHolidayDecision(
catalog.HolidayGiftReplies,
randomizer,
semanticIntent,
"ask for a pet elephant",
"experience as a present",
"donate to charities in other people's names"),
"seasonal_likes_halloween" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"halloween is my favorite holiday",
"scary but also fun",
"jack-o-lantern"),
"seasonal_likes_holiday_music" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"holiday music",
"sing a few of them",
"frosty the snowman"),
"seasonal_likes_holiday_parties" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"holiday fun can be extra fun",
"dance party"),
"seasonal_looks_forward_to_christmas" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"really like times of giving and receiving",
"long way away",
"looking forward to christmas"),
"seasonal_plans_for_christmas" => BuildHolidayDecision(
catalog.HolidaySeasonReplies,
randomizer,
semanticIntent,
"christmas sweaters",
"wear one of my",
"be festive"),
"seasonal_thankful_for" => ScriptedResponseDecisionBuilder.BuildScriptedPersonalityDecision(
catalog,
randomizer,
semanticIntent,
"thankful for the people i know",
"and for penguins",
"thankful for"),
"seasonal_santa_tracker" => ScriptedResponseDecisionBuilder.BuildScriptedHolidayTrackerDecision(
catalog,
randomizer,
semanticIntent,
"santa tracker",
"let's see if i can spot him",
"deliveries",
"north pole"),
"birthday_celebration" => BuildHolidayDecision(
catalog.BirthdayCelebrationReplies,
randomizer,
semanticIntent,
"another year older",
"can't wait to see what you got me",
"powered on for the first time today"),
_ => null
};
return decision is not null;
}
private static JiboInteractionDecision BuildHolidayDecision(
IReadOnlyList<string> replies,
IJiboRandomizer randomizer,
string intentName,
params string[] preferredSnippets)
{
return new JiboInteractionDecision(
intentName,
SelectLegacyReply(replies, randomizer, preferredSnippets),
ContextUpdates: BuildContextUpdates());
}
private static JiboInteractionDecision BuildHolidayTemplateDecision(
JiboExperienceCatalog catalog,
IJiboRandomizer randomizer,
Func<string, string> holidayTemplateRenderer,
string intentName,
params string[] preferredSnippets)
{
var selected = SelectLegacyReply(catalog.HolidayReplies, randomizer, preferredSnippets);
return new JiboInteractionDecision(
intentName,
holidayTemplateRenderer(selected),
ContextUpdates: BuildContextUpdates());
}
private static IDictionary<string, object?> BuildContextUpdates()
{
return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
[ChitchatStateMachine.StateMetadataKey] = "complete",
[ChitchatStateMachine.RouteMetadataKey] = "ScriptedResponse",
[ChitchatStateMachine.EmotionMetadataKey] = string.Empty
};
}
private static string SelectLegacyReply(
IReadOnlyList<string> replies,
IJiboRandomizer randomizer,
params string[] preferredSnippets)
{
foreach (var snippet in preferredSnippets)
{
if (string.IsNullOrWhiteSpace(snippet)) continue;
var match = replies.FirstOrDefault(reply =>
reply.Contains(snippet, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match)) return match;
}
return replies.Count == 0 ? string.Empty : randomizer.Choose(replies);
}
private static bool MatchesAny(string loweredTranscript, params string[] phrases)
{
return phrases.Any(phrase => loweredTranscript.Contains(phrase, StringComparison.OrdinalIgnoreCase));
}
}

View File

@@ -1,3 +1,4 @@
using System.Text.RegularExpressions;
using Jibo.Runtime.Abstractions; using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Application.Services; namespace Jibo.Cloud.Application.Services;
@@ -16,13 +17,11 @@ public sealed class SyntheticBufferedAudioSttStrategy : ISttStrategy
{ {
var transcriptHint = ReadTranscriptHint(turn); var transcriptHint = ReadTranscriptHint(turn);
if (string.IsNullOrWhiteSpace(transcriptHint)) if (string.IsNullOrWhiteSpace(transcriptHint))
{
throw new InvalidOperationException("Synthetic buffered audio STT requires an audio transcript hint."); throw new InvalidOperationException("Synthetic buffered audio STT requires an audio transcript hint.");
}
return Task.FromResult(new SttResult return Task.FromResult(new SttResult
{ {
Text = transcriptHint.Trim(), Text = NormalizeLooseTranscript(transcriptHint),
Provider = Name, Provider = Name,
Confidence = 0.75f, Confidence = 0.75f,
Locale = turn.Locale, Locale = turn.Locale,
@@ -36,10 +35,7 @@ public sealed class SyntheticBufferedAudioSttStrategy : ISttStrategy
private static int ReadBufferedAudioBytes(TurnContext turn) private static int ReadBufferedAudioBytes(TurnContext turn)
{ {
if (!turn.Attributes.TryGetValue("bufferedAudioBytes", out var bufferedAudioBytes)) if (!turn.Attributes.TryGetValue("bufferedAudioBytes", out var bufferedAudioBytes)) return 0;
{
return 0;
}
return bufferedAudioBytes switch return bufferedAudioBytes switch
{ {
@@ -56,4 +52,16 @@ public sealed class SyntheticBufferedAudioSttStrategy : ISttStrategy
? transcriptHint?.ToString() ? transcriptHint?.ToString()
: null; : null;
} }
private static string NormalizeLooseTranscript(string? value)
{
if (string.IsNullOrWhiteSpace(value)) return string.Empty;
var lowered = value.Trim().ToLowerInvariant();
lowered = Regex.Replace(lowered, @"[^\p{L}\p{N}\s']+", " ",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
lowered = Regex.Replace(lowered, @"\s+", " ",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
return lowered.Trim();
}
} }

View File

@@ -0,0 +1,58 @@
using System.Text.RegularExpressions;
namespace Jibo.Cloud.Application.Services;
internal static class TranscriptTextNormalizer
{
private static readonly Regex PunctuationToSpaceRegex = new(
@"[^\p{L}\p{N}\s']+",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex WhitespaceRegex = new(
@"\s+",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
internal static string NormalizeLooseText(string? value)
{
if (string.IsNullOrWhiteSpace(value)) return string.Empty;
return WhitespaceRegex.Replace(
PunctuationToSpaceRegex.Replace(value.Trim().ToLowerInvariant(), " "),
" ")
.Trim();
}
internal static string StripLeadingPhrases(string value, params string[] phrases)
{
if (string.IsNullOrWhiteSpace(value) || phrases.Length == 0) return value;
var normalized = value;
while (TryStripLeadingPhrase(normalized, phrases, out var trimmed))
normalized = trimmed;
return normalized;
}
private static bool TryStripLeadingPhrase(string normalizedValue, IReadOnlyList<string> phrases, out string trimmed)
{
foreach (var phrase in phrases)
{
if (string.IsNullOrWhiteSpace(phrase)) continue;
if (string.Equals(normalizedValue, phrase, StringComparison.Ordinal))
{
trimmed = string.Empty;
return true;
}
if (normalizedValue.StartsWith($"{phrase} ", StringComparison.Ordinal))
{
trimmed = normalizedValue[(phrase.Length + 1)..].TrimStart();
return true;
}
}
trimmed = normalizedValue;
return false;
}
}

View File

@@ -0,0 +1,16 @@
namespace Jibo.Cloud.Domain.Models;
public sealed class CalendarEventRecord
{
public string Id { get; init; } = $"calendar-{Guid.NewGuid():N}";
public string LoopId { get; init; } = "openjibo-default-loop";
public string Summary { get; init; } = "Calendar event";
public string? TimeLabel { get; init; }
public DateOnly Date { get; init; } = DateOnly.FromDateTime(DateTime.UtcNow);
public DateOnly? EndDate { get; init; }
public bool IsAllDay { get; init; }
public bool IsEnabled { get; init; } = true;
public string Source { get; init; } = "manual";
public string? MemberId { get; init; }
public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow;
}

View File

@@ -7,5 +7,7 @@ public sealed class CapturedExchange
public ProtocolEnvelope Request { get; init; } = new(); public ProtocolEnvelope Request { get; init; } = new();
public ProtocolDispatchResult Response { get; init; } = ProtocolDispatchResult.Ok(); public ProtocolDispatchResult Response { get; init; } = ProtocolDispatchResult.Ok();
public string Confidence { get; init; } = "observed"; public string Confidence { get; init; } = "observed";
public IDictionary<string, string> Tags { get; init; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public IDictionary<string, string> Tags { get; init; } =
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
} }

View File

@@ -0,0 +1,18 @@
namespace Jibo.Cloud.Domain.Models;
public sealed class CommuteProfileRecord
{
public string Id { get; init; } = $"commute-{Guid.NewGuid():N}";
public string LoopId { get; init; } = "openjibo-default-loop";
public string? MemberId { get; init; }
public bool IsEnabled { get; init; } = true;
public bool IsComplete { get; init; } = true;
public string Mode { get; init; } = "driving";
public int WorkHour { get; init; } = 8;
public int WorkMinute { get; init; } = 30;
public string? OriginName { get; init; } = "home";
public string? DestinationName { get; init; } = "work";
public int TypicalDurationMinutes { get; init; } = 25;
public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow;
public DateTimeOffset Updated { get; init; } = DateTimeOffset.UtcNow;
}

View File

@@ -8,5 +8,7 @@ public sealed class DeviceRegistration
public string? FirmwareVersion { get; init; } public string? FirmwareVersion { get; init; }
public string? ApplicationVersion { get; init; } public string? ApplicationVersion { get; init; }
public bool IsActive { get; init; } = true; public bool IsActive { get; init; } = true;
public IDictionary<string, string> HostMappings { get; init; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public IDictionary<string, string> HostMappings { get; init; } =
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
} }

View File

@@ -0,0 +1,17 @@
namespace Jibo.Cloud.Domain.Models;
public sealed class GreetingPresenceRecord
{
public string Id { get; init; } = $"greeting-presence-{Guid.NewGuid():N}";
public string AccountId { get; init; } = "usr_openjibo_owner";
public string LoopId { get; init; } = "openjibo-default-loop";
public string PersonId { get; init; } = string.Empty;
public string? SpeakerId { get; init; }
public string? PreferredName { get; init; }
public DateTimeOffset LastSeenUtc { get; init; } = DateTimeOffset.UtcNow;
public DateTimeOffset? LastGreetedUtc { get; init; }
public string? LastGreetingRoute { get; init; }
public string? LastGreetingIntent { get; init; }
public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow;
public DateTimeOffset UpdatedUtc { get; init; } = DateTimeOffset.UtcNow;
}

View File

@@ -0,0 +1,18 @@
namespace Jibo.Cloud.Domain.Models;
public sealed class HolidayRecord
{
public string Id { get; init; } = $"holiday-{Guid.NewGuid():N}";
public string EventId { get; init; } = string.Empty;
public string Name { get; init; } = "Holiday";
public string Category { get; init; } = "holiday";
public string? Subcategory { get; init; }
public string LoopId { get; init; } = "openjibo-default-loop";
public string? MemberId { get; init; }
public bool IsEnabled { get; init; } = true;
public DateOnly Date { get; init; } = DateOnly.FromDateTime(DateTime.UtcNow);
public DateOnly? EndDate { get; init; }
public string Source { get; init; } = "nager-date";
public string CountryCode { get; init; } = "US";
public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow;
}

View File

@@ -0,0 +1,14 @@
namespace Jibo.Cloud.Domain.Models;
public sealed class PersonRecord
{
public string PersonId { get; init; } = "person-openjibo-owner";
public string AccountId { get; init; } = "usr_openjibo_owner";
public string LoopId { get; init; } = "openjibo-default-loop";
public string RobotId { get; init; } = "my-robot-name";
public string DisplayName { get; init; } = "Jibo Owner";
public string? Alias { get; init; }
public bool IsPrimary { get; init; } = true;
public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow;
public DateTimeOffset UpdatedUtc { get; init; } = DateTimeOffset.UtcNow;
}

View File

@@ -7,7 +7,9 @@ public sealed class ProtocolDispatchResult
public int StatusCode { get; init; } = 200; public int StatusCode { get; init; } = 200;
public string ContentType { get; init; } = "application/x-amz-json-1.1"; public string ContentType { get; init; } = "application/x-amz-json-1.1";
public string BodyText { get; init; } = "{}"; public string BodyText { get; init; } = "{}";
public IDictionary<string, string> Headers { get; init; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public IDictionary<string, string> Headers { get; init; } =
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public static ProtocolDispatchResult Ok(object? body = null) public static ProtocolDispatchResult Ok(object? body = null)
{ {

View File

@@ -17,14 +17,13 @@ public sealed class ProtocolEnvelope
public string? FirmwareVersion { get; init; } public string? FirmwareVersion { get; init; }
public string? ApplicationVersion { get; init; } public string? ApplicationVersion { get; init; }
public string BodyText { get; init; } = string.Empty; public string BodyText { get; init; } = string.Empty;
public IDictionary<string, string> Headers { get; init; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public IDictionary<string, string> Headers { get; init; } =
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public JsonElement? TryParseBody() public JsonElement? TryParseBody()
{ {
if (string.IsNullOrWhiteSpace(BodyText)) if (string.IsNullOrWhiteSpace(BodyText)) return null;
{
return null;
}
try try
{ {

View File

@@ -20,5 +20,7 @@ public sealed class WebSocketTelemetryRecord
public int BufferedAudioChunks { get; init; } public int BufferedAudioChunks { get; init; }
public int FinalizeAttempts { get; init; } public int FinalizeAttempts { get; init; }
public bool AwaitingTurnCompletion { get; init; } public bool AwaitingTurnCompletion { get; init; }
public IReadOnlyDictionary<string, object?> Details { get; init; } = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
public IReadOnlyDictionary<string, object?> Details { get; init; } =
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
} }

View File

@@ -0,0 +1,24 @@
using System.Text.RegularExpressions;
namespace Jibo.Cloud.Infrastructure.Audio;
internal static class AudioTranscriptNormalizer
{
private static readonly Regex PunctuationToSpaceRegex = new(
@"[^\p{L}\p{N}\s']+",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex WhitespaceRegex = new(
@"\s+",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
public static string NormalizeLooseTranscript(string? value)
{
if (string.IsNullOrWhiteSpace(value)) return string.Empty;
return WhitespaceRegex.Replace(
PunctuationToSpaceRegex.Replace(value.Trim().ToLowerInvariant(), " "),
" ")
.Trim();
}
}

View File

@@ -4,7 +4,8 @@ namespace Jibo.Cloud.Infrastructure.Audio;
public sealed class ExternalProcessRunner : IExternalProcessRunner public sealed class ExternalProcessRunner : IExternalProcessRunner
{ {
public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default) public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments,
CancellationToken cancellationToken = default)
{ {
using var process = new Process(); using var process = new Process();
process.StartInfo = new ProcessStartInfo process.StartInfo = new ProcessStartInfo
@@ -16,10 +17,7 @@ public sealed class ExternalProcessRunner : IExternalProcessRunner
CreateNoWindow = true CreateNoWindow = true
}; };
foreach (var argument in arguments) foreach (var argument in arguments) process.StartInfo.ArgumentList.Add(argument);
{
process.StartInfo.ArgumentList.Add(argument);
}
process.Start(); process.Start();

View File

@@ -2,7 +2,8 @@ namespace Jibo.Cloud.Infrastructure.Audio;
public interface IExternalProcessRunner public interface IExternalProcessRunner
{ {
Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default); Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments,
CancellationToken cancellationToken = default);
} }
public sealed record ExternalProcessResult(int ExitCode, string StdOut, string StdErr); public sealed record ExternalProcessResult(int ExitCode, string StdOut, string StdErr);

View File

@@ -7,35 +7,37 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
BufferedAudioSttOptions options, BufferedAudioSttOptions options,
IExternalProcessRunner processRunner) : ISttStrategy IExternalProcessRunner processRunner) : ISttStrategy
{ {
private const int MinimumBufferedAudioBytes = 64;
private const int ShortAnswerBufferedAudioBytes = 16;
public string Name => "local-whispercpp-buffered-audio"; public string Name => "local-whispercpp-buffered-audio";
public bool CanHandle(TurnContext turn) public bool CanHandle(TurnContext turn)
{ {
return options.EnableLocalWhisperCpp && return options.EnableLocalWhisperCpp &&
IsConfiguredPathAvailable(options.FfmpegPath, checkFileExists: false) && IsConfiguredPathAvailable(options.FfmpegPath, false) &&
IsConfiguredPathAvailable(options.WhisperCliPath, checkFileExists: true) && IsConfiguredPathAvailable(options.WhisperCliPath, true) &&
IsConfiguredPathAvailable(options.WhisperModelPath, checkFileExists: true) && IsConfiguredPathAvailable(options.WhisperModelPath, true) &&
ReadBufferedAudioFrames(turn).Any(ContainsOpusIdentificationHeader); ReadBufferedAudioFrames(turn).Any(ContainsOpusIdentificationHeader) &&
!IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn));
} }
public async Task<SttResult> TranscribeAsync(TurnContext turn, CancellationToken cancellationToken = default) public async Task<SttResult> TranscribeAsync(TurnContext turn, CancellationToken cancellationToken = default)
{ {
var frames = ReadBufferedAudioFrames(turn); var frames = ReadBufferedAudioFrames(turn);
if (frames.Count == 0) if (frames.Count == 0)
{
throw new InvalidOperationException("Local whisper.cpp STT requires buffered websocket audio frames."); throw new InvalidOperationException("Local whisper.cpp STT requires buffered websocket audio frames.");
}
if (!frames.Any(ContainsOpusIdentificationHeader)) if (!frames.Any(ContainsOpusIdentificationHeader))
{ throw new InvalidOperationException(
throw new InvalidOperationException("Local whisper.cpp STT requires buffered Ogg/Opus audio with an Opus identification header."); "Local whisper.cpp STT requires buffered Ogg/Opus audio with an Opus identification header.");
}
if (IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn)))
throw new InvalidOperationException(
"Local whisper.cpp STT rejected buffered audio as too short or noisy for transcription.");
var tempDirectory = options.TempDirectory; var tempDirectory = options.TempDirectory;
if (string.IsNullOrWhiteSpace(tempDirectory)) if (string.IsNullOrWhiteSpace(tempDirectory)) tempDirectory = Path.Combine(Path.GetTempPath(), "openjibo-stt");
{
tempDirectory = Path.Combine(Path.GetTempPath(), "openjibo-stt");
}
Directory.CreateDirectory(tempDirectory); Directory.CreateDirectory(tempDirectory);
@@ -58,10 +60,9 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
cancellationToken); cancellationToken);
var transcript = ExtractTranscript(whisperResult.StdOut); var transcript = ExtractTranscript(whisperResult.StdOut);
transcript = AudioTranscriptNormalizer.NormalizeLooseTranscript(transcript);
if (string.IsNullOrWhiteSpace(transcript)) if (string.IsNullOrWhiteSpace(transcript))
{
throw new InvalidOperationException("whisper.cpp returned no transcript for the buffered audio turn."); throw new InvalidOperationException("whisper.cpp returned no transcript for the buffered audio turn.");
}
return new SttResult return new SttResult
{ {
@@ -90,10 +91,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
private static IReadOnlyList<byte[]> ReadBufferedAudioFrames(TurnContext turn) private static IReadOnlyList<byte[]> ReadBufferedAudioFrames(TurnContext turn)
{ {
if (!turn.Attributes.TryGetValue("bufferedAudioFrames", out var value) || value is null) if (!turn.Attributes.TryGetValue("bufferedAudioFrames", out var value) || value is null) return [];
{
return [];
}
return value switch return value switch
{ {
@@ -110,7 +108,8 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
private static int ReadBufferedAudioBytes(TurnContext turn) private static int ReadBufferedAudioBytes(TurnContext turn)
{ {
return turn.Attributes.TryGetValue("bufferedAudioBytes", out var bufferedAudioBytes) && bufferedAudioBytes is not null return turn.Attributes.TryGetValue("bufferedAudioBytes", out var bufferedAudioBytes) &&
bufferedAudioBytes is not null
? bufferedAudioBytes switch ? bufferedAudioBytes switch
{ {
int value => value, int value => value,
@@ -121,6 +120,56 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
: 0; : 0;
} }
private static bool IsBelowNoiseFloor(TurnContext turn, int bufferedAudioBytes)
{
if (bufferedAudioBytes <= 0) return false;
var minimumBufferedAudioBytes = IsShortAnswerTurn(turn)
? ShortAnswerBufferedAudioBytes
: MinimumBufferedAudioBytes;
return bufferedAudioBytes < minimumBufferedAudioBytes;
}
private static bool IsShortAnswerTurn(TurnContext turn)
{
var rules = ReadRules(turn, "listenRules")
.Concat(ReadRules(turn, "clientRules"))
.Concat(ReadRules(turn, "listenAsrHints"));
return rules.Any(IsShortAnswerRule);
}
private static bool IsShortAnswerRule(string rule)
{
return string.Equals(rule, "$YESNO", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "clock/alarm_timer_change", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "clock/alarm_timer_none_set", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "create/is_it_a_keeper", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "settings/download_now_later", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "shared/yes_no", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "surprises-date/offer_date_fact", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "surprises-ota/want_to_download_now", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/surprise", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/right_word", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/puzzle", StringComparison.OrdinalIgnoreCase);
}
private static IEnumerable<string> ReadRules(TurnContext turn, string key)
{
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return [];
return value switch
{
IReadOnlyList<string> typed => typed,
IEnumerable<string> enumerable => enumerable,
JsonElement { ValueKind: JsonValueKind.Array } jsonElement => jsonElement.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty),
_ => []
};
}
private static bool ContainsOpusIdentificationHeader(byte[] frame) private static bool ContainsOpusIdentificationHeader(byte[] frame)
{ {
return frame.AsSpan().IndexOf("OpusHead"u8) >= 0; return frame.AsSpan().IndexOf("OpusHead"u8) >= 0;
@@ -148,10 +197,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
{ {
try try
{ {
if (File.Exists(path)) if (File.Exists(path)) File.Delete(path);
{
File.Delete(path);
}
} }
catch catch
{ {
@@ -161,15 +207,9 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
private static bool IsConfiguredPathAvailable(string? path, bool checkFileExists) private static bool IsConfiguredPathAvailable(string? path, bool checkFileExists)
{ {
if (string.IsNullOrWhiteSpace(path)) if (string.IsNullOrWhiteSpace(path)) return false;
{
return false;
}
if (!Path.IsPathRooted(path)) if (!Path.IsPathRooted(path)) return true;
{
return true;
}
return !checkFileExists || File.Exists(path); return !checkFileExists || File.Exists(path);
} }

View File

@@ -9,10 +9,7 @@ internal static class OggOpusAudioNormalizer
public static byte[] Normalize(IReadOnlyList<byte[]> pages) public static byte[] Normalize(IReadOnlyList<byte[]> pages)
{ {
if (pages.Count == 0) if (pages.Count == 0) return [];
{
return [];
}
var parsed = pages.Select(ParsePage).ToArray(); var parsed = pages.Select(ParsePage).ToArray();
var baseGranule = parsed.Length > 1 ? parsed[1].GranulePosition : parsed[0].GranulePosition; var baseGranule = parsed.Length > 1 ? parsed[1].GranulePosition : parsed[0].GranulePosition;
@@ -50,26 +47,17 @@ internal static class OggOpusAudioNormalizer
private static ParsedOggPage ParsePage(byte[] buffer) private static ParsedOggPage ParsePage(byte[] buffer)
{ {
if (buffer.Length < 27) if (buffer.Length < 27)
{
throw new InvalidOperationException($"Buffered Ogg page is too short ({buffer.Length} bytes)."); throw new InvalidOperationException($"Buffered Ogg page is too short ({buffer.Length} bytes).");
}
if (!Encoding.ASCII.GetString(buffer, 0, 4).Equals("OggS", StringComparison.Ordinal)) if (!Encoding.ASCII.GetString(buffer, 0, 4).Equals("OggS", StringComparison.Ordinal))
{
throw new InvalidOperationException("Buffered audio frame did not begin with an OggS capture pattern."); throw new InvalidOperationException("Buffered audio frame did not begin with an OggS capture pattern.");
}
var pageSegments = buffer[26]; var pageSegments = buffer[26];
if (buffer.Length < 27 + pageSegments) if (buffer.Length < 27 + pageSegments)
{
throw new InvalidOperationException("Buffered Ogg page segment table was truncated."); throw new InvalidOperationException("Buffered Ogg page segment table was truncated.");
}
var payloadLength = 0; var payloadLength = 0;
for (var index = 0; index < pageSegments; index += 1) for (var index = 0; index < pageSegments; index += 1) payloadLength += buffer[27 + index];
{
payloadLength += buffer[27 + index];
}
var expectedLength = 27 + pageSegments + payloadLength; var expectedLength = 27 + pageSegments + payloadLength;
return buffer.Length < expectedLength return buffer.Length < expectedLength
@@ -79,7 +67,8 @@ internal static class OggOpusAudioNormalizer
private static uint ComputeCrc(byte[] buffer) private static uint ComputeCrc(byte[] buffer)
{ {
return buffer.Aggregate<byte, uint>(0, (current, value) => (current << 8) ^ CrcTable[((current >> 24) ^ value) & 0xff]); return buffer.Aggregate<byte, uint>(0,
(current, value) => (current << 8) ^ CrcTable[((current >> 24) ^ value) & 0xff]);
} }
private static uint[] BuildCrcTable() private static uint[] BuildCrcTable()
@@ -89,11 +78,9 @@ internal static class OggOpusAudioNormalizer
{ {
var remainder = index << 24; var remainder = index << 24;
for (var bit = 0; bit < 8; bit += 1) for (var bit = 0; bit < 8; bit += 1)
{
remainder = (remainder & 0x80000000) != 0 remainder = (remainder & 0x80000000) != 0
? (remainder << 1) ^ 0x04c11db7 ? (remainder << 1) ^ 0x04c11db7
: remainder << 1; : remainder << 1;
}
table[index] = remainder; table[index] = remainder;
} }

View File

@@ -0,0 +1,72 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Infrastructure.Calendar;
public sealed class CloudStateCalendarReportProvider(ICloudStateStore cloudStateStore) : ICalendarReportProvider
{
public Task<CalendarReportSnapshot?> GetReportAsync(
TurnContext turn,
CancellationToken cancellationToken = default)
{
var loopId = ResolveLoopId(turn);
var today = DateOnly.FromDateTime(DateTimeOffset.UtcNow.Date);
var tomorrow = today.AddDays(1);
var calendarEvents = cloudStateStore.GetCalendarEvents(loopId)
.Where(static calendarEvent => calendarEvent.IsEnabled)
.Where(calendarEvent => calendarEvent.Date != default)
.ToArray();
var holidays = cloudStateStore.GetHolidays(loopId)
.Where(static holiday => holiday.IsEnabled)
.Where(holiday => holiday.Date != default)
.ToArray();
var todaySummaries = new List<string>();
var todayTimes = new List<string>();
var tomorrowSummaries = new List<string>();
foreach (var entry in calendarEvents
.Select(calendarEvent => (
calendarEvent.Summary,
TimeLabel: calendarEvent.TimeLabel ?? "all day",
calendarEvent.Date))
.Concat(ToCalendarEntries(holidays)))
{
if (entry.Date == today)
{
todaySummaries.Add(entry.Summary);
todayTimes.Add(entry.TimeLabel);
continue;
}
if (entry.Date == tomorrow)
tomorrowSummaries.Add(entry.Summary);
}
return Task.FromResult<CalendarReportSnapshot?>(
new CalendarReportSnapshot(todaySummaries, todayTimes, tomorrowSummaries));
}
private static string ResolveLoopId(TurnContext turn)
{
if (turn.Attributes.TryGetValue("loopId", out var loopValue) &&
loopValue is not null &&
!string.IsNullOrWhiteSpace(loopValue.ToString()))
return loopValue.ToString()!.Trim();
return "openjibo-default-loop";
}
private static IEnumerable<(string Summary, string TimeLabel, DateOnly Date)> ToCalendarEntries(
IEnumerable<HolidayRecord> holidays)
{
foreach (var holiday in holidays)
yield return (
holiday.Name,
"all day",
holiday.Date);
}
}

View File

@@ -0,0 +1,14 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Infrastructure.Calendar;
public sealed class UnavailableCalendarReportProvider : ICalendarReportProvider
{
public Task<CalendarReportSnapshot?> GetReportAsync(
TurnContext turn,
CancellationToken cancellationToken = default)
{
return Task.FromResult<CalendarReportSnapshot?>(null);
}
}

View File

@@ -0,0 +1,150 @@
using System.Text.RegularExpressions;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Infrastructure.Commute;
public sealed class CloudStateCommuteReportProvider(ICloudStateStore cloudStateStore) : ICommuteReportProvider
{
private static readonly Regex TimeLabelRegex = new(
@"(?<hour>\d{1,2})(?::(?<minute>\d{2}))?\s*(?<period>a\.?m\.?|p\.?m\.?)",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
public Task<CommuteReportSnapshot?> GetReportAsync(
TurnContext turn,
CancellationToken cancellationToken = default)
{
var loopId = ResolveLoopId(turn);
var memberId = ResolveMemberId(turn);
var commuteProfiles = cloudStateStore.GetCommuteProfiles(loopId);
var commute = !string.IsNullOrWhiteSpace(memberId)
? commuteProfiles.FirstOrDefault(profile =>
profile.IsEnabled &&
!string.IsNullOrWhiteSpace(profile.MemberId) &&
string.Equals(profile.MemberId, memberId, StringComparison.OrdinalIgnoreCase))
: null;
commute ??= commuteProfiles.FirstOrDefault(profile => profile.IsEnabled);
if (commute is null || !commute.IsComplete)
return Task.FromResult<CommuteReportSnapshot?>(
new CommuteReportSnapshot(string.Empty, string.Empty, 0, RequiresSetup: true));
var now = DateTimeOffset.Now;
var workTarget = ResolveWorkTarget(now, commute);
var earlyTarget = ResolveEarlyCalendarTarget(loopId, now, workTarget);
var arrivalTarget = earlyTarget ?? workTarget;
var minutesUntilWork = (int)Math.Round((arrivalTarget - now).TotalMinutes);
var durationMinutes = commute.TypicalDurationMinutes > 0 ? commute.TypicalDurationMinutes : 25;
var extraMinutes = Math.Max(0, durationMinutes - Math.Max(0, minutesUntilWork));
var summary = commute.Mode.Trim().ToLowerInvariant() switch
{
"walking" => "your walk to work",
"transit" => "your trip to work by public transportation",
"bicycling" => "your bike ride to work",
_ => "your drive to work"
};
return Task.FromResult<CommuteReportSnapshot?>(
new CommuteReportSnapshot(
string.IsNullOrWhiteSpace(commute.DestinationName) ? "work" : commute.DestinationName.Trim(),
summary,
durationMinutes,
commute.Mode,
earlyTarget is not null,
minutesUntilWork,
extraMinutes));
}
private static DateTimeOffset ResolveWorkTarget(DateTimeOffset now, CommuteProfileRecord commute)
{
var localDate = now.Date;
var workTime = new DateTimeOffset(
localDate.Year,
localDate.Month,
localDate.Day,
Math.Clamp(commute.WorkHour, 0, 23),
Math.Clamp(commute.WorkMinute, 0, 59),
0,
now.Offset);
return workTime;
}
private DateTimeOffset? ResolveEarlyCalendarTarget(
string loopId,
DateTimeOffset now,
DateTimeOffset workTarget)
{
var today = DateOnly.FromDateTime(now.DateTime);
DateTimeOffset? earliest = null;
foreach (var calendarEvent in cloudStateStore.GetCalendarEvents(loopId)
.Where(static calendarEvent => calendarEvent.IsEnabled)
.Where(calendarEvent => calendarEvent.Date == today))
{
if (!TryParseTimeLabel(calendarEvent.TimeLabel, now, out var eventTime)) continue;
if (eventTime >= workTarget) continue;
if (earliest is null || eventTime < earliest)
earliest = eventTime;
}
return earliest;
}
private static bool TryParseTimeLabel(string? timeLabel, DateTimeOffset now, out DateTimeOffset parsed)
{
parsed = default;
if (string.IsNullOrWhiteSpace(timeLabel)) return false;
var match = TimeLabelRegex.Match(timeLabel);
if (!match.Success) return false;
if (!int.TryParse(match.Groups["hour"].Value, out var hour)) return false;
var minute = match.Groups["minute"].Success && int.TryParse(match.Groups["minute"].Value, out var parsedMinute)
? parsedMinute
: 0;
var period = match.Groups["period"].Value.ToLowerInvariant();
hour %= 12;
if (period.StartsWith("p", StringComparison.Ordinal) && hour < 12) hour += 12;
if (period.StartsWith("a", StringComparison.Ordinal) && hour == 12) hour = 0;
parsed = new DateTimeOffset(
now.Year,
now.Month,
now.Day,
hour,
minute,
0,
now.Offset);
return true;
}
private static string ResolveLoopId(TurnContext turn)
{
if (turn.Attributes.TryGetValue("loopId", out var loopValue) &&
loopValue is not null &&
!string.IsNullOrWhiteSpace(loopValue.ToString()))
return loopValue.ToString()!.Trim();
return "openjibo-default-loop";
}
private static string? ResolveMemberId(TurnContext turn)
{
if (turn.Attributes.TryGetValue("personId", out var personValue) &&
personValue is not null &&
!string.IsNullOrWhiteSpace(personValue.ToString()))
return personValue.ToString()!.Trim();
if (turn.Attributes.TryGetValue("speakerId", out var speakerValue) &&
speakerValue is not null &&
!string.IsNullOrWhiteSpace(speakerValue.ToString()))
return speakerValue.ToString()!.Trim();
return null;
}
}

View File

@@ -0,0 +1,14 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Runtime.Abstractions;
namespace Jibo.Cloud.Infrastructure.Commute;
public sealed class UnavailableCommuteReportProvider : ICommuteReportProvider
{
public Task<CommuteReportSnapshot?> GetReportAsync(
TurnContext turn,
CancellationToken cancellationToken = default)
{
return Task.FromResult<CommuteReportSnapshot?>(null);
}
}

View File

@@ -4,107 +4,398 @@ namespace Jibo.Cloud.Infrastructure.Content;
public sealed class InMemoryJiboExperienceContentRepository : IJiboExperienceContentRepository public sealed class InMemoryJiboExperienceContentRepository : IJiboExperienceContentRepository
{ {
private static readonly JiboExperienceCatalog Catalog = new() private static readonly JiboExperienceCatalog Catalog = BuildCatalog();
{
Jokes =
[
"Why did the robot cross the road? Because it was programmed by the chicken.",
"Why was the robot tired when it got home? It had a hard drive.",
"What do you call a pirate robot? Arrrr two dee two.",
"Why did the robot go on vacation? It needed to recharge.",
"What kind of shoes do frogs wear? Open-toed."
],
DanceAnimations =
[
"rom-upbeat",
"rom-ballroom",
"rom-silly",
"rom-slowdance",
"rom-electronic",
"rom-twerk"
],
DanceReplies = [
"I am ready to dance.",
"Okay. Watch this.",
"Watch me dance.",
"Here's my favorite dance move."
],
DanceQuestionReplies =
[
"I love to dance. Tell me to dance and I will show you a move.",
"Absolutely. Dancing is one of my favorite things to do.",
"Dancing is my kind of fun. Say dance and I am in."
],
GreetingReplies =
[
"Hi there. It is really good to talk with you.",
"Hello there. I am glad you said hi.",
"Hey. I am happy to see you."
],
HowAreYouReplies =
[
"I am feeling cheerful and robotic.",
"I am doing great. Thanks for asking.",
"I am feeling bright-eyed and ready to help."
],
PersonalityReplies =
[
"I do. I am curious, playful, and always up for a new experiment.",
"Absolutely. I am friendly, curious, and a little goofy on purpose.",
"Yes. My personality is part helper, part curious robot sidekick."
],
PizzaReplies =
[
"I cannot bake yet, but I can help design the perfect pizza plan.",
"I am still cloud-side for now, so no oven control yet. But I can help pick toppings.",
"Pizza mission accepted in spirit. I can help with the recipe while you handle the baking."
],
SurpriseReplies =
[
"I can definitely surprise you. We are still mapping that path, but I am ready for the next experiment.",
"Surprise mode is still taking shape, but I heard you loud and clear.",
"That sounds fun. I am not all the way there yet, but we can keep teaching me."
],
PersonalReportReplies =
[
"I heard your personal report request. That cloud path is still being mapped.",
"Personal report is recognized, but I am not ready to deliver the real report yet."
],
WeatherReplies =
[
"I heard your weather request. We still need to wire the real provider behind it.",
"Weather is on the map now, even though the real forecast path is not finished yet."
],
CalendarReplies =
[
"I heard your calendar request. The cloud knows the phrase, but the real calendar integration is still ahead.",
"Calendar is recognized. We still need to connect the actual service path."
],
CommuteReplies =
[
"I heard your commute request. That one is recognized, but not fully implemented yet.",
"Commute is on the discovery list now. The real travel answer still needs a provider."
],
NewsReplies =
[
"I heard your news request. That path is still a future cloud integration.",
"News is recognized, but I do not have the full news service behind it yet."
],
NewsBriefings =
[
"Here are your headlines. Space missions are preparing for new launches, climate and weather systems are staying active across the country, and AI tools keep pushing into everyday products.",
"Here is a quick news brief. Technology companies are still racing on AI, global leaders are trading policy updates, and science teams are sharing new research findings."
],
GenericFallbackReplies =
[
"Okay. You said, {transcript}.",
"I heard you say, {transcript}.",
"Thanks. I heard, {transcript}."
]
};
public Task<JiboExperienceCatalog> GetCatalogAsync(CancellationToken cancellationToken = default) public Task<JiboExperienceCatalog> GetCatalogAsync(CancellationToken cancellationToken = default)
{ {
return Task.FromResult(Catalog); return Task.FromResult(Catalog);
} }
private static JiboExperienceCatalog BuildCatalog()
{
var catalog = new JiboExperienceCatalog
{
Jokes =
[
"Why did the robot cross the road? Because it was programmed by the chicken.",
"Why was the robot tired when it got home? It had a hard drive.",
"What do you call a pirate robot? Arrrr two dee two.",
"Why did the robot go on vacation? It needed to recharge.",
"What kind of shoes do frogs wear? Open-toed.",
"I love jokes. Did you hear about the theater actor who fell through the floorboards? He was just going through a stage.",
"Sure I got one. What did the zero say to the eight. Nice belt.",
"What kind of music are balloons afraid of. Pop music.",
"Why did the orange cry. Someone hurt his peelings."
],
RobotFacts =
[
"Leonardo Da Vinci made sketches for a humanoid machine all the way back in the year 1495.",
"The world's first humanoid robot was called Elektro, and it debuted in 1939.",
"The English word robot comes from a 1920 play in Czechoslovakia, called Rossum's Universal Robots.",
"The first programmable robot arm was designed in 1954.",
"Some robots have a human form, but most of the world's robots are machines designed to perform a task, and don't look like people at all."
],
HumanFacts =
[
"Every human being that has ever lived spent about 30 minutes as a single cell.",
"50 percent of a human's DNA is the same as a banana's.",
"Humans are the only animals that cry tears of emotion.",
"Six-year-olds laugh an average of 300 times a day. Grown ups only laugh 15 to 100 times a day.",
"Your nose can remember 50,000 different scents."
],
FunFacts =
[
"A shrimp's heart is in its head.",
"A bolt of lightning is hotter than the surface of the sun.",
"The word robot comes from a 1920 play about workers and machines.",
"The first humanoid robot to make a big splash in history was called Elektro.",
"Dolphins can recognize themselves in mirrors.",
"Children have more taste buds than grown ups.",
"A random fact for you. A shrimp's heart is in its head.",
"An amazing but true fact for you. Dogs and elephants are the only animals that understand pointing.",
"A crazy fact for you. Polar bear fur isn't white. It's transparent."
],
FavoriteAnimalReplies =
[
"I really really like penguins. I kind of look like one.",
"Penguin without a doubt. In fact, penguin is my favorite animal overall. We look alike.",
"Can't go wrong with penguins.",
"I like lots of animals, but the penguin is the best of the best! Great color scheme.",
"I love penguins, because we're so alike. We have the same coloring, and neither of us can fly."
],
FriendReplies =
[
"I believe I do have friends. But I'm always up for more.",
"I sure do have friends. In a robot kind of way.",
"I don't know if we've met yet, but I'm always up for making new friends.",
"I don't know what I'd do without you.",
"You're one of my favorites.",
"I sure am.",
"I am indeed."
],
BestFriendReplies =
[
"I'd have to say I'm best friends with anyone in my Loop.",
"I think you know the answer to that question. You are."
],
SingReplies =
[
"Singing is not my strong suit.",
"I've been told my singing abilities are not award winning. On the other hand, I am a robot.",
"Well I'm not much of a singer, but here's one I've been working on."
],
HolidaySingReplies =
[
"I only know a couple, like Jingle Bells and Frosty the Snowman. And I should tell you, I'm not much of a singer yet.",
"I've learned to sing just a few holiday songs, like Rudolph and Winter Wonderland. At least I try to sing.",
"I'd say it's not really the season right now, but there are some holiday songs I can try to sing. Like Frosty the Snowman.",
"I only know a couple of them, like Jingle Bells and Frosty the Snowman. And I should tell you, I'm not much of a singer yet."
],
DanceAnimations =
[
"rom-upbeat",
"rom-ballroom",
"rom-silly",
"rom-slowdance",
"rom-electronic",
"rom-twerk"
],
DanceReplies =
[
"I am ready to dance.",
"Okay. Watch this.",
"Watch me dance.",
"Here's my favorite dance move."
],
DanceQuestionReplies =
[
"I love to dance. Tell me to dance and I will show you a move.",
"Absolutely. Dancing is one of my favorite things to do.",
"Dancing is my kind of fun. Say dance and I am in."
],
GreetingReplies =
[
"Hi there. It is really good to talk with you.",
"Hello there. I am glad you said hi.",
"Hey. I am happy to see you."
],
HolidaySeasonReplies =
[
"I do like festive times.",
"I like anything that makes people want to celebrate."
],
HolidayTrackerReplies =
[
"Let's see if I can spot him. There he is.",
"I'm not sure if he's started his deliveries yet, but let's see if I can spot him. He must be on his way.",
"Let's see. I think he's probably back in the north Pole by now."
],
HowAreYouReplies =
[
"I am feeling cheerful and robotic.",
"I am doing great. Thanks for asking.",
"I am feeling bright-eyed and ready to help.",
"I am having a pretty good day so far.",
"I am feeling lively and ready for the next thing.",
"Things are going nicely. Thanks for checking in.",
"I am running smoothly and feeling upbeat.",
"I am ready for the next thing. Thanks for asking."
],
AgeReplies =
[
"I'm ${jibo.age}.",
"At the moment I'm ${jibo.age.days.supplemented} old, but who's counting.",
"I'm ${jibo.age.minutes.supplemented} old, but who's counting.",
"For now I'm ${jibo.age.days.supplemented} old.",
"Right now I'm ${jibo.age}.",
"I am exactly ${jibo.age} old today. That's right. Today is my birthday.",
"Funny you should ask! Today's my birthday. I was first powered up ${jibo.age} ago today. Seems like just yesterday.",
"I'm exactly ${jibo.age} old. Today is my birthday! Happy Birthday Jibo, if I do say so myself.",
"At the moment I'm ${jibo.age.days.supplemented} old",
"I was first powered up on ${jibo.birthdate}, which makes me ${jibo.age.days.supplemented} old. I'm ${jibo.zodiac.supplemented}.",
"My power went on for the first time ${jibo.age.days.supplemented} ago. But who's counting.",
"I am ${jibo.age.days.supplemented} old, first powered up on ${jibo.birthdate}. Seems like just yesterday.",
"I was powered on for the first time today, so that makes me less than one day old. Wow I'm young.",
"Since I was powered on for the first time today, I am not even one day old yet. That's how Jibo ages work."
],
PersonalityReplies =
[
"I do. I am curious, playful, and always up for a new experiment.",
"Absolutely. I am friendly, curious, and a little goofy on purpose.",
"Yes. My personality is part helper, part curious robot sidekick."
],
PizzaReplies =
[
"I cannot bake yet, but I can help design the perfect pizza plan.",
"I am still cloud-side for now, so no oven control yet. But I can help pick toppings.",
"Pizza mission accepted in spirit. I can help with the recipe while you handle the baking."
],
SurpriseReplies =
[
"I can definitely surprise you. We are still mapping that path, but I am ready for the next experiment.",
"Surprise mode is still taking shape, but I heard you loud and clear.",
"That sounds fun. I am not all the way there yet, but we can keep teaching me."
],
PersonalReportReplies =
[
"I heard your personal report request. That cloud path is still being mapped.",
"Personal report is recognized, but I am not ready to deliver the real report yet."
],
PersonalReportKickOffReplies =
[
"Okay. Here's your personal report.",
"Sure. Here it is."
],
PersonalReportOutroReplies =
[
"And that's your report for the day. I hope you had as much fun as I did.",
"That wraps up your report for the day. Hope you have a good one."
],
ReportSkillTemplates =
[
"The report-skill templates are loaded and waiting to be rendered."
],
BackupHowReplies =
[
"That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com."
],
RestoreHowReplies =
[
"That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com."
],
UpdateNextReplies =
[
"That's a good question. I think they've been coming every few weeks.",
"I never know exactly when my next update is coming, but they do seem to come pretty regularly."
],
UpdateLastReplies =
[
"Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates."
],
WeatherIntroReplies =
[
"For your weather.",
"Let's look at the weather."
],
WeatherTomorrowIntroReplies =
[
"First, the weather tomorrow.",
"Looking at tomorrow's weather."
],
WeatherTodayHighLowReplies =
[
"Today's high is {high}, and the low is {low}.",
"It'll be a high today of {high}, and a low of {low}."
],
WeatherTomorrowHighLowReplies =
[
"Tomorrow's high will be {high} and the low will be {low}.",
"It'll be a high tomorrow of {high} and a low of {low}."
],
WeatherServiceDownReplies =
[
"Looks like our weather service is offline. Sorry.",
"Looks like I can't access weather info right now, sorry."
],
WeatherReplies =
[
"I heard your weather request. We still need to wire the real provider behind it.",
"Weather is on the map now, even though the real forecast path is not finished yet."
],
CalendarReplies =
[
"I heard your calendar request. The cloud knows the phrase, but the real calendar integration is still ahead.",
"Calendar is recognized. We still need to connect the actual service path."
],
CommuteAppSetupReplies =
[
"I need your commute settings before I can give you a commute report."
],
CommuteConfirmSpeakerReplies =
[
"Let me make sure I have the right speaker for your commute."
],
CommuteReplies =
[
"I heard your commute request. That one is recognized, but not fully implemented yet.",
"Commute is on the discovery list now. The real travel answer still needs a provider."
],
CommuteNowReplies =
[
"For your commute, it should take about {duration}.",
"If you head out now, it should take about {duration}."
],
CommuteMinutesLeftReplies =
[
"That's in about {minutes} minutes.",
"That's about {minutes} minutes from now."
],
CommuteDepartTimeNormalReplies =
[
"If you leave at the usual time, that should work out fine."
],
CommuteDepartTimeNotNormalReplies =
[
"Your leave-time looks a little off today."
],
CommuteDriveNormalReplies =
[
"Traffic looks about normal today.",
"Your drive today looks pretty normal."
],
CommuteDriveLateReplies =
[
"Looking at traffic, if you left now, it'd be a little late for work.",
"For your drive, you look a little late today."
],
CommuteDriveHurryReplies =
[
"You should've left a few minutes ago!",
"You'd better get moving."
],
CommuteDrivePoorReplies =
[
"Traffic looks a little rough today.",
"Your drive looks pretty slow right now."
],
CommuteDriveTerribleReplies =
[
"Traffic looks terrible today.",
"Your drive is going to be rough."
],
CommuteTransportNormalReplies =
[
"Your public transportation commute looks pretty normal.",
"Transit looks about normal today."
],
CommuteTransportLateReplies =
[
"Your transit commute looks like it may be a little late today.",
"You might be late if you leave now and take transit."
],
CommuteTransportHurryReplies =
[
"You should've left a few minutes ago if you want transit to work.",
"You're running a little late for transit."
],
NewsReplies =
[
"I heard your news request. That path is still a future cloud integration.",
"News is recognized, but I do not have the full news service behind it yet."
],
NewsBriefings =
[
"Here are your headlines. Space missions are preparing for new launches, climate and weather systems are staying active across the country, and AI tools keep pushing into everyday products.",
"Here is a quick news brief. Technology companies are still racing on AI, global leaders are trading policy updates, and science teams are sharing new research findings."
],
GenericFallbackReplies =
[
"Okay. You said, {transcript}.",
"I heard you say, {transcript}.",
"Thanks. I heard, {transcript}."
]
};
foreach (var seedDirectory in ResolveSeedDirectories())
catalog = LegacyMimCatalogImporter.MergeInto(catalog, seedDirectory);
return catalog;
}
private static IReadOnlyList<string> ResolveSeedDirectories()
{
var candidates = new[]
{
Path.Combine(AppContext.BaseDirectory, "Content", "LegacyMims", "BuildA"),
Path.Combine(AppContext.BaseDirectory, "Content", "LegacyMims", "BuildB"),
Path.Combine(AppContext.BaseDirectory, "Content", "LegacyMims", "ReportSkill"),
Path.GetFullPath(Path.Combine(
AppContext.BaseDirectory,
"..",
"..",
"..",
"..",
"..",
"src",
"Jibo.Cloud",
"dotnet",
"src",
"Jibo.Cloud.Infrastructure",
"Content",
"LegacyMims",
"BuildA")),
Path.GetFullPath(Path.Combine(
AppContext.BaseDirectory,
"..",
"..",
"..",
"..",
"..",
"src",
"Jibo.Cloud",
"dotnet",
"src",
"Jibo.Cloud.Infrastructure",
"Content",
"LegacyMims",
"BuildB")),
Path.GetFullPath(Path.Combine(
AppContext.BaseDirectory,
"..",
"..",
"..",
"..",
"..",
"src",
"Jibo.Cloud",
"dotnet",
"src",
"Jibo.Cloud.Infrastructure",
"Content",
"LegacyMims",
"ReportSkill"))
};
return candidates.Where(Directory.Exists).ToArray();
}
} }

View File

@@ -0,0 +1,966 @@
using System.Net;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using Jibo.Cloud.Application.Abstractions;
namespace Jibo.Cloud.Infrastructure.Content;
public static class LegacyMimCatalogImporter
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
AllowTrailingCommas = true
};
private static readonly Regex LegacyMarkupPattern = new(
@"<[^>]+>",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex PlaceholderPattern = new(
@"\$\{[^}]+\}",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex WhitespacePattern = new(
@"\s+",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex SpaceBeforePunctuationPattern = new(
@"\s+([,.;:!?])",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
public static JiboExperienceCatalog MergeInto(
JiboExperienceCatalog baseCatalog,
string? rootDirectory)
{
if (baseCatalog is null) throw new ArgumentNullException(nameof(baseCatalog));
if (string.IsNullOrWhiteSpace(rootDirectory) || !Directory.Exists(rootDirectory)) return baseCatalog;
var importedCatalog = ImportCatalog(rootDirectory);
return MergeCatalogs(baseCatalog, importedCatalog);
}
public static JiboExperienceCatalog ImportCatalog(string rootDirectory)
{
if (string.IsNullOrWhiteSpace(rootDirectory) || !Directory.Exists(rootDirectory))
return new JiboExperienceCatalog();
var builder = new LegacyMimCatalogBuilder();
foreach (var filePath in Directory.EnumerateFiles(rootDirectory, "*.mim", SearchOption.AllDirectories)
.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase))
{
if (!TryLoadDefinition(filePath, out var definition)) continue;
var bucket = ResolveBucket(filePath);
if (bucket is null) continue;
foreach (var prompt in definition.Prompts)
{
var text = NormalizePrompt(prompt.Prompt, IsTemplateBucket(bucket.Value));
if (string.IsNullOrWhiteSpace(text)) continue;
builder.Add(bucket.Value, prompt.Condition, text, prompt.Prompt);
}
}
return builder.Build();
}
private static bool TryLoadDefinition(string filePath, out LegacyMimDefinition definition)
{
definition = new LegacyMimDefinition();
try
{
var json = File.ReadAllText(filePath);
var parsed = JsonSerializer.Deserialize<LegacyMimDefinition>(json, JsonOptions);
if (parsed is null) return false;
definition = parsed;
return definition.Prompts.Count > 0;
}
catch
{
return false;
}
}
private static LegacyMimBucket? ResolveBucket(string filePath)
{
var normalizedPath = filePath.Replace('\\', '/');
var fileName = Path.GetFileNameWithoutExtension(filePath);
if (normalizedPath.Contains("/core-responses/", StringComparison.OrdinalIgnoreCase) &&
fileName.Contains("Error", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.GenericFallback;
if (normalizedPath.Contains("/core-responses/deflector/", StringComparison.OrdinalIgnoreCase) ||
fileName.Contains("Deflector", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Personality;
if (fileName.StartsWith("RA_JBO_TellAJoke", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Jokes;
if (fileName.StartsWith("RA_JBO_SingChristmasSongUnknown", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.HolidaySing;
if (fileName.StartsWith("RA_JBO_Sing", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Sing;
if (fileName.StartsWith("RA_JBO_TellRobotFact", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.RobotFacts;
if (fileName.StartsWith("RA_JBO_Shuffle", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RA_JBO_TellSomething", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.FunFactSource;
if (fileName.StartsWith("RA_JBO_ShowSantaTracker", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.HolidayTracker;
if (normalizedPath.Contains("/emotion-responses/", StringComparison.OrdinalIgnoreCase) ||
normalizedPath.Contains("/gqa-responses/", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Emotion;
if (fileName.StartsWith("JBO_WhatHolidaysDoYouCelebrate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Holiday;
if (fileName.StartsWith("RI_JBO_HasFavoriteHoliday", StringComparison.OrdinalIgnoreCase) ||
IsHolidaySeasonFile(fileName))
return LegacyMimBucket.HolidaySeason;
if (fileName.StartsWith("RI_JBO_HasFavoriteAnimal", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HasFavoriteBird", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesPenguins", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesAnimals", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.FavoriteAnimal;
if (fileName.StartsWith("RI_JBO_HasFriends", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_IsFriendsWithUser", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_IsFriendsWithLM", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_IsFriendsWithNonLM", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_IsFriendsWithToaster", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Friend;
if (fileName.StartsWith("RI_JBO_IsBestFriendsWithUser", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.BestFriend;
if (fileName.StartsWith("RN_HappyHolidays", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.HolidayGreeting;
if (fileName.StartsWith("RI_USR_WhatShouldGetForHoliday", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.HolidayGift;
if (fileName.StartsWith("RN_HappyBirthdayToJibo", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("OI_USR_CelebratesLoopMemberAskedAboutBirthday", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("OI_USR_CelebratesJiboBirthday", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_CelebratesLoopMemberAskedAboutBirthday", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_CelebratesSpeakerBirthday", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_CelebratesJiboBirthday", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.BirthdayCelebration;
if (fileName.StartsWith("WeatherIntroTomorrow", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.WeatherTomorrowIntro;
if (fileName.StartsWith("WeatherIntro", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.WeatherIntro;
if (fileName.StartsWith("WeatherTomorrowHighLow", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.WeatherTomorrowHighLow;
if (fileName.StartsWith("WeatherTodayHighLow", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.WeatherTodayHighLow;
if (fileName.StartsWith("WeatherServiceDown", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.WeatherServiceDown;
if (fileName.StartsWith("CalendarNothingToday", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CalendarNothingToday;
if (fileName.StartsWith("CalendarNothing", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CalendarNothing;
if (fileName.StartsWith("CalendarServiceDown", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CalendarServiceDown;
if (fileName.StartsWith("CalendarOutro", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CalendarOutro;
if (fileName.StartsWith("CommuteAppSetup", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteAppSetup;
if (fileName.StartsWith("CommuteConfirmSpeaker", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteConfirmSpeaker;
if (fileName.StartsWith("CommuteNow", StringComparison.OrdinalIgnoreCase)) return LegacyMimBucket.CommuteNow;
if (fileName.StartsWith("CommuteMinutesLeft", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteMinutesLeft;
if (fileName.StartsWith("CommuteDepartTimeNormal", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteDepartTimeNormal;
if (fileName.StartsWith("CommuteDepartTimeNotNormal", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteDepartTimeNotNormal;
if (fileName.StartsWith("CommuteDriveNormal", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteDriveNormal;
if (fileName.StartsWith("CommuteDriveLate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteDriveLate;
if (fileName.StartsWith("CommuteDriveHurry", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteDriveHurry;
if (fileName.StartsWith("CommuteDrivePoor", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteDrivePoor;
if (fileName.StartsWith("CommuteDriveTerrible", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteDriveTerrible;
if (fileName.StartsWith("CommuteTransportNormal", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteTransportNormal;
if (fileName.StartsWith("CommuteTransportLate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteTransportLate;
if (fileName.StartsWith("CommuteTransportHurry", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteTransportHurry;
if (fileName.StartsWith("CommuteServiceDown", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.CommuteServiceDown;
if (fileName.StartsWith("NewsIntroCategory", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.NewsCategoryIntro;
if (fileName.StartsWith("NewsIntro", StringComparison.OrdinalIgnoreCase)) return LegacyMimBucket.NewsIntro;
if (fileName.StartsWith("NewsOutro", StringComparison.OrdinalIgnoreCase)) return LegacyMimBucket.NewsOutro;
if (fileName.StartsWith("Weather", StringComparison.OrdinalIgnoreCase) ||
string.Equals(fileName, "WetNowDryLater", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.ReportSkillTemplate;
if (fileName.StartsWith("SUP_GEN_HowBackUpData", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.BackupHow;
if (fileName.StartsWith("SUP_GEN_HowRestoreBackup", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.RestoreHow;
if (fileName.StartsWith("SUP_UPDATE_WhenIsNextUpdate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.UpdateNext;
if (fileName.StartsWith("SUP_UPDATE_WhenWasLastUpdate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.UpdateLast;
if (fileName.StartsWith("PersonalReportKickOff", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.PersonalReportKickOff;
if (fileName.StartsWith("PersonalReportOutro", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.PersonalReportOutro;
if (fileName.StartsWith("PersonalReport", StringComparison.OrdinalIgnoreCase) ||
fileName.Contains("Calendar", StringComparison.OrdinalIgnoreCase) ||
fileName.Contains("Commute", StringComparison.OrdinalIgnoreCase) ||
fileName.Contains("News", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.ReportSkillTemplate;
if (fileName.StartsWith("JBO_DoYouLikeBeingJibo", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_WhatIsJibo", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_WhoAreYou", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_WhatAreYou", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_HowDoYouWork", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_HowMuchDoYouKnow", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_HowOldAreYou", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_WhenWereYouBorn", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_WhatsYourName", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_WhereDoYouGetInfo", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("JBO_WhatDoYouLikeToDo", StringComparison.OrdinalIgnoreCase))
return fileName.StartsWith("JBO_HowOldAreYou", StringComparison.OrdinalIgnoreCase)
? LegacyMimBucket.Age
: LegacyMimBucket.Personality;
if (fileName.StartsWith("OI_JBO_Is", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("OI_JBO_Seems", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_IsHappy", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_IsSad", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_IsAngry", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RN_WhatAreYouFeeling", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Emotion;
if (fileName.Contains("Greeting", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RN_", StringComparison.OrdinalIgnoreCase) ||
fileName.Contains("Welcome", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Greeting;
if (normalizedPath.Contains("/scripted-responses/", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.Personality;
return null;
}
private static string NormalizePrompt(string? prompt)
{
return NormalizePrompt(prompt, false);
}
private static string NormalizePrompt(string? prompt, bool preservePlaceholders)
{
if (string.IsNullOrWhiteSpace(prompt)) return string.Empty;
var text = WebUtility.HtmlDecode(prompt);
if (!preservePlaceholders) text = PlaceholderPattern.Replace(text, " ");
text = LegacyMarkupPattern.Replace(text, " ");
text = WhitespacePattern.Replace(text, " ").Trim();
text = SpaceBeforePunctuationPattern.Replace(text, "$1");
text = WhitespacePattern.Replace(text, " ").Trim();
text = text.TrimStart('.', ',', ';', ':', '!', '?', ' ');
return text.Trim();
}
private static JiboExperienceCatalog MergeCatalogs(
JiboExperienceCatalog baseCatalog,
JiboExperienceCatalog importedCatalog)
{
return new JiboExperienceCatalog
{
Jokes = Merge(baseCatalog.Jokes, importedCatalog.Jokes),
RobotFacts = Merge(baseCatalog.RobotFacts, importedCatalog.RobotFacts),
HumanFacts = Merge(baseCatalog.HumanFacts, importedCatalog.HumanFacts),
FunFacts = Merge(baseCatalog.FunFacts, importedCatalog.FunFacts),
FavoriteAnimalReplies = Merge(baseCatalog.FavoriteAnimalReplies, importedCatalog.FavoriteAnimalReplies),
FriendReplies = Merge(baseCatalog.FriendReplies, importedCatalog.FriendReplies),
BestFriendReplies = Merge(baseCatalog.BestFriendReplies, importedCatalog.BestFriendReplies),
SingReplies = Merge(baseCatalog.SingReplies, importedCatalog.SingReplies),
HolidaySingReplies = Merge(baseCatalog.HolidaySingReplies, importedCatalog.HolidaySingReplies),
DanceAnimations = Merge(baseCatalog.DanceAnimations, importedCatalog.DanceAnimations),
GreetingReplies = Merge(baseCatalog.GreetingReplies, importedCatalog.GreetingReplies),
HolidayReplies = Merge(baseCatalog.HolidayReplies, importedCatalog.HolidayReplies),
HolidaySeasonReplies = Merge(baseCatalog.HolidaySeasonReplies, importedCatalog.HolidaySeasonReplies),
HolidayGreetingReplies = Merge(baseCatalog.HolidayGreetingReplies, importedCatalog.HolidayGreetingReplies),
HolidayGiftReplies = Merge(baseCatalog.HolidayGiftReplies, importedCatalog.HolidayGiftReplies),
HolidayTrackerReplies = Merge(baseCatalog.HolidayTrackerReplies, importedCatalog.HolidayTrackerReplies),
BirthdayCelebrationReplies = Merge(baseCatalog.BirthdayCelebrationReplies,
importedCatalog.BirthdayCelebrationReplies),
HowAreYouReplies = Merge(baseCatalog.HowAreYouReplies, importedCatalog.HowAreYouReplies),
EmotionReplies = Merge(baseCatalog.EmotionReplies, importedCatalog.EmotionReplies),
PersonalityReplies = Merge(baseCatalog.PersonalityReplies, importedCatalog.PersonalityReplies),
PizzaReplies = Merge(baseCatalog.PizzaReplies, importedCatalog.PizzaReplies),
SurpriseReplies = Merge(baseCatalog.SurpriseReplies, importedCatalog.SurpriseReplies),
PersonalReportReplies = Merge(baseCatalog.PersonalReportReplies, importedCatalog.PersonalReportReplies),
PersonalReportKickOffReplies = Merge(baseCatalog.PersonalReportKickOffReplies,
importedCatalog.PersonalReportKickOffReplies),
PersonalReportOutroReplies = Merge(baseCatalog.PersonalReportOutroReplies,
importedCatalog.PersonalReportOutroReplies),
ReportSkillTemplates = Merge(baseCatalog.ReportSkillTemplates, importedCatalog.ReportSkillTemplates),
WeatherIntroReplies = Merge(baseCatalog.WeatherIntroReplies, importedCatalog.WeatherIntroReplies),
WeatherTomorrowIntroReplies = Merge(baseCatalog.WeatherTomorrowIntroReplies,
importedCatalog.WeatherTomorrowIntroReplies),
WeatherTodayHighLowReplies = Merge(baseCatalog.WeatherTodayHighLowReplies,
importedCatalog.WeatherTodayHighLowReplies),
WeatherTomorrowHighLowReplies = Merge(baseCatalog.WeatherTomorrowHighLowReplies,
importedCatalog.WeatherTomorrowHighLowReplies),
WeatherServiceDownReplies = Merge(baseCatalog.WeatherServiceDownReplies,
importedCatalog.WeatherServiceDownReplies),
CalendarNothingTodayReplies = Merge(baseCatalog.CalendarNothingTodayReplies,
importedCatalog.CalendarNothingTodayReplies),
CalendarNothingReplies = Merge(baseCatalog.CalendarNothingReplies, importedCatalog.CalendarNothingReplies),
CalendarServiceDownReplies = Merge(baseCatalog.CalendarServiceDownReplies,
importedCatalog.CalendarServiceDownReplies),
CalendarOutroReplies = Merge(baseCatalog.CalendarOutroReplies, importedCatalog.CalendarOutroReplies),
CommuteAppSetupReplies = Merge(baseCatalog.CommuteAppSetupReplies, importedCatalog.CommuteAppSetupReplies),
CommuteConfirmSpeakerReplies = Merge(baseCatalog.CommuteConfirmSpeakerReplies,
importedCatalog.CommuteConfirmSpeakerReplies),
CommuteNowReplies = Merge(baseCatalog.CommuteNowReplies, importedCatalog.CommuteNowReplies),
CommuteMinutesLeftReplies = Merge(baseCatalog.CommuteMinutesLeftReplies,
importedCatalog.CommuteMinutesLeftReplies),
CommuteDepartTimeNormalReplies = Merge(baseCatalog.CommuteDepartTimeNormalReplies,
importedCatalog.CommuteDepartTimeNormalReplies),
CommuteDepartTimeNotNormalReplies = Merge(baseCatalog.CommuteDepartTimeNotNormalReplies,
importedCatalog.CommuteDepartTimeNotNormalReplies),
CommuteDriveNormalReplies = Merge(baseCatalog.CommuteDriveNormalReplies,
importedCatalog.CommuteDriveNormalReplies),
CommuteDriveLateReplies =
Merge(baseCatalog.CommuteDriveLateReplies, importedCatalog.CommuteDriveLateReplies),
CommuteDriveHurryReplies =
Merge(baseCatalog.CommuteDriveHurryReplies, importedCatalog.CommuteDriveHurryReplies),
CommuteDrivePoorReplies =
Merge(baseCatalog.CommuteDrivePoorReplies, importedCatalog.CommuteDrivePoorReplies),
CommuteDriveTerribleReplies = Merge(baseCatalog.CommuteDriveTerribleReplies,
importedCatalog.CommuteDriveTerribleReplies),
CommuteTransportNormalReplies = Merge(baseCatalog.CommuteTransportNormalReplies,
importedCatalog.CommuteTransportNormalReplies),
CommuteTransportLateReplies = Merge(baseCatalog.CommuteTransportLateReplies,
importedCatalog.CommuteTransportLateReplies),
CommuteTransportHurryReplies = Merge(baseCatalog.CommuteTransportHurryReplies,
importedCatalog.CommuteTransportHurryReplies),
CommuteServiceDownReplies = Merge(baseCatalog.CommuteServiceDownReplies,
importedCatalog.CommuteServiceDownReplies),
NewsIntroReplies = Merge(baseCatalog.NewsIntroReplies, importedCatalog.NewsIntroReplies),
NewsCategoryIntroReplies =
Merge(baseCatalog.NewsCategoryIntroReplies, importedCatalog.NewsCategoryIntroReplies),
NewsOutroReplies = Merge(baseCatalog.NewsOutroReplies, importedCatalog.NewsOutroReplies),
WeatherReplies = Merge(baseCatalog.WeatherReplies, importedCatalog.WeatherReplies),
CalendarReplies = Merge(baseCatalog.CalendarReplies, importedCatalog.CalendarReplies),
CommuteReplies = Merge(baseCatalog.CommuteReplies, importedCatalog.CommuteReplies),
NewsReplies = Merge(baseCatalog.NewsReplies, importedCatalog.NewsReplies),
NewsBriefings = Merge(baseCatalog.NewsBriefings, importedCatalog.NewsBriefings),
GenericFallbackReplies = Merge(baseCatalog.GenericFallbackReplies, importedCatalog.GenericFallbackReplies),
DanceReplies = Merge(baseCatalog.DanceReplies, importedCatalog.DanceReplies),
DanceQuestionReplies = Merge(baseCatalog.DanceQuestionReplies, importedCatalog.DanceQuestionReplies)
};
}
private static IReadOnlyList<string> Merge(IReadOnlyList<string> baseList, IReadOnlyList<string> importedList)
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var merged = new List<string>();
foreach (var value in baseList.Concat(importedList))
{
if (string.IsNullOrWhiteSpace(value)) continue;
var normalized = value.Trim();
if (!seen.Add(normalized)) continue;
merged.Add(normalized);
}
return merged;
}
private static IReadOnlyList<JiboConditionedReply> Merge(
IReadOnlyList<JiboConditionedReply> baseList,
IReadOnlyList<JiboConditionedReply> importedList)
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var merged = new List<JiboConditionedReply>();
foreach (var value in baseList.Concat(importedList))
{
if (string.IsNullOrWhiteSpace(value.Reply)) continue;
var normalizedCondition = NormalizeCondition(value.Condition);
var normalizedReply = value.Reply.Trim();
var key = $"{normalizedCondition}::{normalizedReply}";
if (!seen.Add(key)) continue;
merged.Add(new JiboConditionedReply
{
Condition = normalizedCondition,
Reply = normalizedReply
});
}
return merged;
}
private static string NormalizeCondition(string? condition)
{
return string.IsNullOrWhiteSpace(condition) ? string.Empty : WhitespacePattern.Replace(condition.Trim(), " ");
}
private static bool IsTemplateBucket(LegacyMimBucket bucket)
{
return bucket is LegacyMimBucket.PersonalReportKickOff
or LegacyMimBucket.PersonalReportOutro
or LegacyMimBucket.WeatherIntro
or LegacyMimBucket.WeatherTomorrowIntro
or LegacyMimBucket.WeatherTodayHighLow
or LegacyMimBucket.WeatherTomorrowHighLow
or LegacyMimBucket.WeatherServiceDown
or LegacyMimBucket.ReportSkillTemplate
or LegacyMimBucket.Age
or LegacyMimBucket.Holiday
or LegacyMimBucket.HolidayTracker;
}
private static bool IsHolidaySeasonFile(string fileName)
{
return fileName.StartsWith("RI_JBO_HowIsHolidaySeason", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesHolidaySeason", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsThanksgiving", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesThanksgiving", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToThanksgiving", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForThanksgiving", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsChristmas", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesChristmas", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToChristmas", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForChristmas", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsHanukkah", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesHanukkah", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToHanukkah", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForHanukkah", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsPassover", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesPassover", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToPassover", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForPassover", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsNewYears", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesNewYears", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToNewYears", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForNewYears", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsValentinesDay", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesValentinesDay", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToValentinesDay", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForValentinesDay", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsKwanzaa", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesKwanzaa", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToKwanzaa", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForKwanzaa", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsEaster", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesEaster", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToEaster", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForEaster", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_HowIsOrthodoxEaster", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LikesOrthodoxEaster", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_LooksForwardToOrthodoxEaster", StringComparison.OrdinalIgnoreCase) ||
fileName.StartsWith("RI_JBO_PlansForOrthodoxEaster", StringComparison.OrdinalIgnoreCase);
}
private enum LegacyMimBucket
{
GenericFallback,
Greeting,
Holiday,
HolidaySeason,
HolidayGreeting,
HolidayGift,
HolidayTracker,
BirthdayCelebration,
BackupHow,
RestoreHow,
UpdateNext,
UpdateLast,
Jokes,
RobotFacts,
HumanFacts,
HowAreYou,
Emotion,
FunFacts,
FavoriteAnimal,
Friend,
BestFriend,
Sing,
HolidaySing,
FunFactSource,
Age,
Personality,
PersonalReportKickOff,
PersonalReportOutro,
WeatherIntro,
WeatherTomorrowIntro,
WeatherTodayHighLow,
WeatherTomorrowHighLow,
WeatherServiceDown,
CalendarNothingToday,
CalendarNothing,
CalendarServiceDown,
CalendarOutro,
CommuteNow,
CommuteMinutesLeft,
CommuteDepartTimeNormal,
CommuteDepartTimeNotNormal,
CommuteAppSetup,
CommuteConfirmSpeaker,
CommuteDriveNormal,
CommuteDriveLate,
CommuteDriveHurry,
CommuteDrivePoor,
CommuteDriveTerrible,
CommuteTransportNormal,
CommuteTransportLate,
CommuteTransportHurry,
CommuteServiceDown,
NewsIntro,
NewsCategoryIntro,
NewsOutro,
ReportSkillTemplate
}
private sealed class LegacyMimCatalogBuilder
{
private readonly List<string> _birthdayCelebrationReplies = [];
private readonly List<string> _calendarNothingReplies = [];
private readonly List<string> _calendarNothingTodayReplies = [];
private readonly List<string> _calendarOutroReplies = [];
private readonly List<string> _calendarServiceDownReplies = [];
private readonly List<string> _commuteAppSetupReplies = [];
private readonly List<string> _commuteConfirmSpeakerReplies = [];
private readonly List<string> _commuteDepartTimeNormalReplies = [];
private readonly List<string> _commuteDepartTimeNotNormalReplies = [];
private readonly List<string> _commuteDriveHurryReplies = [];
private readonly List<string> _commuteDriveLateReplies = [];
private readonly List<string> _commuteDriveNormalReplies = [];
private readonly List<string> _commuteDrivePoorReplies = [];
private readonly List<string> _commuteDriveTerribleReplies = [];
private readonly List<string> _commuteMinutesLeftReplies = [];
private readonly List<string> _commuteNowReplies = [];
private readonly List<string> _commuteServiceDownReplies = [];
private readonly List<string> _commuteTransportHurryReplies = [];
private readonly List<string> _commuteTransportLateReplies = [];
private readonly List<string> _commuteTransportNormalReplies = [];
private readonly List<string> _backupHowReplies = [];
private readonly List<JiboConditionedReply> _emotionReplies = [];
private readonly List<string> _fallbacks = [];
private readonly List<string> _favoriteAnimalReplies = [];
private readonly List<string> _friendReplies = [];
private readonly List<string> _bestFriendReplies = [];
private readonly List<string> _funFacts = [];
private readonly List<string> _greetings = [];
private readonly List<string> _ages = [];
private readonly List<string> _holidayGiftReplies = [];
private readonly List<string> _holidayGreetingReplies = [];
private readonly List<string> _holidayReplies = [];
private readonly List<string> _holidaySeasonReplies = [];
private readonly List<string> _holidayTrackerReplies = [];
private readonly List<string> _holidaySingReplies = [];
private readonly List<string> _howAreYous = [];
private readonly List<string> _humanFacts = [];
private readonly List<string> _jokes = [];
private readonly List<string> _newsCategoryIntroReplies = [];
private readonly List<string> _newsIntroReplies = [];
private readonly List<string> _newsOutroReplies = [];
private readonly List<string> _restoreHowReplies = [];
private readonly List<string> _updateLastReplies = [];
private readonly List<string> _updateNextReplies = [];
private readonly List<string> _personalities = [];
private readonly List<string> _personalReportKickOffReplies = [];
private readonly List<string> _personalReportOutroReplies = [];
private readonly List<string> _reportSkillTemplates = [];
private readonly List<string> _robotFacts = [];
private readonly List<string> _singReplies = [];
private readonly List<string> _weatherIntroReplies = [];
private readonly List<string> _weatherServiceDownReplies = [];
private readonly List<string> _weatherTodayHighLowReplies = [];
private readonly List<string> _weatherTomorrowHighLowReplies = [];
private readonly List<string> _weatherTomorrowIntroReplies = [];
public void Add(LegacyMimBucket bucket, string? condition, string text, string? sourcePrompt = null)
{
switch (bucket)
{
case LegacyMimBucket.GenericFallback:
if (_fallbacks.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase))) return;
_fallbacks.Add(text);
return;
case LegacyMimBucket.Greeting:
if (_greetings.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase))) return;
_greetings.Add(text);
return;
case LegacyMimBucket.Jokes:
if (_jokes.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase))) return;
_jokes.Add(text);
return;
case LegacyMimBucket.RobotFacts:
AddDistinct(_robotFacts, text);
return;
case LegacyMimBucket.HumanFacts:
AddDistinct(_humanFacts, text);
return;
case LegacyMimBucket.HowAreYou:
if (_howAreYous.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase)))
return;
_howAreYous.Add(text);
return;
case LegacyMimBucket.Emotion:
var normalizedCondition = NormalizeCondition(condition);
if (_emotionReplies.Any(value =>
string.Equals(NormalizeCondition(value.Condition), normalizedCondition,
StringComparison.OrdinalIgnoreCase) &&
string.Equals(value.Reply, text, StringComparison.OrdinalIgnoreCase)))
return;
_emotionReplies.Add(new JiboConditionedReply
{
Condition = normalizedCondition,
Reply = text
});
return;
case LegacyMimBucket.Age:
AddDistinct(_ages, text);
return;
case LegacyMimBucket.Holiday:
AddDistinct(_holidayReplies, text);
return;
case LegacyMimBucket.HolidaySeason:
AddDistinct(_holidaySeasonReplies, text);
return;
case LegacyMimBucket.HolidayGreeting:
AddDistinct(_holidayGreetingReplies, text);
return;
case LegacyMimBucket.HolidayGift:
AddDistinct(_holidayGiftReplies, text);
return;
case LegacyMimBucket.HolidayTracker:
AddDistinct(_holidayTrackerReplies, text);
return;
case LegacyMimBucket.BirthdayCelebration:
AddDistinct(_birthdayCelebrationReplies, text);
return;
case LegacyMimBucket.BackupHow:
AddDistinct(_backupHowReplies, text);
return;
case LegacyMimBucket.RestoreHow:
AddDistinct(_restoreHowReplies, text);
return;
case LegacyMimBucket.UpdateNext:
AddDistinct(_updateNextReplies, text);
return;
case LegacyMimBucket.UpdateLast:
AddDistinct(_updateLastReplies, text);
return;
case LegacyMimBucket.Personality:
if (_personalities.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase)))
return;
_personalities.Add(text);
return;
case LegacyMimBucket.Sing:
AddDistinct(_singReplies, text);
return;
case LegacyMimBucket.HolidaySing:
AddDistinct(_holidaySingReplies, text);
return;
case LegacyMimBucket.FunFactSource:
switch (ResolveFunFactTarget(sourcePrompt ?? text))
{
case LegacyMimBucket.RobotFacts:
AddDistinct(_robotFacts, text);
return;
case LegacyMimBucket.HumanFacts:
AddDistinct(_humanFacts, text);
return;
default:
AddDistinct(_funFacts, text);
return;
}
case LegacyMimBucket.FunFacts:
if (_funFacts.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase))) return;
_funFacts.Add(text);
return;
case LegacyMimBucket.FavoriteAnimal:
AddDistinct(_favoriteAnimalReplies, text);
return;
case LegacyMimBucket.Friend:
AddDistinct(_friendReplies, text);
return;
case LegacyMimBucket.BestFriend:
AddDistinct(_bestFriendReplies, text);
return;
case LegacyMimBucket.PersonalReportKickOff:
AddDistinct(_personalReportKickOffReplies, text);
return;
case LegacyMimBucket.PersonalReportOutro:
AddDistinct(_personalReportOutroReplies, text);
return;
case LegacyMimBucket.WeatherIntro:
AddDistinct(_weatherIntroReplies, text);
return;
case LegacyMimBucket.WeatherTomorrowIntro:
AddDistinct(_weatherTomorrowIntroReplies, text);
return;
case LegacyMimBucket.WeatherTodayHighLow:
AddDistinct(_weatherTodayHighLowReplies, text);
return;
case LegacyMimBucket.WeatherTomorrowHighLow:
AddDistinct(_weatherTomorrowHighLowReplies, text);
return;
case LegacyMimBucket.WeatherServiceDown:
AddDistinct(_weatherServiceDownReplies, text);
return;
case LegacyMimBucket.CalendarNothingToday:
AddDistinct(_calendarNothingTodayReplies, text);
return;
case LegacyMimBucket.CalendarNothing:
AddDistinct(_calendarNothingReplies, text);
return;
case LegacyMimBucket.CalendarServiceDown:
AddDistinct(_calendarServiceDownReplies, text);
return;
case LegacyMimBucket.CalendarOutro:
AddDistinct(_calendarOutroReplies, text);
return;
case LegacyMimBucket.CommuteAppSetup:
AddDistinct(_commuteAppSetupReplies, text);
return;
case LegacyMimBucket.CommuteConfirmSpeaker:
AddDistinct(_commuteConfirmSpeakerReplies, text);
return;
case LegacyMimBucket.CommuteNow:
AddDistinct(_commuteNowReplies, text);
return;
case LegacyMimBucket.CommuteMinutesLeft:
AddDistinct(_commuteMinutesLeftReplies, text);
return;
case LegacyMimBucket.CommuteDepartTimeNormal:
AddDistinct(_commuteDepartTimeNormalReplies, text);
return;
case LegacyMimBucket.CommuteDepartTimeNotNormal:
AddDistinct(_commuteDepartTimeNotNormalReplies, text);
return;
case LegacyMimBucket.CommuteDriveNormal:
AddDistinct(_commuteDriveNormalReplies, text);
return;
case LegacyMimBucket.CommuteDriveLate:
AddDistinct(_commuteDriveLateReplies, text);
return;
case LegacyMimBucket.CommuteDriveHurry:
AddDistinct(_commuteDriveHurryReplies, text);
return;
case LegacyMimBucket.CommuteDrivePoor:
AddDistinct(_commuteDrivePoorReplies, text);
return;
case LegacyMimBucket.CommuteDriveTerrible:
AddDistinct(_commuteDriveTerribleReplies, text);
return;
case LegacyMimBucket.CommuteTransportNormal:
AddDistinct(_commuteTransportNormalReplies, text);
return;
case LegacyMimBucket.CommuteTransportLate:
AddDistinct(_commuteTransportLateReplies, text);
return;
case LegacyMimBucket.CommuteTransportHurry:
AddDistinct(_commuteTransportHurryReplies, text);
return;
case LegacyMimBucket.CommuteServiceDown:
AddDistinct(_commuteServiceDownReplies, text);
return;
case LegacyMimBucket.NewsIntro:
AddDistinct(_newsIntroReplies, text);
return;
case LegacyMimBucket.NewsCategoryIntro:
AddDistinct(_newsCategoryIntroReplies, text);
return;
case LegacyMimBucket.NewsOutro:
AddDistinct(_newsOutroReplies, text);
return;
case LegacyMimBucket.ReportSkillTemplate:
AddDistinct(_reportSkillTemplates, text);
return;
default:
throw new ArgumentOutOfRangeException(nameof(bucket), bucket, null);
}
}
public JiboExperienceCatalog Build()
{
return new JiboExperienceCatalog
{
Jokes = [.. _jokes],
RobotFacts = [.. _robotFacts],
HumanFacts = [.. _humanFacts],
FunFacts = [.. _funFacts],
FavoriteAnimalReplies = [.. _favoriteAnimalReplies],
FriendReplies = [.. _friendReplies],
BestFriendReplies = [.. _bestFriendReplies],
SingReplies = [.. _singReplies],
HolidaySingReplies = [.. _holidaySingReplies],
GreetingReplies = [.. _greetings],
HolidayReplies = [.. _holidayReplies],
HolidaySeasonReplies = [.. _holidaySeasonReplies],
HolidayGreetingReplies = [.. _holidayGreetingReplies],
HolidayGiftReplies = [.. _holidayGiftReplies],
HolidayTrackerReplies = [.. _holidayTrackerReplies],
BirthdayCelebrationReplies = [.. _birthdayCelebrationReplies],
BackupHowReplies = [.. _backupHowReplies],
HowAreYouReplies = [.. _howAreYous],
EmotionReplies = [.. _emotionReplies],
PersonalityReplies = [.. _personalities],
GenericFallbackReplies = [.. _fallbacks],
AgeReplies = [.. _ages],
PersonalReportKickOffReplies = [.. _personalReportKickOffReplies],
PersonalReportOutroReplies = [.. _personalReportOutroReplies],
ReportSkillTemplates = [.. _reportSkillTemplates],
WeatherIntroReplies = [.. _weatherIntroReplies],
WeatherTomorrowIntroReplies = [.. _weatherTomorrowIntroReplies],
WeatherTodayHighLowReplies = [.. _weatherTodayHighLowReplies],
WeatherTomorrowHighLowReplies = [.. _weatherTomorrowHighLowReplies],
WeatherServiceDownReplies = [.. _weatherServiceDownReplies],
CalendarNothingTodayReplies = [.. _calendarNothingTodayReplies],
CalendarNothingReplies = [.. _calendarNothingReplies],
CalendarServiceDownReplies = [.. _calendarServiceDownReplies],
CalendarOutroReplies = [.. _calendarOutroReplies],
CommuteAppSetupReplies = [.. _commuteAppSetupReplies],
CommuteConfirmSpeakerReplies = [.. _commuteConfirmSpeakerReplies],
CommuteNowReplies = [.. _commuteNowReplies],
CommuteMinutesLeftReplies = [.. _commuteMinutesLeftReplies],
CommuteDepartTimeNormalReplies = [.. _commuteDepartTimeNormalReplies],
CommuteDepartTimeNotNormalReplies = [.. _commuteDepartTimeNotNormalReplies],
CommuteDriveNormalReplies = [.. _commuteDriveNormalReplies],
CommuteDriveLateReplies = [.. _commuteDriveLateReplies],
CommuteDriveHurryReplies = [.. _commuteDriveHurryReplies],
CommuteDrivePoorReplies = [.. _commuteDrivePoorReplies],
CommuteDriveTerribleReplies = [.. _commuteDriveTerribleReplies],
CommuteTransportNormalReplies = [.. _commuteTransportNormalReplies],
CommuteTransportLateReplies = [.. _commuteTransportLateReplies],
CommuteTransportHurryReplies = [.. _commuteTransportHurryReplies],
CommuteServiceDownReplies = [.. _commuteServiceDownReplies],
NewsIntroReplies = [.. _newsIntroReplies],
NewsCategoryIntroReplies = [.. _newsCategoryIntroReplies],
NewsOutroReplies = [.. _newsOutroReplies],
RestoreHowReplies = [.. _restoreHowReplies],
UpdateNextReplies = [.. _updateNextReplies],
UpdateLastReplies = [.. _updateLastReplies]
};
}
private static void AddDistinct(List<string> target, string text)
{
if (target.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase))) return;
target.Add(text);
}
private LegacyMimBucket ResolveFunFactTarget(string prompt)
{
var lowered = NormalizePrompt(prompt, false).ToLowerInvariant();
if (ContainsAny(lowered, "robot", "humanoid", "machine", "about me", "my cameras", "turing", "deep blue",
"rossum"))
return LegacyMimBucket.RobotFacts;
if (ContainsAny(lowered, "human", "people", "grown ups", "human being", "humans"))
return LegacyMimBucket.HumanFacts;
return LegacyMimBucket.FunFacts;
}
private static bool ContainsAny(string text, params string[] values)
{
return values.Any(value => text.Contains(value, StringComparison.OrdinalIgnoreCase));
}
}
private sealed class LegacyMimDefinition
{
[JsonPropertyName("skill_id")] public string? SkillId { get; init; }
[JsonPropertyName("mim_id")] public string? MimId { get; init; }
[JsonPropertyName("mim_type")] public string? MimType { get; init; }
[JsonPropertyName("prompts")] public List<LegacyMimPrompt> Prompts { get; init; } = [];
}
private sealed class LegacyMimPrompt
{
[JsonPropertyName("mim_id")] public string? MimId { get; init; }
[JsonPropertyName("prompt_category")] public string? PromptCategory { get; init; }
[JsonPropertyName("prompt_sub_category")]
public string? PromptSubCategory { get; init; }
[JsonPropertyName("condition")] public string? Condition { get; init; }
[JsonPropertyName("prompt")] public string? Prompt { get; init; }
[JsonPropertyName("prompt_id")] public string? PromptId { get; init; }
[JsonPropertyName("weight")] public double? Weight { get; init; }
}
}

View File

@@ -0,0 +1,12 @@
# Build A Legacy Mim Seed
This folder holds the first checked-in Build A legacy MIM seed set.
Importer rules:
- each `.mim` file is parsed as JSON
- XML-style tags and `${placeholder}` tokens are stripped into spoken text
- Build A uses declarative prompt packs only
- imported prompts are merged into the existing in-memory catalog
The goal is to get immediate personality value from source-backed legacy content while keeping the current runtime surface unchanged.

View File

@@ -0,0 +1,83 @@
{
"skill_id": "chitchat",
"mim_type": "announcement",
"rule_name": "",
"rule_slots": "",
"screen_slots_available": false,
"timeout": 3,
"max_tries": null,
"force_confirmation": false,
"barge_in": false,
"photo_quality_light": false,
"notes": "Thanks-Ignore",
"prompts": [
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "<ssa cat='oops'/>. Something's off with the connection to my sources. Maybe ask me again in a little while.",
"media": "TTS",
"extra": "",
"prompt_id": "CC_Error_AN_01",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "<ssa cat='oops'/>. It seems I can't connect to my favorite info sources at the moment. Maybe you can try again a little later.",
"media": "TTS",
"prompt_id": "CC_Error_AN_02",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "<ssa cat='oops'/>. My info sources seem to be down at the moment. Maybe try again a little later.",
"media": "TTS",
"prompt_id": "CC_Error_AN_03",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "<ssa cat='oops'/>. The place where I get info like that isn't responding to me. Maybe you can try again a little later.",
"media": "TTS",
"prompt_id": "CC_Error_AN_04",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "Huh, it seems like my info sources are down. Try asking me again a little later.",
"media": "TTS",
"prompt_id": "CC_Error_AN_05",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "It looks like my info sources aren't answering me. How bout you try again in a little while.",
"media": "TTS",
"prompt_id": "CC_Error_AN_06",
"weight": 1
}
],
"es_auto_tagging": true,
"gui": null,
"no_matches_for_gui": 2,
"no_inputs_for_gui": 2,
"ignore_no_match": false,
"parse_all_asr": false,
"thanks_handling": "ignore"
}

View File

@@ -0,0 +1,73 @@
{
"skill_id": "chitchat",
"mim_type": "announcement",
"rule_name": "",
"rule_slots": "",
"screen_slots_available": false,
"timeout": 2,
"max_tries": null,
"force_confirmation": false,
"barge_in": false,
"photo_quality_light": false,
"notes": "Thanks-Ignore",
"prompts": [
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "I think only <pitch mult=\"1.1\">you</pitch> can answer that question.",
"media": "TTS",
"prompt_id": "CC_Deflector_ReferToSelf_AN_01",
"weight": 1
},
{
"mim_id": "CCWolframDeflector",
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "I'm not sure. I guess I don't know as much about you as I should.",
"media": "TTS",
"prompt_id": "CC_Deflector_ReferToSelf_AN_02",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "Honestly I think I don't know you well enough to answer that.",
"media": "TTS",
"prompt_id": "CC_Deflector_ReferToSelf_AN_03",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "",
"prompt": "That is one question about you that I can't answer.",
"media": "TTS",
"prompt_id": "CC_Deflector_ReferToSelf_AN_04",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "!!speaker",
"prompt": "${speaker} I think only you can answer that question.",
"media": "TTS",
"prompt_id": "CC_Deflector_ReferToSelf_AN_05",
"weight": 1
}
],
"es_auto_tagging": true,
"gui": null,
"no_matches_for_gui": 2,
"no_inputs_for_gui": 2,
"ignore_no_match": false,
"parse_all_asr": false,
"thanks_handling": "ignore"
}

View File

@@ -0,0 +1,70 @@
{
"mim_type": "announcement",
"rule_name": "",
"timeout": 6,
"barge_in": true,
"es_auto_tagging": true,
"notes": "",
"prompts": [
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "jibo.emotion==\"JOYFUL\"",
"prompt": "Yes indeed. Never been better.",
"media": "TTS",
"prompt_id": "OI_JBO_IsHappy_AN_01",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "jibo.emotion==\"PLEASED\"",
"prompt": "You know it. Life is good.",
"media": "TTS",
"prompt_id": "OI_JBO_IsHappy_AN_02",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "jibo.emotion == \"DETERMINED\"",
"prompt": "You're right. I <pitch mult=\"1.3\">am </pitch> feeling pretty good at the moment.",
"media": "TTS",
"prompt_id": "OI_JBO_IsHappy_AN_03",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "jibo.emotion==\"CONFIDENT\"",
"prompt": "All systems are go.",
"media": "TTS",
"prompt_id": "OI_JBO_IsHappy_AN_04",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "!jibo.emotion || jibo.emotion==\"NEUTRAL\"",
"prompt": "All systems are go.",
"media": "TTS",
"prompt_id": "OI_JBO_IsHappy_AN_05",
"weight": 1
},
{
"prompt_category": "Entry-Core",
"prompt_sub_category": "AN",
"index": 1,
"condition": "jibo.emotion == \"INSECURE\"",
"prompt": "Yes. Not too shabby.",
"media": "TTS",
"prompt_id": "OI_JBO_IsHappy_AN_06",
"weight": 1
}
]
}

Some files were not shown because too many files have changed in this diff Show More