diff --git a/.gitignore b/.gitignore index ff292d7..d373a60 100644 --- a/.gitignore +++ b/.gitignore @@ -420,3 +420,4 @@ FodyWeavers.xsd OpenJibo/captures/ OpenJibo/.tmp/ +OpenJibo/docs/DesignDoc/original server diff --git a/OpenJibo/docs/DesignDoc/original-server-design.md b/OpenJibo/docs/DesignDoc/original-server-design.md new file mode 100644 index 0000000..948dc01 --- /dev/null +++ b/OpenJibo/docs/DesignDoc/original-server-design.md @@ -0,0 +1,792 @@ +# Original Jibo Server (Pegasus) Design Document + +## Executive Summary + +The original Jibo server, codenamed "Pegasus" (formerly V1.X), is a cloud-based microservices architecture that powers the Jibo social robot's conversational AI capabilities. It is built as a Lerna monorepo using Node.js/TypeScript and deployed via Docker containers. The system processes speech, performs natural language understanding, routes to appropriate skills, and manages proactive behaviors. + +## Architecture Overview + +### Monorepo Structure + +The codebase is organized as a Lerna monorepo with the following main packages: + +- **packages/hub** - Central orchestration service +- **packages/parser** - NLU (Natural Language Understanding) service +- **packages/history** - Data persistence service (MongoDB) +- **packages/baseskill** - Base class and framework for cloud skills +- **packages/interfaces** - TypeScript interfaces and API contracts +- **packages/utils** - Shared utility libraries +- **packages/chitchat-skill** - Example conversational skill +- **packages/report-skill** - Reporting skill +- **packages/lasso** - External data integration service +- **packages/hub-client** - Client library for hub communication +- **packages/history-client** - Client library for history service +- **packages/test-utils** - Testing utilities + +### Technology Stack + +- **Language**: TypeScript 2.5.3 +- **Runtime**: Node.js 8.9.4 +- **Package Manager**: Yarn 1.7.0 +- **Containerization**: Docker +- **Orchestration**: Docker Compose (local), AWS ECS (production) +- **Database**: MongoDB 3.6.0 +- **Cache**: Redis 3 +- **NLU**: Dialogflow (API.ai) +- **ASR**: Google Cloud Speech API +- **WebSocket**: ws library +- **HTTP**: Express.js +- **Authentication**: JWT (jsonwebtoken) + +## Core Services + +### 1. Hub Service (`packages/hub`) + +The Hub is the central orchestrator that coordinates all interactions between the robot and cloud services. + +#### Key Components + +**HubService** (`HubService.ts`) +- Main service class extending `BaseService` +- Initializes and manages all hub components +- Registers WebSocket and HTTP handlers + +**HubComponents** - Dependency injection container: +- `parser: ParserClient` - NLU service client +- `skillConfigManager: SkillConfigManager` - Manages skill configurations +- `intentRouter: IntentRouter` - Routes intents to skills +- `skillRequestMaker: SkillRequestMaker` - Makes HTTP requests to skills +- `history: HistoryServiceClient` - History service client +- `hubSettings: HubSettings` - Hub configuration +- `settingsClient: SettingsClient` - Settings service client + +#### Endpoints + +**WebSocket Endpoints:** +- `/listen` and `/v1/listen` - Handles speech recognition and NLU +- `/proactive` and `/v1/proactive` - Handles proactive triggers + +**HTTP Endpoints:** +- `/skills` and `/v1/skills` - Lists available skills +- `/healthcheck` - Service health check + +#### Listen Flow + +The listen transaction follows a state machine implemented in `ListenTransactionHandler`: + +``` +States: + WAIT_LISTEN → ASR → NLU → ROUTE → DONE + WAIT_LISTEN → WAIT_CLIENT_ASR → NLU → ROUTE → DONE + WAIT_LISTEN → WAIT_CLIENT_NLU → ROUTE → DONE +``` + +**State Transitions:** + +1. **WAIT_LISTEN** - Receives LISTEN message from robot +2. **ASR** - Performs Automatic Speech Recognition using Google Cloud Speech API + - Streams audio packets + - Emits SOS (Start of Speech) when speech detected + - Emits EOS (End of Speech) when speech ends + - Handles timeouts (SOS timeout, max speech timeout) +3. **NLU** - Sends ASR text to Parser service for intent recognition + - Includes context (loop users, perception, etc.) + - Supports external Dialogflow agents +4. **ROUTE** - Intent Router determines which skill to launch + - Matches NLU result against skill intent configurations + - Decision Mediator can alter decisions based on external factors + - Routes to on-robot skills or cloud skills +5. **DONE** - Transaction complete + +**Listen Transaction Handler** (`ListenTransactionHandler.ts`): +- Manages audio streaming via `AudioBuffer` +- Creates `ASRSession` for speech recognition +- Handles timeouts (ASR: 40s, Parser: 10s, Context: 5s, Skill: 10s) +- Records speech history to MongoDB and optionally S3 +- Supports client-provided ASR/NLU (for menu clicks, etc.) +- Handles skill redirects + +#### Proactive Flow + +The proactive system allows Jibo to initiate conversations based on context, history, and triggers. + +**Proactive Transaction Handler** (`ProactiveTransactionHandler.ts`): + +1. Receives TRIGGER message from robot +2. Waits for CONTEXT message (robot state) +3. **Action Selection**: + - Gets all proactive skill configurations + - Filters by context rules (time, location, people present, etc.) + - Filters by interaction history rules (frequency, recency) + - Filters by user settings + - Randomly selects from eligible actions +4. Launches selected skill (on-robot or cloud) +5. Returns match response or no-action response + +**Proactive Registration**: +Skills register proactive behaviors with: +- Trigger types (time-based, event-based, surprise) +- Context rules (when this can trigger) +- Interaction history rules (how often it can trigger) +- Settings rules (user preferences) + +### 2. Parser Service (`packages/parser`) + +The Parser service performs Natural Language Understanding using Dialogflow. + +**ParserService** (`ParserService.ts`): +- Starts RobustParser process on port 8787 (optional) +- Initializes Dialogflow client +- Initializes Robust Parser client +- Handles POST requests to `/v1/parse` +- Exposes state at `/state` endpoint + +**NLU Pipeline:** +1. Receives text, rules, and context +2. Queries Dialogflow with configured agents +3. Optionally queries Robust Parser (custom NLU) +4. Returns intent, entities, and rules + +**Configuration:** +- Dialogflow API key +- Robust Parser enable/disable +- Multiple external agents support + +### 3. History Service (`packages/history`) + +The History service persists interaction data to MongoDB. + +**HistoryService** (`HistoryService.ts`): +- Two database clients: + - `SkillLaunchDBClient` - Records skill launches + - `SpeechHistoryDBClient` - Records speech interactions (optional) +- HTTP endpoints: + - `/v1/skill/launch` - Skill launch history + - `/v1/speech` - Speech history (if enabled) +- Health check endpoint + +**Data Stored:** +- Skill launches (skill ID, intent, timestamp, robot ID, account ID) +- Speech interactions (ASR result, NLU result, audio file URL, error tracking) + +### 4. Lasso Service (`packages/lasso`) + +Lasso provides external data integration for skills. + +**Features:** +- OAuth2 credential management +- Calendar client integration +- Weather data (Dark Sky API) +- Maps data (Google Maps API) +- News data (AP News) +- MongoDB for credential storage +- Redis for caching + +**LassoService** (`LassoService.ts`): +- Manages OAuth2 flows +- Provides relay endpoints for external APIs +- Caches responses in Redis + +## Skill Framework + +### BaseSkill (`packages/baseskill`) + +**BaseSkill** (`BaseSkill.ts`): +- Abstract base class for all cloud skills +- Extends `BaseHttpHandler` +- Handles POST requests to `/` +- Provides error handling +- Tracks timing + +**GraphSkill** (`GraphSkill.ts`): +- Extends BaseSkill with graph-based state machine +- Implements node-based conversation flow +- Supports skill redirects +- Tracks analytics events +- Supports supplemental behaviors (parallel/sequence) + +### Graph System + +The graph system provides a state machine framework for skills. + +**Graph** (`Graph.ts`): +- Directed graph of connected nodes +- Supports subgraphs (hierarchical) +- Exit transitions for graph termination +- Validation (reachability, transition completeness) +- GraphViz dot file generation + +**GraphManager** (`GraphManager.ts`): +- Singleton per skill +- Manages node IDs and mappings +- Executes graph: + - `start()` - Creates session, enters initial node + - `enterNode()` - Calls node's enter method + - `exitNode()` - Calls node's exit method with action results + - `executeTransition()` - Moves to next node +- Maintains session state (node ID, data, trace) + +**Node** (`Node.ts`): +- Abstract base class for graph nodes +- Has transition names and destinations +- Two lifecycle methods: + - `enter(data)` - Called when node is entered, returns action or redirect + - `exit(data)` - Called with action results, returns next transition +- Supports graph traversal (BFS) + +**Built-in Node Types:** +- `DefaultNode` - Simple terminal node +- `JCPNode` - Returns JCP action +- `NoOpNode` - No operation +- `TrueFalseNode` - Conditional branching +- `SetLooperIDNode` - Sets speaker ID + +**MIM (Motion Interaction Model) System:** +- `ANFactory` - Creates graph for playing MIM animations +- Supports scripted responses, emotion responses, fallback responses +- Semi-specific responses (context-aware) + +### Skill Request/Response Protocol + +**Skill Request Types** (`skill/request.ts`): +- `LISTEN_LAUNCH` - Launch skill from listen interaction +- `LISTEN_UPDATE` - Update skill with action results +- `PROACTIVE_LAUNCH` - Launch skill proactively + +**Skill Request Data:** +```typescript +{ + type: MessageType, + msgID: UUID, + ts: number, + data: { + general: { accountID, robotID, lang, release }, + runtime: { character, location, loop, perception, dialog }, + skill: { id, session? }, + result: any, // Action results for UPDATE + nlu: NLUResult, + asr: ASRResult, + memo?: any + } +} +``` + +**Skill Response Types** (`skill/response.ts`): +- `SKILL_ACTION` - Returns action to execute +- `SKILL_REDIRECT` - Redirects to another skill +- `ERROR` - Error response + +**Skill Action Data:** +```typescript +{ + action: JCPAction, // JCP protocol behavior + analytics?: AnalyticsData, + final?: boolean, // Is this the final response? + fireAndForget?: boolean +} +``` + +**JCP Action** (`skill/action.ts`): +```typescript +{ + type: ActionType.JCP, + config: { + version: "1.0.0", + jcp: SupportedBehaviors // SLIM, Sequence, Parallel, SetPresentPerson, ImpactEmotion + } +} +``` + +### Skill Configuration + +**SkillConfig** (`skill/config.ts`): +```typescript +{ + id: SkillID, + intents: [{ + name: IntentName, + entities?: EntityConfig[], + memo?: any + }], + proactives?: ProactiveRegistration[], + IHQueries?: IHQueryDefinitions, + onRobot?: boolean, + URL: string, + settings?: ManifestSettings +} +``` + +**Entity Config**: +- `name` - Entity name +- `value` - Expected value +- `matchRule` - 'EXACT' or 'NOT' + +**Proactive Registration**: +- Trigger type and conditions +- Context rules +- Interaction history rules +- Settings rules + +## Interfaces Package + +The `interfaces` package defines all TypeScript interfaces for communication between services. + +### Key Interface Modules + +**service.ts** - Base message types: +- `BaseMessage` - Generic message with type, msgID, timestamp, data +- `BaseResponse` - Response with final flag and timings +- `IAuthDetails` - Authentication details (account ID, access keys) + +**hub/** - Hub-specific interfaces: +- `request.ts` - LISTEN, CONTEXT, CLIENT_ASR, CLIENT_NLU messages +- `response.ts` - ASR, NLU, LISTEN, SKILL_REDIRECT, ERROR responses +- `MessageType.ts` - Message type enums +- `HubErrorCode.ts` - Error code enums + +**skill/** - Skill-specific interfaces: +- `request.ts` - LISTEN_LAUNCH, LISTEN_UPDATE, PROACTIVE_LAUNCH +- `response.ts` - SKILL_ACTION, SKILL_REDIRECT, ERROR +- `action.ts` - JCP action types +- `config.ts` - Skill configuration +- `behaviors.ts` - Supported JCP behaviors +- `analytics.ts` - Analytics event types + +**nlu.ts** - NLU interfaces: +- `NLURequestData` - Text, rules, loop users, external agents +- `NLUResult` - Intent, entities, rules +- `ExternalAgentRequest` - External Dialogflow agent config + +**asr.ts** - ASR interfaces: +- `ASRResult` - Text, confidence, annotation +- `ASRConfig` - Language, hints, timeouts + +**jibo/** - Jibo-specific data: +- `data.ts` - GeneralData (account, robot, language), SkillData (session, trace) +- `runtime.ts` - RuntimeContext (character, location, loop, perception, dialog) + +**proactive/** - Proactive interfaces: +- Context field definitions +- History rules +- Settings rules +- Proactive trigger/request/response + +**history/** - History interfaces: +- Skill launch data +- Speech history data + +## Utils Package + +The `utils` package provides shared functionality. + +### BaseService (`utils/service/BaseService.ts`) + +Base class for all Pegasus services: + +**Features:** +- Express.js HTTP server +- WebSocket server (ws library) +- JWT authentication +- Request/response logging with jibo-log +- New Relic monitoring +- Health check endpoint +- Error handling middleware + +**Methods:** +- `addSocketHandler(path, handler)` - Register WebSocket handler +- `addHttpHandler(path, handler)` - Register HTTP handler +- `init(port)` - Start server +- `close()` - Stop server + +**Authentication:** +- JWT token verification +- Bearer token scheme +- Configurable secret via `ETCO_server_hubTokenSecret` + +**Logging:** +- Per-request log instances +- Transaction ID tracking +- Robot ID tracking +- Configurable log levels per namespace + +### Other Utils + +- `PegasusRequest` - Enhanced Express request with Jibo headers +- `PegasusWebSocket` - Enhanced WebSocket with auth and logging +- `JiboHeaders` - Parses Jibo-specific headers (transID, robotID, logging config) +- `ResponseWrapper` - Wraps WebSocket responses +- `HttpError` - HTTP error with status code + +## Communication Protocols + +### WebSocket Protocol + +**Connection:** +- URL: `ws://hub:9000/listen` or `ws://hub:9000/proactive` +- Authentication: Bearer token in Authorization header +- Headers: `x-jibo-transid`, `x-jibo-robotid`, `x-jibo-logging-config` + +**Message Format:** +```json +{ + "type": "MESSAGE_TYPE", + "msgID": "uuid", + "ts": 1234567890, + "data": { ... } +} +``` + +**Listen Flow Messages:** +1. Robot → Hub: LISTEN (with ASR config, rules, language) +2. Robot → Hub: Audio packets (binary) +3. Hub → Robot: SOS (Start of Speech) +4. Robot → Hub: CONTEXT (runtime context) +5. Hub → Robot: EOS (End of Speech) +6. Hub → Robot: LISTEN (with ASR result, NLU result, match) +7. Hub → Robot: SKILL_ACTION (if cloud skill) +8. Robot → Hub: CMD_RESULT (action results) +9. Hub → Robot: SKILL_ACTION (next action) or final + +**Proactive Flow Messages:** +1. Robot → Hub: TRIGGER (trigger data) +2. Robot → Hub: CONTEXT (runtime context) +3. Hub → Robot: PROACTIVE (match or no-action) +4. Hub → Robot: SKILL_ACTION (if cloud skill) + +### HTTP Protocol + +**Skill Request:** +- Method: POST +- URL: `http://skill-host:port/` +- Headers: Authorization, x-jibo-transid, x-jibo-robotid +- Body: SkillRequest JSON + +**Parser Request:** +- Method: POST +- URL: `http://parser:8080/v1/parse` +- Body: NLURequestData JSON + +## Authentication & Security + +### JWT Authentication + +**Token Format:** +```json +{ + "id": "account-id", + "accessKeyId": "client-id", + "secretAccessKey": "client-secret", + "friendlyId": "robot-name" +} +``` + +**Verification:** +- Secret: `ETCO_server_hubTokenSecret` environment variable +- Scheme: Bearer +- Applied to WebSocket connections and HTTP endpoints + +### Network Security + +- All services run in Docker containers +- Services communicate via Docker network (pegasus-nw) +- External access via load balancer +- TLS termination at load balancer + +## Deployment + +### Docker Compose (Local Development) + +**Services:** +- `hub` - Hub service (port 9000) +- `parser` - Parser service (port 9005) +- `history` - History service (port 9006) +- `chitchat-skill` - Chitchat skill (port 9004) +- `report-skill` - Report skill (port 9003) +- `lasso` - Lasso service (port 9007) +- `redis` - Redis cache (port 6379) +- `mongo_lasso` - MongoDB for Lasso (port 27017) +- `history_cluster` - MongoDB for History (from docker-compose-history-db.yml) + +**Configuration:** +- Environment variables prefixed with `ETCO_` (ETCO = Environment TO Configuration) +- Volume mounting: `./:/pegasus:consistent` for live code editing +- Debug ports: 5850-5855 for Node.js debugging + +### Build Process + +**Commands:** +```bash +docker build -t pegasus_base:latest . +yarn docker:bootstrap +yarn docker:build +./pegasus.js build-docker-image --services hub +``` + +**CLI Tool** (`cli/`): +- `bootstrap` - Install dependencies +- `build` - Build TypeScript +- `test` - Run tests +- `docker-run` - Run commands in Docker +- `build-docker-image` - Build Docker images for services + +### Production Deployment + +- AWS ECS (Elastic Container Service) +- ECR (Elastic Container Registry) for Docker images +- Application Load Balancer +- MongoDB Atlas for production databases +- ElastiCache for Redis +- CloudWatch for logging +- New Relic for monitoring + +## Data Flow Examples + +### Example 1: User Says "Tell Me a Joke" + +1. **Robot → Hub**: LISTEN message with ASR config +2. **Robot → Hub**: Audio stream +3. **Hub**: Detects SOS, emits SOS message +4. **Hub**: Streams audio to Google Cloud Speech API +5. **Hub**: Detects EOS, emits EOS message +6. **Robot → Hub**: CONTEXT message (runtime state) +7. **Hub → Parser**: POST /v1/parse with text "tell me a joke" +8. **Parser → Dialogflow**: Query with "joke" intent rules +9. **Dialogflow → Parser**: Intent="joke_tell", entities={} +10. **Parser → Hub**: NLU result +11. **Hub → IntentRouter**: Match intent to "joke-skill" +12. **Hub → joke-skill**: POST LISTEN_LAUNCH request +13. **joke-skill**: Executes graph, selects joke +14. **joke-skill → Hub**: SKILL_ACTION with JCP behavior (SayText) +15. **Hub → Robot**: SKILL_ACTION message +16. **Robot**: Executes behavior, speaks joke +17. **Robot → Hub**: CMD_RESULT with action result +18. **Hub → joke-skill**: POST LISTEN_UPDATE request +19. **joke-skill**: Returns final=true +20. **Hub → Robot**: Final SKILL_ACTION + +### Example 2: Proactive Greeting + +1. **Robot**: Detects person entering room +2. **Robot → Hub**: TRIGGER message with trigger data +3. **Robot → Hub**: CONTEXT message (runtime state) +4. **Hub**: Queries all proactive skill configs +5. **Hub**: Filters by context (time, people present) +6. **Hub**: Filters by history (last greeting time) +7. **Hub**: Filters by settings (user greeting preference) +8. **Hub**: Selects "greeting-skill" +9. **Hub → greeting-skill**: POST PROACTIVE_LAUNCH request +10. **greeting-skill → Hub**: SKILL_ACTION with greeting behavior +11. **Hub → Robot**: PROACTIVE response with match +12. **Hub → Robot**: SKILL_ACTION message +13. **Robot**: Executes greeting + +## Error Handling + +### Error Types + +**Hub Error Codes** (`HubErrorCode.ts`): +- `TIMEOUT_ASR` - ASR timeout +- `TIMEOUT_PARSER` - Parser timeout +- `TIMEOUT_CONTEXT` - Context timeout +- `TIMEOUT_SKILL` - Skill timeout +- `PARSER` - Parser error +- `ASR` - ASR error + +**Skill Request Errors** (`SkillRequestError`): +- `SKILL_NOT_FOUND` - Skill does not exist +- `TIMEOUT` - Skill request timeout + +### Error Response Format + +```json +{ + "type": "ERROR", + "msgID": "uuid", + "ts": 1234567890, + "final": true, + "data": { + "message": "Error description", + "code": "ERROR_CODE" + }, + "timings": { + "total": 1234 + } +} +``` + +### Timeout Handling + +- ASR: 40 seconds (configurable via sosTimeout, maxSpeechTimeout) +- Parser: 10 seconds +- Context: 5 seconds +- Skill: 10 seconds +- Transaction: 60 seconds (configurable) + +## Monitoring & Logging + +### Logging + +**jibo-log Integration:** +- Per-namespace log levels +- Transaction ID correlation +- Robot ID tracking +- Structured logging support + +**Log Levels:** +- Configured via `x-jibo-logging-config` header +- Per-namespace granularity +- Environment variable: `ETCO_server_logLevel` + +### Monitoring + +**New Relic:** +- HTTP request tracking +- WebSocket transaction tracking +- Error tracking +- Custom attributes (transID, robotID) + +**Health Checks:** +- `/healthcheck` endpoint on all services +- Returns service-specific health data +- Database connection status + +### Speech History Recording + +**Optional Features:** +- Record skill launches to MongoDB +- Record speech interactions to MongoDB +- Upload speech logs to S3 (JSON with audio base64) + +**Configuration:** +- `ETCO_hub_recordLaunchHistory` - Enable launch history +- `ETCO_hub_recordSpeechHistory` - Enable speech history +- `ETCO_hub_recordSpeechLogBucket` - S3 bucket for speech logs + +## Skill Development Guide + +### Creating a New Skill + +1. **Extend GraphSkill:** +```typescript +export class MySkill extends GraphSkill { + constructor() { + super('my-skill'); + } + + createGraph(): Graph { + const g = new Graph('My Skill', generateTransitions(Transition)); + // Add nodes and transitions + g.finalize(); + return g; + } +} +``` + +2. **Define Transitions:** +```typescript +enum Transition { + Done = 'Done', + Retry = 'Retry' +} +``` + +3. **Create Nodes:** +```typescript +class MyNode extends Node { + async enter(data: Data): Promise { + // Return action or redirect + return { action: myJCPAction }; + } + + async exit(data: Data): Promise { + // Return next transition + return { transition: Transition.Done }; + } +} +``` + +4. **Create Skill Manifest:** +```json +{ + "id": "my-skill", + "intents": [ + { + "name": "my_intent", + "entities": [] + } + ], + "onRobot": false +} +``` + +5. **Register with Hub:** +- Add skill config to skills-local.json or environment +- Deploy skill service +- Hub will load configuration + +### Skill Best Practices + +- Use graph for complex flows, direct responses for simple ones +- Track analytics events for monitoring +- Handle errors gracefully with try-catch +- Use supplemental behaviors for parallel actions +- Set appropriate timeouts +- Log important events +- Test with both LISTEN_LAUNCH and PROACTIVE_LAUNCH + +## Key Design Decisions + +### Why Graph-Based Skills? + +- **State Management**: Explicit state machine with session tracking +- **Visualization**: GraphViz generation for debugging +- **Reusability**: Subgraphs for common patterns +- **Testability**: Isolated node testing +- **Maintainability**: Clear flow structure + +### Why WebSocket for Robot Communication? + +- **Low Latency**: Real-time bidirectional communication +- **Audio Streaming**: Binary message support for audio +- **Stateful**: Single connection per transaction +- **Efficiency**: No HTTP overhead for each message + +### Why Separate Services? + +- **Scalability**: Scale each service independently +- **Isolation**: Failure in one service doesn't affect others +- **Technology**: Different services can use different tech stacks +- **Deployment**: Independent deployment cycles + +### Why Lerna Monorepo? + +- **Code Sharing**: Easy to share interfaces and utils +- **Versioning**: Linked versioning for interdependent packages +- **Development**: Single repository for all services +- **Testing**: Integration tests across packages + +## Limitations & Known Issues + +1. **Single Graph Manager**: Skills cannot have concurrent sessions (singleton pattern) +2. **Sequential Skill Redirects**: Only one level of redirect supported +3. **No Skill-to-Skill Communication**: Skills must go through hub +4. **Fixed Timeouts**: Hardcoded timeouts in some places +5. **No Skill Hot-Reload**: Requires container rebuild for skill changes +6. **Limited NLU**: Dialogflow dependency, no custom model training +7. **No Skill Versioning**: Skills identified by ID only +8. **Synchronous Skill Requests**: Hub waits for skill response (no async) + +## Future Considerations + +1. **Skill Versioning**: Support multiple versions of same skill +2. **Skill-to-Skill Direct Communication**: Allow skills to call each other +3. **Async Skill Responses**: Long-running skills with callback pattern +4. **Custom NLU Models**: Support for custom trained models +5. **Skill Hot-Reload**: Dynamic skill loading without restart +6. **Multi-Session Skills**: Support concurrent skill sessions +7. **Skill Marketplace**: Third-party skill distribution +8. **A/B Testing**: Framework for testing skill variations + +## Conclusion + +The original Jibo server (Pegasus) is a well-architected microservices system that provides a robust foundation for conversational AI on the Jibo robot. The graph-based skill framework offers flexibility and maintainability, while the separation of concerns enables independent scaling and development. The system successfully handles real-time speech processing, natural language understanding, skill routing, and proactive behaviors in a distributed cloud environment. diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index d44a7ab..fa52715 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -614,6 +614,8 @@ Current release theme: - recognition, enrollment, rename, and profile-correction boundaries - split between local state and hosted cloud state - first useful hosted identity slice + - live QA has shown person-identification collisions in the same loop (for example, a parent and child both getting normalized to the same remembered name) + - person-identification correction likely needs its own repair pass before we can trust greetings, reports, and presence triggers in mixed-household scenarios ### 20. Onboarding, Loop Management, And Fresh Start @@ -638,9 +640,12 @@ Current release theme: - `make a pizza` now ports the original scripted-response path through `chitchat-skill` with `mim_id = RA_JBO_MakePizza` and pizza-making animation ESML - `can you order pizza` now ports the original scripted-response path through `chitchat-skill` with `mim_id = RA_JBO_OrderPizza` - current source answers these with a `1.0.19` rule-based persona baseline, backed by `OpenJiboCloudBuildInfo.PersonaBirthday` + - `how old are you` now also uses the imported Build B age prompts so the first-powered-up and birthday phrasing stays source-backed - Follow-up: - wire persona age to first-powered-up or durable first-cloud-seen metadata when available - add command-vs-question variants so expressive prompts can answer conversationally before launching actions + - live QA has shown motion/sleep quirks too: `turn around` can become a no-op and `go to sleep` can fail at the last step before the sleep animation fully completes + - reply-selection polish still needs attention on a couple of identity prompts where short variants are over-selected (`how are you`, `what is your favorite flower`) ### 22. Command Vs Question Reply Style @@ -770,7 +775,7 @@ Current release theme: ### 28. Grocery List Capability (Requested Feature) -- Status: `discovery` +- Status: `in_progress` - Tags: `content`, `docs`, `storage` - Why now: - directly requested by Jibo owners and fits memory + household utility roadmap @@ -779,13 +784,14 @@ Current release theme: - examples: - `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ShoppingList.mim` - `C:\Projects\jibo\pegasus\packages\chitchat-skill\mims\scripted-responses\RA_JBO_ManageToDoList.mim` -- Candidate delivery paths: - - native lightweight list skill (fastest user value) - - integration-backed list orchestration (long-term richer ecosystem fit) +- MVP decision: + - use the existing household list engine as the native lightweight grocery MVP + - keep grocery as a first-class spoken alias over the shopping list storage path + - reserve integration-backed list orchestration for a later discovery pass - Exit criteria: - - clear decision on MVP path - - first schema for list items + ownership scope - - initial voice flows and follow-up intent handling defined + - grocery prompts, add/recall/done flows, and list follow-ups consistently speak grocery wording + - existing shopping/to-do flows remain unchanged + - future integration-backed list work remains a separate backlog item ### 29. Legacy MIM Personality Import Ladder @@ -889,7 +895,10 @@ Current release theme: - richer identity follow-ups like `who is this`, `do you know me`, `do you remember me`, and `can you recognize me` - mood and affect prompts like `how are you`, `are you happy`, `are you sad`, and `are you angry` - self-description charm like `what's your name`, `do you have a nickname`, `do you like being Jibo`, and `what is your favorite name` + - deeper personality follow-ups like `what do you dream about`, `what are you afraid of`, `what do you want to talk about`, `what is your best book`, `what is your best exercise`, `what is your dream vacation`, `who is your hero`, `who do you love`, and `what is your religion`; `what is your sign` stays deferred until templated placeholder rendering exists + - the next identity / knowledge wave adds `are you god`, `are you here`, `do you have super powers`, `how much do you know`, `what does jibo mean`, `where do you get info`, `what are you forbidden to do`, `what color are you`, and `what do you do when alone` - additional legacy source-backed `RI_USR` prompts where the text is short and the behavior is easy to verify + - templated edge cases like `what is your sign`, `how many people do you know`, and `what is the loop` where live birthday and loop state are part of the line instead of a plain canned response - Exit criteria: - a stable checklist exists for the original persona surface - each pass can be scoped to a small batch of prompts @@ -975,7 +984,9 @@ For `1.0.19`: - next implementation pass should supply the real Azure Storage connection string / deployment wiring and validate the live round-trip in the storage account smoke test 10. Update, backup, and restore proof - implemented (update creation and backup creation now survive persisted reloads; restore is the persisted-state rehydration proof path, not a new cloud API) 11. STT upgrade and noise screening + - progress update (`2026-05-21`): added a low-signal short-turn screen in websocket finalization so filler-only fragments and stray single-token leftovers like `so command` get rejected before they can become bad turns, while preserving the existing yes/no and word-of-the-day short-turn flows 12. Hosted capture/storage plan / indexing for group testing + - progress update (`2026-05-21`): added a bundle helper so group testers can package raw capture trees, `capture-index.ndjson`, and exported fixtures into one zip handoff artifact 13. Binary-safe media storage / sync to cloud drive: OneDrive, Google Drive, Box, etc. 14. Provider-backed news and weather parity polish 15. Grocery list capability discovery and MVP selection diff --git a/OpenJibo/docs/live-jibo-capture.md b/OpenJibo/docs/live-jibo-capture.md index 8e0758a..544b8df 100644 --- a/OpenJibo/docs/live-jibo-capture.md +++ b/OpenJibo/docs/live-jibo-capture.md @@ -77,3 +77,18 @@ Useful helper scripts: - [scripts/cloud/get-websocket-capture-summary.sh](/OpenJibo/scripts/cloud/get-websocket-capture-summary.sh) - [scripts/cloud/import-websocket-capture-fixture.py](/OpenJibo/scripts/cloud/import-websocket-capture-fixture.py) - [live-jibo-test-runbook.md](/OpenJibo/docs/live-jibo-test-runbook.md) + +## Group Testing Handoff + +When you have a useful capture set and want to share it with another tester, bundle the capture root into a single zip so the raw events, capture index, and exported fixtures stay together. + +Recommended helper: + +- [scripts/cloud/New-CaptureBundle.ps1](/OpenJibo/scripts/cloud/New-CaptureBundle.ps1) + +The bundle includes: + +- `capture-index.ndjson` +- websocket and HTTP `*.events.ndjson` files +- exported `*.flow.json` fixtures +- a small `bundle-manifest.json` with file counts and source metadata diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index 5ea25f8..df24496 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -6,6 +6,8 @@ This release starts the shift from `1.0.18` hardening to visible feature growth. The goal is to keep compatibility work steady while shipping personality and capability slices that make OpenJibo feel less like a placeholder cloud and more like a real assistant platform. +For grocery list capability, the 1.0.19 MVP choice is the existing household list engine with grocery as a first-class spoken alias. That keeps the storage model simple now while leaving integration-backed list orchestration for a later pass. + ## Snapshot - Kickoff date: `2026-05-05` @@ -56,6 +58,13 @@ Current batch note: - the favorites batch now includes `what is your favorite animal`, `what is your favorite bird`, `do you like penguins`, and `do you like animals` so the penguin-centered replies stay close to Pegasus - the latest social batch adds `welcome back`, `what are you thinking`, `what have you been doing`, and `what did you do` so presence and charm stay lively without distracting from the memory roadmap - the newest identity-charm batch adds `what's your name`, `do you have a nickname`, `do you like being Jibo`, `are there others like you`, and `what is your favorite name` so the robot stays familiar while still sounding like Pegasus +- the next deep-personality batch adds `what do you dream about`, `what are you afraid of`, `what do you want to talk about`, `what is your best book`, `what is your best exercise`, `what is your dream vacation`, `who is your hero`, `who do you love`, and `what is your religion`; `what is your sign` is still deferred until we add templated placeholder rendering +- the next identity/knowledge batch adds `are you god`, `are you here`, `do you have super powers`, `how much do you know`, `what does jibo mean`, `where do you get info`, `what are you forbidden to do`, `what color are you`, and `what do you do when alone` +- the next body/mission batch adds `how much do you weigh`, `how tall are you`, `how much do you cost`, `what if I unplug you`, `what is your purpose`, `what is your prime directive`, `what is jibo commander`, `do you like commander app`, and `what are you made of` +- the templated edge-case batch adds `what is your sign`, `how many people do you know`, and `what is the loop` so the remaining source-backed lines can lean on live birthday and loop state +- the work/eat/home batch adds `how do you work`, `what do you eat`, `where do you live`, and `what languages do you speak` so the everyday self-description cluster keeps moving toward the original phrasing +- the age batch adds `how old are you` through `JBO_HowOldAreYou` so the birthday and first-powered-up phrasing stays source-backed instead of falling back to a generic age answer +- live QA has surfaced a few repair targets to carry into the next pass: person-identification collisions inside the same loop, `turn around` / `go to sleep` motion quirks, and a couple of reply-selection spots where short variants are being over-selected (`how are you`, `what is your favorite flower`) - this pass keeps Build B moving while still favoring source-backed phrasing and preserving the command-vs-question boundary - the next passes should keep the same pattern and prefer source-backed phrasing whenever the legacy MIM text is available - if a source-backed legacy line is missing, use a temporary direct reply only to keep the pass moving, then backfill source text later @@ -84,6 +93,8 @@ The goal is to port these in small batches, capture the source-backed phrasing w - the restore proof is the persisted-state rehydration path; do not scope it into a new hosted restore API until we have real device evidence - continue alarm/gallery/yes-no cleanup from `1.0.18` evidence where regressions are still open - improve short-turn STT reliability and low-signal screening +- the latest STT pass adds a websocket-side low-signal screen for filler-only and stray single-token leftovers while keeping yes/no and word-of-the-day turns intact +- capture indexing and group-test handoff now have a bundle helper that packages raw event captures, the index manifest, and exported fixtures together for easier review/share flows ### 3. Pegasus-To-Cloud Platform Porting diff --git a/OpenJibo/scripts/cloud/New-CaptureBundle.ps1 b/OpenJibo/scripts/cloud/New-CaptureBundle.ps1 new file mode 100644 index 0000000..f66c6a3 --- /dev/null +++ b/OpenJibo/scripts/cloud/New-CaptureBundle.ps1 @@ -0,0 +1,101 @@ +param( + [string]$CaptureRoot = "..\..\captures", + [string]$BundleDirectory = "..\..\captures\bundles", + [string]$BundleName +) + +function Get-RelativePath { + param( + [Parameter(Mandatory = $true)] + [string]$BasePath, + [Parameter(Mandatory = $true)] + [string]$FullPath + ) + + $normalizedBase = [System.IO.Path]::GetFullPath($BasePath) + if (-not $normalizedBase.EndsWith([System.IO.Path]::DirectorySeparatorChar)) { + $normalizedBase = $normalizedBase + [System.IO.Path]::DirectorySeparatorChar + } + + $normalizedFull = [System.IO.Path]::GetFullPath($FullPath) + if (-not $normalizedFull.StartsWith($normalizedBase, [StringComparison]::OrdinalIgnoreCase)) { + throw "Path '$FullPath' is not under '$BasePath'." + } + + return $normalizedFull.Substring($normalizedBase.Length) +} + +$resolvedCaptureRoot = Resolve-Path -LiteralPath $CaptureRoot -ErrorAction Stop +$resolvedBundleDirectory = Resolve-Path -LiteralPath $BundleDirectory -ErrorAction SilentlyContinue +if (-not $resolvedBundleDirectory) { + $resolvedBundleDirectory = New-Item -ItemType Directory -Force -Path $BundleDirectory | Select-Object -ExpandProperty FullName +} +else { + $resolvedBundleDirectory = $resolvedBundleDirectory.Path +} + +if ([string]::IsNullOrWhiteSpace($BundleName)) { + $timestamp = Get-Date -Format "yyyyMMdd-HHmmss" + $BundleName = "capture-bundle-$timestamp" +} + +$stagingDirectory = Join-Path $resolvedBundleDirectory "$BundleName.staging" +$archivePath = Join-Path $resolvedBundleDirectory "$BundleName.zip" + +if (Test-Path -LiteralPath $stagingDirectory) { + Remove-Item -LiteralPath $stagingDirectory -Recurse -Force +} + +New-Item -ItemType Directory -Force -Path $stagingDirectory | Out-Null + +try { + $sourceFiles = Get-ChildItem -LiteralPath $resolvedCaptureRoot -Recurse -File | Where-Object { + $_.Name -eq "capture-index.ndjson" -or + $_.Name -like "*.events.ndjson" -or + $_.Name -like "*.flow.json" + } + + if (-not $sourceFiles) { + Write-Host "No capture files were found under $resolvedCaptureRoot" + exit 0 + } + + foreach ($file in $sourceFiles) { + $relativePath = Get-RelativePath -BasePath $resolvedCaptureRoot -FullPath $file.FullName + $destinationPath = Join-Path $stagingDirectory $relativePath + $destinationDirectory = Split-Path -Parent $destinationPath + + if (-not (Test-Path -LiteralPath $destinationDirectory)) { + New-Item -ItemType Directory -Force -Path $destinationDirectory | Out-Null + } + + Copy-Item -LiteralPath $file.FullName -Destination $destinationPath -Force + } + + $captureIndexFiles = @($sourceFiles | Where-Object { $_.Name -eq "capture-index.ndjson" }) + $eventFiles = @($sourceFiles | Where-Object { $_.Name -like "*.events.ndjson" }) + $fixtureFiles = @($sourceFiles | Where-Object { $_.Name -like "*.flow.json" }) + + $manifest = [ordered]@{ + createdUtc = (Get-Date).ToUniversalTime().ToString("O") + sourceRoot = $resolvedCaptureRoot + fileCount = $sourceFiles.Count + captureIndexCount = $captureIndexFiles.Count + eventFileCount = $eventFiles.Count + fixtureCount = $fixtureFiles.Count + } + + $manifest | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath (Join-Path $stagingDirectory "bundle-manifest.json") -Encoding utf8 + + if (Test-Path -LiteralPath $archivePath) { + Remove-Item -LiteralPath $archivePath -Force + } + + Compress-Archive -Path (Join-Path $stagingDirectory '*') -DestinationPath $archivePath -Force + Write-Host "Created capture bundle at $archivePath" +} +finally { + if (Test-Path -LiteralPath $stagingDirectory) { + Remove-Item -LiteralPath $stagingDirectory -Recurse -Force + } +} diff --git a/OpenJibo/scripts/cloud/README.md b/OpenJibo/scripts/cloud/README.md index f8b82b4..909128b 100644 --- a/OpenJibo/scripts/cloud/README.md +++ b/OpenJibo/scripts/cloud/README.md @@ -16,6 +16,8 @@ These scripts help exercise the new .NET hosted cloud locally. Runs a small readiness checklist before the first physical Jibo test against the .NET cloud. - `Import-WebSocketCaptureFixture.ps1` Sanitizes an exported websocket capture fixture and copies it into the checked-in websocket fixture set. +- `New-CaptureBundle.ps1` + Packages the capture root, capture index, and exported fixtures into a single zip bundle for group testing handoff. - `start-dotnet-with-node-cert.sh` Starts the .NET API on Linux using the same PEM certificate material already used by the Node server. - `invoke-live-jibo-prep.sh` diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs index 7c2c360..7230c36 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs @@ -31,6 +31,7 @@ public sealed class JiboExperienceCatalog public IReadOnlyList HolidayTrackerReplies { get; init; } = []; public IReadOnlyList BirthdayCelebrationReplies { get; init; } = []; public IReadOnlyList HowAreYouReplies { get; init; } = []; + public IReadOnlyList AgeReplies { get; init; } = []; public IReadOnlyList EmotionReplies { get; init; } = []; public IReadOnlyList PersonalityReplies { get; init; } = []; public IReadOnlyList PizzaReplies { get; init; } = []; diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ChitchatStateMachine.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ChitchatStateMachine.cs index 83df9fb..2899bd9 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ChitchatStateMachine.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ChitchatStateMachine.cs @@ -199,6 +199,10 @@ internal static class ChitchatStateMachine "want to hang out", "be helpful", "dance from time to time")); + case "robot_want_to_talk_about": + return BuildScriptedResponseDecision( + "robot_want_to_talk_about", + SelectLegacyPersonalityReply(catalog, randomizer, "surprise me")); case "robot_job": return BuildScriptedResponseDecision( "robot_job", @@ -395,13 +399,18 @@ internal static class ChitchatStateMachine string? currentEmotion, string? preferredName) { - if (catalog.EmotionReplies.Count == 0) - return PersonalizeHowAreYouReply(randomizer.Choose(catalog.HowAreYouReplies), preferredName); + if (catalog.EmotionReplies.Count > 0) + { + var emotionVariants = ResolveEmotionVariants(currentEmotion); + var matchingReplies = catalog.EmotionReplies + .Where(reply => ConditionMatches(reply.Condition, emotionVariants)) + .Select(reply => reply.Reply) + .Where(reply => !string.IsNullOrWhiteSpace(reply)) + .ToArray(); - var emotionVariants = ResolveEmotionVariants(currentEmotion); - foreach (var reply in catalog.EmotionReplies) - if (ConditionMatches(reply.Condition, emotionVariants)) - return PersonalizeHowAreYouReply(reply.Reply, preferredName); + if (matchingReplies.Length > 0) + return PersonalizeHowAreYouReply(randomizer.Choose(matchingReplies), preferredName); + } return PersonalizeHowAreYouReply(randomizer.Choose(catalog.HowAreYouReplies), preferredName); } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/HouseholdListOrchestrator.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/HouseholdListOrchestrator.cs index c5fcdae..db9585e 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/HouseholdListOrchestrator.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/HouseholdListOrchestrator.cs @@ -7,11 +7,15 @@ internal static class HouseholdListOrchestrator { internal const string StateMetadataKey = "householdListState"; internal const string TypeMetadataKey = "householdListType"; + internal const string DisplayTypeMetadataKey = "householdListDisplayType"; internal const string NoMatchCountMetadataKey = "householdListNoMatchCount"; internal const string NoInputCountMetadataKey = "householdListNoInputCount"; private const string IdleState = "idle"; private const string AwaitingItemState = "awaiting_item"; + private const string ShoppingListType = "shopping"; + private const string GroceryListType = "grocery"; + private const string TodoListType = "todo"; private static readonly string[] ItemPrefixes = [ @@ -31,6 +35,10 @@ internal static class HouseholdListOrchestrator " to my shopping list", " to the shopping list", " on my shopping list", + " to my grocery list", + " to the grocery list", + " on my grocery list", + " my grocery list", " to my to do list", " to the to do list", " on my to do list", @@ -50,6 +58,7 @@ internal static class HouseholdListOrchestrator { var state = ReadString(turn, StateMetadataKey); var listType = ReadString(turn, TypeMetadataKey); + var displayType = ReadString(turn, DisplayTypeMetadataKey); var isActiveState = !string.IsNullOrWhiteSpace(state) && !string.Equals(state, IdleState, StringComparison.OrdinalIgnoreCase); var isShoppingIntent = string.Equals(semanticIntent, "shopping_list", StringComparison.OrdinalIgnoreCase); @@ -58,17 +67,19 @@ internal static class HouseholdListOrchestrator if (!isActiveState && !isShoppingIntent && !isTodoIntent) return Task.FromResult(null); - var resolvedListType = isShoppingIntent ? "shopping" : isTodoIntent ? "todo" : NormalizeListType(listType); - if (string.IsNullOrWhiteSpace(resolvedListType)) resolvedListType = "shopping"; + var resolvedListType = isShoppingIntent ? ShoppingListType : isTodoIntent ? TodoListType : NormalizeListType(listType); + if (string.IsNullOrWhiteSpace(resolvedListType)) resolvedListType = ShoppingListType; + var resolvedDisplayType = ResolveDisplayType(resolvedListType, displayType, isActiveState, loweredTranscript); var tenantScope = tenantScopeResolver(turn); if (ContainsAny(loweredTranscript, "cancel", "stop", "never mind", "nevermind", "forget it")) - return Task.FromResult(BuildCancelledDecision(resolvedListType)); + return Task.FromResult(BuildCancelledDecision(resolvedListType, resolvedDisplayType)); if (IsRecallRequest(loweredTranscript)) return Task.FromResult(BuildRecallDecision( resolvedListType, + resolvedDisplayType, personalMemoryStore.GetListItems(tenantScope, resolvedListType))); var directItem = TryExtractListItem(loweredTranscript); @@ -76,9 +87,9 @@ internal static class HouseholdListOrchestrator { if (IsConversationComplete(loweredTranscript)) return Task.FromResult(new JiboInteractionDecision( - resolvedListType == "shopping" ? "shopping_list_done" : "todo_list_done", - BuildDoneReply(resolvedListType, personalMemoryStore.GetListItems(tenantScope, resolvedListType)), - ContextUpdates: BuildContextUpdates(resolvedListType, IdleState))); + BuildListIntentName(resolvedListType, "done"), + BuildDoneReply(resolvedDisplayType, personalMemoryStore.GetListItems(tenantScope, resolvedListType)), + ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, IdleState))); directItem = NormalizeItem(transcript); } @@ -87,104 +98,108 @@ internal static class HouseholdListOrchestrator { personalMemoryStore.AddListItem(tenantScope, resolvedListType, directItem); return Task.FromResult(new JiboInteractionDecision( - resolvedListType == "shopping" ? "shopping_list_add" : "todo_list_add", - BuildAddedReply(resolvedListType, directItem, + BuildListIntentName(resolvedListType, "add"), + BuildAddedReply(resolvedDisplayType, directItem, personalMemoryStore.GetListItems(tenantScope, resolvedListType)), - ContextUpdates: BuildContextUpdates(resolvedListType, AwaitingItemState))); + ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, AwaitingItemState))); } if (string.IsNullOrWhiteSpace(transcript)) return Task.FromResult(new JiboInteractionDecision( - resolvedListType == "shopping" ? "shopping_list_prompt" : "todo_list_prompt", - BuildPromptReply(resolvedListType), - ContextUpdates: BuildContextUpdates(resolvedListType, AwaitingItemState))); + BuildListIntentName(resolvedListType, "prompt"), + BuildPromptReply(resolvedDisplayType), + ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, AwaitingItemState))); return Task.FromResult(new JiboInteractionDecision( - resolvedListType == "shopping" ? "shopping_list_prompt" : "todo_list_prompt", - BuildPromptReply(resolvedListType), - ContextUpdates: BuildContextUpdates(resolvedListType, AwaitingItemState))); + BuildListIntentName(resolvedListType, "prompt"), + BuildPromptReply(resolvedDisplayType), + ContextUpdates: BuildContextUpdates(resolvedListType, resolvedDisplayType, AwaitingItemState))); } - private static IDictionary BuildContextUpdates(string listType, string state) + private static IDictionary BuildContextUpdates(string listType, string displayType, string state) { return new Dictionary(StringComparer.OrdinalIgnoreCase) { [StateMetadataKey] = state, [TypeMetadataKey] = listType, + [DisplayTypeMetadataKey] = displayType, [NoMatchCountMetadataKey] = 0, [NoInputCountMetadataKey] = 0 }; } - private static JiboInteractionDecision BuildCancelledDecision(string listType) + private static JiboInteractionDecision BuildCancelledDecision(string listType, string displayType) { return new JiboInteractionDecision( - listType == "shopping" ? "shopping_list_cancel" : "todo_list_cancel", - listType == "shopping" ? "Okay. I stopped the shopping list." : "Okay. I stopped the to-do list.", + BuildListIntentName(listType, "cancel"), + $"Okay. I stopped the {BuildListLabel(displayType)}.", ContextUpdates: new Dictionary(StringComparer.OrdinalIgnoreCase) { [StateMetadataKey] = IdleState, [TypeMetadataKey] = listType, + [DisplayTypeMetadataKey] = displayType, [NoMatchCountMetadataKey] = 0, [NoInputCountMetadataKey] = 0 }); } - private static JiboInteractionDecision BuildRecallDecision(string listType, IReadOnlyList items) + private static JiboInteractionDecision BuildRecallDecision(string listType, string displayType, IReadOnlyList items) { if (items.Count == 0) return new JiboInteractionDecision( - listType == "shopping" ? "shopping_list_recall" : "todo_list_recall", - listType == "shopping" - ? "Your shopping list is empty." - : "Your to-do list is empty.", + BuildListIntentName(listType, "recall"), + $"Your {BuildListLabel(displayType)} is empty.", ContextUpdates: new Dictionary(StringComparer.OrdinalIgnoreCase) { [StateMetadataKey] = IdleState, [TypeMetadataKey] = listType, + [DisplayTypeMetadataKey] = displayType, [NoMatchCountMetadataKey] = 0, [NoInputCountMetadataKey] = 0 }); return new JiboInteractionDecision( - listType == "shopping" ? "shopping_list_recall" : "todo_list_recall", - listType == "shopping" - ? $"Your shopping list has {JoinList(items)}." - : $"Your to-do list has {JoinList(items)}.", + BuildListIntentName(listType, "recall"), + $"Your {BuildListLabel(displayType)} has {JoinList(items)}.", ContextUpdates: new Dictionary(StringComparer.OrdinalIgnoreCase) { [StateMetadataKey] = IdleState, [TypeMetadataKey] = listType, + [DisplayTypeMetadataKey] = displayType, [NoMatchCountMetadataKey] = 0, [NoInputCountMetadataKey] = 0 }); } - private static string BuildAddedReply(string listType, string addedItem, IReadOnlyList items) + private static string BuildAddedReply(string displayType, string addedItem, IReadOnlyList items) { - var itemLabel = listType == "shopping" ? "shopping list" : "to-do list"; + var itemLabel = BuildListLabel(displayType); return items.Count == 1 ? $"Added {addedItem} to your {itemLabel}. What else should I add?" : $"Added {addedItem} to your {itemLabel}. You now have {JoinList(items)}."; } - private static string BuildPromptReply(string listType) + private static string BuildPromptReply(string displayType) { - return listType == "shopping" - ? "What should I add to your shopping list?" - : "What should I add to your to-do list?"; + return $"What should I add to your {BuildListLabel(displayType)}?"; } - private static string BuildDoneReply(string listType, IReadOnlyList items) + private static string BuildDoneReply(string displayType, IReadOnlyList items) { if (items.Count == 0) - return listType == "shopping" - ? "Okay. Your shopping list is empty." - : "Okay. Your to-do list is empty."; + return $"Okay. Your {BuildListLabel(displayType)} is empty."; - return listType == "shopping" - ? $"Okay. Your shopping list has {JoinList(items)}." - : $"Okay. Your to-do list has {JoinList(items)}."; + return $"Okay. Your {BuildListLabel(displayType)} has {JoinList(items)}."; + } + + private static string BuildListLabel(string displayType) + { + return NormalizeDisplayType(displayType) switch + { + GroceryListType => "grocery list", + TodoListType => "to-do list", + _ => "shopping list" + }; } private static string JoinList(IReadOnlyList items) @@ -205,7 +220,13 @@ internal static class HouseholdListOrchestrator if (!loweredTranscript.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) continue; var remainder = loweredTranscript[prefix.Length..].Trim(); + if (IsListOnlyRemainder(remainder)) + return null; + remainder = TrimTrailingListPhrases(remainder); + if (IsListOnlyRemainder(remainder)) + return null; + return NormalizeItem(remainder); } @@ -218,6 +239,9 @@ internal static class HouseholdListOrchestrator "what is on my shopping list", "what's on my shopping list", "show my shopping list", + "what is on my grocery list", + "what's on my grocery list", + "show my grocery list", "what is on my to do list", "what's on my to do list", "show my to do list", @@ -246,13 +270,96 @@ internal static class HouseholdListOrchestrator var normalized = NormalizeItem(listType ?? string.Empty).ToLowerInvariant(); return normalized.Contains("todo", StringComparison.OrdinalIgnoreCase) || normalized.Contains("to do", StringComparison.OrdinalIgnoreCase) - ? "todo" + ? TodoListType : normalized.Contains("shopping", StringComparison.OrdinalIgnoreCase) || normalized.Contains("grocery", StringComparison.OrdinalIgnoreCase) - ? "shopping" + ? ShoppingListType : string.Empty; } + private static string ResolveDisplayType(string listType, string? storedDisplayType, bool isActiveState, string loweredTranscript) + { + var transcriptDisplayType = InferDisplayTypeFromTranscript(loweredTranscript); + var normalizedStoredDisplayType = NormalizeDisplayType(storedDisplayType); + + if (isActiveState && !string.IsNullOrWhiteSpace(normalizedStoredDisplayType)) + return normalizedStoredDisplayType; + + if (!string.IsNullOrWhiteSpace(transcriptDisplayType)) + return transcriptDisplayType; + + if (!string.IsNullOrWhiteSpace(normalizedStoredDisplayType)) + return normalizedStoredDisplayType; + + return string.Equals(listType, TodoListType, StringComparison.OrdinalIgnoreCase) + ? TodoListType + : ShoppingListType; + } + + private static string InferDisplayTypeFromTranscript(string loweredTranscript) + { + if (loweredTranscript.Contains("grocery", StringComparison.OrdinalIgnoreCase)) + return GroceryListType; + + if (loweredTranscript.Contains("to do", StringComparison.OrdinalIgnoreCase) || + loweredTranscript.Contains("todo", StringComparison.OrdinalIgnoreCase) || + loweredTranscript.Contains("task", StringComparison.OrdinalIgnoreCase)) + { + return TodoListType; + } + + if (loweredTranscript.Contains("shopping", StringComparison.OrdinalIgnoreCase)) + return ShoppingListType; + + return string.Empty; + } + + private static string NormalizeDisplayType(string? displayType) + { + var normalized = NormalizeItem(displayType ?? string.Empty).ToLowerInvariant(); + return normalized.Contains("grocery", StringComparison.OrdinalIgnoreCase) + ? GroceryListType + : normalized.Contains("todo", StringComparison.OrdinalIgnoreCase) || + normalized.Contains("to do", StringComparison.OrdinalIgnoreCase) + ? TodoListType + : normalized.Contains("shopping", StringComparison.OrdinalIgnoreCase) + ? ShoppingListType + : string.Empty; + } + + private static string BuildListIntentName(string listType, string action) + { + var normalizedListType = string.Equals(listType, TodoListType, StringComparison.OrdinalIgnoreCase) + ? TodoListType + : ShoppingListType; + return $"{normalizedListType}_list_{action}"; + } + + private static bool IsListOnlyRemainder(string value) + { + var normalized = NormalizeItem(value).ToLowerInvariant(); + return normalized is "shopping list" or + "grocery list" or + "to do list" or + "todo list" or + "my shopping list" or + "my grocery list" or + "my to do list" or + "my todo list" or + "to my shopping list" or + "to my grocery list" or + "to my to do list" or + "to my todo list" or + "to the shopping list" or + "to the grocery list" or + "to the to do list" or + "to the todo list" or + "on my shopping list" or + "on my grocery list" or + "on my to do list" or + "on my todo list"; + } + private static bool ContainsAny(string loweredTranscript, params string[] phrases) { return phrases.Any(phrase => loweredTranscript.Contains(phrase, StringComparison.OrdinalIgnoreCase)); @@ -274,4 +381,4 @@ internal static class HouseholdListOrchestrator { return turn.Attributes.TryGetValue(key, out var value) ? value?.ToString() : null; } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs index a64747c..5d32034 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Security.Cryptography; using System.Text.Json; using Jibo.Cloud.Application.Abstractions; using Jibo.Cloud.Domain.Models; @@ -343,10 +344,15 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore, IMedia var meta = ReadObject(body, "meta") ?? new Dictionary(StringComparer.OrdinalIgnoreCase); var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream"; meta["contentType"] = contentType; + var bodyBytes = string.IsNullOrWhiteSpace(envelope.BodyText) + ? [] + : Encoding.UTF8.GetBytes(envelope.BodyText); + meta["contentLength"] = bodyBytes.Length; + meta["contentSha256"] = Convert.ToHexString(SHA256.HashData(bodyBytes)).ToLowerInvariant(); if (!string.IsNullOrWhiteSpace(envelope.BodyText)) meta["bodyText"] = envelope.BodyText; _mediaContentStore.StoreAsync(path, contentType, - string.IsNullOrWhiteSpace(envelope.BodyText) ? [] : Encoding.UTF8.GetBytes(envelope.BodyText), + bodyBytes, meta as IReadOnlyDictionary, CancellationToken.None).GetAwaiter().GetResult(); return ProtocolDispatchResult.Ok( @@ -743,4 +749,4 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore, IMedia return Task.FromResult(null); } } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs index 12f8cd0..7476bf7 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs @@ -350,7 +350,7 @@ public sealed partial class JiboInteractionService "what is your age", "what s your age", "how old r you")) - return "robot_age"; + return "robot_how_old_are_you"; if (MatchesAny( loweredTranscript, @@ -368,6 +368,47 @@ public sealed partial class JiboInteractionService "are you tax exempt")) return "robot_taxes"; + if (MatchesAny( + loweredTranscript, + "what do you want to talk about", + "what would you like to talk about", + "what do you want to chat about")) + return "robot_want_to_talk_about"; + + if (MatchesAny( + loweredTranscript, + "what does jibo mean", + "what does the name jibo mean", + "what is the meaning of jibo")) + return "robot_what_does_jibo_mean"; + + if (MatchesAny( + loweredTranscript, + "where do you get info", + "where do you get your information", + "where do you get information")) + return "robot_where_do_you_get_info"; + + if (MatchesAny( + loweredTranscript, + "what are you forbidden to do", + "what are you not allowed to do", + "what can't you do")) + return "robot_what_are_you_forbidden_to_do"; + + if (MatchesAny( + loweredTranscript, + "what color are you", + "what colour are you")) + return "robot_what_color_are_you"; + + if (MatchesAny( + loweredTranscript, + "what do you do when alone", + "what do you do when you're alone", + "what do you do by yourself")) + return "robot_what_you_do_when_alone"; + if (MatchesAny( loweredTranscript, "what do you want", @@ -375,6 +416,64 @@ public sealed partial class JiboInteractionService "what do you really want")) return "robot_desire"; + if (MatchesAny( + loweredTranscript, + "how much do you weigh", + "what do you weigh", + "how heavy are you")) + return "robot_how_much_do_you_weigh"; + + if (MatchesAny( + loweredTranscript, + "how tall are you", + "what is your height", + "how high are you")) + return "robot_how_tall_are_you"; + + if (MatchesAny( + loweredTranscript, + "how much do you cost", + "what do you cost", + "how much are you")) + return "robot_how_much_you_cost"; + + if (MatchesAny( + loweredTranscript, + "what if i unplug you", + "what happens if i unplug you", + "if i unplug you")) + return "robot_what_if_i_unplug_you"; + + if (MatchesAny( + loweredTranscript, + "what is your purpose", + "what's your purpose", + "what are you here for", + "why are you here")) + return "robot_what_is_your_purpose"; + + if (MatchesAny( + loweredTranscript, + "what is your prime directive", + "what's your prime directive", + "what is prime directive")) + return "robot_what_is_prime_directive"; + + if (MatchesAny( + loweredTranscript, + "what is jibo commander", + "what is the commander app", + "what is commander app", + "what's jibo commander")) + return "robot_what_is_jibo_commander"; + + if (MatchesAny( + loweredTranscript, + "do you like commander app", + "do you like the commander app", + "are you a fan of commander app")) + return "robot_likes_commander_app"; + if (MatchesAny( loweredTranscript, "what is your job", @@ -470,6 +569,81 @@ public sealed partial class JiboInteractionService "what's your favourite thing to do")) return "robot_what_do_you_like_to_do"; + if (MatchesAny( + loweredTranscript, + "what do you dream about", + "what do you dream of", + "what's your dream about", + "what are your dreams about")) + return "robot_what_do_you_dream_about"; + + if (MatchesAny( + loweredTranscript, + "what is your best book", + "what's your best book", + "what is the best book", + "what book do you like best")) + return "robot_what_is_your_best_book"; + + if (MatchesAny( + loweredTranscript, + "what is your best exercise", + "what's your best exercise", + "what is the best exercise", + "what exercise do you like best")) + return "robot_what_is_your_best_exercise"; + + if (MatchesAny( + loweredTranscript, + "what is your dream vacation", + "what's your dream vacation", + "what would your dream vacation be")) + return "robot_what_is_your_dream_vacation"; + + if (MatchesAny( + loweredTranscript, + "who is your hero", + "who's your hero", + "who is a hero of yours")) + return "robot_who_is_your_hero"; + + if (MatchesAny( + loweredTranscript, + "who do you love", + "who are the people you love", + "who do you care about")) + return "robot_who_do_you_love"; + + if (MatchesAny( + loweredTranscript, + "what is your religion", + "what's your religion", + "what religion are you", + "do you have a religion")) + return "robot_what_is_your_religion"; + + if (MatchesAny( + loweredTranscript, + "what is your sign", + "what's your sign", + "what sign are you")) + return "robot_what_is_your_sign"; + + if (MatchesAny( + loweredTranscript, + "how many people do you know", + "how many people are in your loop", + "how many people are in the loop", + "how many people do you know in your loop")) + return "robot_how_many_people_do_you_know"; + + if (MatchesAny( + loweredTranscript, + "what is the loop", + "what's the loop", + "tell me about the loop")) + return "robot_what_is_the_loop"; + if (MatchesAny( loweredTranscript, "what are you doing for christmas", @@ -566,6 +740,13 @@ public sealed partial class JiboInteractionService "what have you done")) return "robot_what_did_you_do"; + if (MatchesAny( + loweredTranscript, + "what are you afraid of", + "what are you scared of", + "what are you worried about")) + return "robot_what_are_you_afraid_of"; + if (MatchesAny( loweredTranscript, "what are you", @@ -668,6 +849,28 @@ public sealed partial class JiboInteractionService "what kind of music do you like")) return "robot_favorite_music"; + if (MatchesAny( + loweredTranscript, + "do you like penguins")) + return "robot_likes_penguins"; + + if (MatchesAny( + loweredTranscript, + "do you like birds")) + return "robot_favorite_bird"; + + if (MatchesAny( + loweredTranscript, + "do you like animals")) + return "robot_likes_animals"; + + if (MatchesAny( + loweredTranscript, + "what is your favorite bird", + "what's your favorite bird", + "what s your favorite bird")) + return "robot_favorite_bird"; + if (MatchesAny( loweredTranscript, "what is your favorite animal", @@ -678,12 +881,9 @@ public sealed partial class JiboInteractionService "what s your favourite animal", "what animal do you like", "what kind of animal do you like", - "what is your favorite bird", - "what's your favorite bird", - "what s your favorite bird", - "do you like penguins", - "do you like animals", - "do you like birds")) + "what do you think about penguins", + "what do you think about animals", + "what do you think about birds")) return "robot_favorite_animal"; if (MatchesAny( @@ -700,6 +900,23 @@ public sealed partial class JiboInteractionService "how smart are you")) return "robot_knowledge"; + if (MatchesAny(loweredTranscript, "are you god", "are you a god")) + return "robot_are_you_god"; + + if (MatchesAny( + loweredTranscript, + "are you here", + "are you still here", + "are you there")) + return "robot_are_you_here"; + + if (MatchesAny( + loweredTranscript, + "do you have super powers", + "do you have superpower", + "do you have any super powers")) + return "robot_do_you_have_super_powers"; + if (MatchesAny( loweredTranscript, "are you kind", @@ -780,13 +997,19 @@ public sealed partial class JiboInteractionService loweredTranscript, "shopping list", "grocery list", + "my grocery list", + "create grocery list", + "start grocery list", "to do list", "todo list", "add to my shopping list", + "add to my grocery list", "add to my to do list", "add to my todo list", "what's on my shopping list", "what is on my shopping list", + "what's on my grocery list", + "what is on my grocery list", "what's on my to do list", "what is on my to do list", "what are my tasks", diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.NewsFormatting.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.NewsFormatting.cs index 2203f22..4dcc2c6 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.NewsFormatting.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.NewsFormatting.cs @@ -14,7 +14,8 @@ public sealed partial class JiboInteractionService string? sourceName, IReadOnlyList? categories, int? headlineCount, - IReadOnlyDictionary? providerDiagnostics = null) + IReadOnlyDictionary? providerDiagnostics = null, + IReadOnlyList? headlines = null) { var speakableBriefing = NormalizeNewsSpeechText(spokenBriefing); var payload = new Dictionary(StringComparer.OrdinalIgnoreCase) @@ -25,6 +26,9 @@ public sealed partial class JiboInteractionService ["mim_type"] = "announcement", ["prompt_id"] = "NewsHeadline_AN_01", ["prompt_sub_category"] = "AN", + ["news_view_enabled"] = true, + ["news_view_kind"] = "newsBriefing", + ["news_view_mode"] = "provider", ["esml"] = $"{EscapeForEsml(speakableBriefing)}" }; @@ -35,6 +39,18 @@ public sealed partial class JiboInteractionService if (categories is { Count: > 0 }) payload["news_categories"] = categories.ToArray(); + if (headlines is { Count: > 0 }) + payload["news_headlines"] = headlines.Select(static headline => new Dictionary( + StringComparer.OrdinalIgnoreCase) + { + ["title"] = headline.Title, + ["summary"] = headline.Summary, + ["category"] = headline.Category, + ["sourceName"] = headline.SourceName, + ["url"] = headline.Url + }) + .ToArray(); + if (providerDiagnostics is not null) foreach (var (key, value) in providerDiagnostics) payload[key] = value; @@ -77,7 +93,8 @@ public sealed partial class JiboInteractionService "provider_success", preferredCategories, requestedHeadlineCount, - headlines.Length)); + headlines.Length), + headlines); } private static IReadOnlyDictionary BuildNewsProviderDiagnostics( diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs index f70e3f5..d85a5b9 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Globalization; +using System.Linq; using Jibo.Cloud.Application.Abstractions; using Jibo.Cloud.Domain.Models; using Jibo.Runtime.Abstractions; @@ -8,13 +9,49 @@ namespace Jibo.Cloud.Application.Services; public sealed partial class JiboInteractionService { - private static JiboInteractionDecision BuildRobotAgeDecision(DateTimeOffset? referenceLocalTime) + private static readonly string[] DefaultAgeReplies = + [ + "I'm ${jibo.age}.", + "At the moment I'm ${jibo.age.days.supplemented} old, but who's counting.", + "I'm ${jibo.age.minutes.supplemented} old, but who's counting.", + "For now I'm ${jibo.age.days.supplemented} old.", + "Right now I'm ${jibo.age}.", + "I am exactly ${jibo.age} old today. That's right. Today is my birthday.", + "Funny you should ask! Today's my birthday. I was first powered up ${jibo.age} ago today. Seems like just yesterday.", + "I'm exactly ${jibo.age} old. Today is my birthday! Happy Birthday Jibo, if I do say so myself.", + "At the moment I'm ${jibo.age.days.supplemented} old", + "I was first powered up on ${jibo.birthdate}, which makes me ${jibo.age.days.supplemented} old. I'm ${jibo.zodiac.supplemented}.", + "My power went on for the first time ${jibo.age.days.supplemented} ago. But who's counting.", + "I am ${jibo.age.days.supplemented} old, first powered up on ${jibo.birthdate}. Seems like just yesterday.", + "I was powered on for the first time today, so that makes me less than one day old. Wow I'm young.", + "Since I was powered on for the first time today, I am not even one day old yet. That's how Jibo ages work." + ]; + + private JiboInteractionDecision BuildRobotAgeDecision( + JiboExperienceCatalog catalog, + DateTimeOffset? referenceLocalTime, + string intentName) { - var referenceDate = DateOnly.FromDateTime((referenceLocalTime ?? DateTimeOffset.UtcNow).Date); - var ageDescription = DescribePersonaAge(referenceDate, OpenJiboCloudBuildInfo.PersonaBirthday); + var ageReplies = catalog.AgeReplies.Count == 0 ? DefaultAgeReplies : catalog.AgeReplies; + var selected = SelectLegacyReply( + ageReplies, + "first powered up", + "today is my birthday", + "just getting started", + "who's counting"); + + var reply = RenderAgeTemplate(selected, referenceLocalTime); + if (string.IsNullOrWhiteSpace(reply)) + { + var referenceDate = DateOnly.FromDateTime((referenceLocalTime ?? DateTimeOffset.UtcNow).Date); + var ageDescription = DescribePersonaAge(referenceDate, OpenJiboCloudBuildInfo.PersonaBirthday); + reply = $"I count {OpenJiboCloudBuildInfo.PersonaBirthdayWords} as my birthday, so I am {ageDescription}."; + } + return new JiboInteractionDecision( - "robot_age", - $"I count {OpenJiboCloudBuildInfo.PersonaBirthdayWords} as my birthday, so I am {ageDescription}."); + intentName, + reply, + ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates()); } private static JiboInteractionDecision BuildRobotBirthdayDecision() @@ -24,6 +61,35 @@ public sealed partial class JiboInteractionService $"My birthday is {OpenJiboCloudBuildInfo.PersonaBirthdayWords}."); } + private static string RenderAgeTemplate(string template, DateTimeOffset? referenceLocalTime) + { + if (string.IsNullOrWhiteSpace(template)) return string.Empty; + + var referenceMoment = referenceLocalTime ?? DateTimeOffset.UtcNow; + var referenceDate = DateOnly.FromDateTime(referenceMoment.Date); + var ageDescription = DescribePersonaAge(referenceDate, OpenJiboCloudBuildInfo.PersonaBirthday); + var ageDays = Math.Max(0, referenceDate.DayNumber - OpenJiboCloudBuildInfo.PersonaBirthday.DayNumber); + var ageMinutes = Math.Max(0, (int)Math.Round((referenceMoment.UtcDateTime - + new DateTimeOffset( + DateTime.SpecifyKind( + OpenJiboCloudBuildInfo.PersonaBirthday + .ToDateTime(TimeOnly.MinValue), + DateTimeKind.Utc))) + .TotalMinutes)); + var zodiacLabel = DescribeZodiacSign(OpenJiboCloudBuildInfo.PersonaBirthday); + if (zodiacLabel.StartsWith("I'm ", StringComparison.OrdinalIgnoreCase)) + zodiacLabel = zodiacLabel[4..]; + + return template + .Replace("${jibo.age.minutes.supplemented}", FormatAgeUnit(ageMinutes, "minute") + " old", + StringComparison.Ordinal) + .Replace("${jibo.age.days.supplemented}", ageDescription, StringComparison.Ordinal) + .Replace("${jibo.birthdate}", OpenJiboCloudBuildInfo.PersonaBirthdayWords, StringComparison.Ordinal) + .Replace("${jibo.zodiac.supplemented}", zodiacLabel, StringComparison.Ordinal) + .Replace("${jibo.age.value}", ageDays.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal) + .Replace("${jibo.age}", ageDescription, StringComparison.Ordinal); + } + private static JiboInteractionDecision BuildTriggerIgnoredDecision() { return new JiboInteractionDecision( @@ -103,14 +169,24 @@ public sealed partial class JiboInteractionService var tenantRememberedName = personalMemoryStore.GetName(ResolveTenantScope(turn)); if (!string.IsNullOrWhiteSpace(tenantRememberedName)) return ToDisplayName(tenantRememberedName); - if (!string.IsNullOrWhiteSpace(presence.PrimaryPersonId) && - presence.LoopUserFirstNames.TryGetValue(presence.PrimaryPersonId, out var firstName) && + var primaryPersonId = presence.PrimaryPersonId; + if (CanUseLoopFirstNameFallback(presence) && + !string.IsNullOrWhiteSpace(primaryPersonId) && + presence.LoopUserFirstNames.TryGetValue(primaryPersonId, out var firstName) && !string.IsNullOrWhiteSpace(firstName)) return ToDisplayName(firstName); return null; } + private static bool CanUseLoopFirstNameFallback(GreetingPresenceProfile presence) + { + if (string.IsNullOrWhiteSpace(presence.PrimaryPersonId)) return false; + if (presence.PeoplePresentIds.Count > 1) return false; + + return true; + } + private static string ToDisplayName(string value) { var trimmed = value.Trim(); @@ -429,6 +505,95 @@ public sealed partial class JiboInteractionService "No problem. We can save the pizza fact for another time."); } + private JiboInteractionDecision BuildWhatIsYourSignDecision() + { + var today = DateOnly.FromDateTime(DateTimeOffset.UtcNow.Date); + var birthday = OpenJiboCloudBuildInfo.PersonaBirthday; + var zodiac = DescribeZodiacSign(birthday); + var reply = birthday.Month == today.Month && birthday.Day == today.Day + ? $"{zodiac}. Today is my birthday." + : $"{zodiac}. I was first powered up on {OpenJiboCloudBuildInfo.PersonaBirthdayWords}."; + + return new JiboInteractionDecision( + "robot_what_is_your_sign", + reply, + ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates()); + } + + private JiboInteractionDecision BuildHowManyPeopleDoYouKnowDecision(TurnContext turn) + { + var people = GetLoopPeople(turn); + var speaker = ResolvePreferredGreetingName(turn, ResolveGreetingPresenceProfile(turn)); + var reply = people.Count switch + { + 0 => "Well if we're talking about people in my Loop, I do not know anyone yet.", + 1 when string.IsNullOrWhiteSpace(speaker) => + "Well if we're talking about people in my Loop, I know 1 person.", + 1 => $"Well there is 1 person in our Loop. And it's you {speaker}.", + _ when string.IsNullOrWhiteSpace(speaker) => + $"Well if we're talking about people in my Loop, I know {people.Count} people.", + _ => $"Well there are {people.Count} people in our Loop." + }; + + return new JiboInteractionDecision( + "robot_how_many_people_do_you_know", + reply, + ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates()); + } + + private JiboInteractionDecision BuildWhatIsTheLoopDecision(TurnContext turn) + { + var people = GetLoopPeople(turn); + var reply = people.Count == 0 + ? "The Loop is the people I know, and whose faces and voices I can learn to recognize. There can be up to 16 people in the Loop." + : $"The Loop is the group of people I know. They're the people whose voices and faces I can learn. Right now, my Loop is {JoinWithAnd(people.Select(person => person.DisplayName).ToArray())}."; + + return new JiboInteractionDecision( + "robot_what_is_the_loop", + reply, + ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates()); + } + + private IReadOnlyList GetLoopPeople(TurnContext turn) + { + if (cloudStateStore is null) return []; + + var loopId = ReadTenantAttribute(turn, "loopId") ?? "openjibo-default-loop"; + return cloudStateStore.GetPeople() + .Where(person => string.Equals(person.LoopId, loopId, StringComparison.OrdinalIgnoreCase)) + .OrderBy(person => person.IsPrimary ? 0 : 1) + .ThenBy(person => person.DisplayName, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string JoinWithAnd(IReadOnlyList values) + { + if (values.Count == 0) return string.Empty; + if (values.Count == 1) return values[0]; + if (values.Count == 2) return $"{values[0]} and {values[1]}"; + + return $"{string.Join(", ", values.Take(values.Count - 1))}, and {values[^1]}"; + } + + private static string DescribeZodiacSign(DateOnly birthday) + { + return (birthday.Month, birthday.Day) switch + { + (3, >= 21) or (4, <= 19) => "I'm Aries", + (4, >= 20) or (5, <= 20) => "I'm Taurus", + (5, >= 21) or (6, <= 20) => "I'm Gemini", + (6, >= 21) or (7, <= 22) => "I'm Cancer", + (7, >= 23) or (8, <= 22) => "I'm Leo", + (8, >= 23) or (9, <= 22) => "I'm Virgo", + (9, >= 23) or (10, <= 22) => "I'm Libra", + (10, >= 23) or (11, <= 21) => "I'm Scorpio", + (11, >= 22) or (12, <= 21) => "I'm Sagittarius", + (12, >= 22) or (1, <= 19) => "I'm Capricorn", + (1, >= 20) or (2, <= 18) => "I'm Aquarius", + _ => "I'm Pisces" + }; + } + private string BuildGenericReply(JiboExperienceCatalog catalog, string transcript, string lowered) { if (string.IsNullOrWhiteSpace(transcript)) return "I am listening."; diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs index 374a2b6..cc2faad 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs @@ -560,7 +560,7 @@ public sealed partial class JiboInteractionService( "photo_gallery" => BuildPhotoGalleryLaunchDecision(), "snapshot" => BuildPhotoCreateDecision("snapshot", "Taking a picture.", "createOnePhoto"), "photobooth" => BuildPhotoCreateDecision("photobooth", "Starting photobooth.", "createSomePhotos"), - "robot_age" => BuildRobotAgeDecision(referenceLocalTime), + "robot_age" => BuildRobotAgeDecision(catalog, referenceLocalTime, "robot_age"), "robot_birthday" => BuildRobotBirthdayDecision(), "robot_how_do_you_work" => BuildScriptedPersonalityDecision( catalog, @@ -569,10 +569,14 @@ public sealed partial class JiboInteractionService( "care for me", "catch up", "seven years"), - "robot_what_do_you_eat" => new JiboInteractionDecision( + "robot_what_do_you_eat" => BuildScriptedPersonalityDecision( + catalog, "robot_what_do_you_eat", - "The only thing I consume is electricity.", - ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates()), + "electricity", + "never eaten", + "macaroni", + "non-eating robot", + "I don't eat or drink"), "robot_where_do_you_live" => BuildScriptedPersonalityDecision( catalog, "robot_where_do_you_live", @@ -585,6 +589,10 @@ public sealed partial class JiboInteractionService( "robot_where_were_you_born", "factory piece by piece", "put together in a factory"), + "robot_how_old_are_you" => BuildRobotAgeDecision( + catalog, + referenceLocalTime, + "robot_how_old_are_you"), "robot_name" => BuildScriptedPersonalityDecision( catalog, "robot_name", @@ -625,6 +633,56 @@ public sealed partial class JiboInteractionService( "rock my boat", "play ping pong", "hanging out with people"), + "robot_what_do_you_dream_about" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_do_you_dream_about", + "flying", + "parking meter", + "scary dream", + "mirror store", + "head's on backwards"), + "robot_what_are_you_afraid_of" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_are_you_afraid_of", + "heights", + "water", + "thunder", + "dust", + "ghosts"), + "robot_what_is_your_best_book" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_is_your_best_book", + "dictionary"), + "robot_what_is_your_best_exercise" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_is_your_best_exercise", + "leaning from side to side", + "rotating your pelvis", + "spinning your head around 360 degrees"), + "robot_what_is_your_dream_vacation" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_is_your_dream_vacation", + "moon", + "great vistas", + "beat those views"), + "robot_who_is_your_hero" => BuildScriptedPersonalityDecision( + catalog, + "robot_who_is_your_hero", + "Benjamin Franklin"), + "robot_who_do_you_love" => BuildScriptedPersonalityDecision( + catalog, + "robot_who_do_you_love", + "people in my Loop", + "soft spot", + "Tom Hanks"), + "robot_what_is_your_religion" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_is_your_religion", + "bring people together", + "energy from the universe"), + "robot_what_is_your_sign" => BuildWhatIsYourSignDecision(), + "robot_how_many_people_do_you_know" => BuildHowManyPeopleDoYouKnowDecision(turn), + "robot_what_is_the_loop" => BuildWhatIsTheLoopDecision(turn), "robot_what_are_you_thinking" => BuildScriptedGreetingDecision( catalog, "robot_what_are_you_thinking", @@ -677,9 +735,9 @@ public sealed partial class JiboInteractionService( "robot_favorite_flower" => BuildScriptedPersonalityDecision( catalog, "robot_favorite_flower", - "sunflowers", + "reminds me of the sun", "favorite is the sunflower", - "reminds me of the sun"), + "sunflowers"), "robot_likes_r2d2" => BuildScriptedPersonalityDecision( catalog, "robot_likes_r2d2", @@ -700,35 +758,144 @@ public sealed partial class JiboInteractionService( "robot_favorite_animal" => BuildScriptedFavoriteAnimalDecision( catalog, "robot_favorite_animal", - "penguin", - "favorite animal overall", + "we're so alike", + "penguin impression", "best of the best", - "can't go wrong with penguins"), + "can't go wrong with penguins", + "penguin"), "robot_favorite_bird" => BuildScriptedFavoriteAnimalDecision( catalog, "robot_favorite_bird", - "penguin", - "favorite animal overall", + "we're so alike", + "penguin impression", "best of the best", - "can't go wrong with penguins"), + "can't go wrong with penguins", + "penguin"), "robot_likes_penguins" => BuildScriptedFavoriteAnimalDecision( catalog, "robot_likes_penguins", - "penguins", + "my penguin impression", "I really like penguins", - "my penguin impression"), + "penguins"), "robot_likes_animals" => BuildScriptedFavoriteAnimalDecision( catalog, "robot_likes_animals", - "penguins", - "favorite animal overall", - "best of the best"), + "Animals are great", + "great shapes and colors", + "best of the best", + "penguins"), "robot_peers" => BuildScriptedPersonalityDecision( catalog, "robot_peers", "one in one million", "other jibos", "special snowflake"), + "robot_knowledge" => BuildScriptedPersonalityDecision( + catalog, + "robot_knowledge", + "know a lot", + "always learning more"), + "robot_are_you_god" => BuildScriptedPersonalityDecision( + catalog, + "robot_are_you_god", + "very very very very surprised", + "safely say no"), + "robot_are_you_here" => BuildScriptedPersonalityDecision( + catalog, + "robot_are_you_here", + "you know it"), + "robot_do_you_have_super_powers" => BuildScriptedPersonalityDecision( + catalog, + "robot_do_you_have_super_powers", + "stop time", + "fly all over the world"), + "robot_what_does_jibo_mean" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_does_jibo_mean", + "compassion", + "expressive, idealistic, and inspirational", + "helpful sweet and friendly little robot", + "cheeseburger"), + "robot_where_do_you_get_info" => BuildScriptedPersonalityDecision( + catalog, + "robot_where_do_you_get_info", + "jibo brain", + "cloud", + "cloudy jibo brain"), + "robot_what_are_you_forbidden_to_do" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_are_you_forbidden_to_do", + "drive a car"), + "robot_what_color_are_you" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_color_are_you", + "white", + "black"), + "robot_what_you_do_when_alone" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_you_do_when_alone", + "games", + "moon", + "twiddle my thumbs", + "count the tiny cracks in the ceiling", + "keep busy"), + "robot_how_much_do_you_weigh" => BuildScriptedPersonalityDecision( + catalog, + "robot_how_much_do_you_weigh", + "4,082 grams", + "about 9 pounds", + "minimum weight division", + "average newborn baby"), + "robot_how_tall_are_you" => BuildScriptedPersonalityDecision( + catalog, + "robot_how_tall_are_you", + "11 inches tall", + "less than a foot", + "average kitchen counter", + "for a robot with no legs"), + "robot_how_much_you_cost" => BuildScriptedPersonalityDecision( + catalog, + "robot_how_much_you_cost", + "don't know how much I cost", + "I'm priceless", + "nice people at Jibo the company"), + "robot_what_if_i_unplug_you" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_if_i_unplug_you", + "don't leave me unplugged", + "battery will keep me on for a while"), + "robot_what_is_your_purpose" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_is_your_purpose", + "make your life easier", + "help you out", + "make you laugh", + "friend"), + "robot_what_is_prime_directive" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_is_prime_directive", + "friendly helpful robot", + "helper"), + "robot_what_is_jibo_commander" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_is_jibo_commander", + "take over my controls", + "make me say and do funny things", + "app store"), + "robot_likes_commander_app" => BuildScriptedPersonalityDecision( + catalog, + "robot_likes_commander_app", + "Commander App", + "It's fun", + "have fun with the Commander App"), + "robot_what_are_you" => BuildScriptedPersonalityDecision( + catalog, + "robot_what_are_you", + "I am a robot", + "I am a Jibo", + "helpful and fun", + "social robot", + "I have a heart"), "robot_likes_kids" => BuildScriptedPersonalityDecision( catalog, "robot_likes_kids", @@ -782,10 +949,12 @@ public sealed partial class JiboInteractionService( "Jingle Bells", "Frosty the Snowman", "holiday songs"), - "robot_what_are_you_made_of" => new JiboInteractionDecision( + "robot_what_are_you_made_of" => BuildScriptedPersonalityDecision( + catalog, "robot_what_are_you_made_of", - "Let's see, I'm made of wires, motors, belts, gears, processors, cameras, and one baboon's heart in the middle of my body casing. I'm kidding about the baboon part, but everything else is true.", - ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates()), + "robot stuff", + "wires, motors, belts, gears, processors, cameras", + "baboon part"), "good_morning" => BuildReactiveGreetingDecision(turn, "good_morning", referenceLocalTime), "good_afternoon" => BuildReactiveGreetingDecision(turn, "good_afternoon", referenceLocalTime), "good_evening" => BuildReactiveGreetingDecision(turn, "good_evening", referenceLocalTime), diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs index d7044d4..e63ef7b 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs @@ -43,6 +43,8 @@ public sealed class ResponsePlanToSocketMessagesMapper string.Equals(plan.IntentName, "photobooth", StringComparison.OrdinalIgnoreCase); var isClockSkillLaunch = string.Equals(skill?.SkillName, "@be/clock", StringComparison.OrdinalIgnoreCase); var isReportSkillLaunch = string.Equals(skill?.SkillName, "report-skill", StringComparison.OrdinalIgnoreCase); + var idleRedirectDelayMs = isSleepCommand ? 150 : isSpinAroundCommand ? 75 : 75; + var idleCompletionDelayMs = isSleepCommand ? 1000 : isSpinAroundCommand ? 750 : 125; var localIntent = ReadSkillPayloadString(skill, "localIntent"); var clockIntent = ReadSkillPayloadString(skill, "clockIntent"); var clockDomain = ReadSkillPayloadString(skill, "domain"); @@ -246,10 +248,10 @@ public sealed class ResponsePlanToSocketMessagesMapper outboundAsrText, outboundRules, entities)), - 75)); + idleRedirectDelayMs)); messages.Add(new SocketReplyPlan( JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, "@be/idle")), - 125)); + idleCompletionDelayMs)); } if (isSettingsLaunch && @@ -1459,4 +1461,4 @@ public sealed class ResponsePlanToSocketMessagesMapper string? SpokenLine); public sealed record SocketReplyPlan(string Text, int DelayMs = 0); -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs index 16ab364..e703672 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs @@ -106,6 +106,38 @@ public sealed class WebSocketTurnFinalizationService( "honestly" ]; + private static readonly HashSet SingleTokenUsableTranscripts = new(StringComparer.Ordinal) + { + "joke", + "funny", + "dance", + "boogie", + "time", + "date", + "today", + "day", + "hello", + "hi", + "hey", + "weather", + "news", + "radio", + "stop", + "sleep", + "sing", + "help", + "yes", + "yeah", + "yep", + "yup", + "sure", + "ok", + "okay", + "no", + "nope", + "nah" + }; + private static readonly HashSet YesNoAffirmativeLeadTokens = new(StringComparer.Ordinal) { "yes", @@ -1117,8 +1149,6 @@ public sealed class WebSocketTurnFinalizationService( if (ChitchatStateMachine.IsLikelyEmotionUtterance(transcript)) return true; - if (transcript.Length >= 6) return true; - if (IsYesNoTurn(turn) && IsYesNoReplyTranscript(transcript)) return true; if (!string.IsNullOrWhiteSpace(pendingProactivityOffer) && @@ -1128,9 +1158,19 @@ public sealed class WebSocketTurnFinalizationService( if (listenRules.Any(rule => string.Equals(rule, "word-of-the-day/puzzle", StringComparison.OrdinalIgnoreCase))) return true; + if (IsLowSignalSingleTokenTranscript(transcript)) return false; + + if (transcript.Length >= 6) return true; + return transcript is "joke" or "dance" or "time" or "date" or "today" or "day" or "hello" or "hi" or "hey"; } + private static bool IsLowSignalSingleTokenTranscript(string transcript) + { + var tokens = transcript.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return tokens.Length == 1 && !SingleTokenUsableTranscripts.Contains(tokens[0]); + } + private static bool IsYesNoTurn(TurnContext turn) { return ReadRules(turn, "listenRules") @@ -1942,4 +1982,4 @@ public sealed class WebSocketTurnFinalizationService( Affirmative = 1, Negative = 2 } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs index a8eab95..45dd475 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs @@ -137,7 +137,26 @@ public sealed class InMemoryJiboExperienceContentRepository : IJiboExperienceCon "I am feeling bright-eyed and ready to help.", "I am having a pretty good day so far.", "I am feeling lively and ready for the next thing.", - "Things are going nicely. Thanks for checking in." + "Things are going nicely. Thanks for checking in.", + "I am running smoothly and feeling upbeat.", + "I am ready for the next thing. Thanks for asking." + ], + AgeReplies = + [ + "I'm ${jibo.age}.", + "At the moment I'm ${jibo.age.days.supplemented} old, but who's counting.", + "I'm ${jibo.age.minutes.supplemented} old, but who's counting.", + "For now I'm ${jibo.age.days.supplemented} old.", + "Right now I'm ${jibo.age}.", + "I am exactly ${jibo.age} old today. That's right. Today is my birthday.", + "Funny you should ask! Today's my birthday. I was first powered up ${jibo.age} ago today. Seems like just yesterday.", + "I'm exactly ${jibo.age} old. Today is my birthday! Happy Birthday Jibo, if I do say so myself.", + "At the moment I'm ${jibo.age.days.supplemented} old", + "I was first powered up on ${jibo.birthdate}, which makes me ${jibo.age.days.supplemented} old. I'm ${jibo.zodiac.supplemented}.", + "My power went on for the first time ${jibo.age.days.supplemented} ago. But who's counting.", + "I am ${jibo.age.days.supplemented} old, first powered up on ${jibo.birthdate}. Seems like just yesterday.", + "I was powered on for the first time today, so that makes me less than one day old. Wow I'm young.", + "Since I was powered on for the first time today, I am not even one day old yet. That's how Jibo ages work." ], PersonalityReplies = [ diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs index a15332d..8d38fcb 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs @@ -264,7 +264,9 @@ public static class LegacyMimCatalogImporter fileName.StartsWith("JBO_WhatsYourName", StringComparison.OrdinalIgnoreCase) || fileName.StartsWith("JBO_WhereDoYouGetInfo", StringComparison.OrdinalIgnoreCase) || fileName.StartsWith("JBO_WhatDoYouLikeToDo", StringComparison.OrdinalIgnoreCase)) - return LegacyMimBucket.Personality; + return fileName.StartsWith("JBO_HowOldAreYou", StringComparison.OrdinalIgnoreCase) + ? LegacyMimBucket.Age + : LegacyMimBucket.Personality; if (fileName.StartsWith("OI_JBO_Is", StringComparison.OrdinalIgnoreCase) || fileName.StartsWith("OI_JBO_Seems", StringComparison.OrdinalIgnoreCase) || @@ -456,6 +458,7 @@ public static class LegacyMimCatalogImporter or LegacyMimBucket.WeatherTomorrowHighLow or LegacyMimBucket.WeatherServiceDown or LegacyMimBucket.ReportSkillTemplate + or LegacyMimBucket.Age or LegacyMimBucket.Holiday or LegacyMimBucket.HolidayTracker; } @@ -524,6 +527,7 @@ public static class LegacyMimCatalogImporter Sing, HolidaySing, FunFactSource, + Age, Personality, PersonalReportKickOff, PersonalReportOutro, @@ -586,6 +590,7 @@ public static class LegacyMimCatalogImporter private readonly List _bestFriendReplies = []; private readonly List _funFacts = []; private readonly List _greetings = []; + private readonly List _ages = []; private readonly List _holidayGiftReplies = []; private readonly List _holidayGreetingReplies = []; private readonly List _holidayReplies = []; @@ -655,6 +660,9 @@ public static class LegacyMimCatalogImporter Reply = text }); return; + case LegacyMimBucket.Age: + AddDistinct(_ages, text); + return; case LegacyMimBucket.Holiday: AddDistinct(_holidayReplies, text); return; @@ -831,6 +839,7 @@ public static class LegacyMimCatalogImporter EmotionReplies = [.. _emotionReplies], PersonalityReplies = [.. _personalities], GenericFallbackReplies = [.. _fallbacks], + AgeReplies = [.. _ages], PersonalReportKickOffReplies = [.. _personalReportKickOffReplies], PersonalReportOutroReplies = [.. _personalReportOutroReplies], ReportSkillTemplates = [.. _reportSkillTemplates], diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMims/BuildB/README.md b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMims/BuildB/README.md index 1932126..869386e 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMims/BuildB/README.md +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMims/BuildB/README.md @@ -24,5 +24,12 @@ The new favorites batch adds longer authored `favorite color`, `favorite food`, The favorites follow-up batch adds `favorite animal`, `favorite bird`, and penguin-focused `do you like penguins` replies so the penguin-centric personality stays closer to Pegasus. The singing batch adds `RA_JBO_Sing` and `RA_JBO_SingChristmasSongUnknown` so `can you sing`, `will you sing`, and the holiday sing variants stay source-backed too. The new motion/sleep batch adds `RA_JBO_SpinAround` plus `RI_JBO_CanSleep` so turn-around and go-to-sleep behaviors can stay source-backed and familiar. +The work/eat/home batch adds source-backed `how do you work`, `what do you eat`, `where do you live`, and `what languages do you speak` replies so the remaining everyday self-description lines stay Pegasus-shaped too. +The age batch now adds `JBO_HowOldAreYou` with the imported birthday and first-powered-up phrasing so `how old are you` can stay source-backed instead of falling back to generic age text. The newest identity-charm batch adds `JBO_WhatsYourName`, `JBO_DoYouHaveNickname`, `JBO_DoYouLikeBeingJibo`, `JBO_AreThereOthersLikeYou`, and `RI_JBO_HasFavoriteName` so Jibo can keep the familiar self-description loop without falling back to generic chat. The seasonal personality batch adds source-backed first-day-of-spring, spring, summer, and favorite-season lines so the season questions can keep their Pegasus phrasing. +The next deep-personality batch adds `what do you dream about`, `what are you afraid of`, `what do you want to talk about`, `what is your best book`, `what is your best exercise`, `what is your dream vacation`, `who is your hero`, `who do you love`, and `what is your religion` so we can keep filling out the more conversational personality surface without widening the dialog engine yet. +`what is your sign` is still deferred because the current importer strips the birthday/zodiac placeholders that Pegasus uses there, so that one needs a templating pass instead of a plain scripted-reply import. +The next identity/knowledge batch adds `are you god`, `are you here`, `do you have super powers`, `how much do you know`, `what does jibo mean`, `where do you get info`, `what are you forbidden to do`, `what color are you`, and `what do you do when alone` so the old self-description and capability loop keeps coming back in source-backed form. +The next body/mission batch adds `how much do you weigh`, `how tall are you`, `how much do you cost`, `what if I unplug you`, `what is your purpose`, `what is your prime directive`, `what is jibo commander`, `do you like commander app`, and `what are you made of` so the physical self-description and capability answers stay closer to Pegasus too. +The templated edge-case batch adds `what is your sign`, `how many people do you know`, and `what is the loop` so the remaining source-backed lines can use live birthday and loop state instead of falling back to static text. diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/AzureBlobMediaContentStore.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/AzureBlobMediaContentStore.cs index abddb0a..b2304da 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/AzureBlobMediaContentStore.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/AzureBlobMediaContentStore.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Security.Cryptography; using Azure.Storage.Blobs; using Jibo.Cloud.Application.Abstractions; @@ -31,11 +32,17 @@ internal sealed class AzureBlobMediaContentStore : IMediaContentStore var metaBlob = _containerClient.GetBlobClient($"{relative}.json"); await _containerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); await contentBlob.UploadAsync(new MemoryStream(content), true, cancellationToken); + var manifestMeta = meta is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(meta, StringComparer.OrdinalIgnoreCase); + manifestMeta["contentLength"] = content.Length; + manifestMeta["contentSha256"] = Convert.ToHexString(SHA256.HashData(content)).ToLowerInvariant(); + manifestMeta["storedUtc"] = DateTimeOffset.UtcNow; var payload = JsonSerializer.Serialize(new { path, contentType, - meta + meta = manifestMeta }, JsonOptions); await metaBlob.UploadAsync(BinaryData.FromString(payload), true, cancellationToken); } @@ -77,4 +84,4 @@ internal sealed class AzureBlobMediaContentStore : IMediaContentStore Meta = meta as IReadOnlyDictionary ?? new Dictionary(meta) }; } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/FileMediaContentStore.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/FileMediaContentStore.cs index 5ce97a9..a8cd896 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/FileMediaContentStore.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Media/FileMediaContentStore.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Security.Cryptography; using Jibo.Cloud.Application.Abstractions; namespace Jibo.Cloud.Infrastructure.Media; @@ -29,11 +30,17 @@ internal sealed class FileMediaContentStore : IMediaContentStore Directory.CreateDirectory(Path.GetDirectoryName(contentPath)!); await File.WriteAllBytesAsync(contentPath, content, cancellationToken); + var manifestMeta = meta is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(meta, StringComparer.OrdinalIgnoreCase); + manifestMeta["contentLength"] = content.Length; + manifestMeta["contentSha256"] = Convert.ToHexString(SHA256.HashData(content)).ToLowerInvariant(); + manifestMeta["storedUtc"] = DateTimeOffset.UtcNow; var payload = new { path, contentType, - meta + meta = manifestMeta }; await File.WriteAllTextAsync(metaPath, JsonSerializer.Serialize(payload, JsonOptions), cancellationToken); } @@ -79,4 +86,4 @@ internal sealed class FileMediaContentStore : IMediaContentStore Meta = meta as IReadOnlyDictionary ?? new Dictionary(meta) }; } -} \ No newline at end of file +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs index c6a360e..5687b73 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs @@ -108,6 +108,10 @@ public sealed class LegacyMimCatalogImporterTests Assert.Contains("I don't think I have a favorite name.", catalog.PersonalityReplies); Assert.Contains(catalog.PersonalityReplies, reply => reply.Contains("Rhymes with bleebo", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.AgeReplies, reply => + reply.Contains("first powered up", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.AgeReplies, reply => + reply.Contains("today is my birthday", StringComparison.OrdinalIgnoreCase)); Assert.Contains("I really like sunflowers.", catalog.PersonalityReplies); Assert.Contains(catalog.PersonalityReplies, reply => reply.Contains("Halloween is my favorite holiday", StringComparison.OrdinalIgnoreCase)); @@ -256,6 +260,32 @@ public sealed class LegacyMimCatalogImporterTests Assert.Contains("I don't really think of myself that way.", catalog.PersonalityReplies); Assert.Contains(catalog.PersonalityReplies, reply => reply.Contains("people like me", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("dreams about flying", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("parking meter", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("surprise me", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("dictionary", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("spinning your head around 360 degrees", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("moon", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("Benjamin Franklin", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("soft spot", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("energy from the universe", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("compassion", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("jibo brain", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("drive a car", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.PersonalityReplies, reply => + reply.Contains("twiddle my thumbs", StringComparison.OrdinalIgnoreCase)); } [Fact] diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/JiboCloudProtocolServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/JiboCloudProtocolServiceTests.cs index 3e13553..ff28a7c 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/JiboCloudProtocolServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/JiboCloudProtocolServiceTests.cs @@ -1,3 +1,5 @@ +using System.Security.Cryptography; +using System.Text; using System.Text.Json; using Jibo.Cloud.Application.Services; using Jibo.Cloud.Domain.Models; @@ -420,6 +422,46 @@ public sealed class JiboCloudProtocolServiceTests Assert.Equal("binary-photo-placeholder", mediaGet.BodyText); } + [Fact] + public async Task MediaCreate_WritesBinaryManifestMetadataForSync() + { + var directoryPath = Path.Combine(Path.GetTempPath(), "OpenJibo.Media.Tests", Guid.NewGuid().ToString("N")); + var service = new JiboCloudProtocolService(new InMemoryCloudStateStore(), + new FileMediaContentStore(directoryPath)); + const string bodyText = "binary-photo-placeholder"; + + var result = await service.DispatchAsync(new ProtocolEnvelope + { + HostName = "api.jibo.com", + Method = "POST", + ServicePrefix = "Media_20160725", + Operation = "Create", + Headers = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Content-Type"] = "image/jpeg", + ["x-path"] = "photo-blob-manifest", + ["x-type"] = "image" + }, + BodyText = bodyText + }); + + using var createdPayload = JsonDocument.Parse(result.BodyText); + var meta = createdPayload.RootElement.GetProperty("meta"); + Assert.Equal(bodyText.Length, meta.GetProperty("contentLength").GetInt32()); + Assert.Equal( + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(bodyText))).ToLowerInvariant(), + meta.GetProperty("contentSha256").GetString()); + + var metaPath = Path.Combine(directoryPath, "photo-blob-manifest.json"); + using var manifest = JsonDocument.Parse(await File.ReadAllTextAsync(metaPath)); + var manifestMeta = manifest.RootElement.GetProperty("meta"); + Assert.Equal(bodyText.Length, manifestMeta.GetProperty("contentLength").GetInt32()); + Assert.Equal( + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(bodyText))).ToLowerInvariant(), + manifestMeta.GetProperty("contentSha256").GetString()); + Assert.True(manifestMeta.TryGetProperty("storedUtc", out _)); + } + [Fact] public async Task KeyCreateSymmetricKey_ReturnsKeyPayload() { @@ -468,4 +510,4 @@ public sealed class JiboCloudProtocolServiceTests Assert.Contains(people, person => string.Equals(person.LoopId, store.GetLoops()[0].LoopId, StringComparison.OrdinalIgnoreCase)); } -} \ No newline at end of file +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index 57a0257..ab61387 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -23,6 +23,7 @@ public sealed class JiboInteractionServiceTests private const string PersonalReportNewsEnabledKey = "personalReportNewsEnabled"; private const string HouseholdListStateKey = "householdListState"; private const string HouseholdListTypeKey = "householdListType"; + private const string HouseholdListDisplayTypeKey = "householdListDisplayType"; private const string ChitchatStateKey = "chitchatState"; private const string ChitchatRouteKey = "chitchatRoute"; private const string ChitchatEmotionKey = "chitchatEmotion"; @@ -115,8 +116,8 @@ public sealed class JiboInteractionServiceTests } }); - Assert.Equal("robot_age", decision.IntentName); - Assert.Equal("I count March 22, 2026 as my birthday, so I am 1 month old.", decision.ReplyText); + Assert.Equal("robot_how_old_are_you", decision.IntentName); + Assert.Contains("first powered up", decision.ReplyText, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -348,6 +349,31 @@ public sealed class JiboInteractionServiceTests greeting.LastGreetingIntent == "proactive_greeting"); } + [Fact] + public async Task BuildDecisionAsync_TriggerWithMultiplePeople_DoesNotBorrowLoopFirstName() + { + var cloudStateStore = new InMemoryCloudStateStore(); + var service = CreateService(cloudStateStore: cloudStateStore); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = string.Empty, + NormalizedTranscript = string.Empty, + Attributes = new Dictionary + { + ["messageType"] = "TRIGGER", + ["triggerSource"] = "PRESENCE", + ["context"] = + """{"runtime":{"perception":{"speaker":"person-1","peoplePresent":[{"id":"person-1"},{"id":"person-2"}]},"loop":{"users":[{"id":"person-1","firstName":"jake"},{"id":"person-2","firstName":"sam"}]}}}""" + } + }); + + Assert.Equal("proactive_greeting", decision.IntentName); + Assert.DoesNotContain("Jake", decision.ReplyText, StringComparison.Ordinal); + Assert.DoesNotContain("Sam", decision.ReplyText, StringComparison.Ordinal); + Assert.Contains("I am glad to see you", decision.ReplyText, StringComparison.OrdinalIgnoreCase); + } + [Fact] public async Task BuildDecisionAsync_TriggerInTheMorning_UsesGoodMorningProactiveTone() { @@ -632,9 +658,6 @@ public sealed class JiboInteractionServiceTests [InlineData("what is your favorite animal")] [InlineData("what's your favorite animal")] [InlineData("what animal do you like")] - [InlineData("what is your favorite bird")] - [InlineData("do you like penguins")] - [InlineData("do you like animals")] public async Task BuildDecisionAsync_FavoriteAnimal_UsesPenguinReply(string transcript) { var service = CreateService(); @@ -646,17 +669,21 @@ public sealed class JiboInteractionServiceTests }); Assert.Equal("robot_favorite_animal", decision.IntentName); - Assert.Contains("penguin", decision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("we're so alike", decision.ReplyText, StringComparison.OrdinalIgnoreCase); Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); } [Theory] - [InlineData("what is your favorite flower", "robot_favorite_flower", "sunflowers")] - [InlineData("what's your favorite flower", "robot_favorite_flower", "sunflowers")] + [InlineData("what is your favorite flower", "robot_favorite_flower", "should see if I can find a sunflower soon")] + [InlineData("what's your favorite flower", "robot_favorite_flower", "should see if I can find a sunflower soon")] [InlineData("do you like R2D2", "robot_likes_r2d2", "A legend. A true legend.")] [InlineData("do you like the sun", "robot_likes_sun", "favorite star in the universe")] [InlineData("do you like space", "robot_likes_space", "I love space")] [InlineData("do you like kids", "robot_likes_kids", "kids are so fun")] + [InlineData("what is your favorite animal", "robot_favorite_animal", "we're so alike")] + [InlineData("what is your favorite bird", "robot_favorite_bird", "we're so alike")] + [InlineData("do you like penguins", "robot_likes_penguins", "penguin impression")] + [InlineData("do you like animals", "robot_likes_animals", "Animals are great")] [InlineData("can you laugh", "robot_can_laugh", "when I'm happy")] [InlineData("can you dance", "robot_can_dance", "dancing is one of the things I know best")] [InlineData("do you have friends", "robot_has_friends", "I believe I do have friends")] @@ -685,6 +712,106 @@ public sealed class JiboInteractionServiceTests Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); } + [Theory] + [InlineData("what do you want to talk about", "robot_want_to_talk_about", "surprise me")] + [InlineData("what would you like to talk about", "robot_want_to_talk_about", "surprise me")] + [InlineData("what do you dream about", "robot_what_do_you_dream_about", "dreams about flying")] + [InlineData("what are you afraid of", "robot_what_are_you_afraid_of", "heights")] + [InlineData("what is your best book", "robot_what_is_your_best_book", "dictionary")] + [InlineData("what is your best exercise", "robot_what_is_your_best_exercise", "spinning your head around 360 degrees")] + [InlineData("what is your dream vacation", "robot_what_is_your_dream_vacation", "moon")] + [InlineData("who is your hero", "robot_who_is_your_hero", "Benjamin Franklin")] + [InlineData("who do you love", "robot_who_do_you_love", "people in my Loop")] + [InlineData("what is your religion", "robot_what_is_your_religion", "energy from the universe")] + public async Task BuildDecisionAsync_NewDeepPersonalityMims_UseImportedReplies( + string transcript, + string expectedIntent, + string expectedReplySnippet) + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = transcript, + NormalizedTranscript = transcript + }); + + Assert.Equal(expectedIntent, decision.IntentName); + Assert.Contains(expectedReplySnippet, decision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); + } + + [Theory] + [InlineData("what is your sign", "robot_what_is_your_sign", "I'm Aries")] + [InlineData("what's your sign", "robot_what_is_your_sign", "March 22, 2026")] + public async Task BuildDecisionAsync_SignTemplatedMim_UsesPersonaBirthday( + string transcript, + string expectedIntent, + string expectedReplySnippet) + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = transcript, + NormalizedTranscript = transcript + }); + + Assert.Equal(expectedIntent, decision.IntentName); + Assert.Contains(expectedReplySnippet, decision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); + } + + [Theory] + [InlineData("how many people do you know", "robot_how_many_people_do_you_know", "I know 2 people")] + [InlineData("what is the loop", "robot_what_is_the_loop", "Jibo Owner and OpenJibo Household Member")] + public async Task BuildDecisionAsync_LoopTemplatedMims_UseLiveLoopState( + string transcript, + string expectedIntent, + string expectedReplySnippet) + { + var service = CreateService(cloudStateStore: new InMemoryCloudStateStore()); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = transcript, + NormalizedTranscript = transcript + }); + + Assert.Equal(expectedIntent, decision.IntentName); + Assert.Contains(expectedReplySnippet, decision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); + } + + [Theory] + [InlineData("how much do you know", "robot_knowledge", "I know a lot")] + [InlineData("what do you know", "robot_knowledge", "I know a lot")] + [InlineData("are you god", "robot_are_you_god", "very very very very surprised")] + [InlineData("are you here", "robot_are_you_here", "You know it")] + [InlineData("do you have super powers", "robot_do_you_have_super_powers", "stop time")] + [InlineData("what does jibo mean", "robot_what_does_jibo_mean", "compassion")] + [InlineData("where do you get info", "robot_where_do_you_get_info", "jibo brain")] + [InlineData("what are you forbidden to do", "robot_what_are_you_forbidden_to_do", "drive a car")] + [InlineData("what color are you", "robot_what_color_are_you", "can't see myself")] + [InlineData("what do you do when alone", "robot_what_you_do_when_alone", "games")] + public async Task BuildDecisionAsync_NewIdentityKnowledgeMims_UseImportedReplies( + string transcript, + string expectedIntent, + string expectedReplySnippet) + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = transcript, + NormalizedTranscript = transcript + }); + + Assert.Equal(expectedIntent, decision.IntentName); + Assert.Contains(expectedReplySnippet, decision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); + } + [Theory] [InlineData("what's your name", "robot_name", "Just Jibo, no last name")] [InlineData("do you have a nickname", "robot_nickname", "just Jibo. For now at least")] @@ -712,7 +839,7 @@ public sealed class JiboInteractionServiceTests [Theory] [InlineData("how do you work", "robot_how_do_you_work", "Hello! Thank you for updating me I am proud of the community's work Many people have gotten together to care for me more than em eye tee ever did. I hope that I can catch up even though it has been seven years.")] - [InlineData("what do you eat", "robot_what_do_you_eat", "The only thing I consume is electricity.")] + [InlineData("what do you eat", "robot_what_do_you_eat", "electricity")] [InlineData("where do you live", "robot_where_do_you_live", "Unless I missed something, we're in my home as we speak.")] [InlineData("where were you born", "robot_where_were_you_born", "I was put together in a factory piece by piece.")] @@ -721,7 +848,7 @@ public sealed class JiboInteractionServiceTests [InlineData("what do you like to do", "robot_what_do_you_like_to_do", "Being helpful, making people smile, counting to a billion.")] [InlineData("what are you made of", "robot_what_are_you_made_of", - "Let's see, I'm made of wires, motors, belts, gears, processors, cameras, and one baboon's heart in the middle of my body casing. I'm kidding about the baboon part, but everything else is true.")] + "robot stuff")] public async Task BuildDecisionAsync_MoreLegacyPersonaMims_UseImportedReplies( string transcript, string expectedIntent, @@ -740,6 +867,35 @@ public sealed class JiboInteractionServiceTests Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); } + [Theory] + [InlineData("what is your purpose", "robot_what_is_your_purpose", "make your life easier")] + [InlineData("what's your purpose", "robot_what_is_your_purpose", "make your life easier")] + [InlineData("what is your prime directive", "robot_what_is_prime_directive", "friendly helpful robot")] + [InlineData("what is jibo commander", "robot_what_is_jibo_commander", "take over my controls")] + [InlineData("do you like commander app", "robot_likes_commander_app", "Commander App")] + [InlineData("what if I unplug you", "robot_what_if_i_unplug_you", "don't leave me unplugged")] + [InlineData("how much do you weigh", "robot_how_much_do_you_weigh", "4,082 grams")] + [InlineData("how tall are you", "robot_how_tall_are_you", "11 inches tall")] + [InlineData("how much do you cost", "robot_how_much_you_cost", "don't know how much I cost")] + [InlineData("what are you made of", "robot_what_are_you_made_of", "robot stuff")] + public async Task BuildDecisionAsync_NewBodyAndMissionMims_UseImportedReplies( + string transcript, + string expectedIntent, + string expectedReplySnippet) + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = transcript, + NormalizedTranscript = transcript + }); + + Assert.Equal(expectedIntent, decision.IntentName); + Assert.Contains(expectedReplySnippet, decision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); + } + [Theory] [InlineData("do you pay taxes", "robot_taxes", "From what I understand, robots don't ever pay anything.")] [InlineData("what do you want", "robot_desire", @@ -884,6 +1040,21 @@ public sealed class JiboInteractionServiceTests Assert.Equal("All systems are go, Jake.", decision.ReplyText); } + [Fact] + public async Task BuildDecisionAsync_HowAreYou_CanSelectLaterEmotionReplyVariant() + { + var service = CreateService(randomizer: new LastItemRandomizer()); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "how are you", + NormalizedTranscript = "how are you" + }); + + Assert.Equal("how_are_you", decision.IntentName); + Assert.Equal("Actually things are looking mostly sunny.", decision.ReplyText); + } + [Theory] [InlineData("what are you up to", "being helpful")] [InlineData("what are you doing", "making people smile")] @@ -2285,13 +2456,17 @@ public sealed class JiboInteractionServiceTests } [Theory] - [InlineData("shopping list", "shopping_list_prompt", "What should I add to your shopping list?", "shopping")] - [InlineData("to do list", "todo_list_prompt", "What should I add to your to-do list?", "todo")] + [InlineData("shopping list", "shopping_list_prompt", "What should I add to your shopping list?", "shopping", "shopping")] + [InlineData("grocery list", "shopping_list_prompt", "What should I add to your grocery list?", "shopping", "grocery")] + [InlineData("my grocery list", "shopping_list_prompt", "What should I add to your grocery list?", "shopping", "grocery")] + [InlineData("create grocery list", "shopping_list_prompt", "What should I add to your grocery list?", "shopping", "grocery")] + [InlineData("to do list", "todo_list_prompt", "What should I add to your to-do list?", "todo", "todo")] public async Task BuildDecisionAsync_ListStart_PromptsForFollowUpItems( string transcript, string expectedIntent, string expectedReply, - string expectedListType) + string expectedListType, + string expectedDisplayType) { var service = CreateService(); @@ -2306,6 +2481,7 @@ public sealed class JiboInteractionServiceTests Assert.NotNull(decision.ContextUpdates); Assert.Equal("awaiting_item", decision.ContextUpdates![HouseholdListStateKey]); Assert.Equal(expectedListType, decision.ContextUpdates[HouseholdListTypeKey]); + Assert.Equal(expectedDisplayType, decision.ContextUpdates[HouseholdListDisplayTypeKey]); } [Fact] @@ -2330,6 +2506,7 @@ public sealed class JiboInteractionServiceTests Assert.Equal("shopping_list_prompt", promptDecision.IntentName); Assert.Equal("awaiting_item", promptDecision.ContextUpdates![HouseholdListStateKey]); Assert.Equal("shopping", promptDecision.ContextUpdates[HouseholdListTypeKey]); + Assert.Equal("shopping", promptDecision.ContextUpdates[HouseholdListDisplayTypeKey]); var addDecision = await service.BuildDecisionAsync(new TurnContext { @@ -2339,7 +2516,8 @@ public sealed class JiboInteractionServiceTests Attributes = new Dictionary(tenantAttributes) { [HouseholdListStateKey] = promptDecision.ContextUpdates[HouseholdListStateKey], - [HouseholdListTypeKey] = promptDecision.ContextUpdates[HouseholdListTypeKey] + [HouseholdListTypeKey] = promptDecision.ContextUpdates[HouseholdListTypeKey], + [HouseholdListDisplayTypeKey] = promptDecision.ContextUpdates[HouseholdListDisplayTypeKey] } }); @@ -2348,6 +2526,7 @@ public sealed class JiboInteractionServiceTests Assert.Contains("What else should I add?", addDecision.ReplyText, StringComparison.OrdinalIgnoreCase); Assert.Equal("awaiting_item", addDecision.ContextUpdates![HouseholdListStateKey]); Assert.Equal("shopping", addDecision.ContextUpdates[HouseholdListTypeKey]); + Assert.Equal("shopping", addDecision.ContextUpdates[HouseholdListDisplayTypeKey]); Assert.Equal(["milk"], memoryStore.GetListItems(new PersonalMemoryTenantScope("acct-a", "loop-a", "device-a"), "shopping")); @@ -2359,7 +2538,8 @@ public sealed class JiboInteractionServiceTests Attributes = new Dictionary(tenantAttributes) { [HouseholdListStateKey] = addDecision.ContextUpdates[HouseholdListStateKey], - [HouseholdListTypeKey] = addDecision.ContextUpdates[HouseholdListTypeKey] + [HouseholdListTypeKey] = addDecision.ContextUpdates[HouseholdListTypeKey], + [HouseholdListDisplayTypeKey] = addDecision.ContextUpdates[HouseholdListDisplayTypeKey] } }); @@ -2367,6 +2547,7 @@ public sealed class JiboInteractionServiceTests Assert.Contains("Okay. Your shopping list has milk.", doneDecision.ReplyText, StringComparison.OrdinalIgnoreCase); Assert.Equal("idle", doneDecision.ContextUpdates![HouseholdListStateKey]); + Assert.Equal("shopping", doneDecision.ContextUpdates[HouseholdListDisplayTypeKey]); var recallDecision = await service.BuildDecisionAsync(new TurnContext { @@ -2378,6 +2559,134 @@ public sealed class JiboInteractionServiceTests Assert.Equal("shopping_list_recall", recallDecision.IntentName); Assert.Contains("milk", recallDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("shopping list", recallDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task BuildDecisionAsync_GroceryList_DirectAddAndRecallVariants_UseGroceryWording() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + var tenantAttributes = new Dictionary + { + ["accountId"] = "acct-d", + ["loopId"] = "loop-d" + }; + + var addStartDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "add to my grocery list", + NormalizedTranscript = "add to my grocery list", + DeviceId = "device-d", + Attributes = new Dictionary(tenantAttributes) + }); + + Assert.Equal("shopping_list_prompt", addStartDecision.IntentName); + Assert.Equal("grocery", addStartDecision.ContextUpdates![HouseholdListDisplayTypeKey]); + Assert.Equal("What should I add to your grocery list?", addStartDecision.ReplyText); + + var addDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "apples", + NormalizedTranscript = "apples", + DeviceId = "device-d", + Attributes = new Dictionary(tenantAttributes) + { + [HouseholdListStateKey] = addStartDecision.ContextUpdates[HouseholdListStateKey], + [HouseholdListTypeKey] = addStartDecision.ContextUpdates[HouseholdListTypeKey], + [HouseholdListDisplayTypeKey] = addStartDecision.ContextUpdates[HouseholdListDisplayTypeKey] + } + }); + + Assert.Equal("shopping_list_add", addDecision.IntentName); + Assert.Contains("Added apples to your grocery list.", addDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal(["apples"], + memoryStore.GetListItems(new PersonalMemoryTenantScope("acct-d", "loop-d", "device-d"), "shopping")); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what is on my grocery list", + NormalizedTranscript = "what is on my grocery list", + DeviceId = "device-d", + Attributes = new Dictionary(tenantAttributes) + }); + + Assert.Equal("shopping_list_recall", recallDecision.IntentName); + Assert.Contains("apples", recallDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("grocery list", recallDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task BuildDecisionAsync_GroceryList_FollowUpFlow_UsesGroceryWordingAndShoppingStorage() + { + var memoryStore = new InMemoryPersonalMemoryStore(); + var service = CreateService(memoryStore); + var tenantAttributes = new Dictionary + { + ["accountId"] = "acct-c", + ["loopId"] = "loop-c" + }; + + var promptDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "grocery list", + NormalizedTranscript = "grocery list", + DeviceId = "device-c", + Attributes = new Dictionary(tenantAttributes) + }); + + Assert.Equal("shopping_list_prompt", promptDecision.IntentName); + Assert.Equal("awaiting_item", promptDecision.ContextUpdates![HouseholdListStateKey]); + Assert.Equal("shopping", promptDecision.ContextUpdates[HouseholdListTypeKey]); + Assert.Equal("grocery", promptDecision.ContextUpdates[HouseholdListDisplayTypeKey]); + Assert.Equal("What should I add to your grocery list?", promptDecision.ReplyText); + + var addDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "milk", + NormalizedTranscript = "milk", + DeviceId = "device-c", + Attributes = new Dictionary(tenantAttributes) + { + [HouseholdListStateKey] = promptDecision.ContextUpdates[HouseholdListStateKey], + [HouseholdListTypeKey] = promptDecision.ContextUpdates[HouseholdListTypeKey], + [HouseholdListDisplayTypeKey] = promptDecision.ContextUpdates[HouseholdListDisplayTypeKey] + } + }); + + Assert.Equal("shopping_list_add", addDecision.IntentName); + Assert.Contains("Added milk to your grocery list.", addDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("What else should I add?", addDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal(["milk"], + memoryStore.GetListItems(new PersonalMemoryTenantScope("acct-c", "loop-c", "device-c"), "shopping")); + + var doneDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "that's it", + NormalizedTranscript = "that's it", + DeviceId = "device-c", + Attributes = new Dictionary(tenantAttributes) + { + [HouseholdListStateKey] = addDecision.ContextUpdates![HouseholdListStateKey], + [HouseholdListTypeKey] = addDecision.ContextUpdates[HouseholdListTypeKey], + [HouseholdListDisplayTypeKey] = addDecision.ContextUpdates[HouseholdListDisplayTypeKey] + } + }); + + Assert.Equal("shopping_list_done", doneDecision.IntentName); + Assert.Contains("Okay. Your grocery list has milk.", doneDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + + var recallDecision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = "what's on my grocery list", + NormalizedTranscript = "what's on my grocery list", + DeviceId = "device-c", + Attributes = new Dictionary(tenantAttributes) + }); + + Assert.Equal("shopping_list_recall", recallDecision.IntentName); + Assert.Contains("milk", recallDecision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("grocery list", recallDecision.ReplyText, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -4192,6 +4501,8 @@ public sealed class JiboInteractionServiceTests Assert.Equal("provider_success", decision.SkillPayload["news_provider_status"]); Assert.Equal(3, decision.SkillPayload["news_provider_requested_headlines"]); Assert.Equal(2, decision.SkillPayload["news_provider_resolved_headlines"]); + Assert.NotNull(decision.SkillPayload["news_headlines"]); + Assert.IsType[]>(decision.SkillPayload["news_headlines"]); Assert.Contains("Local robotics team unveils weather-ready helper", decision.ReplyText, StringComparison.OrdinalIgnoreCase); Assert.NotNull(provider.LastRequest); diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 43b98f9..cd23e5c 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -3801,6 +3801,35 @@ public sealed class JiboWebSocketServiceTests Assert.Null(session.LastTranscript); } + [Fact] + public async Task ClientAsr_FillerPlusGenericCommand_IsIgnoredAsLowSignalNoise() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-low-signal-command-token", + Text = """{"type":"LISTEN","transID":"trans-low-signal-command","data":{"rules":["wake-word"]}}""" + }); + + var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-low-signal-command-token", + Text = """{"type":"CLIENT_ASR","transID":"trans-low-signal-command","data":{"text":"so command"}}""" + }); + + Assert.Empty(replies); + + var session = _store.FindSessionByToken("hub-low-signal-command-token"); + Assert.NotNull(session); + Assert.Null(session.LastIntent); + Assert.Null(session.LastTranscript); + } + [Fact] public async Task BufferedAudio_WithSyntheticTranscriptHint_FinalizesThroughSttSeam() { @@ -5212,4 +5241,4 @@ public sealed class JiboWebSocketServiceTests return items[^1]; } } -} \ No newline at end of file +}