diff --git a/OpenJibo/docs/protocol-inventory.md b/OpenJibo/docs/protocol-inventory.md index af3f7ad..f41c31f 100644 --- a/OpenJibo/docs/protocol-inventory.md +++ b/OpenJibo/docs/protocol-inventory.md @@ -56,9 +56,29 @@ Observed from `open-jibo-link.js`: | Host/path | Flow | Confidence | Current .NET status | | --- | --- | --- | --- | | `api-socket.jibo.com/{token}` | token-authenticated socket for API-side signaling | medium | stub endpoint implemented | -| `neo-hub.jibo.com/{listen-path}` | listen turn flow with JSON and binary audio traffic | medium | initial JSON flow implemented | +| `neo-hub.jibo.com/{listen-path}` | listen turn flow with JSON and binary audio traffic | medium | fixture-backed synthetic turn flow implemented for `LISTEN`, `CONTEXT`, `CLIENT_NLU`, `CLIENT_ASR`, `EOS`, and first chat/joke skill responses | | `neo-hub.jibo.com/v1/proactive` | proactive connection flow | medium | stub endpoint implemented | +### Current WebSocket Parity Slice + +The current .NET pass covers only a narrow, explicitly synthetic subset of observed Neo-Hub behavior: + +- token/session tracking across websocket turns +- `LISTEN` message handling with synthetic `LISTEN` result payload shaping +- `CONTEXT` capture for turn/session state +- `CLIENT_NLU` turn completion using remembered listen/session metadata +- `CLIENT_ASR` text-driven turn completion +- `EOS` emission after completed turns +- first richer vertical slice for joke/chat `SKILL_ACTION` playback + +This does not yet mean parity for: + +- real binary audio buffering and finalization +- external ASR lifecycle timing +- early-EOS behavior +- multi-step skill lifecycles beyond the current synthetic playback response +- broader interaction, animation, or ESML command families + ## Upload Paths | Path | Purpose | Confidence | Current .NET status | diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/README.md b/OpenJibo/src/Jibo.Cloud/dotnet/README.md index 350f6cc..f18cbb0 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/README.md +++ b/OpenJibo/src/Jibo.Cloud/dotnet/README.md @@ -57,9 +57,25 @@ The .NET implementation should: - copy observed behavior where needed - use fixtures captured from Node and real robots - avoid speculative protocol design +- separate HTTP parity, websocket parity, and future discovery work so coverage stays honest ## Current State This folder now contains the first hosted scaffold, not just a README. The intent is to grow from a runnable dev monolith into the real Azure deployment target without abandoning the existing abstractions work. + +Current websocket scope is still intentionally narrow: + +- token-backed socket sessions +- synthetic `LISTEN` result shaping for `LISTEN`, `CLIENT_NLU`, and `CLIENT_ASR` +- `CONTEXT` capture and follow-up turn state +- `EOS` completion +- first skill vertical for joke/chat `SKILL_ACTION` playback + +Not yet covered: + +- real binary audio / ASR finalization parity +- upstream Nimbus or broader skill lifecycle behavior +- animation / expression command families +- ESML feature parity beyond the narrow synthetic playback payloads used in the current scaffold diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs index 3c3eaf7..6aa14b6 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs @@ -57,6 +57,19 @@ public sealed class DemoConversationBroker : IConversationBroker } }; + if (string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase)) + { + plan.Actions.Add(new InvokeNativeSkillAction + { + Sequence = 2, + SkillName = "@be/joke", + Payload = new Dictionary + { + ["replyType"] = "joke" + } + }); + } + return Task.FromResult(plan); } } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs index a11db74..cf16976 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs @@ -15,9 +15,12 @@ public sealed class JiboWebSocketService( { var session = stateStore.FindSessionByToken(envelope.Token ?? string.Empty) ?? stateStore.OpenSession(envelope.Kind, null, envelope.Token, envelope.HostName, envelope.Path); + session.LastSeenUtc = DateTimeOffset.UtcNow; if (envelope.IsBinary) { + session.LastMessageType = "BINARY_AUDIO"; + session.Metadata["lastAudioBytes"] = envelope.Binary?.Length ?? 0; return [ new WebSocketReply @@ -36,14 +39,70 @@ public sealed class JiboWebSocketService( } var parsedType = ReadMessageType(envelope.Text); - session.LastListenType = parsedType; + session.LastMessageType = parsedType; + var parsedTransId = ReadTransId(envelope.Text); + if (!string.IsNullOrWhiteSpace(parsedTransId)) + { + session.LastTransId = parsedTransId; + } + + if (parsedType == "CONTEXT") + { + session.Metadata["context"] = ExtractDataPayload(envelope.Text); + return + [ + new WebSocketReply + { + Text = JsonSerializer.Serialize(new + { + type = "OPENJIBO_CONTEXT_ACK", + data = new + { + sessionId = session.SessionId, + transID = session.LastTransId + } + }) + } + ]; + } if (parsedType is "LISTEN" or "CLIENT_NLU" or "CLIENT_ASR") { - var turn = turnContextMapper.MapListenMessage(envelope, session); - var plan = await conversationBroker.HandleTurnAsync(turn, cancellationToken); + PersistTurnHints(session, envelope.Text, parsedType); - return replyMapper.Map(plan).Select(text => new WebSocketReply + var turn = turnContextMapper.MapListenMessage(envelope, session, parsedType); + if (string.IsNullOrWhiteSpace(turn.NormalizedTranscript) && + string.IsNullOrWhiteSpace(turn.RawTranscript)) + { + return + [ + new WebSocketReply + { + Text = JsonSerializer.Serialize(new + { + type = "OPENJIBO_ACK", + data = new + { + messageType = parsedType, + sessionId = session.SessionId, + transID = session.LastTransId + } + }) + } + ]; + } + + var plan = await conversationBroker.HandleTurnAsync(turn, cancellationToken); + var listenAction = plan.Actions.OfType().OrderBy(action => action.Sequence).LastOrDefault(); + session.LastTranscript = turn.NormalizedTranscript ?? turn.RawTranscript; + session.LastIntent = plan.IntentName; + session.LastListenType = listenAction?.Mode; + session.FollowUpExpiresUtc = plan.FollowUp.KeepMicOpen + ? DateTimeOffset.UtcNow.Add(plan.FollowUp.Timeout) + : null; + + var emitSkillActions = parsedType != "CLIENT_NLU"; + return replyMapper.Map(plan, turn, session, emitSkillActions).Select(text => new WebSocketReply { Text = text }).ToArray(); @@ -66,6 +125,45 @@ public sealed class JiboWebSocketService( ]; } + private static void PersistTurnHints(CloudSession session, string? text, string messageType) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + try + { + using var document = JsonDocument.Parse(text); + var root = document.RootElement; + + if (root.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Object) + { + if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array) + { + session.Metadata["listenRules"] = rules.EnumerateArray() + .Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString()) + .Where(rule => !string.IsNullOrWhiteSpace(rule)) + .ToArray(); + } + + if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String) + { + session.LastIntent = intent.GetString(); + } + + if (messageType == "CONTEXT") + { + session.Metadata["context"] = data.GetRawText(); + } + } + } + catch + { + // Keep the compatibility layer permissive while captures are still incomplete. + } + } + private static string ReadMessageType(string? text) { if (string.IsNullOrWhiteSpace(text)) @@ -88,4 +186,50 @@ public sealed class JiboWebSocketService( return "UNKNOWN"; } + + private static string? ReadTransId(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + try + { + using var document = JsonDocument.Parse(text); + if (document.RootElement.TryGetProperty("transID", out var transId) && transId.ValueKind == JsonValueKind.String) + { + return transId.GetString(); + } + } + catch + { + return null; + } + + return null; + } + + private static string? ExtractDataPayload(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + try + { + using var document = JsonDocument.Parse(text); + if (document.RootElement.TryGetProperty("data", out var data)) + { + return data.GetRawText(); + } + } + catch + { + return null; + } + + return null; + } } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs index 5450fe2..cb0dce2 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs @@ -6,14 +6,34 @@ namespace Jibo.Cloud.Application.Services; public sealed class ProtocolToTurnContextMapper { - public TurnContext MapListenMessage(WebSocketMessageEnvelope envelope, CloudSession session) + public TurnContext MapListenMessage(WebSocketMessageEnvelope envelope, CloudSession session, string messageType) { var text = ExtractTranscript(envelope.Text); + var protocolOperation = messageType.ToLowerInvariant(); + var attributes = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["messageType"] = messageType + }; + + if (!string.IsNullOrWhiteSpace(session.LastTransId)) + { + attributes["transID"] = session.LastTransId; + } + + if (session.Metadata.TryGetValue("context", out var context)) + { + attributes["context"] = context; + } + + if (session.Metadata.TryGetValue("listenRules", out var listenRules)) + { + attributes["listenRules"] = listenRules; + } return new TurnContext { SessionId = session.SessionId, - InputMode = session.LastListenType == "follow-up" ? TurnInputMode.FollowUp : TurnInputMode.DirectText, + InputMode = session.FollowUpOpen ? TurnInputMode.FollowUp : TurnInputMode.DirectText, SourceKind = TurnSourceKind.Api, RawTranscript = text, NormalizedTranscript = text?.Trim(), @@ -21,10 +41,11 @@ public sealed class ProtocolToTurnContextMapper HostName = envelope.HostName, RequestId = envelope.ConnectionId, ProtocolService = "neo-hub", - ProtocolOperation = "listen", + ProtocolOperation = protocolOperation, FirmwareVersion = session.Metadata.TryGetValue("firmwareVersion", out var firmwareVersion) ? firmwareVersion as string : null, ApplicationVersion = session.Metadata.TryGetValue("applicationVersion", out var applicationVersion) ? applicationVersion as string : null, - IsFollowUpEligible = true + IsFollowUpEligible = true, + Attributes = attributes }; } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs index 95e97d3..f8e2668 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs @@ -1,39 +1,144 @@ using System.Text.Json; +using Jibo.Cloud.Domain.Models; using Jibo.Runtime.Abstractions; namespace Jibo.Cloud.Application.Services; public sealed class ResponsePlanToSocketMessagesMapper { - public IReadOnlyList Map(ResponsePlan plan) + public IReadOnlyList Map(ResponsePlan plan, TurnContext turn, CloudSession session, bool emitSkillActions) { var speak = plan.Actions.OfType().FirstOrDefault(); + var skill = plan.Actions.OfType().FirstOrDefault(); + var transId = turn.Attributes.TryGetValue("transID", out var transIdValue) + ? transIdValue?.ToString() ?? string.Empty + : session.LastTransId ?? string.Empty; + var transcript = turn.NormalizedTranscript ?? turn.RawTranscript ?? string.Empty; + var rules = ReadRules(turn); var messages = new List(); - if (speak is not null) + messages.Add(JsonSerializer.Serialize(new { - messages.Add(JsonSerializer.Serialize(new + type = "LISTEN", + transID = transId, + data = new { - type = "OPENJIBO_RESPONSE", - data = new + asr = new { - intent = plan.IntentName, - text = speak.Text, - followUpOpen = plan.FollowUp.KeepMicOpen, - timeoutMs = (int)plan.FollowUp.Timeout.TotalMilliseconds + confidence = 0.95, + final = true, + text = transcript + }, + nlu = new + { + confidence = 0.95, + intent = plan.IntentName ?? "unknown", + rules, + entities = new Dictionary() + }, + match = new + { + intent = plan.IntentName ?? "unknown", + rule = rules.FirstOrDefault() ?? string.Empty, + score = 0.95 } - })); - } + } + })); messages.Add(JsonSerializer.Serialize(new { type = "EOS", data = new { - sessionId = plan.SessionId + sessionId = plan.SessionId, + transID = transId } })); + if (emitSkillActions && speak is not null) + { + messages.Add(JsonSerializer.Serialize(BuildSkillPayload(plan, turn, transId, speak, skill))); + } + return messages; } + + private static IReadOnlyList ReadRules(TurnContext turn) + { + if (!turn.Attributes.TryGetValue("listenRules", out var value)) + { + return []; + } + + return value switch + { + IReadOnlyList typedRules => typedRules, + IEnumerable rules => rules.Where(rule => !string.IsNullOrWhiteSpace(rule)).ToArray(), + _ => [] + }; + } + + private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill) + { + var isJoke = string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase) || + string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase); + var skillId = isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill"; + var esml = isJoke + ? $"{EscapeXml(speak.Text)}" + : $"{EscapeXml(speak.Text)}"; + var mimId = isJoke ? "runtime-joke" : "runtime-chat"; + + return new + { + type = "SKILL_ACTION", + ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + msgID = $"msg-{Guid.NewGuid():N}", + transID = transId, + data = new + { + skill = new + { + id = skillId + }, + action = new + { + config = new + { + jcp = new + { + type = "SLIM", + config = new + { + play = new + { + esml, + meta = new + { + prompt_id = "RUNTIME_PROMPT", + prompt_sub_category = "AN", + mim_id = mimId, + mim_type = "announcement", + intent = plan.IntentName ?? "unknown", + transcript = turn.NormalizedTranscript ?? turn.RawTranscript ?? string.Empty + } + } + } + } + } + }, + analytics = new Dictionary(), + final = true + } + }; + } + + private static string EscapeXml(string value) + { + return value + .Replace("&", "&", StringComparison.Ordinal) + .Replace("<", "<", StringComparison.Ordinal) + .Replace(">", ">", StringComparison.Ordinal) + .Replace("\"", """, StringComparison.Ordinal) + .Replace("'", "'", StringComparison.Ordinal); + } } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Domain/Models/CloudSession.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Domain/Models/CloudSession.cs index 7f1ec41..3ce0eaf 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Domain/Models/CloudSession.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Domain/Models/CloudSession.cs @@ -11,7 +11,12 @@ public sealed class CloudSession public string? Path { get; init; } public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow; public DateTimeOffset LastSeenUtc { get; set; } = DateTimeOffset.UtcNow; + public DateTimeOffset? FollowUpExpiresUtc { get; set; } + public string? LastMessageType { get; set; } public string? LastListenType { get; set; } + public string? LastIntent { get; set; } public string? LastTranscript { get; set; } + public string? LastTransId { get; set; } + public bool FollowUpOpen => FollowUpExpiresUtc.HasValue && FollowUpExpiresUtc > DateTimeOffset.UtcNow; public IDictionary Metadata { get; init; } = new Dictionary(); } diff --git a/OpenJibo/src/Jibo.Cloud/node/fixtures/README.md b/OpenJibo/src/Jibo.Cloud/node/fixtures/README.md index fb58e69..83eacff 100644 --- a/OpenJibo/src/Jibo.Cloud/node/fixtures/README.md +++ b/OpenJibo/src/Jibo.Cloud/node/fixtures/README.md @@ -6,5 +6,7 @@ Current fixture groups: - `http/` Basic `X-Amz-Target` request and response examples for startup flows. +- `websocket/` + Sanitized Neo-Hub turn-flow examples used to replay `LISTEN`, `CONTEXT`, `CLIENT_NLU`, `CLIENT_ASR`, and synthetic `EOS` / `SKILL_ACTION` behavior against the .NET implementation. Expand this folder whenever new robot traffic is captured and cleaned. diff --git a/OpenJibo/src/Jibo.Cloud/node/fixtures/websocket/neo-hub-client-asr-joke.flow.json b/OpenJibo/src/Jibo.Cloud/node/fixtures/websocket/neo-hub-client-asr-joke.flow.json new file mode 100644 index 0000000..41bf798 --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/node/fixtures/websocket/neo-hub-client-asr-joke.flow.json @@ -0,0 +1,42 @@ +{ + "name": "neo-hub client asr joke flow", + "session": { + "hostName": "neo-hub.jibo.com", + "path": "/listen", + "kind": "neo-hub-listen", + "token": "fixture-joke-token" + }, + "steps": [ + { + "text": { + "type": "LISTEN", + "transID": "fixture-trans-joke", + "data": { + "text": "tell me a joke", + "rules": [ + "wake-word" + ] + } + }, + "expectedReplyTypes": [ + "LISTEN", + "EOS", + "SKILL_ACTION" + ] + }, + { + "text": { + "type": "CLIENT_ASR", + "transID": "fixture-trans-joke", + "data": { + "text": "tell me a joke" + } + }, + "expectedReplyTypes": [ + "LISTEN", + "EOS", + "SKILL_ACTION" + ] + } + ] +} diff --git a/OpenJibo/src/Jibo.Cloud/node/fixtures/websocket/neo-hub-context-client-nlu.flow.json b/OpenJibo/src/Jibo.Cloud/node/fixtures/websocket/neo-hub-context-client-nlu.flow.json new file mode 100644 index 0000000..f6322bb --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/node/fixtures/websocket/neo-hub-context-client-nlu.flow.json @@ -0,0 +1,54 @@ +{ + "name": "neo-hub context client nlu flow", + "session": { + "hostName": "neo-hub.jibo.com", + "path": "/listen", + "kind": "neo-hub-listen", + "token": "fixture-nlu-token" + }, + "steps": [ + { + "text": { + "type": "LISTEN", + "transID": "fixture-trans-nlu", + "data": { + "text": "hello jibo", + "rules": [ + "wake-word" + ] + } + }, + "expectedReplyTypes": [ + "LISTEN", + "EOS", + "SKILL_ACTION" + ] + }, + { + "text": { + "type": "CONTEXT", + "transID": "fixture-trans-nlu", + "data": { + "topic": "conversation", + "screen": "home" + } + }, + "expectedReplyTypes": [ + "OPENJIBO_CONTEXT_ACK" + ] + }, + { + "text": { + "type": "CLIENT_NLU", + "transID": "fixture-trans-nlu", + "data": { + "intent": "joke" + } + }, + "expectedReplyTypes": [ + "LISTEN", + "EOS" + ] + } + ] +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Fixtures/WebSocketFixtureLoader.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Fixtures/WebSocketFixtureLoader.cs new file mode 100644 index 0000000..3d7d2e6 --- /dev/null +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Fixtures/WebSocketFixtureLoader.cs @@ -0,0 +1,57 @@ +using System.Text.Json; +using Jibo.Cloud.Domain.Models; + +namespace Jibo.Cloud.Tests.Fixtures; + +internal static class WebSocketFixtureLoader +{ + public static WebSocketFixture Load(string relativePath) + { + var fullPath = Path.Combine(AppContext.BaseDirectory, relativePath); + using var document = JsonDocument.Parse(File.ReadAllText(fullPath)); + var root = document.RootElement; + + var session = root.GetProperty("session"); + var steps = new List(); + foreach (var stepElement in root.GetProperty("steps").EnumerateArray()) + { + steps.Add(new WebSocketFixtureStep + { + Message = new WebSocketMessageEnvelope + { + HostName = session.GetProperty("hostName").GetString() ?? "neo-hub.jibo.com", + Path = session.GetProperty("path").GetString() ?? "/listen", + Kind = session.GetProperty("kind").GetString() ?? "neo-hub-listen", + Token = session.GetProperty("token").GetString(), + Text = stepElement.TryGetProperty("text", out var text) ? text.GetRawText() : null, + Binary = stepElement.TryGetProperty("binary", out var binary) && binary.ValueKind == JsonValueKind.Array + ? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray() + : null + }, + ExpectedReplyTypes = stepElement.GetProperty("expectedReplyTypes") + .EnumerateArray() + .Select(item => item.GetString() ?? string.Empty) + .Where(item => !string.IsNullOrWhiteSpace(item)) + .ToArray() + }); + } + + return new WebSocketFixture + { + Name = root.TryGetProperty("name", out var name) ? name.GetString() ?? Path.GetFileNameWithoutExtension(relativePath) : Path.GetFileNameWithoutExtension(relativePath), + Steps = steps + }; + } +} + +internal sealed class WebSocketFixture +{ + public string Name { get; init; } = string.Empty; + public IReadOnlyList Steps { get; init; } = []; +} + +internal sealed class WebSocketFixtureStep +{ + public WebSocketMessageEnvelope Message { get; init; } = new(); + public IReadOnlyList ExpectedReplyTypes { get; init; } = []; +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Jibo.Cloud.Tests.csproj b/OpenJibo/tests/Jibo.Cloud.Tests/Jibo.Cloud.Tests.csproj index d5ef86d..c95f3c2 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/Jibo.Cloud.Tests.csproj +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Jibo.Cloud.Tests.csproj @@ -24,6 +24,10 @@ fixtures\%(Filename)%(Extension) PreserveNewest + + fixtures\%(Filename)%(Extension) + PreserveNewest + diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs index 861f7f6..f1174d7 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs @@ -2,25 +2,27 @@ using System.Text.Json; using Jibo.Cloud.Application.Services; using Jibo.Cloud.Domain.Models; using Jibo.Cloud.Infrastructure.Persistence; +using Jibo.Cloud.Tests.Fixtures; namespace Jibo.Cloud.Tests.WebSockets; public sealed class JiboWebSocketServiceTests { + private readonly InMemoryCloudStateStore _store; private readonly JiboWebSocketService _service; public JiboWebSocketServiceTests() { - var store = new InMemoryCloudStateStore(); + _store = new InMemoryCloudStateStore(); _service = new JiboWebSocketService( - store, + _store, new ProtocolToTurnContextMapper(), new DemoConversationBroker(), new ResponsePlanToSocketMessagesMapper()); } [Fact] - public async Task ListenMessage_ReturnsResponseAndEos() + public async Task ListenMessage_ReturnsSyntheticListenEosAndSkillAction() { var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope { @@ -28,12 +30,17 @@ public sealed class JiboWebSocketServiceTests Path = "/listen", Kind = "neo-hub-listen", Token = "hub-test-token", - Text = """{"type":"LISTEN","data":{"text":"hello jibo"}}""" + Text = """{"type":"LISTEN","transID":"trans-hello","data":{"text":"hello jibo","rules":["wake-word"]}}""" }); - Assert.Equal(2, replies.Count); - Assert.Contains("OPENJIBO_RESPONSE", replies[0].Text); - Assert.Contains("EOS", replies[1].Text); + Assert.Equal(3, replies.Count); + Assert.Equal("LISTEN", ReadReplyType(replies[0])); + Assert.Equal("EOS", ReadReplyType(replies[1])); + Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2])); + + using var listenPayload = JsonDocument.Parse(replies[0].Text!); + Assert.Equal("hello jibo", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString()); + Assert.Equal("chat", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString()); } [Fact] @@ -52,4 +59,69 @@ public sealed class JiboWebSocketServiceTests Assert.Equal("OPENJIBO_AUDIO_RECEIVED", payload.RootElement.GetProperty("type").GetString()); Assert.Equal(4, payload.RootElement.GetProperty("data").GetProperty("bytes").GetInt32()); } + + [Fact] + public async Task ContextThenClientNlu_UsesFollowUpTurnStateAndSkipsSkillAction() + { + await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-follow-up-token", + Text = """{"type":"LISTEN","transID":"trans-follow-up","data":{"text":"hello jibo","rules":["wake-word"]}}""" + }); + + var contextReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-follow-up-token", + Text = """{"type":"CONTEXT","transID":"trans-follow-up","data":{"topic":"conversation","screen":"home"}}""" + }); + + Assert.Single(contextReplies); + Assert.Equal("OPENJIBO_CONTEXT_ACK", ReadReplyType(contextReplies[0])); + + var nluReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope + { + HostName = "neo-hub.jibo.com", + Path = "/listen", + Kind = "neo-hub-listen", + Token = "hub-follow-up-token", + Text = """{"type":"CLIENT_NLU","transID":"trans-follow-up","data":{"intent":"joke"}}""" + }); + + Assert.Equal(2, nluReplies.Count); + Assert.Equal("LISTEN", ReadReplyType(nluReplies[0])); + Assert.Equal("EOS", ReadReplyType(nluReplies[1])); + + var session = _store.FindSessionByToken("hub-follow-up-token"); + Assert.NotNull(session); + Assert.True(session!.FollowUpOpen); + Assert.Equal("joke", session.LastIntent); + Assert.Equal("trans-follow-up", session.LastTransId); + } + + [Theory] + [InlineData("fixtures\\neo-hub-client-asr-joke.flow.json")] + [InlineData("fixtures\\neo-hub-context-client-nlu.flow.json")] + public async Task WebSocketFixture_ReplaysSuccessfully(string relativePath) + { + var fixture = WebSocketFixtureLoader.Load(relativePath); + + foreach (var step in fixture.Steps) + { + var replies = await _service.HandleMessageAsync(step.Message); + var actualTypes = replies.Select(ReadReplyType).ToArray(); + Assert.Equal(step.ExpectedReplyTypes, actualTypes); + } + } + + private static string ReadReplyType(WebSocketReply reply) + { + using var payload = JsonDocument.Parse(reply.Text!); + return payload.RootElement.GetProperty("type").GetString() ?? string.Empty; + } }