added a first pass at websocket IO
This commit is contained in:
@@ -56,9 +56,29 @@ Observed from `open-jibo-link.js`:
|
|||||||
| Host/path | Flow | Confidence | Current .NET status |
|
| Host/path | Flow | Confidence | Current .NET status |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `api-socket.jibo.com/{token}` | token-authenticated socket for API-side signaling | medium | stub endpoint implemented |
|
| `api-socket.jibo.com/{token}` | token-authenticated socket for API-side signaling | medium | stub endpoint implemented |
|
||||||
| `neo-hub.jibo.com/{listen-path}` | listen turn flow with JSON and binary audio traffic | medium | initial JSON flow implemented |
|
| `neo-hub.jibo.com/{listen-path}` | listen turn flow with JSON and binary audio traffic | medium | fixture-backed synthetic turn flow implemented for `LISTEN`, `CONTEXT`, `CLIENT_NLU`, `CLIENT_ASR`, `EOS`, and first chat/joke skill responses |
|
||||||
| `neo-hub.jibo.com/v1/proactive` | proactive connection flow | medium | stub endpoint implemented |
|
| `neo-hub.jibo.com/v1/proactive` | proactive connection flow | medium | stub endpoint implemented |
|
||||||
|
|
||||||
|
### Current WebSocket Parity Slice
|
||||||
|
|
||||||
|
The current .NET pass covers only a narrow, explicitly synthetic subset of observed Neo-Hub behavior:
|
||||||
|
|
||||||
|
- token/session tracking across websocket turns
|
||||||
|
- `LISTEN` message handling with synthetic `LISTEN` result payload shaping
|
||||||
|
- `CONTEXT` capture for turn/session state
|
||||||
|
- `CLIENT_NLU` turn completion using remembered listen/session metadata
|
||||||
|
- `CLIENT_ASR` text-driven turn completion
|
||||||
|
- `EOS` emission after completed turns
|
||||||
|
- first richer vertical slice for joke/chat `SKILL_ACTION` playback
|
||||||
|
|
||||||
|
This does not yet mean parity for:
|
||||||
|
|
||||||
|
- real binary audio buffering and finalization
|
||||||
|
- external ASR lifecycle timing
|
||||||
|
- early-EOS behavior
|
||||||
|
- multi-step skill lifecycles beyond the current synthetic playback response
|
||||||
|
- broader interaction, animation, or ESML command families
|
||||||
|
|
||||||
## Upload Paths
|
## Upload Paths
|
||||||
|
|
||||||
| Path | Purpose | Confidence | Current .NET status |
|
| Path | Purpose | Confidence | Current .NET status |
|
||||||
|
|||||||
@@ -57,9 +57,25 @@ The .NET implementation should:
|
|||||||
- copy observed behavior where needed
|
- copy observed behavior where needed
|
||||||
- use fixtures captured from Node and real robots
|
- use fixtures captured from Node and real robots
|
||||||
- avoid speculative protocol design
|
- avoid speculative protocol design
|
||||||
|
- separate HTTP parity, websocket parity, and future discovery work so coverage stays honest
|
||||||
|
|
||||||
## Current State
|
## Current State
|
||||||
|
|
||||||
This folder now contains the first hosted scaffold, not just a README.
|
This folder now contains the first hosted scaffold, not just a README.
|
||||||
|
|
||||||
The intent is to grow from a runnable dev monolith into the real Azure deployment target without abandoning the existing abstractions work.
|
The intent is to grow from a runnable dev monolith into the real Azure deployment target without abandoning the existing abstractions work.
|
||||||
|
|
||||||
|
Current websocket scope is still intentionally narrow:
|
||||||
|
|
||||||
|
- token-backed socket sessions
|
||||||
|
- synthetic `LISTEN` result shaping for `LISTEN`, `CLIENT_NLU`, and `CLIENT_ASR`
|
||||||
|
- `CONTEXT` capture and follow-up turn state
|
||||||
|
- `EOS` completion
|
||||||
|
- first skill vertical for joke/chat `SKILL_ACTION` playback
|
||||||
|
|
||||||
|
Not yet covered:
|
||||||
|
|
||||||
|
- real binary audio / ASR finalization parity
|
||||||
|
- upstream Nimbus or broader skill lifecycle behavior
|
||||||
|
- animation / expression command families
|
||||||
|
- ESML feature parity beyond the narrow synthetic playback payloads used in the current scaffold
|
||||||
|
|||||||
@@ -57,6 +57,19 @@ public sealed class DemoConversationBroker : IConversationBroker
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
plan.Actions.Add(new InvokeNativeSkillAction
|
||||||
|
{
|
||||||
|
Sequence = 2,
|
||||||
|
SkillName = "@be/joke",
|
||||||
|
Payload = new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["replyType"] = "joke"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return Task.FromResult(plan);
|
return Task.FromResult(plan);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,9 +15,12 @@ public sealed class JiboWebSocketService(
|
|||||||
{
|
{
|
||||||
var session = stateStore.FindSessionByToken(envelope.Token ?? string.Empty) ??
|
var session = stateStore.FindSessionByToken(envelope.Token ?? string.Empty) ??
|
||||||
stateStore.OpenSession(envelope.Kind, null, envelope.Token, envelope.HostName, envelope.Path);
|
stateStore.OpenSession(envelope.Kind, null, envelope.Token, envelope.HostName, envelope.Path);
|
||||||
|
session.LastSeenUtc = DateTimeOffset.UtcNow;
|
||||||
|
|
||||||
if (envelope.IsBinary)
|
if (envelope.IsBinary)
|
||||||
{
|
{
|
||||||
|
session.LastMessageType = "BINARY_AUDIO";
|
||||||
|
session.Metadata["lastAudioBytes"] = envelope.Binary?.Length ?? 0;
|
||||||
return
|
return
|
||||||
[
|
[
|
||||||
new WebSocketReply
|
new WebSocketReply
|
||||||
@@ -36,14 +39,70 @@ public sealed class JiboWebSocketService(
|
|||||||
}
|
}
|
||||||
|
|
||||||
var parsedType = ReadMessageType(envelope.Text);
|
var parsedType = ReadMessageType(envelope.Text);
|
||||||
session.LastListenType = parsedType;
|
session.LastMessageType = parsedType;
|
||||||
|
var parsedTransId = ReadTransId(envelope.Text);
|
||||||
|
if (!string.IsNullOrWhiteSpace(parsedTransId))
|
||||||
|
{
|
||||||
|
session.LastTransId = parsedTransId;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsedType == "CONTEXT")
|
||||||
|
{
|
||||||
|
session.Metadata["context"] = ExtractDataPayload(envelope.Text);
|
||||||
|
return
|
||||||
|
[
|
||||||
|
new WebSocketReply
|
||||||
|
{
|
||||||
|
Text = JsonSerializer.Serialize(new
|
||||||
|
{
|
||||||
|
type = "OPENJIBO_CONTEXT_ACK",
|
||||||
|
data = new
|
||||||
|
{
|
||||||
|
sessionId = session.SessionId,
|
||||||
|
transID = session.LastTransId
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
if (parsedType is "LISTEN" or "CLIENT_NLU" or "CLIENT_ASR")
|
if (parsedType is "LISTEN" or "CLIENT_NLU" or "CLIENT_ASR")
|
||||||
{
|
{
|
||||||
var turn = turnContextMapper.MapListenMessage(envelope, session);
|
PersistTurnHints(session, envelope.Text, parsedType);
|
||||||
var plan = await conversationBroker.HandleTurnAsync(turn, cancellationToken);
|
|
||||||
|
|
||||||
return replyMapper.Map(plan).Select(text => new WebSocketReply
|
var turn = turnContextMapper.MapListenMessage(envelope, session, parsedType);
|
||||||
|
if (string.IsNullOrWhiteSpace(turn.NormalizedTranscript) &&
|
||||||
|
string.IsNullOrWhiteSpace(turn.RawTranscript))
|
||||||
|
{
|
||||||
|
return
|
||||||
|
[
|
||||||
|
new WebSocketReply
|
||||||
|
{
|
||||||
|
Text = JsonSerializer.Serialize(new
|
||||||
|
{
|
||||||
|
type = "OPENJIBO_ACK",
|
||||||
|
data = new
|
||||||
|
{
|
||||||
|
messageType = parsedType,
|
||||||
|
sessionId = session.SessionId,
|
||||||
|
transID = session.LastTransId
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
var plan = await conversationBroker.HandleTurnAsync(turn, cancellationToken);
|
||||||
|
var listenAction = plan.Actions.OfType<ListenAction>().OrderBy(action => action.Sequence).LastOrDefault();
|
||||||
|
session.LastTranscript = turn.NormalizedTranscript ?? turn.RawTranscript;
|
||||||
|
session.LastIntent = plan.IntentName;
|
||||||
|
session.LastListenType = listenAction?.Mode;
|
||||||
|
session.FollowUpExpiresUtc = plan.FollowUp.KeepMicOpen
|
||||||
|
? DateTimeOffset.UtcNow.Add(plan.FollowUp.Timeout)
|
||||||
|
: null;
|
||||||
|
|
||||||
|
var emitSkillActions = parsedType != "CLIENT_NLU";
|
||||||
|
return replyMapper.Map(plan, turn, session, emitSkillActions).Select(text => new WebSocketReply
|
||||||
{
|
{
|
||||||
Text = text
|
Text = text
|
||||||
}).ToArray();
|
}).ToArray();
|
||||||
@@ -66,6 +125,45 @@ public sealed class JiboWebSocketService(
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void PersistTurnHints(CloudSession session, string? text, string messageType)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(text))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var document = JsonDocument.Parse(text);
|
||||||
|
var root = document.RootElement;
|
||||||
|
|
||||||
|
if (root.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Object)
|
||||||
|
{
|
||||||
|
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
|
||||||
|
{
|
||||||
|
session.Metadata["listenRules"] = rules.EnumerateArray()
|
||||||
|
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
|
||||||
|
.Where(rule => !string.IsNullOrWhiteSpace(rule))
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
|
||||||
|
{
|
||||||
|
session.LastIntent = intent.GetString();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (messageType == "CONTEXT")
|
||||||
|
{
|
||||||
|
session.Metadata["context"] = data.GetRawText();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Keep the compatibility layer permissive while captures are still incomplete.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static string ReadMessageType(string? text)
|
private static string ReadMessageType(string? text)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(text))
|
if (string.IsNullOrWhiteSpace(text))
|
||||||
@@ -88,4 +186,50 @@ public sealed class JiboWebSocketService(
|
|||||||
|
|
||||||
return "UNKNOWN";
|
return "UNKNOWN";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static string? ReadTransId(string? text)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(text))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var document = JsonDocument.Parse(text);
|
||||||
|
if (document.RootElement.TryGetProperty("transID", out var transId) && transId.ValueKind == JsonValueKind.String)
|
||||||
|
{
|
||||||
|
return transId.GetString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string? ExtractDataPayload(string? text)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(text))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var document = JsonDocument.Parse(text);
|
||||||
|
if (document.RootElement.TryGetProperty("data", out var data))
|
||||||
|
{
|
||||||
|
return data.GetRawText();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,14 +6,34 @@ namespace Jibo.Cloud.Application.Services;
|
|||||||
|
|
||||||
public sealed class ProtocolToTurnContextMapper
|
public sealed class ProtocolToTurnContextMapper
|
||||||
{
|
{
|
||||||
public TurnContext MapListenMessage(WebSocketMessageEnvelope envelope, CloudSession session)
|
public TurnContext MapListenMessage(WebSocketMessageEnvelope envelope, CloudSession session, string messageType)
|
||||||
{
|
{
|
||||||
var text = ExtractTranscript(envelope.Text);
|
var text = ExtractTranscript(envelope.Text);
|
||||||
|
var protocolOperation = messageType.ToLowerInvariant();
|
||||||
|
var attributes = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["messageType"] = messageType
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(session.LastTransId))
|
||||||
|
{
|
||||||
|
attributes["transID"] = session.LastTransId;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (session.Metadata.TryGetValue("context", out var context))
|
||||||
|
{
|
||||||
|
attributes["context"] = context;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (session.Metadata.TryGetValue("listenRules", out var listenRules))
|
||||||
|
{
|
||||||
|
attributes["listenRules"] = listenRules;
|
||||||
|
}
|
||||||
|
|
||||||
return new TurnContext
|
return new TurnContext
|
||||||
{
|
{
|
||||||
SessionId = session.SessionId,
|
SessionId = session.SessionId,
|
||||||
InputMode = session.LastListenType == "follow-up" ? TurnInputMode.FollowUp : TurnInputMode.DirectText,
|
InputMode = session.FollowUpOpen ? TurnInputMode.FollowUp : TurnInputMode.DirectText,
|
||||||
SourceKind = TurnSourceKind.Api,
|
SourceKind = TurnSourceKind.Api,
|
||||||
RawTranscript = text,
|
RawTranscript = text,
|
||||||
NormalizedTranscript = text?.Trim(),
|
NormalizedTranscript = text?.Trim(),
|
||||||
@@ -21,10 +41,11 @@ public sealed class ProtocolToTurnContextMapper
|
|||||||
HostName = envelope.HostName,
|
HostName = envelope.HostName,
|
||||||
RequestId = envelope.ConnectionId,
|
RequestId = envelope.ConnectionId,
|
||||||
ProtocolService = "neo-hub",
|
ProtocolService = "neo-hub",
|
||||||
ProtocolOperation = "listen",
|
ProtocolOperation = protocolOperation,
|
||||||
FirmwareVersion = session.Metadata.TryGetValue("firmwareVersion", out var firmwareVersion) ? firmwareVersion as string : null,
|
FirmwareVersion = session.Metadata.TryGetValue("firmwareVersion", out var firmwareVersion) ? firmwareVersion as string : null,
|
||||||
ApplicationVersion = session.Metadata.TryGetValue("applicationVersion", out var applicationVersion) ? applicationVersion as string : null,
|
ApplicationVersion = session.Metadata.TryGetValue("applicationVersion", out var applicationVersion) ? applicationVersion as string : null,
|
||||||
IsFollowUpEligible = true
|
IsFollowUpEligible = true,
|
||||||
|
Attributes = attributes
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,39 +1,144 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
|
using Jibo.Cloud.Domain.Models;
|
||||||
using Jibo.Runtime.Abstractions;
|
using Jibo.Runtime.Abstractions;
|
||||||
|
|
||||||
namespace Jibo.Cloud.Application.Services;
|
namespace Jibo.Cloud.Application.Services;
|
||||||
|
|
||||||
public sealed class ResponsePlanToSocketMessagesMapper
|
public sealed class ResponsePlanToSocketMessagesMapper
|
||||||
{
|
{
|
||||||
public IReadOnlyList<string> Map(ResponsePlan plan)
|
public IReadOnlyList<string> Map(ResponsePlan plan, TurnContext turn, CloudSession session, bool emitSkillActions)
|
||||||
{
|
{
|
||||||
var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault();
|
var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault();
|
||||||
|
var skill = plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault();
|
||||||
|
var transId = turn.Attributes.TryGetValue("transID", out var transIdValue)
|
||||||
|
? transIdValue?.ToString() ?? string.Empty
|
||||||
|
: session.LastTransId ?? string.Empty;
|
||||||
|
var transcript = turn.NormalizedTranscript ?? turn.RawTranscript ?? string.Empty;
|
||||||
|
var rules = ReadRules(turn);
|
||||||
var messages = new List<string>();
|
var messages = new List<string>();
|
||||||
|
|
||||||
if (speak is not null)
|
messages.Add(JsonSerializer.Serialize(new
|
||||||
{
|
{
|
||||||
messages.Add(JsonSerializer.Serialize(new
|
type = "LISTEN",
|
||||||
|
transID = transId,
|
||||||
|
data = new
|
||||||
{
|
{
|
||||||
type = "OPENJIBO_RESPONSE",
|
asr = new
|
||||||
data = new
|
|
||||||
{
|
{
|
||||||
intent = plan.IntentName,
|
confidence = 0.95,
|
||||||
text = speak.Text,
|
final = true,
|
||||||
followUpOpen = plan.FollowUp.KeepMicOpen,
|
text = transcript
|
||||||
timeoutMs = (int)plan.FollowUp.Timeout.TotalMilliseconds
|
},
|
||||||
|
nlu = new
|
||||||
|
{
|
||||||
|
confidence = 0.95,
|
||||||
|
intent = plan.IntentName ?? "unknown",
|
||||||
|
rules,
|
||||||
|
entities = new Dictionary<string, object?>()
|
||||||
|
},
|
||||||
|
match = new
|
||||||
|
{
|
||||||
|
intent = plan.IntentName ?? "unknown",
|
||||||
|
rule = rules.FirstOrDefault() ?? string.Empty,
|
||||||
|
score = 0.95
|
||||||
}
|
}
|
||||||
}));
|
}
|
||||||
}
|
}));
|
||||||
|
|
||||||
messages.Add(JsonSerializer.Serialize(new
|
messages.Add(JsonSerializer.Serialize(new
|
||||||
{
|
{
|
||||||
type = "EOS",
|
type = "EOS",
|
||||||
data = new
|
data = new
|
||||||
{
|
{
|
||||||
sessionId = plan.SessionId
|
sessionId = plan.SessionId,
|
||||||
|
transID = transId
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
if (emitSkillActions && speak is not null)
|
||||||
|
{
|
||||||
|
messages.Add(JsonSerializer.Serialize(BuildSkillPayload(plan, turn, transId, speak, skill)));
|
||||||
|
}
|
||||||
|
|
||||||
return messages;
|
return messages;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static IReadOnlyList<string> ReadRules(TurnContext turn)
|
||||||
|
{
|
||||||
|
if (!turn.Attributes.TryGetValue("listenRules", out var value))
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return value switch
|
||||||
|
{
|
||||||
|
IReadOnlyList<string> typedRules => typedRules,
|
||||||
|
IEnumerable<string> rules => rules.Where(rule => !string.IsNullOrWhiteSpace(rule)).ToArray(),
|
||||||
|
_ => []
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill)
|
||||||
|
{
|
||||||
|
var isJoke = string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
|
||||||
|
var skillId = isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill";
|
||||||
|
var esml = isJoke
|
||||||
|
? $"<speak><es cat='happy' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>"
|
||||||
|
: $"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>";
|
||||||
|
var mimId = isJoke ? "runtime-joke" : "runtime-chat";
|
||||||
|
|
||||||
|
return new
|
||||||
|
{
|
||||||
|
type = "SKILL_ACTION",
|
||||||
|
ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
||||||
|
msgID = $"msg-{Guid.NewGuid():N}",
|
||||||
|
transID = transId,
|
||||||
|
data = new
|
||||||
|
{
|
||||||
|
skill = new
|
||||||
|
{
|
||||||
|
id = skillId
|
||||||
|
},
|
||||||
|
action = new
|
||||||
|
{
|
||||||
|
config = new
|
||||||
|
{
|
||||||
|
jcp = new
|
||||||
|
{
|
||||||
|
type = "SLIM",
|
||||||
|
config = new
|
||||||
|
{
|
||||||
|
play = new
|
||||||
|
{
|
||||||
|
esml,
|
||||||
|
meta = new
|
||||||
|
{
|
||||||
|
prompt_id = "RUNTIME_PROMPT",
|
||||||
|
prompt_sub_category = "AN",
|
||||||
|
mim_id = mimId,
|
||||||
|
mim_type = "announcement",
|
||||||
|
intent = plan.IntentName ?? "unknown",
|
||||||
|
transcript = turn.NormalizedTranscript ?? turn.RawTranscript ?? string.Empty
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
analytics = new Dictionary<string, object?>(),
|
||||||
|
final = true
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string EscapeXml(string value)
|
||||||
|
{
|
||||||
|
return value
|
||||||
|
.Replace("&", "&", StringComparison.Ordinal)
|
||||||
|
.Replace("<", "<", StringComparison.Ordinal)
|
||||||
|
.Replace(">", ">", StringComparison.Ordinal)
|
||||||
|
.Replace("\"", """, StringComparison.Ordinal)
|
||||||
|
.Replace("'", "'", StringComparison.Ordinal);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,12 @@ public sealed class CloudSession
|
|||||||
public string? Path { get; init; }
|
public string? Path { get; init; }
|
||||||
public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow;
|
public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow;
|
||||||
public DateTimeOffset LastSeenUtc { get; set; } = DateTimeOffset.UtcNow;
|
public DateTimeOffset LastSeenUtc { get; set; } = DateTimeOffset.UtcNow;
|
||||||
|
public DateTimeOffset? FollowUpExpiresUtc { get; set; }
|
||||||
|
public string? LastMessageType { get; set; }
|
||||||
public string? LastListenType { get; set; }
|
public string? LastListenType { get; set; }
|
||||||
|
public string? LastIntent { get; set; }
|
||||||
public string? LastTranscript { get; set; }
|
public string? LastTranscript { get; set; }
|
||||||
|
public string? LastTransId { get; set; }
|
||||||
|
public bool FollowUpOpen => FollowUpExpiresUtc.HasValue && FollowUpExpiresUtc > DateTimeOffset.UtcNow;
|
||||||
public IDictionary<string, object?> Metadata { get; init; } = new Dictionary<string, object?>();
|
public IDictionary<string, object?> Metadata { get; init; } = new Dictionary<string, object?>();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,5 +6,7 @@ Current fixture groups:
|
|||||||
|
|
||||||
- `http/`
|
- `http/`
|
||||||
Basic `X-Amz-Target` request and response examples for startup flows.
|
Basic `X-Amz-Target` request and response examples for startup flows.
|
||||||
|
- `websocket/`
|
||||||
|
Sanitized Neo-Hub turn-flow examples used to replay `LISTEN`, `CONTEXT`, `CLIENT_NLU`, `CLIENT_ASR`, and synthetic `EOS` / `SKILL_ACTION` behavior against the .NET implementation.
|
||||||
|
|
||||||
Expand this folder whenever new robot traffic is captured and cleaned.
|
Expand this folder whenever new robot traffic is captured and cleaned.
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
{
|
||||||
|
"name": "neo-hub client asr joke flow",
|
||||||
|
"session": {
|
||||||
|
"hostName": "neo-hub.jibo.com",
|
||||||
|
"path": "/listen",
|
||||||
|
"kind": "neo-hub-listen",
|
||||||
|
"token": "fixture-joke-token"
|
||||||
|
},
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"text": {
|
||||||
|
"type": "LISTEN",
|
||||||
|
"transID": "fixture-trans-joke",
|
||||||
|
"data": {
|
||||||
|
"text": "tell me a joke",
|
||||||
|
"rules": [
|
||||||
|
"wake-word"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"expectedReplyTypes": [
|
||||||
|
"LISTEN",
|
||||||
|
"EOS",
|
||||||
|
"SKILL_ACTION"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"text": {
|
||||||
|
"type": "CLIENT_ASR",
|
||||||
|
"transID": "fixture-trans-joke",
|
||||||
|
"data": {
|
||||||
|
"text": "tell me a joke"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"expectedReplyTypes": [
|
||||||
|
"LISTEN",
|
||||||
|
"EOS",
|
||||||
|
"SKILL_ACTION"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
{
|
||||||
|
"name": "neo-hub context client nlu flow",
|
||||||
|
"session": {
|
||||||
|
"hostName": "neo-hub.jibo.com",
|
||||||
|
"path": "/listen",
|
||||||
|
"kind": "neo-hub-listen",
|
||||||
|
"token": "fixture-nlu-token"
|
||||||
|
},
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"text": {
|
||||||
|
"type": "LISTEN",
|
||||||
|
"transID": "fixture-trans-nlu",
|
||||||
|
"data": {
|
||||||
|
"text": "hello jibo",
|
||||||
|
"rules": [
|
||||||
|
"wake-word"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"expectedReplyTypes": [
|
||||||
|
"LISTEN",
|
||||||
|
"EOS",
|
||||||
|
"SKILL_ACTION"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"text": {
|
||||||
|
"type": "CONTEXT",
|
||||||
|
"transID": "fixture-trans-nlu",
|
||||||
|
"data": {
|
||||||
|
"topic": "conversation",
|
||||||
|
"screen": "home"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"expectedReplyTypes": [
|
||||||
|
"OPENJIBO_CONTEXT_ACK"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"text": {
|
||||||
|
"type": "CLIENT_NLU",
|
||||||
|
"transID": "fixture-trans-nlu",
|
||||||
|
"data": {
|
||||||
|
"intent": "joke"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"expectedReplyTypes": [
|
||||||
|
"LISTEN",
|
||||||
|
"EOS"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using Jibo.Cloud.Domain.Models;
|
||||||
|
|
||||||
|
namespace Jibo.Cloud.Tests.Fixtures;
|
||||||
|
|
||||||
|
internal static class WebSocketFixtureLoader
|
||||||
|
{
|
||||||
|
public static WebSocketFixture Load(string relativePath)
|
||||||
|
{
|
||||||
|
var fullPath = Path.Combine(AppContext.BaseDirectory, relativePath);
|
||||||
|
using var document = JsonDocument.Parse(File.ReadAllText(fullPath));
|
||||||
|
var root = document.RootElement;
|
||||||
|
|
||||||
|
var session = root.GetProperty("session");
|
||||||
|
var steps = new List<WebSocketFixtureStep>();
|
||||||
|
foreach (var stepElement in root.GetProperty("steps").EnumerateArray())
|
||||||
|
{
|
||||||
|
steps.Add(new WebSocketFixtureStep
|
||||||
|
{
|
||||||
|
Message = new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = session.GetProperty("hostName").GetString() ?? "neo-hub.jibo.com",
|
||||||
|
Path = session.GetProperty("path").GetString() ?? "/listen",
|
||||||
|
Kind = session.GetProperty("kind").GetString() ?? "neo-hub-listen",
|
||||||
|
Token = session.GetProperty("token").GetString(),
|
||||||
|
Text = stepElement.TryGetProperty("text", out var text) ? text.GetRawText() : null,
|
||||||
|
Binary = stepElement.TryGetProperty("binary", out var binary) && binary.ValueKind == JsonValueKind.Array
|
||||||
|
? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray()
|
||||||
|
: null
|
||||||
|
},
|
||||||
|
ExpectedReplyTypes = stepElement.GetProperty("expectedReplyTypes")
|
||||||
|
.EnumerateArray()
|
||||||
|
.Select(item => item.GetString() ?? string.Empty)
|
||||||
|
.Where(item => !string.IsNullOrWhiteSpace(item))
|
||||||
|
.ToArray()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return new WebSocketFixture
|
||||||
|
{
|
||||||
|
Name = root.TryGetProperty("name", out var name) ? name.GetString() ?? Path.GetFileNameWithoutExtension(relativePath) : Path.GetFileNameWithoutExtension(relativePath),
|
||||||
|
Steps = steps
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class WebSocketFixture
|
||||||
|
{
|
||||||
|
public string Name { get; init; } = string.Empty;
|
||||||
|
public IReadOnlyList<WebSocketFixtureStep> Steps { get; init; } = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class WebSocketFixtureStep
|
||||||
|
{
|
||||||
|
public WebSocketMessageEnvelope Message { get; init; } = new();
|
||||||
|
public IReadOnlyList<string> ExpectedReplyTypes { get; init; } = [];
|
||||||
|
}
|
||||||
@@ -24,6 +24,10 @@
|
|||||||
<Link>fixtures\%(Filename)%(Extension)</Link>
|
<Link>fixtures\%(Filename)%(Extension)</Link>
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
</None>
|
</None>
|
||||||
|
<None Include="..\..\src\Jibo.Cloud\node\fixtures\websocket\*.json">
|
||||||
|
<Link>fixtures\%(Filename)%(Extension)</Link>
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</None>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -2,25 +2,27 @@ using System.Text.Json;
|
|||||||
using Jibo.Cloud.Application.Services;
|
using Jibo.Cloud.Application.Services;
|
||||||
using Jibo.Cloud.Domain.Models;
|
using Jibo.Cloud.Domain.Models;
|
||||||
using Jibo.Cloud.Infrastructure.Persistence;
|
using Jibo.Cloud.Infrastructure.Persistence;
|
||||||
|
using Jibo.Cloud.Tests.Fixtures;
|
||||||
|
|
||||||
namespace Jibo.Cloud.Tests.WebSockets;
|
namespace Jibo.Cloud.Tests.WebSockets;
|
||||||
|
|
||||||
public sealed class JiboWebSocketServiceTests
|
public sealed class JiboWebSocketServiceTests
|
||||||
{
|
{
|
||||||
|
private readonly InMemoryCloudStateStore _store;
|
||||||
private readonly JiboWebSocketService _service;
|
private readonly JiboWebSocketService _service;
|
||||||
|
|
||||||
public JiboWebSocketServiceTests()
|
public JiboWebSocketServiceTests()
|
||||||
{
|
{
|
||||||
var store = new InMemoryCloudStateStore();
|
_store = new InMemoryCloudStateStore();
|
||||||
_service = new JiboWebSocketService(
|
_service = new JiboWebSocketService(
|
||||||
store,
|
_store,
|
||||||
new ProtocolToTurnContextMapper(),
|
new ProtocolToTurnContextMapper(),
|
||||||
new DemoConversationBroker(),
|
new DemoConversationBroker(),
|
||||||
new ResponsePlanToSocketMessagesMapper());
|
new ResponsePlanToSocketMessagesMapper());
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ListenMessage_ReturnsResponseAndEos()
|
public async Task ListenMessage_ReturnsSyntheticListenEosAndSkillAction()
|
||||||
{
|
{
|
||||||
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
{
|
{
|
||||||
@@ -28,12 +30,17 @@ public sealed class JiboWebSocketServiceTests
|
|||||||
Path = "/listen",
|
Path = "/listen",
|
||||||
Kind = "neo-hub-listen",
|
Kind = "neo-hub-listen",
|
||||||
Token = "hub-test-token",
|
Token = "hub-test-token",
|
||||||
Text = """{"type":"LISTEN","data":{"text":"hello jibo"}}"""
|
Text = """{"type":"LISTEN","transID":"trans-hello","data":{"text":"hello jibo","rules":["wake-word"]}}"""
|
||||||
});
|
});
|
||||||
|
|
||||||
Assert.Equal(2, replies.Count);
|
Assert.Equal(3, replies.Count);
|
||||||
Assert.Contains("OPENJIBO_RESPONSE", replies[0].Text);
|
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
|
||||||
Assert.Contains("EOS", replies[1].Text);
|
Assert.Equal("EOS", ReadReplyType(replies[1]));
|
||||||
|
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
|
||||||
|
|
||||||
|
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
|
||||||
|
Assert.Equal("hello jibo", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
|
||||||
|
Assert.Equal("chat", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -52,4 +59,69 @@ public sealed class JiboWebSocketServiceTests
|
|||||||
Assert.Equal("OPENJIBO_AUDIO_RECEIVED", payload.RootElement.GetProperty("type").GetString());
|
Assert.Equal("OPENJIBO_AUDIO_RECEIVED", payload.RootElement.GetProperty("type").GetString());
|
||||||
Assert.Equal(4, payload.RootElement.GetProperty("data").GetProperty("bytes").GetInt32());
|
Assert.Equal(4, payload.RootElement.GetProperty("data").GetProperty("bytes").GetInt32());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ContextThenClientNlu_UsesFollowUpTurnStateAndSkipsSkillAction()
|
||||||
|
{
|
||||||
|
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-follow-up-token",
|
||||||
|
Text = """{"type":"LISTEN","transID":"trans-follow-up","data":{"text":"hello jibo","rules":["wake-word"]}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
var contextReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-follow-up-token",
|
||||||
|
Text = """{"type":"CONTEXT","transID":"trans-follow-up","data":{"topic":"conversation","screen":"home"}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Single(contextReplies);
|
||||||
|
Assert.Equal("OPENJIBO_CONTEXT_ACK", ReadReplyType(contextReplies[0]));
|
||||||
|
|
||||||
|
var nluReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
|
||||||
|
{
|
||||||
|
HostName = "neo-hub.jibo.com",
|
||||||
|
Path = "/listen",
|
||||||
|
Kind = "neo-hub-listen",
|
||||||
|
Token = "hub-follow-up-token",
|
||||||
|
Text = """{"type":"CLIENT_NLU","transID":"trans-follow-up","data":{"intent":"joke"}}"""
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal(2, nluReplies.Count);
|
||||||
|
Assert.Equal("LISTEN", ReadReplyType(nluReplies[0]));
|
||||||
|
Assert.Equal("EOS", ReadReplyType(nluReplies[1]));
|
||||||
|
|
||||||
|
var session = _store.FindSessionByToken("hub-follow-up-token");
|
||||||
|
Assert.NotNull(session);
|
||||||
|
Assert.True(session!.FollowUpOpen);
|
||||||
|
Assert.Equal("joke", session.LastIntent);
|
||||||
|
Assert.Equal("trans-follow-up", session.LastTransId);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("fixtures\\neo-hub-client-asr-joke.flow.json")]
|
||||||
|
[InlineData("fixtures\\neo-hub-context-client-nlu.flow.json")]
|
||||||
|
public async Task WebSocketFixture_ReplaysSuccessfully(string relativePath)
|
||||||
|
{
|
||||||
|
var fixture = WebSocketFixtureLoader.Load(relativePath);
|
||||||
|
|
||||||
|
foreach (var step in fixture.Steps)
|
||||||
|
{
|
||||||
|
var replies = await _service.HandleMessageAsync(step.Message);
|
||||||
|
var actualTypes = replies.Select(ReadReplyType).ToArray();
|
||||||
|
Assert.Equal(step.ExpectedReplyTypes, actualTypes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ReadReplyType(WebSocketReply reply)
|
||||||
|
{
|
||||||
|
using var payload = JsonDocument.Parse(reply.Text!);
|
||||||
|
return payload.RootElement.GetProperty("type").GetString() ?? string.Empty;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user