Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactors

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-26 20:57:08 -05:00
parent acbba413db
commit 8c97968d95
20 changed files with 547 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
OpenJibo/OpenJibo.sln.DotSettings
View File

@@ -1,3 +1,18 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation"> <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/UserDictionary/Words/=ampm/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Arrrr/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=esml/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Hotphrase/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary> <s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo_0027s/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=multichunk/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=nevermind/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=noinput/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Photobooth/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slnx/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slowdance/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=timecoded/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Todays/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=whispercpp/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=YESNO/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

23
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Program.cs
View File

@@ -25,22 +25,15 @@ app.Use(async (context, next) =>
var kind = ResolveSocketKind(context.Request.Host.Host, context.Request.Path); var kind = ResolveSocketKind(context.Request.Host.Host, context.Request.Path);
var token = ResolveToken(context.Request); var token = ResolveToken(context.Request);
if (kind == "unknown") switch (kind)
{ {
context.Response.StatusCode = StatusCodes.Status404NotFound; case "unknown":
return; context.Response.StatusCode = StatusCodes.Status404NotFound;
} return;
case "api-socket" when string.IsNullOrWhiteSpace(token):
if (kind == "api-socket" && string.IsNullOrWhiteSpace(token)) case "neo-hub-listen" or "neo-hub-proactive" when string.IsNullOrWhiteSpace(token):
{ context.Response.StatusCode = StatusCodes.Status401Unauthorized;
context.Response.StatusCode = StatusCodes.Status401Unauthorized; return;
return;
}
if (kind is "neo-hub-listen" or "neo-hub-proactive" && string.IsNullOrWhiteSpace(token))
{
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
} }
var webSocketService = context.RequestServices.GetRequiredService<JiboWebSocketService>(); var webSocketService = context.RequestServices.GetRequiredService<JiboWebSocketService>();

123
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs
View File

@@ -194,27 +194,25 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}); });
} }
if (operation is "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or switch (operation)
"ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
"VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect")
{ {
return ProtocolDispatchResult.Ok(new case "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or
{ "ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
id = account.AccountId, "VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect":
email = account.Email, return ProtocolDispatchResult.Ok(new
firstName = account.FirstName, {
lastName = account.LastName, id = account.AccountId,
accessKeyId = account.AccessKeyId, email = account.Email,
secretAccessKey = account.SecretAccessKey firstName = account.FirstName,
}); lastName = account.LastName,
} accessKeyId = account.AccessKeyId,
secretAccessKey = account.SecretAccessKey
if (operation is "ChangeEmail" or "SendPhoneVerificationCode") });
{ case "ChangeEmail" or "SendPhoneVerificationCode":
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
id = account.AccountId id = account.AccountId
}); });
} }
if (operation.Equals("GetAccountByAccessToken", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("GetAccountByAccessToken", StringComparison.OrdinalIgnoreCase))
@@ -236,8 +234,8 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}".ToLowerInvariant(); var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}".ToLowerInvariant();
return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query) return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query)
? new[] ?
{ [
new new
{ {
id = account.AccountId, id = account.AccountId,
@@ -245,7 +243,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
firstName = account.FirstName, firstName = account.FirstName,
lastName = account.LastName lastName = account.LastName
} }
} ]
: Array.Empty<object>()); : Array.Empty<object>());
} }
@@ -382,25 +380,24 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray()); return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray());
} }
if (operation.Equals("Create", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
{ return ProtocolDispatchResult.Ok(Array.Empty<object>());
var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId;
var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
var reference = ReadHeader(envelope, "x-reference") ?? ReadString(body, "reference") ?? string.Empty;
var isEncrypted = ReadBooleanHeader(envelope, "x-encrypted") || ReadBool(body, "isEncrypted");
var meta = ReadObject(body, "meta") ?? new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream";
meta["contentType"] = contentType;
if (!string.IsNullOrWhiteSpace(envelope.BodyText))
{
meta["bodyText"] = envelope.BodyText;
}
return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta))); var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId;
var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
var reference = ReadHeader(envelope, "x-reference") ?? ReadString(body, "reference") ?? string.Empty;
var isEncrypted = ReadBooleanHeader(envelope, "x-encrypted") || ReadBool(body, "isEncrypted");
var meta = ReadObject(body, "meta") ?? new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream";
meta["contentType"] = contentType;
if (!string.IsNullOrWhiteSpace(envelope.BodyText))
{
meta["bodyText"] = envelope.BodyText;
} }
return ProtocolDispatchResult.Ok(Array.Empty<object>()); return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta)));
} }
private ProtocolDispatchResult HandlePerson(string operation) private ProtocolDispatchResult HandlePerson(string operation)
@@ -430,9 +427,10 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}); });
} }
string? symmetricKey;
if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase))
{ {
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId); symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
loopId, loopId,
@@ -472,18 +470,17 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(new { ok = true }); return ProtocolDispatchResult.Ok(new { ok = true });
} }
if (operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase))
{ return ProtocolDispatchResult.Ok(new { ok = true, operation });
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
{ return ProtocolDispatchResult.Ok(new
loopId, {
key = symmetricKey, loopId,
symmetricKey key = symmetricKey,
}); symmetricKey
} });
return ProtocolDispatchResult.Ok(new { ok = true, operation });
} }
private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope)
@@ -509,23 +506,22 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}); });
} }
if (operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase))
{
var profile = stateStore.GetRobotProfile();
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
id = ReadString(envelope.TryParseBody(), "id") ?? profile.RobotId, result = "ok"
payload = profile.Payload,
calibrationPayload = profile.CalibrationPayload,
updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(),
created = profile.CreatedUtc.ToUnixTimeMilliseconds()
}); });
}
var profile = stateStore.GetRobotProfile();
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
result = "ok" id = ReadString(envelope.TryParseBody(), "id") ?? profile.RobotId,
payload = profile.Payload,
calibrationPayload = profile.CalibrationPayload,
updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(),
created = profile.CreatedUtc.ToUnixTimeMilliseconds()
}); });
} }
private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope)
@@ -674,10 +670,9 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return []; return [];
} }
return property.EnumerateArray() return [.. property.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString()) .Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(item => !string.IsNullOrWhiteSpace(item)) .Where(item => !string.IsNullOrWhiteSpace(item))];
.ToArray();
} }
private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName) private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName)

150
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -484,14 +484,12 @@ public sealed class JiboInteractionService(
return "hello"; return "hello";
} }
if (isYesNoTurn && MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh")) switch (isYesNoTurn)
{ {
return "yes"; case true when MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh"):
} return "yes";
case true when MatchesAny(loweredTranscript, "no", "nope", "nah"):
if (isYesNoTurn && MatchesAny(loweredTranscript, "no", "nope", "nah")) return "no";
{
return "no";
} }
if (MatchesAny(loweredTranscript, "what time is it", "current time", "the time", "time is it") || if (MatchesAny(loweredTranscript, "what time is it", "current time", "the time", "time is it") ||
@@ -752,12 +750,7 @@ public sealed class JiboInteractionService(
} }
var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints); var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints);
if (!string.IsNullOrWhiteSpace(fuzzyHintMatch)) return !string.IsNullOrWhiteSpace(fuzzyHintMatch) ? fuzzyHintMatch : transcript;
{
return fuzzyHintMatch;
}
return transcript;
} }
private static bool IsYesNoTurn(TurnContext turn) private static bool IsYesNoTurn(TurnContext turn)
@@ -805,11 +798,10 @@ public sealed class JiboInteractionService(
} }
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint); var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance) if (distance >= bestDistance) continue;
{
bestDistance = distance; bestDistance = distance;
bestHint = hint; bestHint = hint;
}
} }
return bestDistance <= 2 ? bestHint : null; return bestDistance <= 2 ? bestHint : null;
@@ -996,14 +988,12 @@ public sealed class JiboInteractionService(
{ {
var compactHour = compact.Length switch var compactHour = compact.Length switch
{ {
3 => compactValue / 100, 3 or 4 => compactValue / 100,
4 => compactValue / 100,
_ => -1 _ => -1
}; };
var compactMinute = compact.Length switch var compactMinute = compact.Length switch
{ {
3 => compactValue % 100, 3 or 4 => compactValue % 100,
4 => compactValue % 100,
_ => -1 _ => -1
}; };
if (compactHour is >= 1 and <= 12 && compactMinute is >= 0 and <= 59) if (compactHour is >= 1 and <= 12 && compactMinute is >= 0 and <= 59)
@@ -1023,13 +1013,13 @@ public sealed class JiboInteractionService(
var hourToken = match.Groups["hour"].Value; var hourToken = match.Groups["hour"].Value;
var minuteToken = match.Groups["minute"].Success ? match.Groups["minute"].Value : "00"; var minuteToken = match.Groups["minute"].Success ? match.Groups["minute"].Value : "00";
var hour = ParseNumberToken(hourToken); var hour = ParseNumberToken(hourToken);
if (hour is null || hour is < 1 or > 12) if (hour is null or < 1 or > 12)
{ {
return null; return null;
} }
var minute = ParseNumberToken(minuteToken); var minute = ParseNumberToken(minuteToken);
if (minute is null || minute is < 0 or > 59) if (minute is null or < 0 or > 59)
{ {
return null; return null;
} }
@@ -1146,7 +1136,7 @@ public sealed class JiboInteractionService(
return lastClockDomain; return lastClockDomain;
} }
var combinedRules = clientRules.Concat(listenRules); var combinedRules = clientRules.Concat(listenRules).ToArray();
if (combinedRules.Any(rule => if (combinedRules.Any(rule =>
rule.Contains("timer", StringComparison.OrdinalIgnoreCase) && rule.Contains("timer", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase))) !rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)))
@@ -1154,14 +1144,9 @@ public sealed class JiboInteractionService(
return "timer"; return "timer";
} }
if (combinedRules.Any(rule => return combinedRules.Any(rule =>
rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) && rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase))) !rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)) ? "alarm" : null;
{
return "alarm";
}
return null;
} }
private static bool IsTimerRequest(string loweredTranscript) private static bool IsTimerRequest(string loweredTranscript)
@@ -1303,12 +1288,7 @@ public sealed class JiboInteractionService(
} }
var match = VolumeLevelPattern.Match(loweredTranscript); var match = VolumeLevelPattern.Match(loweredTranscript);
if (!match.Success) return !match.Success ? null : TryNormalizeVolumeLevel(match.Groups["value"].Value);
{
return null;
}
return TryNormalizeVolumeLevel(match.Groups["value"].Value);
} }
private static string? TryNormalizeVolumeLevel(string token) private static string? TryNormalizeVolumeLevel(string token)
@@ -1367,13 +1347,15 @@ public sealed class JiboInteractionService(
} }
var parts = valueToken.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); var parts = valueToken.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length >= 2) if (parts.Length < 2)
return parts.Length > 0
? ParseNumberToken(parts[^1])
: null;
parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2)));
if (parsed is not null)
{ {
parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2))); return parsed;
if (parsed is not null)
{
return parsed;
}
} }
return parts.Length > 0 return parts.Length > 0
@@ -1389,18 +1371,76 @@ public sealed class JiboInteractionService(
return numeric; return numeric;
} }
if (normalized.Contains(' ')) if (!normalized.Contains(' '))
{ {
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); return normalized switch
if (parts.Length == 2)
{ {
var first = ParseNumberToken(parts[0]); "a" or "an" => 1,
var second = ParseNumberToken(parts[1]); "one" => 1,
if (first is >= 20 && second is >= 0 and < 10) "two" => 2,
{ "three" => 3,
return first + second; "four" => 4,
} "five" => 5,
} "six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length != 2)
{
return normalized switch
{
"a" or "an" => 1,
"one" => 1,
"two" => 2,
"three" => 3,
"four" => 4,
"five" => 5,
"six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var first = ParseNumberToken(parts[0]);
var second = ParseNumberToken(parts[1]);
if (first is >= 20 && second is >= 0 and < 10)
{
return first + second;
} }
return normalized switch return normalized switch

71
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs
View File

@@ -32,47 +32,48 @@ public sealed class JiboWebSocketService(
var parsedType = ReadMessageType(envelope.Text); var parsedType = ReadMessageType(envelope.Text);
session.LastMessageType = parsedType; session.LastMessageType = parsedType;
turnFinalizationService.ObserveIncomingMessage(session, envelope.Text); WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
if (parsedType == "CONTEXT") switch (parsedType)
{ {
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken); case "CONTEXT":
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
{ {
["transID"] = session.TurnState.TransId var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
}, cancellationToken); await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
return replies; {
} ["transID"] = session.TurnState.TransId
}, cancellationToken);
if (parsedType == "LISTEN") return replies;
{ }
var replies = ContainsInlineTurnPayload(envelope.Text) case "LISTEN":
? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
: turnFinalizationService.HandleListenSetup(session, envelope);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
{ {
["messageType"] = parsedType, var replies = ContainsInlineTurnPayload(envelope.Text)
["replyCount"] = replies.Count, ? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
["transcript"] = session.LastTranscript, : WebSocketTurnFinalizationService.HandleListenSetup(session, envelope);
["intent"] = session.LastIntent await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
}, cancellationToken); {
return replies; ["messageType"] = parsedType,
} ["replyCount"] = replies.Count,
["transcript"] = session.LastTranscript,
if (parsedType is "CLIENT_NLU" or "CLIENT_ASR") ["intent"] = session.LastIntent
{ }, cancellationToken);
var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken); return replies;
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?> }
case "CLIENT_NLU" or "CLIENT_ASR":
{ {
["messageType"] = parsedType, var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken);
["replyCount"] = replies.Count, await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
["transcript"] = session.LastTranscript, {
["intent"] = session.LastIntent ["messageType"] = parsedType,
}, cancellationToken); ["replyCount"] = replies.Count,
return replies; ["transcript"] = session.LastTranscript,
["intent"] = session.LastIntent
}, cancellationToken);
return replies;
}
default:
return [];
} }
return [];
} }
private static string ReadMessageType(string? text) private static string ReadMessageType(string? text)

80
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs
View File

@@ -93,52 +93,46 @@ public sealed class ProtocolToTurnContextMapper
using var document = JsonDocument.Parse(text); using var document = JsonDocument.Parse(text);
var root = document.RootElement; var root = document.RootElement;
if (root.TryGetProperty("data", out var data)) if (!root.TryGetProperty("data", out var data)) return null;
if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String)
{ {
if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String) return transcript.GetString();
{
return transcript.GetString();
}
if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("text", out var asrText) &&
asrText.ValueKind == JsonValueKind.String)
{
return asrText.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String)
{
return transcriptHint.GetString();
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
attributes["clientIntent"] = intent.GetString();
}
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
attributes["clientRules"] = rules.EnumerateArray()
.Where(item => item.ValueKind == JsonValueKind.String)
.Select(item => item.GetString() ?? string.Empty)
.Where(rule => !string.IsNullOrWhiteSpace(rule))
.ToArray();
}
if (data.TryGetProperty("entities", out var entities) && entities.ValueKind == JsonValueKind.Object)
{
attributes["clientEntities"] = entities.Clone();
}
if (intent.ValueKind == JsonValueKind.String)
{
return intent.GetString();
}
} }
return null; if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("text", out var asrText) &&
asrText.ValueKind == JsonValueKind.String)
{
return asrText.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String)
{
return transcriptHint.GetString();
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
attributes["clientIntent"] = intent.GetString();
}
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
attributes["clientRules"] = rules.EnumerateArray()
.Where(item => item.ValueKind == JsonValueKind.String)
.Select(item => item.GetString() ?? string.Empty)
.Where(rule => !string.IsNullOrWhiteSpace(rule))
.ToArray();
}
if (data.TryGetProperty("entities", out var entities) && entities.ValueKind == JsonValueKind.Object)
{
attributes["clientEntities"] = entities.Clone();
}
return intent.ValueKind == JsonValueKind.String ? intent.GetString() : null;
} }
catch catch
{ {

178
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -6,7 +6,8 @@ namespace Jibo.Cloud.Application.Services;
public sealed class ResponsePlanToSocketMessagesMapper public sealed class ResponsePlanToSocketMessagesMapper
{ {
public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session, bool emitSkillActions) public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session,
bool emitSkillActions)
{ {
var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault(); var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault();
var skill = plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault(); var skill = plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault();
@@ -22,7 +23,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
var isYesNoIntent = string.Equals(plan.IntentName, "yes", StringComparison.OrdinalIgnoreCase) || var isYesNoIntent = string.Equals(plan.IntentName, "yes", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "no", StringComparison.OrdinalIgnoreCase); string.Equals(plan.IntentName, "no", StringComparison.OrdinalIgnoreCase);
var isWordOfDayLaunch = string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase); var isWordOfDayLaunch = string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase);
var isWordOfDayGuess = string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase); var isWordOfDayGuess =
string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) || var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase); string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase);
var isStopCommand = string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase); var isStopCommand = string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase);
@@ -53,56 +55,68 @@ public sealed class ResponsePlanToSocketMessagesMapper
var outboundIntent = isGlobalCommand && !string.IsNullOrWhiteSpace(globalIntent) var outboundIntent = isGlobalCommand && !string.IsNullOrWhiteSpace(globalIntent)
? globalIntent ? globalIntent
: isWordOfDayLaunch : isWordOfDayLaunch
? "menu" ? "menu"
: isRadioLaunch : isRadioLaunch
? "menu" ? "menu"
: isSettingsLaunch && !string.IsNullOrWhiteSpace(localIntent) : isSettingsLaunch && !string.IsNullOrWhiteSpace(localIntent)
? localIntent ? localIntent
: (isPhotoGalleryLaunch || isPhotoCreateLaunch) && !string.IsNullOrWhiteSpace(localIntent) : (isPhotoGalleryLaunch || isPhotoCreateLaunch) && !string.IsNullOrWhiteSpace(localIntent)
? localIntent ? localIntent
: isClockSkillLaunch && !string.IsNullOrWhiteSpace(clockIntent) : isClockSkillLaunch && !string.IsNullOrWhiteSpace(clockIntent)
? clockIntent ? clockIntent
: isWordOfDayGuess : isWordOfDayGuess
? "guess" ? "guess"
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent) : string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) &&
? clientIntent !string.IsNullOrWhiteSpace(clientIntent)
: plan.IntentName ?? "unknown"; ? clientIntent
: plan.IntentName ?? "unknown";
var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess) var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess)
? wordOfDayGuess ? wordOfDayGuess
: isWordOfDayLaunch : isWordOfDayLaunch
? string.Empty ? string.Empty
: isGlobalCommand : isGlobalCommand
? transcript ? transcript
: isRadioLaunch : isRadioLaunch
? transcript ? transcript
: isSettingsLaunch : isSettingsLaunch
? transcript ? transcript
: isPhotoGalleryLaunch || isPhotoCreateLaunch : isPhotoGalleryLaunch || isPhotoCreateLaunch
? transcript ? transcript
: isClockSkillLaunch : isClockSkillLaunch
? transcript ? transcript
: string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(nluGuess) : string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) &&
? nluGuess !string.IsNullOrWhiteSpace(nluGuess)
: isYesNoTurn && isYesNoIntent ? nluGuess
? transcript : isYesNoTurn && isYesNoIntent
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent) ? transcript
? clientIntent : string.Equals(messageType, "CLIENT_NLU",
: transcript; StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(clientIntent)
? clientIntent
: transcript;
var outboundRules = isWordOfDayLaunch var outboundRules = isWordOfDayLaunch
? ["word-of-the-day/menu"] ? ["word-of-the-day/menu"]
: isGlobalCommand : isGlobalCommand
? BuildGlobalCommandRules(rules) ? BuildGlobalCommandRules(rules)
: isRadioLaunch : isRadioLaunch
? Array.Empty<string>() ? []
: isSettingsLaunch : isSettingsLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>() ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
: isPhotoGalleryLaunch || isPhotoCreateLaunch ? rules
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>() : []
: isClockSkillLaunch : isPhotoGalleryLaunch || isPhotoCreateLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>() ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
: isWordOfDayGuess ? rules
? ["word-of-the-day/puzzle"] : []
: isYesNoTurn && isYesNoIntent ? [yesNoRule!] : rules; : isClockSkillLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isWordOfDayGuess
? ["word-of-the-day/puzzle"]
: isYesNoTurn && isYesNoIntent
? [yesNoRule!]
: rules;
var entities = ReadEntities( var entities = ReadEntities(
turn, turn,
messageType, messageType,
@@ -280,7 +294,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return messages; return messages;
} }
public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId, IReadOnlyList<string> rules) public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId,
IReadOnlyList<string> rules)
{ {
return return
[ [
@@ -376,12 +391,12 @@ public sealed class ResponsePlanToSocketMessagesMapper
var messages = new List<SocketReplyPlan>(MapNoInput(transId, rules)) var messages = new List<SocketReplyPlan>(MapNoInput(transId, rules))
{ {
new(JsonSerializer.Serialize(BuildSkillRedirectPayload( new(JsonSerializer.Serialize(BuildSkillRedirectPayload(
transId, transId,
skillId, skillId,
string.Empty, string.Empty,
string.Empty, string.Empty,
[], [],
new Dictionary<string, object?>())), new Dictionary<string, object?>())),
redirectDelayMs) redirectDelayMs)
}; };
@@ -402,7 +417,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch return value switch
{ {
IReadOnlyList<string> typedRules => typedRules, IReadOnlyList<string> typedRules => typedRules,
IEnumerable<string> rules => rules.Where(rule => !string.IsNullOrWhiteSpace(rule)).ToArray(), IEnumerable<string> rules => [.. rules.Where(rule => !string.IsNullOrWhiteSpace(rule))],
_ => [] _ => []
}; };
} }
@@ -487,12 +502,11 @@ public sealed class ResponsePlanToSocketMessagesMapper
entities["seconds"] = timerSeconds ?? "null"; entities["seconds"] = timerSeconds ?? "null";
} }
if (string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) && if (!string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) ||
(!string.IsNullOrWhiteSpace(alarmTime) || !string.IsNullOrWhiteSpace(alarmAmPm))) (string.IsNullOrWhiteSpace(alarmTime) && string.IsNullOrWhiteSpace(alarmAmPm))) return entities;
{
entities["time"] = alarmTime ?? string.Empty; entities["time"] = alarmTime ?? string.Empty;
entities["ampm"] = alarmAmPm ?? string.Empty; entities["ampm"] = alarmAmPm ?? string.Empty;
}
return entities; return entities;
} }
@@ -505,12 +519,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
}; };
} }
if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)) if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ||
{ !turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
return new Dictionary<string, object?>();
}
if (!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
{ {
return new Dictionary<string, object?>(); return new Dictionary<string, object?>();
} }
@@ -582,7 +592,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch return value switch
{ {
JsonElement { ValueKind: JsonValueKind.Object } jsonElement JsonElement { ValueKind: JsonValueKind.Object } jsonElement
when jsonElement.TryGetProperty(entityName, out var property) && property.ValueKind == JsonValueKind.String when jsonElement.TryGetProperty(entityName, out var property) &&
property.ValueKind == JsonValueKind.String
=> property.GetString(), => property.GetString(),
IReadOnlyDictionary<string, string> typed when typed.TryGetValue(entityName, out var entityValue) IReadOnlyDictionary<string, string> typed when typed.TryGetValue(entityName, out var entityValue)
=> entityValue, => entityValue,
@@ -602,7 +613,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value?.ToString(); return value?.ToString();
} }
private static string? ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess) private static string ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess)
{ {
if (!string.IsNullOrWhiteSpace(nluGuess)) if (!string.IsNullOrWhiteSpace(nluGuess))
{ {
@@ -662,11 +673,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
} }
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint); var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance) if (distance >= bestDistance) continue;
{
bestDistance = distance; bestDistance = distance;
bestHint = hint; bestHint = hint;
}
} }
return bestDistance <= 2 ? bestHint : null; return bestDistance <= 2 ? bestHint : null;
@@ -704,10 +714,12 @@ public sealed class ResponsePlanToSocketMessagesMapper
return previous[right.Length]; return previous[right.Length];
} }
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill) private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak,
InvokeNativeSkillAction? skill)
{ {
var skillPayload = skill?.Payload; var skillPayload = skill?.Payload;
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only", StringComparison.OrdinalIgnoreCase)) if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only",
StringComparison.OrdinalIgnoreCase))
{ {
return BuildCompletionOnlySkillPayload( return BuildCompletionOnlySkillPayload(
transId, transId,
@@ -718,12 +730,14 @@ public sealed class ResponsePlanToSocketMessagesMapper
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase); string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase); var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase);
var payloadSkill = ReadPayloadString(skillPayload, "skillId"); var payloadSkill = ReadPayloadString(skillPayload, "skillId");
var skillId = string.IsNullOrWhiteSpace(payloadSkill) ? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill" : payloadSkill; var skillId = string.IsNullOrWhiteSpace(payloadSkill)
? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill"
: payloadSkill;
var esml = ReadPayloadString(skillPayload, "esml") ?? (isDance var esml = ReadPayloadString(skillPayload, "esml") ?? (isDance
? "<speak>Okay.<break size='0.2'/> Watch this.<anim cat='dance' filter='music, rom-upbeat' /></speak>" ? "<speak>Okay.<break size='0.2'/> Watch this.<anim cat='dance' filter='music, rom-upbeat' /></speak>"
: isJoke : isJoke
? $"<speak><es cat='happy' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>" ? $"<speak><es cat='happy' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>"
: $"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>"); : $"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>");
var mimId = ReadPayloadString(skillPayload, "mim_id") ?? (isJoke ? "runtime-joke" : "runtime-chat"); var mimId = ReadPayloadString(skillPayload, "mim_id") ?? (isJoke ? "runtime-joke" : "runtime-chat");
var mimType = ReadPayloadString(skillPayload, "mim_type") ?? "announcement"; var mimType = ReadPayloadString(skillPayload, "mim_type") ?? "announcement";
@@ -799,9 +813,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static IReadOnlyList<string> BuildGlobalCommandRules(IReadOnlyList<string> rules) private static IReadOnlyList<string> BuildGlobalCommandRules(IReadOnlyList<string> rules)
{ {
return rules.Any(static rule => string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase)) return rules.Any(static rule =>
string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase))
? ["globals/global_commands_launch"] ? ["globals/global_commands_launch"]
: Array.Empty<string>(); : [];
} }
private static object BuildGenericFallbackSkillPayload(string transId) private static object BuildGenericFallbackSkillPayload(string transId)
@@ -829,7 +844,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
{ {
play = new play = new
{ {
esml = "<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>", esml =
"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>",
meta = new meta = new
{ {
prompt_id = "RUNTIME_PROMPT", prompt_id = "RUNTIME_PROMPT",

187
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -6,10 +6,8 @@ using System.Text.RegularExpressions;
namespace Jibo.Cloud.Application.Services; namespace Jibo.Cloud.Application.Services;
public sealed class WebSocketTurnFinalizationService( public sealed partial class WebSocketTurnFinalizationService(
ProtocolToTurnContextMapper turnContextMapper,
IConversationBroker conversationBroker, IConversationBroker conversationBroker,
ResponsePlanToSocketMessagesMapper replyMapper,
ISttStrategySelector sttStrategySelector, ISttStrategySelector sttStrategySelector,
ITurnTelemetrySink sink ITurnTelemetrySink sink
) )
@@ -18,7 +16,7 @@ public sealed class WebSocketTurnFinalizationService(
private const int AutoFinalizeMinBufferedAudioChunks = 4; private const int AutoFinalizeMinBufferedAudioChunks = 4;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400); private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400);
public void ObserveIncomingMessage(CloudSession session, string? text) public static void ObserveIncomingMessage(CloudSession session, string? text)
{ {
if (!TryReadTransId(text, out var nextTransId) || string.IsNullOrWhiteSpace(nextTransId)) if (!TryReadTransId(text, out var nextTransId) || string.IsNullOrWhiteSpace(nextTransId))
{ {
@@ -39,12 +37,7 @@ public sealed class WebSocketTurnFinalizationService(
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
var turnState = session.TurnState; var turnState = session.TurnState;
if (ShouldIgnoreLateAudio(session)) if (ShouldIgnoreLateAudio(session) || !turnState.AwaitingTurnCompletion &&
{
return [];
}
if (!turnState.AwaitingTurnCompletion &&
!session.FollowUpOpen && !session.FollowUpOpen &&
!turnState.SawListen && !turnState.SawListen &&
!string.IsNullOrWhiteSpace(turnState.TransId)) !string.IsNullOrWhiteSpace(turnState.TransId))
@@ -58,7 +51,7 @@ public sealed class WebSocketTurnFinalizationService(
turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0; turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0;
if (envelope.Binary is { Length: > 0 }) if (envelope.Binary is { Length: > 0 })
{ {
turnState.BufferedAudioFrames.Add(envelope.Binary.ToArray()); turnState.BufferedAudioFrames.Add([.. envelope.Binary]);
} }
turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow; turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow;
turnState.AwaitingTurnCompletion = true; turnState.AwaitingTurnCompletion = true;
@@ -116,7 +109,7 @@ public sealed class WebSocketTurnFinalizationService(
return await FinalizeTurnAsync(session, envelope, messageType, allowFallbackOnMissingTranscript: false, cancellationToken); return await FinalizeTurnAsync(session, envelope, messageType, allowFallbackOnMissingTranscript: false, cancellationToken);
} }
public IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope) public static IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope)
{ {
PersistTurnHints(session, envelope.Text); PersistTurnHints(session, envelope.Text);
@@ -129,7 +122,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session); ResetBufferedAudio(session);
session.TurnState.SawListen = false; session.TurnState.SawListen = false;
session.TurnState.SawContext = false; session.TurnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill( return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
session.TurnState.TransId ?? session.LastTransId ?? string.Empty, session.TurnState.TransId ?? session.LastTransId ?? string.Empty,
session.TurnState.ListenRules, session.TurnState.ListenRules,
"@be/idle") "@be/idle")
@@ -137,8 +130,7 @@ public sealed class WebSocketTurnFinalizationService(
{ {
Text = map.Text, Text = map.Text,
DelayMs = map.DelayMs DelayMs = map.DelayMs
}) })];
.ToArray();
} }
session.TurnState.AwaitingTurnCompletion = true; session.TurnState.AwaitingTurnCompletion = true;
@@ -147,17 +139,12 @@ public sealed class WebSocketTurnFinalizationService(
private async Task<TurnContext> ResolveTranscriptAsync(TurnContext turn, CloudSession session, CancellationToken cancellationToken) private async Task<TurnContext> ResolveTranscriptAsync(TurnContext turn, CloudSession session, CancellationToken cancellationToken)
{ {
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript)) if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript) || session.TurnState.BufferedAudioBytes <= 0)
{ {
return turn; return turn;
} }
if (session.TurnState.BufferedAudioBytes <= 0) ISttStrategy? strategy;
{
return turn;
}
ISttStrategy? strategy = null;
try try
{ {
strategy = await sttStrategySelector.SelectAsync(turn, cancellationToken); strategy = await sttStrategySelector.SelectAsync(turn, cancellationToken);
@@ -254,47 +241,44 @@ public sealed class WebSocketTurnFinalizationService(
} }
} }
if (root.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Object) if (!root.TryGetProperty("data", out var data) || data.ValueKind != JsonValueKind.Object) return;
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{ {
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array) turnState.ListenRules = [.. rules.EnumerateArray()
{
turnState.ListenRules = rules.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString()) .Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(rule => !string.IsNullOrWhiteSpace(rule)) .Where(rule => !string.IsNullOrWhiteSpace(rule))];
.ToArray(); session.Metadata["listenRules"] = turnState.ListenRules;
session.Metadata["listenRules"] = turnState.ListenRules;
}
if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("hints", out var hints) &&
hints.ValueKind == JsonValueKind.Array)
{
turnState.ListenAsrHints = hints.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty)
.Where(static hint => !string.IsNullOrWhiteSpace(hint))
.ToArray();
}
if (data.TryGetProperty("hotphrase", out var hotphrase) &&
(hotphrase.ValueKind == JsonValueKind.True || hotphrase.ValueKind == JsonValueKind.False))
{
turnState.ListenHotphrase = hotphrase.GetBoolean();
turnState.HotphraseEmptyTurnCount = 0;
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
session.LastIntent = intent.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String)
{
turnState.AudioTranscriptHint = transcriptHint.GetString();
session.Metadata["audioTranscriptHint"] = turnState.AudioTranscriptHint;
}
} }
if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("hints", out var hints) &&
hints.ValueKind == JsonValueKind.Array)
{
turnState.ListenAsrHints = [.. hints.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty)
.Where(static hint => !string.IsNullOrWhiteSpace(hint))];
}
if (data.TryGetProperty("hotphrase", out var hotphrase) &&
hotphrase.ValueKind is JsonValueKind.True or JsonValueKind.False)
{
turnState.ListenHotphrase = hotphrase.GetBoolean();
turnState.HotphraseEmptyTurnCount = 0;
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
session.LastIntent = intent.GetString();
}
if (!data.TryGetProperty("transcriptHint", out var transcriptHint) ||
transcriptHint.ValueKind != JsonValueKind.String) return;
turnState.AudioTranscriptHint = transcriptHint.GetString();
session.Metadata["audioTranscriptHint"] = turnState.AudioTranscriptHint;
} }
catch catch
{ {
@@ -394,7 +378,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session); ResetBufferedAudio(session);
turnState.SawListen = false; turnState.SawListen = false;
turnState.SawContext = false; turnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill( return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.TransId ?? session.LastTransId ?? string.Empty,
turnState.ListenRules, turnState.ListenRules,
"@be/idle") "@be/idle")
@@ -402,8 +386,7 @@ public sealed class WebSocketTurnFinalizationService(
{ {
Text = map.Text, Text = map.Text,
DelayMs = map.DelayMs DelayMs = map.DelayMs
}) })];
.ToArray();
} }
if (ShouldHandleAsLocalNoInput(finalizedTurn)) if (ShouldHandleAsLocalNoInput(finalizedTurn))
@@ -448,36 +431,38 @@ public sealed class WebSocketTurnFinalizationService(
turnState.FinalizeAttemptCount += 1; turnState.FinalizeAttemptCount += 1;
} }
if (allowFallbackOnMissingTranscript && switch (allowFallbackOnMissingTranscript)
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
IsYesNoTurn(finalizedTurn))
{ {
turnState.AwaitingTurnCompletion = false; case true when
session.LastTranscript = string.Empty; turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
session.LastIntent = null; IsYesNoTurn(finalizedTurn):
session.LastListenType = "no-input"; {
var localRule = ReadPrimaryYesNoRule(finalizedTurn); turnState.AwaitingTurnCompletion = false;
var noInputReplies = BuildLocalNoInputReplies(session, turnState, localRule); session.LastTranscript = string.Empty;
ResetBufferedAudio(session); session.LastIntent = null;
return noInputReplies; session.LastListenType = "no-input";
var localRule = ReadPrimaryYesNoRule(finalizedTurn);
var noInputReplies = BuildLocalNoInputReplies(session, turnState, localRule);
ResetBufferedAudio(session);
return noInputReplies;
}
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
string.IsNullOrWhiteSpace(turnState.LastSttError):
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = "heyJibo";
session.LastListenType = "fallback";
var fallbackReplies = ResponsePlanToSocketMessagesMapper.MapFallback(session, turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.ListenRules)
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
ResetBufferedAudio(session);
return fallbackReplies;
}
default:
return [];
} }
if (allowFallbackOnMissingTranscript &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
string.IsNullOrWhiteSpace(turnState.LastSttError))
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = "heyJibo";
session.LastListenType = "fallback";
var fallbackReplies = ResponsePlanToSocketMessagesMapper.MapFallback(session, turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.ListenRules)
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
ResetBufferedAudio(session);
return fallbackReplies;
}
return [];
} }
var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken); var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken);
@@ -487,7 +472,7 @@ public sealed class WebSocketTurnFinalizationService(
session.LastListenType = listenAction?.Mode; session.LastListenType = listenAction?.Mode;
turnState.LastLocalNoInputRule = null; turnState.LastLocalNoInputRule = null;
turnState.LocalNoInputCount = 0; turnState.LocalNoInputCount = 0;
if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock", Payload: not null } clockAction && if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock" } clockAction &&
clockAction.Payload.TryGetValue("domain", out var lastClockDomainValue) && clockAction.Payload.TryGetValue("domain", out var lastClockDomainValue) &&
lastClockDomainValue is not null) lastClockDomainValue is not null)
{ {
@@ -545,10 +530,7 @@ public sealed class WebSocketTurnFinalizationService(
var turnAge = turnState.FirstAudioReceivedUtc.HasValue var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value ? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero; : TimeSpan.Zero;
return turnState.AwaitingTurnCompletion && return turnState is { AwaitingTurnCompletion: true, SawListen: true, BufferedAudioChunkCount: >= AutoFinalizeMinBufferedAudioChunks, BufferedAudioBytes: >= AutoFinalizeMinBufferedAudioBytes } &&
turnState.SawListen &&
turnState.BufferedAudioChunkCount >= AutoFinalizeMinBufferedAudioChunks &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
turnAge >= AutoFinalizeMinTurnAge; turnAge >= AutoFinalizeMinTurnAge;
} }
@@ -755,7 +737,7 @@ public sealed class WebSocketTurnFinalizationService(
.FirstOrDefault(IsConstrainedYesNoRule); .FirstOrDefault(IsConstrainedYesNoRule);
} }
private static IReadOnlyList<WebSocketReply> BuildLocalNoInputReplies( private static WebSocketReply[] BuildLocalNoInputReplies(
CloudSession session, CloudSession session,
WebSocketTurnState turnState, WebSocketTurnState turnState,
string? localRule) string? localRule)
@@ -764,14 +746,12 @@ public sealed class WebSocketTurnFinalizationService(
var effectiveRule = string.IsNullOrWhiteSpace(localRule) var effectiveRule = string.IsNullOrWhiteSpace(localRule)
? turnState.ListenRules.FirstOrDefault(IsLocalNoInputRule) ? turnState.ListenRules.FirstOrDefault(IsLocalNoInputRule)
: localRule; : localRule;
IReadOnlyList<string> rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule]; var rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule];
var maps = ShouldRedirectRepeatedNoInputToIdle(turnState, effectiveRule) var maps = ShouldRedirectRepeatedNoInputToIdle(turnState, effectiveRule)
? ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(transId, rules, "@be/idle") ? ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(transId, rules, "@be/idle")
: ResponsePlanToSocketMessagesMapper.MapNoInput(transId, rules); : ResponsePlanToSocketMessagesMapper.MapNoInput(transId, rules);
return maps return [.. maps.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })];
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
} }
private static bool ShouldRedirectRepeatedNoInputToIdle(WebSocketTurnState turnState, string? localRule) private static bool ShouldRedirectRepeatedNoInputToIdle(WebSocketTurnState turnState, string? localRule)
@@ -853,7 +833,7 @@ public sealed class WebSocketTurnFinalizationService(
return string.Empty; return string.Empty;
} }
return Regex.Replace(transcript.Trim().ToLowerInvariant(), @"[^\w\s]", " ") return TranscriptNormalizationRegex().Replace(transcript.Trim().ToLowerInvariant(), " ")
.Replace(" ", " ", StringComparison.Ordinal) .Replace(" ", " ", StringComparison.Ordinal)
.Trim(); .Trim();
} }
@@ -1036,4 +1016,7 @@ public sealed class WebSocketTurnFinalizationService(
_ => false _ => false
}; };
} }
[GeneratedRegex(@"[^\w\s]")]
private static partial Regex TranscriptNormalizationRegex();
} }

16
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/ExternalProcessRunner.cs
View File

@@ -6,16 +6,14 @@ public sealed class ExternalProcessRunner : IExternalProcessRunner
{ {
public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default) public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default)
{ {
using var process = new Process using var process = new Process();
process.StartInfo = new ProcessStartInfo
{ {
StartInfo = new ProcessStartInfo FileName = fileName,
{ RedirectStandardOutput = true,
FileName = fileName, RedirectStandardError = true,
RedirectStandardOutput = true, UseShellExecute = false,
RedirectStandardError = true, CreateNoWindow = true
UseShellExecute = false,
CreateNoWindow = true
}
}; };
foreach (var argument in arguments) foreach (var argument in arguments)

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs
View File

@@ -132,7 +132,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var timecoded = lines var timecoded = lines
.Where(static line => line.StartsWith("[", StringComparison.Ordinal) && line.Contains("-->", StringComparison.Ordinal)) .Where(static line => line.StartsWith('[') && line.Contains("-->", StringComparison.Ordinal))
.Select(static line => .Select(static line =>
{ {
var closingBracket = line.IndexOf(']'); var closingBracket = line.IndexOf(']');
@@ -171,6 +171,6 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
return true; return true;
} }
return checkFileExists ? File.Exists(path) : true; return !checkFileExists || File.Exists(path);
} }
} }

8
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/OggOpusAudioNormalizer.cs
View File

@@ -79,13 +79,7 @@ internal static class OggOpusAudioNormalizer
private static uint ComputeCrc(byte[] buffer) private static uint ComputeCrc(byte[] buffer)
{ {
uint crc = 0; return buffer.Aggregate<byte, uint>(0, (current, value) => (current << 8) ^ CrcTable[((current >> 24) ^ value) & 0xff]);
foreach (var value in buffer)
{
crc = (crc << 8) ^ CrcTable[((crc >> 24) ^ value) & 0xff];
}
return crc;
} }
private static uint[] BuildCrcTable() private static uint[] BuildCrcTable()

1
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs
View File

@@ -7,7 +7,6 @@ using Jibo.Cloud.Infrastructure.Telemetry;
using Jibo.Runtime.Abstractions; using Jibo.Runtime.Abstractions;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using System.IO;
namespace Jibo.Cloud.Infrastructure.DependencyInjection; namespace Jibo.Cloud.Infrastructure.DependencyInjection;

29
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryCloudStateStore.cs
View File

@@ -18,7 +18,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
private readonly ConcurrentDictionary<string, string> _symmetricKeys = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary<string, string> _symmetricKeys = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, KeyRequestRecord> _keyRequests = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary<string, KeyRequestRecord> _keyRequests = new(StringComparer.OrdinalIgnoreCase);
private readonly string? _persistencePath; private readonly string? _persistencePath;
private readonly object _syncRoot = new(); private readonly Lock _syncRoot = new();
private readonly List<UpdateManifest> _updates; private readonly List<UpdateManifest> _updates;
private readonly List<MediaRecord> _media = []; private readonly List<MediaRecord> _media = [];
private readonly List<BackupRecord> _backups = []; private readonly List<BackupRecord> _backups = [];
@@ -186,21 +186,20 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
public UpdateManifest RemoveUpdate(string? updateId) public UpdateManifest RemoveUpdate(string? updateId)
{ {
var existing = _updates.FirstOrDefault(update => update.UpdateId == updateId); var existing = _updates.FirstOrDefault(update => update.UpdateId == updateId);
if (existing is not null) if (existing is null)
{ return new UpdateManifest
_updates.Remove(existing); {
PersistState(); UpdateId = updateId ?? "unknown-update",
return existing; Changes = "Update not found",
} Url = "https://api.jibo.com/update/missing",
ShaHash = "missing",
Subsystem = "unknown"
};
_updates.Remove(existing);
PersistState();
return existing;
return new UpdateManifest
{
UpdateId = updateId ?? "unknown-update",
Changes = "Update not found",
Url = "https://api.jibo.com/update/missing",
ShaHash = "missing",
Subsystem = "unknown"
};
} }
public IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null) public IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null)

18
OpenJibo/src/Playground/AsrEvent.cs Normal file
View File

@@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}

12
OpenJibo/src/Playground/AsrUtterance.cs Normal file
View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
}

42
OpenJibo/src/Playground/Program.cs
View File

@@ -2,7 +2,7 @@
using System.Net.WebSockets; using System.Net.WebSockets;
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Serialization; using Playground;
Console.Write("Enter Jibo IP: "); Console.Write("Enter Jibo IP: ");
var jiboIp = (Console.ReadLine() ?? "").Trim(); var jiboIp = (Console.ReadLine() ?? "").Trim();
@@ -67,7 +67,7 @@ while (!cts.IsCancellationRequested)
var json = Encoding.UTF8.GetString(ms.ToArray()); var json = Encoding.UTF8.GetString(ms.ToArray());
AsrEvent? evt = null; AsrEvent? evt;
try try
{ {
evt = JsonSerializer.Deserialize<AsrEvent>(json); evt = JsonSerializer.Deserialize<AsrEvent>(json);
@@ -86,15 +86,11 @@ while (!cts.IsCancellationRequested)
Console.WriteLine($"[{evt.EventType}] {json}"); Console.WriteLine($"[{evt.EventType}] {json}");
if (evt.EventType == "speech_to_text_final") if (evt.EventType != "speech_to_text_final") continue;
{ var best = PickBestUtterance(evt.Utterances);
var best = PickBestUtterance(evt.Utterances); if (string.IsNullOrWhiteSpace(best)) continue;
if (!string.IsNullOrWhiteSpace(best)) utteranceTcs.TrySetResult(best);
{ return;
utteranceTcs.TrySetResult(best);
return;
}
}
} }
}, cts.Token); }, cts.Token);
@@ -219,28 +215,4 @@ static string BuildReply(string heard)
return "Hello! I heard you loud and clear."; return "Hello! I heard you loud and clear.";
return text.Contains("your name") ? "I am Jibo, running with a local demo bridge." : $"You said: {heard}"; return text.Contains("your name") ? "I am Jibo, running with a local demo bridge." : $"You said: {heard}";
}
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
} }

16
OpenJibo/tests/Jibo.Cloud.Tests/Fixtures/WebSocketFixtureLoader.cs
View File

@@ -17,10 +17,9 @@ internal static class WebSocketFixtureLoader
var root = document.RootElement; var root = document.RootElement;
var session = root.GetProperty("session"); var session = root.GetProperty("session");
var steps = new List<WebSocketFixtureStep>(); var steps = root.GetProperty("steps")
foreach (var stepElement in root.GetProperty("steps").EnumerateArray()) .EnumerateArray()
{ .Select(stepElement => new WebSocketFixtureStep
steps.Add(new WebSocketFixtureStep
{ {
Message = new WebSocketMessageEnvelope Message = new WebSocketMessageEnvelope
{ {
@@ -33,16 +32,15 @@ internal static class WebSocketFixtureLoader
? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray() ? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray()
: null : null
}, },
ExpectedReplyTypes = stepElement.GetProperty("expectedReplyTypes") ExpectedReplyTypes = [.. stepElement.GetProperty("expectedReplyTypes")
.EnumerateArray() .EnumerateArray()
.Select(item => item.GetString() ?? string.Empty) .Select(item => item.GetString() ?? string.Empty)
.Where(item => !string.IsNullOrWhiteSpace(item)) .Where(item => !string.IsNullOrWhiteSpace(item))],
.ToArray(),
ExpectedReplies = stepElement.TryGetProperty("expectedReplies", out var expectedReplies) && expectedReplies.ValueKind == JsonValueKind.Array ExpectedReplies = stepElement.TryGetProperty("expectedReplies", out var expectedReplies) && expectedReplies.ValueKind == JsonValueKind.Array
? JsonSerializer.Deserialize<List<ExpectedWebSocketReply>>(expectedReplies.GetRawText(), SerializerOptions) ?? [] ? JsonSerializer.Deserialize<List<ExpectedWebSocketReply>>(expectedReplies.GetRawText(), SerializerOptions) ?? []
: [] : []
}); })
} .ToList();
return new WebSocketFixture return new WebSocketFixture
{ {

40
OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs
View File

@@ -16,18 +16,18 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>())) sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Exception("dummy")); .ThrowsAsync(new Exception("dummy"));
var turnService = new WebSocketTurnFinalizationService( var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
sttStrategySelector.Object, sttStrategySelector.Object,
sink.Object sink.Object
); );
await turnService.HandleTurnAsync(new CloudSession() { TurnState = { BufferedAudioBytes = 100 }}, new WebSocketMessageEnvelope(), "dummy", await turnService.HandleTurnAsync(new CloudSession { TurnState = { BufferedAudioBytes = 100 } },
new WebSocketMessageEnvelope(), "dummy",
CancellationToken.None); CancellationToken.None);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once()); sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
} }
[Fact] [Fact]
@@ -38,21 +38,23 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>())) sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new InvalidOperationException("ffmpeg failed")); .ThrowsAsync(new InvalidOperationException("ffmpeg failed"));
var turnService = new WebSocketTurnFinalizationService( var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
sttStrategySelector.Object, sttStrategySelector.Object,
sink.Object sink.Object
); );
var session = new CloudSession(); var session = new CloudSession
session.TurnState.AwaitingTurnCompletion = true; {
session.TurnState.SawListen = true; TurnState =
session.TurnState.SawContext = true; {
session.TurnState.BufferedAudioBytes = 12000; AwaitingTurnCompletion = true,
session.TurnState.BufferedAudioChunkCount = 5; SawListen = true,
session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); SawContext = true,
BufferedAudioBytes = 12000,
BufferedAudioChunkCount = 5,
FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2)
}
};
var replies = await turnService.HandleContextAsync( var replies = await turnService.HandleContextAsync(
session, session,
@@ -64,6 +66,8 @@ public sealed class FileTurnTelemetrySinkTests
Assert.Equal(12000, session.TurnState.BufferedAudioBytes); Assert.Equal(12000, session.TurnState.BufferedAudioBytes);
Assert.Equal("ffmpeg failed", session.TurnState.LastSttError); Assert.Equal("ffmpeg failed", session.TurnState.LastSttError);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once()); sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
} }
} }

40
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -16,11 +16,9 @@ public sealed class JiboWebSocketServiceTests
public JiboWebSocketServiceTests() public JiboWebSocketServiceTests()
{ {
_store = new InMemoryCloudStateStore(); _store = new InMemoryCloudStateStore();
var turnContextMapper = new ProtocolToTurnContextMapper();
var contentRepository = new InMemoryJiboExperienceContentRepository(); var contentRepository = new InMemoryJiboExperienceContentRepository();
var contentCache = new JiboExperienceContentCache(contentRepository); var contentCache = new JiboExperienceContentCache(contentRepository);
var conversationBroker = new DemoConversationBroker(new JiboInteractionService(contentCache, new DefaultJiboRandomizer())); var conversationBroker = new DemoConversationBroker(new JiboInteractionService(contentCache, new DefaultJiboRandomizer()));
var replyMapper = new ResponsePlanToSocketMessagesMapper();
var sttSelector = new DefaultSttStrategySelector( var sttSelector = new DefaultSttStrategySelector(
[ [
new SyntheticBufferedAudioSttStrategy() new SyntheticBufferedAudioSttStrategy()
@@ -30,10 +28,7 @@ public sealed class JiboWebSocketServiceTests
_service = new JiboWebSocketService( _service = new JiboWebSocketService(
_store, _store,
new NullWebSocketTelemetrySink(), new NullWebSocketTelemetrySink(),
new WebSocketTurnFinalizationService( new WebSocketTurnFinalizationService(conversationBroker,
turnContextMapper,
conversationBroker,
replyMapper,
sttSelector, sttSelector,
sink)); sink));
} }
@@ -2639,7 +2634,7 @@ public sealed class JiboWebSocketServiceTests
Path = "/listen", Path = "/listen",
Kind = "neo-hub-listen", Kind = "neo-hub-listen",
Token = "hub-context-reset-token", Token = "hub-context-reset-token",
Binary = [9, 9, 9, 9] Binary = "\t\t\t\t"u8.ToArray()
}); });
var session = _store.FindSessionByToken("hub-context-reset-token"); var session = _store.FindSessionByToken("hub-context-reset-token");
@@ -2681,26 +2676,24 @@ public sealed class JiboWebSocketServiceTests
var actualTypes = replies.Select(ReadReplyType).ToArray(); var actualTypes = replies.Select(ReadReplyType).ToArray();
Assert.Equal(step.ExpectedReplyTypes, actualTypes); Assert.Equal(step.ExpectedReplyTypes, actualTypes);
if (step.ExpectedReplies.Count > 0) if (step.ExpectedReplies.Count <= 0) continue;
Assert.Equal(replies.Count, step.ExpectedReplies.Count);
for (var index = 0; index < step.ExpectedReplies.Count; index += 1)
{ {
Assert.Equal(replies.Count, step.ExpectedReplies.Count); var expectedReply = step.ExpectedReplies[index];
Assert.Equal(expectedReply.Type, actualTypes[index]);
for (var index = 0; index < step.ExpectedReplies.Count; index += 1) if (expectedReply.DelayMs.HasValue)
{ {
var expectedReply = step.ExpectedReplies[index]; Assert.Equal(expectedReply.DelayMs.Value, replies[index].DelayMs);
Assert.Equal(expectedReply.Type, actualTypes[index]);
if (expectedReply.DelayMs.HasValue)
{
Assert.Equal(expectedReply.DelayMs.Value, replies[index].DelayMs);
}
if (expectedReply.JsonSubset is { ValueKind: JsonValueKind.Object } jsonSubset)
{
using var actualPayload = JsonDocument.Parse(replies[index].Text!);
AssertJsonContains(jsonSubset, actualPayload.RootElement);
}
} }
if (expectedReply.JsonSubset is not { ValueKind: JsonValueKind.Object } jsonSubset) continue;
using var actualPayload = JsonDocument.Parse(replies[index].Text!);
AssertJsonContains(jsonSubset, actualPayload.RootElement);
} }
} }
} }
@@ -2709,6 +2702,7 @@ public sealed class JiboWebSocketServiceTests
{ {
Assert.Equal(expected.ValueKind, actual.ValueKind); Assert.Equal(expected.ValueKind, actual.ValueKind);
// ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault
switch (expected.ValueKind) switch (expected.ValueKind)
{ {
case JsonValueKind.Object: case JsonValueKind.Object:

14
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs
View File

@@ -158,14 +158,14 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests
{ {
Calls.Add((fileName, arguments)); Calls.Add((fileName, arguments));
if (string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase)) if (!string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase))
{ return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke",
var outputPath = arguments[^1]; string.Empty));
File.WriteAllBytes(outputPath, "RIFF"u8);
return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty)); var outputPath = arguments[^1];
} File.WriteAllBytes(outputPath, "RIFF"u8);
return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty));
return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke", string.Empty));
} }
} }
} }