Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactors

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-26 20:57:08 -05:00
parent acbba413db
commit 8c97968d95
20 changed files with 547 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
OpenJibo/OpenJibo.sln.DotSettings
View File

@@ -1,3 +1,18 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation"> <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/UserDictionary/Words/=ampm/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Arrrr/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=esml/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Hotphrase/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary> <s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo_0027s/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=multichunk/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=nevermind/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=noinput/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Photobooth/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slnx/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slowdance/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=timecoded/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Todays/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=whispercpp/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=YESNO/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

15
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Program.cs
View File

@@ -25,20 +25,13 @@ app.Use(async (context, next) =>
var kind = ResolveSocketKind(context.Request.Host.Host, context.Request.Path); var kind = ResolveSocketKind(context.Request.Host.Host, context.Request.Path);
var token = ResolveToken(context.Request); var token = ResolveToken(context.Request);
if (kind == "unknown") switch (kind)
{ {
case "unknown":
context.Response.StatusCode = StatusCodes.Status404NotFound; context.Response.StatusCode = StatusCodes.Status404NotFound;
return; return;
} case "api-socket" when string.IsNullOrWhiteSpace(token):
case "neo-hub-listen" or "neo-hub-proactive" when string.IsNullOrWhiteSpace(token):
if (kind == "api-socket" && string.IsNullOrWhiteSpace(token))
{
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
}
if (kind is "neo-hub-listen" or "neo-hub-proactive" && string.IsNullOrWhiteSpace(token))
{
context.Response.StatusCode = StatusCodes.Status401Unauthorized; context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return; return;
} }

53
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs
View File

@@ -194,10 +194,11 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}); });
} }
if (operation is "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or switch (operation)
"ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
"VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect")
{ {
case "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or
"ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
"VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect":
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
id = account.AccountId, id = account.AccountId,
@@ -207,10 +208,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
accessKeyId = account.AccessKeyId, accessKeyId = account.AccessKeyId,
secretAccessKey = account.SecretAccessKey secretAccessKey = account.SecretAccessKey
}); });
} case "ChangeEmail" or "SendPhoneVerificationCode":
if (operation is "ChangeEmail" or "SendPhoneVerificationCode")
{
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
id = account.AccountId id = account.AccountId
@@ -236,8 +234,8 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}".ToLowerInvariant(); var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}".ToLowerInvariant();
return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query) return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query)
? new[] ?
{ [
new new
{ {
id = account.AccountId, id = account.AccountId,
@@ -245,7 +243,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
firstName = account.FirstName, firstName = account.FirstName,
lastName = account.LastName lastName = account.LastName
} }
} ]
: Array.Empty<object>()); : Array.Empty<object>());
} }
@@ -382,8 +380,9 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray()); return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray());
} }
if (operation.Equals("Create", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
{ return ProtocolDispatchResult.Ok(Array.Empty<object>());
var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId; var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId;
var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown"; var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
@@ -398,9 +397,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
} }
return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta))); return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta)));
}
return ProtocolDispatchResult.Ok(Array.Empty<object>());
} }
private ProtocolDispatchResult HandlePerson(string operation) private ProtocolDispatchResult HandlePerson(string operation)
@@ -430,9 +427,10 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}); });
} }
string? symmetricKey;
if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase)) if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase))
{ {
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId); symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
loopId, loopId,
@@ -472,18 +470,17 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(new { ok = true }); return ProtocolDispatchResult.Ok(new { ok = true });
} }
if (operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase))
{ return ProtocolDispatchResult.Ok(new { ok = true, operation });
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
loopId, loopId,
key = symmetricKey, key = symmetricKey,
symmetricKey symmetricKey
}); });
}
return ProtocolDispatchResult.Ok(new { ok = true, operation });
} }
private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope)
@@ -509,8 +506,12 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}); });
} }
if (operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase)) if (!operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(new
{ {
result = "ok"
});
var profile = stateStore.GetRobotProfile(); var profile = stateStore.GetRobotProfile();
return ProtocolDispatchResult.Ok(new return ProtocolDispatchResult.Ok(new
{ {
@@ -520,12 +521,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(), updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(),
created = profile.CreatedUtc.ToUnixTimeMilliseconds() created = profile.CreatedUtc.ToUnixTimeMilliseconds()
}); });
}
return ProtocolDispatchResult.Ok(new
{
result = "ok"
});
} }
private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope) private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope)
@@ -674,10 +670,9 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return []; return [];
} }
return property.EnumerateArray() return [.. property.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString()) .Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(item => !string.IsNullOrWhiteSpace(item)) .Where(item => !string.IsNullOrWhiteSpace(item))];
.ToArray();
} }
private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName) private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName)

122
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -484,13 +484,11 @@ public sealed class JiboInteractionService(
return "hello"; return "hello";
} }
if (isYesNoTurn && MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh")) switch (isYesNoTurn)
{ {
case true when MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh"):
return "yes"; return "yes";
} case true when MatchesAny(loweredTranscript, "no", "nope", "nah"):
if (isYesNoTurn && MatchesAny(loweredTranscript, "no", "nope", "nah"))
{
return "no"; return "no";
} }
@@ -752,12 +750,7 @@ public sealed class JiboInteractionService(
} }
var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints); var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints);
if (!string.IsNullOrWhiteSpace(fuzzyHintMatch)) return !string.IsNullOrWhiteSpace(fuzzyHintMatch) ? fuzzyHintMatch : transcript;
{
return fuzzyHintMatch;
}
return transcript;
} }
private static bool IsYesNoTurn(TurnContext turn) private static bool IsYesNoTurn(TurnContext turn)
@@ -805,12 +798,11 @@ public sealed class JiboInteractionService(
} }
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint); var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance) if (distance >= bestDistance) continue;
{
bestDistance = distance; bestDistance = distance;
bestHint = hint; bestHint = hint;
} }
}
return bestDistance <= 2 ? bestHint : null; return bestDistance <= 2 ? bestHint : null;
} }
@@ -996,14 +988,12 @@ public sealed class JiboInteractionService(
{ {
var compactHour = compact.Length switch var compactHour = compact.Length switch
{ {
3 => compactValue / 100, 3 or 4 => compactValue / 100,
4 => compactValue / 100,
_ => -1 _ => -1
}; };
var compactMinute = compact.Length switch var compactMinute = compact.Length switch
{ {
3 => compactValue % 100, 3 or 4 => compactValue % 100,
4 => compactValue % 100,
_ => -1 _ => -1
}; };
if (compactHour is >= 1 and <= 12 && compactMinute is >= 0 and <= 59) if (compactHour is >= 1 and <= 12 && compactMinute is >= 0 and <= 59)
@@ -1023,13 +1013,13 @@ public sealed class JiboInteractionService(
var hourToken = match.Groups["hour"].Value; var hourToken = match.Groups["hour"].Value;
var minuteToken = match.Groups["minute"].Success ? match.Groups["minute"].Value : "00"; var minuteToken = match.Groups["minute"].Success ? match.Groups["minute"].Value : "00";
var hour = ParseNumberToken(hourToken); var hour = ParseNumberToken(hourToken);
if (hour is null || hour is < 1 or > 12) if (hour is null or < 1 or > 12)
{ {
return null; return null;
} }
var minute = ParseNumberToken(minuteToken); var minute = ParseNumberToken(minuteToken);
if (minute is null || minute is < 0 or > 59) if (minute is null or < 0 or > 59)
{ {
return null; return null;
} }
@@ -1146,7 +1136,7 @@ public sealed class JiboInteractionService(
return lastClockDomain; return lastClockDomain;
} }
var combinedRules = clientRules.Concat(listenRules); var combinedRules = clientRules.Concat(listenRules).ToArray();
if (combinedRules.Any(rule => if (combinedRules.Any(rule =>
rule.Contains("timer", StringComparison.OrdinalIgnoreCase) && rule.Contains("timer", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase))) !rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)))
@@ -1154,14 +1144,9 @@ public sealed class JiboInteractionService(
return "timer"; return "timer";
} }
if (combinedRules.Any(rule => return combinedRules.Any(rule =>
rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) && rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase))) !rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)) ? "alarm" : null;
{
return "alarm";
}
return null;
} }
private static bool IsTimerRequest(string loweredTranscript) private static bool IsTimerRequest(string loweredTranscript)
@@ -1303,12 +1288,7 @@ public sealed class JiboInteractionService(
} }
var match = VolumeLevelPattern.Match(loweredTranscript); var match = VolumeLevelPattern.Match(loweredTranscript);
if (!match.Success) return !match.Success ? null : TryNormalizeVolumeLevel(match.Groups["value"].Value);
{
return null;
}
return TryNormalizeVolumeLevel(match.Groups["value"].Value);
} }
private static string? TryNormalizeVolumeLevel(string token) private static string? TryNormalizeVolumeLevel(string token)
@@ -1367,14 +1347,16 @@ public sealed class JiboInteractionService(
} }
var parts = valueToken.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); var parts = valueToken.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length >= 2) if (parts.Length < 2)
{ return parts.Length > 0
? ParseNumberToken(parts[^1])
: null;
parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2))); parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2)));
if (parsed is not null) if (parsed is not null)
{ {
return parsed; return parsed;
} }
}
return parts.Length > 0 return parts.Length > 0
? ParseNumberToken(parts[^1]) ? ParseNumberToken(parts[^1])
@@ -1389,19 +1371,77 @@ public sealed class JiboInteractionService(
return numeric; return numeric;
} }
if (normalized.Contains(' ')) if (!normalized.Contains(' '))
{ {
return normalized switch
{
"a" or "an" => 1,
"one" => 1,
"two" => 2,
"three" => 3,
"four" => 4,
"five" => 5,
"six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length == 2) if (parts.Length != 2)
{ {
return normalized switch
{
"a" or "an" => 1,
"one" => 1,
"two" => 2,
"three" => 3,
"four" => 4,
"five" => 5,
"six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var first = ParseNumberToken(parts[0]); var first = ParseNumberToken(parts[0]);
var second = ParseNumberToken(parts[1]); var second = ParseNumberToken(parts[1]);
if (first is >= 20 && second is >= 0 and < 10) if (first is >= 20 && second is >= 0 and < 10)
{ {
return first + second; return first + second;
} }
}
}
return normalized switch return normalized switch
{ {

17
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs
View File

@@ -32,9 +32,11 @@ public sealed class JiboWebSocketService(
var parsedType = ReadMessageType(envelope.Text); var parsedType = ReadMessageType(envelope.Text);
session.LastMessageType = parsedType; session.LastMessageType = parsedType;
turnFinalizationService.ObserveIncomingMessage(session, envelope.Text); WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
if (parsedType == "CONTEXT") switch (parsedType)
{
case "CONTEXT":
{ {
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken); var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
@@ -43,12 +45,11 @@ public sealed class JiboWebSocketService(
}, cancellationToken); }, cancellationToken);
return replies; return replies;
} }
case "LISTEN":
if (parsedType == "LISTEN")
{ {
var replies = ContainsInlineTurnPayload(envelope.Text) var replies = ContainsInlineTurnPayload(envelope.Text)
? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken) ? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
: turnFinalizationService.HandleListenSetup(session, envelope); : WebSocketTurnFinalizationService.HandleListenSetup(session, envelope);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
{ {
["messageType"] = parsedType, ["messageType"] = parsedType,
@@ -58,8 +59,7 @@ public sealed class JiboWebSocketService(
}, cancellationToken); }, cancellationToken);
return replies; return replies;
} }
case "CLIENT_NLU" or "CLIENT_ASR":
if (parsedType is "CLIENT_NLU" or "CLIENT_ASR")
{ {
var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken); var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?> await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
@@ -71,9 +71,10 @@ public sealed class JiboWebSocketService(
}, cancellationToken); }, cancellationToken);
return replies; return replies;
} }
default:
return []; return [];
} }
}
private static string ReadMessageType(string? text) private static string ReadMessageType(string? text)
{ {

12
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs
View File

@@ -93,8 +93,8 @@ public sealed class ProtocolToTurnContextMapper
using var document = JsonDocument.Parse(text); using var document = JsonDocument.Parse(text);
var root = document.RootElement; var root = document.RootElement;
if (root.TryGetProperty("data", out var data)) if (!root.TryGetProperty("data", out var data)) return null;
{
if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String) if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String)
{ {
return transcript.GetString(); return transcript.GetString();
@@ -132,13 +132,7 @@ public sealed class ProtocolToTurnContextMapper
attributes["clientEntities"] = entities.Clone(); attributes["clientEntities"] = entities.Clone();
} }
if (intent.ValueKind == JsonValueKind.String) return intent.ValueKind == JsonValueKind.String ? intent.GetString() : null;
{
return intent.GetString();
}
}
return null;
} }
catch catch
{ {

82
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -6,7 +6,8 @@ namespace Jibo.Cloud.Application.Services;
public sealed class ResponsePlanToSocketMessagesMapper public sealed class ResponsePlanToSocketMessagesMapper
{ {
public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session, bool emitSkillActions) public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session,
bool emitSkillActions)
{ {
var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault(); var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault();
var skill = plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault(); var skill = plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault();
@@ -22,7 +23,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
var isYesNoIntent = string.Equals(plan.IntentName, "yes", StringComparison.OrdinalIgnoreCase) || var isYesNoIntent = string.Equals(plan.IntentName, "yes", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "no", StringComparison.OrdinalIgnoreCase); string.Equals(plan.IntentName, "no", StringComparison.OrdinalIgnoreCase);
var isWordOfDayLaunch = string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase); var isWordOfDayLaunch = string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase);
var isWordOfDayGuess = string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase); var isWordOfDayGuess =
string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) || var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase); string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase);
var isStopCommand = string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase); var isStopCommand = string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase);
@@ -64,7 +66,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
? clockIntent ? clockIntent
: isWordOfDayGuess : isWordOfDayGuess
? "guess" ? "guess"
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent) : string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(clientIntent)
? clientIntent ? clientIntent
: plan.IntentName ?? "unknown"; : plan.IntentName ?? "unknown";
var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess) var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess)
@@ -81,11 +84,14 @@ public sealed class ResponsePlanToSocketMessagesMapper
? transcript ? transcript
: isClockSkillLaunch : isClockSkillLaunch
? transcript ? transcript
: string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(nluGuess) : string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(nluGuess)
? nluGuess ? nluGuess
: isYesNoTurn && isYesNoIntent : isYesNoTurn && isYesNoIntent
? transcript ? transcript
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent) : string.Equals(messageType, "CLIENT_NLU",
StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(clientIntent)
? clientIntent ? clientIntent
: transcript; : transcript;
var outboundRules = isWordOfDayLaunch var outboundRules = isWordOfDayLaunch
@@ -93,16 +99,24 @@ public sealed class ResponsePlanToSocketMessagesMapper
: isGlobalCommand : isGlobalCommand
? BuildGlobalCommandRules(rules) ? BuildGlobalCommandRules(rules)
: isRadioLaunch : isRadioLaunch
? Array.Empty<string>() ? []
: isSettingsLaunch : isSettingsLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>() ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isPhotoGalleryLaunch || isPhotoCreateLaunch : isPhotoGalleryLaunch || isPhotoCreateLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>() ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isClockSkillLaunch : isClockSkillLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>() ? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isWordOfDayGuess : isWordOfDayGuess
? ["word-of-the-day/puzzle"] ? ["word-of-the-day/puzzle"]
: isYesNoTurn && isYesNoIntent ? [yesNoRule!] : rules; : isYesNoTurn && isYesNoIntent
? [yesNoRule!]
: rules;
var entities = ReadEntities( var entities = ReadEntities(
turn, turn,
messageType, messageType,
@@ -280,7 +294,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return messages; return messages;
} }
public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId, IReadOnlyList<string> rules) public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId,
IReadOnlyList<string> rules)
{ {
return return
[ [
@@ -402,7 +417,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch return value switch
{ {
IReadOnlyList<string> typedRules => typedRules, IReadOnlyList<string> typedRules => typedRules,
IEnumerable<string> rules => rules.Where(rule => !string.IsNullOrWhiteSpace(rule)).ToArray(), IEnumerable<string> rules => [.. rules.Where(rule => !string.IsNullOrWhiteSpace(rule))],
_ => [] _ => []
}; };
} }
@@ -487,12 +502,11 @@ public sealed class ResponsePlanToSocketMessagesMapper
entities["seconds"] = timerSeconds ?? "null"; entities["seconds"] = timerSeconds ?? "null";
} }
if (string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) && if (!string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) ||
(!string.IsNullOrWhiteSpace(alarmTime) || !string.IsNullOrWhiteSpace(alarmAmPm))) (string.IsNullOrWhiteSpace(alarmTime) && string.IsNullOrWhiteSpace(alarmAmPm))) return entities;
{
entities["time"] = alarmTime ?? string.Empty; entities["time"] = alarmTime ?? string.Empty;
entities["ampm"] = alarmAmPm ?? string.Empty; entities["ampm"] = alarmAmPm ?? string.Empty;
}
return entities; return entities;
} }
@@ -505,12 +519,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
}; };
} }
if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)) if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ||
{ !turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
return new Dictionary<string, object?>();
}
if (!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
{ {
return new Dictionary<string, object?>(); return new Dictionary<string, object?>();
} }
@@ -582,7 +592,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch return value switch
{ {
JsonElement { ValueKind: JsonValueKind.Object } jsonElement JsonElement { ValueKind: JsonValueKind.Object } jsonElement
when jsonElement.TryGetProperty(entityName, out var property) && property.ValueKind == JsonValueKind.String when jsonElement.TryGetProperty(entityName, out var property) &&
property.ValueKind == JsonValueKind.String
=> property.GetString(), => property.GetString(),
IReadOnlyDictionary<string, string> typed when typed.TryGetValue(entityName, out var entityValue) IReadOnlyDictionary<string, string> typed when typed.TryGetValue(entityName, out var entityValue)
=> entityValue, => entityValue,
@@ -602,7 +613,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value?.ToString(); return value?.ToString();
} }
private static string? ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess) private static string ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess)
{ {
if (!string.IsNullOrWhiteSpace(nluGuess)) if (!string.IsNullOrWhiteSpace(nluGuess))
{ {
@@ -662,12 +673,11 @@ public sealed class ResponsePlanToSocketMessagesMapper
} }
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint); var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance) if (distance >= bestDistance) continue;
{
bestDistance = distance; bestDistance = distance;
bestHint = hint; bestHint = hint;
} }
}
return bestDistance <= 2 ? bestHint : null; return bestDistance <= 2 ? bestHint : null;
} }
@@ -704,10 +714,12 @@ public sealed class ResponsePlanToSocketMessagesMapper
return previous[right.Length]; return previous[right.Length];
} }
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill) private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak,
InvokeNativeSkillAction? skill)
{ {
var skillPayload = skill?.Payload; var skillPayload = skill?.Payload;
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only", StringComparison.OrdinalIgnoreCase)) if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only",
StringComparison.OrdinalIgnoreCase))
{ {
return BuildCompletionOnlySkillPayload( return BuildCompletionOnlySkillPayload(
transId, transId,
@@ -718,7 +730,9 @@ public sealed class ResponsePlanToSocketMessagesMapper
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase); string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase); var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase);
var payloadSkill = ReadPayloadString(skillPayload, "skillId"); var payloadSkill = ReadPayloadString(skillPayload, "skillId");
var skillId = string.IsNullOrWhiteSpace(payloadSkill) ? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill" : payloadSkill; var skillId = string.IsNullOrWhiteSpace(payloadSkill)
? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill"
: payloadSkill;
var esml = ReadPayloadString(skillPayload, "esml") ?? (isDance var esml = ReadPayloadString(skillPayload, "esml") ?? (isDance
? "<speak>Okay.<break size='0.2'/> Watch this.<anim cat='dance' filter='music, rom-upbeat' /></speak>" ? "<speak>Okay.<break size='0.2'/> Watch this.<anim cat='dance' filter='music, rom-upbeat' /></speak>"
: isJoke : isJoke
@@ -799,9 +813,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static IReadOnlyList<string> BuildGlobalCommandRules(IReadOnlyList<string> rules) private static IReadOnlyList<string> BuildGlobalCommandRules(IReadOnlyList<string> rules)
{ {
return rules.Any(static rule => string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase)) return rules.Any(static rule =>
string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase))
? ["globals/global_commands_launch"] ? ["globals/global_commands_launch"]
: Array.Empty<string>(); : [];
} }
private static object BuildGenericFallbackSkillPayload(string transId) private static object BuildGenericFallbackSkillPayload(string transId)
@@ -829,7 +844,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
{ {
play = new play = new
{ {
esml = "<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>", esml =
"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>",
meta = new meta = new
{ {
prompt_id = "RUNTIME_PROMPT", prompt_id = "RUNTIME_PROMPT",

93
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -6,10 +6,8 @@ using System.Text.RegularExpressions;
namespace Jibo.Cloud.Application.Services; namespace Jibo.Cloud.Application.Services;
public sealed class WebSocketTurnFinalizationService( public sealed partial class WebSocketTurnFinalizationService(
ProtocolToTurnContextMapper turnContextMapper,
IConversationBroker conversationBroker, IConversationBroker conversationBroker,
ResponsePlanToSocketMessagesMapper replyMapper,
ISttStrategySelector sttStrategySelector, ISttStrategySelector sttStrategySelector,
ITurnTelemetrySink sink ITurnTelemetrySink sink
) )
@@ -18,7 +16,7 @@ public sealed class WebSocketTurnFinalizationService(
private const int AutoFinalizeMinBufferedAudioChunks = 4; private const int AutoFinalizeMinBufferedAudioChunks = 4;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400); private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400);
public void ObserveIncomingMessage(CloudSession session, string? text) public static void ObserveIncomingMessage(CloudSession session, string? text)
{ {
if (!TryReadTransId(text, out var nextTransId) || string.IsNullOrWhiteSpace(nextTransId)) if (!TryReadTransId(text, out var nextTransId) || string.IsNullOrWhiteSpace(nextTransId))
{ {
@@ -39,12 +37,7 @@ public sealed class WebSocketTurnFinalizationService(
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
var turnState = session.TurnState; var turnState = session.TurnState;
if (ShouldIgnoreLateAudio(session)) if (ShouldIgnoreLateAudio(session) || !turnState.AwaitingTurnCompletion &&
{
return [];
}
if (!turnState.AwaitingTurnCompletion &&
!session.FollowUpOpen && !session.FollowUpOpen &&
!turnState.SawListen && !turnState.SawListen &&
!string.IsNullOrWhiteSpace(turnState.TransId)) !string.IsNullOrWhiteSpace(turnState.TransId))
@@ -58,7 +51,7 @@ public sealed class WebSocketTurnFinalizationService(
turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0; turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0;
if (envelope.Binary is { Length: > 0 }) if (envelope.Binary is { Length: > 0 })
{ {
turnState.BufferedAudioFrames.Add(envelope.Binary.ToArray()); turnState.BufferedAudioFrames.Add([.. envelope.Binary]);
} }
turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow; turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow;
turnState.AwaitingTurnCompletion = true; turnState.AwaitingTurnCompletion = true;
@@ -116,7 +109,7 @@ public sealed class WebSocketTurnFinalizationService(
return await FinalizeTurnAsync(session, envelope, messageType, allowFallbackOnMissingTranscript: false, cancellationToken); return await FinalizeTurnAsync(session, envelope, messageType, allowFallbackOnMissingTranscript: false, cancellationToken);
} }
public IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope) public static IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope)
{ {
PersistTurnHints(session, envelope.Text); PersistTurnHints(session, envelope.Text);
@@ -129,7 +122,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session); ResetBufferedAudio(session);
session.TurnState.SawListen = false; session.TurnState.SawListen = false;
session.TurnState.SawContext = false; session.TurnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill( return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
session.TurnState.TransId ?? session.LastTransId ?? string.Empty, session.TurnState.TransId ?? session.LastTransId ?? string.Empty,
session.TurnState.ListenRules, session.TurnState.ListenRules,
"@be/idle") "@be/idle")
@@ -137,8 +130,7 @@ public sealed class WebSocketTurnFinalizationService(
{ {
Text = map.Text, Text = map.Text,
DelayMs = map.DelayMs DelayMs = map.DelayMs
}) })];
.ToArray();
} }
session.TurnState.AwaitingTurnCompletion = true; session.TurnState.AwaitingTurnCompletion = true;
@@ -147,17 +139,12 @@ public sealed class WebSocketTurnFinalizationService(
private async Task<TurnContext> ResolveTranscriptAsync(TurnContext turn, CloudSession session, CancellationToken cancellationToken) private async Task<TurnContext> ResolveTranscriptAsync(TurnContext turn, CloudSession session, CancellationToken cancellationToken)
{ {
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript)) if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript) || session.TurnState.BufferedAudioBytes <= 0)
{ {
return turn; return turn;
} }
if (session.TurnState.BufferedAudioBytes <= 0) ISttStrategy? strategy;
{
return turn;
}
ISttStrategy? strategy = null;
try try
{ {
strategy = await sttStrategySelector.SelectAsync(turn, cancellationToken); strategy = await sttStrategySelector.SelectAsync(turn, cancellationToken);
@@ -254,14 +241,13 @@ public sealed class WebSocketTurnFinalizationService(
} }
} }
if (root.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Object) if (!root.TryGetProperty("data", out var data) || data.ValueKind != JsonValueKind.Object) return;
{
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array) if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{ {
turnState.ListenRules = rules.EnumerateArray() turnState.ListenRules = [.. rules.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString()) .Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(rule => !string.IsNullOrWhiteSpace(rule)) .Where(rule => !string.IsNullOrWhiteSpace(rule))];
.ToArray();
session.Metadata["listenRules"] = turnState.ListenRules; session.Metadata["listenRules"] = turnState.ListenRules;
} }
@@ -270,15 +256,14 @@ public sealed class WebSocketTurnFinalizationService(
asr.TryGetProperty("hints", out var hints) && asr.TryGetProperty("hints", out var hints) &&
hints.ValueKind == JsonValueKind.Array) hints.ValueKind == JsonValueKind.Array)
{ {
turnState.ListenAsrHints = hints.EnumerateArray() turnState.ListenAsrHints = [.. hints.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String) .Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty) .Select(static item => item.GetString() ?? string.Empty)
.Where(static hint => !string.IsNullOrWhiteSpace(hint)) .Where(static hint => !string.IsNullOrWhiteSpace(hint))];
.ToArray();
} }
if (data.TryGetProperty("hotphrase", out var hotphrase) && if (data.TryGetProperty("hotphrase", out var hotphrase) &&
(hotphrase.ValueKind == JsonValueKind.True || hotphrase.ValueKind == JsonValueKind.False)) hotphrase.ValueKind is JsonValueKind.True or JsonValueKind.False)
{ {
turnState.ListenHotphrase = hotphrase.GetBoolean(); turnState.ListenHotphrase = hotphrase.GetBoolean();
turnState.HotphraseEmptyTurnCount = 0; turnState.HotphraseEmptyTurnCount = 0;
@@ -289,13 +274,12 @@ public sealed class WebSocketTurnFinalizationService(
session.LastIntent = intent.GetString(); session.LastIntent = intent.GetString();
} }
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String) if (!data.TryGetProperty("transcriptHint", out var transcriptHint) ||
{ transcriptHint.ValueKind != JsonValueKind.String) return;
turnState.AudioTranscriptHint = transcriptHint.GetString(); turnState.AudioTranscriptHint = transcriptHint.GetString();
session.Metadata["audioTranscriptHint"] = turnState.AudioTranscriptHint; session.Metadata["audioTranscriptHint"] = turnState.AudioTranscriptHint;
} }
}
}
catch catch
{ {
// Keep the compatibility layer permissive while captures are still incomplete. // Keep the compatibility layer permissive while captures are still incomplete.
@@ -394,7 +378,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session); ResetBufferedAudio(session);
turnState.SawListen = false; turnState.SawListen = false;
turnState.SawContext = false; turnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill( return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.TransId ?? session.LastTransId ?? string.Empty,
turnState.ListenRules, turnState.ListenRules,
"@be/idle") "@be/idle")
@@ -402,8 +386,7 @@ public sealed class WebSocketTurnFinalizationService(
{ {
Text = map.Text, Text = map.Text,
DelayMs = map.DelayMs DelayMs = map.DelayMs
}) })];
.ToArray();
} }
if (ShouldHandleAsLocalNoInput(finalizedTurn)) if (ShouldHandleAsLocalNoInput(finalizedTurn))
@@ -448,9 +431,11 @@ public sealed class WebSocketTurnFinalizationService(
turnState.FinalizeAttemptCount += 1; turnState.FinalizeAttemptCount += 1;
} }
if (allowFallbackOnMissingTranscript && switch (allowFallbackOnMissingTranscript)
{
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes && turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
IsYesNoTurn(finalizedTurn)) IsYesNoTurn(finalizedTurn):
{ {
turnState.AwaitingTurnCompletion = false; turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty; session.LastTranscript = string.Empty;
@@ -461,10 +446,9 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session); ResetBufferedAudio(session);
return noInputReplies; return noInputReplies;
} }
case true when
if (allowFallbackOnMissingTranscript &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes && turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
string.IsNullOrWhiteSpace(turnState.LastSttError)) string.IsNullOrWhiteSpace(turnState.LastSttError):
{ {
turnState.AwaitingTurnCompletion = false; turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty; session.LastTranscript = string.Empty;
@@ -476,9 +460,10 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session); ResetBufferedAudio(session);
return fallbackReplies; return fallbackReplies;
} }
default:
return []; return [];
} }
}
var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken); var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken);
var listenAction = plan.Actions.OfType<ListenAction>().OrderBy(action => action.Sequence).LastOrDefault(); var listenAction = plan.Actions.OfType<ListenAction>().OrderBy(action => action.Sequence).LastOrDefault();
@@ -487,7 +472,7 @@ public sealed class WebSocketTurnFinalizationService(
session.LastListenType = listenAction?.Mode; session.LastListenType = listenAction?.Mode;
turnState.LastLocalNoInputRule = null; turnState.LastLocalNoInputRule = null;
turnState.LocalNoInputCount = 0; turnState.LocalNoInputCount = 0;
if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock", Payload: not null } clockAction && if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock" } clockAction &&
clockAction.Payload.TryGetValue("domain", out var lastClockDomainValue) && clockAction.Payload.TryGetValue("domain", out var lastClockDomainValue) &&
lastClockDomainValue is not null) lastClockDomainValue is not null)
{ {
@@ -545,10 +530,7 @@ public sealed class WebSocketTurnFinalizationService(
var turnAge = turnState.FirstAudioReceivedUtc.HasValue var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value ? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero; : TimeSpan.Zero;
return turnState.AwaitingTurnCompletion && return turnState is { AwaitingTurnCompletion: true, SawListen: true, BufferedAudioChunkCount: >= AutoFinalizeMinBufferedAudioChunks, BufferedAudioBytes: >= AutoFinalizeMinBufferedAudioBytes } &&
turnState.SawListen &&
turnState.BufferedAudioChunkCount >= AutoFinalizeMinBufferedAudioChunks &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
turnAge >= AutoFinalizeMinTurnAge; turnAge >= AutoFinalizeMinTurnAge;
} }
@@ -755,7 +737,7 @@ public sealed class WebSocketTurnFinalizationService(
.FirstOrDefault(IsConstrainedYesNoRule); .FirstOrDefault(IsConstrainedYesNoRule);
} }
private static IReadOnlyList<WebSocketReply> BuildLocalNoInputReplies( private static WebSocketReply[] BuildLocalNoInputReplies(
CloudSession session, CloudSession session,
WebSocketTurnState turnState, WebSocketTurnState turnState,
string? localRule) string? localRule)
@@ -764,14 +746,12 @@ public sealed class WebSocketTurnFinalizationService(
var effectiveRule = string.IsNullOrWhiteSpace(localRule) var effectiveRule = string.IsNullOrWhiteSpace(localRule)
? turnState.ListenRules.FirstOrDefault(IsLocalNoInputRule) ? turnState.ListenRules.FirstOrDefault(IsLocalNoInputRule)
: localRule; : localRule;
IReadOnlyList<string> rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule]; var rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule];
var maps = ShouldRedirectRepeatedNoInputToIdle(turnState, effectiveRule) var maps = ShouldRedirectRepeatedNoInputToIdle(turnState, effectiveRule)
? ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(transId, rules, "@be/idle") ? ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(transId, rules, "@be/idle")
: ResponsePlanToSocketMessagesMapper.MapNoInput(transId, rules); : ResponsePlanToSocketMessagesMapper.MapNoInput(transId, rules);
return maps return [.. maps.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })];
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
} }
private static bool ShouldRedirectRepeatedNoInputToIdle(WebSocketTurnState turnState, string? localRule) private static bool ShouldRedirectRepeatedNoInputToIdle(WebSocketTurnState turnState, string? localRule)
@@ -853,7 +833,7 @@ public sealed class WebSocketTurnFinalizationService(
return string.Empty; return string.Empty;
} }
return Regex.Replace(transcript.Trim().ToLowerInvariant(), @"[^\w\s]", " ") return TranscriptNormalizationRegex().Replace(transcript.Trim().ToLowerInvariant(), " ")
.Replace(" ", " ", StringComparison.Ordinal) .Replace(" ", " ", StringComparison.Ordinal)
.Trim(); .Trim();
} }
@@ -1036,4 +1016,7 @@ public sealed class WebSocketTurnFinalizationService(
_ => false _ => false
}; };
} }
[GeneratedRegex(@"[^\w\s]")]
private static partial Regex TranscriptNormalizationRegex();
} }

6
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/ExternalProcessRunner.cs
View File

@@ -6,16 +6,14 @@ public sealed class ExternalProcessRunner : IExternalProcessRunner
{ {
public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default) public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default)
{ {
using var process = new Process using var process = new Process();
{ process.StartInfo = new ProcessStartInfo
StartInfo = new ProcessStartInfo
{ {
FileName = fileName, FileName = fileName,
RedirectStandardOutput = true, RedirectStandardOutput = true,
RedirectStandardError = true, RedirectStandardError = true,
UseShellExecute = false, UseShellExecute = false,
CreateNoWindow = true CreateNoWindow = true
}
}; };
foreach (var argument in arguments) foreach (var argument in arguments)

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs
View File

@@ -132,7 +132,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var timecoded = lines var timecoded = lines
.Where(static line => line.StartsWith("[", StringComparison.Ordinal) && line.Contains("-->", StringComparison.Ordinal)) .Where(static line => line.StartsWith('[') && line.Contains("-->", StringComparison.Ordinal))
.Select(static line => .Select(static line =>
{ {
var closingBracket = line.IndexOf(']'); var closingBracket = line.IndexOf(']');
@@ -171,6 +171,6 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
return true; return true;
} }
return checkFileExists ? File.Exists(path) : true; return !checkFileExists || File.Exists(path);
} }
} }

8
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/OggOpusAudioNormalizer.cs
View File

@@ -79,13 +79,7 @@ internal static class OggOpusAudioNormalizer
private static uint ComputeCrc(byte[] buffer) private static uint ComputeCrc(byte[] buffer)
{ {
uint crc = 0; return buffer.Aggregate<byte, uint>(0, (current, value) => (current << 8) ^ CrcTable[((current >> 24) ^ value) & 0xff]);
foreach (var value in buffer)
{
crc = (crc << 8) ^ CrcTable[((crc >> 24) ^ value) & 0xff];
}
return crc;
} }
private static uint[] BuildCrcTable() private static uint[] BuildCrcTable()

1
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs
View File

@@ -7,7 +7,6 @@ using Jibo.Cloud.Infrastructure.Telemetry;
using Jibo.Runtime.Abstractions; using Jibo.Runtime.Abstractions;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using System.IO;
namespace Jibo.Cloud.Infrastructure.DependencyInjection; namespace Jibo.Cloud.Infrastructure.DependencyInjection;

15
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryCloudStateStore.cs
View File

@@ -18,7 +18,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
private readonly ConcurrentDictionary<string, string> _symmetricKeys = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary<string, string> _symmetricKeys = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, KeyRequestRecord> _keyRequests = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary<string, KeyRequestRecord> _keyRequests = new(StringComparer.OrdinalIgnoreCase);
private readonly string? _persistencePath; private readonly string? _persistencePath;
private readonly object _syncRoot = new(); private readonly Lock _syncRoot = new();
private readonly List<UpdateManifest> _updates; private readonly List<UpdateManifest> _updates;
private readonly List<MediaRecord> _media = []; private readonly List<MediaRecord> _media = [];
private readonly List<BackupRecord> _backups = []; private readonly List<BackupRecord> _backups = [];
@@ -186,13 +186,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
public UpdateManifest RemoveUpdate(string? updateId) public UpdateManifest RemoveUpdate(string? updateId)
{ {
var existing = _updates.FirstOrDefault(update => update.UpdateId == updateId); var existing = _updates.FirstOrDefault(update => update.UpdateId == updateId);
if (existing is not null) if (existing is null)
{
_updates.Remove(existing);
PersistState();
return existing;
}
return new UpdateManifest return new UpdateManifest
{ {
UpdateId = updateId ?? "unknown-update", UpdateId = updateId ?? "unknown-update",
@@ -201,6 +195,11 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
ShaHash = "missing", ShaHash = "missing",
Subsystem = "unknown" Subsystem = "unknown"
}; };
_updates.Remove(existing);
PersistState();
return existing;
} }
public IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null) public IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null)

18
OpenJibo/src/Playground/AsrEvent.cs Normal file
View File

@@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}

12
OpenJibo/src/Playground/AsrUtterance.cs Normal file
View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
}

36
OpenJibo/src/Playground/Program.cs
View File

@@ -2,7 +2,7 @@
using System.Net.WebSockets; using System.Net.WebSockets;
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Serialization; using Playground;
Console.Write("Enter Jibo IP: "); Console.Write("Enter Jibo IP: ");
var jiboIp = (Console.ReadLine() ?? "").Trim(); var jiboIp = (Console.ReadLine() ?? "").Trim();
@@ -67,7 +67,7 @@ while (!cts.IsCancellationRequested)
var json = Encoding.UTF8.GetString(ms.ToArray()); var json = Encoding.UTF8.GetString(ms.ToArray());
AsrEvent? evt = null; AsrEvent? evt;
try try
{ {
evt = JsonSerializer.Deserialize<AsrEvent>(json); evt = JsonSerializer.Deserialize<AsrEvent>(json);
@@ -86,16 +86,12 @@ while (!cts.IsCancellationRequested)
Console.WriteLine($"[{evt.EventType}] {json}"); Console.WriteLine($"[{evt.EventType}] {json}");
if (evt.EventType == "speech_to_text_final") if (evt.EventType != "speech_to_text_final") continue;
{
var best = PickBestUtterance(evt.Utterances); var best = PickBestUtterance(evt.Utterances);
if (!string.IsNullOrWhiteSpace(best)) if (string.IsNullOrWhiteSpace(best)) continue;
{
utteranceTcs.TrySetResult(best); utteranceTcs.TrySetResult(best);
return; return;
} }
}
}
}, cts.Token); }, cts.Token);
var startPayload = new var startPayload = new
@@ -220,27 +216,3 @@ static string BuildReply(string heard)
return text.Contains("your name") ? "I am Jibo, running with a local demo bridge." : $"You said: {heard}"; return text.Contains("your name") ? "I am Jibo, running with a local demo bridge." : $"You said: {heard}";
} }
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
}

16
OpenJibo/tests/Jibo.Cloud.Tests/Fixtures/WebSocketFixtureLoader.cs
View File

@@ -17,10 +17,9 @@ internal static class WebSocketFixtureLoader
var root = document.RootElement; var root = document.RootElement;
var session = root.GetProperty("session"); var session = root.GetProperty("session");
var steps = new List<WebSocketFixtureStep>(); var steps = root.GetProperty("steps")
foreach (var stepElement in root.GetProperty("steps").EnumerateArray()) .EnumerateArray()
{ .Select(stepElement => new WebSocketFixtureStep
steps.Add(new WebSocketFixtureStep
{ {
Message = new WebSocketMessageEnvelope Message = new WebSocketMessageEnvelope
{ {
@@ -33,16 +32,15 @@ internal static class WebSocketFixtureLoader
? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray() ? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray()
: null : null
}, },
ExpectedReplyTypes = stepElement.GetProperty("expectedReplyTypes") ExpectedReplyTypes = [.. stepElement.GetProperty("expectedReplyTypes")
.EnumerateArray() .EnumerateArray()
.Select(item => item.GetString() ?? string.Empty) .Select(item => item.GetString() ?? string.Empty)
.Where(item => !string.IsNullOrWhiteSpace(item)) .Where(item => !string.IsNullOrWhiteSpace(item))],
.ToArray(),
ExpectedReplies = stepElement.TryGetProperty("expectedReplies", out var expectedReplies) && expectedReplies.ValueKind == JsonValueKind.Array ExpectedReplies = stepElement.TryGetProperty("expectedReplies", out var expectedReplies) && expectedReplies.ValueKind == JsonValueKind.Array
? JsonSerializer.Deserialize<List<ExpectedWebSocketReply>>(expectedReplies.GetRawText(), SerializerOptions) ?? [] ? JsonSerializer.Deserialize<List<ExpectedWebSocketReply>>(expectedReplies.GetRawText(), SerializerOptions) ?? []
: [] : []
}); })
} .ToList();
return new WebSocketFixture return new WebSocketFixture
{ {

40
OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs
View File

@@ -16,18 +16,18 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>())) sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Exception("dummy")); .ThrowsAsync(new Exception("dummy"));
var turnService = new WebSocketTurnFinalizationService( var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
sttStrategySelector.Object, sttStrategySelector.Object,
sink.Object sink.Object
); );
await turnService.HandleTurnAsync(new CloudSession() { TurnState = { BufferedAudioBytes = 100 }}, new WebSocketMessageEnvelope(), "dummy", await turnService.HandleTurnAsync(new CloudSession { TurnState = { BufferedAudioBytes = 100 } },
new WebSocketMessageEnvelope(), "dummy",
CancellationToken.None); CancellationToken.None);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once()); sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
} }
[Fact] [Fact]
@@ -38,21 +38,23 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>())) sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new InvalidOperationException("ffmpeg failed")); .ThrowsAsync(new InvalidOperationException("ffmpeg failed"));
var turnService = new WebSocketTurnFinalizationService( var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
sttStrategySelector.Object, sttStrategySelector.Object,
sink.Object sink.Object
); );
var session = new CloudSession(); var session = new CloudSession
session.TurnState.AwaitingTurnCompletion = true; {
session.TurnState.SawListen = true; TurnState =
session.TurnState.SawContext = true; {
session.TurnState.BufferedAudioBytes = 12000; AwaitingTurnCompletion = true,
session.TurnState.BufferedAudioChunkCount = 5; SawListen = true,
session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2); SawContext = true,
BufferedAudioBytes = 12000,
BufferedAudioChunkCount = 5,
FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2)
}
};
var replies = await turnService.HandleContextAsync( var replies = await turnService.HandleContextAsync(
session, session,
@@ -64,6 +66,8 @@ public sealed class FileTurnTelemetrySinkTests
Assert.Equal(12000, session.TurnState.BufferedAudioBytes); Assert.Equal(12000, session.TurnState.BufferedAudioBytes);
Assert.Equal("ffmpeg failed", session.TurnState.LastSttError); Assert.Equal("ffmpeg failed", session.TurnState.LastSttError);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once()); sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
} }
} }

20
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -16,11 +16,9 @@ public sealed class JiboWebSocketServiceTests
public JiboWebSocketServiceTests() public JiboWebSocketServiceTests()
{ {
_store = new InMemoryCloudStateStore(); _store = new InMemoryCloudStateStore();
var turnContextMapper = new ProtocolToTurnContextMapper();
var contentRepository = new InMemoryJiboExperienceContentRepository(); var contentRepository = new InMemoryJiboExperienceContentRepository();
var contentCache = new JiboExperienceContentCache(contentRepository); var contentCache = new JiboExperienceContentCache(contentRepository);
var conversationBroker = new DemoConversationBroker(new JiboInteractionService(contentCache, new DefaultJiboRandomizer())); var conversationBroker = new DemoConversationBroker(new JiboInteractionService(contentCache, new DefaultJiboRandomizer()));
var replyMapper = new ResponsePlanToSocketMessagesMapper();
var sttSelector = new DefaultSttStrategySelector( var sttSelector = new DefaultSttStrategySelector(
[ [
new SyntheticBufferedAudioSttStrategy() new SyntheticBufferedAudioSttStrategy()
@@ -30,10 +28,7 @@ public sealed class JiboWebSocketServiceTests
_service = new JiboWebSocketService( _service = new JiboWebSocketService(
_store, _store,
new NullWebSocketTelemetrySink(), new NullWebSocketTelemetrySink(),
new WebSocketTurnFinalizationService( new WebSocketTurnFinalizationService(conversationBroker,
turnContextMapper,
conversationBroker,
replyMapper,
sttSelector, sttSelector,
sink)); sink));
} }
@@ -2639,7 +2634,7 @@ public sealed class JiboWebSocketServiceTests
Path = "/listen", Path = "/listen",
Kind = "neo-hub-listen", Kind = "neo-hub-listen",
Token = "hub-context-reset-token", Token = "hub-context-reset-token",
Binary = [9, 9, 9, 9] Binary = "\t\t\t\t"u8.ToArray()
}); });
var session = _store.FindSessionByToken("hub-context-reset-token"); var session = _store.FindSessionByToken("hub-context-reset-token");
@@ -2681,8 +2676,8 @@ public sealed class JiboWebSocketServiceTests
var actualTypes = replies.Select(ReadReplyType).ToArray(); var actualTypes = replies.Select(ReadReplyType).ToArray();
Assert.Equal(step.ExpectedReplyTypes, actualTypes); Assert.Equal(step.ExpectedReplyTypes, actualTypes);
if (step.ExpectedReplies.Count > 0) if (step.ExpectedReplies.Count <= 0) continue;
{
Assert.Equal(replies.Count, step.ExpectedReplies.Count); Assert.Equal(replies.Count, step.ExpectedReplies.Count);
for (var index = 0; index < step.ExpectedReplies.Count; index += 1) for (var index = 0; index < step.ExpectedReplies.Count; index += 1)
@@ -2695,20 +2690,19 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal(expectedReply.DelayMs.Value, replies[index].DelayMs); Assert.Equal(expectedReply.DelayMs.Value, replies[index].DelayMs);
} }
if (expectedReply.JsonSubset is { ValueKind: JsonValueKind.Object } jsonSubset) if (expectedReply.JsonSubset is not { ValueKind: JsonValueKind.Object } jsonSubset) continue;
{
using var actualPayload = JsonDocument.Parse(replies[index].Text!); using var actualPayload = JsonDocument.Parse(replies[index].Text!);
AssertJsonContains(jsonSubset, actualPayload.RootElement); AssertJsonContains(jsonSubset, actualPayload.RootElement);
} }
} }
} }
}
}
private static void AssertJsonContains(JsonElement expected, JsonElement actual) private static void AssertJsonContains(JsonElement expected, JsonElement actual)
{ {
Assert.Equal(expected.ValueKind, actual.ValueKind); Assert.Equal(expected.ValueKind, actual.ValueKind);
// ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault
switch (expected.ValueKind) switch (expected.ValueKind)
{ {
case JsonValueKind.Object: case JsonValueKind.Object:

8
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs
View File

@@ -158,14 +158,14 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests
{ {
Calls.Add((fileName, arguments)); Calls.Add((fileName, arguments));
if (string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase)) if (!string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase))
{ return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke",
string.Empty));
var outputPath = arguments[^1]; var outputPath = arguments[^1];
File.WriteAllBytes(outputPath, "RIFF"u8); File.WriteAllBytes(outputPath, "RIFF"u8);
return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty)); return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty));
}
return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke", string.Empty));
} }
} }
} }