Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactors

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-26 20:57:08 -05:00
parent acbba413db
commit 8c97968d95
20 changed files with 547 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
OpenJibo/OpenJibo.sln.DotSettings
View File

@@ -1,3 +1,18 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/UserDictionary/Words/=ampm/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Arrrr/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=esml/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Hotphrase/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo_0027s/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=multichunk/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=nevermind/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=noinput/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Photobooth/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slnx/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slowdance/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=timecoded/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Todays/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=whispercpp/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=YESNO/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

15
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Program.cs
View File

@@ -25,20 +25,13 @@ app.Use(async (context, next) =>
var kind = ResolveSocketKind(context.Request.Host.Host, context.Request.Path);
var token = ResolveToken(context.Request);
if (kind == "unknown")
switch (kind)
{
case "unknown":
context.Response.StatusCode = StatusCodes.Status404NotFound;
return;
}
if (kind == "api-socket" && string.IsNullOrWhiteSpace(token))
{
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
}
if (kind is "neo-hub-listen" or "neo-hub-proactive" && string.IsNullOrWhiteSpace(token))
{
case "api-socket" when string.IsNullOrWhiteSpace(token):
case "neo-hub-listen" or "neo-hub-proactive" when string.IsNullOrWhiteSpace(token):
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
}

53
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs
View File

@@ -194,10 +194,11 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
});
}
if (operation is "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or
"ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
"VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect")
switch (operation)
{
case "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or
"ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
"VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect":
return ProtocolDispatchResult.Ok(new
{
id = account.AccountId,
@@ -207,10 +208,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
accessKeyId = account.AccessKeyId,
secretAccessKey = account.SecretAccessKey
});
}
if (operation is "ChangeEmail" or "SendPhoneVerificationCode")
{
case "ChangeEmail" or "SendPhoneVerificationCode":
return ProtocolDispatchResult.Ok(new
{
id = account.AccountId
@@ -236,8 +234,8 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}".ToLowerInvariant();
return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query)
? new[]
{
?
[
new
{
id = account.AccountId,
@@ -245,7 +243,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
firstName = account.FirstName,
lastName = account.LastName
}
}
]
: Array.Empty<object>());
}
@@ -382,8 +380,9 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray());
}
if (operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
{
if (!operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(Array.Empty<object>());
var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId;
var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
@@ -398,9 +397,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
}
return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta)));
}
return ProtocolDispatchResult.Ok(Array.Empty<object>());
}
private ProtocolDispatchResult HandlePerson(string operation)
@@ -430,9 +427,10 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
});
}
string? symmetricKey;
if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase))
{
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new
{
loopId,
@@ -472,18 +470,17 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(new { ok = true });
}
if (operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase))
{
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
if (!operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(new { ok = true, operation });
symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new
{
loopId,
key = symmetricKey,
symmetricKey
});
}
return ProtocolDispatchResult.Ok(new { ok = true, operation });
}
private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope)
@@ -509,8 +506,12 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
});
}
if (operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase))
if (!operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(new
{
result = "ok"
});
var profile = stateStore.GetRobotProfile();
return ProtocolDispatchResult.Ok(new
{
@@ -520,12 +521,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(),
created = profile.CreatedUtc.ToUnixTimeMilliseconds()
});
}
return ProtocolDispatchResult.Ok(new
{
result = "ok"
});
}
private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope)
@@ -674,10 +670,9 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return [];
}
return property.EnumerateArray()
return [.. property.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(item => !string.IsNullOrWhiteSpace(item))
.ToArray();
.Where(item => !string.IsNullOrWhiteSpace(item))];
}
private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName)

122
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -484,13 +484,11 @@ public sealed class JiboInteractionService(
return "hello";
}
if (isYesNoTurn && MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh"))
switch (isYesNoTurn)
{
case true when MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh"):
return "yes";
}
if (isYesNoTurn && MatchesAny(loweredTranscript, "no", "nope", "nah"))
{
case true when MatchesAny(loweredTranscript, "no", "nope", "nah"):
return "no";
}
@@ -752,12 +750,7 @@ public sealed class JiboInteractionService(
}
var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints);
if (!string.IsNullOrWhiteSpace(fuzzyHintMatch))
{
return fuzzyHintMatch;
}
return transcript;
return !string.IsNullOrWhiteSpace(fuzzyHintMatch) ? fuzzyHintMatch : transcript;
}
private static bool IsYesNoTurn(TurnContext turn)
@@ -805,12 +798,11 @@ public sealed class JiboInteractionService(
}
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance)
{
if (distance >= bestDistance) continue;
bestDistance = distance;
bestHint = hint;
}
}
return bestDistance <= 2 ? bestHint : null;
}
@@ -996,14 +988,12 @@ public sealed class JiboInteractionService(
{
var compactHour = compact.Length switch
{
3 => compactValue / 100,
4 => compactValue / 100,
3 or 4 => compactValue / 100,
_ => -1
};
var compactMinute = compact.Length switch
{
3 => compactValue % 100,
4 => compactValue % 100,
3 or 4 => compactValue % 100,
_ => -1
};
if (compactHour is >= 1 and <= 12 && compactMinute is >= 0 and <= 59)
@@ -1023,13 +1013,13 @@ public sealed class JiboInteractionService(
var hourToken = match.Groups["hour"].Value;
var minuteToken = match.Groups["minute"].Success ? match.Groups["minute"].Value : "00";
var hour = ParseNumberToken(hourToken);
if (hour is null || hour is < 1 or > 12)
if (hour is null or < 1 or > 12)
{
return null;
}
var minute = ParseNumberToken(minuteToken);
if (minute is null || minute is < 0 or > 59)
if (minute is null or < 0 or > 59)
{
return null;
}
@@ -1146,7 +1136,7 @@ public sealed class JiboInteractionService(
return lastClockDomain;
}
var combinedRules = clientRules.Concat(listenRules);
var combinedRules = clientRules.Concat(listenRules).ToArray();
if (combinedRules.Any(rule =>
rule.Contains("timer", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)))
@@ -1154,14 +1144,9 @@ public sealed class JiboInteractionService(
return "timer";
}
if (combinedRules.Any(rule =>
return combinedRules.Any(rule =>
rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)))
{
return "alarm";
}
return null;
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)) ? "alarm" : null;
}
private static bool IsTimerRequest(string loweredTranscript)
@@ -1303,12 +1288,7 @@ public sealed class JiboInteractionService(
}
var match = VolumeLevelPattern.Match(loweredTranscript);
if (!match.Success)
{
return null;
}
return TryNormalizeVolumeLevel(match.Groups["value"].Value);
return !match.Success ? null : TryNormalizeVolumeLevel(match.Groups["value"].Value);
}
private static string? TryNormalizeVolumeLevel(string token)
@@ -1367,14 +1347,16 @@ public sealed class JiboInteractionService(
}
var parts = valueToken.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length >= 2)
{
if (parts.Length < 2)
return parts.Length > 0
? ParseNumberToken(parts[^1])
: null;
parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2)));
if (parsed is not null)
{
return parsed;
}
}
return parts.Length > 0
? ParseNumberToken(parts[^1])
@@ -1389,19 +1371,77 @@ public sealed class JiboInteractionService(
return numeric;
}
if (normalized.Contains(' '))
if (!normalized.Contains(' '))
{
return normalized switch
{
"a" or "an" => 1,
"one" => 1,
"two" => 2,
"three" => 3,
"four" => 4,
"five" => 5,
"six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length == 2)
if (parts.Length != 2)
{
return normalized switch
{
"a" or "an" => 1,
"one" => 1,
"two" => 2,
"three" => 3,
"four" => 4,
"five" => 5,
"six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var first = ParseNumberToken(parts[0]);
var second = ParseNumberToken(parts[1]);
if (first is >= 20 && second is >= 0 and < 10)
{
return first + second;
}
}
}
return normalized switch
{

17
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs
View File

@@ -32,9 +32,11 @@ public sealed class JiboWebSocketService(
var parsedType = ReadMessageType(envelope.Text);
session.LastMessageType = parsedType;
turnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
if (parsedType == "CONTEXT")
switch (parsedType)
{
case "CONTEXT":
{
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
@@ -43,12 +45,11 @@ public sealed class JiboWebSocketService(
}, cancellationToken);
return replies;
}
if (parsedType == "LISTEN")
case "LISTEN":
{
var replies = ContainsInlineTurnPayload(envelope.Text)
? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
: turnFinalizationService.HandleListenSetup(session, envelope);
: WebSocketTurnFinalizationService.HandleListenSetup(session, envelope);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
{
["messageType"] = parsedType,
@@ -58,8 +59,7 @@ public sealed class JiboWebSocketService(
}, cancellationToken);
return replies;
}
if (parsedType is "CLIENT_NLU" or "CLIENT_ASR")
case "CLIENT_NLU" or "CLIENT_ASR":
{
var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
@@ -71,9 +71,10 @@ public sealed class JiboWebSocketService(
}, cancellationToken);
return replies;
}
default:
return [];
}
}
private static string ReadMessageType(string? text)
{

12
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs
View File

@@ -93,8 +93,8 @@ public sealed class ProtocolToTurnContextMapper
using var document = JsonDocument.Parse(text);
var root = document.RootElement;
if (root.TryGetProperty("data", out var data))
{
if (!root.TryGetProperty("data", out var data)) return null;
if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String)
{
return transcript.GetString();
@@ -132,13 +132,7 @@ public sealed class ProtocolToTurnContextMapper
attributes["clientEntities"] = entities.Clone();
}
if (intent.ValueKind == JsonValueKind.String)
{
return intent.GetString();
}
}
return null;
return intent.ValueKind == JsonValueKind.String ? intent.GetString() : null;
}
catch
{

82
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -6,7 +6,8 @@ namespace Jibo.Cloud.Application.Services;
public sealed class ResponsePlanToSocketMessagesMapper
{
public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session, bool emitSkillActions)
public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session,
bool emitSkillActions)
{
var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault();
var skill = plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault();
@@ -22,7 +23,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
var isYesNoIntent = string.Equals(plan.IntentName, "yes", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "no", StringComparison.OrdinalIgnoreCase);
var isWordOfDayLaunch = string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase);
var isWordOfDayGuess = string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var isWordOfDayGuess =
string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase);
var isStopCommand = string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase);
@@ -64,7 +66,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
? clockIntent
: isWordOfDayGuess
? "guess"
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent)
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(clientIntent)
? clientIntent
: plan.IntentName ?? "unknown";
var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess)
@@ -81,11 +84,14 @@ public sealed class ResponsePlanToSocketMessagesMapper
? transcript
: isClockSkillLaunch
? transcript
: string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(nluGuess)
: string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(nluGuess)
? nluGuess
: isYesNoTurn && isYesNoIntent
? transcript
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent)
: string.Equals(messageType, "CLIENT_NLU",
StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(clientIntent)
? clientIntent
: transcript;
var outboundRules = isWordOfDayLaunch
@@ -93,16 +99,24 @@ public sealed class ResponsePlanToSocketMessagesMapper
: isGlobalCommand
? BuildGlobalCommandRules(rules)
: isRadioLaunch
? Array.Empty<string>()
? []
: isSettingsLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>()
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isPhotoGalleryLaunch || isPhotoCreateLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>()
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isClockSkillLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>()
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isWordOfDayGuess
? ["word-of-the-day/puzzle"]
: isYesNoTurn && isYesNoIntent ? [yesNoRule!] : rules;
: isYesNoTurn && isYesNoIntent
? [yesNoRule!]
: rules;
var entities = ReadEntities(
turn,
messageType,
@@ -280,7 +294,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return messages;
}
public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId, IReadOnlyList<string> rules)
public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId,
IReadOnlyList<string> rules)
{
return
[
@@ -402,7 +417,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch
{
IReadOnlyList<string> typedRules => typedRules,
IEnumerable<string> rules => rules.Where(rule => !string.IsNullOrWhiteSpace(rule)).ToArray(),
IEnumerable<string> rules => [.. rules.Where(rule => !string.IsNullOrWhiteSpace(rule))],
_ => []
};
}
@@ -487,12 +502,11 @@ public sealed class ResponsePlanToSocketMessagesMapper
entities["seconds"] = timerSeconds ?? "null";
}
if (string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) &&
(!string.IsNullOrWhiteSpace(alarmTime) || !string.IsNullOrWhiteSpace(alarmAmPm)))
{
if (!string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) ||
(string.IsNullOrWhiteSpace(alarmTime) && string.IsNullOrWhiteSpace(alarmAmPm))) return entities;
entities["time"] = alarmTime ?? string.Empty;
entities["ampm"] = alarmAmPm ?? string.Empty;
}
return entities;
}
@@ -505,12 +519,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
};
}
if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase))
{
return new Dictionary<string, object?>();
}
if (!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ||
!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
{
return new Dictionary<string, object?>();
}
@@ -582,7 +592,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch
{
JsonElement { ValueKind: JsonValueKind.Object } jsonElement
when jsonElement.TryGetProperty(entityName, out var property) && property.ValueKind == JsonValueKind.String
when jsonElement.TryGetProperty(entityName, out var property) &&
property.ValueKind == JsonValueKind.String
=> property.GetString(),
IReadOnlyDictionary<string, string> typed when typed.TryGetValue(entityName, out var entityValue)
=> entityValue,
@@ -602,7 +613,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value?.ToString();
}
private static string? ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess)
private static string ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess)
{
if (!string.IsNullOrWhiteSpace(nluGuess))
{
@@ -662,12 +673,11 @@ public sealed class ResponsePlanToSocketMessagesMapper
}
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance)
{
if (distance >= bestDistance) continue;
bestDistance = distance;
bestHint = hint;
}
}
return bestDistance <= 2 ? bestHint : null;
}
@@ -704,10 +714,12 @@ public sealed class ResponsePlanToSocketMessagesMapper
return previous[right.Length];
}
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill)
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak,
InvokeNativeSkillAction? skill)
{
var skillPayload = skill?.Payload;
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only", StringComparison.OrdinalIgnoreCase))
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only",
StringComparison.OrdinalIgnoreCase))
{
return BuildCompletionOnlySkillPayload(
transId,
@@ -718,7 +730,9 @@ public sealed class ResponsePlanToSocketMessagesMapper
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase);
var payloadSkill = ReadPayloadString(skillPayload, "skillId");
var skillId = string.IsNullOrWhiteSpace(payloadSkill) ? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill" : payloadSkill;
var skillId = string.IsNullOrWhiteSpace(payloadSkill)
? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill"
: payloadSkill;
var esml = ReadPayloadString(skillPayload, "esml") ?? (isDance
? "<speak>Okay.<break size='0.2'/> Watch this.<anim cat='dance' filter='music, rom-upbeat' /></speak>"
: isJoke
@@ -799,9 +813,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static IReadOnlyList<string> BuildGlobalCommandRules(IReadOnlyList<string> rules)
{
return rules.Any(static rule => string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase))
return rules.Any(static rule =>
string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase))
? ["globals/global_commands_launch"]
: Array.Empty<string>();
: [];
}
private static object BuildGenericFallbackSkillPayload(string transId)
@@ -829,7 +844,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
{
play = new
{
esml = "<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>",
esml =
"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>",
meta = new
{
prompt_id = "RUNTIME_PROMPT",

93
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -6,10 +6,8 @@ using System.Text.RegularExpressions;
namespace Jibo.Cloud.Application.Services;
public sealed class WebSocketTurnFinalizationService(
ProtocolToTurnContextMapper turnContextMapper,
public sealed partial class WebSocketTurnFinalizationService(
IConversationBroker conversationBroker,
ResponsePlanToSocketMessagesMapper replyMapper,
ISttStrategySelector sttStrategySelector,
ITurnTelemetrySink sink
)
@@ -18,7 +16,7 @@ public sealed class WebSocketTurnFinalizationService(
private const int AutoFinalizeMinBufferedAudioChunks = 4;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400);
public void ObserveIncomingMessage(CloudSession session, string? text)
public static void ObserveIncomingMessage(CloudSession session, string? text)
{
if (!TryReadTransId(text, out var nextTransId) || string.IsNullOrWhiteSpace(nextTransId))
{
@@ -39,12 +37,7 @@ public sealed class WebSocketTurnFinalizationService(
CancellationToken cancellationToken = default)
{
var turnState = session.TurnState;
if (ShouldIgnoreLateAudio(session))
{
return [];
}
if (!turnState.AwaitingTurnCompletion &&
if (ShouldIgnoreLateAudio(session) || !turnState.AwaitingTurnCompletion &&
!session.FollowUpOpen &&
!turnState.SawListen &&
!string.IsNullOrWhiteSpace(turnState.TransId))
@@ -58,7 +51,7 @@ public sealed class WebSocketTurnFinalizationService(
turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0;
if (envelope.Binary is { Length: > 0 })
{
turnState.BufferedAudioFrames.Add(envelope.Binary.ToArray());
turnState.BufferedAudioFrames.Add([.. envelope.Binary]);
}
turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow;
turnState.AwaitingTurnCompletion = true;
@@ -116,7 +109,7 @@ public sealed class WebSocketTurnFinalizationService(
return await FinalizeTurnAsync(session, envelope, messageType, allowFallbackOnMissingTranscript: false, cancellationToken);
}
public IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope)
public static IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope)
{
PersistTurnHints(session, envelope.Text);
@@ -129,7 +122,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session);
session.TurnState.SawListen = false;
session.TurnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
session.TurnState.TransId ?? session.LastTransId ?? string.Empty,
session.TurnState.ListenRules,
"@be/idle")
@@ -137,8 +130,7 @@ public sealed class WebSocketTurnFinalizationService(
{
Text = map.Text,
DelayMs = map.DelayMs
})
.ToArray();
})];
}
session.TurnState.AwaitingTurnCompletion = true;
@@ -147,17 +139,12 @@ public sealed class WebSocketTurnFinalizationService(
private async Task<TurnContext> ResolveTranscriptAsync(TurnContext turn, CloudSession session, CancellationToken cancellationToken)
{
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript))
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript) || session.TurnState.BufferedAudioBytes <= 0)
{
return turn;
}
if (session.TurnState.BufferedAudioBytes <= 0)
{
return turn;
}
ISttStrategy? strategy = null;
ISttStrategy? strategy;
try
{
strategy = await sttStrategySelector.SelectAsync(turn, cancellationToken);
@@ -254,14 +241,13 @@ public sealed class WebSocketTurnFinalizationService(
}
}
if (root.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Object)
{
if (!root.TryGetProperty("data", out var data) || data.ValueKind != JsonValueKind.Object) return;
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
turnState.ListenRules = rules.EnumerateArray()
turnState.ListenRules = [.. rules.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(rule => !string.IsNullOrWhiteSpace(rule))
.ToArray();
.Where(rule => !string.IsNullOrWhiteSpace(rule))];
session.Metadata["listenRules"] = turnState.ListenRules;
}
@@ -270,15 +256,14 @@ public sealed class WebSocketTurnFinalizationService(
asr.TryGetProperty("hints", out var hints) &&
hints.ValueKind == JsonValueKind.Array)
{
turnState.ListenAsrHints = hints.EnumerateArray()
turnState.ListenAsrHints = [.. hints.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty)
.Where(static hint => !string.IsNullOrWhiteSpace(hint))
.ToArray();
.Where(static hint => !string.IsNullOrWhiteSpace(hint))];
}
if (data.TryGetProperty("hotphrase", out var hotphrase) &&
(hotphrase.ValueKind == JsonValueKind.True || hotphrase.ValueKind == JsonValueKind.False))
hotphrase.ValueKind is JsonValueKind.True or JsonValueKind.False)
{
turnState.ListenHotphrase = hotphrase.GetBoolean();
turnState.HotphraseEmptyTurnCount = 0;
@@ -289,13 +274,12 @@ public sealed class WebSocketTurnFinalizationService(
session.LastIntent = intent.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String)
{
if (!data.TryGetProperty("transcriptHint", out var transcriptHint) ||
transcriptHint.ValueKind != JsonValueKind.String) return;
turnState.AudioTranscriptHint = transcriptHint.GetString();
session.Metadata["audioTranscriptHint"] = turnState.AudioTranscriptHint;
}
}
}
catch
{
// Keep the compatibility layer permissive while captures are still incomplete.
@@ -394,7 +378,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session);
turnState.SawListen = false;
turnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
turnState.TransId ?? session.LastTransId ?? string.Empty,
turnState.ListenRules,
"@be/idle")
@@ -402,8 +386,7 @@ public sealed class WebSocketTurnFinalizationService(
{
Text = map.Text,
DelayMs = map.DelayMs
})
.ToArray();
})];
}
if (ShouldHandleAsLocalNoInput(finalizedTurn))
@@ -448,9 +431,11 @@ public sealed class WebSocketTurnFinalizationService(
turnState.FinalizeAttemptCount += 1;
}
if (allowFallbackOnMissingTranscript &&
switch (allowFallbackOnMissingTranscript)
{
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
IsYesNoTurn(finalizedTurn))
IsYesNoTurn(finalizedTurn):
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
@@ -461,10 +446,9 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session);
return noInputReplies;
}
if (allowFallbackOnMissingTranscript &&
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
string.IsNullOrWhiteSpace(turnState.LastSttError))
string.IsNullOrWhiteSpace(turnState.LastSttError):
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
@@ -476,9 +460,10 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session);
return fallbackReplies;
}
default:
return [];
}
}
var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken);
var listenAction = plan.Actions.OfType<ListenAction>().OrderBy(action => action.Sequence).LastOrDefault();
@@ -487,7 +472,7 @@ public sealed class WebSocketTurnFinalizationService(
session.LastListenType = listenAction?.Mode;
turnState.LastLocalNoInputRule = null;
turnState.LocalNoInputCount = 0;
if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock", Payload: not null } clockAction &&
if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock" } clockAction &&
clockAction.Payload.TryGetValue("domain", out var lastClockDomainValue) &&
lastClockDomainValue is not null)
{
@@ -545,10 +530,7 @@ public sealed class WebSocketTurnFinalizationService(
var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero;
return turnState.AwaitingTurnCompletion &&
turnState.SawListen &&
turnState.BufferedAudioChunkCount >= AutoFinalizeMinBufferedAudioChunks &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
return turnState is { AwaitingTurnCompletion: true, SawListen: true, BufferedAudioChunkCount: >= AutoFinalizeMinBufferedAudioChunks, BufferedAudioBytes: >= AutoFinalizeMinBufferedAudioBytes } &&
turnAge >= AutoFinalizeMinTurnAge;
}
@@ -755,7 +737,7 @@ public sealed class WebSocketTurnFinalizationService(
.FirstOrDefault(IsConstrainedYesNoRule);
}
private static IReadOnlyList<WebSocketReply> BuildLocalNoInputReplies(
private static WebSocketReply[] BuildLocalNoInputReplies(
CloudSession session,
WebSocketTurnState turnState,
string? localRule)
@@ -764,14 +746,12 @@ public sealed class WebSocketTurnFinalizationService(
var effectiveRule = string.IsNullOrWhiteSpace(localRule)
? turnState.ListenRules.FirstOrDefault(IsLocalNoInputRule)
: localRule;
IReadOnlyList<string> rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule];
var rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule];
var maps = ShouldRedirectRepeatedNoInputToIdle(turnState, effectiveRule)
? ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(transId, rules, "@be/idle")
: ResponsePlanToSocketMessagesMapper.MapNoInput(transId, rules);
return maps
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
return [.. maps.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })];
}
private static bool ShouldRedirectRepeatedNoInputToIdle(WebSocketTurnState turnState, string? localRule)
@@ -853,7 +833,7 @@ public sealed class WebSocketTurnFinalizationService(
return string.Empty;
}
return Regex.Replace(transcript.Trim().ToLowerInvariant(), @"[^\w\s]", " ")
return TranscriptNormalizationRegex().Replace(transcript.Trim().ToLowerInvariant(), " ")
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
}
@@ -1036,4 +1016,7 @@ public sealed class WebSocketTurnFinalizationService(
_ => false
};
}
[GeneratedRegex(@"[^\w\s]")]
private static partial Regex TranscriptNormalizationRegex();
}

6
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/ExternalProcessRunner.cs
View File

@@ -6,16 +6,14 @@ public sealed class ExternalProcessRunner : IExternalProcessRunner
{
public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default)
{
using var process = new Process
{
StartInfo = new ProcessStartInfo
using var process = new Process();
process.StartInfo = new ProcessStartInfo
{
FileName = fileName,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
}
};
foreach (var argument in arguments)

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs
View File

@@ -132,7 +132,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var timecoded = lines
.Where(static line => line.StartsWith("[", StringComparison.Ordinal) && line.Contains("-->", StringComparison.Ordinal))
.Where(static line => line.StartsWith('[') && line.Contains("-->", StringComparison.Ordinal))
.Select(static line =>
{
var closingBracket = line.IndexOf(']');
@@ -171,6 +171,6 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
return true;
}
return checkFileExists ? File.Exists(path) : true;
return !checkFileExists || File.Exists(path);
}
}

8
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/OggOpusAudioNormalizer.cs
View File

@@ -79,13 +79,7 @@ internal static class OggOpusAudioNormalizer
private static uint ComputeCrc(byte[] buffer)
{
uint crc = 0;
foreach (var value in buffer)
{
crc = (crc << 8) ^ CrcTable[((crc >> 24) ^ value) & 0xff];
}
return crc;
return buffer.Aggregate<byte, uint>(0, (current, value) => (current << 8) ^ CrcTable[((current >> 24) ^ value) & 0xff]);
}
private static uint[] BuildCrcTable()

1
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs
View File

@@ -7,7 +7,6 @@ using Jibo.Cloud.Infrastructure.Telemetry;
using Jibo.Runtime.Abstractions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Configuration;
using System.IO;
namespace Jibo.Cloud.Infrastructure.DependencyInjection;

15
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryCloudStateStore.cs
View File

@@ -18,7 +18,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
private readonly ConcurrentDictionary<string, string> _symmetricKeys = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, KeyRequestRecord> _keyRequests = new(StringComparer.OrdinalIgnoreCase);
private readonly string? _persistencePath;
private readonly object _syncRoot = new();
private readonly Lock _syncRoot = new();
private readonly List<UpdateManifest> _updates;
private readonly List<MediaRecord> _media = [];
private readonly List<BackupRecord> _backups = [];
@@ -186,13 +186,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
public UpdateManifest RemoveUpdate(string? updateId)
{
var existing = _updates.FirstOrDefault(update => update.UpdateId == updateId);
if (existing is not null)
{
_updates.Remove(existing);
PersistState();
return existing;
}
if (existing is null)
return new UpdateManifest
{
UpdateId = updateId ?? "unknown-update",
@@ -201,6 +195,11 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
ShaHash = "missing",
Subsystem = "unknown"
};
_updates.Remove(existing);
PersistState();
return existing;
}
public IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null)

18
OpenJibo/src/Playground/AsrEvent.cs Normal file
View File

@@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}

12
OpenJibo/src/Playground/AsrUtterance.cs Normal file
View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
}

36
OpenJibo/src/Playground/Program.cs
View File

@@ -2,7 +2,7 @@
using System.Net.WebSockets;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Playground;
Console.Write("Enter Jibo IP: ");
var jiboIp = (Console.ReadLine() ?? "").Trim();
@@ -67,7 +67,7 @@ while (!cts.IsCancellationRequested)
var json = Encoding.UTF8.GetString(ms.ToArray());
AsrEvent? evt = null;
AsrEvent? evt;
try
{
evt = JsonSerializer.Deserialize<AsrEvent>(json);
@@ -86,16 +86,12 @@ while (!cts.IsCancellationRequested)
Console.WriteLine($"[{evt.EventType}] {json}");
if (evt.EventType == "speech_to_text_final")
{
if (evt.EventType != "speech_to_text_final") continue;
var best = PickBestUtterance(evt.Utterances);
if (!string.IsNullOrWhiteSpace(best))
{
if (string.IsNullOrWhiteSpace(best)) continue;
utteranceTcs.TrySetResult(best);
return;
}
}
}
}, cts.Token);
var startPayload = new
@@ -220,27 +216,3 @@ static string BuildReply(string heard)
return text.Contains("your name") ? "I am Jibo, running with a local demo bridge." : $"You said: {heard}";
}
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
}

16
OpenJibo/tests/Jibo.Cloud.Tests/Fixtures/WebSocketFixtureLoader.cs
View File

@@ -17,10 +17,9 @@ internal static class WebSocketFixtureLoader
var root = document.RootElement;
var session = root.GetProperty("session");
var steps = new List<WebSocketFixtureStep>();
foreach (var stepElement in root.GetProperty("steps").EnumerateArray())
{
steps.Add(new WebSocketFixtureStep
var steps = root.GetProperty("steps")
.EnumerateArray()
.Select(stepElement => new WebSocketFixtureStep
{
Message = new WebSocketMessageEnvelope
{
@@ -33,16 +32,15 @@ internal static class WebSocketFixtureLoader
? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray()
: null
},
ExpectedReplyTypes = stepElement.GetProperty("expectedReplyTypes")
ExpectedReplyTypes = [.. stepElement.GetProperty("expectedReplyTypes")
.EnumerateArray()
.Select(item => item.GetString() ?? string.Empty)
.Where(item => !string.IsNullOrWhiteSpace(item))
.ToArray(),
.Where(item => !string.IsNullOrWhiteSpace(item))],
ExpectedReplies = stepElement.TryGetProperty("expectedReplies", out var expectedReplies) && expectedReplies.ValueKind == JsonValueKind.Array
? JsonSerializer.Deserialize<List<ExpectedWebSocketReply>>(expectedReplies.GetRawText(), SerializerOptions) ?? []
: []
});
}
})
.ToList();
return new WebSocketFixture
{

40
OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs
View File

@@ -16,18 +16,18 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Exception("dummy"));
var turnService = new WebSocketTurnFinalizationService(
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
sttStrategySelector.Object,
sink.Object
);
await turnService.HandleTurnAsync(new CloudSession() { TurnState = { BufferedAudioBytes = 100 }}, new WebSocketMessageEnvelope(), "dummy",
await turnService.HandleTurnAsync(new CloudSession { TurnState = { BufferedAudioBytes = 100 } },
new WebSocketMessageEnvelope(), "dummy",
CancellationToken.None);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once());
sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
}
[Fact]
@@ -38,21 +38,23 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new InvalidOperationException("ffmpeg failed"));
var turnService = new WebSocketTurnFinalizationService(
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
sttStrategySelector.Object,
sink.Object
);
var session = new CloudSession();
session.TurnState.AwaitingTurnCompletion = true;
session.TurnState.SawListen = true;
session.TurnState.SawContext = true;
session.TurnState.BufferedAudioBytes = 12000;
session.TurnState.BufferedAudioChunkCount = 5;
session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2);
var session = new CloudSession
{
TurnState =
{
AwaitingTurnCompletion = true,
SawListen = true,
SawContext = true,
BufferedAudioBytes = 12000,
BufferedAudioChunkCount = 5,
FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2)
}
};
var replies = await turnService.HandleContextAsync(
session,
@@ -64,6 +66,8 @@ public sealed class FileTurnTelemetrySinkTests
Assert.Equal(12000, session.TurnState.BufferedAudioBytes);
Assert.Equal("ffmpeg failed", session.TurnState.LastSttError);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once());
sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
}
}

20
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -16,11 +16,9 @@ public sealed class JiboWebSocketServiceTests
public JiboWebSocketServiceTests()
{
_store = new InMemoryCloudStateStore();
var turnContextMapper = new ProtocolToTurnContextMapper();
var contentRepository = new InMemoryJiboExperienceContentRepository();
var contentCache = new JiboExperienceContentCache(contentRepository);
var conversationBroker = new DemoConversationBroker(new JiboInteractionService(contentCache, new DefaultJiboRandomizer()));
var replyMapper = new ResponsePlanToSocketMessagesMapper();
var sttSelector = new DefaultSttStrategySelector(
[
new SyntheticBufferedAudioSttStrategy()
@@ -30,10 +28,7 @@ public sealed class JiboWebSocketServiceTests
_service = new JiboWebSocketService(
_store,
new NullWebSocketTelemetrySink(),
new WebSocketTurnFinalizationService(
turnContextMapper,
conversationBroker,
replyMapper,
new WebSocketTurnFinalizationService(conversationBroker,
sttSelector,
sink));
}
@@ -2639,7 +2634,7 @@ public sealed class JiboWebSocketServiceTests
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-context-reset-token",
Binary = [9, 9, 9, 9]
Binary = "\t\t\t\t"u8.ToArray()
});
var session = _store.FindSessionByToken("hub-context-reset-token");
@@ -2681,8 +2676,8 @@ public sealed class JiboWebSocketServiceTests
var actualTypes = replies.Select(ReadReplyType).ToArray();
Assert.Equal(step.ExpectedReplyTypes, actualTypes);
if (step.ExpectedReplies.Count > 0)
{
if (step.ExpectedReplies.Count <= 0) continue;
Assert.Equal(replies.Count, step.ExpectedReplies.Count);
for (var index = 0; index < step.ExpectedReplies.Count; index += 1)
@@ -2695,20 +2690,19 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal(expectedReply.DelayMs.Value, replies[index].DelayMs);
}
if (expectedReply.JsonSubset is { ValueKind: JsonValueKind.Object } jsonSubset)
{
if (expectedReply.JsonSubset is not { ValueKind: JsonValueKind.Object } jsonSubset) continue;
using var actualPayload = JsonDocument.Parse(replies[index].Text!);
AssertJsonContains(jsonSubset, actualPayload.RootElement);
}
}
}
}
}
private static void AssertJsonContains(JsonElement expected, JsonElement actual)
{
Assert.Equal(expected.ValueKind, actual.ValueKind);
// ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault
switch (expected.ValueKind)
{
case JsonValueKind.Object:

8
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs
View File

@@ -158,14 +158,14 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests
{
Calls.Add((fileName, arguments));
if (string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase))
{
if (!string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase))
return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke",
string.Empty));
var outputPath = arguments[^1];
File.WriteAllBytes(outputPath, "RIFF"u8);
return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty));
}
return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke", string.Empty));
}
}
}