Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactors

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-26 20:57:08 -05:00
parent acbba413db
commit 8c97968d95
20 changed files with 547 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
OpenJibo/OpenJibo.sln.DotSettings
View File

@@ -1,3 +1,18 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/UserDictionary/Words/=ampm/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Arrrr/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=esml/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Hotphrase/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jibo_0027s/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=multichunk/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=nevermind/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=noinput/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=openjibo/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Photobooth/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slnx/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=slowdance/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=timecoded/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Todays/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=whispercpp/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=YESNO/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

23
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Program.cs
View File

@@ -25,22 +25,15 @@ app.Use(async (context, next) =>
var kind = ResolveSocketKind(context.Request.Host.Host, context.Request.Path);
var token = ResolveToken(context.Request);
if (kind == "unknown")
switch (kind)
{
context.Response.StatusCode = StatusCodes.Status404NotFound;
return;
}
if (kind == "api-socket" && string.IsNullOrWhiteSpace(token))
{
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
}
if (kind is "neo-hub-listen" or "neo-hub-proactive" && string.IsNullOrWhiteSpace(token))
{
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
case "unknown":
context.Response.StatusCode = StatusCodes.Status404NotFound;
return;
case "api-socket" when string.IsNullOrWhiteSpace(token):
case "neo-hub-listen" or "neo-hub-proactive" when string.IsNullOrWhiteSpace(token):
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
}
var webSocketService = context.RequestServices.GetRequiredService<JiboWebSocketService>();

123
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboCloudProtocolService.cs
View File

@@ -194,27 +194,25 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
});
}
if (operation is "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or
"ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
"VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect")
switch (operation)
{
return ProtocolDispatchResult.Ok(new
{
id = account.AccountId,
email = account.Email,
firstName = account.FirstName,
lastName = account.LastName,
accessKeyId = account.AccessKeyId,
secretAccessKey = account.SecretAccessKey
});
}
if (operation is "ChangeEmail" or "SendPhoneVerificationCode")
{
return ProtocolDispatchResult.Ok(new
{
id = account.AccountId
});
case "Update" or "ResetKeys" or "Remove" or "ActivateByCode" or "ResendActivationCode" or
"ChangePassword" or "SendPasswordReset" or "PasswordResetByCode" or "UpdatePhoto" or "RemovePhoto" or
"VerifyPhoneByCode" or "AcceptTerms" or "FacebookConnect" or "FacebookMobileConnect":
return ProtocolDispatchResult.Ok(new
{
id = account.AccountId,
email = account.Email,
firstName = account.FirstName,
lastName = account.LastName,
accessKeyId = account.AccessKeyId,
secretAccessKey = account.SecretAccessKey
});
case "ChangeEmail" or "SendPhoneVerificationCode":
return ProtocolDispatchResult.Ok(new
{
id = account.AccountId
});
}
if (operation.Equals("GetAccountByAccessToken", StringComparison.OrdinalIgnoreCase))
@@ -236,8 +234,8 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var haystack = $"{account.Email} {account.FirstName} {account.LastName} {account.AccountId}".ToLowerInvariant();
return ProtocolDispatchResult.Ok(query.Length > 0 && haystack.Contains(query)
? new[]
{
?
[
new
{
id = account.AccountId,
@@ -245,7 +243,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
firstName = account.FirstName,
lastName = account.LastName
}
}
]
: Array.Empty<object>());
}
@@ -382,25 +380,24 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(stateStore.RemoveMedia(ReadStringArray(body, "paths")).Select(MapMedia).ToArray());
}
if (operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
{
var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId;
var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
var reference = ReadHeader(envelope, "x-reference") ?? ReadString(body, "reference") ?? string.Empty;
var isEncrypted = ReadBooleanHeader(envelope, "x-encrypted") || ReadBool(body, "isEncrypted");
var meta = ReadObject(body, "meta") ?? new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream";
meta["contentType"] = contentType;
if (!string.IsNullOrWhiteSpace(envelope.BodyText))
{
meta["bodyText"] = envelope.BodyText;
}
if (!operation.Equals("Create", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(Array.Empty<object>());
return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta)));
var loopId = ReadHeader(envelope, "x-loop-id") ?? ReadString(body, "loopId") ?? stateStore.GetLoops()[0].LoopId;
var path = ReadHeader(envelope, "x-path") ?? ReadString(body, "path") ?? $"/media/{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
var reference = ReadHeader(envelope, "x-reference") ?? ReadString(body, "reference") ?? string.Empty;
var isEncrypted = ReadBooleanHeader(envelope, "x-encrypted") || ReadBool(body, "isEncrypted");
var meta = ReadObject(body, "meta") ?? new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream";
meta["contentType"] = contentType;
if (!string.IsNullOrWhiteSpace(envelope.BodyText))
{
meta["bodyText"] = envelope.BodyText;
}
return ProtocolDispatchResult.Ok(Array.Empty<object>());
return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta)));
}
private ProtocolDispatchResult HandlePerson(string operation)
@@ -430,9 +427,10 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
});
}
string? symmetricKey;
if (operation.Equals("CreateSymmetricKey", StringComparison.OrdinalIgnoreCase))
{
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new
{
loopId,
@@ -472,18 +470,17 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return ProtocolDispatchResult.Ok(new { ok = true });
}
if (operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase))
{
var symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new
{
loopId,
key = symmetricKey,
symmetricKey
});
}
if (!operation.Equals("LoadSymmetricKey", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(new { ok = true, operation });
symmetricKey = stateStore.GetOrCreateSymmetricKey(loopId);
return ProtocolDispatchResult.Ok(new
{
loopId,
key = symmetricKey,
symmetricKey
});
return ProtocolDispatchResult.Ok(new { ok = true, operation });
}
private ProtocolDispatchResult HandleRobot(string operation, ProtocolEnvelope envelope)
@@ -509,23 +506,22 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
});
}
if (operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase))
{
var profile = stateStore.GetRobotProfile();
if (!operation.Equals("GetRobot", StringComparison.OrdinalIgnoreCase))
return ProtocolDispatchResult.Ok(new
{
id = ReadString(envelope.TryParseBody(), "id") ?? profile.RobotId,
payload = profile.Payload,
calibrationPayload = profile.CalibrationPayload,
updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(),
created = profile.CreatedUtc.ToUnixTimeMilliseconds()
result = "ok"
});
}
var profile = stateStore.GetRobotProfile();
return ProtocolDispatchResult.Ok(new
{
result = "ok"
id = ReadString(envelope.TryParseBody(), "id") ?? profile.RobotId,
payload = profile.Payload,
calibrationPayload = profile.CalibrationPayload,
updated = profile.UpdatedUtc.ToUnixTimeMilliseconds(),
created = profile.CreatedUtc.ToUnixTimeMilliseconds()
});
}
private ProtocolDispatchResult HandleUpdate(string operation, ProtocolEnvelope envelope)
@@ -674,10 +670,9 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return [];
}
return property.EnumerateArray()
return [.. property.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(item => !string.IsNullOrWhiteSpace(item))
.ToArray();
.Where(item => !string.IsNullOrWhiteSpace(item))];
}
private static IDictionary<string, object?>? ReadObject(JsonElement? element, string propertyName)

150
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -484,14 +484,12 @@ public sealed class JiboInteractionService(
return "hello";
}
if (isYesNoTurn && MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh"))
switch (isYesNoTurn)
{
return "yes";
}
if (isYesNoTurn && MatchesAny(loweredTranscript, "no", "nope", "nah"))
{
return "no";
case true when MatchesAny(loweredTranscript, "yes", "yeah", "yup", "sure", "uh huh"):
return "yes";
case true when MatchesAny(loweredTranscript, "no", "nope", "nah"):
return "no";
}
if (MatchesAny(loweredTranscript, "what time is it", "current time", "the time", "time is it") ||
@@ -752,12 +750,7 @@ public sealed class JiboInteractionService(
}
var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints);
if (!string.IsNullOrWhiteSpace(fuzzyHintMatch))
{
return fuzzyHintMatch;
}
return transcript;
return !string.IsNullOrWhiteSpace(fuzzyHintMatch) ? fuzzyHintMatch : transcript;
}
private static bool IsYesNoTurn(TurnContext turn)
@@ -805,11 +798,10 @@ public sealed class JiboInteractionService(
}
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance)
{
bestDistance = distance;
bestHint = hint;
}
if (distance >= bestDistance) continue;
bestDistance = distance;
bestHint = hint;
}
return bestDistance <= 2 ? bestHint : null;
@@ -996,14 +988,12 @@ public sealed class JiboInteractionService(
{
var compactHour = compact.Length switch
{
3 => compactValue / 100,
4 => compactValue / 100,
3 or 4 => compactValue / 100,
_ => -1
};
var compactMinute = compact.Length switch
{
3 => compactValue % 100,
4 => compactValue % 100,
3 or 4 => compactValue % 100,
_ => -1
};
if (compactHour is >= 1 and <= 12 && compactMinute is >= 0 and <= 59)
@@ -1023,13 +1013,13 @@ public sealed class JiboInteractionService(
var hourToken = match.Groups["hour"].Value;
var minuteToken = match.Groups["minute"].Success ? match.Groups["minute"].Value : "00";
var hour = ParseNumberToken(hourToken);
if (hour is null || hour is < 1 or > 12)
if (hour is null or < 1 or > 12)
{
return null;
}
var minute = ParseNumberToken(minuteToken);
if (minute is null || minute is < 0 or > 59)
if (minute is null or < 0 or > 59)
{
return null;
}
@@ -1146,7 +1136,7 @@ public sealed class JiboInteractionService(
return lastClockDomain;
}
var combinedRules = clientRules.Concat(listenRules);
var combinedRules = clientRules.Concat(listenRules).ToArray();
if (combinedRules.Any(rule =>
rule.Contains("timer", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)))
@@ -1154,14 +1144,9 @@ public sealed class JiboInteractionService(
return "timer";
}
if (combinedRules.Any(rule =>
rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)))
{
return "alarm";
}
return null;
return combinedRules.Any(rule =>
rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) &&
!rule.Contains("alarm_timer_query_menu", StringComparison.OrdinalIgnoreCase)) ? "alarm" : null;
}
private static bool IsTimerRequest(string loweredTranscript)
@@ -1303,12 +1288,7 @@ public sealed class JiboInteractionService(
}
var match = VolumeLevelPattern.Match(loweredTranscript);
if (!match.Success)
{
return null;
}
return TryNormalizeVolumeLevel(match.Groups["value"].Value);
return !match.Success ? null : TryNormalizeVolumeLevel(match.Groups["value"].Value);
}
private static string? TryNormalizeVolumeLevel(string token)
@@ -1367,13 +1347,15 @@ public sealed class JiboInteractionService(
}
var parts = valueToken.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length >= 2)
if (parts.Length < 2)
return parts.Length > 0
? ParseNumberToken(parts[^1])
: null;
parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2)));
if (parsed is not null)
{
parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2)));
if (parsed is not null)
{
return parsed;
}
return parsed;
}
return parts.Length > 0
@@ -1389,18 +1371,76 @@ public sealed class JiboInteractionService(
return numeric;
}
if (normalized.Contains(' '))
if (!normalized.Contains(' '))
{
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length == 2)
return normalized switch
{
var first = ParseNumberToken(parts[0]);
var second = ParseNumberToken(parts[1]);
if (first is >= 20 && second is >= 0 and < 10)
{
return first + second;
}
}
"a" or "an" => 1,
"one" => 1,
"two" => 2,
"three" => 3,
"four" => 4,
"five" => 5,
"six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length != 2)
{
return normalized switch
{
"a" or "an" => 1,
"one" => 1,
"two" => 2,
"three" => 3,
"four" => 4,
"five" => 5,
"six" => 6,
"seven" => 7,
"eight" => 8,
"nine" => 9,
"ten" => 10,
"eleven" => 11,
"twelve" => 12,
"thirteen" => 13,
"fourteen" => 14,
"fifteen" => 15,
"sixteen" => 16,
"seventeen" => 17,
"eighteen" => 18,
"nineteen" => 19,
"twenty" => 20,
"thirty" => 30,
"forty" => 40,
"fifty" => 50,
_ => null
};
}
var first = ParseNumberToken(parts[0]);
var second = ParseNumberToken(parts[1]);
if (first is >= 20 && second is >= 0 and < 10)
{
return first + second;
}
return normalized switch

71
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboWebSocketService.cs
View File

@@ -32,47 +32,48 @@ public sealed class JiboWebSocketService(
var parsedType = ReadMessageType(envelope.Text);
session.LastMessageType = parsedType;
turnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
if (parsedType == "CONTEXT")
switch (parsedType)
{
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
case "CONTEXT":
{
["transID"] = session.TurnState.TransId
}, cancellationToken);
return replies;
}
if (parsedType == "LISTEN")
{
var replies = ContainsInlineTurnPayload(envelope.Text)
? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
: turnFinalizationService.HandleListenSetup(session, envelope);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received", new Dictionary<string, object?>
{
["transID"] = session.TurnState.TransId
}, cancellationToken);
return replies;
}
case "LISTEN":
{
["messageType"] = parsedType,
["replyCount"] = replies.Count,
["transcript"] = session.LastTranscript,
["intent"] = session.LastIntent
}, cancellationToken);
return replies;
}
if (parsedType is "CLIENT_NLU" or "CLIENT_ASR")
{
var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
var replies = ContainsInlineTurnPayload(envelope.Text)
? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
: WebSocketTurnFinalizationService.HandleListenSetup(session, envelope);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
{
["messageType"] = parsedType,
["replyCount"] = replies.Count,
["transcript"] = session.LastTranscript,
["intent"] = session.LastIntent
}, cancellationToken);
return replies;
}
case "CLIENT_NLU" or "CLIENT_ASR":
{
["messageType"] = parsedType,
["replyCount"] = replies.Count,
["transcript"] = session.LastTranscript,
["intent"] = session.LastIntent
}, cancellationToken);
return replies;
var replies = await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken);
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed", new Dictionary<string, object?>
{
["messageType"] = parsedType,
["replyCount"] = replies.Count,
["transcript"] = session.LastTranscript,
["intent"] = session.LastIntent
}, cancellationToken);
return replies;
}
default:
return [];
}
return [];
}
private static string ReadMessageType(string? text)

80
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs
View File

@@ -93,52 +93,46 @@ public sealed class ProtocolToTurnContextMapper
using var document = JsonDocument.Parse(text);
var root = document.RootElement;
if (root.TryGetProperty("data", out var data))
if (!root.TryGetProperty("data", out var data)) return null;
if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String)
{
if (data.TryGetProperty("text", out var transcript) && transcript.ValueKind == JsonValueKind.String)
{
return transcript.GetString();
}
if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("text", out var asrText) &&
asrText.ValueKind == JsonValueKind.String)
{
return asrText.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String)
{
return transcriptHint.GetString();
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
attributes["clientIntent"] = intent.GetString();
}
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
attributes["clientRules"] = rules.EnumerateArray()
.Where(item => item.ValueKind == JsonValueKind.String)
.Select(item => item.GetString() ?? string.Empty)
.Where(rule => !string.IsNullOrWhiteSpace(rule))
.ToArray();
}
if (data.TryGetProperty("entities", out var entities) && entities.ValueKind == JsonValueKind.Object)
{
attributes["clientEntities"] = entities.Clone();
}
if (intent.ValueKind == JsonValueKind.String)
{
return intent.GetString();
}
return transcript.GetString();
}
return null;
if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("text", out var asrText) &&
asrText.ValueKind == JsonValueKind.String)
{
return asrText.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String)
{
return transcriptHint.GetString();
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
attributes["clientIntent"] = intent.GetString();
}
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
attributes["clientRules"] = rules.EnumerateArray()
.Where(item => item.ValueKind == JsonValueKind.String)
.Select(item => item.GetString() ?? string.Empty)
.Where(rule => !string.IsNullOrWhiteSpace(rule))
.ToArray();
}
if (data.TryGetProperty("entities", out var entities) && entities.ValueKind == JsonValueKind.Object)
{
attributes["clientEntities"] = entities.Clone();
}
return intent.ValueKind == JsonValueKind.String ? intent.GetString() : null;
}
catch
{

178
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -6,7 +6,8 @@ namespace Jibo.Cloud.Application.Services;
public sealed class ResponsePlanToSocketMessagesMapper
{
public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session, bool emitSkillActions)
public static IReadOnlyList<SocketReplyPlan> Map(ResponsePlan plan, TurnContext turn, CloudSession session,
bool emitSkillActions)
{
var speak = plan.Actions.OfType<SpeakAction>().FirstOrDefault();
var skill = plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault();
@@ -22,7 +23,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
var isYesNoIntent = string.Equals(plan.IntentName, "yes", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "no", StringComparison.OrdinalIgnoreCase);
var isWordOfDayLaunch = string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase);
var isWordOfDayGuess = string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var isWordOfDayGuess =
string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var isRadioLaunch = string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase);
var isStopCommand = string.Equals(plan.IntentName, "stop", StringComparison.OrdinalIgnoreCase);
@@ -53,56 +55,68 @@ public sealed class ResponsePlanToSocketMessagesMapper
var outboundIntent = isGlobalCommand && !string.IsNullOrWhiteSpace(globalIntent)
? globalIntent
: isWordOfDayLaunch
? "menu"
: isRadioLaunch
? "menu"
: isSettingsLaunch && !string.IsNullOrWhiteSpace(localIntent)
? localIntent
: (isPhotoGalleryLaunch || isPhotoCreateLaunch) && !string.IsNullOrWhiteSpace(localIntent)
? localIntent
: isClockSkillLaunch && !string.IsNullOrWhiteSpace(clockIntent)
? clockIntent
: isWordOfDayGuess
? "guess"
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent)
? clientIntent
: plan.IntentName ?? "unknown";
? "menu"
: isRadioLaunch
? "menu"
: isSettingsLaunch && !string.IsNullOrWhiteSpace(localIntent)
? localIntent
: (isPhotoGalleryLaunch || isPhotoCreateLaunch) && !string.IsNullOrWhiteSpace(localIntent)
? localIntent
: isClockSkillLaunch && !string.IsNullOrWhiteSpace(clockIntent)
? clockIntent
: isWordOfDayGuess
? "guess"
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(clientIntent)
? clientIntent
: plan.IntentName ?? "unknown";
var outboundAsrText = isWordOfDayGuess && !string.IsNullOrWhiteSpace(wordOfDayGuess)
? wordOfDayGuess
: isWordOfDayLaunch
? string.Empty
: isGlobalCommand
? transcript
: isRadioLaunch
? transcript
: isSettingsLaunch
? transcript
: isPhotoGalleryLaunch || isPhotoCreateLaunch
? transcript
: isClockSkillLaunch
? transcript
: string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(nluGuess)
? nluGuess
: isYesNoTurn && isYesNoIntent
? transcript
: string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(clientIntent)
? clientIntent
: transcript;
? string.Empty
: isGlobalCommand
? transcript
: isRadioLaunch
? transcript
: isSettingsLaunch
? transcript
: isPhotoGalleryLaunch || isPhotoCreateLaunch
? transcript
: isClockSkillLaunch
? transcript
: string.Equals(clientIntent, "guess", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(nluGuess)
? nluGuess
: isYesNoTurn && isYesNoIntent
? transcript
: string.Equals(messageType, "CLIENT_NLU",
StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(clientIntent)
? clientIntent
: transcript;
var outboundRules = isWordOfDayLaunch
? ["word-of-the-day/menu"]
: isGlobalCommand
? BuildGlobalCommandRules(rules)
: isRadioLaunch
? Array.Empty<string>()
: isSettingsLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>()
: isPhotoGalleryLaunch || isPhotoCreateLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>()
: isClockSkillLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ? rules : Array.Empty<string>()
: isWordOfDayGuess
? ["word-of-the-day/puzzle"]
: isYesNoTurn && isYesNoIntent ? [yesNoRule!] : rules;
? BuildGlobalCommandRules(rules)
: isRadioLaunch
? []
: isSettingsLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isPhotoGalleryLaunch || isPhotoCreateLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isClockSkillLaunch
? string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
? rules
: []
: isWordOfDayGuess
? ["word-of-the-day/puzzle"]
: isYesNoTurn && isYesNoIntent
? [yesNoRule!]
: rules;
var entities = ReadEntities(
turn,
messageType,
@@ -280,7 +294,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return messages;
}
public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId, IReadOnlyList<string> rules)
public static IReadOnlyList<SocketReplyPlan> MapFallback(CloudSession session, string transId,
IReadOnlyList<string> rules)
{
return
[
@@ -376,12 +391,12 @@ public sealed class ResponsePlanToSocketMessagesMapper
var messages = new List<SocketReplyPlan>(MapNoInput(transId, rules))
{
new(JsonSerializer.Serialize(BuildSkillRedirectPayload(
transId,
skillId,
string.Empty,
string.Empty,
[],
new Dictionary<string, object?>())),
transId,
skillId,
string.Empty,
string.Empty,
[],
new Dictionary<string, object?>())),
redirectDelayMs)
};
@@ -402,7 +417,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch
{
IReadOnlyList<string> typedRules => typedRules,
IEnumerable<string> rules => rules.Where(rule => !string.IsNullOrWhiteSpace(rule)).ToArray(),
IEnumerable<string> rules => [.. rules.Where(rule => !string.IsNullOrWhiteSpace(rule))],
_ => []
};
}
@@ -487,12 +502,11 @@ public sealed class ResponsePlanToSocketMessagesMapper
entities["seconds"] = timerSeconds ?? "null";
}
if (string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) &&
(!string.IsNullOrWhiteSpace(alarmTime) || !string.IsNullOrWhiteSpace(alarmAmPm)))
{
entities["time"] = alarmTime ?? string.Empty;
entities["ampm"] = alarmAmPm ?? string.Empty;
}
if (!string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) ||
(string.IsNullOrWhiteSpace(alarmTime) && string.IsNullOrWhiteSpace(alarmAmPm))) return entities;
entities["time"] = alarmTime ?? string.Empty;
entities["ampm"] = alarmAmPm ?? string.Empty;
return entities;
}
@@ -505,12 +519,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
};
}
if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase))
{
return new Dictionary<string, object?>();
}
if (!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
if (!string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase) ||
!turn.Attributes.TryGetValue("clientEntities", out var value) || value is null)
{
return new Dictionary<string, object?>();
}
@@ -582,7 +592,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value switch
{
JsonElement { ValueKind: JsonValueKind.Object } jsonElement
when jsonElement.TryGetProperty(entityName, out var property) && property.ValueKind == JsonValueKind.String
when jsonElement.TryGetProperty(entityName, out var property) &&
property.ValueKind == JsonValueKind.String
=> property.GetString(),
IReadOnlyDictionary<string, string> typed when typed.TryGetValue(entityName, out var entityValue)
=> entityValue,
@@ -602,7 +613,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return value?.ToString();
}
private static string? ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess)
private static string ResolveWordOfDayGuess(TurnContext turn, string transcript, string? nluGuess)
{
if (!string.IsNullOrWhiteSpace(nluGuess))
{
@@ -662,11 +673,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
}
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance)
{
bestDistance = distance;
bestHint = hint;
}
if (distance >= bestDistance) continue;
bestDistance = distance;
bestHint = hint;
}
return bestDistance <= 2 ? bestHint : null;
@@ -704,10 +714,12 @@ public sealed class ResponsePlanToSocketMessagesMapper
return previous[right.Length];
}
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill)
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak,
InvokeNativeSkillAction? skill)
{
var skillPayload = skill?.Payload;
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only", StringComparison.OrdinalIgnoreCase))
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only",
StringComparison.OrdinalIgnoreCase))
{
return BuildCompletionOnlySkillPayload(
transId,
@@ -718,12 +730,14 @@ public sealed class ResponsePlanToSocketMessagesMapper
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase);
var payloadSkill = ReadPayloadString(skillPayload, "skillId");
var skillId = string.IsNullOrWhiteSpace(payloadSkill) ? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill" : payloadSkill;
var skillId = string.IsNullOrWhiteSpace(payloadSkill)
? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill"
: payloadSkill;
var esml = ReadPayloadString(skillPayload, "esml") ?? (isDance
? "<speak>Okay.<break size='0.2'/> Watch this.<anim cat='dance' filter='music, rom-upbeat' /></speak>"
: isJoke
? $"<speak><es cat='happy' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>"
: $"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>");
? $"<speak><es cat='happy' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>"
: $"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>{EscapeXml(speak.Text)}</es></speak>");
var mimId = ReadPayloadString(skillPayload, "mim_id") ?? (isJoke ? "runtime-joke" : "runtime-chat");
var mimType = ReadPayloadString(skillPayload, "mim_type") ?? "announcement";
@@ -799,9 +813,10 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static IReadOnlyList<string> BuildGlobalCommandRules(IReadOnlyList<string> rules)
{
return rules.Any(static rule => string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase))
return rules.Any(static rule =>
string.Equals(rule, "globals/global_commands_launch", StringComparison.OrdinalIgnoreCase))
? ["globals/global_commands_launch"]
: Array.Empty<string>();
: [];
}
private static object BuildGenericFallbackSkillPayload(string transId)
@@ -829,7 +844,8 @@ public sealed class ResponsePlanToSocketMessagesMapper
{
play = new
{
esml = "<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>",
esml =
"<speak><es cat='neutral' filter='!ssa-only, !sfx-only' endNeutral='true'>I heard you.</es></speak>",
meta = new
{
prompt_id = "RUNTIME_PROMPT",

187
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -6,10 +6,8 @@ using System.Text.RegularExpressions;
namespace Jibo.Cloud.Application.Services;
public sealed class WebSocketTurnFinalizationService(
ProtocolToTurnContextMapper turnContextMapper,
public sealed partial class WebSocketTurnFinalizationService(
IConversationBroker conversationBroker,
ResponsePlanToSocketMessagesMapper replyMapper,
ISttStrategySelector sttStrategySelector,
ITurnTelemetrySink sink
)
@@ -18,7 +16,7 @@ public sealed class WebSocketTurnFinalizationService(
private const int AutoFinalizeMinBufferedAudioChunks = 4;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400);
public void ObserveIncomingMessage(CloudSession session, string? text)
public static void ObserveIncomingMessage(CloudSession session, string? text)
{
if (!TryReadTransId(text, out var nextTransId) || string.IsNullOrWhiteSpace(nextTransId))
{
@@ -39,12 +37,7 @@ public sealed class WebSocketTurnFinalizationService(
CancellationToken cancellationToken = default)
{
var turnState = session.TurnState;
if (ShouldIgnoreLateAudio(session))
{
return [];
}
if (!turnState.AwaitingTurnCompletion &&
if (ShouldIgnoreLateAudio(session) || !turnState.AwaitingTurnCompletion &&
!session.FollowUpOpen &&
!turnState.SawListen &&
!string.IsNullOrWhiteSpace(turnState.TransId))
@@ -58,7 +51,7 @@ public sealed class WebSocketTurnFinalizationService(
turnState.BufferedAudioBytes += envelope.Binary?.Length ?? 0;
if (envelope.Binary is { Length: > 0 })
{
turnState.BufferedAudioFrames.Add(envelope.Binary.ToArray());
turnState.BufferedAudioFrames.Add([.. envelope.Binary]);
}
turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow;
turnState.AwaitingTurnCompletion = true;
@@ -116,7 +109,7 @@ public sealed class WebSocketTurnFinalizationService(
return await FinalizeTurnAsync(session, envelope, messageType, allowFallbackOnMissingTranscript: false, cancellationToken);
}
public IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope)
public static IReadOnlyList<WebSocketReply> HandleListenSetup(CloudSession session, WebSocketMessageEnvelope envelope)
{
PersistTurnHints(session, envelope.Text);
@@ -129,7 +122,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session);
session.TurnState.SawListen = false;
session.TurnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
session.TurnState.TransId ?? session.LastTransId ?? string.Empty,
session.TurnState.ListenRules,
"@be/idle")
@@ -137,8 +130,7 @@ public sealed class WebSocketTurnFinalizationService(
{
Text = map.Text,
DelayMs = map.DelayMs
})
.ToArray();
})];
}
session.TurnState.AwaitingTurnCompletion = true;
@@ -147,17 +139,12 @@ public sealed class WebSocketTurnFinalizationService(
private async Task<TurnContext> ResolveTranscriptAsync(TurnContext turn, CloudSession session, CancellationToken cancellationToken)
{
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript))
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript) || session.TurnState.BufferedAudioBytes <= 0)
{
return turn;
}
if (session.TurnState.BufferedAudioBytes <= 0)
{
return turn;
}
ISttStrategy? strategy = null;
ISttStrategy? strategy;
try
{
strategy = await sttStrategySelector.SelectAsync(turn, cancellationToken);
@@ -254,47 +241,44 @@ public sealed class WebSocketTurnFinalizationService(
}
}
if (root.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Object)
if (!root.TryGetProperty("data", out var data) || data.ValueKind != JsonValueKind.Object) return;
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
turnState.ListenRules = rules.EnumerateArray()
turnState.ListenRules = [.. rules.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
.Where(rule => !string.IsNullOrWhiteSpace(rule))
.ToArray();
session.Metadata["listenRules"] = turnState.ListenRules;
}
if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("hints", out var hints) &&
hints.ValueKind == JsonValueKind.Array)
{
turnState.ListenAsrHints = hints.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty)
.Where(static hint => !string.IsNullOrWhiteSpace(hint))
.ToArray();
}
if (data.TryGetProperty("hotphrase", out var hotphrase) &&
(hotphrase.ValueKind == JsonValueKind.True || hotphrase.ValueKind == JsonValueKind.False))
{
turnState.ListenHotphrase = hotphrase.GetBoolean();
turnState.HotphraseEmptyTurnCount = 0;
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
session.LastIntent = intent.GetString();
}
if (data.TryGetProperty("transcriptHint", out var transcriptHint) && transcriptHint.ValueKind == JsonValueKind.String)
{
turnState.AudioTranscriptHint = transcriptHint.GetString();
session.Metadata["audioTranscriptHint"] = turnState.AudioTranscriptHint;
}
.Where(rule => !string.IsNullOrWhiteSpace(rule))];
session.Metadata["listenRules"] = turnState.ListenRules;
}
if (data.TryGetProperty("asr", out var asr) &&
asr.ValueKind == JsonValueKind.Object &&
asr.TryGetProperty("hints", out var hints) &&
hints.ValueKind == JsonValueKind.Array)
{
turnState.ListenAsrHints = [.. hints.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty)
.Where(static hint => !string.IsNullOrWhiteSpace(hint))];
}
if (data.TryGetProperty("hotphrase", out var hotphrase) &&
hotphrase.ValueKind is JsonValueKind.True or JsonValueKind.False)
{
turnState.ListenHotphrase = hotphrase.GetBoolean();
turnState.HotphraseEmptyTurnCount = 0;
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
session.LastIntent = intent.GetString();
}
if (!data.TryGetProperty("transcriptHint", out var transcriptHint) ||
transcriptHint.ValueKind != JsonValueKind.String) return;
turnState.AudioTranscriptHint = transcriptHint.GetString();
session.Metadata["audioTranscriptHint"] = turnState.AudioTranscriptHint;
}
catch
{
@@ -394,7 +378,7 @@ public sealed class WebSocketTurnFinalizationService(
ResetBufferedAudio(session);
turnState.SawListen = false;
turnState.SawContext = false;
return ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
return [.. ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(
turnState.TransId ?? session.LastTransId ?? string.Empty,
turnState.ListenRules,
"@be/idle")
@@ -402,8 +386,7 @@ public sealed class WebSocketTurnFinalizationService(
{
Text = map.Text,
DelayMs = map.DelayMs
})
.ToArray();
})];
}
if (ShouldHandleAsLocalNoInput(finalizedTurn))
@@ -448,36 +431,38 @@ public sealed class WebSocketTurnFinalizationService(
turnState.FinalizeAttemptCount += 1;
}
if (allowFallbackOnMissingTranscript &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
IsYesNoTurn(finalizedTurn))
switch (allowFallbackOnMissingTranscript)
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = null;
session.LastListenType = "no-input";
var localRule = ReadPrimaryYesNoRule(finalizedTurn);
var noInputReplies = BuildLocalNoInputReplies(session, turnState, localRule);
ResetBufferedAudio(session);
return noInputReplies;
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
IsYesNoTurn(finalizedTurn):
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = null;
session.LastListenType = "no-input";
var localRule = ReadPrimaryYesNoRule(finalizedTurn);
var noInputReplies = BuildLocalNoInputReplies(session, turnState, localRule);
ResetBufferedAudio(session);
return noInputReplies;
}
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
string.IsNullOrWhiteSpace(turnState.LastSttError):
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = "heyJibo";
session.LastListenType = "fallback";
var fallbackReplies = ResponsePlanToSocketMessagesMapper.MapFallback(session, turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.ListenRules)
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
ResetBufferedAudio(session);
return fallbackReplies;
}
default:
return [];
}
if (allowFallbackOnMissingTranscript &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
string.IsNullOrWhiteSpace(turnState.LastSttError))
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = "heyJibo";
session.LastListenType = "fallback";
var fallbackReplies = ResponsePlanToSocketMessagesMapper.MapFallback(session, turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.ListenRules)
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
ResetBufferedAudio(session);
return fallbackReplies;
}
return [];
}
var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken);
@@ -487,7 +472,7 @@ public sealed class WebSocketTurnFinalizationService(
session.LastListenType = listenAction?.Mode;
turnState.LastLocalNoInputRule = null;
turnState.LocalNoInputCount = 0;
if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock", Payload: not null } clockAction &&
if (plan.Actions.OfType<InvokeNativeSkillAction>().FirstOrDefault() is { SkillName: "@be/clock" } clockAction &&
clockAction.Payload.TryGetValue("domain", out var lastClockDomainValue) &&
lastClockDomainValue is not null)
{
@@ -545,10 +530,7 @@ public sealed class WebSocketTurnFinalizationService(
var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero;
return turnState.AwaitingTurnCompletion &&
turnState.SawListen &&
turnState.BufferedAudioChunkCount >= AutoFinalizeMinBufferedAudioChunks &&
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
return turnState is { AwaitingTurnCompletion: true, SawListen: true, BufferedAudioChunkCount: >= AutoFinalizeMinBufferedAudioChunks, BufferedAudioBytes: >= AutoFinalizeMinBufferedAudioBytes } &&
turnAge >= AutoFinalizeMinTurnAge;
}
@@ -755,7 +737,7 @@ public sealed class WebSocketTurnFinalizationService(
.FirstOrDefault(IsConstrainedYesNoRule);
}
private static IReadOnlyList<WebSocketReply> BuildLocalNoInputReplies(
private static WebSocketReply[] BuildLocalNoInputReplies(
CloudSession session,
WebSocketTurnState turnState,
string? localRule)
@@ -764,14 +746,12 @@ public sealed class WebSocketTurnFinalizationService(
var effectiveRule = string.IsNullOrWhiteSpace(localRule)
? turnState.ListenRules.FirstOrDefault(IsLocalNoInputRule)
: localRule;
IReadOnlyList<string> rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule];
var rules = string.IsNullOrWhiteSpace(effectiveRule) ? turnState.ListenRules : [effectiveRule];
var maps = ShouldRedirectRepeatedNoInputToIdle(turnState, effectiveRule)
? ResponsePlanToSocketMessagesMapper.MapNoInputAndRedirectToSkill(transId, rules, "@be/idle")
: ResponsePlanToSocketMessagesMapper.MapNoInput(transId, rules);
return maps
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
return [.. maps.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })];
}
private static bool ShouldRedirectRepeatedNoInputToIdle(WebSocketTurnState turnState, string? localRule)
@@ -853,7 +833,7 @@ public sealed class WebSocketTurnFinalizationService(
return string.Empty;
}
return Regex.Replace(transcript.Trim().ToLowerInvariant(), @"[^\w\s]", " ")
return TranscriptNormalizationRegex().Replace(transcript.Trim().ToLowerInvariant(), " ")
.Replace(" ", " ", StringComparison.Ordinal)
.Trim();
}
@@ -1036,4 +1016,7 @@ public sealed class WebSocketTurnFinalizationService(
_ => false
};
}
[GeneratedRegex(@"[^\w\s]")]
private static partial Regex TranscriptNormalizationRegex();
}

16
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/ExternalProcessRunner.cs
View File

@@ -6,16 +6,14 @@ public sealed class ExternalProcessRunner : IExternalProcessRunner
{
public async Task<ExternalProcessResult> RunAsync(string fileName, IReadOnlyList<string> arguments, CancellationToken cancellationToken = default)
{
using var process = new Process
using var process = new Process();
process.StartInfo = new ProcessStartInfo
{
StartInfo = new ProcessStartInfo
{
FileName = fileName,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
}
FileName = fileName,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
foreach (var argument in arguments)

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs
View File

@@ -132,7 +132,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var timecoded = lines
.Where(static line => line.StartsWith("[", StringComparison.Ordinal) && line.Contains("-->", StringComparison.Ordinal))
.Where(static line => line.StartsWith('[') && line.Contains("-->", StringComparison.Ordinal))
.Select(static line =>
{
var closingBracket = line.IndexOf(']');
@@ -171,6 +171,6 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
return true;
}
return checkFileExists ? File.Exists(path) : true;
return !checkFileExists || File.Exists(path);
}
}

8
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/OggOpusAudioNormalizer.cs
View File

@@ -79,13 +79,7 @@ internal static class OggOpusAudioNormalizer
private static uint ComputeCrc(byte[] buffer)
{
uint crc = 0;
foreach (var value in buffer)
{
crc = (crc << 8) ^ CrcTable[((crc >> 24) ^ value) & 0xff];
}
return crc;
return buffer.Aggregate<byte, uint>(0, (current, value) => (current << 8) ^ CrcTable[((current >> 24) ^ value) & 0xff]);
}
private static uint[] BuildCrcTable()

1
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs
View File

@@ -7,7 +7,6 @@ using Jibo.Cloud.Infrastructure.Telemetry;
using Jibo.Runtime.Abstractions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Configuration;
using System.IO;
namespace Jibo.Cloud.Infrastructure.DependencyInjection;

29
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/InMemoryCloudStateStore.cs
View File

@@ -18,7 +18,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
private readonly ConcurrentDictionary<string, string> _symmetricKeys = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, KeyRequestRecord> _keyRequests = new(StringComparer.OrdinalIgnoreCase);
private readonly string? _persistencePath;
private readonly object _syncRoot = new();
private readonly Lock _syncRoot = new();
private readonly List<UpdateManifest> _updates;
private readonly List<MediaRecord> _media = [];
private readonly List<BackupRecord> _backups = [];
@@ -186,21 +186,20 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
public UpdateManifest RemoveUpdate(string? updateId)
{
var existing = _updates.FirstOrDefault(update => update.UpdateId == updateId);
if (existing is not null)
{
_updates.Remove(existing);
PersistState();
return existing;
}
if (existing is null)
return new UpdateManifest
{
UpdateId = updateId ?? "unknown-update",
Changes = "Update not found",
Url = "https://api.jibo.com/update/missing",
ShaHash = "missing",
Subsystem = "unknown"
};
_updates.Remove(existing);
PersistState();
return existing;
return new UpdateManifest
{
UpdateId = updateId ?? "unknown-update",
Changes = "Update not found",
Url = "https://api.jibo.com/update/missing",
ShaHash = "missing",
Subsystem = "unknown"
};
}
public IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null)

18
OpenJibo/src/Playground/AsrEvent.cs Normal file
View File

@@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}

12
OpenJibo/src/Playground/AsrUtterance.cs Normal file
View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Playground;
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
}

42
OpenJibo/src/Playground/Program.cs
View File

@@ -2,7 +2,7 @@
using System.Net.WebSockets;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Playground;
Console.Write("Enter Jibo IP: ");
var jiboIp = (Console.ReadLine() ?? "").Trim();
@@ -67,7 +67,7 @@ while (!cts.IsCancellationRequested)
var json = Encoding.UTF8.GetString(ms.ToArray());
AsrEvent? evt = null;
AsrEvent? evt;
try
{
evt = JsonSerializer.Deserialize<AsrEvent>(json);
@@ -86,15 +86,11 @@ while (!cts.IsCancellationRequested)
Console.WriteLine($"[{evt.EventType}] {json}");
if (evt.EventType == "speech_to_text_final")
{
var best = PickBestUtterance(evt.Utterances);
if (!string.IsNullOrWhiteSpace(best))
{
utteranceTcs.TrySetResult(best);
return;
}
}
if (evt.EventType != "speech_to_text_final") continue;
var best = PickBestUtterance(evt.Utterances);
if (string.IsNullOrWhiteSpace(best)) continue;
utteranceTcs.TrySetResult(best);
return;
}
}, cts.Token);
@@ -220,27 +216,3 @@ static string BuildReply(string heard)
return text.Contains("your name") ? "I am Jibo, running with a local demo bridge." : $"You said: {heard}";
}
public sealed class AsrEvent
{
[JsonPropertyName("event_type")]
public string? EventType { get; set; }
[JsonPropertyName("task_id")]
public string? TaskId { get; set; }
[JsonPropertyName("request_id")]
public string? RequestId { get; set; }
[JsonPropertyName("utterances")]
public List<AsrUtterance>? Utterances { get; set; }
}
public sealed class AsrUtterance
{
[JsonPropertyName("utterance")]
public string? Utterance { get; set; }
[JsonPropertyName("score")]
public double Score { get; set; }
}

16
OpenJibo/tests/Jibo.Cloud.Tests/Fixtures/WebSocketFixtureLoader.cs
View File

@@ -17,10 +17,9 @@ internal static class WebSocketFixtureLoader
var root = document.RootElement;
var session = root.GetProperty("session");
var steps = new List<WebSocketFixtureStep>();
foreach (var stepElement in root.GetProperty("steps").EnumerateArray())
{
steps.Add(new WebSocketFixtureStep
var steps = root.GetProperty("steps")
.EnumerateArray()
.Select(stepElement => new WebSocketFixtureStep
{
Message = new WebSocketMessageEnvelope
{
@@ -33,16 +32,15 @@ internal static class WebSocketFixtureLoader
? binary.EnumerateArray().Select(item => (byte)item.GetInt32()).ToArray()
: null
},
ExpectedReplyTypes = stepElement.GetProperty("expectedReplyTypes")
ExpectedReplyTypes = [.. stepElement.GetProperty("expectedReplyTypes")
.EnumerateArray()
.Select(item => item.GetString() ?? string.Empty)
.Where(item => !string.IsNullOrWhiteSpace(item))
.ToArray(),
.Where(item => !string.IsNullOrWhiteSpace(item))],
ExpectedReplies = stepElement.TryGetProperty("expectedReplies", out var expectedReplies) && expectedReplies.ValueKind == JsonValueKind.Array
? JsonSerializer.Deserialize<List<ExpectedWebSocketReply>>(expectedReplies.GetRawText(), SerializerOptions) ?? []
: []
});
}
})
.ToList();
return new WebSocketFixture
{

40
OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs
View File

@@ -16,18 +16,18 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Exception("dummy"));
var turnService = new WebSocketTurnFinalizationService(
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
sttStrategySelector.Object,
sink.Object
);
await turnService.HandleTurnAsync(new CloudSession() { TurnState = { BufferedAudioBytes = 100 }}, new WebSocketMessageEnvelope(), "dummy",
await turnService.HandleTurnAsync(new CloudSession { TurnState = { BufferedAudioBytes = 100 } },
new WebSocketMessageEnvelope(), "dummy",
CancellationToken.None);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once());
sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
}
[Fact]
@@ -38,21 +38,23 @@ public sealed class FileTurnTelemetrySinkTests
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new InvalidOperationException("ffmpeg failed"));
var turnService = new WebSocketTurnFinalizationService(
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
var turnService = new WebSocketTurnFinalizationService(Mock.Of<IConversationBroker>(),
sttStrategySelector.Object,
sink.Object
);
var session = new CloudSession();
session.TurnState.AwaitingTurnCompletion = true;
session.TurnState.SawListen = true;
session.TurnState.SawContext = true;
session.TurnState.BufferedAudioBytes = 12000;
session.TurnState.BufferedAudioChunkCount = 5;
session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2);
var session = new CloudSession
{
TurnState =
{
AwaitingTurnCompletion = true,
SawListen = true,
SawContext = true,
BufferedAudioBytes = 12000,
BufferedAudioChunkCount = 5,
FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2)
}
};
var replies = await turnService.HandleContextAsync(
session,
@@ -64,6 +66,8 @@ public sealed class FileTurnTelemetrySinkTests
Assert.Equal(12000, session.TurnState.BufferedAudioBytes);
Assert.Equal("ffmpeg failed", session.TurnState.LastSttError);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once());
sink.Verify(
s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Once());
}
}

40
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -16,11 +16,9 @@ public sealed class JiboWebSocketServiceTests
public JiboWebSocketServiceTests()
{
_store = new InMemoryCloudStateStore();
var turnContextMapper = new ProtocolToTurnContextMapper();
var contentRepository = new InMemoryJiboExperienceContentRepository();
var contentCache = new JiboExperienceContentCache(contentRepository);
var conversationBroker = new DemoConversationBroker(new JiboInteractionService(contentCache, new DefaultJiboRandomizer()));
var replyMapper = new ResponsePlanToSocketMessagesMapper();
var sttSelector = new DefaultSttStrategySelector(
[
new SyntheticBufferedAudioSttStrategy()
@@ -30,10 +28,7 @@ public sealed class JiboWebSocketServiceTests
_service = new JiboWebSocketService(
_store,
new NullWebSocketTelemetrySink(),
new WebSocketTurnFinalizationService(
turnContextMapper,
conversationBroker,
replyMapper,
new WebSocketTurnFinalizationService(conversationBroker,
sttSelector,
sink));
}
@@ -2639,7 +2634,7 @@ public sealed class JiboWebSocketServiceTests
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-context-reset-token",
Binary = [9, 9, 9, 9]
Binary = "\t\t\t\t"u8.ToArray()
});
var session = _store.FindSessionByToken("hub-context-reset-token");
@@ -2681,26 +2676,24 @@ public sealed class JiboWebSocketServiceTests
var actualTypes = replies.Select(ReadReplyType).ToArray();
Assert.Equal(step.ExpectedReplyTypes, actualTypes);
if (step.ExpectedReplies.Count > 0)
if (step.ExpectedReplies.Count <= 0) continue;
Assert.Equal(replies.Count, step.ExpectedReplies.Count);
for (var index = 0; index < step.ExpectedReplies.Count; index += 1)
{
Assert.Equal(replies.Count, step.ExpectedReplies.Count);
var expectedReply = step.ExpectedReplies[index];
Assert.Equal(expectedReply.Type, actualTypes[index]);
for (var index = 0; index < step.ExpectedReplies.Count; index += 1)
if (expectedReply.DelayMs.HasValue)
{
var expectedReply = step.ExpectedReplies[index];
Assert.Equal(expectedReply.Type, actualTypes[index]);
if (expectedReply.DelayMs.HasValue)
{
Assert.Equal(expectedReply.DelayMs.Value, replies[index].DelayMs);
}
if (expectedReply.JsonSubset is { ValueKind: JsonValueKind.Object } jsonSubset)
{
using var actualPayload = JsonDocument.Parse(replies[index].Text!);
AssertJsonContains(jsonSubset, actualPayload.RootElement);
}
Assert.Equal(expectedReply.DelayMs.Value, replies[index].DelayMs);
}
if (expectedReply.JsonSubset is not { ValueKind: JsonValueKind.Object } jsonSubset) continue;
using var actualPayload = JsonDocument.Parse(replies[index].Text!);
AssertJsonContains(jsonSubset, actualPayload.RootElement);
}
}
}
@@ -2709,6 +2702,7 @@ public sealed class JiboWebSocketServiceTests
{
Assert.Equal(expected.ValueKind, actual.ValueKind);
// ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault
switch (expected.ValueKind)
{
case JsonValueKind.Object:

14
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs
View File

@@ -158,14 +158,14 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests
{
Calls.Add((fileName, arguments));
if (string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase))
{
var outputPath = arguments[^1];
File.WriteAllBytes(outputPath, "RIFF"u8);
return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty));
}
if (!string.Equals(fileName, "ffmpeg", StringComparison.OrdinalIgnoreCase))
return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke",
string.Empty));
var outputPath = arguments[^1];
File.WriteAllBytes(outputPath, "RIFF"u8);
return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty));
return Task.FromResult(new ExternalProcessResult(0, "[00:00:00.000 --> 00:00:01.000] tell me a joke", string.Empty));
}
}
}