Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added more logging around $YESNO so we can get more consistent yes and no replies processed.... they are spotty currently

Browse Source
This commit is contained in:
Jacob Dubin
2026-05-03 22:42:41 -05:00
parent 2ec4902189
commit 573911de0f
8 changed files with 185 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/ITurnTelemetrySink.cs
View File

@@ -2,5 +2,7 @@ namespace Jibo.Cloud.Application.Abstractions;
public interface ITurnTelemetrySink
{
Task RecordTurnDiagnosticAsync(string category, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default);
Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default);
}
}

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/NullTurnTelemetrySink.cs
View File

@@ -4,5 +4,7 @@ namespace Jibo.Cloud.Application.Services;
public sealed class NullTurnTelemetrySink : ITurnTelemetrySink
{
public Task RecordTurnDiagnosticAsync(string category, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default) => Task.CompletedTask;
public Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default) => Task.CompletedTask;
}
}

99
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -37,8 +37,21 @@ public sealed partial class WebSocketTurnFinalizationService(
CancellationToken cancellationToken = default)
{
var turnState = session.TurnState;
if (ShouldIgnoreLateAudio(session) || ShouldIgnoreAudioWithoutListen(turnState))
var ignoreLateAudio = ShouldIgnoreLateAudio(session);
var ignoreAudioWithoutListen = ShouldIgnoreAudioWithoutListen(turnState);
if (ignoreLateAudio || ignoreAudioWithoutListen)
{
await sink.RecordTurnDiagnosticAsync("binary_audio_ignored", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["ignored"] = true,
["ignoreLateAudio"] = ignoreLateAudio,
["ignoreAudioWithoutListen"] = ignoreAudioWithoutListen,
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
["sawListen"] = turnState.SawListen,
["sawContext"] = turnState.SawContext
}), cancellationToken);
return [];
}
@@ -53,6 +66,17 @@ public sealed partial class WebSocketTurnFinalizationService(
turnState.LastAudioReceivedUtc = DateTimeOffset.UtcNow;
turnState.AwaitingTurnCompletion = true;
session.Metadata["lastAudioBytes"] = envelope.Binary?.Length ?? 0;
await sink.RecordTurnDiagnosticAsync("binary_audio_received", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
["sawListen"] = turnState.SawListen,
["sawContext"] = turnState.SawContext,
["listenRules"] = turnState.ListenRules,
["listenAsrHints"] = turnState.ListenAsrHints,
["yesNoRule"] = turnState.ListenRules.FirstOrDefault(IsConstrainedYesNoRule)
}), cancellationToken);
if (ShouldAutoFinalize(session))
{
@@ -328,6 +352,25 @@ public sealed partial class WebSocketTurnFinalizationService(
CancellationToken cancellationToken)
{
var turn = ProtocolToTurnContextMapper.MapListenMessage(envelope, session, messageType);
var turnState = session.TurnState;
if (IsYesNoTurn(turn) || ReadPrimaryYesNoRule(turn) is not null)
{
await sink.RecordTurnDiagnosticAsync("yes_no_turn_received", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["messageType"] = messageType,
["listenRules"] = ReadRules(turn, "listenRules").ToArray(),
["clientRules"] = ReadRules(turn, "clientRules").ToArray(),
["listenAsrHints"] = ReadRules(turn, "listenAsrHints").ToArray(),
["yesNoRule"] = ReadPrimaryYesNoRule(turn),
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
["sawListen"] = turnState.SawListen,
["sawContext"] = turnState.SawContext,
["followUpOpen"] = session.FollowUpOpen,
["followUpExpiresUtc"] = session.FollowUpExpiresUtc
}), cancellationToken);
}
if (ShouldIgnoreBlankAudioHotphraseTurn(turn))
{
session.TurnState.AwaitingTurnCompletion = false;
@@ -366,7 +409,6 @@ public sealed partial class WebSocketTurnFinalizationService(
};
}
var turnState = session.TurnState;
if (ShouldTreatBufferedHotphraseAsGreeting(finalizedTurn, turnState, allowFallbackOnMissingTranscript))
{
finalizedTurn = WithSyntheticTranscript(finalizedTurn, "hello");
@@ -393,6 +435,22 @@ public sealed partial class WebSocketTurnFinalizationService(
if (ShouldHandleAsLocalNoInput(finalizedTurn))
{
if (IsYesNoTurn(finalizedTurn))
{
await sink.RecordTurnDiagnosticAsync("yes_no_no_input", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["messageType"] = messageType,
["listenRules"] = ReadRules(finalizedTurn, "listenRules").ToArray(),
["clientRules"] = ReadRules(finalizedTurn, "clientRules").ToArray(),
["listenAsrHints"] = ReadRules(finalizedTurn, "listenAsrHints").ToArray(),
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
["sawListen"] = turnState.SawListen,
["sawContext"] = turnState.SawContext,
["followUpOpen"] = session.FollowUpOpen
}), cancellationToken);
}
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = null;
@@ -522,6 +580,24 @@ public sealed partial class WebSocketTurnFinalizationService(
DelayMs = map.DelayMs
}).ToArray();
if (IsYesNoTurn(finalizedTurn))
{
await sink.RecordTurnDiagnosticAsync("yes_no_turn_resolved", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["messageType"] = messageType,
["transcript"] = finalizedTurn.NormalizedTranscript ?? finalizedTurn.RawTranscript,
["intent"] = plan.IntentName,
["listenRules"] = ReadRules(finalizedTurn, "listenRules").ToArray(),
["clientRules"] = ReadRules(finalizedTurn, "clientRules").ToArray(),
["listenAsrHints"] = ReadRules(finalizedTurn, "listenAsrHints").ToArray(),
["awaitingTurnCompletion"] = turnState.AwaitingTurnCompletion,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
["followUpOpen"] = session.FollowUpOpen,
["followUpExpiresUtc"] = session.FollowUpExpiresUtc
}), cancellationToken);
}
ResetBufferedAudio(session);
turnState.SawListen = false;
turnState.SawContext = false;
@@ -1045,6 +1121,25 @@ public sealed partial class WebSocketTurnFinalizationService(
.Any(static rule => string.Equals(rule, "launch", StringComparison.OrdinalIgnoreCase));
}
private static Dictionary<string, object?> BuildTurnDiagnosticSnapshot(
CloudSession session,
WebSocketMessageEnvelope envelope,
Dictionary<string, object?> details)
{
details["sessionToken"] = session.Token;
details["hostName"] = envelope.HostName;
details["path"] = envelope.Path;
details["kind"] = envelope.Kind;
details["transID"] = session.TurnState.TransId ?? session.LastTransId;
details["lastMessageType"] = session.LastMessageType;
details["awaitingTurnCompletion"] = session.TurnState.AwaitingTurnCompletion;
details["bufferedAudioBytes"] = session.TurnState.BufferedAudioBytes;
details["bufferedAudioChunks"] = session.TurnState.BufferedAudioChunkCount;
details["sawListen"] = session.TurnState.SawListen;
details["sawContext"] = session.TurnState.SawContext;
return details;
}
private static TurnContext WithSyntheticTranscript(TurnContext turn, string transcript)
{
var attributes = new Dictionary<string, object?>(turn.Attributes, StringComparer.OrdinalIgnoreCase)

31
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileTurnTelemetrySink.cs
View File

@@ -15,6 +15,20 @@ public sealed class FileTurnTelemetrySink(ILogger<FileTurnTelemetrySink> logger,
private readonly SemaphoreSlim _writeLock = new(1, 1);
public async Task RecordTurnDiagnosticAsync(string category, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default)
{
if (!options.Value.Enabled)
{
return;
}
await WriteEventAsync(new
{
Type = category,
Details = details
}, "Turn telemetry diagnostic", LogLevel.Information, cancellationToken);
}
public async Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default)
{
if (!options.Value.Enabled)
@@ -22,15 +36,20 @@ public sealed class FileTurnTelemetrySink(ILogger<FileTurnTelemetrySink> logger,
return;
}
await WriteErrorAsync(ex, message, cancellationToken);
await WriteEventAsync(new
{
Exception = ex.ToString(),
Message = message,
Type = "transcript_error"
}, "Turn telemetry error", LogLevel.Error, cancellationToken);
}
private async Task WriteErrorAsync(Exception ex, string message, CancellationToken cancellationToken)
private async Task WriteEventAsync(object payload, string logMessage, LogLevel level, CancellationToken cancellationToken)
{
var directory = GetBaseDirectory();
Directory.CreateDirectory(directory);
var filePath = Path.Combine(directory, $"{DateTimeOffset.UtcNow:yyyyMMdd}.events.ndjson");
var line = JsonSerializer.Serialize(new { Exception = ex.ToString(), Message = message }, JsonOptions) + Environment.NewLine;
var line = JsonSerializer.Serialize(payload, JsonOptions) + Environment.NewLine;
await _writeLock.WaitAsync(cancellationToken);
try
@@ -42,7 +61,7 @@ public sealed class FileTurnTelemetrySink(ILogger<FileTurnTelemetrySink> logger,
_writeLock.Release();
}
logger.LogError("Turn telemetry Message={Message} Exception={Exception}", message, ex);
logger.Log(level, "{LogMessage} {Payload}", logMessage, payload);
}
private string GetBaseDirectory()
@@ -52,4 +71,4 @@ public sealed class FileTurnTelemetrySink(ILogger<FileTurnTelemetrySink> logger,
Directory.GetCurrentDirectory(),
AppContext.BaseDirectory);
}
}
}