Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden weather fallback and turn finalization

Browse Source
This commit is contained in:
Jacob Dubin
2026-05-06 10:49:24 -05:00
parent ede694afdd
commit 60b8616239
11 changed files with 58623 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -382,83 +382,44 @@ public sealed class JiboInteractionService(
string transcript,
CancellationToken cancellationToken)
{
var payload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "report-skill",
["localIntent"] = "requestWeatherPR"
};
var dateEntity = TryResolveWeatherDateEntity(transcript);
if (dateEntity is not null)
{
payload["date"] = dateEntity;
}
var weatherConditionEntity = TryResolveWeatherConditionEntity(transcript);
if (weatherConditionEntity is not null)
{
payload["weatherCondition"] = weatherConditionEntity;
}
var replyText = "Checking your weather report.";
if (weatherReportProvider is null)
{
return new JiboInteractionDecision(
"weather",
replyText,
"report-skill",
payload);
"I can check weather once my weather service is connected.");
}
var locationQuery = TryResolveWeatherLocationQuery(transcript);
if (!string.IsNullOrWhiteSpace(locationQuery))
{
payload["locationQuery"] = locationQuery;
}
var weatherCoordinates = TryResolveWeatherCoordinates(turn);
if (weatherCoordinates is not null)
var useCelsius = ShouldUseCelsius(turn, transcript);
WeatherReportSnapshot? snapshot;
try
{
payload["latitude"] = weatherCoordinates.Value.Latitude;
payload["longitude"] = weatherCoordinates.Value.Longitude;
snapshot = await weatherReportProvider.GetReportAsync(
new WeatherReportRequest(
locationQuery,
weatherCoordinates?.Latitude,
weatherCoordinates?.Longitude,
string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase),
useCelsius),
cancellationToken);
}
catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
snapshot = null;
}
var useCelsius = ShouldUseCelsius(turn, transcript);
var snapshot = await weatherReportProvider.GetReportAsync(
new WeatherReportRequest(
locationQuery,
weatherCoordinates?.Latitude,
weatherCoordinates?.Longitude,
string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase),
useCelsius),
cancellationToken);
if (snapshot is not null)
if (snapshot is null)
{
payload["provider"] = "openweather";
payload["temperature"] = snapshot.Temperature;
if (snapshot.HighTemperature is not null)
{
payload["highTemperature"] = snapshot.HighTemperature.Value;
}
if (snapshot.LowTemperature is not null)
{
payload["lowTemperature"] = snapshot.LowTemperature.Value;
}
if (!string.IsNullOrWhiteSpace(snapshot.Condition))
{
payload["weatherCondition"] = snapshot.Condition;
}
replyText = BuildWeatherSpokenReply(snapshot, dateEntity);
return new JiboInteractionDecision(
"weather",
"I couldn't fetch the weather right now. Please try again.");
}
return new JiboInteractionDecision(
"weather",
replyText,
"report-skill",
payload);
BuildWeatherSpokenReply(snapshot, dateEntity));
}
private static string BuildWeatherSpokenReply(

118
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -12,9 +12,12 @@ public sealed partial class WebSocketTurnFinalizationService(
ITurnTelemetrySink sink
)
{
private const int AutoFinalizeMinBufferedAudioBytes = 12000;
private const int AutoFinalizeMinBufferedAudioChunks = 4;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400);
private const int AutoFinalizeMinBufferedAudioBytes = 15000;
private const int AutoFinalizeMinBufferedAudioChunks = 5;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1800);
private static readonly TimeSpan AutoFinalizeMissingTranscriptFallbackAge = TimeSpan.FromMilliseconds(4200);
private static readonly TimeSpan AutoFinalizeContinuationDeferralMaxAge = TimeSpan.FromMilliseconds(3600);
private const int AutoFinalizeContinuationDeferralMaxAttempts = 2;
public static void ObserveIncomingMessage(CloudSession session, string? text)
{
@@ -491,8 +494,37 @@ public sealed partial class WebSocketTurnFinalizationService(
turnState.FinalizeAttemptCount += 1;
}
var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero;
switch (allowFallbackOnMissingTranscript)
{
case true when
turnState.FinalizeAttemptCount >= 2 &&
turnAge >= AutoFinalizeMissingTranscriptFallbackAge:
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = "heyJibo";
session.LastListenType = "fallback";
await sink.RecordTurnDiagnosticAsync("auto_finalize_forced_fallback", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["messageType"] = messageType,
["finalizeAttemptCount"] = turnState.FinalizeAttemptCount,
["turnAgeMs"] = (int)turnAge.TotalMilliseconds,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
["lastSttError"] = turnState.LastSttError
}), cancellationToken);
var fallbackReplies = ResponsePlanToSocketMessagesMapper.MapFallback(session, turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.ListenRules)
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
ResetBufferedAudio(session);
turnState.SawListen = false;
turnState.SawContext = false;
return fallbackReplies;
}
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
IsYesNoTurn(finalizedTurn):
@@ -525,6 +557,26 @@ public sealed partial class WebSocketTurnFinalizationService(
}
}
if (ShouldDeferForLikelyContinuation(finalizedTurn, turnState, messageType, allowFallbackOnMissingTranscript, out var deferralReason))
{
turnState.AwaitingTurnCompletion = true;
turnState.FinalizeAttemptCount += 1;
var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero;
await sink.RecordTurnDiagnosticAsync("auto_finalize_deferred_for_continuation", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["messageType"] = messageType,
["transcript"] = finalizedTurn.NormalizedTranscript ?? finalizedTurn.RawTranscript,
["reason"] = deferralReason,
["finalizeAttemptCount"] = turnState.FinalizeAttemptCount,
["turnAgeMs"] = (int)turnAge.TotalMilliseconds,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount
}), cancellationToken);
return [];
}
var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken);
var listenAction = plan.Actions.OfType<ListenAction>().OrderBy(action => action.Sequence).LastOrDefault();
session.LastTranscript = finalizedTurn.NormalizedTranscript ?? finalizedTurn.RawTranscript;
@@ -1142,6 +1194,66 @@ public sealed partial class WebSocketTurnFinalizationService(
.Any(static rule => string.Equals(rule, "launch", StringComparison.OrdinalIgnoreCase));
}
private static bool ShouldDeferForLikelyContinuation(
TurnContext turn,
WebSocketTurnState turnState,
string messageType,
bool allowFallbackOnMissingTranscript,
out string reason)
{
reason = string.Empty;
if (!allowFallbackOnMissingTranscript ||
!string.Equals(messageType, "AUTO_FINALIZE", StringComparison.OrdinalIgnoreCase))
{
return false;
}
if (!turnState.FirstAudioReceivedUtc.HasValue ||
DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value >= AutoFinalizeContinuationDeferralMaxAge ||
turnState.FinalizeAttemptCount >= AutoFinalizeContinuationDeferralMaxAttempts)
{
return false;
}
var normalized = NormalizeTranscript(turn.NormalizedTranscript ?? turn.RawTranscript);
if (string.IsNullOrWhiteSpace(normalized))
{
return false;
}
if (normalized is "my birthday" or "my birthday is")
{
reason = "birthday_set_incomplete";
return true;
}
if (normalized.StartsWith("my favorite ", StringComparison.Ordinal) ||
normalized.StartsWith("my favourite ", StringComparison.Ordinal))
{
if (normalized.EndsWith(" is", StringComparison.Ordinal) ||
normalized.EndsWith(" are", StringComparison.Ordinal) ||
!normalized.Contains(" is ", StringComparison.Ordinal))
{
reason = "preference_set_incomplete";
return true;
}
}
if (normalized.StartsWith("what s my favorite", StringComparison.Ordinal) ||
normalized.StartsWith("what is my favorite", StringComparison.Ordinal) ||
normalized.StartsWith("what s my favourite", StringComparison.Ordinal) ||
normalized.StartsWith("what is my favourite", StringComparison.Ordinal))
{
if (normalized is "what s my favorite" or "what is my favorite" or "what s my favourite" or "what is my favourite")
{
reason = "preference_recall_incomplete";
return true;
}
}
return false;
}
private static Dictionary<string, object?> BuildTurnDiagnosticSnapshot(
CloudSession session,
WebSocketMessageEnvelope envelope,