Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden weather fallback and turn finalization

Browse Source
This commit is contained in:
Jacob Dubin
2026-05-06 10:49:24 -05:00
parent ede694afdd
commit 60b8616239
11 changed files with 58623 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -382,83 +382,44 @@ public sealed class JiboInteractionService(
string transcript,
CancellationToken cancellationToken)
{
var payload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "report-skill",
["localIntent"] = "requestWeatherPR"
};
var dateEntity = TryResolveWeatherDateEntity(transcript);
if (dateEntity is not null)
{
payload["date"] = dateEntity;
}
var weatherConditionEntity = TryResolveWeatherConditionEntity(transcript);
if (weatherConditionEntity is not null)
{
payload["weatherCondition"] = weatherConditionEntity;
}
var replyText = "Checking your weather report.";
if (weatherReportProvider is null)
{
return new JiboInteractionDecision(
"weather",
replyText,
"report-skill",
payload);
"I can check weather once my weather service is connected.");
}
var locationQuery = TryResolveWeatherLocationQuery(transcript);
if (!string.IsNullOrWhiteSpace(locationQuery))
{
payload["locationQuery"] = locationQuery;
}
var weatherCoordinates = TryResolveWeatherCoordinates(turn);
if (weatherCoordinates is not null)
var useCelsius = ShouldUseCelsius(turn, transcript);
WeatherReportSnapshot? snapshot;
try
{
payload["latitude"] = weatherCoordinates.Value.Latitude;
payload["longitude"] = weatherCoordinates.Value.Longitude;
snapshot = await weatherReportProvider.GetReportAsync(
new WeatherReportRequest(
locationQuery,
weatherCoordinates?.Latitude,
weatherCoordinates?.Longitude,
string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase),
useCelsius),
cancellationToken);
}
catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
snapshot = null;
}
var useCelsius = ShouldUseCelsius(turn, transcript);
var snapshot = await weatherReportProvider.GetReportAsync(
new WeatherReportRequest(
locationQuery,
weatherCoordinates?.Latitude,
weatherCoordinates?.Longitude,
string.Equals(dateEntity, "tomorrow", StringComparison.OrdinalIgnoreCase),
useCelsius),
cancellationToken);
if (snapshot is not null)
if (snapshot is null)
{
payload["provider"] = "openweather";
payload["temperature"] = snapshot.Temperature;
if (snapshot.HighTemperature is not null)
{
payload["highTemperature"] = snapshot.HighTemperature.Value;
}
if (snapshot.LowTemperature is not null)
{
payload["lowTemperature"] = snapshot.LowTemperature.Value;
}
if (!string.IsNullOrWhiteSpace(snapshot.Condition))
{
payload["weatherCondition"] = snapshot.Condition;
}
replyText = BuildWeatherSpokenReply(snapshot, dateEntity);
return new JiboInteractionDecision(
"weather",
"I couldn't fetch the weather right now. Please try again.");
}
return new JiboInteractionDecision(
"weather",
replyText,
"report-skill",
payload);
BuildWeatherSpokenReply(snapshot, dateEntity));
}
private static string BuildWeatherSpokenReply(

118
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -12,9 +12,12 @@ public sealed partial class WebSocketTurnFinalizationService(
ITurnTelemetrySink sink
)
{
private const int AutoFinalizeMinBufferedAudioBytes = 12000;
private const int AutoFinalizeMinBufferedAudioChunks = 4;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400);
private const int AutoFinalizeMinBufferedAudioBytes = 15000;
private const int AutoFinalizeMinBufferedAudioChunks = 5;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1800);
private static readonly TimeSpan AutoFinalizeMissingTranscriptFallbackAge = TimeSpan.FromMilliseconds(4200);
private static readonly TimeSpan AutoFinalizeContinuationDeferralMaxAge = TimeSpan.FromMilliseconds(3600);
private const int AutoFinalizeContinuationDeferralMaxAttempts = 2;
public static void ObserveIncomingMessage(CloudSession session, string? text)
{
@@ -491,8 +494,37 @@ public sealed partial class WebSocketTurnFinalizationService(
turnState.FinalizeAttemptCount += 1;
}
var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero;
switch (allowFallbackOnMissingTranscript)
{
case true when
turnState.FinalizeAttemptCount >= 2 &&
turnAge >= AutoFinalizeMissingTranscriptFallbackAge:
{
turnState.AwaitingTurnCompletion = false;
session.LastTranscript = string.Empty;
session.LastIntent = "heyJibo";
session.LastListenType = "fallback";
await sink.RecordTurnDiagnosticAsync("auto_finalize_forced_fallback", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["messageType"] = messageType,
["finalizeAttemptCount"] = turnState.FinalizeAttemptCount,
["turnAgeMs"] = (int)turnAge.TotalMilliseconds,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount,
["lastSttError"] = turnState.LastSttError
}), cancellationToken);
var fallbackReplies = ResponsePlanToSocketMessagesMapper.MapFallback(session, turnState.TransId ?? session.LastTransId ?? string.Empty, turnState.ListenRules)
.Select(map => new WebSocketReply { Text = map.Text, DelayMs = map.DelayMs })
.ToArray();
ResetBufferedAudio(session);
turnState.SawListen = false;
turnState.SawContext = false;
return fallbackReplies;
}
case true when
turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
IsYesNoTurn(finalizedTurn):
@@ -525,6 +557,26 @@ public sealed partial class WebSocketTurnFinalizationService(
}
}
if (ShouldDeferForLikelyContinuation(finalizedTurn, turnState, messageType, allowFallbackOnMissingTranscript, out var deferralReason))
{
turnState.AwaitingTurnCompletion = true;
turnState.FinalizeAttemptCount += 1;
var turnAge = turnState.FirstAudioReceivedUtc.HasValue
? DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value
: TimeSpan.Zero;
await sink.RecordTurnDiagnosticAsync("auto_finalize_deferred_for_continuation", BuildTurnDiagnosticSnapshot(session, envelope, new Dictionary<string, object?>
{
["messageType"] = messageType,
["transcript"] = finalizedTurn.NormalizedTranscript ?? finalizedTurn.RawTranscript,
["reason"] = deferralReason,
["finalizeAttemptCount"] = turnState.FinalizeAttemptCount,
["turnAgeMs"] = (int)turnAge.TotalMilliseconds,
["bufferedAudioBytes"] = turnState.BufferedAudioBytes,
["bufferedAudioChunks"] = turnState.BufferedAudioChunkCount
}), cancellationToken);
return [];
}
var plan = await conversationBroker.HandleTurnAsync(finalizedTurn, cancellationToken);
var listenAction = plan.Actions.OfType<ListenAction>().OrderBy(action => action.Sequence).LastOrDefault();
session.LastTranscript = finalizedTurn.NormalizedTranscript ?? finalizedTurn.RawTranscript;
@@ -1142,6 +1194,66 @@ public sealed partial class WebSocketTurnFinalizationService(
.Any(static rule => string.Equals(rule, "launch", StringComparison.OrdinalIgnoreCase));
}
private static bool ShouldDeferForLikelyContinuation(
TurnContext turn,
WebSocketTurnState turnState,
string messageType,
bool allowFallbackOnMissingTranscript,
out string reason)
{
reason = string.Empty;
if (!allowFallbackOnMissingTranscript ||
!string.Equals(messageType, "AUTO_FINALIZE", StringComparison.OrdinalIgnoreCase))
{
return false;
}
if (!turnState.FirstAudioReceivedUtc.HasValue ||
DateTimeOffset.UtcNow - turnState.FirstAudioReceivedUtc.Value >= AutoFinalizeContinuationDeferralMaxAge ||
turnState.FinalizeAttemptCount >= AutoFinalizeContinuationDeferralMaxAttempts)
{
return false;
}
var normalized = NormalizeTranscript(turn.NormalizedTranscript ?? turn.RawTranscript);
if (string.IsNullOrWhiteSpace(normalized))
{
return false;
}
if (normalized is "my birthday" or "my birthday is")
{
reason = "birthday_set_incomplete";
return true;
}
if (normalized.StartsWith("my favorite ", StringComparison.Ordinal) ||
normalized.StartsWith("my favourite ", StringComparison.Ordinal))
{
if (normalized.EndsWith(" is", StringComparison.Ordinal) ||
normalized.EndsWith(" are", StringComparison.Ordinal) ||
!normalized.Contains(" is ", StringComparison.Ordinal))
{
reason = "preference_set_incomplete";
return true;
}
}
if (normalized.StartsWith("what s my favorite", StringComparison.Ordinal) ||
normalized.StartsWith("what is my favorite", StringComparison.Ordinal) ||
normalized.StartsWith("what s my favourite", StringComparison.Ordinal) ||
normalized.StartsWith("what is my favourite", StringComparison.Ordinal))
{
if (normalized is "what s my favorite" or "what is my favorite" or "what s my favourite" or "what is my favourite")
{
reason = "preference_recall_incomplete";
return true;
}
}
return false;
}
private static Dictionary<string, object?> BuildTurnDiagnosticSnapshot(
CloudSession session,
WebSocketMessageEnvelope envelope,

4
OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs
View File

@@ -81,7 +81,7 @@ public sealed class FileTurnTelemetrySinkTests
AwaitingTurnCompletion = true,
SawListen = true,
SawContext = true,
BufferedAudioBytes = 12000,
BufferedAudioBytes = 15000,
BufferedAudioChunkCount = 5,
FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2)
}
@@ -94,7 +94,7 @@ public sealed class FileTurnTelemetrySinkTests
Assert.Empty(replies);
Assert.True(session.TurnState.AwaitingTurnCompletion);
Assert.Equal(12000, session.TurnState.BufferedAudioBytes);
Assert.Equal(15000, session.TurnState.BufferedAudioBytes);
Assert.Equal("ffmpeg failed", session.TurnState.LastSttError);
sink.Verify(

29
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs
View File

@@ -642,7 +642,7 @@ public sealed class JiboInteractionServiceTests
}
[Fact]
public async Task BuildDecisionAsync_WeatherQuery_LaunchesReportSkillWithPegasusIntent()
public async Task BuildDecisionAsync_WeatherQuery_WithoutProvider_UsesSpokenFallback()
{
var service = CreateService();
@@ -653,13 +653,13 @@ public sealed class JiboInteractionServiceTests
});
Assert.Equal("weather", decision.IntentName);
Assert.Equal("report-skill", decision.SkillName);
Assert.Equal("requestWeatherPR", decision.SkillPayload!["localIntent"]);
Assert.False(decision.SkillPayload!.ContainsKey("cloudSkill"));
Assert.Null(decision.SkillName);
Assert.Null(decision.SkillPayload);
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
}
[Fact]
public async Task BuildDecisionAsync_WeatherTomorrowQuery_SetsTomorrowEntity()
public async Task BuildDecisionAsync_WeatherTomorrowQuery_WithoutProvider_StillReturnsFallback()
{
var service = CreateService();
@@ -670,11 +670,11 @@ public sealed class JiboInteractionServiceTests
});
Assert.Equal("weather", decision.IntentName);
Assert.Equal("tomorrow", decision.SkillPayload!["date"]);
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
}
[Fact]
public async Task BuildDecisionAsync_WeatherConditionQuery_SetsWeatherConditionEntity()
public async Task BuildDecisionAsync_WeatherConditionQuery_WithoutProvider_StillReturnsFallback()
{
var service = CreateService();
@@ -685,12 +685,11 @@ public sealed class JiboInteractionServiceTests
});
Assert.Equal("weather", decision.IntentName);
Assert.Equal("rain", decision.SkillPayload!["weatherCondition"]);
Assert.Equal("tomorrow", decision.SkillPayload["date"]);
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
}
[Fact]
public async Task BuildDecisionAsync_ClientNluRequestWeatherPR_LaunchesReportSkill()
public async Task BuildDecisionAsync_ClientNluRequestWeatherPR_WithoutProvider_StillReturnsFallback()
{
var service = CreateService();
@@ -705,8 +704,7 @@ public sealed class JiboInteractionServiceTests
});
Assert.Equal("weather", decision.IntentName);
Assert.Equal("report-skill", decision.SkillName);
Assert.Equal("requestWeatherPR", decision.SkillPayload!["localIntent"]);
Assert.Equal("I can check weather once my weather service is connected.", decision.ReplyText);
}
[Fact]
@@ -725,10 +723,11 @@ public sealed class JiboInteractionServiceTests
});
Assert.Equal("weather", decision.IntentName);
Assert.Null(decision.SkillName);
Assert.Null(decision.SkillPayload);
Assert.Equal("Right now in Boston, US, it is light rain and 61 degrees Fahrenheit.", decision.ReplyText);
Assert.Equal("openweather", decision.SkillPayload!["provider"]);
Assert.Equal(61, decision.SkillPayload["temperature"]);
Assert.Equal("rain", decision.SkillPayload["weatherCondition"]);
Assert.NotNull(provider.LastRequest);
Assert.False(provider.LastRequest!.IsTomorrow);
}
[Fact]

121
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -286,6 +286,75 @@ public sealed class JiboWebSocketServiceTests
Assert.True(listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skipSurprises").GetBoolean());
}
[Fact]
public async Task BufferedAudio_WithIncompletePreferenceHint_DefersThenFinalizesWhenContinuationArrives()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-preference-continuation-token",
Text = """{"type":"LISTEN","transID":"trans-preference-continuation","data":{"rules":["launch"]}}"""
});
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-preference-continuation-token",
Text = """{"type":"CONTEXT","transID":"trans-preference-continuation","data":{"audioTranscriptHint":"my favorite sport"}}"""
});
for (var index = 0; index < 4; index += 1)
{
var chunkReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-preference-continuation-token",
Binary = new byte[3000]
});
Assert.Empty(chunkReplies);
}
var session = _store.FindSessionByToken("hub-preference-continuation-token");
Assert.NotNull(session);
session.TurnState.FirstAudioReceivedUtc = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(2);
var deferredReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-preference-continuation-token",
Binary = new byte[3000]
});
Assert.Empty(deferredReplies);
var finalizedReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-preference-continuation-token",
Text = """{"type":"CONTEXT","transID":"trans-preference-continuation","data":{"audioTranscriptHint":"my favorite sport is football"}}"""
});
Assert.Equal(3, finalizedReplies.Count);
Assert.Equal("LISTEN", ReadReplyType(finalizedReplies[0]));
Assert.Equal("EOS", ReadReplyType(finalizedReplies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(finalizedReplies[2]));
using var listenPayload = JsonDocument.Parse(finalizedReplies[0].Text!);
Assert.Equal("memory_set_preference", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("my favorite sport is football", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
}
[Fact]
public async Task MultiChunkAudio_AccumulatesBufferedStateAcrossMessages()
{
@@ -1697,7 +1766,7 @@ public sealed class JiboWebSocketServiceTests
}
[Fact]
public async Task ClientAsr_HowIsTheWeather_EmitsReportSkillRedirectAndCompletion()
public async Task ClientAsr_HowIsTheWeather_EmitsSpokenWeatherFallbackWithoutRedirect()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
@@ -1717,26 +1786,17 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-weather","data":{"text":"how is the weather"}}"""
});
Assert.Equal(5, replies.Count);
Assert.Equal(3, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_REDIRECT", ReadReplyType(replies[2]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[4]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("requestWeatherPR", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("report-skill", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString());
Assert.Equal("weather", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.False(listenPayload.RootElement.GetProperty("data").GetProperty("nlu").TryGetProperty("skill", out _));
Assert.Equal(JsonValueKind.Null, listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("cloudSkill").ValueKind);
using var redirectPayload = JsonDocument.Parse(replies[2].Text!);
Assert.Equal("report-skill", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString());
Assert.Equal("requestWeatherPR", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
using var completionPayload = JsonDocument.Parse(replies[3].Text!);
Assert.Equal("report-skill", completionPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString());
using var speakPayload = JsonDocument.Parse(replies[4].Text!);
using var speakPayload = JsonDocument.Parse(replies[2].Text!);
var esml = speakPayload.RootElement
.GetProperty("data")
.GetProperty("action")
@@ -1746,11 +1806,11 @@ public sealed class JiboWebSocketServiceTests
.GetProperty("play")
.GetProperty("esml")
.GetString();
Assert.Contains("Checking your weather report", esml, StringComparison.OrdinalIgnoreCase);
Assert.Contains("weather service is connected", esml, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task ClientAsr_WillItRainTomorrow_EmitsReportSkillWeatherEntities()
public async Task ClientAsr_WillItRainTomorrow_EmitsSpokenWeatherFallbackWithoutRedirect()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
@@ -1770,16 +1830,25 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-weather-entities","data":{"text":"will it rain tomorrow"}}"""
});
Assert.Equal(5, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
var entities = listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities");
Assert.Equal("tomorrow", entities.GetProperty("date").GetString());
Assert.Equal("rain", entities.GetProperty("Weather").GetString());
Assert.Equal(3, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
using var redirectPayload = JsonDocument.Parse(replies[2].Text!);
var redirectEntities = redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities");
Assert.Equal("tomorrow", redirectEntities.GetProperty("date").GetString());
Assert.Equal("rain", redirectEntities.GetProperty("Weather").GetString());
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("weather", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
using var speakPayload = JsonDocument.Parse(replies[2].Text!);
var esml = speakPayload.RootElement
.GetProperty("data")
.GetProperty("action")
.GetProperty("config")
.GetProperty("jcp")
.GetProperty("config")
.GetProperty("play")
.GetProperty("esml")
.GetString();
Assert.Contains("weather service is connected", esml, StringComparison.OrdinalIgnoreCase);
}
[Fact]