hey jibo and word of the day fixes

This commit is contained in:
Jacob Dubin
2026-04-18 22:02:31 -05:00
parent 16768a0c80
commit d4dbfefa06
13 changed files with 4416 additions and 7 deletions

View File

@@ -99,6 +99,12 @@ Evidence from the continued `2026-04-18` word-of-the-day and time captures:
- post-game hotphrase blank-audio turns should be treated as cleanup noise, not a new cloud conversation turn
- clock replies should use the user-facing hour format without a leading zero
Evidence from the smaller `2026-04-18/19` hotphrase and word-of-the-day verification bundle:
- hotphrase silence can still auto-finalize into a generic `heyJibo` fallback, which sounds confused on-robot compared with a dedicated greeting path
- voice-triggered `loadMenu + destination=word-of-the-day` reaches Nimbus successfully, but Nimbus still expects a follow-up cloud skill response and times out if launch stops at `LISTEN` + `EOS`
- the local buffered-audio seam is still producing repeated `whisper.cpp returned no transcript` and `ffmpeg ... Codec not found` failures, so lightweight waveform or energy screening is worth considering once the core launch flow is stable
Near-term interaction work should now prioritize:
1. preserve and interpret yes/no turn constraints from observed listen rules

View File

@@ -16,7 +16,7 @@ public sealed class WebSocketTurnFinalizationService(
{
private const int AutoFinalizeMinBufferedAudioBytes = 12000;
private const int AutoFinalizeMinBufferedAudioChunks = 5;
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1800);
private static readonly TimeSpan AutoFinalizeMinTurnAge = TimeSpan.FromMilliseconds(1400);
public void ObserveIncomingMessage(CloudSession session, string? text)
{
@@ -475,7 +475,6 @@ public sealed class WebSocketTurnFinalizationService(
: DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
var emitSkillActions = messageType != "CLIENT_NLU" &&
!string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var replies = ResponsePlanToSocketMessagesMapper.Map(plan, finalizedTurn, session, emitSkillActions).Select(map => new WebSocketReply
{
@@ -745,8 +744,7 @@ public sealed class WebSocketTurnFinalizationService(
return false;
}
return turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes &&
(turnState.FinalizeAttemptCount > 0 || !string.IsNullOrWhiteSpace(turnState.LastSttError));
return turnState.BufferedAudioBytes >= AutoFinalizeMinBufferedAudioBytes;
}
private static bool ShouldTreatEmptyHotphraseTurnAsGreeting(TurnContext turn)

View File

@@ -466,7 +466,7 @@ public sealed class JiboWebSocketServiceTests
}
[Fact]
public async Task ClientAsr_WordOfDayLaunch_EmitsMenuStyleLoadMenuWithoutSkillAction()
public async Task ClientAsr_WordOfDayLaunch_EmitsMenuStyleLoadMenuWithSkillAction()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
@@ -486,13 +486,16 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-launch","data":{"text":"Play word of the day."}}"""
});
Assert.Equal(2, replies.Count);
Assert.Equal(3, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("loadMenu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("Play word of the day.", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("word-of-the-day", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("destination").GetString());
Assert.Equal("main-menu/execute_fun_stuff", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString());
using var skillPayload = JsonDocument.Parse(replies[2].Text!);
Assert.Equal("@be/word-of-the-day", skillPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString());
var session = _store.FindSessionByToken("hub-wod-launch-token");
Assert.NotNull(session);
Assert.False(session.FollowUpOpen);
@@ -547,9 +550,10 @@ public sealed class JiboWebSocketServiceTests
Binary = new byte[3000]
});
Assert.Equal(2, replies.Count);
Assert.Equal(3, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
var lateReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{