Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more fixes

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-18 22:27:46 -05:00
parent b77f332350
commit a26c64efbe
16 changed files with 5406 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
OpenJibo/docs/development-plan.md
View File

@@ -104,6 +104,8 @@ Evidence from the smaller `2026-04-18/19` hotphrase and word-of-the-day verifica
- hotphrase silence can still auto-finalize into a generic `heyJibo` fallback, which sounds confused on-robot compared with a dedicated greeting path
- voice-triggered `loadMenu + destination=word-of-the-day` reaches Nimbus successfully, but Nimbus still expects a follow-up cloud skill response and times out if launch stops at `LISTEN` + `EOS`
- the newer `jibo test 2` bundle shows voice launch now reaches Nimbus and receives a cloud response, but a generic `SLIM/RUNTIME_PROMPT` just says "starting word of the day" instead of performing the menu-style redirect the on-screen path uses
- the `jibo test 3` bundle confirms Nimbus rejects `REDIRECT` in that cloud-skill slot, so the better next experiment is to hint the on-robot target skill directly on the synthetic `LISTEN` result and skip Nimbus `SKILL_ACTION` entirely for word-of-the-day launch
- the same bundle also shows `word-of-the-day/right_word` cleanup turns need a short ignore window for trailing audio or the robot can stay stuck in a blue-ring listening state
- the local buffered-audio seam is still producing repeated `whisper.cpp returned no transcript` and `ffmpeg ... Codec not found` failures, so lightweight waveform or energy screening is worth considering once the core launch flow is stable
Near-term interaction work should now prioritize:

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -231,9 +231,7 @@ public sealed class JiboInteractionService(
SkillPayload: new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["destination"] = "word-of-the-day",
["skillId"] = "@be/word-of-the-day",
["redirectIntent"] = "menu",
["redirectDomain"] = "word-of-the-day"
["skillId"] = "@be/word-of-the-day"
});
}

100
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -45,9 +45,42 @@ public sealed class ResponsePlanToSocketMessagesMapper
? ["main-menu/execute_fun_stuff"]
: isYesNoTurn && isYesNoIntent ? [yesNoCreateRule!] : rules;
var entities = ReadEntities(turn, messageType, isYesNoTurn && isYesNoIntent, isWordOfDayLaunch, isWordOfDayGuess, wordOfDayGuess);
var messages = new List<SocketReplyPlan>
object listenMessage;
if (isWordOfDayLaunch)
{
new(JsonSerializer.Serialize(new
listenMessage = new
{
type = "LISTEN",
transID = transId,
skillID = "@be/word-of-the-day",
onRobot = true,
data = new
{
asr = new
{
confidence = 0.95,
final = true,
text = outboundAsrText
},
nlu = new
{
confidence = 0.95,
intent = outboundIntent,
rules = outboundRules,
entities
},
match = new
{
intent = outboundIntent,
rule = outboundRules.FirstOrDefault() ?? string.Empty,
score = 0.95
}
}
};
}
else
{
listenMessage = new
{
type = "LISTEN",
transID = transId,
@@ -73,7 +106,12 @@ public sealed class ResponsePlanToSocketMessagesMapper
score = 0.95
}
}
})),
};
}
var messages = new List<SocketReplyPlan>
{
new(JsonSerializer.Serialize(listenMessage)),
new(JsonSerializer.Serialize(new
{
type = "EOS",
@@ -302,15 +340,9 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill)
{
var skillPayload = skill?.Payload;
var isWordOfTheDay = plan.IntentName is not null && string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase);
var isJoke = string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase) ||
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase);
if (isWordOfTheDay)
{
return BuildWordOfTheDayLaunchSkillPayload(transId, skillPayload);
}
var payloadSkill = ReadPayloadString(skillPayload, "skillId");
var skillId = string.IsNullOrWhiteSpace(payloadSkill) ? isJoke ? "@be/joke" : skill?.SkillName ?? "chitchat-skill" : payloadSkill;
var esml = ReadPayloadString(skillPayload, "esml") ?? (isDance
@@ -363,56 +395,6 @@ public sealed class ResponsePlanToSocketMessagesMapper
};
}
private static object BuildWordOfTheDayLaunchSkillPayload(string transId, IDictionary<string, object?>? payload)
{
var skillId = ReadPayloadString(payload, "skillId") ?? "@be/word-of-the-day";
var redirectIntent = ReadPayloadString(payload, "redirectIntent") ?? "menu";
var redirectDomain = ReadPayloadString(payload, "redirectDomain") ?? "word-of-the-day";
return new
{
type = "SKILL_ACTION",
ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
msgID = CreateHubMessageId(),
transID = transId,
data = new
{
skill = new
{
id = skillId
},
action = new
{
config = new
{
jcp = new
{
type = "REDIRECT",
config = new
{
nlu = new
{
intent = redirectIntent,
entities = new
{
domain = redirectDomain
}
},
asr = new
{
text = string.Empty,
confidence = 1.0
}
}
}
}
},
analytics = new Dictionary<string, object?>(),
final = true
}
};
}
private static object BuildGenericFallbackSkillPayload(string transId)
{
return new

3
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -373,6 +373,8 @@ public sealed class WebSocketTurnFinalizationService(
if (ShouldIgnoreCompletedWordOfDayTurn(finalizedTurn))
{
turnState.AwaitingTurnCompletion = false;
turnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
session.FollowUpExpiresUtc = null;
ResetBufferedAudio(session);
return [];
}
@@ -475,6 +477,7 @@ public sealed class WebSocketTurnFinalizationService(
: DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
var emitSkillActions = messageType != "CLIENT_NLU" &&
!string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var replies = ResponsePlanToSocketMessagesMapper.Map(plan, finalizedTurn, session, emitSkillActions).Select(map => new WebSocketReply
{

3
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs
View File

@@ -145,8 +145,7 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("Starting word of the day.", decision.ReplyText);
Assert.Equal("@be/word-of-the-day", decision.SkillName);
Assert.Equal("word-of-the-day", decision.SkillPayload!["destination"]);
Assert.Equal("menu", decision.SkillPayload["redirectIntent"]);
Assert.Equal("word-of-the-day", decision.SkillPayload["redirectDomain"]);
Assert.Equal("@be/word-of-the-day", decision.SkillPayload["skillId"]);
}
[Fact]

47
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -486,19 +486,14 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-launch","data":{"text":"Play word of the day."}}"""
});
Assert.Equal(3, replies.Count);
Assert.Equal(2, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("loadMenu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("Play word of the day.", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("word-of-the-day", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("destination").GetString());
Assert.Equal("main-menu/execute_fun_stuff", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString());
using var skillPayload = JsonDocument.Parse(replies[2].Text!);
Assert.Equal("@be/word-of-the-day", skillPayload.RootElement.GetProperty("data").GetProperty("skill").GetProperty("id").GetString());
Assert.Equal("REDIRECT", skillPayload.RootElement.GetProperty("data").GetProperty("action").GetProperty("config").GetProperty("jcp").GetProperty("type").GetString());
Assert.Equal("menu", skillPayload.RootElement.GetProperty("data").GetProperty("action").GetProperty("config").GetProperty("jcp").GetProperty("config").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("word-of-the-day", skillPayload.RootElement.GetProperty("data").GetProperty("action").GetProperty("config").GetProperty("jcp").GetProperty("config").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString());
Assert.Equal(string.Empty, skillPayload.RootElement.GetProperty("data").GetProperty("action").GetProperty("config").GetProperty("jcp").GetProperty("config").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("@be/word-of-the-day", listenPayload.RootElement.GetProperty("skillID").GetString());
Assert.True(listenPayload.RootElement.GetProperty("onRobot").GetBoolean());
var session = _store.FindSessionByToken("hub-wod-launch-token");
Assert.NotNull(session);
@@ -554,10 +549,13 @@ public sealed class JiboWebSocketServiceTests
Binary = new byte[3000]
});
Assert.Equal(3, replies.Count);
Assert.Equal(2, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("@be/word-of-the-day", listenPayload.RootElement.GetProperty("skillID").GetString());
Assert.True(listenPayload.RootElement.GetProperty("onRobot").GetBoolean());
var lateReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
@@ -631,6 +629,35 @@ public sealed class JiboWebSocketServiceTests
Assert.Empty(replies);
}
[Fact]
public async Task BinaryAudio_AfterWordOfDayRightWordListen_IsIgnoredDuringCleanupWindow()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-wod-right-word-audio-token",
Text = """{"type":"LISTEN","transID":"trans-wod-right-word-audio","data":{"rules":["word-of-the-day/right_word","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"]}}"""
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-wod-right-word-audio-token",
Binary = new byte[4096]
});
Assert.Empty(replies);
var session = _store.FindSessionByToken("hub-wod-right-word-audio-token");
Assert.NotNull(session);
Assert.False(session.TurnState.AwaitingTurnCompletion);
Assert.True(session.TurnState.IgnoreAdditionalAudioUntilUtc.HasValue);
}
[Fact]
public async Task BlankAudioHotphraseTurn_IsIgnored()
{