Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes for jibo test fails

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-21 22:53:14 -05:00
parent b595766ac9
commit 685d1515f6
14 changed files with 5214 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
OpenJibo/docs/development-plan.md
View File

@@ -181,7 +181,8 @@ Latest clock discovery findings:
- `@be/clock` is a real local skill with `clock`, `timer`, and `alarm` domains.
- Menu launches use `intent = "menu"` with `entities.domain` set to the target sub-area.
- Direct timer and alarm actions use `timerValue` and `alarmValue` utterances, not a generic chat path.
- The `jibo test 15` bundle shows stock OS 1.9 rejecting our older top-level `timerValue` launch with `found no matching transition`, so the safer cloud contract is a stock-style `start` intent with the timer/alarm entities attached.
- The same bundle also shows local follow-up rules like `clock/timer_set_value`, so bare replies such as `five minutes` or `ten twenty five` need to be parsed when the robot is already collecting a timer/alarm value.
- The newest `.NET` pass now routes `open the clock` into the direct `askForTime` clock-view path, moves plain time/date/day questions onto stock-shaped local `@be/clock` handoffs, and keeps malformed timer/alarm requests on a clarification reply path instead of generic chat echo.
Latest photo discovery findings:

4
OpenJibo/docs/feature-backlog.md
View File

@@ -127,9 +127,11 @@ Parallel tags:
- Current evidence:
- [protocol-inventory.md](C:/Projects/JiboExperiments/OpenJibo/docs/protocol-inventory.md) already tracks menu intents for `askForTime`, `askForDate`, `timerValue`, and `alarmValue`
- `@be/clock` exists in the robot skill inventory
- `JiboOs` shows `@be/clock` branches on `entities.domain = clock | timer | alarm`, uses `intent = menu` for menu launches, and accepts direct `timerValue` / `alarmValue` utterances with structured entities
- `JiboOs` shows `@be/clock` branches on `entities.domain = clock | timer | alarm`, uses `intent = menu` for menu launches, and has distinct local value-collection rules such as `clock/timer_set_value`
- [artifact-output/jibo-test-15](C:/Projects/JiboExperiments/artifact-output/jibo-test-15) shows stock OS 1.9 rejecting our older `timerValue` top-level launch with `found no matching transition`, which points to a stock-style `start` flow plus local follow-up value rules instead
- Implementation notes:
- compare our custom time/date path against actual menu payloads
- keep direct clock/date/day local, but treat timer and alarm as a two-part flow: stock start intent plus bare follow-up parsing on `clock/*_set_value`
- decide whether timer and alarm should stay robot-local with cloud acknowledgement, or whether cloud needs to shape the launch and follow-up turns
- Progress so far:
- voice `open the clock` now routes to the direct local `askForTime` clock-view path instead of the broader clock menu

124
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -23,7 +23,17 @@ public sealed class JiboInteractionService(
var clientEntities = ReadEntities(turn);
var isYesNoTurn = IsYesNoTurn(turn);
var semanticIntent = ResolveSemanticIntent(lowered, clientIntent, clientRules, listenRules, clientEntities, isYesNoTurn);
var isTimerValueTurn = IsClockTimerValueTurn(clientRules, listenRules);
var isAlarmValueTurn = IsClockAlarmValueTurn(clientRules, listenRules);
var semanticIntent = ResolveSemanticIntent(
lowered,
clientIntent,
clientRules,
listenRules,
clientEntities,
isYesNoTurn,
isTimerValueTurn,
isAlarmValueTurn);
return semanticIntent switch
{
"joke" => BuildJokeDecision(catalog),
@@ -39,8 +49,8 @@ public sealed class JiboInteractionService(
"clock_menu" => BuildClockLaunchDecision("clock_menu", "clock", "menu", "Opening the clock menu."),
"timer_menu" => BuildClockLaunchDecision("timer", "Opening the timer."),
"alarm_menu" => BuildClockLaunchDecision("alarm", "Opening the alarm."),
"timer_value" => BuildTimerValueDecision(lowered),
"alarm_value" => BuildAlarmValueDecision(lowered),
"timer_value" => BuildTimerValueDecision(lowered, isTimerValueTurn),
"alarm_value" => BuildAlarmValueDecision(lowered, isAlarmValueTurn),
"timer_clarify" => new JiboInteractionDecision("timer_clarify", "How long should I set the timer for?"),
"alarm_clarify" => new JiboInteractionDecision("alarm_clarify", "What time should I set the alarm for?"),
"photo_gallery" => BuildPhotoGalleryLaunchDecision(),
@@ -141,7 +151,9 @@ public sealed class JiboInteractionService(
IReadOnlyList<string> clientRules,
IReadOnlyList<string> listenRules,
IReadOnlyDictionary<string, string> clientEntities,
bool isYesNoTurn)
bool isYesNoTurn,
bool isTimerValueTurn,
bool isAlarmValueTurn)
{
var wordOfDayPuzzleTurn = clientRules.Concat(listenRules)
.Any(rule => string.Equals(rule, "word-of-the-day/puzzle", StringComparison.OrdinalIgnoreCase));
@@ -196,6 +208,18 @@ public sealed class JiboInteractionService(
return "alarm_value";
}
if ((string.Equals(clientIntent, "start", StringComparison.OrdinalIgnoreCase) ||
string.Equals(clientIntent, "set", StringComparison.OrdinalIgnoreCase)) &&
clientEntities.TryGetValue("domain", out var startDomain))
{
return startDomain.ToLowerInvariant() switch
{
"timer" => "timer_value",
"alarm" => "alarm_value",
_ => "chat"
};
}
if (string.Equals(clientIntent, "menu", StringComparison.OrdinalIgnoreCase) &&
clientEntities.TryGetValue("domain", out var clockDomain))
{
@@ -261,22 +285,22 @@ public sealed class JiboInteractionService(
return "alarm_menu";
}
if (TryParseAlarmValue(loweredTranscript) is not null)
if (TryParseAlarmValue(loweredTranscript, isAlarmValueTurn) is not null)
{
return "alarm_value";
}
if (TryParseTimerValue(loweredTranscript) is not null)
if (TryParseTimerValue(loweredTranscript, isTimerValueTurn) is not null)
{
return "timer_value";
}
if (IsAlarmRequest(loweredTranscript))
if (IsAlarmRequest(loweredTranscript) || isAlarmValueTurn)
{
return "alarm_clarify";
}
if (IsTimerRequest(loweredTranscript))
if (IsTimerRequest(loweredTranscript) || isTimerValueTurn)
{
return "timer_clarify";
}
@@ -469,9 +493,9 @@ public sealed class JiboInteractionService(
return BuildClockLaunchDecision($"{domain}_menu", domain, "menu", replyText);
}
private static JiboInteractionDecision BuildTimerValueDecision(string loweredTranscript)
private static JiboInteractionDecision BuildTimerValueDecision(string loweredTranscript, bool allowImplicit)
{
var timer = TryParseTimerValue(loweredTranscript) ?? new ClockTimerValue("0", "1", "null");
var timer = TryParseTimerValue(loweredTranscript, allowImplicit) ?? new ClockTimerValue("0", "1", "null");
return new JiboInteractionDecision(
"timer_value",
@@ -481,16 +505,16 @@ public sealed class JiboInteractionService(
{
["skillId"] = "@be/clock",
["domain"] = "timer",
["clockIntent"] = "timerValue",
["clockIntent"] = "start",
["hours"] = timer.Hours,
["minutes"] = timer.Minutes,
["seconds"] = timer.Seconds
});
}
private static JiboInteractionDecision BuildAlarmValueDecision(string loweredTranscript)
private static JiboInteractionDecision BuildAlarmValueDecision(string loweredTranscript, bool allowImplicit)
{
var alarm = TryParseAlarmValue(loweredTranscript) ?? new ClockAlarmValue("7:00", "am");
var alarm = TryParseAlarmValue(loweredTranscript, allowImplicit) ?? new ClockAlarmValue("7:00", "am");
return new JiboInteractionDecision(
"alarm_value",
@@ -500,7 +524,7 @@ public sealed class JiboInteractionService(
{
["skillId"] = "@be/clock",
["domain"] = "alarm",
["clockIntent"] = "alarmValue",
["clockIntent"] = "start",
["time"] = alarm.Time,
["ampm"] = alarm.AmPm
});
@@ -734,9 +758,9 @@ public sealed class JiboInteractionService(
};
}
private static ClockTimerValue? TryParseTimerValue(string loweredTranscript)
private static ClockTimerValue? TryParseTimerValue(string loweredTranscript, bool allowImplicit = false)
{
if (!loweredTranscript.Contains("timer", StringComparison.Ordinal))
if (!allowImplicit && !loweredTranscript.Contains("timer", StringComparison.Ordinal))
{
return null;
}
@@ -756,9 +780,9 @@ public sealed class JiboInteractionService(
seconds is null ? "null" : seconds.Value.ToString());
}
private static ClockAlarmValue? TryParseAlarmValue(string loweredTranscript)
private static ClockAlarmValue? TryParseAlarmValue(string loweredTranscript, bool allowImplicit = false)
{
if (!loweredTranscript.Contains("alarm", StringComparison.Ordinal))
if (!allowImplicit && !loweredTranscript.Contains("alarm", StringComparison.Ordinal))
{
return null;
}
@@ -803,7 +827,8 @@ public sealed class JiboInteractionService(
return null;
}
if (!int.TryParse(minuteToken, out var minute) || minute is < 0 or > 59)
var minute = ParseNumberToken(minuteToken);
if (minute is null || minute is < 0 or > 59)
{
return null;
}
@@ -838,26 +863,79 @@ public sealed class JiboInteractionService(
"alarm for");
}
private static bool IsClockTimerValueTurn(
IReadOnlyList<string> clientRules,
IReadOnlyList<string> listenRules)
{
return clientRules.Concat(listenRules).Any(static rule =>
rule.Contains("clock/", StringComparison.OrdinalIgnoreCase) &&
rule.Contains("timer", StringComparison.OrdinalIgnoreCase) &&
rule.Contains("value", StringComparison.OrdinalIgnoreCase));
}
private static bool IsClockAlarmValueTurn(
IReadOnlyList<string> clientRules,
IReadOnlyList<string> listenRules)
{
return clientRules.Concat(listenRules).Any(static rule =>
rule.Contains("clock/", StringComparison.OrdinalIgnoreCase) &&
rule.Contains("alarm", StringComparison.OrdinalIgnoreCase) &&
rule.Contains("value", StringComparison.OrdinalIgnoreCase));
}
private static int? ExtractDurationValue(string loweredTranscript, string unitStem)
{
var pattern = new Regex($@"\b(?<value>\d+|[a-z\-]+)\s+{unitStem}s?\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
var pattern = new Regex($@"\b(?<value>\d+|[a-z\-]+(?:\s+[a-z\-]+)?)\s+{unitStem}s?\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
var match = pattern.Match(loweredTranscript);
if (!match.Success)
{
return null;
}
return ParseNumberToken(match.Groups["value"].Value);
var valueToken = match.Groups["value"].Value.Trim();
var parsed = ParseNumberToken(valueToken);
if (parsed is not null)
{
return parsed;
}
var parts = valueToken.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length >= 2)
{
parsed = ParseNumberToken(string.Join(' ', parts.TakeLast(2)));
if (parsed is not null)
{
return parsed;
}
}
return parts.Length > 0
? ParseNumberToken(parts[^1])
: null;
}
private static int? ParseNumberToken(string token)
{
var normalized = token.Trim().ToLowerInvariant();
var normalized = token.Trim().ToLowerInvariant().Replace("-", " ", StringComparison.Ordinal);
if (int.TryParse(normalized, out var numeric))
{
return numeric;
}
if (normalized.Contains(' '))
{
var parts = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
var first = ParseNumberToken(parts[0]);
var second = ParseNumberToken(parts[1]);
if (first is >= 20 && second is >= 0 and < 10)
{
return first + second;
}
}
}
return normalized switch
{
"a" or "an" => 1,
@@ -893,7 +971,7 @@ public sealed class JiboInteractionService(
private sealed record ClockAlarmValue(string Time, string AmPm);
private static readonly Regex SplitAlarmPattern = new(
@"\b(?<hour>\d{1,2}|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)(?:[:\s](?<minute>\d{2}))?\s*(?<ampm>a\.?m\.?|p\.?m\.?)?\b",
@"\b(?<hour>\d{1,2}|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)(?:[:\s](?<minute>\d{2}|[a-z\-]+(?:\s+[a-z\-]+)?))?\s*(?<ampm>a\.?m\.?|p\.?m\.?)?\b",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex CompactAlarmPattern = new(

6
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -408,14 +408,16 @@ public sealed class ResponsePlanToSocketMessagesMapper
entities["domain"] = clockDomain;
}
if (string.Equals(clockIntent, "timerValue", StringComparison.OrdinalIgnoreCase))
if (string.Equals(clockDomain, "timer", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(timerHours + timerMinutes + timerSeconds))
{
entities["hours"] = timerHours ?? "0";
entities["minutes"] = timerMinutes ?? "0";
entities["seconds"] = timerSeconds ?? "null";
}
if (string.Equals(clockIntent, "alarmValue", StringComparison.OrdinalIgnoreCase))
if (string.Equals(clockDomain, "alarm", StringComparison.OrdinalIgnoreCase) &&
(!string.IsNullOrWhiteSpace(alarmTime) || !string.IsNullOrWhiteSpace(alarmAmPm)))
{
entities["time"] = alarmTime ?? string.Empty;
entities["ampm"] = alarmAmPm ?? string.Empty;

61
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -105,6 +105,29 @@ public sealed class WebSocketTurnFinalizationService(
session.Metadata["audioTranscriptHint"] = transcriptHint;
}
if (ShouldIgnorePassiveLocalSkillContext(session, envelope.Text))
{
turnState.AwaitingTurnCompletion = false;
turnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
ResetBufferedAudio(session);
turnState.SawContext = false;
return
[
new WebSocketReply
{
Text = JsonSerializer.Serialize(new
{
type = "OPENJIBO_CONTEXT_ACK",
data = new
{
sessionId = session.SessionId,
transID = session.LastTransId
}
})
}
];
}
if (ShouldAutoFinalize(session))
{
return await FinalizeTurnAsync(session, envelope, "AUTO_FINALIZE", allowFallbackOnMissingTranscript: true, cancellationToken);
@@ -614,6 +637,18 @@ public sealed class WebSocketTurnFinalizationService(
ignoreUntilUtc.Value > DateTimeOffset.UtcNow;
}
private static bool ShouldIgnorePassiveLocalSkillContext(CloudSession session, string? text)
{
if (session.FollowUpOpen || session.TurnState.SawListen)
{
return false;
}
var skillId = TryReadContextSkillId(text);
return string.Equals(skillId, "@be/gallery", StringComparison.OrdinalIgnoreCase) ||
string.Equals(skillId, "@be/create", StringComparison.OrdinalIgnoreCase);
}
private static string? ExtractDataPayload(string? text)
{
if (string.IsNullOrWhiteSpace(text))
@@ -664,6 +699,32 @@ public sealed class WebSocketTurnFinalizationService(
}
}
private static string? TryReadContextSkillId(string? text)
{
if (string.IsNullOrWhiteSpace(text))
{
return null;
}
try
{
using var document = JsonDocument.Parse(text);
if (!document.RootElement.TryGetProperty("data", out var data) ||
!data.TryGetProperty("skill", out var skill) ||
!skill.TryGetProperty("id", out var id) ||
id.ValueKind != JsonValueKind.String)
{
return null;
}
return id.GetString();
}
catch (JsonException)
{
return null;
}
}
private static bool TryReadTransId(string? text, out string? transId)
{
transId = null;

51
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs
View File

@@ -250,7 +250,7 @@ public sealed class JiboInteractionServiceTests
}
[Fact]
public async Task BuildDecisionAsync_SetTimerForFiveMinutes_MapsToTimerValue()
public async Task BuildDecisionAsync_SetTimerForFiveMinutes_MapsToClockStartIntent()
{
var service = CreateService();
@@ -263,14 +263,14 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("timer_value", decision.IntentName);
Assert.Equal("@be/clock", decision.SkillName);
Assert.Equal("timer", decision.SkillPayload!["domain"]);
Assert.Equal("timerValue", decision.SkillPayload["clockIntent"]);
Assert.Equal("start", decision.SkillPayload["clockIntent"]);
Assert.Equal("0", decision.SkillPayload["hours"]);
Assert.Equal("5", decision.SkillPayload["minutes"]);
Assert.Equal("null", decision.SkillPayload["seconds"]);
}
[Fact]
public async Task BuildDecisionAsync_SetAlarmForSevenThirtyAm_MapsToAlarmValue()
public async Task BuildDecisionAsync_SetAlarmForSevenThirtyAm_MapsToClockStartIntent()
{
var service = CreateService();
@@ -283,7 +283,7 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("alarm_value", decision.IntentName);
Assert.Equal("@be/clock", decision.SkillName);
Assert.Equal("alarm", decision.SkillPayload!["domain"]);
Assert.Equal("alarmValue", decision.SkillPayload["clockIntent"]);
Assert.Equal("start", decision.SkillPayload["clockIntent"]);
Assert.Equal("7:30", decision.SkillPayload["time"]);
Assert.Equal("am", decision.SkillPayload["ampm"]);
}
@@ -320,6 +320,49 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("am", decision.SkillPayload["ampm"]);
}
[Fact]
public async Task BuildDecisionAsync_TimerValueFollowUp_ParsesBareDuration()
{
var service = CreateService();
var decision = await service.BuildDecisionAsync(new TurnContext
{
RawTranscript = "twenty five minutes",
NormalizedTranscript = "twenty five minutes",
Attributes = new Dictionary<string, object?>
{
["listenRules"] = new[] { "clock/timer_set_value" }
}
});
Assert.Equal("timer_value", decision.IntentName);
Assert.Equal("@be/clock", decision.SkillName);
Assert.Equal("start", decision.SkillPayload!["clockIntent"]);
Assert.Equal("25", decision.SkillPayload["minutes"]);
}
[Fact]
public async Task BuildDecisionAsync_AlarmValueFollowUp_ParsesBareSpokenTime()
{
var service = CreateService();
var decision = await service.BuildDecisionAsync(new TurnContext
{
RawTranscript = "ten twenty five",
NormalizedTranscript = "ten twenty five",
Attributes = new Dictionary<string, object?>
{
["listenRules"] = new[] { "clock/alarm_set_value" }
}
});
Assert.Equal("alarm_value", decision.IntentName);
Assert.Equal("@be/clock", decision.SkillName);
Assert.Equal("start", decision.SkillPayload!["clockIntent"]);
Assert.Equal("10:25", decision.SkillPayload["time"]);
Assert.Equal("am", decision.SkillPayload["ampm"]);
}
[Fact]
public async Task BuildDecisionAsync_SetAlarmWithoutTime_AsksForClarification()
{

114
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -371,7 +371,7 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[3]));
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("timerValue", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("start", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("@be/clock", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString());
Assert.Equal("timer", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString());
Assert.Equal("0", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("hours").GetString());
@@ -380,7 +380,7 @@ public sealed class JiboWebSocketServiceTests
using var redirectPayload = JsonDocument.Parse(replies[2].Text!);
Assert.Equal("@be/clock", redirectPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("skillID").GetString());
Assert.Equal("timerValue", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("start", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
}
[Fact]
@@ -468,7 +468,7 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal(4, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("alarmValue", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("start", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("@be/clock", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("skill").GetString());
Assert.Equal("alarm", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString());
Assert.Equal("7:30", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("time").GetString());
@@ -503,6 +503,65 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal("am", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("ampm").GetString());
}
[Fact]
public async Task ClientAsr_TimerValueFollowUp_ParsesBareDurationIntoClockStartIntent()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-clock-timer-followup-token",
Text = """{"type":"LISTEN","transID":"trans-clock-timer-followup","data":{"rules":["clock/timer_set_value"]}}"""
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-clock-timer-followup-token",
Text = """{"type":"CLIENT_ASR","transID":"trans-clock-timer-followup","data":{"text":"twenty five minutes"}}"""
});
Assert.Equal(4, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("start", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("timer", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString());
Assert.Equal("25", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("minutes").GetString());
}
[Fact]
public async Task ClientAsr_AlarmValueFollowUp_ParsesBareSpokenTimeIntoClockStartIntent()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-clock-alarm-followup-token",
Text = """{"type":"LISTEN","transID":"trans-clock-alarm-followup","data":{"rules":["clock/alarm_set_value"]}}"""
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-clock-alarm-followup-token",
Text = """{"type":"CLIENT_ASR","transID":"trans-clock-alarm-followup","data":{"text":"ten twenty five"}}"""
});
Assert.Equal(4, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("start", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("alarm", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("domain").GetString());
Assert.Equal("10:25", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("time").GetString());
Assert.Equal("am", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("ampm").GetString());
}
[Fact]
public async Task ClientAsr_SetAlarmWithoutTime_UsesClarificationSpeechInsteadOfClockRedirect()
{
@@ -573,6 +632,55 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal("menu", redirectPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
}
[Fact]
public async Task Context_FromGalleryOpen_DoesNotReopenPendingTurnOrLeaveBufferedAudioArmed()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-photo-gallery-context-token",
Text = """{"type":"LISTEN","transID":"trans-photo-gallery-context","data":{"rules":["globals/global_commands_launch"]}}"""
});
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-photo-gallery-context-token",
Text = """{"type":"CLIENT_ASR","transID":"trans-photo-gallery-context","data":{"text":"open photo gallery"}}"""
});
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-photo-gallery-context-token",
Binary = [1, 2, 3, 4, 5]
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-photo-gallery-context-token",
Text = """{"type":"CONTEXT","transID":"trans-photo-gallery-context","data":{"skill":{"id":"@be/gallery"}}}"""
});
Assert.Single(replies);
Assert.Equal("OPENJIBO_CONTEXT_ACK", ReadReplyType(replies[0]));
var session = _store.FindSessionByToken("hub-photo-gallery-context-token");
Assert.NotNull(session);
Assert.False(session.TurnState.AwaitingTurnCompletion);
Assert.Equal(0, session.TurnState.BufferedAudioBytes);
Assert.Equal(0, session.TurnState.BufferedAudioChunkCount);
}
[Fact]
public async Task ClientAsr_SnapAPicture_RedirectsIntoCreateSkill()
{

65
artifact-output/jibo-test-15/captures/http/20260422.events.ndjson Normal file
View File

File diff suppressed because one or more lines are too long

4
artifact-output/jibo-test-15/captures/turn/20260421.events.ndjson Normal file
View File

@@ -0,0 +1,4 @@
{
"exception": "System.InvalidOperationException: whisper.cpp returned no transcript for the buffered audio turn.\n at Jibo.Cloud.Infrastructure.Audio.LocalWhisperCppBufferedAudioSttStrategy.TranscribeAsync(TurnContext turn, CancellationToken cancellationToken) in /home/jake-dubin/JiboExperiments/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs:line 58\n at Jibo.Cloud.Application.Services.WebSocketTurnFinalizationService.ResolveTranscriptAsync(TurnContext turn, CloudSession session, CancellationToken cancellationToken) in /home/jake-dubin/JiboExperiments/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs:line 219",
"message": "Error during STT processing"
}

2926
artifact-output/jibo-test-15/captures/websocket/20260422.events.ndjson Normal file
View File

File diff suppressed because it is too large Load Diff

127
artifact-output/jibo-test-15/captures/websocket/fixtures/neohubjibocom-neohublisten-tid52b6f46e3b6a11f191515cf821ea55ae.flow.json Normal file
View File

@@ -0,0 +1,127 @@
{
"name": "neohubjibocom-neohublisten-tid52b6f46e3b6a11f191515cf821ea55ae",
"session": {
"hostName": "neo-hub.jibo.com",
"path": "/v1/listen",
"kind": "neo-hub-listen",
"token": "hub-usr_openjibo_owner-1776545569192"
},
"steps": [
{
"text": {
"type": "LISTEN",
"ts": 1776546327052,
"msgID": "mid-5303b5f6-3b6a-11f1-adc4-5cf821ea55ae",
"transID": "tid-52b6f46e-3b6a-11f1-9151-5cf821ea55ae",
"data": {
"lang": "en-US",
"hotphrase": true,
"rules": [
"launch",
"globals/global_commands_launch"
],
"mode": "CLIENT_ASR",
"asr": {
"hints": [],
"earlyEOS": [],
"encoding": "OGG_OPUS",
"sampleRate": 16000,
"sosTimeout": 7000,
"maxSpeechTimeout": 20000
}
}
},
"binary": null,
"expectedReplyTypes": [
"OPENJIBO_TURN_PENDING"
]
},
{
"text": {
"type": "CLIENT_ASR",
"ts": 1776546327052,
"msgID": "mid-5303bd76-3b6a-11f1-81d4-5cf821ea55ae",
"transID": "tid-52b6f46e-3b6a-11f1-9151-5cf821ea55ae",
"data": {
"text": "tell me about the news"
}
},
"binary": null,
"expectedReplyTypes": [
"LISTEN",
"EOS",
"SKILL_ACTION"
]
},
{
"text": {
"type": "CONTEXT",
"ts": 1776546327177,
"msgID": "mid-5316cf2e-3b6a-11f1-8aa7-5cf821ea55ae",
"transID": "tid-52b6f46e-3b6a-11f1-9151-5cf821ea55ae",
"data": {
"runtime": {
"character": {
"emotion": {
"name": "NEUTRAL",
"valence": 0.45,
"confidence": 0.2
},
"motivation": {
"social": 1,
"playful": 1
}
},
"perception": {
"speaker": null,
"peoplePresent": []
},
"location": {
"city": "Pleasant Hill",
"state": "Missouri",
"stateAbbr": "MO",
"country": "United States",
"countryCode": "US",
"lat": 38.8358494,
"lng": -94.1427229,
"iso": "2026-04-18T16:05:27.073-05:00"
},
"loop": {
"loopId": "5c0b221fdf9d450019c5e253",
"users": [
{
"firstName": "Erin",
"lastName": "Picone",
"phoneticName": "Erin",
"gender": "female",
"birthdate": 649209600000,
"id": "5c0b221fdf9d450019c5e255",
"accountId": "5c0b20547c46170019235759"
}
],
"jibo": {
"color": "WHITE",
"birthdate": 1544234645598,
"id": "5c0b221fdf9d450019c5e254"
},
"owner": "5c0b221fdf9d450019c5e255"
},
"dialog": {
"referent": null
}
},
"skill": {
"id": null
},
"general": {
"release": "1.9.2"
}
}
},
"binary": null,
"expectedReplyTypes": [
"OPENJIBO_CONTEXT_ACK"
]
}
]
}

174
artifact-output/jibo-test-15/captures/websocket/fixtures/neohubjibocom-neohublisten-tida8165b823b9411f195545cf821ea55ae.flow.json Normal file
View File

@@ -0,0 +1,174 @@
{
"name": "neohubjibocom-neohublisten-tida8165b823b9411f195545cf821ea55ae",
"session": {
"hostName": "neo-hub.jibo.com",
"path": "/v1/listen",
"kind": "neo-hub-listen",
"token": "hub-usr_openjibo_owner-1776563295273"
},
"steps": [
{
"text": {
"type": "LISTEN",
"ts": 1776564508900,
"msgID": "mid-a83d9076-3b94-11f1-978d-5cf821ea55ae",
"transID": "tid-a8165b82-3b94-11f1-9554-5cf821ea55ae",
"data": {
"lang": "en-US",
"hotphrase": true,
"rules": [
"launch",
"globals/global_commands_launch"
],
"mode": "",
"asr": {
"hints": [],
"earlyEOS": [],
"encoding": "OGG_OPUS",
"sampleRate": 16000,
"sosTimeout": 7000,
"maxSpeechTimeout": 20000
}
}
},
"binary": null,
"expectedReplyTypes": [
"LISTEN",
"EOS",
"SKILL_ACTION"
]
},
{
"text": null,
"binary": [
79,
103,
103,
83,
0,
2,
0,
0,
0,
0,
0,
0,
0,
0,
62,
200,
6,
48,
0,
0,
0,
0,
146,
142,
146,
101,
1,
19,
79,
112,
117,
115,
72,
101,
97,
100,
1,
1,
0,
0,
128,
62,
0,
0,
0,
0,
0
],
"expectedReplyTypes": [
"OPENJIBO_AUDIO_RECEIVED"
]
},
{
"text": {
"type": "CONTEXT",
"ts": 1776564510009,
"msgID": "mid-a8e6d640-3b94-11f1-b2b0-5cf821ea55ae",
"transID": "tid-a8165b82-3b94-11f1-9554-5cf821ea55ae",
"data": {
"runtime": {
"character": {
"emotion": {
"name": "NEUTRAL",
"valence": 0.45,
"confidence": 0.2
},
"motivation": {
"social": 1,
"playful": 1
}
},
"perception": {
"speaker": null,
"peoplePresent": [
{
"id": "NOT_TRAINED",
"entityId": 1282,
"type": "fused",
"confidence": 0.25
}
]
},
"location": {
"city": "Pleasant Hill",
"state": "Missouri",
"stateAbbr": "MO",
"country": "United States",
"countryCode": "US",
"lat": 38.8358494,
"lng": -94.1427229,
"iso": "2026-04-18T21:08:29.921-05:00"
},
"loop": {
"loopId": "5c0b221fdf9d450019c5e253",
"users": [
{
"firstName": "Erin",
"lastName": "Picone",
"phoneticName": "Erin",
"gender": "female",
"birthdate": 649209600000,
"id": "5c0b221fdf9d450019c5e255",
"accountId": "5c0b20547c46170019235759"
}
],
"jibo": {
"color": "WHITE",
"birthdate": 1544234645598,
"id": "5c0b221fdf9d450019c5e254"
},
"owner": "5c0b221fdf9d450019c5e255"
},
"dialog": {
"referent": null
}
},
"skill": {
"id": null
},
"general": {
"release": "1.9.2"
}
}
},
"binary": null,
"expectedReplyTypes": [
"OPENJIBO_CONTEXT_ACK"
]
}
]
}

95
artifact-output/jibo-test-15/captures/websocket/fixtures/neohubjibocom-neohublisten-tidd2b591403b6811f1a5735cf821ea55ae.flow.json Normal file
View File

@@ -0,0 +1,95 @@
{
"name": "neohubjibocom-neohublisten-tidd2b591403b6811f1a5735cf821ea55ae",
"session": {
"hostName": "neo-hub.jibo.com",
"path": "/v1/listen",
"kind": "neo-hub-listen",
"token": "hub-usr_openjibo_owner-1776545569192"
},
"steps": [
{
"text": {
"type": "LISTEN",
"ts": 1776545682509,
"msgID": "mid-d2d657ea-3b68-11f1-985b-5cf821ea55ae",
"transID": "tid-d2b59140-3b68-11f1-a573-5cf821ea55ae",
"data": {
"lang": "en-US",
"hotphrase": true,
"rules": [
"launch",
"globals/global_commands_launch"
],
"mode": "",
"asr": {
"hints": [],
"earlyEOS": [],
"encoding": "OGG_OPUS",
"sampleRate": 16000,
"sosTimeout": 7000,
"maxSpeechTimeout": 20000
}
}
},
"binary": null,
"expectedReplyTypes": [
"OPENJIBO_TURN_PENDING"
]
},
{
"text": null,
"binary": [
79,
103,
103,
83,
0,
2,
0,
0,
0,
0,
0,
0,
0,
0,
94,
132,
99,
103,
0,
0,
0,
0,
234,
141,
12,
246,
1,
19,
79,
112,
117,
115,
72,
101,
97,
100,
1,
1,
0,
0,
128,
62,
0,
0,
0,
0,
0
],
"expectedReplyTypes": [
"OPENJIBO_AUDIO_RECEIVED"
]
}
]
}

1492
artifact-output/jibo-test-15/jibo test 15.txt Normal file
View File

File diff suppressed because it is too large Load Diff