Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

yes no fix and better wod answers

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-19 21:44:04 -05:00
parent 1310bf47e3
commit 18c6617087
18 changed files with 8214 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
OpenJibo/docs/development-plan.md
View File

@@ -152,6 +152,8 @@ Latest stock-OS WOD findings:
- Voice `play word of the day` hotphrase launch still enters Global Service first, so a synthetic `LISTEN` result alone is not enough. The next-most-correct transport hint is a direct `SKILL_REDIRECT` event aimed at `@be/word-of-the-day`, alongside the menu-shaped `LISTEN` payload.
- Stock OS also keeps the original hotphrase/global launch cloud response promise alive even after the redirect succeeds, so voice WOD launch needs an explicit silent `SKILL_ACTION` completion on the same transID to avoid later cloud-response culling and an interrupted game state.
- Auto-dismissing `word-of-the-day/right_word` with a no-input `LISTEN`/`EOS` stops the listening ring, but it does not close the WOD UI by itself. Pairing that no-input closeout with an explicit redirect back to `@be/idle` is the current cleanest approximation.
- OTA/update yes-no prompts can advertise `$YESNO` only through ASR hints rather than `listenRules`, so short denials like `no` need to be recognized from `listenAsrHints` too.
- Spoken WOD guesses should preferentially snap to the closest offered hint when Whisper lands very close to one of the menu words, since near-misses like `haglet` for `aglet` are common in live testing.
## Speech, Animation, And ESML

83
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -269,7 +269,7 @@ public sealed class JiboInteractionService(
return guessValue;
}
var loweredTranscript = transcript.Trim().TrimEnd('.', '!', '?', ',').ToLowerInvariant();
var loweredTranscript = NormalizeGuessToken(transcript);
var hintIndex = loweredTranscript switch
{
"1" or "one" or "first" => 0,
@@ -283,17 +283,96 @@ public sealed class JiboInteractionService(
return listenAsrHints[hintIndex];
}
var fuzzyHintMatch = FindClosestHint(loweredTranscript, listenAsrHints);
if (!string.IsNullOrWhiteSpace(fuzzyHintMatch))
{
return fuzzyHintMatch;
}
return transcript;
}
private static bool IsYesNoTurn(TurnContext turn)
{
return ReadRules(turn, "listenRules").Concat(ReadRules(turn, "clientRules"))
return ReadRules(turn, "listenRules")
.Concat(ReadRules(turn, "clientRules"))
.Concat(ReadRules(turn, "listenAsrHints"))
.Any(static rule =>
string.Equals(rule, "$YESNO", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "create/is_it_a_keeper", StringComparison.OrdinalIgnoreCase));
}
private static string? FindClosestHint(string normalizedTranscript, IReadOnlyList<string> hints)
{
if (string.IsNullOrWhiteSpace(normalizedTranscript))
{
return null;
}
string? bestHint = null;
var bestDistance = int.MaxValue;
foreach (var hint in hints)
{
if (string.IsNullOrWhiteSpace(hint))
{
continue;
}
var normalizedHint = NormalizeGuessToken(hint);
if (string.IsNullOrWhiteSpace(normalizedHint))
{
continue;
}
if (string.Equals(normalizedTranscript, normalizedHint, StringComparison.Ordinal))
{
return hint;
}
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance)
{
bestDistance = distance;
bestHint = hint;
}
}
return bestDistance <= 2 ? bestHint : null;
}
private static string NormalizeGuessToken(string value)
{
return value.Trim().TrimEnd('.', '!', '?', ',').ToLowerInvariant();
}
private static int ComputeEditDistance(string left, string right)
{
var previous = new int[right.Length + 1];
var current = new int[right.Length + 1];
for (var column = 0; column <= right.Length; column += 1)
{
previous[column] = column;
}
for (var row = 1; row <= left.Length; row += 1)
{
current[0] = row;
for (var column = 1; column <= right.Length; column += 1)
{
var substitutionCost = left[row - 1] == right[column - 1] ? 0 : 1;
current[column] = Math.Min(
Math.Min(current[column - 1] + 1, previous[column] + 1),
previous[column - 1] + substitutionCost);
}
(previous, current) = (current, previous);
}
return previous[right.Length];
}
private static IEnumerable<string> ReadRules(TurnContext turn, string key)
{
if (!turn.Attributes.TryGetValue(key, out var value) || value is null)

89
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -358,7 +358,7 @@ public sealed class ResponsePlanToSocketMessagesMapper
return nluGuess;
}
var normalized = transcript.Trim().TrimEnd('.', '!', '?', ',').ToLowerInvariant();
var normalized = NormalizeGuessToken(transcript);
var hintIndex = normalized switch
{
"1" or "one" or "first" => 0,
@@ -367,15 +367,90 @@ public sealed class ResponsePlanToSocketMessagesMapper
_ => -1
};
if (hintIndex < 0)
var hints = ReadRuleValues(turn, "listenAsrHints").ToArray();
if (hintIndex >= 0)
{
return transcript;
return hintIndex < hints.Length
? hints[hintIndex]
: transcript;
}
var hints = ReadRuleValues(turn, "listenAsrHints").ToArray();
return hintIndex < hints.Length
? hints[hintIndex]
: transcript;
var fuzzyHintMatch = FindClosestHint(normalized, hints);
return string.IsNullOrWhiteSpace(fuzzyHintMatch)
? transcript
: fuzzyHintMatch;
}
private static string? FindClosestHint(string normalizedTranscript, IReadOnlyList<string> hints)
{
if (string.IsNullOrWhiteSpace(normalizedTranscript))
{
return null;
}
string? bestHint = null;
var bestDistance = int.MaxValue;
foreach (var hint in hints)
{
if (string.IsNullOrWhiteSpace(hint))
{
continue;
}
var normalizedHint = NormalizeGuessToken(hint);
if (string.IsNullOrWhiteSpace(normalizedHint))
{
continue;
}
if (string.Equals(normalizedTranscript, normalizedHint, StringComparison.Ordinal))
{
return hint;
}
var distance = ComputeEditDistance(normalizedTranscript, normalizedHint);
if (distance < bestDistance)
{
bestDistance = distance;
bestHint = hint;
}
}
return bestDistance <= 2 ? bestHint : null;
}
private static string NormalizeGuessToken(string value)
{
return value.Trim().TrimEnd('.', '!', '?', ',').ToLowerInvariant();
}
private static int ComputeEditDistance(string left, string right)
{
var previous = new int[right.Length + 1];
var current = new int[right.Length + 1];
for (var column = 0; column <= right.Length; column += 1)
{
previous[column] = column;
}
for (var row = 1; row <= left.Length; row += 1)
{
current[0] = row;
for (var column = 1; column <= right.Length; column += 1)
{
var substitutionCost = left[row - 1] == right[column - 1] ? 0 : 1;
current[column] = Math.Min(
Math.Min(current[column - 1] + 1, previous[column] + 1),
previous[column - 1] + substitutionCost);
}
(previous, current) = (current, previous);
}
return previous[right.Length];
}
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill)

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -701,7 +701,9 @@ public sealed class WebSocketTurnFinalizationService(
private static bool IsYesNoTurn(TurnContext turn)
{
return ReadRules(turn, "listenRules").Concat(ReadRules(turn, "clientRules"))
return ReadRules(turn, "listenRules")
.Concat(ReadRules(turn, "clientRules"))
.Concat(ReadRules(turn, "listenAsrHints"))
.Any(static rule =>
string.Equals(rule, "$YESNO", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "create/is_it_a_keeper", StringComparison.OrdinalIgnoreCase));

41
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs
View File

@@ -76,6 +76,26 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("Yes.", decision.ReplyText);
}
[Fact]
public async Task BuildDecisionAsync_YesNoFollowUp_FromAsrHints_MapsShortDenialToNoIntent()
{
var service = CreateService();
var decision = await service.BuildDecisionAsync(new TurnContext
{
RawTranscript = "no",
NormalizedTranscript = "no",
Attributes = new Dictionary<string, object?>
{
["listenRules"] = new[] { "surprises-ota/want_to_download_now" },
["listenAsrHints"] = new[] { "$YESNO" }
}
});
Assert.Equal("no", decision.IntentName);
Assert.Equal("No.", decision.ReplyText);
}
[Fact]
public async Task BuildDecisionAsync_SkillPhraseVariant_MapsToKnownIntent()
{
@@ -172,6 +192,27 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("completion_only", decision.SkillPayload["cloudResponseMode"]);
}
[Fact]
public async Task BuildDecisionAsync_WordOfDayGuess_FuzzyMatchesClosestHint()
{
var service = CreateService();
var decision = await service.BuildDecisionAsync(new TurnContext
{
RawTranscript = "Haglet.",
NormalizedTranscript = "Haglet.",
Attributes = new Dictionary<string, object?>
{
["listenRules"] = new[] { "word-of-the-day/puzzle" },
["listenAsrHints"] = new[] { "aglet", "hovel", "wisenheimer" }
}
});
Assert.Equal("word_of_the_day_guess", decision.IntentName);
Assert.Equal("I heard aglet.", decision.ReplyText);
Assert.Equal("aglet", decision.SkillPayload!["guess"]);
}
private static JiboInteractionService CreateService()
{
return new JiboInteractionService(

57
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -373,6 +373,34 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal("create/is_it_a_keeper", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString());
}
[Fact]
public async Task ClientAsr_YesNoPromptFromAsrHints_MapsShortDenialToNoIntent()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-yesno-hints-token",
Text = """{"type":"LISTEN","transID":"trans-yesno-hints","data":{"rules":["surprises-ota/want_to_download_now"],"asr":{"hints":["$YESNO"]}}}"""
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-yesno-hints-token",
Text = """{"type":"CLIENT_ASR","transID":"trans-yesno-hints","data":{"text":"no"}}"""
});
Assert.Equal(3, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("no", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("no", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
}
[Fact]
public async Task ClientNlu_WordOfDayGuess_UsesGuessEntityAsAsrTextAndCompletesTurn()
{
@@ -468,6 +496,35 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
}
[Fact]
public async Task ClientAsr_WordOfDayGuess_FuzzyMatchesClosestHint()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-wod-fuzzy-guess-token",
Text = """{"type":"LISTEN","transID":"trans-wod-fuzzy-guess","data":{"rules":["word-of-the-day/puzzle"],"asr":{"hints":["aglet","hovel","wisenheimer"]}}}"""
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-wod-fuzzy-guess-token",
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-fuzzy-guess","data":{"text":"Haglet."}}"""
});
Assert.Equal(3, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("aglet", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("aglet", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("guess").GetString());
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
}
[Fact]
public async Task ClientAsr_WordOfDayLaunch_EmitsMenuStyleLoadMenuAndRedirect()
{