Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support voice routes and short-answer STT handling

Browse Source
This commit is contained in:
Jacob Dubin
2026-05-23 20:49:09 -05:00
parent b017fd9f60
commit c36a01b142
12 changed files with 318 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
OpenJibo/docs/feature-backlog.md
View File

@@ -1005,3 +1005,4 @@ For `1.0.20` and beyond:
6. Advanced Jibo features such as pizza delivery, Uber/Lyft integration, calendar management, smart home control (Home Assistant), etc. can be added after the conversion process is smooth and stable, with a focus on features that leverage the new cloud capabilities and content personalization enabled by Open Jibo
7. LLM integration for more natural dialog, question answering, and content generation can be explored as a longer-term goal after the core platform is stable and has a growing user base to provide feedback and use cases for LLM-powered features
8. Tiered Jibo brain/orchestration plan from README.md can be implemented in parallel with the above, starting with the simplest cloud features and gradually adding more complex capabilities as the platform matures and user feedback is collected, always preserving his unique charm and original features.
9. Accessibility-first voice parity for menu actions, starting with backup / restore / update and extending to other critical app flows so menu functionality remains available through voice in a later release

1
OpenJibo/docs/release-1.0.19-plan.md
View File

@@ -382,6 +382,7 @@ First completed slice in this personal-report parity track:
9. Provider-backed news expansion and deeper weather parity
10. Capture indexing and retention boundary for group testing, including a lightweight manifest beside raw capture files
11. Binary-safe media storage seam with file and Azure Blob adapters, ready for original/thumbnails follow-up
12. Accessibility voice parity planning for menu-equivalent flows, starting with backup / restore / update voice coverage and broader critical-path accessibility in a later release
For slice 1, use the new import ladder above to keep the work grounded in what OpenJibo can already render today versus what needs new scaffolding.
For slices 2-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements.

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs
View File

@@ -40,6 +40,10 @@ public sealed class JiboExperienceCatalog
public IReadOnlyList<string> PersonalReportKickOffReplies { get; init; } = [];
public IReadOnlyList<string> PersonalReportOutroReplies { get; init; } = [];
public IReadOnlyList<string> ReportSkillTemplates { get; init; } = [];
public IReadOnlyList<string> BackupHowReplies { get; init; } = [];
public IReadOnlyList<string> RestoreHowReplies { get; init; } = [];
public IReadOnlyList<string> UpdateNextReplies { get; init; } = [];
public IReadOnlyList<string> UpdateLastReplies { get; init; } = [];
public IReadOnlyList<string> WeatherIntroReplies { get; init; } = [];
public IReadOnlyList<string> WeatherTomorrowIntroReplies { get; init; } = [];
public IReadOnlyList<string> WeatherTodayHighLowReplies { get; init; } = [];

19
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.DecisionDispatch.cs
View File

@@ -119,6 +119,25 @@ public sealed partial class JiboInteractionService
"day" => BuildClockLaunchDecision("day", "clock", "askForDay", "Showing the day."),
"current_location" => BuildCurrentLocationDecision(turn),
"cloud_version" => BuildCloudVersionDecision(),
"backup_help" => BuildScriptedSupportDecision(
catalog.BackupHowReplies,
"backup_help",
"cloud backup",
"back up",
"restore"),
"restore_backup" => BuildScriptedSupportDecision(
catalog.RestoreHowReplies,
"restore_backup",
"restore you from a backup",
"restore from a backup"),
"update_next" => BuildScriptedSupportDecision(
catalog.UpdateNextReplies,
"update_next",
"next update"),
"update_last" => BuildScriptedSupportDecision(
catalog.UpdateLastReplies,
"update_last",
"last update"),
"radio" => BuildRadioLaunchDecision(),
"radio_genre" => BuildRadioGenreLaunchDecision(lowered),
"stop" => BuildStopDecision(),

38
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs
View File

@@ -1028,6 +1028,44 @@ public sealed partial class JiboInteractionService
if (MatchesAny(loweredTranscript, "commute", "traffic", "drive to work", "how long to work")) return "commute";
if (MatchesAny(
loweredTranscript,
"can i backup my jibo",
"can i back up my jibo",
"how can i backup my jibo",
"how can i back up my jibo",
"how do i backup my jibo",
"how do i back up my jibo",
"can you be backed up",
"how can i store you in the cloud",
"how can i store you online",
"how do i store you in the cloud",
"how do i store you online"))
return "backup_help";
if (MatchesAny(
loweredTranscript,
"can i restore you from a backup",
"how can i restore you from a backup",
"how do i restore you from a backup",
"restore you from a backup",
"restore from a backup"))
return "restore_backup";
if (MatchesAny(
loweredTranscript,
"when is your next update",
"when is my next update",
"when's your next update",
"when s your next update",
"when was your last update",
"when was my last update",
"when's your last update",
"when s your last update"))
return loweredTranscript.Contains("last update", StringComparison.OrdinalIgnoreCase)
? "update_last"
: "update_next";
if (MatchesAny(loweredTranscript, "news", "headlines", "news update", "tell me the news")) return "news";
if (IsWelcomeBackGreeting(loweredTranscript) ||

27
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs
View File

@@ -714,6 +714,21 @@ public sealed partial class JiboInteractionService
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedSupportDecision(
IReadOnlyList<string> replies,
string intentName,
params string[] preferredSnippets)
{
var selected = SelectLegacyReply(replies, preferredSnippets);
if (string.IsNullOrWhiteSpace(selected))
selected = GetSupportFallbackReply(intentName);
return new JiboInteractionDecision(
intentName,
selected,
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildScriptedHolidayGreetingDecision(
JiboExperienceCatalog catalog,
string intentName,
@@ -758,6 +773,18 @@ public sealed partial class JiboInteractionService
return ScriptedResponseDecisionBuilder.SelectLegacyReply(replies, randomizer, preferredSnippets);
}
private static string GetSupportFallbackReply(string intentName)
{
return intentName switch
{
"backup_help" => "That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com.",
"restore_backup" => "That sounds a little too complicated for me, I think your best bet is to get some guidance from Jibo Customer Care. Check the Help section of the Jibo App, or go to the website, support dot jibo dot com.",
"update_next" => "That's a good question. I think they've been coming every few weeks.",
"update_last" => "Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates.",
_ => string.Empty
};
}
private string RenderHolidayTemplate(string template, TurnContext turn, GreetingPresenceProfile presence)
{
var ownerName = ResolvePreferredGreetingName(turn, presence);

56
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs
View File

@@ -8,6 +8,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
IExternalProcessRunner processRunner) : ISttStrategy
{
private const int MinimumBufferedAudioBytes = 64;
private const int ShortAnswerBufferedAudioBytes = 16;
public string Name => "local-whispercpp-buffered-audio";
@@ -18,7 +19,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
IsConfiguredPathAvailable(options.WhisperCliPath, true) &&
IsConfiguredPathAvailable(options.WhisperModelPath, true) &&
ReadBufferedAudioFrames(turn).Any(ContainsOpusIdentificationHeader) &&
!IsBelowNoiseFloor(ReadBufferedAudioBytes(turn));
!IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn));
}
public async Task<SttResult> TranscribeAsync(TurnContext turn, CancellationToken cancellationToken = default)
@@ -31,7 +32,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
throw new InvalidOperationException(
"Local whisper.cpp STT requires buffered Ogg/Opus audio with an Opus identification header.");
if (IsBelowNoiseFloor(ReadBufferedAudioBytes(turn)))
if (IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn)))
throw new InvalidOperationException(
"Local whisper.cpp STT rejected buffered audio as too short or noisy for transcription.");
@@ -119,9 +120,54 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
: 0;
}
private static bool IsBelowNoiseFloor(int bufferedAudioBytes)
private static bool IsBelowNoiseFloor(TurnContext turn, int bufferedAudioBytes)
{
return bufferedAudioBytes > 0 && bufferedAudioBytes < MinimumBufferedAudioBytes;
if (bufferedAudioBytes <= 0) return false;
var minimumBufferedAudioBytes = IsShortAnswerTurn(turn)
? ShortAnswerBufferedAudioBytes
: MinimumBufferedAudioBytes;
return bufferedAudioBytes < minimumBufferedAudioBytes;
}
private static bool IsShortAnswerTurn(TurnContext turn)
{
var rules = ReadRules(turn, "listenRules")
.Concat(ReadRules(turn, "clientRules"))
.Concat(ReadRules(turn, "listenAsrHints"));
return rules.Any(IsShortAnswerRule);
}
private static bool IsShortAnswerRule(string rule)
{
return string.Equals(rule, "$YESNO", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "clock/alarm_timer_change", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "clock/alarm_timer_none_set", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "create/is_it_a_keeper", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "settings/download_now_later", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "shared/yes_no", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "surprises-date/offer_date_fact", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "surprises-ota/want_to_download_now", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/surprise", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/right_word", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/puzzle", StringComparison.OrdinalIgnoreCase);
}
private static IEnumerable<string> ReadRules(TurnContext turn, string key)
{
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return [];
return value switch
{
IReadOnlyList<string> typed => typed,
IEnumerable<string> enumerable => enumerable,
JsonElement { ValueKind: JsonValueKind.Array } jsonElement => jsonElement.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty),
_ => []
};
}
private static bool ContainsOpusIdentificationHeader(byte[] frame)
@@ -167,4 +213,4 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
return !checkFileExists || File.Exists(path);
}
}
}

17
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs
View File

@@ -195,6 +195,23 @@ public sealed class InMemoryJiboExperienceContentRepository : IJiboExperienceCon
[
"The report-skill templates are loaded and waiting to be rendered."
],
BackupHowReplies =
[
"That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com."
],
RestoreHowReplies =
[
"That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com."
],
UpdateNextReplies =
[
"That's a good question. I think they've been coming every few weeks.",
"I never know exactly when my next update is coming, but they do seem to come pretty regularly."
],
UpdateLastReplies =
[
"Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates."
],
WeatherIntroReplies =
[
"For your weather.",

38
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs
View File

@@ -241,6 +241,18 @@ public static class LegacyMimCatalogImporter
string.Equals(fileName, "WetNowDryLater", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.ReportSkillTemplate;
if (fileName.StartsWith("SUP_GEN_HowBackUpData", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.BackupHow;
if (fileName.StartsWith("SUP_GEN_HowRestoreBackup", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.RestoreHow;
if (fileName.StartsWith("SUP_UPDATE_WhenIsNextUpdate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.UpdateNext;
if (fileName.StartsWith("SUP_UPDATE_WhenWasLastUpdate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.UpdateLast;
if (fileName.StartsWith("PersonalReportKickOff", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.PersonalReportKickOff;
@@ -515,6 +527,10 @@ public static class LegacyMimCatalogImporter
HolidayGift,
HolidayTracker,
BirthdayCelebration,
BackupHow,
RestoreHow,
UpdateNext,
UpdateLast,
Jokes,
RobotFacts,
HumanFacts,
@@ -583,6 +599,7 @@ public static class LegacyMimCatalogImporter
private readonly List<string> _commuteTransportHurryReplies = [];
private readonly List<string> _commuteTransportLateReplies = [];
private readonly List<string> _commuteTransportNormalReplies = [];
private readonly List<string> _backupHowReplies = [];
private readonly List<JiboConditionedReply> _emotionReplies = [];
private readonly List<string> _fallbacks = [];
private readonly List<string> _favoriteAnimalReplies = [];
@@ -603,6 +620,9 @@ public static class LegacyMimCatalogImporter
private readonly List<string> _newsCategoryIntroReplies = [];
private readonly List<string> _newsIntroReplies = [];
private readonly List<string> _newsOutroReplies = [];
private readonly List<string> _restoreHowReplies = [];
private readonly List<string> _updateLastReplies = [];
private readonly List<string> _updateNextReplies = [];
private readonly List<string> _personalities = [];
private readonly List<string> _personalReportKickOffReplies = [];
private readonly List<string> _personalReportOutroReplies = [];
@@ -681,6 +701,18 @@ public static class LegacyMimCatalogImporter
case LegacyMimBucket.BirthdayCelebration:
AddDistinct(_birthdayCelebrationReplies, text);
return;
case LegacyMimBucket.BackupHow:
AddDistinct(_backupHowReplies, text);
return;
case LegacyMimBucket.RestoreHow:
AddDistinct(_restoreHowReplies, text);
return;
case LegacyMimBucket.UpdateNext:
AddDistinct(_updateNextReplies, text);
return;
case LegacyMimBucket.UpdateLast:
AddDistinct(_updateLastReplies, text);
return;
case LegacyMimBucket.Personality:
if (_personalities.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase)))
return;
@@ -835,6 +867,7 @@ public static class LegacyMimCatalogImporter
HolidayGiftReplies = [.. _holidayGiftReplies],
HolidayTrackerReplies = [.. _holidayTrackerReplies],
BirthdayCelebrationReplies = [.. _birthdayCelebrationReplies],
BackupHowReplies = [.. _backupHowReplies],
HowAreYouReplies = [.. _howAreYous],
EmotionReplies = [.. _emotionReplies],
PersonalityReplies = [.. _personalities],
@@ -869,7 +902,10 @@ public static class LegacyMimCatalogImporter
CommuteServiceDownReplies = [.. _commuteServiceDownReplies],
NewsIntroReplies = [.. _newsIntroReplies],
NewsCategoryIntroReplies = [.. _newsCategoryIntroReplies],
NewsOutroReplies = [.. _newsOutroReplies]
NewsOutroReplies = [.. _newsOutroReplies],
RestoreHowReplies = [.. _restoreHowReplies],
UpdateNextReplies = [.. _updateNextReplies],
UpdateLastReplies = [.. _updateLastReplies]
};
}

23
OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs
View File

@@ -190,6 +190,29 @@ public sealed class LegacyMimCatalogImporterTests
reply.Contains("north Pole", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void ImportCatalog_ImportsBuildBSupportResponsesIntoDedicatedBuckets()
{
var rootDirectory = Path.Combine(
AppContext.BaseDirectory,
"Content",
"LegacyMims",
"BuildB");
var catalog = LegacyMimCatalogImporter.ImportCatalog(rootDirectory);
Assert.Contains(catalog.BackupHowReplies, reply =>
reply.Contains("Help section of the Jibo App", StringComparison.OrdinalIgnoreCase));
Assert.Contains(catalog.RestoreHowReplies, reply =>
reply.Contains("Jibo Customer Care", StringComparison.OrdinalIgnoreCase));
Assert.Contains(catalog.UpdateNextReplies, reply =>
reply.Contains("coming every few weeks", StringComparison.OrdinalIgnoreCase));
Assert.Contains(catalog.UpdateNextReplies, reply =>
reply.Contains("pretty regularly", StringComparison.OrdinalIgnoreCase));
Assert.Contains(catalog.UpdateLastReplies, reply =>
reply.Contains("release notes page", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void ImportCatalog_ImportsBuildBFriendshipResponsesIntoFriendBuckets()
{

23
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs
View File

@@ -712,6 +712,29 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]);
}
[Theory]
[InlineData("can i backup my jibo", "backup_help", "Help section of the Jibo App")]
[InlineData("how can i restore you from a backup", "restore_backup", "Jibo Customer Care")]
[InlineData("when is your next update", "update_next", "coming every few weeks")]
[InlineData("when was your last update", "update_last", "release notes page")]
public async Task BuildDecisionAsync_SupportHelpQuestions_UseImportedReplies(
string transcript,
string expectedIntent,
string expectedReplySnippet)
{
var service = CreateService();
var decision = await service.BuildDecisionAsync(new TurnContext
{
RawTranscript = transcript,
NormalizedTranscript = transcript
});
Assert.Equal(expectedIntent, decision.IntentName);
Assert.Contains(expectedReplySnippet, decision.ReplyText, StringComparison.OrdinalIgnoreCase);
Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]);
}
[Theory]
[InlineData("what do you want to talk about", "robot_want_to_talk_about", "surprise me")]
[InlineData("what would you like to talk about", "robot_want_to_talk_about", "surprise me")]

78
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs
View File

@@ -102,6 +102,34 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests
Assert.False(strategy.CanHandle(turn));
}
[Theory]
[InlineData("shared/yes_no")]
[InlineData("word-of-the-day/surprise")]
public void CanHandle_ReturnsTrue_WhenShortAnswerTurnsStayUnderTheStandardNoiseFloor(string listenRule)
{
var strategy = new LocalWhisperCppBufferedAudioSttStrategy(
new BufferedAudioSttOptions
{
EnableLocalWhisperCpp = true,
FfmpegPath = "ffmpeg",
WhisperCliPath = "whisper-cli",
WhisperModelPath = "model.bin"
},
new FakeExternalProcessRunner());
var turn = new TurnContext
{
Attributes = new Dictionary<string, object?>
{
["bufferedAudioBytes"] = 47,
["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() },
["listenRules"] = new[] { listenRule }
}
};
Assert.True(strategy.CanHandle(turn));
}
[Fact]
public async Task TranscribeAsync_UsesFfmpegAndWhisperCpp_WhenConfigured()
{
@@ -148,6 +176,54 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests
}
}
[Theory]
[InlineData("shared/yes_no")]
[InlineData("word-of-the-day/surprise")]
public async Task TranscribeAsync_HandlesShortAnswerTurnsWithoutHittingTheStandardNoiseFloor(string listenRule)
{
var tempDirectory = Path.Combine(Path.GetTempPath(), $"openjibo-stt-test-{Guid.NewGuid():N}");
Directory.CreateDirectory(tempDirectory);
try
{
var runner = new FakeExternalProcessRunner("[00:00:00.000 --> 00:00:01.000] yes.");
var strategy = new LocalWhisperCppBufferedAudioSttStrategy(
new BufferedAudioSttOptions
{
EnableLocalWhisperCpp = true,
FfmpegPath = "ffmpeg",
WhisperCliPath = "whisper-cli",
WhisperModelPath = "model.bin",
TempDirectory = tempDirectory
},
runner);
var turn = new TurnContext
{
TurnId = listenRule == "shared/yes_no"
? "turn-short-yes-no"
: "turn-short-word-of-the-day",
Locale = "en-US",
Attributes = new Dictionary<string, object?>
{
["bufferedAudioBytes"] = 47,
["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() },
["listenRules"] = new[] { listenRule }
}
};
var result = await strategy.TranscribeAsync(turn);
Assert.Equal("yes", result.Text);
Assert.Equal("local-whispercpp-buffered-audio", result.Provider);
Assert.Equal(2, runner.Calls.Count);
}
finally
{
if (Directory.Exists(tempDirectory)) Directory.Delete(tempDirectory, true);
}
}
[Fact]
public async Task TranscribeAsync_NormalizesLoosePunctuationFromWhisperOutput()
{
@@ -275,4 +351,4 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests
return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty));
}
}
}
}