From eb509a66e090783434cab0ccc700d8fc564d8b66 Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Thu, 21 May 2026 06:09:27 -0500 Subject: [PATCH] Polish weather news and STT filtering --- OpenJibo/docs/feature-backlog.md | 9 ++- OpenJibo/docs/release-1.0.19-plan.md | 2 + .../Services/JiboInteractionService.cs | 3 + ...LocalWhisperCppBufferedAudioSttStrategy.cs | 16 +++- .../News/NewsApiBriefingProvider.cs | 24 ++++-- .../Infrastructure/ProviderCachingTests.cs | 42 ++++++++++- .../WebSockets/JiboInteractionServiceTests.cs | 2 + ...WhisperCppBufferedAudioSttStrategyTests.cs | 74 ++++++++++++++++++- 8 files changed, 157 insertions(+), 15 deletions(-) diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index 16df88e..8f75e09 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -436,7 +436,7 @@ Current release theme: ### 9. STT Upgrade And Noise Screening -- Status: `ready` +- Status: `in progress` - Tags: `stt` - Why next: - feature paths are now often correct when a transcript exists, but short replies and low-quality audio still block otherwise-correct flows @@ -448,6 +448,10 @@ Current release theme: - `jibo test 26` had long no-`LISTEN` binary buffering and alarm-delete mishears now patched; remaining short-answer failures still need STT/noise work - current source now skips local whisper when buffered audio does not contain an Opus identification header - yes/no and alarm flows are especially sensitive to short or collapsed transcripts +- Progress update (`2026-05-21`): + - added a small local whisper noise floor so obviously tiny buffered audio can be screened before ffmpeg/whisper work runs + - short/noisy buffered turns now fail fast instead of wasting a transcription cycle + - focused tests now cover the new low-audio rejection behavior - Implementation notes: - add lightweight waveform or energy screening before transcription - compare managed STT against the local toolchain @@ -741,6 +745,9 @@ Current release theme: - added TTL caching for weather/news provider calls to reduce repeated external requests - vendored Pegasus `report-skill` templates for weather and personal-report phrasing so the next pass can focus on renderer coverage for calendar, commute, and news templates instead of rediscovering source text - commute now has a loop-scoped provider seam plus persisted commute profiles, so the next pass can focus on richer travel-time data instead of basic storage shape +- Progress update (`2026-05-21`): + - weather payloads now distinguish current-vs-weekly view modes so renderer parity can key off the payload shape more cleanly + - news provider now skips summaryless correction headlines before falling back to broader sources - Source anchors: - `C:\Projects\jibo\pegasus\packages\report-skill\src\subskills\weather\WeatherMimLogic.ts` - `C:\Projects\jibo\pegasus\packages\report-skill\resources\views\weatherHiLo.json` diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index 63787d1..67df4c7 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -351,6 +351,8 @@ First completed slice in this personal-report parity track: - added provider-side request caching for both news and weather to reduce integration churn and repeated lookups - added focused interaction + websocket tests for provider-backed news speech output and request-hint plumbing - added loop-scoped calendar and commute provider seams so personal report can use persisted household context instead of static placeholders +- weather payloads now distinguish current vs weekly view modes so renderer parity can key off the payload shape +- news provider now skips summaryless correction headlines before falling back to broader sources ## Next Slices diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs index c6c6ff3..d241453 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs @@ -1969,6 +1969,8 @@ public sealed class JiboInteractionService( DateTimeOffset? referenceLocalTime) { var payload = BuildWeatherSkillPayload(spokenReply, snapshot, referenceLocalTime); + payload["weather_view_kind"] = "weatherWeekly"; + payload["weather_view_mode"] = "forecast"; payload["weather_weekly_cards"] = segments .Select(static segment => new Dictionary(StringComparer.OrdinalIgnoreCase) { @@ -2118,6 +2120,7 @@ public sealed class JiboInteractionService( ["prompt_sub_category"] = "AN", ["weather_view_enabled"] = true, ["weather_view_kind"] = "weatherHiLo", + ["weather_view_mode"] = "current", ["weather_icon"] = weatherIcon, ["weather_summary"] = snapshot.Summary, ["weather_location"] = snapshot.LocationName, diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs index 05f4d58..0f09ea9 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs @@ -7,6 +7,8 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( BufferedAudioSttOptions options, IExternalProcessRunner processRunner) : ISttStrategy { + private const int MinimumBufferedAudioBytes = 64; + public string Name => "local-whispercpp-buffered-audio"; public bool CanHandle(TurnContext turn) @@ -15,7 +17,8 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( IsConfiguredPathAvailable(options.FfmpegPath, false) && IsConfiguredPathAvailable(options.WhisperCliPath, true) && IsConfiguredPathAvailable(options.WhisperModelPath, true) && - ReadBufferedAudioFrames(turn).Any(ContainsOpusIdentificationHeader); + ReadBufferedAudioFrames(turn).Any(ContainsOpusIdentificationHeader) && + !IsBelowNoiseFloor(ReadBufferedAudioBytes(turn)); } public async Task TranscribeAsync(TurnContext turn, CancellationToken cancellationToken = default) @@ -28,6 +31,10 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( throw new InvalidOperationException( "Local whisper.cpp STT requires buffered Ogg/Opus audio with an Opus identification header."); + if (IsBelowNoiseFloor(ReadBufferedAudioBytes(turn))) + throw new InvalidOperationException( + "Local whisper.cpp STT rejected buffered audio as too short or noisy for transcription."); + var tempDirectory = options.TempDirectory; if (string.IsNullOrWhiteSpace(tempDirectory)) tempDirectory = Path.Combine(Path.GetTempPath(), "openjibo-stt"); @@ -112,6 +119,11 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( : 0; } + private static bool IsBelowNoiseFloor(int bufferedAudioBytes) + { + return bufferedAudioBytes > 0 && bufferedAudioBytes < MinimumBufferedAudioBytes; + } + private static bool ContainsOpusIdentificationHeader(byte[] frame) { return frame.AsSpan().IndexOf("OpusHead"u8) >= 0; @@ -155,4 +167,4 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( return !checkFileExists || File.Exists(path); } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/News/NewsApiBriefingProvider.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/News/NewsApiBriefingProvider.cs index d650ddd..f71a098 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/News/NewsApiBriefingProvider.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/News/NewsApiBriefingProvider.cs @@ -131,9 +131,9 @@ public sealed class NewsApiBriefingProvider( foreach (var article in articles.EnumerateArray()) { var title = NormalizeHeadlineTitle(ReadString(article, "title")); - if (string.IsNullOrWhiteSpace(title) || !seenTitles.Add(title)) continue; - var summary = ReadString(article, "description"); + if (title is null || !IsUsableHeadline(title, summary) || !seenTitles.Add(title)) continue; + var source = article.TryGetProperty("source", out var sourceNode) && sourceNode.ValueKind == JsonValueKind.Object ? ReadString(sourceNode, "name") @@ -176,9 +176,9 @@ public sealed class NewsApiBriefingProvider( foreach (var article in broadArticles.EnumerateArray()) { var title = NormalizeHeadlineTitle(ReadString(article, "title")); - if (string.IsNullOrWhiteSpace(title) || !seenTitles.Add(title)) continue; - var summary = ReadString(article, "description"); + if (title is null || !IsUsableHeadline(title, summary) || !seenTitles.Add(title)) continue; + var source = article.TryGetProperty("source", out var sourceNode) && sourceNode.ValueKind == JsonValueKind.Object ? ReadString(sourceNode, "name") @@ -239,9 +239,9 @@ public sealed class NewsApiBriefingProvider( foreach (var article in everythingArticles.EnumerateArray()) { var title = NormalizeHeadlineTitle(ReadString(article, "title")); - if (string.IsNullOrWhiteSpace(title) || !seenTitles.Add(title)) continue; - var summary = ReadString(article, "description"); + if (title is null || !IsUsableHeadline(title, summary) || !seenTitles.Add(title)) continue; + var source = article.TryGetProperty("source", out var sourceNode) && sourceNode.ValueKind == JsonValueKind.Object ? ReadString(sourceNode, "name") @@ -453,6 +453,16 @@ public sealed class NewsApiBriefingProvider( return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed; } + private static bool IsUsableHeadline(string? title, string? summary) + { + if (string.IsNullOrWhiteSpace(title) || string.IsNullOrWhiteSpace(summary)) return false; + + var loweredTitle = title.Trim().ToLowerInvariant(); + return !loweredTitle.StartsWith("correction:", StringComparison.Ordinal) && + !loweredTitle.StartsWith("corrected:", StringComparison.Ordinal) && + !loweredTitle.Contains(" correction", StringComparison.Ordinal); + } + private static ApiError? TryParseApiError(string? responseBody) { if (string.IsNullOrWhiteSpace(responseBody)) return null; @@ -529,4 +539,4 @@ public sealed class NewsApiBriefingProvider( private sealed record ApiError(string? Code, string? Message); private sealed record CacheEntry(T Value, DateTimeOffset ExpiresUtc); -} \ No newline at end of file +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Infrastructure/ProviderCachingTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Infrastructure/ProviderCachingTests.cs index ee960d7..2035ae9 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/Infrastructure/ProviderCachingTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Infrastructure/ProviderCachingTests.cs @@ -284,6 +284,46 @@ public sealed class ProviderCachingTests Assert.Equal(2, handler.GetCallCount("/v2/top-headlines")); } + [Fact] + public async Task NewsApiBriefingProvider_SkipsCorrectionAndSummarylessHeadlines_BeforeFallback() + { + var handler = new CountingHttpMessageHandler(message => + { + var path = message.RequestUri?.AbsolutePath ?? string.Empty; + if (!string.Equals(path, "/v2/top-headlines", StringComparison.OrdinalIgnoreCase)) + return new HttpResponseMessage(HttpStatusCode.NotFound); + + var query = message.RequestUri?.Query ?? string.Empty; + return JsonResponse(query.Contains("category=sports", StringComparison.OrdinalIgnoreCase) + ? """ + {"status":"ok","articles":[ + {"title":"Correction: robots everywhere","description":"","source":{"name":"AP News"},"url":"https://example.com/correction"} + ]} + """ + : """ + {"status":"ok","articles":[ + {"title":"General robotics update","description":"Top story","source":{"name":"AP News"},"url":"https://example.com/general"} + ]} + """); + }); + var provider = new NewsApiBriefingProvider( + new HttpClient(handler), + new NewsApiOptions + { + ApiKey = "test-key", + CacheTtlSeconds = 300, + FailureCacheTtlSeconds = 30 + }, + NullLogger.Instance); + + var result = await provider.GetBriefingAsync(new NewsBriefingRequest(["sports"])); + + Assert.NotNull(result); + Assert.Single(result!.Headlines); + Assert.Equal("General robotics update", result.Headlines[0].Title); + Assert.Equal(2, handler.GetCallCount("/v2/top-headlines")); + } + [Fact] public async Task NewsApiBriefingProvider_FallsBackToEverything_WhenTopHeadlinesAreEmpty() { @@ -457,4 +497,4 @@ public sealed class ProviderCachingTests return Task.FromResult(responseFactory(request)); } } -} \ No newline at end of file +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index 7261c1c..437c684 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -2770,6 +2770,8 @@ public sealed class JiboInteractionServiceTests Assert.Contains("Temperatures are in Fahrenheit.", decision.ReplyText, StringComparison.OrdinalIgnoreCase); Assert.NotNull(decision.SkillPayload); Assert.True(decision.SkillPayload!.TryGetValue("weather_weekly_cards", out var weeklyCardsValue)); + Assert.Equal("weatherWeekly", decision.SkillPayload["weather_view_kind"]); + Assert.Equal("forecast", decision.SkillPayload["weather_view_mode"]); var weeklyCards = Assert.IsAssignableFrom>>(weeklyCardsValue); Assert.Equal(5, weeklyCards.Count); var firstCard = weeklyCards[0]; diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs index 2aff883..b5f11e7 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs @@ -77,6 +77,31 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests Assert.False(strategy.CanHandle(turn)); } + [Fact] + public void CanHandle_ReturnsFalse_WhenBufferedAudioIsBelowNoiseFloor() + { + var strategy = new LocalWhisperCppBufferedAudioSttStrategy( + new BufferedAudioSttOptions + { + EnableLocalWhisperCpp = true, + FfmpegPath = "ffmpeg", + WhisperCliPath = "whisper-cli", + WhisperModelPath = "model.bin" + }, + new FakeExternalProcessRunner()); + + var turn = new TurnContext + { + Attributes = new Dictionary + { + ["bufferedAudioBytes"] = 47, + ["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() } + } + }; + + Assert.False(strategy.CanHandle(turn)); + } + [Fact] public async Task TranscribeAsync_UsesFfmpegAndWhisperCpp_WhenConfigured() { @@ -103,7 +128,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests Locale = "en-US", Attributes = new Dictionary { - ["bufferedAudioBytes"] = 47, + ["bufferedAudioBytes"] = 147, ["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() } } }; @@ -115,7 +140,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests Assert.Equal(2, runner.Calls.Count); Assert.Equal("ffmpeg", runner.Calls[0].FileName); Assert.Equal("whisper-cli", runner.Calls[1].FileName); - Assert.Equal(47, result.Metadata["bufferedAudioBytes"]); + Assert.Equal(147, result.Metadata["bufferedAudioBytes"]); } finally { @@ -149,7 +174,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests Locale = "en-US", Attributes = new Dictionary { - ["bufferedAudioBytes"] = 47, + ["bufferedAudioBytes"] = 147, ["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() } } }; @@ -165,6 +190,47 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests } } + [Fact] + public async Task TranscribeAsync_Throws_WhenBufferedAudioIsBelowNoiseFloor() + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"openjibo-stt-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDirectory); + + try + { + var runner = new FakeExternalProcessRunner(); + var strategy = new LocalWhisperCppBufferedAudioSttStrategy( + new BufferedAudioSttOptions + { + EnableLocalWhisperCpp = true, + FfmpegPath = "ffmpeg", + WhisperCliPath = "whisper-cli", + WhisperModelPath = "model.bin", + TempDirectory = tempDirectory + }, + runner); + + var turn = new TurnContext + { + TurnId = "turn-local-stt-noise-floor", + Locale = "en-US", + Attributes = new Dictionary + { + ["bufferedAudioBytes"] = 47, + ["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() } + } + }; + + var ex = await Assert.ThrowsAsync(() => strategy.TranscribeAsync(turn)); + Assert.Contains("too short or noisy", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Empty(runner.Calls); + } + finally + { + if (Directory.Exists(tempDirectory)) Directory.Delete(tempDirectory, true); + } + } + private static byte[] BuildMinimalOggPage() { return @@ -209,4 +275,4 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty)); } } -} \ No newline at end of file +}