added some logging to see why processing audio is not working

This commit is contained in:
Jacob Dubin
2026-04-17 23:57:52 -05:00
parent b030d6faeb
commit bd394ecbcd
21 changed files with 232 additions and 94 deletions

View File

@@ -1,15 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Infrastructure\Jibo.Cloud.Infrastructure.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Infrastructure\Jibo.Cloud.Infrastructure.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
</Project>

View File

@@ -43,6 +43,7 @@ app.Use(async (context, next) =>
var webSocketService = context.RequestServices.GetRequiredService<JiboWebSocketService>();
var telemetrySink = context.RequestServices.GetRequiredService<IWebSocketTelemetrySink>();
using var socket = await context.WebSockets.AcceptWebSocketAsync();
var openEnvelope = new WebSocketMessageEnvelope
@@ -56,15 +57,29 @@ app.Use(async (context, next) =>
var openSession = ResolveSession(webSocketService, openEnvelope);
await telemetrySink.RecordConnectionOpenedAsync(openEnvelope, openSession, context.RequestAborted);
var isPrematureClose = false;
while (socket.State == WebSocketState.Open)
{
var received = await ReceiveAsync(socket, context.RequestAborted);
if (received.MessageType == WebSocketMessageType.Close)
ReceivedSocketMessage received = null!;
try
{
await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", context.RequestAborted);
break;
received = await ReceiveAsync(socket, context.RequestAborted);
if (received.MessageType == WebSocketMessageType.Close)
{
await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", context.RequestAborted);
break;
}
}
catch (WebSocketException exception)
{
if (exception.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely)
{
isPrematureClose = true;
break;
}
}
var envelope = new WebSocketMessageEnvelope
{
ConnectionId = Guid.NewGuid().ToString("N"),
@@ -107,7 +122,7 @@ app.Use(async (context, next) =>
Token = token
};
var closeSession = ResolveSession(webSocketService, closeEnvelope);
await telemetrySink.RecordConnectionClosedAsync(closeEnvelope, closeSession, "socket-loop-ended", context.RequestAborted);
await telemetrySink.RecordConnectionClosedAsync(closeEnvelope, closeSession, $"socket-loop-ended{(isPrematureClose ? "-prematurely" : string.Empty)}", context.RequestAborted);
});
app.MapGet("/health", () => Results.Json(new { ok = true, service = "OpenJibo Cloud Api" }));

View File

@@ -9,6 +9,10 @@
"Enabled": true,
"DirectoryPath": "captures/http"
},
"TurnTelemetry": {
"Enabled": true,
"DirectoryPath": "captures/turn"
},
"Stt": {
"EnableLocalWhisperCpp": true,
"FfmpegPath": "/usr/bin/ffmpeg",

View File

@@ -0,0 +1,6 @@
namespace Jibo.Cloud.Application.Abstractions;
public interface ITurnTelemetrySink
{
Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default);
}

View File

@@ -9,4 +9,4 @@ public interface IWebSocketTelemetrySink
Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default);
Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default);
Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason, CancellationToken cancellationToken = default);
}
}

View File

@@ -1,14 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View File

@@ -0,0 +1,8 @@
using Jibo.Cloud.Application.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed class NullTurnTelemetrySink : ITurnTelemetrySink
{
public Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default) => Task.CompletedTask;
}

View File

@@ -10,4 +10,4 @@ public sealed class NullWebSocketTelemetrySink : IWebSocketTelemetrySink
public Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default) => Task.CompletedTask;
public Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default) => Task.CompletedTask;
public Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason, CancellationToken cancellationToken = default) => Task.CompletedTask;
}
}

View File

@@ -1,4 +1,5 @@
using System.Text.Json;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
@@ -8,7 +9,9 @@ public sealed class WebSocketTurnFinalizationService(
ProtocolToTurnContextMapper turnContextMapper,
IConversationBroker conversationBroker,
ResponsePlanToSocketMessagesMapper replyMapper,
ISttStrategySelector sttStrategySelector)
ISttStrategySelector sttStrategySelector,
ITurnTelemetrySink sink
)
{
private const int AutoFinalizeMinBufferedAudioBytes = 12000;
private const int AutoFinalizeMinBufferedAudioChunks = 5;
@@ -155,8 +158,9 @@ public sealed class WebSocketTurnFinalizationService(
Attributes = attributes
};
}
catch
catch (Exception ex)
{
await sink.RecordTranscriptError(ex, "Error during STT processing", cancellationToken);
return turn;
}
}

View File

@@ -1,9 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View File

@@ -19,6 +19,7 @@ public static class ServiceCollectionExtensions
{
services.Configure<WebSocketTelemetryOptions>(configuration.GetSection("OpenJibo:Telemetry"));
services.Configure<ProtocolTelemetryOptions>(configuration.GetSection("OpenJibo:ProtocolTelemetry"));
services.Configure<TurnTelemetryOptions>(configuration.GetSection("OpenJibo:TurnTelemetry"));
configuration.GetSection("OpenJibo:Stt").Bind(sttOptions);
}
@@ -35,6 +36,7 @@ public static class ServiceCollectionExtensions
services.AddSingleton<ISttStrategySelector, DefaultSttStrategySelector>();
services.AddSingleton<IWebSocketTelemetrySink, FileWebSocketTelemetrySink>();
services.AddSingleton<IProtocolTelemetrySink, FileProtocolTelemetrySink>();
services.AddSingleton<ITurnTelemetrySink, FileTurnTelemetrySink>();
services.AddSingleton<ProtocolToTurnContextMapper>();
services.AddSingleton<ResponsePlanToSocketMessagesMapper>();
services.AddSingleton<WebSocketTurnFinalizationService>();

View File

@@ -1,15 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View File

@@ -0,0 +1,55 @@
using System.Text.Json;
using Jibo.Cloud.Application.Abstractions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Jibo.Cloud.Infrastructure.Telemetry;
public sealed class FileTurnTelemetrySink(ILogger<FileTurnTelemetrySink> logger,
IOptions<TurnTelemetryOptions> options) : ITurnTelemetrySink
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = true
};
private readonly SemaphoreSlim _writeLock = new(1, 1);
public async Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default)
{
if (!options.Value.Enabled)
{
return;
}
await WriteErrorAsync(ex, message, cancellationToken);
}
private async Task WriteErrorAsync(Exception ex, string message, CancellationToken cancellationToken)
{
var directory = GetBaseDirectory();
Directory.CreateDirectory(directory);
var filePath = Path.Combine(directory, $"{DateTimeOffset.UtcNow:yyyyMMdd}.events.ndjson");
var line = JsonSerializer.Serialize(new { Exception = ex.ToString(), Message = message }, JsonOptions) + Environment.NewLine;
await _writeLock.WaitAsync(cancellationToken);
try
{
await File.AppendAllTextAsync(filePath, line, cancellationToken);
}
finally
{
_writeLock.Release();
}
logger.LogError("Turn telemetry Message={Message} Exception={Exception}", message, ex);
}
private string GetBaseDirectory()
{
return CapturePathResolver.Resolve(
options.Value.DirectoryPath,
Directory.GetCurrentDirectory(),
AppContext.BaseDirectory);
}
}

View File

@@ -0,0 +1,7 @@
namespace Jibo.Cloud.Infrastructure.Telemetry;
public sealed class TurnTelemetryOptions
{
public bool Enabled { get; set; } = true;
public string DirectoryPath { get; set; } = "captures/turn";
}

View File

@@ -5,4 +5,4 @@ public sealed class WebSocketTelemetryOptions
public bool Enabled { get; set; } = true;
public bool ExportFixtures { get; set; } = true;
public string DirectoryPath { get; set; } = "captures/websocket";
}
}

View File

@@ -1,9 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View File

@@ -1,10 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View File

@@ -1,33 +1,34 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
<PackageReference Include="Moq" Version="4.20.72" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\..\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Infrastructure\Jibo.Cloud.Infrastructure.csproj" />
<ProjectReference Include="..\..\src\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\..\src\Jibo.Cloud\dotnet\src\Jibo.Cloud.Infrastructure\Jibo.Cloud.Infrastructure.csproj" />
<ProjectReference Include="..\..\src\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="..\..\src\Jibo.Cloud\node\fixtures\http\*.json">
<Link>fixtures\%(Filename)%(Extension)</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="..\..\src\Jibo.Cloud\node\fixtures\websocket\*.json">
<Link>fixtures\%(Filename)%(Extension)</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<None Include="..\..\src\Jibo.Cloud\node\fixtures\http\*.json">
<Link>fixtures\%(Filename)%(Extension)</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="..\..\src\Jibo.Cloud\node\fixtures\websocket\*.json">
<Link>fixtures\%(Filename)%(Extension)</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,32 @@
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Application.Services;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
using Moq;
namespace Jibo.Cloud.Tests.Turn;
public sealed class FileTurnTelemetrySinkTests
{
[Fact]
public async Task RecordsTranscriptErrorOnTurnError()
{
var sink = new Mock<ITurnTelemetrySink>();
var sttStrategySelector = new Mock<ISttStrategySelector>();
sttStrategySelector.Setup(s => s.SelectAsync(It.IsAny<TurnContext>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Exception("dummy"));
var turnService = new WebSocketTurnFinalizationService(
new ProtocolToTurnContextMapper(),
Mock.Of<IConversationBroker>(),
new ResponsePlanToSocketMessagesMapper(),
sttStrategySelector.Object,
sink.Object
);
await turnService.HandleTurnAsync(new CloudSession() { TurnState = { BufferedAudioBytes = 100 }}, new WebSocketMessageEnvelope(), "dummy",
CancellationToken.None);
sink.Verify(s => s.RecordTranscriptError(It.IsAny<Exception>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once());
}
}

View File

@@ -36,9 +36,12 @@ public sealed class FileWebSocketTelemetrySinkTests : IDisposable
{
Token = "token-1",
HostName = "neo-hub.jibo.com",
Path = "/listen"
Path = "/listen",
TurnState =
{
TransId = "trans-1"
}
};
session.TurnState.TransId = "trans-1";
await sink.RecordConnectionOpenedAsync(envelope, session);
await sink.RecordInboundAsync(envelope, session, "LISTEN");
@@ -70,7 +73,7 @@ public sealed class FileWebSocketTelemetrySinkTests : IDisposable
{
Directory.CreateDirectory(_repoRoot);
Directory.CreateDirectory(_appBaseDirectory);
File.WriteAllText(Path.Combine(_repoRoot, "OpenJibo.slnx"), string.Empty);
await File.WriteAllTextAsync(Path.Combine(_repoRoot, "OpenJibo.slnx"), string.Empty);
var captureDirectory = CapturePathResolver.Resolve("captures/websocket", _repoRoot, _appBaseDirectory);
var sink = new FileWebSocketTelemetrySink(
@@ -118,4 +121,4 @@ public sealed class FileWebSocketTelemetrySinkTests : IDisposable
DirectoryPath = _directoryPath
}));
}
}
}

View File

@@ -1,5 +1,4 @@
using System.Text.Json;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Application.Services;
using Jibo.Cloud.Domain.Models;
using Jibo.Cloud.Infrastructure.Content;
@@ -25,6 +24,7 @@ public sealed class JiboWebSocketServiceTests
[
new SyntheticBufferedAudioSttStrategy()
]);
var sink = new NullTurnTelemetrySink();
_service = new JiboWebSocketService(
_store,
@@ -33,7 +33,8 @@ public sealed class JiboWebSocketServiceTests
turnContextMapper,
conversationBroker,
replyMapper,
sttSelector));
sttSelector,
sink));
}
[Fact]
@@ -118,7 +119,7 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CONTEXT","transID":"trans-auto","data":{"audioTranscriptHint":"tell me a joke"}}"""
});
IReadOnlyList<WebSocketReply> replies = [];
IReadOnlyList<WebSocketReply> replies;
for (var index = 0; index < 4; index += 1)
{
replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
@@ -175,7 +176,7 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CONTEXT","transID":"trans-auto-fallback","data":{"topic":"conversation"}}"""
});
IReadOnlyList<WebSocketReply> replies = [];
IReadOnlyList<WebSocketReply> replies;
for (var index = 0; index < 4; index += 1)
{
replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
@@ -249,7 +250,7 @@ public sealed class JiboWebSocketServiceTests
var session = _store.FindSessionByToken("hub-multichunk-token");
Assert.NotNull(session);
Assert.Equal(7, session!.TurnState.BufferedAudioBytes);
Assert.Equal(7, session.TurnState.BufferedAudioBytes);
Assert.Equal(2, session.TurnState.BufferedAudioChunkCount);
}
@@ -292,7 +293,7 @@ public sealed class JiboWebSocketServiceTests
var session = _store.FindSessionByToken("hub-follow-up-token");
Assert.NotNull(session);
Assert.True(session!.FollowUpOpen);
Assert.True(session.FollowUpOpen);
Assert.Equal("joke", session.LastIntent);
Assert.Equal("trans-follow-up", session.LastTransId);
}
@@ -393,7 +394,7 @@ public sealed class JiboWebSocketServiceTests
var session = _store.FindSessionByToken("hub-audio-token");
Assert.NotNull(session);
Assert.Equal(0, session!.TurnState.BufferedAudioBytes);
Assert.Equal(0, session.TurnState.BufferedAudioBytes);
Assert.Equal(0, session.TurnState.BufferedAudioChunkCount);
Assert.False(session.Metadata.ContainsKey("audioTranscriptHint"));
}
@@ -632,7 +633,7 @@ public sealed class JiboWebSocketServiceTests
var session = _store.FindSessionByToken("hub-followup-audio-token");
Assert.NotNull(session);
Assert.Equal("trans-second", session!.TurnState.TransId);
Assert.Equal("trans-second", session.TurnState.TransId);
Assert.Equal(0, session.TurnState.BufferedAudioBytes);
Assert.Equal(0, session.TurnState.BufferedAudioChunkCount);
}