Jibo-Revival-Group/JiboExperiments
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added some logging to see why processing audio is not working

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-17 23:57:52 -05:00
parent b030d6faeb
commit bd394ecbcd
21 changed files with 232 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Jibo.Cloud.Api.csproj
View File

@@ -1,15 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Infrastructure\Jibo.Cloud.Infrastructure.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Infrastructure\Jibo.Cloud.Infrastructure.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
</Project>

27
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Program.cs
View File

@@ -43,6 +43,7 @@ app.Use(async (context, next) =>
var webSocketService = context.RequestServices.GetRequiredService<JiboWebSocketService>();
var telemetrySink = context.RequestServices.GetRequiredService<IWebSocketTelemetrySink>();
using var socket = await context.WebSockets.AcceptWebSocketAsync();
var openEnvelope = new WebSocketMessageEnvelope
@@ -56,15 +57,29 @@ app.Use(async (context, next) =>
var openSession = ResolveSession(webSocketService, openEnvelope);
await telemetrySink.RecordConnectionOpenedAsync(openEnvelope, openSession, context.RequestAborted);
var isPrematureClose = false;
while (socket.State == WebSocketState.Open)
{
var received = await ReceiveAsync(socket, context.RequestAborted);
if (received.MessageType == WebSocketMessageType.Close)
ReceivedSocketMessage received = null!;
try
{
await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", context.RequestAborted);
break;
received = await ReceiveAsync(socket, context.RequestAborted);
if (received.MessageType == WebSocketMessageType.Close)
{
await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "bye", context.RequestAborted);
break;
}
}
catch (WebSocketException exception)
{
if (exception.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely)
{
isPrematureClose = true;
break;
}
}
var envelope = new WebSocketMessageEnvelope
{
ConnectionId = Guid.NewGuid().ToString("N"),
@@ -107,7 +122,7 @@ app.Use(async (context, next) =>
Token = token
};
var closeSession = ResolveSession(webSocketService, closeEnvelope);
await telemetrySink.RecordConnectionClosedAsync(closeEnvelope, closeSession, "socket-loop-ended", context.RequestAborted);
await telemetrySink.RecordConnectionClosedAsync(closeEnvelope, closeSession, $"socket-loop-ended{(isPrematureClose ? "-prematurely" : string.Empty)}", context.RequestAborted);
});
app.MapGet("/health", () => Results.Json(new { ok = true, service = "OpenJibo Cloud Api" }));

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.json
View File

@@ -9,6 +9,10 @@
"Enabled": true,
"DirectoryPath": "captures/http"
},
"TurnTelemetry": {
"Enabled": true,
"DirectoryPath": "captures/turn"
},
"Stt": {
"EnableLocalWhisperCpp": true,
"FfmpegPath": "/usr/bin/ffmpeg",

6
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/ITurnTelemetrySink.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace Jibo.Cloud.Application.Abstractions;
public interface ITurnTelemetrySink
{
Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default);
}

2
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IWebSocketTelemetrySink.cs
View File

@@ -9,4 +9,4 @@ public interface IWebSocketTelemetrySink
Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default);
Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default);
Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason, CancellationToken cancellationToken = default);
}
}

18
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Jibo.Cloud.Application.csproj
View File

@@ -1,14 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<ProjectReference Include="..\..\..\..\Jibo.Runtime.Abstractions\Jibo.Runtime.Abstractions.csproj" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

8
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/NullTurnTelemetrySink.cs Normal file
View File

@@ -0,0 +1,8 @@
using Jibo.Cloud.Application.Abstractions;
namespace Jibo.Cloud.Application.Services;
public sealed class NullTurnTelemetrySink : ITurnTelemetrySink
{
public Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default) => Task.CompletedTask;
}

2
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/NullWebSocketTelemetrySink.cs
View File

@@ -10,4 +10,4 @@ public sealed class NullWebSocketTelemetrySink : IWebSocketTelemetrySink
public Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType, IReadOnlyDictionary<string, object?> details, CancellationToken cancellationToken = default) => Task.CompletedTask;
public Task RecordOutboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, IReadOnlyList<WebSocketReply> replies, CancellationToken cancellationToken = default) => Task.CompletedTask;
public Task RecordConnectionClosedAsync(WebSocketMessageEnvelope envelope, CloudSession session, string reason, CancellationToken cancellationToken = default) => Task.CompletedTask;
}
}

8
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -1,4 +1,5 @@
using System.Text.Json;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
using Jibo.Runtime.Abstractions;
@@ -8,7 +9,9 @@ public sealed class WebSocketTurnFinalizationService(
ProtocolToTurnContextMapper turnContextMapper,
IConversationBroker conversationBroker,
ResponsePlanToSocketMessagesMapper replyMapper,
ISttStrategySelector sttStrategySelector)
ISttStrategySelector sttStrategySelector,
ITurnTelemetrySink sink
)
{
private const int AutoFinalizeMinBufferedAudioBytes = 12000;
private const int AutoFinalizeMinBufferedAudioChunks = 5;
@@ -155,8 +158,9 @@ public sealed class WebSocketTurnFinalizationService(
Attributes = attributes
};
}
catch
catch (Exception ex)
{
await sink.RecordTranscriptError(ex, "Error during STT processing", cancellationToken);
return turn;
}
}

10
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Domain/Jibo.Cloud.Domain.csproj
View File

@@ -1,9 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

2
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs
View File

@@ -19,6 +19,7 @@ public static class ServiceCollectionExtensions
{
services.Configure<WebSocketTelemetryOptions>(configuration.GetSection("OpenJibo:Telemetry"));
services.Configure<ProtocolTelemetryOptions>(configuration.GetSection("OpenJibo:ProtocolTelemetry"));
services.Configure<TurnTelemetryOptions>(configuration.GetSection("OpenJibo:TurnTelemetry"));
configuration.GetSection("OpenJibo:Stt").Bind(sttOptions);
}
@@ -35,6 +36,7 @@ public static class ServiceCollectionExtensions
services.AddSingleton<ISttStrategySelector, DefaultSttStrategySelector>();
services.AddSingleton<IWebSocketTelemetrySink, FileWebSocketTelemetrySink>();
services.AddSingleton<IProtocolTelemetrySink, FileProtocolTelemetrySink>();
services.AddSingleton<ITurnTelemetrySink, FileTurnTelemetrySink>();
services.AddSingleton<ProtocolToTurnContextMapper>();
services.AddSingleton<ResponsePlanToSocketMessagesMapper>();
services.AddSingleton<WebSocketTurnFinalizationService>();

20
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Jibo.Cloud.Infrastructure.csproj
View File

@@ -1,15 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Jibo.Cloud.Application\Jibo.Cloud.Application.csproj" />
<ProjectReference Include="..\Jibo.Cloud.Domain\Jibo.Cloud.Domain.csproj" />
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

55
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileTurnTelemetrySink.cs Normal file
View File

@@ -0,0 +1,55 @@
using System.Text.Json;
using Jibo.Cloud.Application.Abstractions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Jibo.Cloud.Infrastructure.Telemetry;
public sealed class FileTurnTelemetrySink(ILogger<FileTurnTelemetrySink> logger,
IOptions<TurnTelemetryOptions> options) : ITurnTelemetrySink
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = true
};
private readonly SemaphoreSlim _writeLock = new(1, 1);
public async Task RecordTranscriptError(Exception ex, string message, CancellationToken cancellationToken = default)
{
if (!options.Value.Enabled)
{
return;
}
await WriteErrorAsync(ex, message, cancellationToken);
}
private async Task WriteErrorAsync(Exception ex, string message, CancellationToken cancellationToken)
{
var directory = GetBaseDirectory();
Directory.CreateDirectory(directory);
var filePath = Path.Combine(directory, $"{DateTimeOffset.UtcNow:yyyyMMdd}.events.ndjson");
var line = JsonSerializer.Serialize(new { Exception = ex.ToString(), Message = message }, JsonOptions) + Environment.NewLine;
await _writeLock.WaitAsync(cancellationToken);
try
{
await File.AppendAllTextAsync(filePath, line, cancellationToken);
}
finally
{
_writeLock.Release();
}
logger.LogError("Turn telemetry Message={Message} Exception={Exception}", message, ex);
}
private string GetBaseDirectory()
{
return CapturePathResolver.Resolve(
options.Value.DirectoryPath,
Directory.GetCurrentDirectory(),
AppContext.BaseDirectory);
}
}

7
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/TurnTelemetryOptions.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace Jibo.Cloud.Infrastructure.Telemetry;
public sealed class TurnTelemetryOptions
{
public bool Enabled { get; set; } = true;
public string DirectoryPath { get; set; } = "captures/turn";
}

2
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/WebSocketTelemetryOptions.cs
View File

@@ -5,4 +5,4 @@ public sealed class WebSocketTelemetryOptions
public bool Enabled { get; set; } = true;
public bool ExportFixtures { get; set; } = true;
public string DirectoryPath { get; set; } = "captures/websocket";
}
}

10
OpenJibo/src/Jibo.Runtime.Abstractions/Jibo.Runtime.Abstractions.csproj
View File

@@ -1,9 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

12
OpenJibo/src/Playground/Playground.csproj
View File

@@ -1,10 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>