From 14b5cb74ccbf8661f13d9f2828ae29c59fd9b1ee Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Sun, 17 May 2026 14:07:56 -0500 Subject: [PATCH] Add capture index manifest for group testing --- OpenJibo/docs/feature-backlog.md | 1 + OpenJibo/docs/live-jibo-capture.md | 7 ++- OpenJibo/docs/release-1.0.19-plan.md | 2 +- .../Telemetry/CaptureIndexWriter.cs | 48 ++++++++++++++++ .../Telemetry/FileProtocolTelemetrySink.cs | 19 ++++++- .../Telemetry/FileTurnTelemetrySink.cs | 19 +++++-- .../Telemetry/FileWebSocketTelemetrySink.cs | 57 +++++++++++++++++-- .../FileProtocolTelemetrySinkTests.cs | 7 ++- .../Turn/FileTurnTelemetrySinkTests.cs | 31 +++++++++- .../FileWebSocketTelemetrySinkTests.cs | 24 +++++++- 10 files changed, 198 insertions(+), 17 deletions(-) create mode 100644 OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/CaptureIndexWriter.cs diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index e3793e3..b93b5f7 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -461,6 +461,7 @@ Current release theme: - Implementation notes: - define local capture sinks versus hosted retention - decide how testers submit noteworthy sessions + - keep a lightweight `capture-index.ndjson` manifest beside raw captures so testers can quickly find sessions, operations, and fixture exports - preserve sanitized fixtures as the durable parity artifact ### 11. Binary-Safe Media Storage diff --git a/OpenJibo/docs/live-jibo-capture.md b/OpenJibo/docs/live-jibo-capture.md index f20e346..8e0758a 100644 --- a/OpenJibo/docs/live-jibo-capture.md +++ b/OpenJibo/docs/live-jibo-capture.md @@ -41,6 +41,7 @@ The `.NET` cloud now supports structured live capture intended for first robot r - HTTP request/response event streams written as NDJSON - websocket event streams written as NDJSON - per-session websocket fixture export for replay +- a small `capture-index.ndjson` manifest beside the raw files so group testers can quickly find the session type, operation, and export artifacts - turn metadata including `transID`, buffered audio counts, finalize attempts, and reply types Default capture location: @@ -54,6 +55,7 @@ Artifacts: - `websocket/*.events.ndjson` - `*.events.ndjson` - `websocket/fixtures/*.flow.json` +- `capture-index.ndjson` ## Suggested First Hookup Plan @@ -61,8 +63,9 @@ Artifacts: 2. Confirm HTTP bootstrap and websocket acceptance with the existing smoke/routing helpers. 3. Run one or two controlled listen turns with Jibo. 4. Inspect the captured HTTP and websocket events plus exported websocket fixtures. -5. Convert the best captures into sanitized checked-in fixtures and tests. -6. Keep Node available to compare any surprising turn behavior before changing infrastructure. +5. Use `capture-index.ndjson` to quickly locate the important sessions and exported fixtures. +6. Convert the best captures into sanitized checked-in fixtures and tests. +7. Keep Node available to compare any surprising turn behavior before changing infrastructure. Useful helper scripts: diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index e49587e..d636e98 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -342,7 +342,7 @@ First completed slice in this personal-report parity track: 7. Update/backup/restore end-to-end proof - implemented 8. STT noise-screening and short-utterance reliability pass 9. Provider-backed news expansion and deeper weather parity -10. Capture indexing and retention boundary for group testing +10. Capture indexing and retention boundary for group testing, including a lightweight manifest beside raw capture files For slice 1, use the new import ladder above to keep the work grounded in what OpenJibo can already render today versus what needs new scaffolding. For slices 2-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements. diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/CaptureIndexWriter.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/CaptureIndexWriter.cs new file mode 100644 index 0000000..68f04d0 --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/CaptureIndexWriter.cs @@ -0,0 +1,48 @@ +using System.Collections.Concurrent; +using System.Text.Json; + +namespace Jibo.Cloud.Infrastructure.Telemetry; + +internal static class CaptureIndexWriter +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false + }; + + private static readonly ConcurrentDictionary DirectoryLocks = new(StringComparer.OrdinalIgnoreCase); + + public static async Task AppendAsync( + string directoryPath, + string sinkName, + string eventType, + IReadOnlyDictionary details, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(directoryPath)) return; + + var directory = Path.GetFullPath(directoryPath); + Directory.CreateDirectory(directory); + var indexPath = Path.Combine(directory, "capture-index.ndjson"); + var payload = new + { + capturedUtc = DateTimeOffset.UtcNow, + sink = sinkName, + eventType, + details + }; + + var line = JsonSerializer.Serialize(payload, JsonOptions) + Environment.NewLine; + var gate = DirectoryLocks.GetOrAdd(directory, static _ => new SemaphoreSlim(1, 1)); + + await gate.WaitAsync(cancellationToken); + try + { + await File.AppendAllTextAsync(indexPath, line, cancellationToken); + } + finally + { + gate.Release(); + } + } +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileProtocolTelemetrySink.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileProtocolTelemetrySink.cs index 43fbb71..7d12e91 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileProtocolTelemetrySink.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileProtocolTelemetrySink.cs @@ -64,6 +64,23 @@ public sealed class FileProtocolTelemetrySink( _writeLock.Release(); } + await CaptureIndexWriter.AppendAsync( + directory, + "http", + "protocol_record", + new Dictionary + { + ["method"] = envelope.Method, + ["host"] = envelope.HostName, + ["path"] = envelope.Path, + ["servicePrefix"] = envelope.ServicePrefix, + ["operation"] = envelope.Operation, + ["statusCode"] = result.StatusCode, + ["contentType"] = result.ContentType, + ["requestId"] = envelope.RequestId + }, + cancellationToken); + logger.LogInformation( "HTTP telemetry {Method} {Host}{Path} target={Target} status={StatusCode}", envelope.Method, @@ -72,4 +89,4 @@ public sealed class FileProtocolTelemetrySink( $"{envelope.ServicePrefix}.{envelope.Operation}".Trim('.'), result.StatusCode); } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileTurnTelemetrySink.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileTurnTelemetrySink.cs index f6560e9..df1b0b5 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileTurnTelemetrySink.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileTurnTelemetrySink.cs @@ -21,7 +21,7 @@ public sealed class FileTurnTelemetrySink( { if (!options.Value.Enabled) return; - await WriteEventAsync(new + await WriteEventAsync(category, new { Type = category, Details = details @@ -32,7 +32,7 @@ public sealed class FileTurnTelemetrySink( { if (!options.Value.Enabled) return; - await WriteEventAsync(new + await WriteEventAsync("transcript_error", new { Exception = ex.ToString(), Message = message, @@ -40,7 +40,7 @@ public sealed class FileTurnTelemetrySink( }, "Turn telemetry error", LogLevel.Error, cancellationToken); } - private async Task WriteEventAsync(object payload, string logMessage, LogLevel level, + private async Task WriteEventAsync(string eventType, object payload, string logMessage, LogLevel level, CancellationToken cancellationToken) { var directory = GetBaseDirectory(); @@ -58,6 +58,17 @@ public sealed class FileTurnTelemetrySink( _writeLock.Release(); } + await CaptureIndexWriter.AppendAsync( + directory, + "turn", + eventType, + new Dictionary + { + ["message"] = logMessage, + ["level"] = level.ToString() + }, + cancellationToken); + logger.Log(level, "{LogMessage} {Payload}", logMessage, payload); } @@ -68,4 +79,4 @@ public sealed class FileTurnTelemetrySink( Directory.GetCurrentDirectory(), AppContext.BaseDirectory); } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileWebSocketTelemetrySink.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileWebSocketTelemetrySink.cs index 44ab24a..16e0934 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileWebSocketTelemetrySink.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Telemetry/FileWebSocketTelemetrySink.cs @@ -39,15 +39,20 @@ public sealed class FileWebSocketTelemetrySink( await WriteRecordAsync(BuildRecord("connection_opened", envelope, session, null, "internal", null, null), cancellationToken); + await AppendIndexAsync(envelope, session, "connection_opened", null, cancellationToken); } - public Task RecordInboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, string? messageType, + public async Task RecordInboundAsync(WebSocketMessageEnvelope envelope, CloudSession session, string? messageType, CancellationToken cancellationToken = default) { - return !options.Value.Enabled - ? Task.CompletedTask - : WriteRecordAsync(BuildRecord("message_in", envelope, session, messageType, "in", null, null), - cancellationToken); + if (!options.Value.Enabled) return; + + await WriteRecordAsync(BuildRecord("message_in", envelope, session, messageType, "in", null, null), + cancellationToken); + await AppendIndexAsync(envelope, session, "message_in", new Dictionary + { + ["messageType"] = messageType + }, cancellationToken); } public Task RecordTurnEventAsync(WebSocketMessageEnvelope envelope, CloudSession session, string eventType, @@ -72,6 +77,10 @@ public sealed class FileWebSocketTelemetrySink( await WriteRecordAsync(BuildRecord("message_out", envelope, session, null, "out", replyTypes, null), cancellationToken); + await AppendIndexAsync(envelope, session, "message_out", new Dictionary + { + ["replyTypes"] = replyTypes + }, cancellationToken); if (_fixtures.TryGetValue(session.SessionId, out var fixture)) fixture.Steps.Add(new CapturedWebSocketFixtureStep @@ -95,6 +104,10 @@ public sealed class FileWebSocketTelemetrySink( "internal", null, new Dictionary { ["reason"] = reason }), cancellationToken); + await AppendIndexAsync(envelope, session, "connection_closed", new Dictionary + { + ["reason"] = reason + }, cancellationToken); if (!options.Value.ExportFixtures || !_fixtures.TryRemove(session.SessionId, out var fixture) || fixture.Steps.Count == 0) return; @@ -122,6 +135,13 @@ public sealed class FileWebSocketTelemetrySink( _writeLock.Release(); } + await AppendIndexAsync(envelope, session, "fixture_export", new Dictionary + { + ["fixturePath"] = fixturePath, + ["stepCount"] = fixture.Steps.Count, + ["fixtureName"] = fixtureName + }, cancellationToken); + logger.LogInformation("Exported websocket fixture {FixturePath}", fixturePath); } @@ -223,6 +243,31 @@ public sealed class FileWebSocketTelemetrySink( AppContext.BaseDirectory); } + private async Task AppendIndexAsync( + WebSocketMessageEnvelope envelope, + CloudSession session, + string eventType, + IReadOnlyDictionary? details, + CancellationToken cancellationToken) + { + var directory = GetBaseDirectory(); + await CaptureIndexWriter.AppendAsync( + directory, + "websocket", + eventType, + new Dictionary + { + ["sessionId"] = session.SessionId, + ["hostName"] = envelope.HostName, + ["path"] = envelope.Path, + ["kind"] = envelope.Kind, + ["token"] = envelope.Token, + ["transId"] = session.TurnState.TransId ?? session.LastTransId, + ["details"] = details + }, + cancellationToken); + } + private static string BuildFixtureName(CloudSession session, CapturedWebSocketFixtureBuilder fixture) { var host = SanitizeName(fixture.Session.HostName); @@ -246,4 +291,4 @@ public sealed class FileWebSocketTelemetrySink( public CapturedWebSocketFixtureSession Session { get; init; } = new(); public List Steps { get; } = []; } -} \ No newline at end of file +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/FileProtocolTelemetrySinkTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/FileProtocolTelemetrySinkTests.cs index 2f0ee5c..21936e9 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/FileProtocolTelemetrySinkTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Protocol/FileProtocolTelemetrySinkTests.cs @@ -55,8 +55,13 @@ public sealed class FileProtocolTelemetrySinkTests : IDisposable var captureFile = Directory.GetFiles(captureDirectory, "*.events.ndjson").Single(); var contents = await File.ReadAllTextAsync(captureFile); + var indexPath = Path.Combine(captureDirectory, "capture-index.ndjson"); + var indexLines = await File.ReadAllLinesAsync(indexPath); Assert.Contains("Notification_20150505", contents); Assert.DoesNotContain(Path.Combine("bin", "Debug"), captureFile, StringComparison.OrdinalIgnoreCase); + Assert.Contains(indexLines, line => line.Contains("\"eventType\":\"protocol_record\"", StringComparison.Ordinal)); + Assert.Contains(indexLines, line => line.Contains("\"servicePrefix\":\"Notification_20150505\"", + StringComparison.Ordinal)); } -} \ No newline at end of file +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs index fc26da0..ad8c985 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Turn/FileTurnTelemetrySinkTests.cs @@ -39,6 +39,35 @@ public sealed class FileTurnTelemetrySinkTests Assert.Equal(1234, payload.GetProperty("details").GetProperty("bufferedAudioBytes").GetInt32()); } + [Fact] + public async Task RecordsCaptureIndexForTurnDiagnosticsAndErrors() + { + var directoryPath = Path.Combine(Path.GetTempPath(), "OpenJibo.Tests", Guid.NewGuid().ToString("N")); + var sink = new FileTurnTelemetrySink( + NullLogger.Instance, + Options.Create(new TurnTelemetryOptions + { + Enabled = true, + DirectoryPath = directoryPath + })); + + await sink.RecordTurnDiagnosticAsync("yes_no_turn_received", new Dictionary + { + ["transID"] = "trans-1", + ["bufferedAudioBytes"] = 1234 + }); + + await sink.RecordTranscriptError(new InvalidOperationException("boom"), "turn error"); + + var indexPath = Path.Combine(directoryPath, "capture-index.ndjson"); + var lines = await File.ReadAllLinesAsync(indexPath); + + Assert.Contains(lines, line => line.Contains("\"eventType\":\"yes_no_turn_received\"", StringComparison.Ordinal)); + Assert.Contains(lines, line => line.Contains("\"eventType\":\"transcript_error\"", StringComparison.Ordinal)); + Assert.Contains(lines, line => line.Contains("\"message\":\"Turn telemetry diagnostic\"", StringComparison.Ordinal)); + Assert.Contains(lines, line => line.Contains("\"message\":\"Turn telemetry error\"", StringComparison.Ordinal)); + } + [Fact] public async Task RecordsTranscriptErrorOnTurnError() { @@ -148,4 +177,4 @@ public sealed class FileTurnTelemetrySinkTests It.IsAny()), Times.AtLeastOnce()); } -} \ No newline at end of file +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/FileWebSocketTelemetrySinkTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/FileWebSocketTelemetrySinkTests.cs index a03881b..7ed7baf 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/FileWebSocketTelemetrySinkTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/FileWebSocketTelemetrySinkTests.cs @@ -66,6 +66,14 @@ public sealed class FileWebSocketTelemetrySinkTests : IDisposable Assert.Equal(1, document.RootElement.GetProperty("steps").GetArrayLength()); Assert.Equal("LISTEN", document.RootElement.GetProperty("steps")[0].GetProperty("expectedReplyTypes")[0].GetString()); + + var indexPath = Path.Combine(_directoryPath, "capture-index.ndjson"); + var indexEntries = await ReadNdjsonAsync(indexPath); + Assert.Contains(indexEntries, entry => entry.GetProperty("eventType").GetString() == "connection_opened"); + Assert.Contains(indexEntries, entry => entry.GetProperty("eventType").GetString() == "message_in"); + Assert.Contains(indexEntries, entry => entry.GetProperty("eventType").GetString() == "message_out"); + Assert.Contains(indexEntries, entry => entry.GetProperty("eventType").GetString() == "connection_closed"); + Assert.Contains(indexEntries, entry => entry.GetProperty("eventType").GetString() == "fixture_export"); } [Fact] @@ -124,4 +132,18 @@ public sealed class FileWebSocketTelemetrySinkTests : IDisposable DirectoryPath = _directoryPath })); } -} \ No newline at end of file + + private static async Task> ReadNdjsonAsync(string filePath) + { + var entries = new List(); + foreach (var line in await File.ReadAllLinesAsync(filePath)) + { + if (string.IsNullOrWhiteSpace(line)) continue; + + using var document = JsonDocument.Parse(line); + entries.Add(document.RootElement.Clone()); + } + + return entries; + } +}