diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index 587ee77..4619e8f 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -889,8 +889,8 @@ For `1.0.19`: 9. Durable memory persistence path (multi-tenant backing store) - reference design captured in `docs/persistence-architecture.md` - store contracts are now tightened around account/loop/device/person scoping, revision tracking, and explicit load/save boundaries - - the backend seam is now selectable, with file-backed local persistence as default and an Azure SQL slot wired for future deployment when credentials are available - - next implementation pass should supply the real Azure SQL connection string / deployment wiring and validate the live round-trip in Azure + - the backend seam is now selectable, with file-backed local persistence as default and an Azure Blob Storage slot wired for future deployment when a storage account connection string is available + - next implementation pass should supply the real Azure Storage connection string / deployment wiring and validate the live round-trip in the storage account smoke test 10. Update, backup, and restore proof 11. STT upgrade and noise screening 12. Hosted capture/storage plan / indexing for group testing diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index 44fede4..dbc969d 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -95,7 +95,7 @@ The goal is to port these in small batches, capture the source-backed phrasing w - seed person-aware state keys now so future interactions can scope to account + loop + device + person without another shape change - keep stateful interaction flows repository-backed instead of embedding more ad hoc metadata in the websocket layer - the store seam now exposes revision metadata plus explicit load/save boundaries so durable adapters can drop in later without changing behavior code -- the backend seam is now selectable, with file-backed local persistence as the default and an Azure SQL slot wired for future deployment wiring +- the backend seam is now selectable, with file-backed local persistence as the default and an Azure Blob Storage slot wired for future deployment wiring ### 6. Multi-Server Sync Path diff --git a/OpenJibo/src/Jibo.Cloud/README.md b/OpenJibo/src/Jibo.Cloud/README.md index 19b45d8..ccf34f6 100644 --- a/OpenJibo/src/Jibo.Cloud/README.md +++ b/OpenJibo/src/Jibo.Cloud/README.md @@ -45,6 +45,47 @@ Human-facing entry points will live on domains such as: Robot traffic may still arrive using legacy hostnames routed to the OpenJibo service. +## Azure Storage Wiring Sample + +For local or hosted Blob-backed persistence, use the Azure sample config in: + +- [appsettings.AzureBlob.sample.json](dotnet/src/Jibo.Cloud.Api/appsettings.AzureBlob.sample.json) + +It shows the expected keys for: + +- `OpenJibo:State:Backend` +- `OpenJibo:State:ConnectionString` +- `OpenJibo:PersonalMemory:Backend` +- `OpenJibo:PersonalMemory:ConnectionString` + +The connection string can also come from: + +- `OPENJIBO_STATE_STORAGE_CONNECTION_STRING` +- `OPENJIBO_PERSONAL_MEMORY_STORAGE_CONNECTION_STRING` + +For a real storage account, swap `UseDevelopmentStorage=true` with your Azure Storage connection string. + +## Local Startup Note + +To run the API with the Blob-backed sample config in Visual Studio or `dotnet run`, choose the +`Jibo.Cloud.Api.AzureBlob` launch profile. + +The test project also has a matching `Jibo.Cloud.Tests.AzureBlob` profile so the smoke test can use +the same environment-variable shape when you run it from an IDE. + +Equivalent environment variables: + +```powershell +$env:OpenJibo__State__Backend = "AzureBlob" +$env:OpenJibo__State__ConnectionString = "UseDevelopmentStorage=true" +$env:OpenJibo__PersonalMemory__Backend = "AzureBlob" +$env:OpenJibo__PersonalMemory__ConnectionString = "UseDevelopmentStorage=true" +dotnet run --project dotnet/src/Jibo.Cloud.Api/Jibo.Cloud.Api.csproj +``` + +Replace `UseDevelopmentStorage=true` with your real storage account connection string when you move +from local emulation to Azure. + ## Recovery Strategy The first supported device path is: diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Properties/launchSettings.json b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Properties/launchSettings.json index dd31341..dbd50b4 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Properties/launchSettings.json +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/Properties/launchSettings.json @@ -7,6 +7,18 @@ "ASPNETCORE_ENVIRONMENT": "Development" }, "applicationUrl": "https://localhost:24604;http://localhost:24605" + }, + "Jibo.Cloud.Api.AzureBlob": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "OpenJibo__State__Backend": "AzureBlob", + "OpenJibo__State__ConnectionString": "UseDevelopmentStorage=true", + "OpenJibo__PersonalMemory__Backend": "AzureBlob", + "OpenJibo__PersonalMemory__ConnectionString": "UseDevelopmentStorage=true" + }, + "applicationUrl": "https://localhost:24604;http://localhost:24605" } } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.AzureBlob.sample.json b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.AzureBlob.sample.json new file mode 100644 index 0000000..3134933 --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Api/appsettings.AzureBlob.sample.json @@ -0,0 +1,14 @@ +{ + "OpenJibo": { + "State": { + "Backend": "AzureBlob", + "ConnectionString": "UseDevelopmentStorage=true", + "PersistencePath": "App_Data/cloud-state.json" + }, + "PersonalMemory": { + "Backend": "AzureBlob", + "ConnectionString": "UseDevelopmentStorage=true", + "PersistencePath": "App_Data/personal-memory.json" + } + } +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs index 4cafbcf..999a987 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/DependencyInjection/ServiceCollectionExtensions.cs @@ -58,8 +58,12 @@ public static class ServiceCollectionExtensions ?? Path.Combine(AppContext.BaseDirectory, "App_Data", "personal-memory.json"); var stateBackendKind = ParseBackendKind(configuration?["OpenJibo:State:Backend"]); var personalMemoryBackendKind = ParseBackendKind(configuration?["OpenJibo:PersonalMemory:Backend"]); - var stateConnectionString = configuration?["OpenJibo:State:ConnectionString"] ?? Environment.GetEnvironmentVariable("OPENJIBO_STATE_SQL_CONNECTION_STRING"); - var personalMemoryConnectionString = configuration?["OpenJibo:PersonalMemory:ConnectionString"] ?? Environment.GetEnvironmentVariable("OPENJIBO_PERSONAL_MEMORY_SQL_CONNECTION_STRING"); + var stateConnectionString = configuration?["OpenJibo:State:ConnectionString"] + ?? Environment.GetEnvironmentVariable("OPENJIBO_STATE_STORAGE_CONNECTION_STRING") + ?? Environment.GetEnvironmentVariable("OPENJIBO_STATE_SQL_CONNECTION_STRING"); + var personalMemoryConnectionString = configuration?["OpenJibo:PersonalMemory:ConnectionString"] + ?? Environment.GetEnvironmentVariable("OPENJIBO_PERSONAL_MEMORY_STORAGE_CONNECTION_STRING") + ?? Environment.GetEnvironmentVariable("OPENJIBO_PERSONAL_MEMORY_SQL_CONNECTION_STRING"); services.AddSingleton(); services.AddSingleton(provider => { diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Jibo.Cloud.Infrastructure.csproj b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Jibo.Cloud.Infrastructure.csproj index b3bdced..1178c88 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Jibo.Cloud.Infrastructure.csproj +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Jibo.Cloud.Infrastructure.csproj @@ -25,6 +25,7 @@ + diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/AzureBlobSnapshotStore.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/AzureBlobSnapshotStore.cs new file mode 100644 index 0000000..aec7c69 --- /dev/null +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/AzureBlobSnapshotStore.cs @@ -0,0 +1,68 @@ +using System.Text.Json; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; + +namespace Jibo.Cloud.Infrastructure.Persistence; + +internal sealed class AzureBlobSnapshotStore : ISnapshotStore +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true, + PropertyNameCaseInsensitive = true + }; + + private readonly BlobContainerClient _containerClient; + private readonly string _blobName; + + public AzureBlobSnapshotStore(string connectionString, string snapshotName, string containerName = "openjibo-snapshots") + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new InvalidOperationException("Azure Blob persistence requires a storage connection string."); + } + + if (string.IsNullOrWhiteSpace(snapshotName)) + { + throw new ArgumentException("A snapshot name is required for Azure Blob persistence.", nameof(snapshotName)); + } + + _containerClient = new BlobContainerClient(connectionString, string.IsNullOrWhiteSpace(containerName) ? "openjibo-snapshots" : containerName); + _blobName = $"{snapshotName}.json"; + } + + public TSnapshot? Load() where TSnapshot : class + { + try + { + if (!_containerClient.Exists()) + { + return default; + } + + var blobClient = _containerClient.GetBlobClient(_blobName); + if (!blobClient.Exists()) + { + return default; + } + + var content = blobClient.DownloadContent(); + var json = content.Value.Content.ToString(); + return string.IsNullOrWhiteSpace(json) + ? default + : JsonSerializer.Deserialize(json, JsonOptions); + } + catch + { + return default; + } + } + + public void Save(TSnapshot snapshot) where TSnapshot : class + { + _containerClient.CreateIfNotExists(PublicAccessType.None); + var blobClient = _containerClient.GetBlobClient(_blobName); + var json = JsonSerializer.Serialize(snapshot, JsonOptions); + blobClient.Upload(BinaryData.FromString(json), overwrite: true); + } +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceBackendKind.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceBackendKind.cs index fd5c47a..7dda13b 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceBackendKind.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceBackendKind.cs @@ -3,5 +3,6 @@ namespace Jibo.Cloud.Infrastructure.Persistence; public enum PersistenceBackendKind { File, + AzureBlob, AzureSql } diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceSnapshotStoreFactory.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceSnapshotStoreFactory.cs index aac9a58..bab5326 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceSnapshotStoreFactory.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Persistence/PersistenceSnapshotStoreFactory.cs @@ -15,6 +15,9 @@ public sealed class PersistenceSnapshotStoreFactory : IPersistenceSnapshotStoreF return backendKind switch { PersistenceBackendKind.File => new JsonFileSnapshotStore(persistencePath, JsonOptions), + PersistenceBackendKind.AzureBlob => new AzureBlobSnapshotStore( + connectionString ?? throw new InvalidOperationException("Azure Blob persistence requires a connection string."), + snapshotName), PersistenceBackendKind.AzureSql => new AzureSqlSnapshotStore( connectionString ?? throw new InvalidOperationException("Azure SQL persistence requires a connection string."), snapshotName), diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Infrastructure/AzureBlobPersistenceSmokeTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Infrastructure/AzureBlobPersistenceSmokeTests.cs new file mode 100644 index 0000000..7d9b87e --- /dev/null +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Infrastructure/AzureBlobPersistenceSmokeTests.cs @@ -0,0 +1,48 @@ +using Jibo.Cloud.Infrastructure.Persistence; + +namespace Jibo.Cloud.Tests.Infrastructure; + +public sealed class AzureBlobPersistenceSmokeTests +{ + [Fact] + public void AzureBlobSnapshotStore_RoundTripsState_WhenAzureBlobProfileIsConfigured() + { + var stateBackend = Environment.GetEnvironmentVariable("OpenJibo__State__Backend"); + var stateConnectionString = Environment.GetEnvironmentVariable("OpenJibo__State__ConnectionString"); + var personalMemoryBackend = Environment.GetEnvironmentVariable("OpenJibo__PersonalMemory__Backend"); + var personalMemoryConnectionString = Environment.GetEnvironmentVariable("OpenJibo__PersonalMemory__ConnectionString"); + + if (!string.Equals(stateBackend, "AzureBlob", StringComparison.OrdinalIgnoreCase) || + string.IsNullOrWhiteSpace(stateConnectionString)) + { + return; + } + + var factory = new PersistenceSnapshotStoreFactory(); + var snapshotName = $"smoke-{Guid.NewGuid():N}"; + var store = factory.Create(null, PersistenceBackendKind.AzureBlob, snapshotName, stateConnectionString); + + var payload = new SmokeSnapshot + { + Name = "azure-smoke", + Value = "round-trip" + }; + + store.Save(payload); + + var loaded = store.Load(); + + Assert.NotNull(loaded); + Assert.Equal(payload.Name, loaded!.Name); + Assert.Equal(payload.Value, loaded.Value); + Assert.Equal("AzureBlob", stateBackend); + Assert.Equal("AzureBlob", personalMemoryBackend); + Assert.False(string.IsNullOrWhiteSpace(personalMemoryConnectionString)); + } + + private sealed class SmokeSnapshot + { + public string Name { get; init; } = string.Empty; + public string Value { get; init; } = string.Empty; + } +} diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Properties/launchSettings.json b/OpenJibo/tests/Jibo.Cloud.Tests/Properties/launchSettings.json new file mode 100644 index 0000000..ea3a924 --- /dev/null +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Properties/launchSettings.json @@ -0,0 +1,13 @@ +{ + "profiles": { + "Jibo.Cloud.Tests.AzureBlob": { + "commandName": "Project", + "environmentVariables": { + "OpenJibo__State__Backend": "AzureBlob", + "OpenJibo__State__ConnectionString": "UseDevelopmentStorage=true", + "OpenJibo__PersonalMemory__Backend": "AzureBlob", + "OpenJibo__PersonalMemory__ConnectionString": "UseDevelopmentStorage=true" + } + } + } +}