fixes for clock and photo gallery

This commit is contained in:
Jacob Dubin
2026-04-21 21:28:15 -05:00
parent 1049f8c038
commit 6b070140bb
13 changed files with 603 additions and 62 deletions

View File

@@ -124,6 +124,9 @@ Current raw-audio behavior is still a compatibility bridge:
- hotphrase `[BLANK_AUDIO]` cleanup turns are ignored instead of reopening the cloud into a stale blank-audio comment path after word-of-the-day completion
- phrase matching has been widened slightly for known test prompts such as joke, dance, surprise, weather, calendar, commute, and news variants
- time replies now use the natural hour format without a leading zero
- plain time/date/day questions now travel through stock-shaped local `@be/clock` handoffs, and `open the clock` uses the direct clock-view path instead of the menu path
- timer/alarm voice launches now accept compact alarm forms like `830` and `8 30`, and malformed timer/alarm requests stay on a clarification reply instead of generic cloud chat
- media and update metadata now persist to a local state file so gallery/update behavior is not lost on every process restart
## Buffered Audio STT
@@ -167,6 +170,12 @@ Capture-storage guidance while moving toward hosted group testing:
- hosted deployments should keep runtime request handling decoupled from long-term capture retention
- sanitized fixtures remain the preferred durable artifact for parity work and bug reproduction
Current local state persistence:
- default path: `App_Data/cloud-state.json` under the running API directory
- current contents: media metadata, backup metadata, and staged update metadata
- current limitation: media bodies are only preserved through the existing text-based HTTP body capture seam, so this is a hosted-gallery bridge, not final binary-safe media storage
## Current Interaction Paths
The working cloud model currently looks like three main paths:

View File

@@ -14,7 +14,7 @@ public interface ICloudStateStore
CloudSession? FindSessionByToken(string token);
IReadOnlyList<LoopRecord> GetLoops();
IReadOnlyList<UpdateManifest> ListUpdates(string? subsystem = null, string? filter = null);
UpdateManifest GetUpdateFrom(string? subsystem, string? fromVersion, string? filter);
UpdateManifest? GetUpdateFrom(string? subsystem, string? fromVersion, string? filter);
UpdateManifest CreateUpdate(string? fromVersion, string? toVersion, string? changes, string? shaHash, long? length, string? subsystem, string? filter, IDictionary<string, object?>? dependencies);
UpdateManifest RemoveUpdate(string? updateId);
IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null);

View File

@@ -74,6 +74,10 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
"word_of_the_day_guess" => false,
"radio" => false,
"radio_genre" => false,
"time" => false,
"date" => false,
"day" => false,
"clock_open" => false,
"clock_menu" => false,
"timer_menu" => false,
"alarm_menu" => false,

View File

@@ -29,6 +29,12 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
return Task.FromResult(ProtocolDispatchResult.Ok(new { ok = true, host = envelope.HostName }));
}
if (envelope.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) &&
envelope.Path.StartsWith("/media/", StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(HandleMediaContent(envelope));
}
if (envelope.Method.Equals("PUT", StringComparison.OrdinalIgnoreCase) &&
(envelope.Path.Equals("/upload/asr-binary", StringComparison.OrdinalIgnoreCase) ||
envelope.Path.Equals("/upload/log-events", StringComparison.OrdinalIgnoreCase) ||
@@ -383,7 +389,13 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
var type = ReadHeader(envelope, "x-type") ?? ReadString(body, "type") ?? "unknown";
var reference = ReadHeader(envelope, "x-reference") ?? ReadString(body, "reference") ?? string.Empty;
var isEncrypted = ReadBooleanHeader(envelope, "x-encrypted") || ReadBool(body, "isEncrypted");
var meta = ReadObject(body, "meta");
var meta = ReadObject(body, "meta") ?? new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
var contentType = ReadHeader(envelope, "Content-Type") ?? "application/octet-stream";
meta["contentType"] = contentType;
if (!string.IsNullOrWhiteSpace(envelope.BodyText))
{
meta["bodyText"] = envelope.BodyText;
}
return ProtocolDispatchResult.Ok(MapMedia(stateStore.CreateMedia(loopId, path, type, reference, isEncrypted, meta)));
}
@@ -530,7 +542,7 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
.Where(update => fromVersion is null || update.FromVersion.Equals(fromVersion, StringComparison.OrdinalIgnoreCase))
.Select(MapUpdate)
.ToArray()),
"GetUpdateFrom" => ProtocolDispatchResult.Ok(MapUpdate(stateStore.GetUpdateFrom(subsystem, fromVersion, filter))),
"GetUpdateFrom" => HandleGetUpdateFrom(subsystem, fromVersion, filter),
"CreateUpdate" => ProtocolDispatchResult.Ok(MapUpdate(stateStore.CreateUpdate(
fromVersion,
ReadString(body, "toVersion"),
@@ -545,6 +557,29 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
};
}
private ProtocolDispatchResult HandleMediaContent(ProtocolEnvelope envelope)
{
var path = Uri.UnescapeDataString(envelope.Path["/media/".Length..]);
var candidatePaths = new[] { path, $"/{path}" };
var media = stateStore.GetMedia(candidatePaths).FirstOrDefault();
if (media is null || media.IsDeleted)
{
return ProtocolDispatchResult.Raw(404, string.Empty);
}
var contentType = TryReadMetaString(media.Meta, "contentType") ?? "application/octet-stream";
var bodyText = TryReadMetaString(media.Meta, "bodyText") ?? string.Empty;
return ProtocolDispatchResult.Raw(200, bodyText, contentType);
}
private ProtocolDispatchResult HandleGetUpdateFrom(string? subsystem, string? fromVersion, string? filter)
{
var update = stateStore.GetUpdateFrom(subsystem, fromVersion, filter);
return update is null
? ProtocolDispatchResult.Ok(new { })
: ProtocolDispatchResult.Ok(MapUpdate(update));
}
private static object MapUpdate(UpdateManifest update)
{
return new
@@ -575,12 +610,21 @@ public sealed class JiboCloudProtocolService(ICloudStateStore stateStore)
accountId = item.AccountId,
loopId = item.LoopId,
url = item.Url,
thumbnailUrl = item.Url,
originalUrl = item.Url,
isEncrypted = item.IsEncrypted,
isDeleted = item.IsDeleted,
meta = item.Meta
};
}
private static string? TryReadMetaString(IDictionary<string, object?> meta, string key)
{
return meta.TryGetValue(key, out var value)
? value?.ToString()
: null;
}
private static string? ReadString(JsonElement? element, string propertyName)
{
if (element is null || !element.Value.TryGetProperty(propertyName, out var property))

View File

@@ -28,17 +28,20 @@ public sealed class JiboInteractionService(
{
"joke" => BuildJokeDecision(catalog),
"dance" => BuildDanceDecision(catalog),
"time" => new JiboInteractionDecision("time", $"It is {DateTime.Now:h:mm tt}."),
"date" => new JiboInteractionDecision("date", $"Today is {DateTime.Now:dddd, MMMM d}."),
"day" => new JiboInteractionDecision("day", $"Today is {DateTime.Now:dddd}."),
"time" => BuildClockLaunchDecision("time", "clock", "askForTime", "Showing the time."),
"date" => BuildClockLaunchDecision("date", "clock", "askForDate", "Showing the date."),
"day" => BuildClockLaunchDecision("day", "clock", "askForDay", "Showing the day."),
"cloud_version" => new JiboInteractionDecision("cloud_version", OpenJiboCloudBuildInfo.SpokenVersion),
"radio" => BuildRadioLaunchDecision(),
"radio_genre" => BuildRadioGenreLaunchDecision(lowered),
"clock_menu" => BuildClockLaunchDecision("clock", "Opening the clock."),
"clock_open" => BuildClockLaunchDecision("clock_open", "clock", "askForTime", "Opening the clock."),
"clock_menu" => BuildClockLaunchDecision("clock_menu", "clock", "menu", "Opening the clock menu."),
"timer_menu" => BuildClockLaunchDecision("timer", "Opening the timer."),
"alarm_menu" => BuildClockLaunchDecision("alarm", "Opening the alarm."),
"timer_value" => BuildTimerValueDecision(lowered),
"alarm_value" => BuildAlarmValueDecision(lowered),
"timer_clarify" => new JiboInteractionDecision("timer_clarify", "How long should I set the timer for?"),
"alarm_clarify" => new JiboInteractionDecision("alarm_clarify", "What time should I set the alarm for?"),
"photo_gallery" => BuildPhotoGalleryLaunchDecision(),
"snapshot" => BuildPhotoCreateDecision("snapshot", "Taking a picture.", "createOnePhoto"),
"photobooth" => BuildPhotoCreateDecision("photobooth", "Starting photobooth.", "createSomePhotos"),
@@ -236,19 +239,9 @@ public sealed class JiboInteractionService(
return "radio_genre";
}
if (TryParseAlarmValue(loweredTranscript) is not null)
{
return "alarm_value";
}
if (TryParseTimerValue(loweredTranscript) is not null)
{
return "timer_value";
}
if (MatchesAny(loweredTranscript, "open the clock", "open clock", "show the clock", "show clock"))
{
return "clock_menu";
return "clock_open";
}
if (MatchesAny(loweredTranscript, "open the timer", "open timer", "show the timer", "show timer"))
@@ -261,6 +254,26 @@ public sealed class JiboInteractionService(
return "alarm_menu";
}
if (TryParseAlarmValue(loweredTranscript) is not null)
{
return "alarm_value";
}
if (TryParseTimerValue(loweredTranscript) is not null)
{
return "timer_value";
}
if (IsAlarmRequest(loweredTranscript))
{
return "alarm_clarify";
}
if (IsTimerRequest(loweredTranscript))
{
return "timer_clarify";
}
if (MatchesAny(loweredTranscript, "open the radio", "play the radio", "turn on the radio", "radio"))
{
return "radio";
@@ -425,20 +438,25 @@ public sealed class JiboInteractionService(
});
}
private static JiboInteractionDecision BuildClockLaunchDecision(string domain, string replyText)
private static JiboInteractionDecision BuildClockLaunchDecision(string intentName, string domain, string clockIntent, string replyText)
{
return new JiboInteractionDecision(
$"{domain}_menu",
intentName,
replyText,
"@be/clock",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["skillId"] = "@be/clock",
["domain"] = domain,
["clockIntent"] = "menu"
["clockIntent"] = clockIntent
});
}
private static JiboInteractionDecision BuildClockLaunchDecision(string domain, string replyText)
{
return BuildClockLaunchDecision($"{domain}_menu", domain, "menu", replyText);
}
private static JiboInteractionDecision BuildTimerValueDecision(string loweredTranscript)
{
var timer = TryParseTimerValue(loweredTranscript) ?? new ClockTimerValue("0", "1", "null");
@@ -733,7 +751,33 @@ public sealed class JiboInteractionService(
return null;
}
var match = AlarmPattern.Match(loweredTranscript);
var compactMatch = CompactAlarmPattern.Match(loweredTranscript);
if (compactMatch.Success)
{
var compact = compactMatch.Groups["compact"].Value;
if (int.TryParse(compact, out var compactValue))
{
var compactHour = compact.Length switch
{
3 => compactValue / 100,
4 => compactValue / 100,
_ => -1
};
var compactMinute = compact.Length switch
{
3 => compactValue % 100,
4 => compactValue % 100,
_ => -1
};
if (compactHour is >= 1 and <= 12 && compactMinute is >= 0 and <= 59)
{
var compactAmPm = ResolveAmPm(compactMatch.Groups["ampm"].Value);
return new ClockAlarmValue($"{compactHour}:{compactMinute:00}", compactAmPm);
}
}
}
var match = SplitAlarmPattern.Match(loweredTranscript);
if (!match.Success)
{
return null;
@@ -752,10 +796,36 @@ public sealed class JiboInteractionService(
return null;
}
var ampm = match.Groups["ampm"].Value.StartsWith("p", StringComparison.Ordinal) ? "pm" : "am";
var ampm = ResolveAmPm(match.Groups["ampm"].Value);
return new ClockAlarmValue($"{hour}:{minute:00}", ampm);
}
private static string ResolveAmPm(string token)
{
return token.StartsWith("p", StringComparison.OrdinalIgnoreCase) ? "pm" : "am";
}
private static bool IsTimerRequest(string loweredTranscript)
{
return MatchesAny(
loweredTranscript,
"set a timer",
"set timer",
"start a timer",
"start timer",
"timer for");
}
private static bool IsAlarmRequest(string loweredTranscript)
{
return MatchesAny(
loweredTranscript,
"set an alarm",
"set alarm",
"wake me up",
"alarm for");
}
private static int? ExtractDurationValue(string loweredTranscript, string unitStem)
{
var pattern = new Regex($@"\b(?<value>\d+|[a-z\-]+)\s+{unitStem}s?\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
@@ -810,8 +880,12 @@ public sealed class JiboInteractionService(
private sealed record ClockAlarmValue(string Time, string AmPm);
private static readonly Regex AlarmPattern = new(
@"\b(?<hour>\d{1,2}|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)(?:[:\s](?<minute>\d{2}))?\s*(?<ampm>a\.?m\.?|p\.?m\.?)\b",
private static readonly Regex SplitAlarmPattern = new(
@"\b(?<hour>\d{1,2}|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)(?:[:\s](?<minute>\d{2}))?\s*(?<ampm>a\.?m\.?|p\.?m\.?)?\b",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex CompactAlarmPattern = new(
@"\b(?<compact>\d{3,4})\s*(?<ampm>a\.?m\.?|p\.?m\.?)?\b",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly (string Phrase, string Station)[] RadioGenreAliases =

View File

@@ -564,6 +564,10 @@ public sealed class WebSocketTurnFinalizationService(
var emitSkillActions = !string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "radio", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "radio_genre", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "time", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "date", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "day", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "clock_open", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "clock_menu", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "timer_menu", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "alarm_menu", StringComparison.OrdinalIgnoreCase) &&

View File

@@ -7,6 +7,7 @@ using Jibo.Cloud.Infrastructure.Telemetry;
using Jibo.Runtime.Abstractions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Configuration;
using System.IO;
namespace Jibo.Cloud.Infrastructure.DependencyInjection;
@@ -24,7 +25,9 @@ public static class ServiceCollectionExtensions
}
services.AddSingleton(sttOptions);
services.AddSingleton<ICloudStateStore, InMemoryCloudStateStore>();
var statePersistencePath = configuration?["OpenJibo:State:PersistencePath"]
?? Path.Combine(AppContext.BaseDirectory, "App_Data", "cloud-state.json");
services.AddSingleton<ICloudStateStore>(_ => new InMemoryCloudStateStore(statePersistencePath));
services.AddSingleton<IJiboExperienceContentRepository, InMemoryJiboExperienceContentRepository>();
services.AddSingleton<JiboExperienceContentCache>();
services.AddSingleton<IJiboRandomizer, DefaultJiboRandomizer>();

View File

@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Text.Json;
using Jibo.Cloud.Application.Abstractions;
using Jibo.Cloud.Domain.Models;
@@ -6,11 +7,18 @@ namespace Jibo.Cloud.Infrastructure.Persistence;
public sealed class InMemoryCloudStateStore : ICloudStateStore
{
private static readonly JsonSerializerOptions PersistenceJsonOptions = new()
{
WriteIndented = true
};
private readonly AccountProfile _account = new();
private readonly ConcurrentDictionary<string, DeviceRegistration> _devices = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, CloudSession> _sessionsByToken = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, string> _symmetricKeys = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, KeyRequestRecord> _keyRequests = new(StringComparer.OrdinalIgnoreCase);
private readonly string? _persistencePath;
private readonly object _syncRoot = new();
private readonly List<UpdateManifest> _updates;
private readonly List<MediaRecord> _media = [];
private readonly List<BackupRecord> _backups = [];
@@ -18,8 +26,9 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
private DeviceRegistration _robot;
private RobotProfile _robotProfile;
public InMemoryCloudStateStore()
public InMemoryCloudStateStore(string? persistencePath = null)
{
_persistencePath = persistencePath;
_robot = new DeviceRegistration
{
HostMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
@@ -52,19 +61,8 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
}
];
_updates =
[
new UpdateManifest
{
UpdateId = "noop-update-robot",
FromVersion = "unknown",
ToVersion = "unknown",
Changes = "No update available",
Url = "https://api.jibo.com/update/noop",
ShaHash = "noop",
Subsystem = "robot"
}
];
_updates = [];
LoadPersistentState();
}
public AccountProfile GetAccount() => _account;
@@ -159,16 +157,10 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
.ToArray();
}
public UpdateManifest GetUpdateFrom(string? subsystem, string? fromVersion, string? filter)
public UpdateManifest? GetUpdateFrom(string? subsystem, string? fromVersion, string? filter)
{
return ListUpdates(subsystem, filter).FirstOrDefault() ?? new UpdateManifest
{
UpdateId = $"noop-update-{subsystem ?? "robot"}-{fromVersion ?? "unknown"}",
FromVersion = fromVersion ?? "unknown",
ToVersion = fromVersion ?? "unknown",
Filter = filter,
Subsystem = subsystem ?? "robot"
};
return ListUpdates(subsystem, filter)
.FirstOrDefault(update => fromVersion is null || update.FromVersion.Equals(fromVersion, StringComparison.OrdinalIgnoreCase));
}
public UpdateManifest CreateUpdate(string? fromVersion, string? toVersion, string? changes, string? shaHash, long? length, string? subsystem, string? filter, IDictionary<string, object?>? dependencies)
@@ -187,6 +179,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
};
_updates.Add(update);
PersistState();
return update;
}
@@ -196,6 +189,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
if (existing is not null)
{
_updates.Remove(existing);
PersistState();
return existing;
}
@@ -212,6 +206,7 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
public IReadOnlyList<MediaRecord> ListMedia(IReadOnlyList<string>? loopIds = null, long? after = null, long? before = null)
{
return _media
.Where(item => !item.IsDeleted)
.Where(item => loopIds is null || loopIds.Count == 0 || loopIds.Contains(item.LoopId))
.Where(item => after is null || item.CreatedUtc.ToUnixTimeMilliseconds() > after)
.Where(item => before is null || item.CreatedUtc.ToUnixTimeMilliseconds() < before)
@@ -251,6 +246,11 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
replacements.Add(updated);
}
if (replacements.Count > 0)
{
PersistState();
}
return replacements;
}
@@ -268,13 +268,23 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
Meta = meta ?? new Dictionary<string, object?>()
};
_media.Add(item);
var existingIndex = _media.FindIndex(existing => existing.Path.Equals(path, StringComparison.OrdinalIgnoreCase));
if (existingIndex >= 0)
{
_media[existingIndex] = item;
}
else
{
_media.Add(item);
}
PersistState();
return item;
}
public IReadOnlyList<BackupRecord> GetBackups() => _backups.ToArray();
public bool ShouldCreateSymmetricKey(string loopId) => true;
public bool ShouldCreateSymmetricKey(string loopId) => !_symmetricKeys.ContainsKey(loopId);
public string GetOrCreateSymmetricKey(string loopId)
{
@@ -350,5 +360,69 @@ public sealed class InMemoryCloudStateStore : ICloudStateStore
},
UpdatedUtc = DateTimeOffset.UtcNow
};
PersistState();
}
private void LoadPersistentState()
{
if (string.IsNullOrWhiteSpace(_persistencePath) || !File.Exists(_persistencePath))
{
return;
}
try
{
var snapshot = JsonSerializer.Deserialize<PersistentStateSnapshot>(File.ReadAllText(_persistencePath), PersistenceJsonOptions);
if (snapshot is null)
{
return;
}
_updates.Clear();
_updates.AddRange(snapshot.Updates ?? []);
_media.Clear();
_media.AddRange(snapshot.Media ?? []);
_backups.Clear();
_backups.AddRange(snapshot.Backups ?? []);
}
catch
{
// Ignore corrupt state and continue with the in-memory defaults.
}
}
private void PersistState()
{
if (string.IsNullOrWhiteSpace(_persistencePath))
{
return;
}
lock (_syncRoot)
{
var directory = Path.GetDirectoryName(_persistencePath);
if (!string.IsNullOrWhiteSpace(directory))
{
Directory.CreateDirectory(directory);
}
var snapshot = new PersistentStateSnapshot
{
Updates = _updates.ToArray(),
Media = _media.ToArray(),
Backups = _backups.ToArray()
};
File.WriteAllText(_persistencePath, JsonSerializer.Serialize(snapshot, PersistenceJsonOptions));
}
}
private sealed class PersistentStateSnapshot
{
public UpdateManifest[]? Updates { get; init; }
public MediaRecord[]? Media { get; init; }
public BackupRecord[]? Backups { get; init; }
}
}