Fix Azure Unicode metadata round trips (10.0.14)

Directory.Build.props

@@ -29,8 +29,8 @@
         <RepositoryUrl>https://github.com/managedcode/Storage</RepositoryUrl>
         <PackageProjectUrl>https://github.com/managedcode/Storage</PackageProjectUrl>
         <Product>Managed Code - Storage</Product>
-        <Version>10.0.13</Version>
-        <PackageVersion>10.0.13</PackageVersion>
+        <Version>10.0.14</Version>
+        <PackageVersion>10.0.14</PackageVersion>
 
     </PropertyGroup>
 

README.md

@@ -105,6 +105,7 @@ Cloud storage vendors expose distinct SDKs, option models, and authentication pa
 - ASP.NET storage controllers, chunk orchestration services, and a SignalR hub/client pair that deliver resumable uploads, ranged downloads, CRC32 validation, and real-time progress.
 - `ManagedCode.Storage.Client` brings streaming uploads/downloads, CRC32 helpers, and MIME discovery via `MimeHelper` to any .NET app.
 - Strongly typed option objects (`UploadOptions`, `DownloadOptions`, `DeleteOptions`, `MetadataOptions`, `LegalHoldOptions`, etc.) let you configure directories, metadata, and legal holds in one place.
+- Azure metadata preserves Unicode filenames and other logical string values through a provider-owned ASCII transport envelope; ordinary native ASCII metadata remains interoperable. See [metadata transport](https://github.com/managed-code-hub/Storage/blob/main/docs/Features/provider-azure-blob.md#metadata-transport).
 - Virtual File System package provides a file/directory API (`IVirtualFileSystem`) on top of the configured `IStorage` and can cache metadata for faster repeated operations, including browser storage verified through real Playwright flows in both Blazor WebAssembly and Interactive Server hosts.
 - For decisions requiring current storage state, `IVirtualFileSystem.StorageFileExistsAsync` bypasses that cache and propagates provider errors. `WriteBytesIfAbsentOrSameAsync` uses an atomic provider capability and accepts only an exact immutable retry. `ManagedCode.Storage.Core.Primitives.VerifiedContentSnapshot` verifies bounded reads by length and SHA-256.
 - Comprehensive automated test suite with cross-provider sync fixtures, multi-gigabyte streaming simulations (4 MB units per "GB"), ASP.NET controller harnesses, SFTP/local filesystem coverage, and Playwright browser verification for browser storage small-file overwrites, concurrent tabs, VFS flows, a fast `128 MiB` browser large-file lane, and a separate `256 MiB` browser stress lane in both Interactive Server and Blazor WebAssembly hosts.

Storages/ManagedCode.Storage.Azure/AzureMetadataTransport.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+
+namespace ManagedCode.Storage.Azure;
+
+internal static class AzureMetadataTransport
+{
+    private const string EnvelopeKey = "managedcode_storage_metadata_v1";
+    private const string EnvelopePrefix = "utf8-json-base64:";
+    private static readonly UTF8Encoding StrictUtf8 = new(false, true);
+
+    internal static Dictionary<string, string>? Encode(IEnumerable<KeyValuePair<string, string>>? metadata)
+    {
+        if (metadata is null)
+            return null;
+
+        var values = metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.OrdinalIgnoreCase);
+        if (values.All(pair => IsNativeKey(pair.Key) && IsNativeValue(pair.Value)) && !values.ContainsKey(EnvelopeKey))
+            return values;
+
+        foreach (var pair in values)
+        {
+            ArgumentException.ThrowIfNullOrEmpty(pair.Key);
+            ArgumentNullException.ThrowIfNull(pair.Value);
+            _ = StrictUtf8.GetByteCount(pair.Key);
+            _ = StrictUtf8.GetByteCount(pair.Value);
+        }
+        return new Dictionary<string, string>
+        {
+            [EnvelopeKey] = EnvelopePrefix + Convert.ToBase64String(JsonSerializer.SerializeToUtf8Bytes(values))
+        };
+    }
+
+    internal static Dictionary<string, string> Decode(IEnumerable<KeyValuePair<string, string>> metadata)
+    {
+        var values = metadata.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.OrdinalIgnoreCase);
+        if (!values.TryGetValue(EnvelopeKey, out var envelope) ||
+            !envelope.StartsWith(EnvelopePrefix, StringComparison.Ordinal))
+            return values;
+
+        if (values.Count != 1)
+            throw new InvalidDataException("Invalid Azure metadata envelope.");
+
+        try
+        {
+            using var document = JsonDocument.Parse(Convert.FromBase64String(envelope[EnvelopePrefix.Length..]));
+            if (document.RootElement.ValueKind != JsonValueKind.Object)
+                throw new InvalidDataException("Invalid Azure metadata envelope.");
+
+            var decoded = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            foreach (var property in document.RootElement.EnumerateObject())
+            {
+                if (string.IsNullOrEmpty(property.Name) || property.Value.ValueKind != JsonValueKind.String ||
+                    !decoded.TryAdd(property.Name, property.Value.GetString()!))
+                    throw new InvalidDataException("Invalid Azure metadata envelope.");
+            }
+            return decoded;
+        }
+        catch (Exception exception) when (exception is FormatException or JsonException)
+        {
+            throw new InvalidDataException("Invalid Azure metadata envelope.", exception);
+        }
+    }
+
+    private static bool IsNativeKey(string key) => key.Length > 0 && IsInitialKeyCharacter(key[0]) &&
+        key.Skip(1).All(character => IsInitialKeyCharacter(character) || character is >= '0' and <= '9');
+
+    private static bool IsInitialKeyCharacter(char character) => character is >= 'a' and <= 'z' or >= 'A' and <= 'Z' or '_';
+
+    private static bool IsNativeValue(string value) => value is not null && value.All(character => character is >= ' ' and <= '~');
+}

Storages/ManagedCode.Storage.Azure/AzureObjectOperations.cs

@@ -20,18 +20,18 @@ public Task<StorageContainerInfo> GetContainerInfoAsync(CancellationToken cancel
     {
         var value = (await container.GetPropertiesAsync(cancellationToken: cancellationToken)).Value;
         return new StorageContainerInfo(value.ETag.ToString(), value.PublicAccess == PublicAccessType.None,
-            new Dictionary<string, string>(value.Metadata));
+            AzureMetadataTransport.Decode(value.Metadata));
     });
 
     public Task CreatePrivateContainerAsync(IReadOnlyDictionary<string, string>? metadata = null, CancellationToken cancellationToken = default) => ExecuteAsync(async () =>
     {
-        await container.CreateIfNotExistsAsync(PublicAccessType.None, Copy(metadata), cancellationToken: cancellationToken);
+        await container.CreateIfNotExistsAsync(PublicAccessType.None, AzureMetadataTransport.Encode(metadata), cancellationToken: cancellationToken);
         return true;
     });
 
     public Task SetContainerMetadataAsync(IReadOnlyDictionary<string, string> metadata, CancellationToken cancellationToken = default) => ExecuteAsync(async () =>
     {
-        await container.SetMetadataAsync(Copy(metadata), cancellationToken: cancellationToken);
+        await container.SetMetadataAsync(AzureMetadataTransport.Encode(metadata), cancellationToken: cancellationToken);
         return true;
     });
 
@@ -74,15 +74,15 @@ public Task<StorageObjectInfo> WriteObjectAsync(string path, Stream content, Sto
         {
             Conditions = Conditions(options),
             HttpHeaders = Headers(options),
-            Metadata = Copy(options.Metadata),
+            Metadata = AzureMetadataTransport.Encode(options.Metadata),
             TransferOptions = new global::Azure.Storage.StorageTransferOptions { MaximumConcurrency = 1, InitialTransferSize = 4 * 1024 * 1024, MaximumTransferSize = 4 * 1024 * 1024 }
         }, cancellationToken);
         return await ReadWrittenInfoAsync(path, response.Value.ETag, cancellationToken);
     });
 
     public Task SetObjectMetadataAsync(string path, IReadOnlyDictionary<string, string> metadata, string? ifMatch = null, CancellationToken cancellationToken = default) => ExecuteAsync(async () =>
     {
-        await container.GetBlobClient(path).SetMetadataAsync(Copy(metadata), new BlobRequestConditions { IfMatch = ETagOrNull(ifMatch) }, cancellationToken);
+        await container.GetBlobClient(path).SetMetadataAsync(AzureMetadataTransport.Encode(metadata), new BlobRequestConditions { IfMatch = ETagOrNull(ifMatch) }, cancellationToken);
         return true;
     });
 
@@ -100,7 +100,7 @@ public Task<StorageObjectPage> ListObjectsAsync(string? prefix = null, string? c
             return new StorageObjectPage(page.Values.Select(item => new StorageObjectInfo(item.Name,
                 item.Properties.ETag?.ToString() ?? throw new InvalidDataException("Object listing returned no ETag."),
                 item.Properties.ContentLength ?? throw new InvalidDataException("Object listing returned no length."),
-                item.Properties.ContentType, item.Properties.ContentEncoding, new Dictionary<string, string>(item.Metadata), item.Properties.LastModified)).ToArray(), page.ContinuationToken);
+                item.Properties.ContentType, item.Properties.ContentEncoding, AzureMetadataTransport.Decode(item.Metadata), item.Properties.LastModified)).ToArray(), page.ContinuationToken);
         }
         return new StorageObjectPage([], null);
     });
@@ -118,7 +118,7 @@ public Task<StorageObjectInfo> CommitPartsAsync(string path, IReadOnlyList<strin
         {
             Conditions = Conditions(options),
             HttpHeaders = Headers(options),
-            Metadata = Copy(options.Metadata)
+            Metadata = AzureMetadataTransport.Encode(options.Metadata)
         }, cancellationToken);
         return await ReadWrittenInfoAsync(path, response.Value.ETag, cancellationToken);
     });
@@ -127,7 +127,7 @@ private async Task<StorageObjectInfo> ReadWrittenInfoAsync(string path, ETag eta
         Info(path, (await container.GetBlobClient(path).GetPropertiesAsync(new BlobRequestConditions { IfMatch = etag }, cancellationToken)).Value);
 
     private static StorageObjectInfo Info(string path, BlobProperties value) => new(path, value.ETag.ToString(), value.ContentLength,
-        value.ContentType, value.ContentEncoding, new Dictionary<string, string>(value.Metadata), value.LastModified);
+        value.ContentType, value.ContentEncoding, AzureMetadataTransport.Decode(value.Metadata), value.LastModified);
 
     private static BlobRequestConditions Conditions(StorageWriteOptions options)
     {
@@ -137,7 +137,6 @@ private static BlobRequestConditions Conditions(StorageWriteOptions options)
 
     private static BlobHttpHeaders Headers(StorageWriteOptions options) => new() { ContentType = options.ContentType, ContentEncoding = options.ContentEncoding };
     private static ETag? ETagOrNull(string? value) => value is null ? null : new ETag(value);
-    private static Dictionary<string, string>? Copy(IReadOnlyDictionary<string, string>? value) => value is null ? null : new(value);
 
     private static async Task<T> ExecuteAsync<T>(Func<Task<T>> operation)
     {

Storages/ManagedCode.Storage.Azure/AzureStorage.cs

@@ -74,7 +74,7 @@ public override async IAsyncEnumerable<BlobMetadata> GetBlobMetadataListAsync(st
                     Uri = new Uri(StorageClient.Uri, $"{StorageOptions.Container}/{blobItem.Name}"),
                     Container = StorageOptions.Container,
                     Length = (ulong)blobItem.Properties.ContentLength!.Value,
-                    Metadata = blobItem.Metadata.ToDictionary(k => k.Key, v => v.Value),
+                    Metadata = AzureMetadataTransport.Decode(blobItem.Metadata),
                     LastModified = blobItem.Properties.LastModified!.Value,
                     CreatedOn = blobItem.Properties.CreatedOn!.Value,
                     MimeType = blobItem.Properties.ContentType
@@ -247,7 +247,7 @@ protected override async Task<Result<BlobMetadata>> UploadInternalAsync(Stream s
 
         var uploadOptions = new BlobUploadOptions
         {
-            Metadata = options.Metadata,
+            Metadata = AzureMetadataTransport.Encode(options.Metadata),
             HttpHeaders = new BlobHttpHeaders
             {
                 ContentType = options.MimeType
@@ -315,7 +315,7 @@ protected override async Task<Result<LocalFile>> DownloadInternalAsync(LocalFile
                 Length = (ulong)response.Value.ContentLength,
                 CreatedOn = response.Value.Details.LastModified,
                 LastModified = response.Value.Details.LastModified,
-                Metadata = response.Value.Details.Metadata.ToDictionary(k => k.Key, v => v.Value),
+                Metadata = AzureMetadataTransport.Decode(response.Value.Details.Metadata),
                 MimeType = response.Value.ContentType
             };
 
@@ -390,7 +390,7 @@ protected override async Task<Result<BlobMetadata>> GetBlobMetadataInternalAsync
                 Length = (ulong)properties.Value.ContentLength,
                 CreatedOn = properties.Value.CreatedOn,
                 LastModified = properties.Value.LastModified,
-                Metadata = properties.Value.Metadata.ToDictionary(k => k.Key, v => v.Value),
+                Metadata = AzureMetadataTransport.Decode(properties.Value.Metadata),
                 MimeType = properties.Value.ContentType
             });
         }