diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6603a99..67d762d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,14 +36,18 @@ jobs: azure: name: "Test: AzureFileSystem" runs-on: ubuntu-latest - services: - azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - 10000:10000 - - 10001:10001 - - 10002:10002 steps: + - name: Start Azurite + run: | + docker run -d \ + -p 10000:10000 \ + -p 10001:10001 \ + -p 10002:10002 \ + mcr.microsoft.com/azure-storage/azurite \ + azurite --skipApiVersionCheck \ + --blobHost 0.0.0.0 \ + --queueHost 0.0.0.0 \ + --tableHost 0.0.0.0 - name: Setup .NET uses: actions/setup-dotnet@v4 with: @@ -58,7 +62,7 @@ jobs: - name: Build run: dotnet build -c Debug - name: Test - run: dotnet test --no-build --filter TestCategory=Cloud:Azure + run: dotnet test --no-build --filter TestCategory=Cloud:Azure -maxcpucount:1 s3: name: "Test: S3FileSystem" diff --git a/Directory.Packages.props b/Directory.Packages.props index 944c2cc..12c4c02 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,12 +4,12 @@ true - - - - - - + + + + + + @@ -18,4 +18,4 @@ - \ No newline at end of file + diff --git a/Ramstack.FileSystem.slnx b/Ramstack.FileSystem.slnx index 7347946..e14578d 100644 --- a/Ramstack.FileSystem.slnx +++ b/Ramstack.FileSystem.slnx @@ -4,6 +4,7 @@ + diff --git a/docker-compose.yml b/docker-compose.yml index 1ea5f76..5409b66 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: - "10000:10000" - "10001:10001" - "10002:10002" - command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 + command: azurite --skipApiVersionCheck --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 rustfs: image: docker.io/rustfs/rustfs:latest diff --git a/src/Ramstack.FileSystem.Amazon/S3Directory.cs b/src/Ramstack.FileSystem.Amazon/S3Directory.cs index 517e6a5..43fd1f4 100644 --- a/src/Ramstack.FileSystem.Amazon/S3Directory.cs +++ b/src/Ramstack.FileSystem.Amazon/S3Directory.cs @@ -48,6 +48,8 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio BucketName = _fs.BucketName }; + dr.Objects ??= []; + do { // The maximum number of objects returned is MaxKeys, which is 1000, @@ -59,8 +61,9 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio .ListObjectsV2Async(lr, cancellationToken) .ConfigureAwait(false); - foreach (var obj in response.S3Objects) - dr.Objects.Add(new KeyVersion { Key = obj.Key }); + if (response.S3Objects is not null) + foreach (var obj in response.S3Objects) + dr.Objects.Add(new KeyVersion { Key = obj.Key }); if (dr.Objects.Count != 0) await _fs.AmazonClient @@ -89,11 +92,13 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync([En .ListObjectsV2Async(request, cancellationToken) .ConfigureAwait(false); - foreach (var prefix in response.CommonPrefixes) - yield return new S3Directory(_fs, VirtualPath.Normalize(prefix)); + if (response.CommonPrefixes is not null) + foreach (var prefix in response.CommonPrefixes) + yield return new S3Directory(_fs, VirtualPath.Normalize(prefix)); - foreach (var obj in response.S3Objects) - yield return CreateVirtualFile(obj); + if (response.S3Objects is not null) + foreach (var obj in response.S3Objects) + yield return CreateVirtualFile(obj); request.ContinuationToken = response.NextContinuationToken; } @@ -116,8 +121,9 @@ protected override async IAsyncEnumerable GetFilesCoreAsync([Enumer .ListObjectsV2Async(request, cancellationToken) .ConfigureAwait(false); - foreach (var obj in response.S3Objects) - yield return CreateVirtualFile(obj); + if (response.S3Objects is not null) + foreach (var obj in response.S3Objects) + yield return CreateVirtualFile(obj); request.ContinuationToken = response.NextContinuationToken; } @@ -140,8 +146,9 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs .ListObjectsV2Async(request, cancellationToken) .ConfigureAwait(false); - foreach (var prefix in response.CommonPrefixes) - yield return new S3Directory(_fs, VirtualPath.Normalize(prefix)); + if (response.CommonPrefixes is not null) + foreach (var prefix in response.CommonPrefixes) + yield return new S3Directory(_fs, VirtualPath.Normalize(prefix)); request.ContinuationToken = response.NextContinuationToken; } @@ -179,28 +186,31 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync(str .ListObjectsV2Async(request, cancellationToken) .ConfigureAwait(false); - foreach (var obj in response.S3Objects) + if (response.S3Objects is not null) { - var path = VirtualPath.Normalize(obj.Key); - var directoryPath = VirtualPath.GetDirectoryName(path); - - while (directoryPath.Length != 0 && directories.Add(directoryPath)) + foreach (var obj in response.S3Objects) { - // - // Directories are yielded in reverse order (deepest first). - // - // Note: We could use a Stack to control the order, - // but since order isn't guaranteed anyway and to avoid - // unnecessary memory allocation, we process them directly. - // - if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) - yield return new S3Directory(_fs, directoryPath); - - directoryPath = VirtualPath.GetDirectoryName(directoryPath); + var path = VirtualPath.Normalize(obj.Key); + var directoryPath = VirtualPath.GetDirectoryName(path); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new S3Directory(_fs, directoryPath); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } + + if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(obj, path); } - - if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes)) - yield return CreateVirtualFile(obj, path); } request.ContinuationToken = response.NextContinuationToken; @@ -234,9 +244,10 @@ protected override async IAsyncEnumerable GetFilesCoreAsync(string[ .ListObjectsV2Async(request, cancellationToken) .ConfigureAwait(false); - foreach (var obj in response.S3Objects) - if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes)) - yield return CreateVirtualFile(obj); + if (response.S3Objects is not null) + foreach (var obj in response.S3Objects) + if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes)) + yield return CreateVirtualFile(obj); request.ContinuationToken = response.NextContinuationToken; } @@ -274,24 +285,27 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs .ListObjectsV2Async(request, cancellationToken) .ConfigureAwait(false); - foreach (var obj in response.S3Objects) + if (response.S3Objects is not null) { - var directoryPath = VirtualPath.GetDirectoryName( - VirtualPath.Normalize(obj.Key)); - - while (directoryPath.Length != 0 && directories.Add(directoryPath)) + foreach (var obj in response.S3Objects) { - // - // Directories are yielded in reverse order (deepest first). - // - // Note: We could use a Stack to control the order, - // but since order isn't guaranteed anyway and to avoid - // unnecessary memory allocation, we process them directly. - // - if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) - yield return new S3Directory(_fs, directoryPath); - - directoryPath = VirtualPath.GetDirectoryName(directoryPath); + var directoryPath = VirtualPath.GetDirectoryName( + VirtualPath.Normalize(obj.Key)); + + while (directoryPath.Length != 0 && directories.Add(directoryPath)) + { + // + // Directories are yielded in reverse order (deepest first). + // + // Note: We could use a Stack to control the order, + // but since order isn't guaranteed anyway and to avoid + // unnecessary memory allocation, we process them directly. + // + if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes)) + yield return new S3Directory(_fs, directoryPath); + + directoryPath = VirtualPath.GetDirectoryName(directoryPath); + } } } @@ -314,8 +328,8 @@ private S3File CreateVirtualFile(S3Object obj, string? normalizedName = null) .CreateFileProperties( creationTime: default, lastAccessTime: default, - lastWriteTime: obj.LastModified, - length: obj.Size); + lastWriteTime: obj.LastModified.GetValueOrDefault(), + length: obj.Size ?? 0); var path = normalizedName ?? VirtualPath.Normalize(obj.Key); return new S3File(_fs, path, properties); diff --git a/src/Ramstack.FileSystem.Amazon/S3File.cs b/src/Ramstack.FileSystem.Amazon/S3File.cs index cdca028..1420a7b 100644 --- a/src/Ramstack.FileSystem.Amazon/S3File.cs +++ b/src/Ramstack.FileSystem.Amazon/S3File.cs @@ -47,7 +47,7 @@ public S3File(AmazonS3FileSystem fileSystem, string path, VirtualNodeProperties? return VirtualNodeProperties.CreateFileProperties( creationTime: default, lastAccessTime: default, - lastWriteTime: metadata.LastModified, + lastWriteTime: metadata.LastModified.GetValueOrDefault(), length: metadata.ContentLength); } catch (AmazonS3Exception e) when (e.StatusCode == HttpStatusCode.NotFound) diff --git a/src/Ramstack.FileSystem.Azure/AzureDirectory.cs b/src/Ramstack.FileSystem.Azure/AzureDirectory.cs index f14a92c..59c3a74 100644 --- a/src/Ramstack.FileSystem.Azure/AzureDirectory.cs +++ b/src/Ramstack.FileSystem.Azure/AzureDirectory.cs @@ -47,6 +47,8 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio var collection = _fs.AzureClient .GetBlobsAsync( prefix: GetPrefix(FullName), + traits: BlobTraits.None, + states: BlobStates.None, cancellationToken: cancellationToken); var client = _fs.AzureClient.GetBlobBatchClient(); @@ -105,6 +107,8 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync([En .GetBlobsByHierarchyAsync( delimiter: "/", prefix: GetPrefix(FullName), + traits: BlobTraits.None, + states: BlobStates.None, cancellationToken: cancellationToken); await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -121,6 +125,8 @@ protected override async IAsyncEnumerable GetFilesCoreAsync([Enumer .GetBlobsByHierarchyAsync( delimiter: "/", prefix: GetPrefix(FullName), + traits: BlobTraits.None, + states: BlobStates.None, cancellationToken: cancellationToken); await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -136,6 +142,8 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs .GetBlobsByHierarchyAsync( delimiter: "/", prefix: GetPrefix(FullName), + traits: BlobTraits.None, + states: BlobStates.None, cancellationToken: cancellationToken); await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -162,7 +170,11 @@ protected override async IAsyncEnumerable GetFileNodesCoreAsync(str var directories = new HashSet { FullName }; await foreach (var page in _fs.AzureClient - .GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken) + .GetBlobsAsync( + prefix: prefix, + traits: BlobTraits.None, + states: BlobStates.None, + cancellationToken: cancellationToken) .AsPages() .WithCancellation(cancellationToken) .ConfigureAwait(false)) @@ -210,7 +222,11 @@ protected override async IAsyncEnumerable GetFilesCoreAsync(string[ var prefix = GetPrefix(FullName); await foreach (var page in _fs.AzureClient - .GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken) + .GetBlobsAsync( + prefix: prefix, + traits: BlobTraits.None, + states: BlobStates.None, + cancellationToken: cancellationToken) .AsPages() .WithCancellation(cancellationToken) .ConfigureAwait(false)) @@ -239,7 +255,11 @@ protected override async IAsyncEnumerable GetDirectoriesCoreAs var directories = new HashSet { FullName }; await foreach (var page in _fs.AzureClient - .GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken) + .GetBlobsAsync( + prefix: prefix, + traits: BlobTraits.None, + states: BlobStates.None, + cancellationToken: cancellationToken) .AsPages() .WithCancellation(cancellationToken) .ConfigureAwait(false)) diff --git a/src/Ramstack.FileSystem.Azure/Ramstack.FileSystem.Azure.csproj b/src/Ramstack.FileSystem.Azure/Ramstack.FileSystem.Azure.csproj index bf5aee4..1462936 100644 --- a/src/Ramstack.FileSystem.Azure/Ramstack.FileSystem.Azure.csproj +++ b/src/Ramstack.FileSystem.Azure/Ramstack.FileSystem.Azure.csproj @@ -1,7 +1,7 @@ - net6.0 + net6.0;net8.0 Provides an implementation of Ramstack.FileSystem based on Azure Blob Storage. enable enable @@ -40,9 +40,17 @@ - + + + + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Ramstack.FileSystem.Azure.Tests/Ramstack.FileSystem.Azure.Tests.csproj b/tests/Ramstack.FileSystem.Azure.Tests/Ramstack.FileSystem.Azure.Tests.csproj index de1581a..1365519 100644 --- a/tests/Ramstack.FileSystem.Azure.Tests/Ramstack.FileSystem.Azure.Tests.csproj +++ b/tests/Ramstack.FileSystem.Azure.Tests/Ramstack.FileSystem.Azure.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net6.0;net8.0 enable enable preview diff --git a/tests/Ramstack.FileSystem.Azure.Tests/ReadonlyAzureFileSystemTests.cs b/tests/Ramstack.FileSystem.Azure.Tests/ReadonlyAzureFileSystemTests.cs index e3551a5..8d51ba2 100644 --- a/tests/Ramstack.FileSystem.Azure.Tests/ReadonlyAzureFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Azure.Tests/ReadonlyAzureFileSystemTests.cs @@ -7,7 +7,8 @@ namespace Ramstack.FileSystem.Azure; [Category("Cloud:Azure")] public class ReadonlyAzureFileSystemTests : VirtualFileSystemSpecificationTests { - private readonly TempFileStorage _storage = new TempFileStorage(); + private readonly TempFileStorage _storage = new(); + private readonly string _storageName = Guid.NewGuid().ToString("N"); [OneTimeSetUp] public async Task Setup() @@ -38,11 +39,11 @@ protected override IVirtualFileSystem GetFileSystem() => protected override DirectoryInfo GetDirectoryInfo() => new DirectoryInfo(_storage.Root); - private static AzureFileSystem CreateFileSystem(bool isReadonly) + private AzureFileSystem CreateFileSystem(bool isReadonly) { const string ConnectionString = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"; - return new AzureFileSystem(ConnectionString, "storage") + return new AzureFileSystem(ConnectionString, _storageName) { IsReadOnly = isReadonly }; diff --git a/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs b/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs index 8a81080..a411b71 100644 --- a/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Azure.Tests/WritableAzureFileSystemTests.cs @@ -9,7 +9,8 @@ namespace Ramstack.FileSystem.Azure; public class WritableAzureFileSystemTests : VirtualFileSystemSpecificationTests { private readonly HashSet _list = []; - private readonly TempFileStorage _storage = new TempFileStorage(); + private readonly TempFileStorage _storage = new(); + private readonly string _storageName = Guid.NewGuid().ToString("N"); [OneTimeSetUp] public async Task Setup() @@ -124,7 +125,7 @@ await fs.GetFilesAsync("/temp").AnyAsync(), } protected override AzureFileSystem GetFileSystem() => - CreateFileSystem("storage"); + CreateFileSystem(_storageName); protected override DirectoryInfo GetDirectoryInfo() => new DirectoryInfo(_storage.Root); diff --git a/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs b/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs index 261e911..6fe037e 100644 --- a/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Google.Tests/ReadonlyGoogleFileSystemTests.cs @@ -43,8 +43,10 @@ protected override DirectoryInfo GetDirectoryInfo() => private static GoogleFileSystem CreateFileSystem(bool isReadOnly) { + #pragma warning disable CS0618 // Type or member is obsolete var client = StorageClient.Create( GoogleCredential.FromFile("credentials.json")); + #pragma warning restore CS0618 // Type or member is obsolete return new GoogleFileSystem(client, "ramstack-test-bucket") { diff --git a/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs b/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs index da61f77..e4c2894 100644 --- a/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs +++ b/tests/Ramstack.FileSystem.Google.Tests/WritableGoogleFileSystemTests.cs @@ -204,8 +204,10 @@ private GoogleFileSystem CreateFileSystem(string bucket) { _buckets.Add(bucket); + #pragma warning disable CS0618 // Type or member is obsolete var client = StorageClient.Create( GoogleCredential.FromFile("credentials.json")); + #pragma warning restore CS0618 // Type or member is obsolete return new GoogleFileSystem(client, bucketName: bucket); }