Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,18 @@ jobs:
azure:
name: "Test: AzureFileSystem"
runs-on: ubuntu-latest
services:
azurite:
image: mcr.microsoft.com/azure-storage/azurite
ports:
- 10000:10000
- 10001:10001
- 10002:10002
steps:
- name: Start Azurite
run: |
docker run -d \
-p 10000:10000 \
-p 10001:10001 \
-p 10002:10002 \
mcr.microsoft.com/azure-storage/azurite \
azurite --skipApiVersionCheck \
--blobHost 0.0.0.0 \
--queueHost 0.0.0.0 \
--tableHost 0.0.0.0
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
Expand All @@ -58,7 +62,7 @@ jobs:
- name: Build
run: dotnet build -c Debug
- name: Test
run: dotnet test --no-build --filter TestCategory=Cloud:Azure
run: dotnet test --no-build --filter TestCategory=Cloud:Azure -maxcpucount:1

s3:
name: "Test: S3FileSystem"
Expand Down
14 changes: 7 additions & 7 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="AWSSDK.S3" Version="3.7.402.11" />
<PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
<PackageVersion Include="Azure.Storage.Blobs.Batch" Version="12.18.1" />
<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.10.0" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="6.0.0" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="6.0.0" />
<PackageVersion Include="AWSSDK.S3" Version="4.0.18.6" />
<PackageVersion Include="Azure.Storage.Blobs" Version="12.27.0" />
<PackageVersion Include="Azure.Storage.Blobs.Batch" Version="12.24.0" />
<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.14.0" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="6.0.1" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="6.0.1" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<PackageVersion Include="MinVer" Version="6.0.0" />
Expand All @@ -18,4 +18,4 @@
<PackageVersion Include="Ramstack.Globbing" Version="2.3.3" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
</Project>
</Project>
1 change: 1 addition & 0 deletions Ramstack.FileSystem.slnx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<File Path=".gitignore" />
<File Path="Directory.Build.props" />
<File Path="Directory.Packages.props" />
<File Path="docker-compose.yml" />
<File Path="LICENSE" />
<File Path="README.md" />
</Folder>
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ services:
- "10000:10000"
- "10001:10001"
- "10002:10002"
command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0
command: azurite --skipApiVersionCheck --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0

rustfs:
image: docker.io/rustfs/rustfs:latest
Expand Down
114 changes: 64 additions & 50 deletions src/Ramstack.FileSystem.Amazon/S3Directory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio
BucketName = _fs.BucketName
};

dr.Objects ??= [];

do
{
// The maximum number of objects returned is MaxKeys, which is 1000,
Expand All @@ -59,8 +61,9 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio
.ListObjectsV2Async(lr, cancellationToken)
.ConfigureAwait(false);

foreach (var obj in response.S3Objects)
dr.Objects.Add(new KeyVersion { Key = obj.Key });
if (response.S3Objects is not null)
foreach (var obj in response.S3Objects)
dr.Objects.Add(new KeyVersion { Key = obj.Key });

if (dr.Objects.Count != 0)
await _fs.AmazonClient
Expand Down Expand Up @@ -89,11 +92,13 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync([En
.ListObjectsV2Async(request, cancellationToken)
.ConfigureAwait(false);

foreach (var prefix in response.CommonPrefixes)
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));
if (response.CommonPrefixes is not null)
foreach (var prefix in response.CommonPrefixes)
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));

foreach (var obj in response.S3Objects)
yield return CreateVirtualFile(obj);
if (response.S3Objects is not null)
foreach (var obj in response.S3Objects)
yield return CreateVirtualFile(obj);

request.ContinuationToken = response.NextContinuationToken;
}
Expand All @@ -116,8 +121,9 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync([Enumer
.ListObjectsV2Async(request, cancellationToken)
.ConfigureAwait(false);

foreach (var obj in response.S3Objects)
yield return CreateVirtualFile(obj);
if (response.S3Objects is not null)
foreach (var obj in response.S3Objects)
yield return CreateVirtualFile(obj);

request.ContinuationToken = response.NextContinuationToken;
}
Expand All @@ -140,8 +146,9 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
.ListObjectsV2Async(request, cancellationToken)
.ConfigureAwait(false);

foreach (var prefix in response.CommonPrefixes)
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));
if (response.CommonPrefixes is not null)
foreach (var prefix in response.CommonPrefixes)
yield return new S3Directory(_fs, VirtualPath.Normalize(prefix));

request.ContinuationToken = response.NextContinuationToken;
}
Expand Down Expand Up @@ -179,28 +186,31 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync(str
.ListObjectsV2Async(request, cancellationToken)
.ConfigureAwait(false);

foreach (var obj in response.S3Objects)
if (response.S3Objects is not null)
{
var path = VirtualPath.Normalize(obj.Key);
var directoryPath = VirtualPath.GetDirectoryName(path);

while (directoryPath.Length != 0 && directories.Add(directoryPath))
foreach (var obj in response.S3Objects)
{
//
// Directories are yielded in reverse order (deepest first).
//
// Note: We could use a Stack<string> to control the order,
// but since order isn't guaranteed anyway and to avoid
// unnecessary memory allocation, we process them directly.
//
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
yield return new S3Directory(_fs, directoryPath);

directoryPath = VirtualPath.GetDirectoryName(directoryPath);
var path = VirtualPath.Normalize(obj.Key);
var directoryPath = VirtualPath.GetDirectoryName(path);

while (directoryPath.Length != 0 && directories.Add(directoryPath))
{
//
// Directories are yielded in reverse order (deepest first).
//
// Note: We could use a Stack<string> to control the order,
// but since order isn't guaranteed anyway and to avoid
// unnecessary memory allocation, we process them directly.
//
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
yield return new S3Directory(_fs, directoryPath);

directoryPath = VirtualPath.GetDirectoryName(directoryPath);
}

if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
yield return CreateVirtualFile(obj, path);
}

if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
yield return CreateVirtualFile(obj, path);
}

request.ContinuationToken = response.NextContinuationToken;
Expand Down Expand Up @@ -234,9 +244,10 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync(string[
.ListObjectsV2Async(request, cancellationToken)
.ConfigureAwait(false);

foreach (var obj in response.S3Objects)
if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
yield return CreateVirtualFile(obj);
if (response.S3Objects is not null)
foreach (var obj in response.S3Objects)
if (IsMatched(obj.Key.AsSpan(request.Prefix.Length), patterns, excludes))
yield return CreateVirtualFile(obj);

request.ContinuationToken = response.NextContinuationToken;
}
Expand Down Expand Up @@ -274,24 +285,27 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
.ListObjectsV2Async(request, cancellationToken)
.ConfigureAwait(false);

foreach (var obj in response.S3Objects)
if (response.S3Objects is not null)
{
var directoryPath = VirtualPath.GetDirectoryName(
VirtualPath.Normalize(obj.Key));

while (directoryPath.Length != 0 && directories.Add(directoryPath))
foreach (var obj in response.S3Objects)
{
//
// Directories are yielded in reverse order (deepest first).
//
// Note: We could use a Stack<string> to control the order,
// but since order isn't guaranteed anyway and to avoid
// unnecessary memory allocation, we process them directly.
//
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
yield return new S3Directory(_fs, directoryPath);

directoryPath = VirtualPath.GetDirectoryName(directoryPath);
var directoryPath = VirtualPath.GetDirectoryName(
VirtualPath.Normalize(obj.Key));

while (directoryPath.Length != 0 && directories.Add(directoryPath))
{
//
// Directories are yielded in reverse order (deepest first).
//
// Note: We could use a Stack<string> to control the order,
// but since order isn't guaranteed anyway and to avoid
// unnecessary memory allocation, we process them directly.
//
if (IsMatched(directoryPath.AsSpan(FullName.Length), patterns, excludes))
yield return new S3Directory(_fs, directoryPath);

directoryPath = VirtualPath.GetDirectoryName(directoryPath);
}
}
}

Expand All @@ -314,8 +328,8 @@ private S3File CreateVirtualFile(S3Object obj, string? normalizedName = null)
.CreateFileProperties(
creationTime: default,
lastAccessTime: default,
lastWriteTime: obj.LastModified,
length: obj.Size);
lastWriteTime: obj.LastModified.GetValueOrDefault(),
length: obj.Size ?? 0);

var path = normalizedName ?? VirtualPath.Normalize(obj.Key);
return new S3File(_fs, path, properties);
Expand Down
2 changes: 1 addition & 1 deletion src/Ramstack.FileSystem.Amazon/S3File.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public S3File(AmazonS3FileSystem fileSystem, string path, VirtualNodeProperties?
return VirtualNodeProperties.CreateFileProperties(
creationTime: default,
lastAccessTime: default,
lastWriteTime: metadata.LastModified,
lastWriteTime: metadata.LastModified.GetValueOrDefault(),
length: metadata.ContentLength);
}
catch (AmazonS3Exception e) when (e.StatusCode == HttpStatusCode.NotFound)
Expand Down
26 changes: 23 additions & 3 deletions src/Ramstack.FileSystem.Azure/AzureDirectory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ protected override async ValueTask DeleteCoreAsync(CancellationToken cancellatio
var collection = _fs.AzureClient
.GetBlobsAsync(
prefix: GetPrefix(FullName),
traits: BlobTraits.None,
states: BlobStates.None,
cancellationToken: cancellationToken);

var client = _fs.AzureClient.GetBlobBatchClient();
Expand Down Expand Up @@ -105,6 +107,8 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync([En
.GetBlobsByHierarchyAsync(
delimiter: "/",
prefix: GetPrefix(FullName),
traits: BlobTraits.None,
states: BlobStates.None,
cancellationToken: cancellationToken);

await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false))
Expand All @@ -121,6 +125,8 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync([Enumer
.GetBlobsByHierarchyAsync(
delimiter: "/",
prefix: GetPrefix(FullName),
traits: BlobTraits.None,
states: BlobStates.None,
cancellationToken: cancellationToken);

await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false))
Expand All @@ -136,6 +142,8 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
.GetBlobsByHierarchyAsync(
delimiter: "/",
prefix: GetPrefix(FullName),
traits: BlobTraits.None,
states: BlobStates.None,
cancellationToken: cancellationToken);

await foreach (var page in collection.AsPages().WithCancellation(cancellationToken).ConfigureAwait(false))
Expand All @@ -162,7 +170,11 @@ protected override async IAsyncEnumerable<VirtualNode> GetFileNodesCoreAsync(str
var directories = new HashSet<string> { FullName };

await foreach (var page in _fs.AzureClient
.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken)
.GetBlobsAsync(
prefix: prefix,
traits: BlobTraits.None,
states: BlobStates.None,
cancellationToken: cancellationToken)
.AsPages()
.WithCancellation(cancellationToken)
.ConfigureAwait(false))
Expand Down Expand Up @@ -210,7 +222,11 @@ protected override async IAsyncEnumerable<VirtualFile> GetFilesCoreAsync(string[
var prefix = GetPrefix(FullName);

await foreach (var page in _fs.AzureClient
.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken)
.GetBlobsAsync(
prefix: prefix,
traits: BlobTraits.None,
states: BlobStates.None,
cancellationToken: cancellationToken)
.AsPages()
.WithCancellation(cancellationToken)
.ConfigureAwait(false))
Expand Down Expand Up @@ -239,7 +255,11 @@ protected override async IAsyncEnumerable<VirtualDirectory> GetDirectoriesCoreAs
var directories = new HashSet<string> { FullName };

await foreach (var page in _fs.AzureClient
.GetBlobsAsync(prefix: prefix, cancellationToken: cancellationToken)
.GetBlobsAsync(
prefix: prefix,
traits: BlobTraits.None,
states: BlobStates.None,
cancellationToken: cancellationToken)
.AsPages()
.WithCancellation(cancellationToken)
.ConfigureAwait(false))
Expand Down
12 changes: 10 additions & 2 deletions src/Ramstack.FileSystem.Azure/Ramstack.FileSystem.Azure.csproj
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
<Description>Provides an implementation of Ramstack.FileSystem based on Azure Blob Storage.</Description>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
Expand Down Expand Up @@ -40,9 +40,17 @@
</Compile>
</ItemGroup>

<ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net6.0'">
<PackageReference Include="Azure.Storage.Blobs" VersionOverride="12.21.2" />
<PackageReference Include="Azure.Storage.Blobs.Batch" VersionOverride="12.18.1" />
</ItemGroup>

<ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
<PackageReference Include="Azure.Storage.Blobs" />
<PackageReference Include="Azure.Storage.Blobs.Batch" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.SourceLink.GitHub">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
Expand Down
Loading
Loading