Skip to content

Commit 1a2af0c

Browse files
mocsharpmocsharp
authored
gh-347 Remove incomplete payloads on timeout (#348)
* gh-347 Remove incomplete payloads on timeout Signed-off-by: Victor Chang <vicchang@nvidia.com> * gh-347 Update unit test Signed-off-by: Victor Chang <vicchang@nvidia.com> * gh-347 Update integration tests Signed-off-by: Victor Chang <vicchang@nvidia.com> * gh-347 Fix unit test Signed-off-by: Victor Chang <vicchang@nvidia.com> * gh-347 Fix integration test Signed-off-by: Victor Chang <vicchang@nvidia.com> --------- Signed-off-by: Victor Chang <vicchang@nvidia.com>
1 parent 4906d18 commit 1a2af0c

File tree

18 files changed

+71
-51
lines changed

18 files changed

+71
-51
lines changed

‎src/Api/MonaiApplicationEntity.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2021-2022 MONAI Consortium
2+
* Copyright 2021-2023 MONAI Consortium
33
* Copyright 2019-2021 NVIDIA Corporation
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +15,6 @@
1515
* limitations under the License.
1616
*/
1717

18-
using System;
1918
using System.Collections.Generic;
2019
using System.ComponentModel.DataAnnotations;
2120
using System.ComponentModel.DataAnnotations.Schema;

‎src/Api/Storage/Payload.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2021-2022 MONAI Consortium
2+
* Copyright 2021-2023 MONAI Consortium
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.

‎src/Configuration/DicomWebConfiguration.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2021-2022 MONAI Consortium
2+
* Copyright 2021-2023 MONAI Consortium
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -55,7 +55,8 @@ public class DicomWebConfiguration
5555
/// single POST request, therefore, the timeout value may be insignificant unless the load of the
5656
/// network affects the upload speed.
5757
/// </summary>
58-
public uint Timeout { get; set; } = 2;
58+
[ConfigurationKeyName("timeout")]
59+
public uint Timeout { get; set; } = 10;
5960

6061
public DicomWebConfiguration()
6162
{

‎src/InformaticsGateway/Logging/Log.3000.PayloadAssembler.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public static partial class Log
5454
[LoggerMessage(EventId = 3012, Level = LogLevel.Information, Message = "Bucket {key} created with timeout {timeout}s.")]
5555
public static partial void BucketCreated(this ILogger logger, string key, uint timeout);
5656

57-
[LoggerMessage(EventId = 3014, Level = LogLevel.Error, Message = "Payload deleted due to upload failure(s) {key}.")]
58-
public static partial void PayloadRemovedWithFailureUploads(this ILogger logger, string key);
57+
[LoggerMessage(EventId = 3014, Level = LogLevel.Error, Message = "Payload ({key}) with {totalNumberOfFiles} files deleted due to {failures} upload failure(s).")]
58+
public static partial void PayloadRemovedWithFailureUploads(this ILogger logger, string key, int totalNumberOfFiles, int failures);
5959
}
6060
}

‎src/InformaticsGateway/Services/Connectors/PayloadAssembler.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2021-2022 MONAI Consortium
2+
* Copyright 2021-2023 MONAI Consortium
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
1717
using System;
1818
using System.Collections.Concurrent;
1919
using System.Collections.Generic;
20+
using System.Linq;
2021
using System.Threading;
2122
using System.Threading.Tasks;
2223
using Ardalis.GuardClauses;
@@ -149,10 +150,10 @@ private async void OnTimedEvent(Object source, System.Timers.ElapsedEventArgs e)
149150
_logger.BucketRemoveError(key);
150151
}
151152
}
152-
else if (payload.AnyUploadFailures())
153+
else if (payload.IsUploadCompletedWithFailures())
153154
{
154155
_payloads.TryRemove(key, out _);
155-
_logger.PayloadRemovedWithFailureUploads(key);
156+
_logger.PayloadRemovedWithFailureUploads(key, payload.Count, payload.Files.Count(p => p.IsUploadFailed));
156157
}
157158
}
158159
}

‎src/InformaticsGateway/Services/Connectors/PayloadExtensions.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2022 MONAI Consortium
2+
* Copyright 2022-2023 MONAI Consortium
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -28,9 +28,9 @@ public static bool IsUploadCompleted(this Payload payload)
2828
return payload.Files.All(p => p.IsUploaded);
2929
}
3030

31-
public static bool AnyUploadFailures(this Payload payload)
31+
public static bool IsUploadCompletedWithFailures(this Payload payload)
3232
{
33-
return payload.Files.Any(p => p.IsUploadFailed);
33+
return payload.Files.Count(p => p.IsUploadFailed) + payload.Files.Count(p => p.IsUploaded) == payload.Count; ;
3434
}
3535

3636
public static bool IsMoveCompleted(this Payload payload)

‎src/InformaticsGateway/Services/Http/InferenceController.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2021-2022 MONAI Consortium
2+
* Copyright 2021-2023 MONAI Consortium
33
* Copyright 2019-2021 NVIDIA Corporation
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -90,7 +90,6 @@ public async Task<ActionResult> NewInferenceRequest([FromBody] InferenceRequest
9090
using var _ = _logger.BeginScope(new LoggingDataDictionary<string, object> { { "TransactionId", request.TransactionId } });
9191
try
9292
{
93-
9493
if (await _inferenceRequestRepository.ExistsAsync(request.TransactionId, HttpContext.RequestAborted).ConfigureAwait(false))
9594
{
9695
return Problem(title: "Conflict", statusCode: (int)HttpStatusCode.Conflict, detail: "An existing request with same transaction ID already exists.");

‎src/InformaticsGateway/Services/Storage/ObjectUploadService.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2021-2022 MONAI Consortium
2+
* Copyright 2021-2023 MONAI Consortium
33
* Copyright 2019-2021 NVIDIA Corporation
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -103,7 +103,7 @@ private async Task StartWorker(int thread, CancellationToken cancellationToken)
103103
{
104104
try
105105
{
106-
var item = await _uplaodQueue.Dequeue(cancellationToken);
106+
var item = await _uplaodQueue.Dequeue(cancellationToken).ConfigureAwait(false);
107107
await ProcessObject(item).ConfigureAwait(false);
108108
}
109109
catch (OperationCanceledException ex)
@@ -164,6 +164,7 @@ private async Task ProcessObject(FileStorageMetadata blob)
164164
}
165165
catch (Exception ex)
166166
{
167+
blob.SetFailed();
167168
_logger.FailedToUploadFile(blob.Id, ex);
168169
}
169170
finally

‎src/InformaticsGateway/Test/Services/Connectors/PayloadAssemblerTest.cs‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,18 +116,23 @@ public async Task GivenAPayloadAssembler_WhenDisposed_ExpectResourceToBeCleanedU
116116
}
117117

118118
[RetryFact(10, 200)]
119-
public async Task GivenAPayloadThatHasNotCompleteUploads_WhenProcessedByTimedEvent_ExpectToBeAddedToQueue()
119+
public async Task GivenAPayloadThatHasNotCompleteUploads_WhenProcessedByTimedEvent_ExpectToBeRemovedFromQueue()
120120
{
121121
var payloadAssembler = new PayloadAssembler(_options, _logger.Object, _serviceScopeFactory.Object);
122122

123-
var file = new TestStorageInfo(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "file1", ".txt");
124-
file.File.SetUploaded("bucket");
123+
var file1 = new TestStorageInfo(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "file1", ".txt");
124+
var file2 = new TestStorageInfo(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "file1", ".txt");
125125

126-
await payloadAssembler.Queue("A", file, 1);
126+
await payloadAssembler.Queue("A", file1, 1);
127+
await payloadAssembler.Queue("A", file2, 1);
128+
129+
file1.SetFailed();
130+
file2.SetUploaded();
127131
await Task.Delay(1001);
128132
payloadAssembler.Dispose();
129133

130-
_repository.Verify(p => p.UpdateAsync(It.Is<Payload>(p => p.State == Payload.PayloadState.Move), It.IsAny<CancellationToken>()), Times.Once());
134+
_repository.Verify(p => p.UpdateAsync(It.Is<Payload>(p => p.State == Payload.PayloadState.Move), It.IsAny<CancellationToken>()), Times.Never());
135+
_logger.VerifyLoggingMessageBeginsWith($"Payload (A) with 2 files deleted due to 1 upload failure(s).", LogLevel.Error, Times.Once());
131136
}
132137

133138
[RetryFact(10, 200)]

‎src/InformaticsGateway/appsettings.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,4 @@
112112
"InformaticsGatewayServerEndpoint": "http://localhost:5000",
113113
"DockerImagePrefix": "ghcr.io/project-monai/monai-deploy-informatics-gateway"
114114
}
115-
}
115+
}

0 commit comments

Comments
 (0)