From 877a7e41fd092a67ddfdc8688e4b73ae90c5b8b2 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Mon, 28 Sep 2026 21:52:09 +0200 Subject: [PATCH 01/15] feat: async chunk upload, dynamic part sizing for multi-TB uploads, and RUFH compliance improvements --- .gitignore | 1 + AGENTS.md | 1 + README.md | 9 +- docs/AZURE_BLOB_STORAGE.md | 14 +- docs/CONFORMITY_TESTING.md | 149 +- docs/S3_STORAGE.md | 15 +- scripts/tus_conformity_test.py | 1227 +++++++++++++++++ .../tus/server/TusFileUploadService.java | 23 + .../RufhAppendPatchRequestHandler.java | 7 + .../RufhCreationPostRequestHandler.java | 14 +- .../rufh/util/RufhInterimResponseUtil.java | 11 +- .../server/upload/UploadStorageService.java | 18 + .../azure/AzureBlobConcatenationService.java | 41 +- .../upload/azure/AzureBlobStorageService.java | 193 ++- .../tus/server/upload/azure/AzureUtils.java | 6 +- ...adLocalCachedStorageAndLockingService.java | 10 + .../upload/disk/DiskStorageService.java | 41 +- .../server/upload/s3/S3StorageService.java | 480 +++++-- .../upload/util/AsyncChunkUploader.java | 290 ++++ .../tus/server/AbstractITRufhProtocol.java | 89 ++ .../tus/server/TusFileUploadServiceTest.java | 16 + .../RufhAppendPatchRequestHandlerTest.java | 90 ++ .../RufhCreationPostRequestHandlerTest.java | 42 + .../util/RufhInterimResponseUtilTest.java | 23 +- .../upload/UploadStorageServiceTest.java | 3 + .../AzureBlobConcatenationServiceTest.java | 27 + .../azure/AzureBlobStorageServiceTest.java | 42 + .../server/upload/azure/AzureUtilsTest.java | 3 + .../azure/ITAzureBlobStorageService.java | 19 + ...calCachedStorageAndLockingServiceTest.java | 24 + .../upload/disk/DiskStorageServiceTest.java | 17 + .../upload/s3/S3StorageServiceTest.java | 421 ++++++ .../upload/util/AsyncChunkUploaderTest.java | 576 ++++++++ 33 files changed, 3728 insertions(+), 214 deletions(-) create mode 100755 scripts/tus_conformity_test.py create mode 100644 src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java create mode 100644 src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java diff --git a/.gitignore b/.gitignore index 5a5a0675..5ffdb6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,4 @@ SFTP_STORAGE_ANALYSIS.md NFS_LOCKING.md RELEASE-REVIEW.md ASYNC_REVIEW.md +ASYNC_REVIEW_2.md diff --git a/AGENTS.md b/AGENTS.md index 6ff7fb7d..9a35c4e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -211,6 +211,7 @@ When updating the IETF protocol implementation for a new draft revision, follow ### 4. Conformity Test Suite Maintenance & Subagent Isolation Whenever a new draft revision of the RUFH specification is published, the repository's Python conformity test suite (`scripts/rufh_conformity_test.py`) MUST be reviewed and updated by a separate, dedicated subagent. - **Strict Isolation Rule**: The subagent tasked with updating `scripts/rufh_conformity_test.py` MUST ONLY consult the official IETF specification document (and RFC 9530) and MUST NOT inspect the Java server implementation code under `src/main/java/`. This ensures the conformity test suite remains an independent, unbiased specification benchmark. +- **Default Multi-Backend Execution**: When executing the conformity test suite (`scripts/rufh_conformity_test.py`), tests MUST be run by default against all three supported storage backend types (Disk: `/test/api/upload`, S3: `/test-s3/api/upload`, and Azure Blob: `/test-azure/api/upload`) as documented in [`docs/CONFORMITY_TESTING.md`](docs/CONFORMITY_TESTING.md). ### 5. Conformity Test Suite Audit — Repeatable Procedure Use this procedure to audit `scripts/rufh_conformity_test.py` against the current (or a new) specification revision. The goal is to identify untested MUST/SHOULD/MAY requirements and produce an actionable improvement report. diff --git a/README.md b/README.md index bc082a23..2fe648d2 100644 --- a/README.md +++ b/README.md @@ -191,11 +191,12 @@ After creating the object, you can configure it using the following methods: | `disableTusExtension(String)` | None | Disables a built-in extension (`creation`, `checksum`, `expiration`, `concatenation`, `termination`, `download`, `cors`). | | `withUploadIdFactory(UploadIdFactory)` | `UuidUploadIdFactory` | Custom ID generator for upload resources (e.g., `UuidUploadIdFactory` or `TimeBasedUploadIdFactory`). | | `withUploadCompletionListener(UploadCompletionListener)` | None | Registers a callback invoked immediately when an upload finishes transferring all bytes and is completed. | +| `withCloudUploadThreadPoolSize(int)` | `10` | Maximum number of worker threads used for asynchronous background chunk uploading in cloud storage backends (S3, Azure). | | `withJsonSerialization()` | Java serialization | Enables JSON serialization for upload metadata (`UploadInfo`). Jackson is bundled by default. | | `withUploadStorageService(UploadStorageService)` | `DiskStorageService` | Configures custom or cloud storage backend (`DiskStorageService`, `S3StorageService`, `AzureBlobStorageService`). | | `withUploadLockingService(UploadLockingService)` | `LeaseFileLockingService` | Configures custom or cloud locking backend (`LeaseFileLockingService`, `S3LockingService`, `AzureBlobLockingService`). | -The library provides filesystem-based storage (`DiskStorageService` / `LeaseFileLockingService`), S3-compatible object storage (`S3StorageService` / `S3LockingService`), and Azure Blob Storage (`AzureBlobStorageService` / `AzureBlobLockingService`). See the **[Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md)**, **[S3 Storage Guide](docs/S3_STORAGE.md)**, and **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md)** for detailed instructions on multi-replica container deployments in Kubernetes, post-upload processing, and legacy locking opt-out. +The library provides filesystem-based storage (`DiskStorageService` / `LeaseFileLockingService`), S3-compatible object storage (`S3StorageService` / `S3LockingService`), and Azure Blob Storage (`AzureBlobStorageService` / `AzureBlobLockingService`). Cloud backends feature an asynchronous 3-slot chunk pipeline (`AsyncChunkUploader`) that overlaps client payload streaming with cloud staging in the background. See the **[Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md)**, **[S3 Storage Guide](docs/S3_STORAGE.md)**, and **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md)** for detailed instructions on multi-replica container deployments in Kubernetes, post-upload processing, and legacy locking opt-out. ### 2. Receiving a Resumable Upload To process an upload request you have to pass the current `jakarta.servlet.http.HttpServletRequest` and `jakarta.servlet.http.HttpServletResponse` objects to the `me.desair.tus.server.TusFileUploadService.process()` method. Typical places were you can do this are inside Servlets, Filters or REST API Controllers. @@ -409,10 +410,10 @@ public TomcatServletWebServerFactory tomcatFactory(TusFileUploadService tusFileU ## Compatible Client Implementations & Conformity Testing This server implementation has been tested with: -- **Tus 1.0.0 Clients**: Tested with [Uppy](https://uppy.io/) and `tus-js-client`. -- **IETF Resumable Uploads Clients & Conformity Tests**: The implementation has been thoroughly tested with our own built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH protocol specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). +- **Tus 1.0.0 Clients & Conformity Tests**: Tested with [Uppy](https://uppy.io/), `tus-js-client`, and our built-in Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) validating all core protocol mechanisms and protocol extensions (creation, creation-with-upload, checksum, termination, concatenation, and expiration) across Disk, S3, and Azure Blob backends. +- **IETF Resumable Uploads Clients & Conformity Tests**: The implementation has been thoroughly tested with our built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH protocol specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). -For detailed instructions on running our native conformity test suite and interpreting results, see the **[Conformity Testing Guide (docs/CONFORMITY_TESTING.md)](docs/CONFORMITY_TESTING.md)**. +For detailed instructions on running our native conformity test suites across all storage backends (Disk, S3, Azure Blob) and interpreting results, see the **[Conformity Testing Guide (docs/CONFORMITY_TESTING.md)](docs/CONFORMITY_TESTING.md)**. This repository also contains comprehensive automated integration test suites (`ITTusFileUploadService`, `RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`) validating both protocol specifications. diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index 99101956..7a1fb930 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -157,15 +157,23 @@ BlobContainerClient containerClient = new BlobContainerClientBuilder() --- -## 6. Local Disk Buffer & Block Size Auto-Calibration +## 6. Local Disk Buffer & Asynchronous Chunk Pipelining -`AzureBlobStorageService` streams incoming PATCH payloads in chunks of `optimalBlockSize` into temporary files, staging each block to Azure as it completes. Peak disk usage per upload is capped at `1 × optimalBlockSize` (e.g. 8 MB). +`AzureBlobStorageService` streams incoming PATCH payloads in chunks of `optimalBlockSize` into temporary files, pipelining block staging to Azure Blob Storage asynchronously via `AsyncChunkUploader`: +- **Asynchronous 3-Slot Pipeline**: + - **Slot 1 (Receiving)**: Streaming incoming bytes from the client into a local temporary file. + - **Slot 2 (Waiting)**: Holds one ready chunk on disk buffer. + - **Slot 3 (Uploading)**: Actively staging the block (`stageBlock`) to Azure Blob Storage on a background daemon worker thread. + - Falls back to synchronous caller execution (`CallerRunsPolicy` on a `SynchronousQueue`) when the thread pool is saturated, preventing worker queue latency and unbounded buffering. +- **Thread Pool Sizing**: Worker threads are managed via a shared executor, default 10 threads, configurable on `TusFileUploadService` via `.withCloudUploadThreadPoolSize(int)` or directly on `AzureBlobStorageService.setCloudUploadThreadPoolSize(int)`. +- **Bounded Disk Footprint**: Peak disk usage per active upload is bounded to at most $2 \times \text{optimalBlockSize}$ (one receiving slot + one waiting slot). Block sizes auto-calibrate based on total upload size: -- **Baseline Preferred Size**: 8 MB (configurable via constructor) +- **Baseline Preferred Size**: 8 MB (configurable via constructor or `setPreferredBlockSize(long)`) - **Minimum Block Size**: 4 MB - **Maximum Block Size**: 4000 MiB (Azure limit) - **Maximum Blocks per Blob**: 50,000 (Azure limit) +- **Multi-TB Support & Auto-Calibration**: When upload length exceeds 400 GB ($50,000 \times 8\text{ MB}$), block size automatically scales up proportionally (e.g. ~22 MB for 1 TB, ~110 MB for 5 TB, supporting blobs up to 190 TB) to ensure the upload finishes within Azure's 50,000 blocks ceiling. --- diff --git a/docs/CONFORMITY_TESTING.md b/docs/CONFORMITY_TESTING.md index 0255a0d8..660b9e4f 100644 --- a/docs/CONFORMITY_TESTING.md +++ b/docs/CONFORMITY_TESTING.md @@ -1,17 +1,19 @@ -# Conformity Testing Guide (IETF Resumable Uploads for HTTP) +# Protocol Conformity Testing Guide -This guide describes how to execute the RUFH (Resumable Uploads for HTTP) conformity tests against a locally running instance of the Spring Boot demo server or any RUFH compliant server endpoint. +This guide describes how to execute the automated conformity test suites for both supported resumable upload protocols: +1. **Tus v1.0.0 Protocol**: [`scripts/tus_conformity_test.py`](../scripts/tus_conformity_test.py) validating the official [Tus v1.0.0 Resumable Upload Protocol](https://tus.io/protocols/resumable-upload). +2. **IETF Resumable Uploads for HTTP (RUFH)**: [`scripts/rufh_conformity_test.py`](../scripts/rufh_conformity_test.py) validating [draft-ietf-httpbis-resumable-upload-12](https://www.ietf.org/archive/id/draft-ietf-httpbis-resumable-upload-12.txt) and [RFC 9530 HTTP Digests](https://www.rfc-editor.org/rfc/rfc9530.html). -The test suite validates compliance with [draft-ietf-httpbis-resumable-upload-12](https://www.ietf.org/archive/id/draft-ietf-httpbis-resumable-upload-12.txt) and [RFC 9530 HTTP Digests](https://www.rfc-editor.org/rfc/rfc9530.html). +Both conformity test suites perform end-to-end, socket-level protocol verification against a running server, ensuring strict adherence to HTTP status codes, structured response headers, error semantics, concurrency guarantees, and storage engine consistency. --- ## 1. Build and Install the Server Library -First, compile and install the core `tus-java-server` library to your local Maven repository: +First, compile and install the core `tus-java-server` library into your local Maven repository: ```bash -# In the root of the tus-java-server repository +# In the root directory of the tus-java-server repository mvn clean install -DskipTests ``` @@ -19,7 +21,7 @@ mvn clean install -DskipTests ## 2. Start the Demo Server -1. Verify the dependency in `tus-java-server-spring-demo` project's `spring-boot-rest/pom.xml` points to the snapshot version: +1. Verify that the dependency in `tus-java-server-spring-demo` project's `spring-boot-rest/pom.xml` references the snapshot version: ```xml me.desair.tus @@ -28,64 +30,157 @@ mvn clean install -DskipTests ``` -2. Build and start the Spring Boot REST demo server with a `1 KB` maximum upload size parameter (`--tus.server.max-upload-size=1024`) to enable full limit discovery & limit enforcement verification: +2. Build and start the Spring Boot REST demo server with a `1 KB` maximum upload size limit (`--tus.server.max-upload-size=1024`) to enable full limit discovery & limit enforcement verification: ```bash cd ../tus-java-server-spring-demo mvn clean package -DskipTests java -jar spring-boot-rest/target/spring-boot-rest-0.0.1-SNAPSHOT.jar --tus.server.max-upload-size=1024 ``` - The server will start on port `8080` with the upload endpoint exposed at: - `http://localhost:8080/test/api/upload` + The server will start on port `8080`, exposing upload endpoints for all three storage backends: + - **Disk Storage (Default)**: `http://localhost:8080/test/api/upload` + - **S3 Object Storage**: `http://localhost:8080/test-s3/api/upload` + - **Azure Blob Storage**: `http://localhost:8080/test-azure/api/upload` --- -## 3. Run the Built-In RUFH Conformity Test Suite +## 3. Python Prerequisites -The repository includes its own native Python conformity test suite located at `scripts/rufh_conformity_test.py`. It requires `pytest` and `requests`. - -### Prerequisites -Install Python dependencies if not already installed: +Install required Python test packages if not already available in your environment: ```bash pip install pytest requests ``` -### Running the Test Suite +--- + +## 4. Running the Tus v1.0.0 Conformity Test Suite + +The Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) verifies 50 specification rules covering: +- **Core Protocol (§5 & §6)**: OPTIONS feature discovery, Tus-Resumable version handshake (412 Precondition Failed), HEAD offset retrieval, cache control, PATCH data append, Content-Type enforcement, offset mismatch prevention (409 Conflict), length bounds validation, and `X-HTTP-Method-Override`. +- **Creation Extension (§7.1)**: Upload-Length validation, Upload-Defer-Length handling, defer-to-known length transition, and Upload-Metadata base64 decoding. +- **Creation With Upload Extension (§7.2)**: Single-request POST creations with payload body, partial upload chunking, and content-type enforcement. +- **Expiration Extension (§7.3)**: Upload-Expires header validation formatted in RFC 9110 HTTP datetime format. +- **Checksum Extension (§7.4)**: Upload-Checksum verification (SHA-1), 460 Checksum Mismatch handling and payload discarding, and unsupported checksum algorithm rejection (400 Bad Request). +- **Termination Extension (§7.5)**: DELETE cancellation of in-progress and completed uploads and 404/410 verification. +- **Concatenation Extension (§7.6)**: Partial upload creation, final upload creation by merging partials, offset summation, and 403 Forbidden enforcement on final upload PATCH requests. +- **Concurrency & Workflows**: Concurrent PATCH race contention prevention at identical offset and end-to-end multi-chunk upload flows. + +### Running Against All Three Storage Backends + +The test suite **MUST be run against all three supported storage backend types** (Disk, S3, and Azure Blob) to ensure cross-engine protocol compliance: + +#### Option A: Standalone Python Runner (Custom Formatted Summary) + +```bash +# 1. Disk Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + +# 2. S3 Object Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test-s3/api/upload + +# 3. Azure Blob Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test-azure/api/upload +``` + +Or execute all three backends in a single command loop: +```bash +for endpoint in /test/api/upload /test-s3/api/upload /test-azure/api/upload; do + echo "======================================================================" + echo " Running Tus v1.0.0 Conformity Tests: http://localhost:8080$endpoint" + echo "======================================================================" + python3 scripts/tus_conformity_test.py --url "http://localhost:8080$endpoint" +done +``` + +#### Option B: PyTest Runner + +```bash +# Execute via pytest +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test-s3/api/upload +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test-azure/api/upload +``` + +--- + +## 5. Running the IETF RUFH Conformity Test Suite + +The RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validates compliance with the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`) and RFC 9530 HTTP Digests. -You can execute the test suite using Python directly or via PyTest: +### Running Against All Three Storage Backends -#### Option A: Running directly with Python (Recommended for structured AI / Agent reporting) ```bash +# 1. Disk Storage Backend python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload + +# 2. S3 Object Storage Backend +python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test-s3/api/upload + +# 3. Azure Blob Storage Backend +python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test-azure/api/upload ``` -#### Option B: Running with PyTest +Or via PyTest: ```bash -pytest scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload +for endpoint in /test/api/upload /test-s3/api/upload /test-azure/api/upload; do + echo "======================================================================" + echo " Running PyTest RUFH Conformity: http://localhost:8080$endpoint" + echo "======================================================================" + pytest scripts/rufh_conformity_test.py --url "http://localhost:8080$endpoint" +done ``` --- -## 4. Understanding Test Results & AI Agent Remediation +## 6. Understanding Test Results & AI Agent Remediation -When executed, the script produces a structured summary report detailing: +When executed directly via Python, both conformity test scripts produce a structured summary report detailing: 1. **Total Tests Executed**: Count of total specification compliance tests run. -2. **Passed Tests**: Number of tests matching draft-12 specification requirements. -3. **Failed Tests**: Detailed list of failing tests including test method names, exact error tracebacks, expected status codes/headers, and corresponding RFC section references. -4. **104 Interim Responses**: Count of tests where `HTTP/1.1 104 Upload Resumption Supported` interim responses were detected from the server socket. +2. **Passed Tests**: Number of tests matching specification requirements. +3. **Failed Tests**: Detailed list of failing tests including test method names, exact error tracebacks, expected status codes/headers, and corresponding specification section references. -AI agents and developers can analyze the detailed failure breakdown in the script's console output to pinpoint specific compliance gaps and adjust server logic accordingly. +### Example Output: +``` +====================================================================== + Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite + Specification: https://tus.io/protocols/resumable-upload + Target Endpoint: http://localhost:8080/test/api/upload +====================================================================== +.................................................. [100%] +50 passed in 0.43s + +====================================================================== + CONFORMITY TEST RESULTS +====================================================================== + Total Tests Executed: 50 + Passed: 50 + Failed: 0 +====================================================================== + +[✓] ALL TUS V1.0.0 CONFORMITY TESTS PASSED SUCCESSFULLY! +``` + +If a test fails, the runner outputs a structured failure breakdown containing: +- Test function name and class +- Verbatim specification quote defining the required behavior +- Exact failure assertion and returned HTTP status or header values + +Developers and automated AI agents can use this failure breakdown to immediately diagnose compliance discrepancies and make targeted fixes in the server code. --- -## 5. Running Community (IETF Hackathon) Tests +## 7. Running Community (IETF Hackathon) Tests -Alternatively, you can also run the external community test suite from the `ietf-hackathon` repository: +Alternatively, you can also run the external community test suite from the `ietf-hackathon` repository across all three storage backends: ```bash git clone https://github.com/tus/ietf-hackathon.git cd ietf-hackathon/tests pip install -r requirements.txt + +# Run against Disk, S3, and Azure Blob endpoints pytest --url http://localhost:8080/test/api/upload +pytest --url http://localhost:8080/test-s3/api/upload +pytest --url http://localhost:8080/test-azure/api/upload ``` diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 87a18626..14eff747 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -160,8 +160,19 @@ S3StorageService s3Storage = new S3StorageService(minioClient, "my-bucket"); S3 requires every part chunk of a multipart upload to be at least 5 MB (except the final part). -- **Disk Buffering**: `S3StorageService` buffers incoming bytes to local disk in chunks (default 50 MB) before uploading them to S3. -- **Incomplete Parts**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as a `/.part` object in S3. On the next `PATCH` request, this chunk is downloaded, prepended to the incoming stream, and upload proceeds seamlessly. +- **Disk Buffering & Preferred Part Size**: `S3StorageService` buffers incoming bytes to local disk in chunks of **8 MB** (`DEFAULT_PREFERRED_PART_SIZE`), matching Azure Blob Storage's block size. Preferred part size is configurable via `setPreferredPartSize(long)` (between 5 MB and 5 GB). +- **Multi-TB Support & Dynamic Part Size Auto-Calibration**: AWS S3 enforces a maximum ceiling of 10,000 parts per multipart upload. When an upload length exceeds 80 GB ($10,000 \times 8\text{ MB}$), `S3StorageService` automatically scales the part size up proportionally: + - **1 TB upload**: auto-calibrated to ~105 MB per part. + - **5 TB upload** (S3 maximum limit): auto-calibrated to ~525 MB per part. + - Guarantees the entire upload completes within 10,000 parts without exceeding S3's 5 GB maximum part limit. + - Peak local disk buffer usage remains bounded to at most $2 \times \text{optimalPartSize}$ (one receiving slot + one waiting slot). +- **Asynchronous Chunk Pipelining (`AsyncChunkUploader`)**: Instead of blocking the HTTP client thread while uploading chunks to S3, `S3StorageService` pipelines chunks using a bounded 3-slot model: + - **Slot 1 (Receiving)**: Streaming incoming bytes from the client into a local temporary file. + - **Slot 2 (Waiting)**: Holds one ready chunk on local disk buffer. + - **Slot 3 (Uploading)**: Actively uploading a chunk to S3 on a background daemon worker thread. + - When the background thread pool is fully utilized, work seamlessly falls back to the client thread without blocking (`ThreadPoolExecutor.CallerRunsPolicy` on a `SynchronousQueue`), ensuring zero queuing overhead and immediate backpressure. +- **Thread Pool Sizing**: Worker threads are managed via a shared executor, default 10 threads, configurable on `TusFileUploadService` via `.withCloudUploadThreadPoolSize(int)` or directly on `S3StorageService.setCloudUploadThreadPoolSize(int)`. +- **Incomplete Parts & Stale Buffer Protection**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as a `/.part` object in S3. On subsequent requests, an arithmetic budget guard ($\text{existingPartsTotalSize} + \text{size}(.part) == \text{length}$) verifies whether the `.part` represents a legitimate final part or an obsolete buffer from a previous attempt, preventing file corruption and size inflation. - **Configurable Temp Directory**: The temporary buffer directory can be configured in the constructor or builder: ```java diff --git a/scripts/tus_conformity_test.py b/scripts/tus_conformity_test.py new file mode 100755 index 00000000..78fb3b9c --- /dev/null +++ b/scripts/tus_conformity_test.py @@ -0,0 +1,1227 @@ +#!/usr/bin/env python3 +""" +Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite + +This conformity test suite validates a server implementation against the +official Tus v1.0.0 Resumable Upload Protocol specification: + https://tus.io/protocols/resumable-upload + +Usage: + pytest scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload +""" + +import argparse +import base64 +import email.utils +import hashlib +import os +import socket +import sys +import threading +import time +from urllib.parse import urlparse + +import pytest + +# Protocol Constants +TUS_RESUMABLE = "Tus-Resumable" +TUS_VERSION = "Tus-Version" +TUS_EXTENSION = "Tus-Extension" +TUS_MAX_SIZE = "Tus-Max-Size" +TUS_CHECKSUM_ALGORITHM = "Tus-Checksum-Algorithm" +UPLOAD_OFFSET = "Upload-Offset" +UPLOAD_LENGTH = "Upload-Length" +UPLOAD_DEFER_LENGTH = "Upload-Defer-Length" +UPLOAD_METADATA = "Upload-Metadata" +UPLOAD_CHECKSUM = "Upload-Checksum" +UPLOAD_CONCAT = "Upload-Concat" +UPLOAD_EXPIRES = "Upload-Expires" +LOCATION = "Location" +CONTENT_TYPE = "Content-Type" +CONTENT_LENGTH = "Content-Length" +CACHE_CONTROL = "Cache-Control" +X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override" +APPLICATION_OFFSET_OCTET_STREAM = "application/offset+octet-stream" +TUS_API_VERSION = "1.0.0" + + +def pytest_addoption(parser): + """Add command line options to pytest.""" + parser.addoption( + "--url", + action="store", + default="http://localhost:8080/test/api/upload", + help="Target Tus upload endpoint URL", + ) + + +@pytest.fixture(scope="session") +def target_url(request): + """Fixture providing the target upload URL.""" + try: + return request.config.getoption("--url") + except (ValueError, AttributeError): + return os.environ.get("TUS_URL", "http://localhost:8080/test/api/upload") + + +class CaseInsensitiveDict(dict): + """A case-insensitive dictionary for HTTP headers.""" + + def __init__(self, data=None, **kwargs): + super().__init__() + self._keys = {} + if data: + self.update(data) + if kwargs: + self.update(kwargs) + + def __setitem__(self, key, value): + super().__setitem__(key.lower(), value) + self._keys[key.lower()] = key + + def __getitem__(self, key): + return super().__getitem__(key.lower()) + + def __delitem__(self, key): + super().__delitem__(key.lower()) + del self._keys[key.lower()] + + def __contains__(self, key): + return super().__contains__(key.lower()) + + def get(self, key, default=None): + return super().get(key.lower(), default) + + def update(self, other=None, **kwargs): + if hasattr(other, "items"): + for k, v in other.items(): + self[k] = v + elif other: + for k, v in other: + self[k] = v + for k, v in kwargs.items(): + self[k] = v + + def items(self): + return ((self._keys[k], v) for k, v in super().items()) + + +def parse_headers(lines): + """Parse HTTP header lines into a CaseInsensitiveDict.""" + res_headers = CaseInsensitiveDict() + for line in lines: + if ":" in line: + k, v = line.split(":", 1) + res_headers[k.strip()] = v.strip() + return res_headers + + +def http_request(method, url, headers=None, body=None): + """ + Socket-based HTTP client helper for raw HTTP/1.1 request execution. + Returns (status_code, headers_dict, body_bytes). + """ + if headers is None: + headers = {} + parsed = urlparse(url) + host = parsed.hostname or "localhost" + port = parsed.port or (443 if parsed.scheme == "https" else 80) + path = parsed.path + ("?" + parsed.query if parsed.query else "") + if not path: + path = "/" + + s = socket.create_connection((host, port), timeout=10) + try: + req_headers = CaseInsensitiveDict(headers) + if "Connection" not in req_headers: + req_headers["Connection"] = "close" + + req_lines = [f"{method} {path} HTTP/1.1", f"Host: {host}:{port}"] + for k, v in req_headers.items(): + req_lines.append(f"{k}: {v}") + if body is not None and "Content-Length" not in req_headers: + body_len = len(body) if isinstance(body, bytes) else len(body.encode("utf-8")) + req_lines.append(f"Content-Length: {body_len}") + elif body is None and method in ("POST", "PATCH", "PUT") and "Content-Length" not in req_headers: + req_lines.append("Content-Length: 0") + + req_lines.append("") + req_lines.append("") + req_data = "\r\n".join(req_lines).encode("latin1") + if body: + req_data += body if isinstance(body, bytes) else body.encode("utf-8") + + s.sendall(req_data) + + # Read response data + resp_bytes = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + resp_bytes += chunk + + raw_str = resp_bytes.decode("latin1", errors="replace") + parts = raw_str.split("\r\n\r\n", 1) + headers_part = parts[0] + body_part = parts[1].encode("latin1") if len(parts) > 1 else b"" + + lines = headers_part.split("\r\n") + status_line = lines[0] + status_code = int(status_line.split()[1]) if len(status_line.split()) > 1 else 0 + + res_headers = parse_headers(lines[1:]) + return status_code, res_headers, body_part + except (socket.timeout, ConnectionRefusedError, socket.error) as e: + pytest.fail(f"HTTP request failed: {e}") + finally: + s.close() + + +def create_upload( + target_url, + upload_length="100", + defer_length=False, + metadata=None, + concat=None, + extra_headers=None, + body=None, + content_type=None, +): + """ + Helper to create an upload resource via POST. + Returns (absolute_location_url, response_headers). + """ + headers = {TUS_RESUMABLE: TUS_API_VERSION} + if defer_length: + headers[UPLOAD_DEFER_LENGTH] = "1" + elif upload_length is not None: + headers[UPLOAD_LENGTH] = str(upload_length) + + if metadata: + headers[UPLOAD_METADATA] = metadata + if concat: + headers[UPLOAD_CONCAT] = concat + if content_type: + headers[CONTENT_TYPE] = content_type + if extra_headers: + headers.update(extra_headers) + + status, resp_headers, _ = http_request("POST", target_url, headers=headers, body=body) + assert status == 201, f"Expected 201 Created for upload creation, got {status}" + + loc = resp_headers.get(LOCATION) + assert loc, "Location header MUST be returned upon 201 Created" + if not loc.startswith("http"): + parsed = urlparse(target_url) + loc = f"{parsed.scheme}://{parsed.netloc}{loc}" + return loc, resp_headers + + +class TestCoreProtocol: + """Core Protocol compliance tests (§6).""" + + def test_options_server_configuration(self, target_url): + """ + §6 OPTIONS: Gathering Server Configuration. + Quote: "A successful response indicated by the 204 No Content or 200 OK status MUST contain + the Tus-Version header. It MAY include the Tus-Extension and Tus-Max-Size headers." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204), f"OPTIONS request MUST return 200 or 204, got {status}" + assert TUS_VERSION in resp_headers, "OPTIONS response MUST contain Tus-Version header" + assert TUS_API_VERSION in resp_headers.get(TUS_VERSION, ""), "Tus-Version MUST include 1.0.0" + assert TUS_EXTENSION in resp_headers, "OPTIONS response SHOULD include Tus-Extension header" + + def test_options_ignores_tus_resumable_header(self, target_url): + """ + §6 OPTIONS: Server MUST Ignore Tus-Resumable Header on OPTIONS. + Quote: "The Client SHOULD NOT include the Tus-Resumable header in the request and the + Server MUST ignore the header." + """ + headers = {TUS_RESUMABLE: "unsupported.version.9.9.9"} + status, resp_headers, _ = http_request("OPTIONS", target_url, headers=headers) + assert status in (200, 204), "OPTIONS MUST succeed and ignore Tus-Resumable header" + assert status != 412, "Server MUST NOT respond with 412 on OPTIONS even with invalid Tus-Resumable" + + def test_unsupported_tus_version_returns_412(self, target_url): + """ + §6 Tus-Resumable: Protocol Version Mismatch Returns 412 Precondition Failed. + Quote: "If the version specified by the Client is not supported by the Server, it MUST + respond with the 412 Precondition Failed status and MUST include the Tus-Version header + into the response. In addition, the Server MUST NOT process the request." + """ + headers = { + TUS_RESUMABLE: "0.1.0", + UPLOAD_LENGTH: "100", + } + status, resp_headers, _ = http_request("POST", target_url, headers=headers) + assert status == 412, f"Unsupported version MUST be rejected with 412, got {status}" + assert TUS_VERSION in resp_headers, "412 response MUST include Tus-Version header" + + def test_missing_tus_resumable_header_rejected(self, target_url): + """ + §6 Tus-Resumable: Header Required in Requests. + Quote: "The Tus-Resumable header MUST be included in every request and response except + for OPTIONS requests." + """ + headers = {UPLOAD_LENGTH: "100"} + status, resp_headers, _ = http_request("POST", target_url, headers=headers) + assert status in (400, 412), f"Missing Tus-Resumable header MUST be rejected, got {status}" + + def test_tus_resumable_present_in_responses(self, target_url): + """ + §6 Tus-Resumable: Header Present in Every Response. + Quote: "The Tus-Resumable header MUST be included in every request and response except + for OPTIONS requests. The value MUST be the version of the protocol used by the Client or Server." + """ + upload_url, resp_headers = create_upload(target_url, upload_length="100") + assert resp_headers.get(TUS_RESUMABLE) == TUS_API_VERSION, "POST response MUST contain Tus-Resumable: 1.0.0" + + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(TUS_RESUMABLE) == TUS_API_VERSION, "HEAD response MUST contain Tus-Resumable: 1.0.0" + + def test_head_requires_upload_offset_even_if_zero(self, target_url): + """ + §6 HEAD: Server MUST Always Include Upload-Offset. + Quote: "The Server MUST always include the Upload-Offset header in the response for a HEAD + request, even if the offset is 0, or the upload is already considered completed." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204), f"HEAD request MUST succeed with 200 or 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST be present and 0 initially" + + def test_head_requires_upload_length_when_known(self, target_url): + """ + §6 HEAD: Server MUST Include Upload-Length When Known. + Quote: "If the size of the upload is known, the Server MUST include the Upload-Length + header in the response." + """ + upload_url, _ = create_upload(target_url, upload_length="256") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert resp_headers.get(UPLOAD_LENGTH) == "256", "HEAD response MUST include Upload-Length" + + def test_head_cache_control_no_store(self, target_url): + """ + §6 HEAD: Cache Prevention via Cache-Control: no-store. + Quote: "The Server MUST prevent the client and/or proxies from caching the response by + adding the Cache-Control: no-store header to the response." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + cache_ctrl = resp_headers.get(CACHE_CONTROL, "") + assert "no-store" in cache_ctrl, f"HEAD response MUST include Cache-Control: no-store, got '{cache_ctrl}'" + + def test_head_non_existent_upload_resource(self, target_url): + """ + §6 HEAD: Non-Existent Resource Returns 404, 410, or 403 Without Upload-Offset. + Quote: "If the resource is not found, the Server SHOULD return either the 404 Not Found, + 410 Gone or 403 Forbidden status without the Upload-Offset header." + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + status, resp_headers, _ = http_request("HEAD", non_existent_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (404, 410, 403), f"HEAD on non-existent resource SHOULD return 404, 410, or 403, got {status}" + assert UPLOAD_OFFSET not in resp_headers, "Non-existent HEAD response MUST NOT include Upload-Offset" + + def test_patch_content_type_required(self, target_url): + """ + §6 PATCH: Content-Type MUST Be application/offset+octet-stream. + Quote: "All PATCH requests MUST use Content-Type: application/offset+octet-stream, + otherwise the server SHOULD return a 415 Unsupported Media Type status." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: "text/plain", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 10) + assert status in (400, 406, 415), f"PATCH with incorrect Content-Type MUST be rejected, got {status}" + + def test_patch_offset_mismatch_returns_409(self, target_url): + """ + §6 PATCH: Mismatching Upload-Offset MUST Return 409 Conflict. + Quote: "The Upload-Offset header’s value MUST be equal to the current offset of the resource... + If the offsets do not match, the Server MUST respond with the 409 Conflict status without + modifying the upload resource." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "50", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 10) + assert status == 409, f"PATCH with mismatching offset MUST return 409 Conflict, got {status}" + + # Verify upload resource was not modified + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be modified after 409 Conflict" + + def test_patch_successful_append(self, target_url): + """ + §6 PATCH: Successful Append Returns 204 No Content With New Upload-Offset. + Quote: "The Server MUST acknowledge successful PATCH requests with the 204 No Content status. + It MUST include the Upload-Offset header containing the new offset. The new offset MUST be + the sum of the offset before the PATCH request and the number of bytes received and processed." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + chunk = b"A" * 40 + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, resp_headers, _ = http_request("PATCH", upload_url, headers=headers, body=chunk) + assert status == 204, f"PATCH MUST return 204 No Content on success, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "40", "Upload-Offset in response MUST be updated to 40" + + # Verify via HEAD + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "40", "HEAD MUST report new offset of 40" + + def test_patch_non_existent_resource(self, target_url): + """ + §6 PATCH: Non-Existent Resource Returns 404 Not Found. + Quote: "If the server receives a PATCH request against a non-existent resource it SHOULD + return a 404 Not Found status." + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", non_existent_url, headers=headers, body=b"A" * 10) + assert status in (404, 410), f"PATCH against non-existent upload SHOULD return 404, got {status}" + + def test_patch_exceeding_upload_length_rejected(self, target_url): + """ + §6 PATCH: Appending Beyond Declared Upload-Length Must Be Rejected. + """ + upload_url, _ = create_upload(target_url, upload_length="30") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 50) + assert status in (400, 409, 413), f"PATCH exceeding Upload-Length MUST be rejected, got {status}" + + def test_x_http_method_override(self, target_url): + """ + §6 X-HTTP-Method-Override: Request Method Override. + Quote: "The X-HTTP-Method-Override request header MUST be a string which MUST be interpreted + as the request’s method by the Server, if the header is presented. The actual method of the + request MUST be ignored." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + + # Test overriding POST to PATCH + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + X_HTTP_METHOD_OVERRIDE: "PATCH", + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, resp_headers, _ = http_request("POST", upload_url, headers=headers, body=b"A" * 40) + assert status == 204, f"Overridden PATCH request MUST succeed with 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "40" + + # Test overriding POST to HEAD + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + X_HTTP_METHOD_OVERRIDE: "HEAD", + } + status, head_headers, _ = http_request("POST", upload_url, headers=headers) + assert status in (200, 204), f"Overridden HEAD request MUST succeed with 200 or 204, got {status}" + assert head_headers.get(UPLOAD_OFFSET) == "40" + + +class TestCreationExtension: + """Creation Extension compliance tests (§7.1).""" + + def test_creation_extension_advertised(self, target_url): + """ + §7.1 Creation: Extension Advertisement in OPTIONS. + Quote: "If the Server supports this extension, it MUST add creation to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "creation" in extensions, "Tus-Extension header MUST include 'creation'" + + def test_create_upload_empty_post(self, target_url): + """ + §7.1 Creation: Empty POST Creation Request. + Quote: "An empty POST request is used to create a new upload resource. The Upload-Length + header indicates the size of the entire upload in bytes... The Server MUST acknowledge a + successful upload creation with the 201 Created status. The Server MUST set the Location + header to the URL of the created resource." + """ + upload_url, resp_headers = create_upload(target_url, upload_length="100") + assert upload_url, "Location header pointing to created resource MUST be returned" + assert resp_headers.get(TUS_RESUMABLE) == TUS_API_VERSION + + def test_create_zero_byte_upload(self, target_url): + """ + §7.1 Creation: Zero-Byte Upload Creation. + Quote: "The Upload-Length header MAY be set to 0, indicating that the Client wants to + upload an empty file. Such an upload is immediately complete after its creation without + transferring data using PATCH requests." + """ + upload_url, _ = create_upload(target_url, upload_length="0") + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST be 0 for zero-byte upload" + assert head_headers.get(UPLOAD_LENGTH) == "0", "Upload-Length MUST be 0 for zero-byte upload" + + def test_create_upload_defer_length(self, target_url): + """ + §7.1 Creation: Deferred Length Creation. + Quote: "Upload-Defer-Length: 1 if upload size is not known at the time... As long as the + length of the upload is not known, the Server MUST set Upload-Defer-Length: 1 in all + responses to HEAD requests. If the length was deferred using Upload-Defer-Length: 1, the + Client MUST set the Upload-Length header in the next PATCH request, once the length is known." + """ + upload_url, _ = create_upload(target_url, defer_length=True) + + # Initial HEAD should return Upload-Defer-Length: 1 and no Upload-Length + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_DEFER_LENGTH) == "1", "HEAD MUST include Upload-Defer-Length: 1 when length deferred" + assert UPLOAD_LENGTH not in head_headers, "Upload-Length MUST NOT be present while deferred" + + # PATCH providing length + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + UPLOAD_LENGTH: "80", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"B" * 40) + assert status == 204, f"PATCH providing deferred Upload-Length MUST succeed with 204, got {status}" + + # Subsequent HEAD should now report Upload-Length: 80 and no Upload-Defer-Length + status, head_headers2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers2.get(UPLOAD_LENGTH) == "80", "HEAD MUST now report Upload-Length: 80" + assert UPLOAD_DEFER_LENGTH not in head_headers2, "Upload-Defer-Length MUST be omitted once length is known" + + def test_create_upload_invalid_defer_length_value(self, target_url): + """ + §7.1 Creation: Invalid Upload-Defer-Length Value Returns 400 Bad Request. + Quote: "If the Upload-Defer-Length header contains any other value than 1 the server + should return a 400 Bad Request status." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_DEFER_LENGTH: "2", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Upload-Defer-Length != 1 MUST return 400 Bad Request, got {status}" + + def test_create_upload_missing_both_length_and_defer_length(self, target_url): + """ + §7.1 Creation: POST Missing Both Upload-Length and Upload-Defer-Length. + Quote: "The request MUST include one of the following headers: a) Upload-Length ... + b) Upload-Defer-Length: 1" + """ + headers = {TUS_RESUMABLE: TUS_API_VERSION} + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"POST without length headers MUST return 400 Bad Request, got {status}" + + def test_create_upload_negative_length(self, target_url): + """ + §6 Upload-Length: Value MUST Be Non-Negative Integer. + Quote: "The Upload-Length request and response header indicates the size of the entire + upload in bytes. The value MUST be a non-negative integer." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "-10", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Negative Upload-Length MUST return 400 Bad Request, got {status}" + + def test_create_upload_non_integer_length(self, target_url): + """ + §6 Upload-Length: Non-Integer Value Rejected. + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "not_a_number", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Non-integer Upload-Length MUST return 400 Bad Request, got {status}" + + def test_create_upload_with_metadata(self, target_url): + """ + §7.1 Creation: Upload-Metadata Header Preservation. + Quote: "The Client MAY supply the Upload-Metadata header to add additional metadata to the + upload creation request... If an upload contains additional metadata, responses to HEAD + requests MUST include the Upload-Metadata header and its value as specified by the Client." + """ + meta_filename = base64.b64encode(b"report.pdf").decode("ascii") + meta_author = base64.b64encode(b"Jane Doe").decode("ascii") + metadata = f"filename {meta_filename},author {meta_author}" + + upload_url, _ = create_upload(target_url, upload_length="100", metadata=metadata) + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + resp_meta = head_headers.get(UPLOAD_METADATA, "") + assert f"filename {meta_filename}" in resp_meta, "Upload-Metadata in HEAD MUST contain filename" + assert f"author {meta_author}" in resp_meta, "Upload-Metadata in HEAD MUST contain author" + + def test_create_upload_metadata_empty_value(self, target_url): + """ + §7.1 Creation: Upload-Metadata with Empty Value. + Quote: "The value MAY be empty. In these cases, the space, which would normally separate + the key and the value, MAY be left out." + """ + metadata = "is_confidential" + upload_url, _ = create_upload(target_url, upload_length="100", metadata=metadata) + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert "is_confidential" in head_headers.get(UPLOAD_METADATA, ""), "Upload-Metadata in HEAD MUST contain empty-value key" + + def test_create_upload_exceeding_tus_max_size(self, target_url): + """ + §7.1 Creation: Upload Exceeding Tus-Max-Size Returns 413 Request Entity Too Large. + Quote: "If the length of the upload exceeds the maximum, which MAY be specified using + the Tus-Max-Size header, the Server MUST respond with the 413 Request Entity Too Large status." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + max_size_str = resp_headers.get(TUS_MAX_SIZE) + if not max_size_str: + pytest.skip("Server does not advertise Tus-Max-Size in OPTIONS response") + + max_size = int(max_size_str) + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: str(max_size + 1), + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 413, f"Upload exceeding Tus-Max-Size ({max_size}) MUST return 413, got {status}" + + def test_deferred_length_cannot_be_changed_once_set(self, target_url): + """ + §7.1 Creation: Deferred Length Cannot Be Changed Once Set. + Quote: "Once set the length MUST NOT be changed." + """ + upload_url, _ = create_upload(target_url, defer_length=True) + + # Set length to 100 + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + UPLOAD_LENGTH: "100", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 40) + assert status == 204 + + # Attempt to change length to 120 in subsequent PATCH + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "40", + UPLOAD_LENGTH: "120", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status2, _, _ = http_request("PATCH", upload_url, headers=headers2, body=b"B" * 40) + assert status2 in (400, 409), f"Attempting to modify already set Upload-Length MUST be rejected, got {status2}" + + +class TestCreationWithUploadExtension: + """Creation With Upload Extension compliance tests (§7.2).""" + + def test_creation_with_upload_advertised(self, target_url): + """ + §7.2 Creation With Upload: Extension Advertisement. + Quote: "If the Server supports this extension, it MUST advertise this by including + creation-with-upload in the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "creation-with-upload" in extensions, "Tus-Extension MUST include 'creation-with-upload'" + + def test_creation_with_full_upload(self, target_url): + """ + §7.2 Creation With Upload: Initial POST Containing Entire File Payload. + Quote: "The Client MAY include either the entirety or a chunk of the upload data in the + body of the POST request... The Server SHOULD accept as many bytes as possible and MUST + include the Upload-Offset header in the response and MUST set its value to the offset of + the upload after applying the accepted bytes." + """ + payload = b"Hello, Full Tus Creation With Upload!" + upload_url, resp_headers = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)), "Response MUST return Upload-Offset equal to body length" + + # Verify via HEAD + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == str(len(payload)) + assert head_headers.get(UPLOAD_LENGTH) == str(len(payload)) + + def test_creation_with_partial_upload_then_patch(self, target_url): + """ + §7.2 Creation With Upload: Initial POST With Partial Payload Followed by PATCH. + Quote: "The Server SHOULD accept as many bytes as possible and MUST include the + Upload-Offset header in the response... The Client MUST perform the actual upload + using the core protocol." + """ + part1 = b"Part 1 Data with at least 32 bytes!!" + part2 = b"Part 2 Data with at least 32 bytes!!" + total_length = len(part1) + len(part2) + + upload_url, resp_headers = create_upload( + target_url, + upload_length=str(total_length), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part1, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(part1)) + + # Complete upload using PATCH + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: str(len(part1)), + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, patch_headers, _ = http_request("PATCH", upload_url, headers=headers, body=part2) + assert status == 204 + assert patch_headers.get(UPLOAD_OFFSET) == str(total_length) + + # Final verification via HEAD + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == str(total_length) + + def test_creation_with_upload_and_deferred_length(self, target_url): + """ + §7.2 Creation With Upload: Creation With Upload Combined With Deferred Length. + """ + payload = b"Initial chunk with deferred length" + upload_url, resp_headers = create_upload( + target_url, + defer_length=True, + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + # HEAD verification + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == str(len(payload)) + assert head_headers.get(UPLOAD_DEFER_LENGTH) == "1" + + def test_creation_with_upload_invalid_content_type(self, target_url): + """ + §7.2 Creation With Upload: Non-Empty POST With Invalid Content-Type Must Be Rejected. + Quote: "The Client MUST include the Content-Type: application/offset+octet-stream header." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "30", + CONTENT_TYPE: "text/plain", + } + status, _, _ = http_request("POST", target_url, headers=headers, body=b"A" * 20) + assert status in (400, 406, 415), f"Non-empty POST with invalid Content-Type MUST be rejected, got {status}" + + def test_creation_with_upload_exceeds_upload_length(self, target_url): + """ + §7.2 Creation With Upload: Body Exceeding Declared Upload-Length Must Be Rejected. + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "10", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("POST", target_url, headers=headers, body=b"A" * 20) + assert status in (400, 413), f"Body exceeding declared length MUST be rejected, got {status}" + + +class TestChecksumExtension: + """Checksum Extension compliance tests (§7.4).""" + + def test_checksum_extension_advertised(self, target_url): + """ + §7.4 Checksum: Extension and Algorithms Advertisement. + Quote: "If supported, the Server MUST add checksum to the Tus-Extension header. + The Tus-Checksum-Algorithm header MUST be included in the response to an OPTIONS request. + The Server MUST support at least the SHA1 checksum algorithm identified by sha1." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "checksum" in extensions, "Tus-Extension header MUST include 'checksum'" + algorithms = [a.strip() for a in resp_headers.get(TUS_CHECKSUM_ALGORITHM, "").split(",")] + assert "sha1" in algorithms, "Tus-Checksum-Algorithm header MUST support 'sha1'" + + def test_patch_with_valid_sha1_checksum(self, target_url): + """ + §7.4 Checksum: Valid SHA1 Checksum Verification. + Quote: "A Client MAY include the Upload-Checksum header in a PATCH request. Once the entire + request has been received, the Server MUST verify the uploaded chunk against the provided + checksum using the specified algorithm... 3. 204 No Content if the checksums match and + the processing of the data succeeded." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Tus SHA1 checksum verified payload" + digest_bytes = hashlib.sha1(payload).digest() + b64_digest = base64.b64encode(digest_bytes).decode("ascii") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: f"sha1 {b64_digest}", + } + status, resp_headers, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 204, f"PATCH with valid checksum MUST succeed with 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + def test_patch_with_invalid_checksum_mismatch(self, target_url): + """ + §7.4 Checksum: Checksum Mismatch Returns 460 Checksum Mismatch. + Quote: "2. 460 Checksum Mismatch if the checksums mismatch... In the first two cases the + uploaded chunk MUST be discarded, and the upload and its offset MUST NOT be updated." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Tus SHA1 payload with at least 32 bytes in length" + invalid_digest = base64.b64encode(b"0" * 20).decode("ascii") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: f"sha1 {invalid_digest}", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 460, f"Checksum mismatch MUST return 460 Checksum Mismatch, got {status}" + + # Verify chunk was discarded and offset remains 0 + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be updated after checksum mismatch" + + def test_patch_with_unsupported_checksum_algorithm(self, target_url): + """ + §7.4 Checksum: Unsupported Checksum Algorithm Returns 400 Bad Request. + Quote: "1. 400 Bad Request if the checksum algorithm is not supported by the server... + the uploaded chunk MUST be discarded, and the upload and its offset MUST NOT be updated." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Payload with unknown checksum algorithm at least 32 bytes" + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: "unknown_algo AAAA", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 400, f"Unsupported checksum algorithm MUST return 400 Bad Request, got {status}" + + # Verify chunk was discarded + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be updated after invalid algorithm" + + +class TestTerminationExtension: + """Termination Extension compliance tests (§7.5).""" + + def test_termination_extension_advertised(self, target_url): + """ + §7.5 Termination: Extension Advertisement. + Quote: "If this extension is supported by the Server, it MUST be announced by adding + termination to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "termination" in extensions, "Tus-Extension header MUST include 'termination'" + + def test_delete_existing_upload(self, target_url): + """ + §7.5 Termination: Deletion of Existing Upload. + Quote: "When receiving a DELETE request for an existing upload the Server SHOULD free + associated resources and MUST respond with the 204 No Content status confirming that the + upload was terminated. For all future requests to this URL, the Server SHOULD respond with + the 404 Not Found or 410 Gone status." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + + # Terminate + status, _, _ = http_request("DELETE", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status == 204, f"DELETE on existing upload MUST return 204 No Content, got {status}" + + # Future HEAD request MUST return 404 or 410 + head_status, _, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_status in (404, 410), f"HEAD after DELETE MUST return 404 or 410, got {head_status}" + + # Future PATCH request MUST return 404 or 410 + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + patch_status, _, _ = http_request("PATCH", upload_url, headers=patch_headers, body=b"A" * 10) + assert patch_status in (404, 410), f"PATCH after DELETE MUST return 404 or 410, got {patch_status}" + + def test_delete_non_existent_upload(self, target_url): + """ + §7.5 Termination: DELETE on Non-Existent Resource Returns 404 Not Found. + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + status, _, _ = http_request("DELETE", non_existent_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (404, 410), f"DELETE on non-existent upload SHOULD return 404, got {status}" + + def test_delete_completed_upload(self, target_url): + """ + §7.5 Termination: Termination of Completed Upload. + Quote: "This extension defines a way for the Client to terminate completed and unfinished + uploads allowing the Server to free up used resources." + """ + payload = b"Terminating completed upload test payload" + upload_url, _ = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + + status, _, _ = http_request("DELETE", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status == 204, f"DELETE on completed upload MUST return 204 No Content, got {status}" + + head_status, _, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_status in (404, 410), f"HEAD on deleted completed upload MUST return 404 or 410, got {head_status}" + + +class TestConcatenationExtension: + """Concatenation Extension compliance tests (§7.6).""" + + def test_concatenation_extension_advertised(self, target_url): + """ + §7.6 Concatenation: Extension Advertisement. + Quote: "If the Server supports this extension, it MUST add concatenation to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "concatenation" in extensions, "Tus-Extension header MUST include 'concatenation'" + + def test_partial_upload_creation_and_head(self, target_url): + """ + §7.6 Concatenation: Partial Upload Creation and HEAD Response. + Quote: "A partial upload represents a chunk of a file. It is constructed by including the + Upload-Concat: partial header while creating a new upload using the Creation extension... + The response to a HEAD request for a partial upload MUST contain the Upload-Offset header. + Response to HEAD request against partial or final upload MUST include the Upload-Concat + header and its value as received in the upload creation request." + """ + upload_url, _ = create_upload(target_url, upload_length="40", concat="partial") + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == "0" + assert head_headers.get(UPLOAD_CONCAT) == "partial", "HEAD response MUST include Upload-Concat: partial" + + def test_final_upload_concatenation_success(self, target_url): + """ + §7.6 Concatenation: Final Concatenation of Partial Uploads. + Quote: "In order to create a new final upload, the Client MUST add the Upload-Concat header + to the upload creation request. The value MUST be final followed by a semicolon and a + space-separated list of the partial upload URLs that need to be concatenated... The length of + the final upload MUST be the sum of the length of all partial uploads. After successful + concatenation, the Upload-Offset and Upload-Length MUST be set and their values MUST be equal." + """ + # Create and fill partial 1 + part1_data = b"Hello, Concatenated World Part 1!" + part1_url, _ = create_upload( + target_url, + upload_length=str(len(part1_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part1_data, + ) + + # Create and fill partial 2 + part2_data = b"Hello, Concatenated World Part 2!" + part2_url, _ = create_upload( + target_url, + upload_length=str(len(part2_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part2_data, + ) + + total_length = len(part1_data) + len(part2_data) + + # Create final concatenated upload + concat_val = f"final;{part1_url} {part2_url}" + final_url, _ = create_upload(target_url, upload_length=None, concat=concat_val) + + # Verify final upload HEAD + status, head_headers, _ = http_request("HEAD", final_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == str(total_length), "Upload-Offset MUST equal total combined length" + assert head_headers.get(UPLOAD_LENGTH) == str(total_length), "Upload-Length MUST equal total combined length" + assert head_headers.get(UPLOAD_CONCAT) == concat_val, "Upload-Concat MUST match final concatenation specification" + + def test_final_creation_must_not_include_upload_length(self, target_url): + """ + §7.6 Concatenation: Final Upload Creation MUST NOT Include Upload-Length. + Quote: "The Client MUST NOT include the Upload-Length header in the final upload creation." + """ + # Create a partial upload + part_url, _ = create_upload(target_url, upload_length="20", concat="partial") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_CONCAT: f"final;{part_url}", + UPLOAD_LENGTH: "20", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Final creation including Upload-Length MUST be rejected with 400, got {status}" + + def test_patch_against_final_upload_forbidden(self, target_url): + """ + §7.6 Concatenation: PATCH on Final Upload Returns 403 Forbidden. + Quote: "The Server MUST respond with the 403 Forbidden status to PATCH requests against + a final upload URL and MUST NOT modify the final or its partial uploads." + """ + part_data = b"Fixed Part Data with at least 32 bytes!" + part_url, _ = create_upload( + target_url, + upload_length=str(len(part_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part_data, + ) + final_url, _ = create_upload(target_url, upload_length=None, concat=f"final;{part_url}") + + # Retrieve current offset of final upload via HEAD + _, head_headers, _ = http_request("HEAD", final_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + offset = head_headers.get(UPLOAD_OFFSET, str(len(part_data))) + + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: offset, + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", final_url, headers=patch_headers, body=b"") + assert status == 403, f"PATCH against final upload MUST return 403 Forbidden, got {status}" + + +class TestExpirationExtension: + """Expiration Extension compliance tests (§7.3).""" + + def test_expiration_extension_advertised(self, target_url): + """ + §7.3 Expiration: Extension Advertisement. + Quote: "In order to indicate this behavior to the Client, the Server MUST add expiration + to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "expiration" in extensions, "Tus-Extension header MUST include 'expiration'" + + def test_upload_expires_header_format(self, target_url): + """ + §7.3 Expiration: Upload-Expires Header in Datetime Format. + Quote: "The Upload-Expires response header indicates the time after which the unfinished + upload expires... This header MUST be included in every PATCH response if the upload is + going to expire. If the expiration is known at the creation, the Upload-Expires header MUST + be included in the response to the initial POST request... The value of the Upload-Expires + header MUST be in RFC 9110 datetime format." + """ + upload_url, post_headers = create_upload(target_url, upload_length="100") + expires_header = post_headers.get(UPLOAD_EXPIRES) + + if not expires_header: + # Perform a PATCH to observe Upload-Expires on intermediate append + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, patch_resp_headers, _ = http_request("PATCH", upload_url, headers=patch_headers, body=b"A" * 40) + assert status == 204 + expires_header = patch_resp_headers.get(UPLOAD_EXPIRES) + + if expires_header: + dt = email.utils.parsedate_to_datetime(expires_header) + assert dt is not None, f"Upload-Expires '{expires_header}' could not be parsed as valid RFC 9110 datetime" + + +class TestConcurrencyAndWorkflow: + """Concurrency and end-to-end upload workflows.""" + + def test_concurrent_patches_same_offset(self, target_url): + """ + §6 PATCH: Concurrency Contention Prevention. + Quote: "The Upload-Offset header’s value MUST be equal to the current offset of the resource. + If the offsets do not match, the Server MUST respond with the 409 Conflict status without + modifying the upload resource." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + results = [] + + def do_patch(): + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 40) + results.append(st) + + t1 = threading.Thread(target=do_patch) + t2 = threading.Thread(target=do_patch) + t1.start() + t2.start() + t1.join() + t2.join() + + successes = [r for r in results if r == 204] + assert len(successes) <= 1, f"At most one concurrent PATCH at offset 0 can succeed, got {results}" + + def test_multi_chunk_resumable_upload_workflow(self, target_url): + """ + End-to-End multi-chunk resumable upload workflow. + Create upload -> PATCH chunk 1 -> HEAD verify -> PATCH chunk 2 -> complete. + """ + total_data = b"0123456789" * 12 # 120 bytes + chunk1 = total_data[:40] + chunk2 = total_data[40:80] + chunk3 = total_data[80:] + + upload_url, _ = create_upload(target_url, upload_length=str(len(total_data))) + + # Chunk 1 + headers1 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st1, _, _ = http_request("PATCH", upload_url, headers=headers1, body=chunk1) + assert st1 == 204 + + # HEAD verification + _, head1, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head1.get(UPLOAD_OFFSET) == "40" + + # Chunk 2 + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "40", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st2, _, _ = http_request("PATCH", upload_url, headers=headers2, body=chunk2) + assert st2 == 204 + + # HEAD verification + _, head2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head2.get(UPLOAD_OFFSET) == "80" + + # Chunk 3 (final) + headers3 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "80", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st3, _, _ = http_request("PATCH", upload_url, headers=headers3, body=chunk3) + assert st3 == 204 + + # Final verification + _, head_final, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_final.get(UPLOAD_OFFSET) == "120" + assert head_final.get(UPLOAD_LENGTH) == "120" + + +# CLI Entry Point & Custom Formatted Summary Reporter +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Tus v1.0.0 Conformity Test Runner") + parser.add_argument( + "--url", + default="http://localhost:8080/test/api/upload", + help="Target Tus upload endpoint URL", + ) + args = parser.parse_args() + + os.environ["TUS_URL"] = args.url + + print("=" * 70) + print(" Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite") + print(" Specification: https://tus.io/protocols/resumable-upload") + print(" Target Endpoint:", args.url) + print("=" * 70) + + class CustomReporter: + def __init__(self): + self.passed = [] + self.failed = [] + self.docs = {} + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() + if report.when == "call": + doc = item.obj.__doc__ or "No description provided." + self.docs[report.nodeid] = doc.strip() + if report.passed: + self.passed.append(report.nodeid) + elif report.failed: + if hasattr(report.longrepr, "reprcrash"): + err_text = report.longrepr.reprcrash.message + elif hasattr(report, "longreprtext"): + err_lines = [ + l.strip() + for l in report.longreprtext.splitlines() + if l.strip().startswith("E ") or l.strip().startswith("AssertionError") + ] + err_text = "\n ".join(err_lines) if err_lines else str(report.longrepr) + else: + err_text = str(report.longrepr) + self.failed.append((report.nodeid, err_text)) + + reporter = CustomReporter() + pytest.main([__file__, "-q", f"--url={args.url}"], plugins=[reporter]) + + total_tests = len(reporter.passed) + len(reporter.failed) + + print("\n" + "=" * 70) + print(" CONFORMITY TEST RESULTS") + print("=" * 70) + print(f" Total Tests Executed: {total_tests}") + print(f" Passed: {len(reporter.passed)}") + print(f" Failed: {len(reporter.failed)}") + print("=" * 70) + + if reporter.failed: + print("\n[!] DETAILED FAILURE BREAKDOWN FOR REMEDIATION:") + print("-" * 70) + for idx, (test_id, err_text) in enumerate(reporter.failed, 1): + print(f"\n{idx}. Test: {test_id}") + func_name = test_id.split("::")[-1] + print(f" Function: {func_name}") + print(f" Specification Goal:\n " + reporter.docs.get(test_id, "").replace("\n", "\n ")) + print(f" Failure Reason:\n " + err_text.replace("\n", "\n ")) + print("-" * 70) + else: + print("\n[✓] ALL TUS V1.0.0 CONFORMITY TESTS PASSED SUCCESSFULLY!") + + sys.exit(0 if not reporter.failed else 1) diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 5b5c20fa..8f203f23 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -235,6 +235,20 @@ public TusFileUploadService withMinSize(Long minSize) { return this; } + /** + * Set the maximum number of worker threads used for asynchronous background chunk uploading in + * cloud storage backends (S3, Azure). Defaults to 10. + * + * @param cloudUploadThreadPoolSize Number of worker threads (must be > 0) + * @return The current service + */ + public TusFileUploadService withCloudUploadThreadPoolSize(int cloudUploadThreadPoolSize) { + Validate.isTrue( + cloudUploadThreadPoolSize > 0, "The cloud upload thread pool size must be greater than 0"); + this.uploadStorageService.setCloudUploadThreadPoolSize(cloudUploadThreadPoolSize); + return this; + } + /** * Provide a custom {@link UploadIdFactory} implementation that should be used to generate * identifiers for the different uploads. Example implementation are {@link @@ -268,6 +282,8 @@ public TusFileUploadService withUploadStorageService(UploadStorageService upload uploadStorageService.setMaxAppendSize(this.uploadStorageService.getMaxAppendSize()); uploadStorageService.setMinAppendSize(this.uploadStorageService.getMinAppendSize()); uploadStorageService.setMinSize(this.uploadStorageService.getMinSize()); + uploadStorageService.setCloudUploadThreadPoolSize( + this.uploadStorageService.getCloudUploadThreadPoolSize()); uploadStorageService.setUploadExpirationPeriod( this.uploadStorageService.getUploadExpirationPeriod()); uploadStorageService.setUploadDeduplicationEnabled( @@ -1085,6 +1101,13 @@ protected void processTusException( response.setStatus(status); if (problemDetails != null) { problemDetails.writeTo(response); + } else if (status == 460) { + // Non-standard HTTP status 460 (Checksum Mismatch) is not recognized by standard + // servlet container error controllers (such as Spring Boot's BasicErrorController), + // which would cause sendError() to fail with an unhandled IllegalArgumentException + // and degrade to HTTP 500. We explicitly set the status and omit sendError(). + response.setHeader(HttpHeader.CONTENT_LENGTH, null); + response.setStatus(status); } else { response.setHeader(HttpHeader.CONTENT_LENGTH, null); response.sendError(status, message); diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index 5ec06fa4..53974029 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -55,6 +55,13 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); + long cl = servletRequest.getContentLengthLong(); + if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && cl >= 0) { + long currentOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; + uploadInfo.setLength(currentOffset + cl); + uploadStorageService.update(uploadInfo); + } + InputStream is = servletRequest.getContentInputStream(); if (is != null) { if (uploadLockingService != null) { diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 1c47d68c..4ff1232d 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -66,17 +66,23 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); + long cl = servletRequest.getContentLengthLong(); + Long announcedLength = uploadLength; + if (announcedLength == null && Boolean.TRUE.equals(uploadComplete) && cl >= 0) { + announcedLength = cl; + } + UploadInfo uploadInfo; if (preCreatedUploadInfo != null) { uploadInfo = preCreatedUploadInfo; - if (uploadLength != null && uploadLength >= 0) { - uploadInfo.setLength(uploadLength); + if (announcedLength != null && announcedLength >= 0) { + uploadInfo.setLength(announcedLength); } uploadStorageService.update(uploadInfo); } else { uploadInfo = new UploadInfo(); - if (uploadLength != null && uploadLength >= 0) { - uploadInfo.setLength(uploadLength); + if (announcedLength != null && announcedLength >= 0) { + uploadInfo.setLength(announcedLength); } uploadInfo = uploadStorageService.create(uploadInfo, ownerKey); } diff --git a/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java b/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java index 4bbb7749..a09dc67e 100644 --- a/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java +++ b/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java @@ -56,13 +56,22 @@ public static String getRawInterimResponse( String uploadUri; try { - UploadInfo uploadInfo = uploadStorageService.getUploadInfo(existingUploadUri, ownerKey); + UploadInfo uploadInfo = + existingUploadUri != null + ? uploadStorageService.getUploadInfo(existingUploadUri, ownerKey) + : null; if (uploadInfo != null) { long offset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; return getRawInterimResponseForAppend(offset); } + // If there is no existing upload, an interim response with Location is only valid for + // an upload creation request matching this storage service's creation endpoint. + if (!Utils.isCreationEndpoint(servletRequest, uploadStorageService)) { + return null; + } + uploadInfo = new UploadInfo(); uploadInfo = uploadStorageService.create(uploadInfo, ownerKey); uploadUri = Utils.getUploadUriOnCreation(uploadInfo, servletRequest, uploadStorageService); diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index ea5d9f1d..bdce0298 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -279,6 +279,24 @@ default boolean isJsonSerializationEnabled() { return false; } + /** + * Set the thread pool size used for asynchronous background chunk uploading to cloud storage. + * + * @param size Number of worker threads + */ + default void setCloudUploadThreadPoolSize(int size) { + // Default no-op for non-cloud implementations + } + + /** + * Get the thread pool size used for asynchronous background chunk uploading to cloud storage. + * + * @return Thread pool size, defaults to 10 + */ + default int getCloudUploadThreadPoolSize() { + return 10; + } + /** * Closes any underlying storage resources. * diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java index 19be0000..697f9be1 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java @@ -3,11 +3,14 @@ import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.sas.BlobSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; import com.azure.storage.blob.specialized.BlockBlobClient; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -109,16 +112,20 @@ public void merge(UploadInfo finalUpload) throws IOException, UploadNotFoundExce BlobClient partialBlob = containerClient.getBlobClient(uploadPrefix + partialInfo.getId()); try { - // 1. Attempt zero-copy server-side block copying on Azure Storage cluster - stageBlockFromUrl(finalBlockBlob, blockId, partialBlob.getBlobUrl()); + // 1. Attempt zero-copy server-side block copying on Azure Storage cluster using an + // authorized blob URL (with read SAS token when shared key credentials are present) + String sourceUrl = getAuthorizedBlobUrl(partialBlob); + stageBlockFromUrl(finalBlockBlob, blockId, sourceUrl); } catch (BlobStorageException e) { // 2. In private Azure containers without SAS tokens or in emulators, stageBlockFromUrl - // fails with 403 (ACCESS_DENIED) or 400/501 (API_NOT_IMPLEMENTED). Fall back + // fails with 401/403 (ACCESS_DENIED / CannotVerifyCopySource) or 400/501 + // (API_NOT_IMPLEMENTED). Fall back // gracefully to streaming block staging via getUploadedBytes(). AzureErrorType errorType = AzureUtils.parseErrorResponse(e); if (errorType == AzureErrorType.API_NOT_IMPLEMENTED || errorType == AzureErrorType.ACCESS_DENIED || e.getStatusCode() == 400 + || e.getStatusCode() == 401 || e.getStatusCode() == 403) { try (InputStream partIs = storageService.getUploadedBytes(partialInfo.getId())) { finalBlockBlob.stageBlock(blockId, partIs, partialInfo.getOffset()); @@ -230,4 +237,32 @@ private String sanitizePrefix(String prefix) { void stageBlockFromUrl(BlockBlobClient finalBlockBlob, String blockId, String sourceUrl) { finalBlockBlob.stageBlockFromUrl(blockId, sourceUrl, null); } + + /** + * Resolves an authorized source URL for server-side block copying. If the client possesses shared + * key credentials, generates a short-lived read SAS token so that private Azure containers can be + * accessed directly by the storage service via stageBlockFromUrl. + * + * @param blobClient The BlobClient of the source partial upload + * @return The authorized blob URL (with SAS token if available) or raw blob URL + */ + String getAuthorizedBlobUrl(BlobClient blobClient) { + if (blobClient == null) { + return null; + } + try { + BlobServiceSasSignatureValues sasValues = + new BlobServiceSasSignatureValues( + OffsetDateTime.now().plusMinutes(15), + new BlobSasPermission().setReadPermission(true)); + String sasToken = blobClient.generateSas(sasValues); + if (sasToken != null && !sasToken.isEmpty()) { + return blobClient.getBlobUrl() + "?" + sasToken; + } + } catch (Exception ignored) { + // Client lacks shared key credentials (e.g. TokenCredential without user delegation key) + // or SAS generation is unsupported; fall back to raw blob URL. + } + return blobClient.getBlobUrl(); + } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 69c6e70e..686797a2 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -25,6 +25,10 @@ import java.util.Base64; import java.util.List; import java.util.Objects; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinAppendSizeNotMetException; @@ -37,6 +41,7 @@ import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import me.desair.tus.server.upload.util.AsyncChunkUploader; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.IOUtils; @@ -86,6 +91,9 @@ public class AzureBlobStorageService implements UploadStorageService { private long preferredBlockSize = DEFAULT_PREFERRED_BLOCK_SIZE; + private int cloudUploadThreadPoolSize = 10; + private final ThreadPoolExecutor uploadExecutor; + private Long maxUploadSize; private Long maxAppendSize; private Long minAppendSize; @@ -144,6 +152,21 @@ public AzureBlobStorageService( } Utils.cleanupTempFiles(this.tempBufferDir, "tus-azure-chunk-*.tmp", 24L * 3600_000L); + AtomicInteger threadNum = new AtomicInteger(1); + this.uploadExecutor = + new ThreadPoolExecutor( + cloudUploadThreadPoolSize, + cloudUploadThreadPoolSize, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + r -> { + Thread t = new Thread(r, "tus-azure-upload-" + threadNum.getAndIncrement()); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.CallerRunsPolicy()); + this.concatenationService = new AzureBlobConcatenationService(containerClient, this.uploadPrefix, this); } @@ -212,56 +235,84 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) IOException streamException = null; TusException pendingTusException = null; - // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob - while (true) { - File chunkFile = null; - try { - chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile(); - ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, optimalBlockSize); - long chunkSize = chunkResult.bytesRead; - if (chunkResult.exception != null) { - streamException = chunkResult.exception; - } + List plannedBlockIds = new ArrayList<>(); + List plannedChunkSizes = new ArrayList<>(); - if (chunkSize <= 0) { - break; - } - - if (effectiveMaxAppendSize != null - && (totalAppended + chunkSize) > effectiveMaxAppendSize) { - // If maxAppendSize is exceeded, stop reading from the stream but commit - // previously staged blocks so no uploaded data or offset is corrupted. - pendingTusException = - new MaxAppendSizeExceededException( - "Append payload size " - + (totalAppended + chunkSize) - + " exceeded limit of " - + effectiveMaxAppendSize); - break; - } - - totalAppended += chunkSize; + try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob + while (true) { + File chunkFile = null; + boolean handedOff = false; + try { + chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile(); + ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, optimalBlockSize); + long chunkSize = chunkResult.bytesRead; + if (chunkResult.exception != null) { + streamException = chunkResult.exception; + } - // Validate chunk against remaining block budget - validateRemainingBlockBudget(upload, blockIds.size()); + if (chunkSize <= 0) { + break; + } - // Stage block directly to Azure Block Blob - stageChunkFile(chunkFile, chunkSize, blockBlobClient, blockIds); + if (effectiveMaxAppendSize != null + && (totalAppended + chunkSize) > effectiveMaxAppendSize) { + // If maxAppendSize is exceeded, stop reading from the stream but commit + // previously staged blocks so no uploaded data or offset is corrupted. + pendingTusException = + new MaxAppendSizeExceededException( + "Append payload size " + + (totalAppended + chunkSize) + + " exceeded limit of " + + effectiveMaxAppendSize); + break; + } - if (streamException != null) { + int plannedIndex = blockIds.size() + plannedBlockIds.size(); + // Validate chunk against remaining block budget + validateRemainingBlockBudget(upload, plannedIndex); + + String blockId = generateBlockId(plannedIndex); + plannedBlockIds.add(blockId); + plannedChunkSizes.add(chunkSize); + totalAppended += chunkSize; + + File fileToUpload = chunkFile; + uploader.submitChunk( + chunkFile, + chunkSize, + blockId, + () -> stageBlock(blockBlobClient, blockId, fileToUpload, chunkSize)); + handedOff = true; + + if (streamException != null) { + break; + } + } catch (IOException e) { + streamException = e; + break; + } catch (TusException te) { + pendingTusException = te; break; + } finally { + if (!handedOff) { + deleteFileQuietly(chunkFile); + } } - } catch (IOException e) { - streamException = e; - break; - } catch (TusException te) { - pendingTusException = te; - break; - } finally { - deleteFileQuietly(chunkFile); + } + + int confirmedCount = uploader.drainAndComplete(4000); + for (int i = 0; i < confirmedCount; i++) { + blockIds.add(plannedBlockIds.get(i)); } } + // Calculate actual confirmed bytes appended + long confirmedAppended = 0L; + for (int i = 0; i < blockIds.size() - initialBlockCount; i++) { + confirmedAppended += plannedChunkSizes.get(i); + } + // 5. Commit any newly staged blocks in batch (1 single commit call for entire request) // Batching block commits into a single call at the end eliminates redundant network // round-trips for every chunk, drastically improving performance. In addition, committing @@ -270,7 +321,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) if (blockIds.size() > initialBlockCount) { try { blockBlobClient.commitBlockList(blockIds, true); - upload.setOffset(initialOffset + totalAppended); + upload.setOffset(initialOffset + confirmedAppended); if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { upload.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); } @@ -301,7 +352,16 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } // 6. Validate minimum append size constraints if configured - validateMinAppendSize(totalAppended); + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + boolean isCompletingOrEmpty = + !upload.isUploadInProgress() + || (upload.getLength() != null + && upload.getOffset() != null + && upload.getOffset() >= upload.getLength()); + if (!isCompletingOrEmpty) { + validateMinAppendSize(confirmedAppended); + } return upload; } @@ -653,6 +713,40 @@ public long getPreferredBlockSize() { return preferredBlockSize; } + @Override + public void setCloudUploadThreadPoolSize(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "The cloud upload thread pool size must be greater than 0"); + } + this.cloudUploadThreadPoolSize = size; + if (size > uploadExecutor.getMaximumPoolSize()) { + uploadExecutor.setMaximumPoolSize(size); + uploadExecutor.setCorePoolSize(size); + } else { + uploadExecutor.setCorePoolSize(size); + uploadExecutor.setMaximumPoolSize(size); + } + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + + @Override + public void close() throws IOException { + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + uploadExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + uploadExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + // --- Helper Methods --- /** Calculates auto-calibrated optimal block size based on total upload length. */ @@ -766,16 +860,17 @@ private void validateMinAppendSize(long totalAppended) throws MinAppendSizeNotMe } } - /** Stages local chunk temp file as a Block Blob block. */ - private void stageChunkFile( - File chunkFile, long chunkSize, BlockBlobClient blockBlobClient, List blockIds) + /** + * Stages local chunk temp file as a Block Blob block without mutating blockIds on the worker + * thread. + */ + private void stageBlock( + BlockBlobClient blockBlobClient, String blockId, File chunkFile, long chunkSize) throws IOException { if (chunkSize > 0) { - String chunkBlockId = generateBlockId(blockIds.size()); try (InputStream chunkIs = new java.io.BufferedInputStream(new FileInputStream(chunkFile))) { - blockBlobClient.stageBlock(chunkBlockId, chunkIs, chunkSize); + blockBlobClient.stageBlock(blockId, chunkIs, chunkSize); } - blockIds.add(chunkBlockId); } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java index b7e1a105..af579f28 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java @@ -58,7 +58,11 @@ public static AzureErrorType parseErrorResponse(BlobStorageException exception) return AzureErrorType.API_NOT_IMPLEMENTED; } - if (statusCode == 403 || errorCodeStr.contains("authorizationfailure")) { + if (statusCode == 401 + || statusCode == 403 + || errorCodeStr.contains("authorizationfailure") + || errorCodeStr.contains("cannotverifycopysource") + || errorCodeStr.contains("noauthenticationinformation")) { return AzureErrorType.ACCESS_DENIED; } diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index 76241689..ee83f592 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -233,6 +233,16 @@ public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm alg return storageServiceDelegate.getUploadInfoByChecksum(checksum, algorithm); } + @Override + public void setCloudUploadThreadPoolSize(int size) { + storageServiceDelegate.setCloudUploadThreadPoolSize(size); + } + + @Override + public int getCloudUploadThreadPoolSize() { + return storageServiceDelegate.getCloudUploadThreadPoolSize(); + } + @Override public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { UploadLock uploadLock = lockingServiceDelegate.lockUploadByUri(requestUri); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 2269c5e2..5a428699 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -34,6 +34,7 @@ import me.desair.tus.server.util.Utils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +51,7 @@ public class DiskStorageService extends AbstractDiskBasedService implements Uplo private Long maxAppendSize = null; private Long minAppendSize = null; private Long minSize = null; + private int cloudUploadThreadPoolSize = 10; private Long uploadExpirationPeriod = null; private UploadIdFactory idFactory; private UploadConcatenationService uploadConcatenationService; @@ -117,6 +119,17 @@ public Long getMinSize() { return minSize; } + @Override + public void setCloudUploadThreadPoolSize(int size) { + Validate.isTrue(size > 0, "The cloud upload thread pool size must be greater than 0"); + this.cloudUploadThreadPoolSize = size; + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + @Override public void setUploadDeduplicationEnabled(boolean enabled) { this.isUploadDeduplicationEnabled = enabled; @@ -338,8 +351,18 @@ public UploadInfo append(UploadInfo info, InputStream inputStream) + " bytes. You can only append to the end of an upload"); } - // write all bytes in the channel up to the configured maximum - transferred = file.transferFrom(uploadedBytes, offset, max - offset); + // write all bytes in the channel up to the configured maximum in a loop to support + // transfers larger than 2GB (OS syscall limits) + long bytesTransferred = 0; + while (offset + bytesTransferred < max) { + long toTransfer = max - (offset + bytesTransferred); + long n = file.transferFrom(uploadedBytes, offset + bytesTransferred, toTransfer); + if (n <= 0) { + break; + } + bytesTransferred += n; + } + transferred = bytesTransferred; file.force(true); newOffset = offset + transferred; @@ -492,8 +515,18 @@ public void copyUploadTo(UploadInfo info, OutputStream outputStream) "The upload bytes for id " + readId + " could not be found."); } try (FileChannel file = FileChannel.open(bytesPath, READ)) { - // Efficiently copy the bytes to the output stream - file.transferTo(0, upload.getLength(), outputChannel); + // Efficiently copy the bytes to the output stream in a loop to handle transfers + // exceeding OS single-syscall limits (e.g. 2GB sendfile limit) + long position = 0; + long remaining = upload.getLength() != null ? upload.getLength() : file.size(); + while (remaining > 0) { + long bytesTransferred = file.transferTo(position, remaining, outputChannel); + if (bytesTransferred <= 0) { + break; + } + position += bytesTransferred; + remaining -= bytesTransferred; + } } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index d3b9c911..8b059137 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -30,6 +30,10 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MaxUploadLengthExceededException; @@ -45,6 +49,7 @@ import me.desair.tus.server.upload.UploadType; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import me.desair.tus.server.upload.util.AsyncChunkUploader; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.FileUtils; @@ -86,12 +91,12 @@ public class S3StorageService implements UploadStorageService { public static final String DEFAULT_LOCKS_PREFIX = "locks/"; // Part Sizing Constraints (per AWS S3 & MinIO specifications) - private static final long DEFAULT_MIN_PART_SIZE = - 5L * 1024 * 1024; // 5 MB (S3 minimum part limit) - private static final long DEFAULT_PREFERRED_PART_SIZE = - 50L * 1024 * 1024; // 50 MB (Optimal chunk size) - private static final long DEFAULT_MAX_PART_SIZE = + public static final long DEFAULT_MIN_PART_SIZE = 5L * 1024 * 1024; // 5 MB (S3 minimum part limit) + public static final long DEFAULT_PREFERRED_PART_SIZE = + 8L * 1024 * 1024; // 8 MB (Optimal chunk size aligned with Azure) + public static final long DEFAULT_MAX_PART_SIZE = 5L * 1024 * 1024 * 1024L; // 5 GB (S3 maximum object/part limit) + public static final int MAX_PARTS_PER_UPLOAD = 10_000; private final MinioClient minioClient; private final String bucket; @@ -104,6 +109,9 @@ public class S3StorageService implements UploadStorageService { private long minPartSize = DEFAULT_MIN_PART_SIZE; private long preferredPartSize = DEFAULT_PREFERRED_PART_SIZE; + private int cloudUploadThreadPoolSize = 10; + private final ThreadPoolExecutor uploadExecutor; + private Long maxUploadSize; private Long maxAppendSize; private Long minAppendSize; @@ -169,6 +177,21 @@ public S3StorageService( log.debug("Unable to ensure temporary directory exists: {}", e.getMessage()); } + AtomicInteger threadNum = new AtomicInteger(1); + this.uploadExecutor = + new ThreadPoolExecutor( + cloudUploadThreadPoolSize, + cloudUploadThreadPoolSize, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + r -> { + Thread t = new Thread(r, "tus-s3-upload-" + threadNum.getAndIncrement()); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.CallerRunsPolicy()); + this.concatenationService = new S3ConcatenationService( this.minioClient, this.bucket, this.objectPrefix, this, this.temporaryDirectory); @@ -307,6 +330,9 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) throws IOException, TusException { // Step 1: Verify upload existence and check configured size limits UploadInfo info = fetchAndValidateUpload(upload.getId()); + if (upload.getLength() != null && info.getLength() == null) { + info.setLength(upload.getLength()); + } String objectKey = getS3ObjectKey(info); String partObjectKey = buildIncompletePartKey(info.getId()); @@ -316,18 +342,39 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) PreparedStream preparedStream = prepareStreamWithExistingIncompletePart(info.getId(), partObjectKey, inputStream); + // Validate that the upload has not exceeded S3's 10,000 multipart parts ceiling + validateRemainingPartBudget(info, preparedStream.remainingPartKeys.size()); + // Step 3: Process payload stream in optimal chunk parts and upload to S3 boolean successfullyFinished = false; try { AppendResult appendResult = processPayloadChunks(info, preparedStream, info.getId(), partObjectKey); - // Step 4: Validate minimum append size constraints if configured + // Step 4: Recalculate total uploaded byte offset across all uploaded part objects in S3. + // S3 listObjects can exhibit eventual consistency; take the maximum of remote parts query + // and locally verified stream progression to ensure newOffset accurately reflects bytes + // successfully written. + long calculatedOffset = + calculateCurrentOffset(objectKey, info.getId(), partObjectKey, info.getLength()); + long currentTotalOffset = + (info.getOffset() != null ? info.getOffset() : 0L) + + appendResult.totalBytesAppended + - preparedStream.prependedBytes; + long newOffset = Math.max(calculatedOffset, currentTotalOffset); + info.setOffset(newOffset); + upload.setOffset(newOffset); + + // Step 5: Validate minimum append size constraints if configured // Subtract prependedBytes so minAppendSize accurately measures the payload transferred // in THIS request rather than earlier buffered bytes. + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + boolean isCompletingOrEmpty = + !info.isUploadInProgress() || (info.getLength() != null && newOffset >= info.getLength()); long requestPayloadAppended = Math.max(0L, appendResult.totalBytesAppended - preparedStream.prependedBytes); - if (minAppendSize != null && requestPayloadAppended < minAppendSize) { + if (minAppendSize != null && !isCompletingOrEmpty && requestPayloadAppended < minAppendSize) { throw new MinAppendSizeNotMetException( "Append payload size " + requestPayloadAppended @@ -335,11 +382,6 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) + minAppendSize); } - // Step 5: Recalculate total uploaded byte offset across all uploaded part objects in S3 - long newOffset = calculateCurrentOffset(objectKey, info.getId(), partObjectKey); - info.setOffset(newOffset); - upload.setOffset(newOffset); - // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), appendResult, newOffset); update(info); @@ -347,7 +389,8 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) return info; } finally { if (!successfullyFinished) { - long newOffset = calculateCurrentOffset(objectKey, info.getId(), partObjectKey); + long newOffset = + calculateCurrentOffset(objectKey, info.getId(), partObjectKey, info.getLength()); info.setOffset(newOffset); upload.setOffset(newOffset); update(info); @@ -687,6 +730,66 @@ public void setIdFactory(UploadIdFactory idFactory) { } } + @Override + public void setCloudUploadThreadPoolSize(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "The cloud upload thread pool size must be greater than 0"); + } + this.cloudUploadThreadPoolSize = size; + if (size > uploadExecutor.getMaximumPoolSize()) { + uploadExecutor.setMaximumPoolSize(size); + uploadExecutor.setCorePoolSize(size); + } else { + uploadExecutor.setCorePoolSize(size); + uploadExecutor.setMaximumPoolSize(size); + } + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + + /** + * Set the preferred chunk part size in bytes used when buffering and uploading parts to S3. + * + * @param preferredPartSize Part size in bytes (must be between 5 MB and 5 GB) + */ + public void setPreferredPartSize(long preferredPartSize) { + if (preferredPartSize < minPartSize || preferredPartSize > DEFAULT_MAX_PART_SIZE) { + throw new IllegalArgumentException( + "Preferred part size must be between " + + minPartSize + + " and " + + DEFAULT_MAX_PART_SIZE + + " bytes"); + } + this.preferredPartSize = preferredPartSize; + } + + /** + * Return the preferred chunk part size in bytes used when buffering and uploading parts to S3. + * + * @return Preferred part size in bytes + */ + public long getPreferredPartSize() { + return preferredPartSize; + } + + @Override + public void close() throws IOException { + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + uploadExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + uploadExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + // PRIVATE HELPER METHODS & S3 PROCESSING LOGIC private UploadInfo fetchAndValidateUpload(UploadId uploadId) @@ -791,8 +894,9 @@ private PreparedStream prepareStreamWithExistingIncompletePart( /** * Reads bytes from the incoming stream into temporary local files of optimal part size (default - * 50MB). Parts ≥ 5MB are uploaded immediately to S3 as part chunk objects. Any trailing chunk - * under 5MB is saved as a temporary .part object unless it completes the overall upload. + * 8MB). Parts ≥ 5MB are uploaded asynchronously to S3 via {@link AsyncChunkUploader}, + * overlapping client stream reading with cloud upload. Any trailing chunk under 5MB is saved as a + * temporary .part object unless it completes the overall upload. */ private AppendResult processPayloadChunks( UploadInfo info, PreparedStream preparedStream, UploadId id, String partObjectKey) @@ -802,7 +906,7 @@ private AppendResult processPayloadChunks( int nextPartNumber = allPartKeys.size() + 1; InputStream streamToRead = preparedStream.stream; - long optimalPartSize = calcOptimalPartSize(info.getLength() != null ? info.getLength() : 0); + long optimalPartSize = calcOptimalPartSize(info.getLength()); byte[] buffer = new byte[8192]; long totalBytesAppended = 0; @@ -813,95 +917,150 @@ private AppendResult processPayloadChunks( boolean streamFinished = false; MaxAppendSizeExceededException maxAppendSizeException = null; + IOException readException = null; - while (!streamFinished) { - File tempChunkFile = - Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); - // Do not call tempChunkFile.deleteOnExit() here. In high-throughput long-running services, - // deleteOnExit() registers entries in a static JVM set that cannot be garbage collected, - // creating an unbounded memory leak. Temp files are deleted in try-finally blocks below. + List plannedPartKeys = new ArrayList<>(); - long chunkBytesWritten = 0; - IOException readException = null; - try { - try (FileOutputStream fos = new FileOutputStream(tempChunkFile)) { - int bytesRead; - while (chunkBytesWritten < optimalPartSize - && (bytesRead = streamToRead.read(buffer)) != -1) { - long requestBytesSoFar = - (totalBytesAppended + bytesRead) - preparedStream.prependedBytes; - if (maxAppendSize != null && requestBytesSoFar > maxAppendSize) { - // If maxAppendSize is exceeded, do not discard tempChunkFile immediately. - // If preparedStream had prepended bytes from a previous incomplete .part, discarding - // tempChunkFile would permanently lose those bytes. Instead, stop reading and record - // maxAppendSizeException so the bytes currently in tempChunkFile are flushed to S3 - // and UploadInfo offset is accurately preserved. - maxAppendSizeException = - new MaxAppendSizeExceededException( - "Append payload exceeded limit of " + maxAppendSize); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + while (!streamFinished) { + File tempChunkFile = + Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); + // Do not call tempChunkFile.deleteOnExit() here. In high-throughput long-running services, + // deleteOnExit() registers entries in a static JVM set that cannot be garbage collected, + // creating an unbounded memory leak. Temp files are deleted in try-finally blocks below. + + long chunkBytesWritten = 0; + boolean handedOff = false; + try { + try (FileOutputStream fos = new FileOutputStream(tempChunkFile)) { + int bytesRead; + while (chunkBytesWritten < optimalPartSize + && (bytesRead = streamToRead.read(buffer)) != -1) { + long requestBytesSoFar = + (totalBytesAppended + bytesRead) - preparedStream.prependedBytes; + if (maxAppendSize != null && requestBytesSoFar > maxAppendSize) { + // If maxAppendSize is exceeded, do not discard tempChunkFile immediately. + // If preparedStream had prepended bytes from a previous incomplete .part, + // discarding + // tempChunkFile would permanently lose those bytes. Instead, stop reading and + // record + // maxAppendSizeException so the bytes currently in tempChunkFile are flushed to S3 + // and UploadInfo offset is accurately preserved. + maxAppendSizeException = + new MaxAppendSizeExceededException( + "Append payload exceeded limit of " + maxAppendSize); + streamFinished = true; + break; + } + fos.write(buffer, 0, bytesRead); + chunkBytesWritten += bytesRead; + totalBytesAppended += bytesRead; + } + + if (chunkBytesWritten < optimalPartSize) { streamFinished = true; - break; } - fos.write(buffer, 0, bytesRead); - chunkBytesWritten += bytesRead; - totalBytesAppended += bytesRead; + } catch (IOException e) { + readException = e; + streamFinished = true; } - if (chunkBytesWritten < optimalPartSize) { - streamFinished = true; + if (chunkBytesWritten == 0) { + break; } - } catch (IOException e) { - readException = e; - streamFinished = true; - } - if (chunkBytesWritten == 0) { - if (readException != null) { - throw readException; + // Base offset plus total bytes appended accurately measures uploaded progress without + // double-counting + long currentTotalOffset = baseOffset + totalBytesAppended; + + // Interruption Guard: If an IOException or limit exception occurred (e.g. client pause or + // connection drop), + // the chunk must NEVER be considered complete, preventing sub-5MB chunks from being + // promoted. + boolean isUploadComplete = + readException == null + && maxAppendSizeException == null + && info.getLength() != null + && currentTotalOffset >= info.getLength(); + + // Validate remaining part capacity before allocating next S3 part number + if (allPartKeys.size() + plannedPartKeys.size() >= MAX_PARTS_PER_UPLOAD + || nextPartNumber > MAX_PARTS_PER_UPLOAD) { + maxAppendSizeException = + new MaxAppendSizeExceededException( + "Upload has reached the maximum allowed S3 limit of " + + MAX_PARTS_PER_UPLOAD + + " parts."); + if (chunkBytesWritten > 0) { + int confirmedCount = uploader.drainAndComplete(4000); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + handedOff = true; + } + streamFinished = true; + break; } - if (maxAppendSizeException != null) { - throw maxAppendSizeException; + + // AWS S3 / MinIO Rule: Parts must be >= 5 MB unless it's the final part completing the + // upload + if (chunkBytesWritten >= minPartSize) { + // Full part chunk (>= 5 MB): Submit to AsyncChunkUploader pipeline for background + // upload + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + plannedPartKeys.add(chunkKey); + long bytesToUpload = chunkBytesWritten; + uploader.submitChunk( + tempChunkFile, + bytesToUpload, + chunkKey, + () -> uploadChunkToS3(chunkKey, tempChunkFile, bytesToUpload)); + handedOff = true; + + } else if (streamFinished && isUploadComplete) { + // Sub-5MB final chunk that completes the overall upload: + // First drain all preceding parts in the pipeline + int confirmedCount = uploader.drainAndComplete(4000); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); + allPartKeys.add(chunkKey); + handedOff = true; + + } else { + // Sub-5MB incomplete chunk (e.g. upload paused midway or interrupted): + // First drain all preceding parts in the pipeline + int confirmedCount = uploader.drainAndComplete(4000); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + + storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + handedOff = true; } - break; - } - // Base offset plus total bytes appended accurately measures uploaded progress without - // double-counting - long currentTotalOffset = baseOffset + totalBytesAppended; - - // Interruption Guard: If an IOException or limit exception occurred (e.g. client pause or - // connection drop), - // the chunk must NEVER be considered complete, preventing sub-5MB chunks from being - // promoted. - boolean isUploadComplete = - readException == null - && maxAppendSizeException == null - && info.getLength() != null - && currentTotalOffset >= info.getLength(); - - // AWS S3 / MinIO Rule: Parts must be >= 5 MB unless it's the final part completing the - // upload - if (chunkBytesWritten >= minPartSize || (streamFinished && isUploadComplete)) { - String chunkKey = buildChunkPartKey(id, nextPartNumber); - uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); - allPartKeys.add(chunkKey); - nextPartNumber++; - } else { - // Store sub-5MB tail chunk as temporary .part object in S3 for subsequent appends - storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + } finally { + if (!handedOff) { + FileUtils.deleteQuietly(tempChunkFile); + } } + } - if (readException != null) { - throw readException; - } - if (maxAppendSizeException != null) { - throw maxAppendSizeException; - } - } finally { - FileUtils.deleteQuietly(tempChunkFile); + // Drain any remaining in-flight chunks in the pipeline + int confirmedCount = uploader.drainAndComplete(4000); + int previouslyConfirmed = allPartKeys.size() - preparedStream.remainingPartKeys.size(); + if (confirmedCount > previouslyConfirmed) { + allPartKeys.clear(); + allPartKeys.addAll(preparedStream.remainingPartKeys); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); } } + if (readException != null) { + throw readException; + } + if (maxAppendSizeException != null) { + throw maxAppendSizeException; + } + return new AppendResult(totalBytesAppended, allPartKeys); } @@ -946,24 +1105,72 @@ private void finalizeCompletedUploadIfFinished( if (info.getLength() != null && newOffset >= info.getLength()) { List partKeys = fetchExistingPartKeys(id); - // If leftover sub-5MB .part exists, save it as final part chunk + long existingPartsTotalSize = 0L; + for (String pk : partKeys) { + try { + StatObjectResponse stat = + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(pk).build()); + existingPartsTotalSize += stat.size(); + } catch (Exception ignored) { + } + } + + // If leftover sub-5MB .part exists, apply arithmetic budget invariants String leftoverPartKey = buildIncompletePartKey(id); if (objectExists(leftoverPartKey)) { - int nextPartNum = partKeys.size() + 1; - String finalChunkKey = buildChunkPartKey(id, nextPartNum); - try (InputStream stream = - minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build())) { - byte[] bytes = IOUtils.toByteArray(stream); - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(finalChunkKey).stream( - new ByteArrayInputStream(bytes), (long) bytes.length, -1L) - .build()); - partKeys.add(finalChunkKey); - } catch (Exception e) { - throw new IOException("Failed to finalize incomplete part for ID " + id, e); + long leftoverSize = 0L; + try { + StatObjectResponse partHead = + minioClient.statObject( + StatObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build()); + if (partHead != null) { + leftoverSize = partHead.size(); + } + } catch (Exception ignored) { + } + + // Arithmetic Decision Matrix: + // Case 1: Numbered parts already satisfy the entire upload length. + // The .part buffer is an orphaned/stale duplicate (e.g. from prior pause). + // MUST DELETE to prevent byte duplication. + if (existingPartsTotalSize >= info.getLength()) { + log.info( + "Purging stale incomplete part {} (size: {}) as numbered parts already cover upload length ({})", + leftoverPartKey, + leftoverSize, + info.getLength()); + deleteObjectQuietly(leftoverPartKey); + + } else if (existingPartsTotalSize + leftoverSize == info.getLength()) { + // Case 2: Numbered parts plus this leftover buffer match the expected length exactly. + // This is a legitimate new tail written during this request. + // MUST PROMOTE to the final numbered part. + int nextPartNum = partKeys.size() + 1; + String finalChunkKey = buildChunkPartKey(id, nextPartNum); + try (InputStream stream = + minioClient.getObject( + GetObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build())) { + byte[] bytes = IOUtils.toByteArray(stream); + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(finalChunkKey).stream( + new ByteArrayInputStream(bytes), (long) bytes.length, -1L) + .build()); + partKeys.add(finalChunkKey); + } catch (Exception e) { + throw new IOException("Failed to finalize incomplete part for ID " + id, e); + } + deleteObjectQuietly(leftoverPartKey); + + } else { + // Case 3: Oversized or inconsistent leftover buffer. + log.warn( + "Discarding inconsistent incomplete part {} (size: {}, numbered parts: {}, total length: {})", + leftoverPartKey, + leftoverSize, + existingPartsTotalSize, + info.getLength()); + deleteObjectQuietly(leftoverPartKey); } - deleteObjectQuietly(leftoverPartKey); } if (!partKeys.isEmpty()) { @@ -1020,6 +1227,16 @@ private void finalizeCompletedUploadIfFinished( for (String pk : partKeys) { deleteObjectQuietly(pk); } + } else if (info.getLength() == 0L) { + // Zero-byte upload: create the empty destination object in S3 + try { + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(objectKey).stream( + new ByteArrayInputStream(new byte[0]), 0L, -1L) + .build()); + } catch (Exception e) { + throw new IOException("Failed to create empty completed object for ID " + id, e); + } } // Add checksum index if deduplication is enabled @@ -1137,11 +1354,12 @@ private void calculateAndSetOffset(UploadInfo info) { String objectKey = getS3ObjectKey(info); String partKey = buildIncompletePartKey(info.getId()); - long offset = calculateCurrentOffset(objectKey, info.getId(), partKey); + long offset = calculateCurrentOffset(objectKey, info.getId(), partKey, info.getLength()); info.setOffset(offset); } - private long calculateCurrentOffset(String objectKey, UploadId id, String partKey) { + private long calculateCurrentOffset( + String objectKey, UploadId id, String partKey, Long expectedLength) { long offset = 0; if (objectExists(objectKey)) { @@ -1155,21 +1373,29 @@ private long calculateCurrentOffset(String objectKey, UploadId id, String partKe } List partKeys = fetchExistingPartKeys(id); + long numberedPartsSize = 0L; for (String pk : partKeys) { try { StatObjectResponse stat = minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(pk).build()); - offset += stat.size(); + numberedPartsSize += stat.size(); } catch (Exception ignored) { } } + offset += numberedPartsSize; - if (!partKeys.contains(partKey)) { + if (!partKeys.contains(partKey) && objectExists(partKey)) { try { StatObjectResponse partHead = minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(partKey).build()); if (partHead != null) { - offset += partHead.size(); + long partSize = partHead.size(); + if (expectedLength == null || numberedPartsSize + partSize <= expectedLength) { + offset += partSize; + } else if (numberedPartsSize >= expectedLength) { + // Numbered parts already cover the file; partKey is a stale orphan + deleteObjectQuietly(partKey); + } } } catch (ErrorResponseException ignored) { } catch (Exception e) { @@ -1201,14 +1427,44 @@ private void deleteAllPartObjectsQuietly(UploadId id) { } } - private long calcOptimalPartSize(long totalSize) { + /** + * Calculates auto-calibrated optimal chunk part size based on total upload length. + * + *

AWS S3 multipart uploads enforce a strict ceiling of 10,000 parts per object. When an upload + * length exceeds 80 GB (10,000 * 8 MB), the part size dynamically scales up (e.g. ~105 MB for 1 + * TB, ~525 MB for 5 TB) so that the entire upload is guaranteed to fit within 10,000 parts, + * bounded by S3's 5 GB maximum part limit. + * + * @param totalLength The announced total upload length in bytes, or null if unknown/deferred + * @return The calibrated optimal part size in bytes + */ + long calcOptimalPartSize(Long totalLength) { long partSize = preferredPartSize; - if (totalSize > 0 && totalSize / partSize >= 10000) { - partSize = (totalSize / 10000) + 1; + if (totalLength != null && totalLength > 0 && totalLength / partSize >= MAX_PARTS_PER_UPLOAD) { + partSize = (totalLength / MAX_PARTS_PER_UPLOAD) + 1; } return Math.max(minPartSize, Math.min(partSize, DEFAULT_MAX_PART_SIZE)); } + /** + * Validates that the upload has sufficient part budget remaining within S3's 10,000 parts limit. + * + *

Why: AWS S3 enforces a strict maximum ceiling of 10,000 parts per multipart upload. + * If an upload receives too many small chunks, it risks hitting this limit before finishing. This + * check runs in O(1) time at the trust boundary to prevent deadlocked uploads. + * + * @param upload The current upload metadata + * @param currentPartCount Number of already committed parts + * @throws MaxAppendSizeExceededException If remaining part capacity is exhausted + */ + void validateRemainingPartBudget(UploadInfo upload, int currentPartCount) + throws MaxAppendSizeExceededException { + if (currentPartCount >= MAX_PARTS_PER_UPLOAD) { + throw new MaxAppendSizeExceededException( + "Upload has reached the maximum allowed S3 limit of " + MAX_PARTS_PER_UPLOAD + " parts."); + } + } + private void putChecksumIndex(String checksum, ChecksumAlgorithm algorithm, UploadId parentId) { String key = buildChecksumKey(checksum, algorithm); try { diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java new file mode 100644 index 00000000..f523e40c --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -0,0 +1,290 @@ +package me.desair.tus.server.upload.util; + +import java.io.File; +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Coordinates asynchronous, pipelined uploading of chunk files to cloud storage backends (S3, + * Azure). + * + *

Uses a bounded 3-slot pipeline (triple-buffering): + * + *

    + *
  • Slot 1 (Receiving): The caller/servlet thread reads incoming request bytes into a + * local temp chunk file. + *
  • Slot 2 (Waiting): At most one completed chunk file held on disk waiting for the + * active cloud upload to complete. + *
  • Slot 3 (Uploading): A discrete task running on the shared thread pool actively + * streaming a chunk to the cloud. + *
+ * + *

This pipeline overlaps network reception from the client with upstream cloud staging, + * eliminating client TCP stall periods while maintaining strict upper bounds on memory, disk space, + * and thread usage. + */ +public class AsyncChunkUploader implements AutoCloseable { + + private static final Logger log = LoggerFactory.getLogger(AsyncChunkUploader.class); + + @FunctionalInterface + public interface ChunkUploadAction { + /** + * Executes the cloud upload action for a single chunk. + * + * @throws Exception If the upload to the cloud provider fails + */ + void upload() throws Exception; + } + + private final ExecutorService executor; + + // Slot 3: In-flight upload task and its associated local file and key + private Future inFlightUpload; + private File inFlightFile; + private String inFlightKey; + + // Slot 2: Waiting chunk awaiting execution + private File waitingChunkFile; + private long waitingChunkSize; + private String waitingKey; + private ChunkUploadAction waitingAction; + + // Number of chunks confirmed uploaded to the cloud + private int confirmedCount; + + // Set to true once drainAndComplete() has successfully finished + private boolean completed; + + /** + * Constructs an uploader using the given shared executor. + * + * @param executor Shared thread pool executor for background chunk uploads + */ + public AsyncChunkUploader(ExecutorService executor) { + this.executor = Objects.requireNonNull(executor, "ExecutorService must not be null"); + } + + /** + * Submits a newly completed local chunk file into the upload pipeline. + * + *

If Slot 3 is idle or completed, the chunk begins uploading immediately. If Slot 3 is busy + * and Slot 2 is empty, the chunk is placed into Slot 2. If both Slot 3 and Slot 2 are occupied, + * this call blocks until Slot 3 finishes (applying backpressure to the client stream). + * + * @param tempFile The local chunk file containing the chunk bytes + * @param size The size of the chunk in bytes + * @param preassignedKey The pre-assigned cloud object key or block ID + * @param uploadAction The upload action to execute on the worker thread + * @throws IOException If a previous chunk upload failed or thread was interrupted + */ + public void submitChunk( + File tempFile, long size, String preassignedKey, ChunkUploadAction uploadAction) + throws IOException { + Objects.requireNonNull(tempFile, "tempFile must not be null"); + Objects.requireNonNull(uploadAction, "uploadAction must not be null"); + + // Check if previous in-flight upload has finished, advancing confirmation count + if (inFlightUpload != null && inFlightUpload.isDone()) { + try { + checkAndConfirmInFlight(); + } catch (IOException e) { + FileUtils.deleteQuietly(tempFile); + throw e; + } + } + + try { + if (waitingChunkFile == null) { + if (inFlightUpload == null) { + // Branch 1: Slot 3 is free and Slot 2 is empty. Submit directly to Slot 3. + submitToSlot3(tempFile, size, preassignedKey, uploadAction); + } else { + // Branch 2: Slot 3 is actively uploading, but Slot 2 (waiting) is free. + // Store in Slot 2 and return immediately so the caller can read the next chunk. + this.waitingChunkFile = tempFile; + this.waitingChunkSize = size; + this.waitingKey = preassignedKey; + this.waitingAction = uploadAction; + } + } else { + // Branch 3: Slot 2 already has a waiting chunk. + // To preserve strict sequential chunk ordering, Slot 2's chunk must be uploaded before the + // new chunk. + // If Slot 3 is still active, apply backpressure by waiting for it to finish. + if (inFlightUpload != null) { + waitForInFlight(); + } + + // Slot 3 is now free. Promote Slot 2 into Slot 3. + submitToSlot3(waitingChunkFile, waitingChunkSize, waitingKey, waitingAction); + this.waitingChunkFile = null; + this.waitingAction = null; + + // Place the newly arrived chunk into the now-empty Slot 2. + this.waitingChunkFile = tempFile; + this.waitingChunkSize = size; + this.waitingKey = preassignedKey; + this.waitingAction = uploadAction; + } + } catch (IOException | RuntimeException e) { + FileUtils.deleteQuietly(tempFile); + throw e; + } + } + + /** + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) up to the specified timeout. + * + *

Used upon stream completion or client interruption to ensure all received bytes are + * persisted to cloud storage before releasing the upload lock. + * + * @param timeoutMs Maximum time in milliseconds to wait for in-flight and waiting chunks + * @return The total number of confirmed successfully uploaded chunks + * @throws IOException If any chunk upload fails or times out + */ + public int drainAndComplete(long timeoutMs) throws IOException { + long deadline = System.currentTimeMillis() + timeoutMs; + + // 1. Await Slot 3 if active + if (inFlightUpload != null) { + long remaining = Math.max(1L, deadline - System.currentTimeMillis()); + awaitFuture(inFlightUpload, remaining, inFlightKey); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } + + // 2. If Slot 2 has a waiting chunk, promote it to Slot 3 and await its completion + if (waitingChunkFile != null) { + submitToSlot3(waitingChunkFile, waitingChunkSize, waitingKey, waitingAction); + waitingChunkFile = null; + waitingAction = null; + + long remaining = Math.max(1L, deadline - System.currentTimeMillis()); + awaitFuture(inFlightUpload, remaining, inFlightKey); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } + + completed = true; + return confirmedCount; + } + + /** + * Aborts the pipeline, cancelling any active cloud upload and deleting any remaining temp files. + */ + public void abort() { + if (inFlightUpload != null && !inFlightUpload.isDone()) { + inFlightUpload.cancel(true); + } + if (inFlightFile != null) { + FileUtils.deleteQuietly(inFlightFile); + inFlightFile = null; + } + if (waitingChunkFile != null) { + FileUtils.deleteQuietly(waitingChunkFile); + waitingChunkFile = null; + waitingAction = null; + } + } + + /** + * Returns the number of chunks confirmed successfully uploaded. + * + * @return Confirmed chunk count + */ + public int getConfirmedCount() { + return confirmedCount; + } + + @Override + public void close() { + // If drainAndComplete() did not finish successfully, abort and clean up uncommitted files + if (!completed) { + abort(); + } + } + + private void submitToSlot3( + File tempFile, long size, String preassignedKey, ChunkUploadAction uploadAction) { + this.inFlightFile = tempFile; + this.inFlightKey = preassignedKey; + + // Wrap the upload action so temp file cleanup is guaranteed on the worker thread + this.inFlightUpload = + executor.submit( + () -> { + try { + uploadAction.upload(); + } finally { + // The worker owns tempFile once submitted; delete it immediately upon upload + // completion + FileUtils.deleteQuietly(tempFile); + } + return null; + }); + } + + private void checkAndConfirmInFlight() throws IOException { + try { + inFlightUpload.get(); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException( + "Interrupted while checking in-flight chunk upload for " + inFlightKey, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, inFlightKey); + } + } + + private void waitForInFlight() throws IOException { + checkAndConfirmInFlight(); + } + + private void awaitFuture(Future future, long timeoutMs, String key) throws IOException { + try { + future.get(timeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException("Interrupted waiting for chunk upload for key " + key, e); + } catch (TimeoutException e) { + abort(); + throw new IOException( + "Timed out after " + timeoutMs + "ms waiting for chunk upload for key " + key, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, key); + } + } + + private IOException translateExecutionException(ExecutionException e, String key) { + Throwable cause = e.getCause(); + if (cause instanceof IOException) { + return (IOException) cause; + } else if (cause instanceof RuntimeException) { + return new IOException("Upload failed for key " + key + ": " + cause.getMessage(), cause); + } else { + return new IOException( + "Unexpected error during chunk upload for key " + key, cause != null ? cause : e); + } + } +} diff --git a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java index e818de35..504dd8b2 100644 --- a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java +++ b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java @@ -801,6 +801,95 @@ public void testUploadCompletionListenerRufhPatchAppend() throws Exception { } } + // =============================================================================================== + // USE CASE 16: Min-Append-Size Exemption for Completing Uploads (§4.1.4) + // =============================================================================================== + + /** + * Section 4.1.4 (Limits): "min-append-size: An Integer indicating the minimum number of bytes + * that MUST be appended with each upload append request. This limit does not apply to upload + * creation requests with no content, or to requests completing the upload by including the + * Upload-Complete: ?1 header field." + * + *

Use Case: Configure min-append-size (1 MB), then verify that an optimistic single-request + * creation upload with Upload-Complete: ?1 and a payload smaller than 1 MB bypasses the + * min-append-size limit and completes successfully. + */ + @Test + public void testCompletingCreationBypassesMinAppendSize() throws Exception { + tusFileUploadService.withMinAppendSize(1024L * 1024L); + + String payload = "Small optimistic payload under min-append-size."; + + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + servletRequest.setContent(payload.getBytes(StandardCharsets.UTF_8)); + + UploadInfo completedInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + + assertResponseStatus(HttpServletResponse.SC_OK); + assertResponseHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + assertNotNull(completedInfo); + assertFalse(completedInfo.isUploadInProgress()); + assertEquals(Long.valueOf(payload.length()), completedInfo.getOffset()); + + try (InputStream stream = tusFileUploadService.getUploadedBytes(completedInfo)) { + assertThat(IOUtils.toString(stream, StandardCharsets.UTF_8), is(payload)); + } + } + + /** + * Section 4.1.4 (Limits): "min-append-size: An Integer indicating the minimum number of bytes + * that MUST be appended with each upload append request. This limit does not apply to upload + * creation requests with no content, or to requests completing the upload by including the + * Upload-Complete: ?1 header field." + * + *

Use Case: Configure min-append-size (1 MB), initiate an upload without content, and append a + * completing chunk smaller than 1 MB with Upload-Complete: ?1. Verify that the completing append + * is exempt from min-append-size and completes the upload cleanly. + */ + @Test + public void testCompletingAppendPatchBypassesMinAppendSize() throws Exception { + tusFileUploadService.withMinAppendSize(1024L * 1024L); + + String uploadContent = "Small final chunk under min-append-size."; + + // Step 1: Initiate upload with POST, Upload-Complete: ?0, declared Upload-Length, and no body + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, String.valueOf(uploadContent.length())); + + UploadInfo createdInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + assertNotNull(createdInfo); + String uploadLocation = servletResponse.getHeader(HttpHeader.LOCATION); + + // Step 2: Append completing chunk with Upload-Complete: ?1 + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(uploadLocation); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + servletRequest.setContent(uploadContent.getBytes(StandardCharsets.UTF_8)); + + UploadInfo completedInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_OK); + assertResponseHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + assertNotNull(completedInfo); + assertFalse(completedInfo.isUploadInProgress()); + assertEquals(Long.valueOf(uploadContent.length()), completedInfo.getOffset()); + + try (InputStream stream = tusFileUploadService.getUploadedBytes(completedInfo)) { + assertThat(IOUtils.toString(stream, StandardCharsets.UTF_8), is(uploadContent)); + } + } + // =============================================================================================== // ASSERTION HELPERS // =============================================================================================== diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 90dc8a99..981ebace 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -331,6 +331,7 @@ public void testWithUploadStorageServicePreservesConfiguration() { service.withMaxAppendSize(1024L); service.withMinAppendSize(512L); service.withMinSize(2048L); + service.withCloudUploadThreadPoolSize(18); service.withUploadDeduplication(true); UploadStorageService newStorage = mock(UploadStorageService.class); @@ -340,6 +341,7 @@ public void testWithUploadStorageServicePreservesConfiguration() { verify(newStorage).setMaxAppendSize(1024L); verify(newStorage).setMinAppendSize(512L); verify(newStorage).setMinSize(2048L); + verify(newStorage).setCloudUploadThreadPoolSize(18); verify(newStorage).setUploadDeduplicationEnabled(true); } @@ -349,11 +351,25 @@ public void testThreadLocalCacheDelegatesAppendAndMinSizes() { service.withMaxAppendSize(1024L); service.withMinAppendSize(512L); service.withMinSize(2048L); + service.withCloudUploadThreadPoolSize(16); service.withThreadLocalCache(true); assertThat(service.getUploadStorageService().getMaxAppendSize(), is(1024L)); assertThat(service.getUploadStorageService().getMinAppendSize(), is(512L)); assertThat(service.getUploadStorageService().getMinSize(), is(2048L)); + assertThat(service.getUploadStorageService().getCloudUploadThreadPoolSize(), is(16)); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithCloudUploadThreadPoolSizeZeroThrowsException() { + TusFileUploadService service = new TusFileUploadService(); + service.withCloudUploadThreadPoolSize(0); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithCloudUploadThreadPoolSizeNegativeThrowsException() { + TusFileUploadService service = new TusFileUploadService(); + service.withCloudUploadThreadPoolSize(-5); } @Test diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index 46f3a3d1..609f1a96 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -162,6 +162,7 @@ public void testProcessCallFiveParameterMethodAndAppendReturnsNull() throws Exce request.setRequestURI("/files/append-id"); request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.addHeader(HttpHeader.CONTENT_LENGTH, "4"); request.setContent("data".getBytes()); UploadInfo info = new UploadInfo(); @@ -215,4 +216,93 @@ public void testProcessIsFinishedCombinations() throws Exception { assertThat(response.getStatus(), is(200)); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + * + *

§4.2.1: "If the upload length is not known when creating the upload resource, the + * Upload-Length header field is omitted, and the length is deferred... In subsequent requests, + * the upload length can be indicated by including the Upload-Length header field or by completing + * the upload using the Upload-Complete: ?1 header field." + */ + @Test + public void testProcessCompletingAppendOnDeferredLengthUploadSetsLength() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "100"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "final-bytes".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(100L); + info.setLength(null); // Deferred length + + UploadInfo updated = new UploadInfo(); + updated.setId(info.getId()); + updated.setOffset(100L + content.length); + updated.setLength(100L + content.length); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(updated); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).update(info); + assertThat(info.getLength(), is(100L + content.length)); + assertThat(response.getStatus(), is(200)); + assertThat( + response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(100L + content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } + + @Test + public void testProcessCompletingAppendOnDeferredLengthUploadWithNullOffsetSetsLength() + throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "initial-and-final-bytes".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(null); + info.setLength(null); // Deferred length + + UploadInfo updated = new UploadInfo(); + updated.setId(info.getId()); + updated.setOffset((long) content.length); + updated.setLength((long) content.length); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(updated); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).update(info); + assertThat(info.getLength(), is((long) content.length)); + assertThat(response.getStatus(), is(200)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index f6dce0c7..e68c1776 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -473,4 +473,46 @@ public void testProcessCompletedUploadCreationWithAbsoluteUploadUri() throws Exc assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("100")); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + * + *

§4.2.1: "If the upload length is not known when creating the upload resource, the + * Upload-Length header field is omitted, and the length is deferred... In subsequent requests, + * the upload length can be indicated by including the Upload-Length header field or by completing + * the upload using the Upload-Complete: ?1 header field." + */ + @Test + public void testProcessCreationWithUploadCompleteAndContentLengthSetsAnnouncedLength() + throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "hello world".getBytes(); + request.setContent(content); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("complete-no-length-id")); + createdInfo.setLength((long) content.length); + createdInfo.setOffset((long) content.length); + + when(storageService.create(captor.capture(), nullable(String.class))).thenReturn(createdInfo); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(createdInfo); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(captor.getValue().getLength(), is((long) content.length)); + assertThat(response.getStatus(), is(200)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java b/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java index 27f03d77..406d7b52 100644 --- a/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java +++ b/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java @@ -96,8 +96,8 @@ public void testGetRawInterimResponseWithExistingUploadAndStorageException() thr @Test public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() throws Exception { MockHttpServletRequest request = new MockHttpServletRequest(); - request.setMethod("PATCH"); - request.setRequestURI("/files/not-found-123"); + request.setMethod("POST"); + request.setRequestURI("/files"); request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); // Host header is not set (null host) @@ -107,8 +107,6 @@ public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() thr me.desair.tus.server.upload.UploadInfo created = new me.desair.tus.server.upload.UploadInfo(); created.setId(new me.desair.tus.server.upload.UploadId("created-456")); - org.mockito.Mockito.when(mockStorage.getUploadInfo("/files/not-found-123", "owner")) - .thenReturn(null); org.mockito.Mockito.when( mockStorage.create( org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.eq("owner"))) @@ -119,6 +117,23 @@ public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() thr assertTrue(raw.contains("Location: /files/created-456")); } + @Test + public void testGetRawInterimResponseWithPatchUploadNotFoundReturnsNull() throws Exception { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("PATCH"); + request.setRequestURI("/files/not-found-123"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + + me.desair.tus.server.upload.UploadStorageService mockStorage = + org.mockito.Mockito.mock(me.desair.tus.server.upload.UploadStorageService.class); + org.mockito.Mockito.when(mockStorage.getUploadUri()).thenReturn("/files"); + org.mockito.Mockito.when(mockStorage.getUploadInfo("/files/not-found-123", "owner")) + .thenReturn(null); + + String raw = RufhInterimResponseUtil.getRawInterimResponse(request, mockStorage, "owner"); + assertNull(raw); + } + @Test public void testGetRawInterimResponseWithAbsoluteUploadUri() throws Exception { MockHttpServletRequest request = new MockHttpServletRequest(); diff --git a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java index a3d2307e..1933b822 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java @@ -142,6 +142,9 @@ public void testDefaultAppendAndSizeConfigurationMethods() throws Exception { dummyStorageService.setJsonSerializationEnabled(true); assertThat(dummyStorageService.isJsonSerializationEnabled(), is(false)); + dummyStorageService.setCloudUploadThreadPoolSize(15); + assertThat(dummyStorageService.getCloudUploadThreadPoolSize(), is(10)); + // Default close is a no-op dummyStorageService.close(); } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java index 32360897..464f526f 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java @@ -250,4 +250,31 @@ public void getPartialUploadsShouldThrowOnOwnerMismatch() throws Exception { // final upload's ownerKey, preventing unauthorized partial stitching. concatenationService.getPartialUploads(finalInfo); } + + @Test + public void getAuthorizedBlobUrlShouldReturnNullOnNullBlobClient() { + org.junit.Assert.assertNull(concatenationService.getAuthorizedBlobUrl(null)); + } + + @Test + public void getAuthorizedBlobUrlShouldAppendSasWhenGenerationSucceeds() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())).thenReturn("sig=mocked-token&sp=r"); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals( + "https://account.blob.core.windows.net/container/blob?sig=mocked-token&sp=r", result); + } + + @Test + public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasGenerationThrows() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())) + .thenThrow(new IllegalStateException("No shared key credentials")); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals("https://account.blob.core.windows.net/container/blob", result); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index afffb6ef..9f4c052e 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -233,4 +233,46 @@ public void calcOptimalBlockSizeCalculations() { long largeLength = 50_000L * 16 * 1024 * 1024L; assertEquals((largeLength / 50_000L) + 1, storageService.calcOptimalBlockSize(largeLength)); } + + @Test + public void testCloudUploadThreadPoolSizeConfiguration() { + assertEquals( + "Default thread pool size is 10", 10, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(30); + assertEquals( + "Updated thread pool size is 30", 30, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(4); + assertEquals("Reduced thread pool size is 4", 4, storageService.getCloudUploadThreadPoolSize()); + + try { + storageService.setCloudUploadThreadPoolSize(0); + org.junit.Assert.fail("Should reject 0 pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + + try { + storageService.setCloudUploadThreadPoolSize(-1); + org.junit.Assert.fail("Should reject negative pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + } + + @Test + public void testCloseGracefulShutdown() throws Exception { + // Verify closing storage service cleanly terminates background upload executor without error + storageService.close(); + // KISS: verifying method executes cleanly without throwing an exception + + // Verify close handles thread interruption gracefully + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java index d11f2a7f..067e4be0 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java @@ -69,6 +69,9 @@ public void testParseErrorResponseApiNotImplemented() { public void testParseErrorResponseAccessDenied() { BlobStorageException ex = createException(403, BlobErrorCode.AUTHORIZATION_FAILURE); assertEquals(AzureErrorType.ACCESS_DENIED, AzureUtils.parseErrorResponse(ex)); + + BlobStorageException ex401 = createException(401, null); + assertEquals(AzureErrorType.ACCESS_DENIED, AzureUtils.parseErrorResponse(ex401)); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 6f2bbc67..44505494 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -73,6 +73,25 @@ public void minAppendSizeNotMetShouldThrow() throws Exception { storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); } + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + */ + @Test + public void completingUploadBypassesMinAppendSize() throws Exception { + storageService.setMinAppendSize(100L); + + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + UploadInfo completed = + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertNotNull(completed); + assertEquals(Long.valueOf(10L), completed.getOffset()); + assertFalse(completed.isUploadInProgress()); + } + @Test(expected = MaxAppendSizeExceededException.class) public void appendExceedsMaxAppendSizeShouldThrow() throws Exception { storageService.setMaxAppendSize(5L); diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index becc861d..d27e22cb 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -161,6 +161,30 @@ public void testDelegateMethods() throws Exception { assertEquals(0, service.getMaxUploadSize()); verify(mockStorage, times(1)).getMaxUploadSize(); + service.setMaxAppendSize(500L); + verify(mockStorage, times(1)).setMaxAppendSize(500L); + + service.getMaxAppendSize(); + verify(mockStorage, times(1)).getMaxAppendSize(); + + service.setMinAppendSize(200L); + verify(mockStorage, times(1)).setMinAppendSize(200L); + + service.getMinAppendSize(); + verify(mockStorage, times(1)).getMinAppendSize(); + + service.setMinSize(300L); + verify(mockStorage, times(1)).setMinSize(300L); + + service.getMinSize(); + verify(mockStorage, times(1)).getMinSize(); + + service.setCloudUploadThreadPoolSize(12); + verify(mockStorage, times(1)).setCloudUploadThreadPoolSize(12); + + service.getCloudUploadThreadPoolSize(); + verify(mockStorage, times(1)).getCloudUploadThreadPoolSize(); + when(mockStorage.create(info, "owner")).thenReturn(info); assertEquals(info, service.create(info, "owner")); verify(mockStorage, times(1)).create(info, "owner"); diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index cd08415d..486be745 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -1106,4 +1106,21 @@ public void testUnsafeChildDataFileDeletionThrowsIllegalArgumentException() thro storageService.update(child); } + + @Test + public void testSetAndGetCloudUploadThreadPoolSize() { + assertThat(storageService.getCloudUploadThreadPoolSize(), is(10)); + storageService.setCloudUploadThreadPoolSize(24); + assertThat(storageService.getCloudUploadThreadPoolSize(), is(24)); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetCloudUploadThreadPoolSizeZeroThrowsIllegalArgumentException() { + storageService.setCloudUploadThreadPoolSize(0); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetCloudUploadThreadPoolSizeNegativeThrowsIllegalArgumentException() { + storageService.setCloudUploadThreadPoolSize(-1); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 868a3c1e..1b43b0d8 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -33,7 +33,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import java.util.Collections; import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinUploadLengthNotReachedException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -782,6 +784,26 @@ public void testAppendThrowsMinAppendSizeNotMetException() throws Exception { storageService.append(info, new ByteArrayInputStream(new byte[100])); } + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + */ + @Test + public void testAppendCompletingUploadBypassesMinAppendSize() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e")); + info.setLength(100L); + + String json = UploadInfoJsonSerializer.serialize(info); + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer(invocation -> mockGetObjectResponse(json.getBytes())); + + storageService.setMinAppendSize(500L); + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[100])); + assertEquals(Long.valueOf(100L), result.getOffset()); + assertFalse(result.isUploadInProgress()); + } + @Test(expected = me.desair.tus.server.exception.MaxUploadLengthExceededException.class) public void testAppendThrowsMaxUploadLengthExceededException() throws Exception { UploadInfo info = new UploadInfo(); @@ -928,6 +950,91 @@ public void testFinalizeCompletedUploadWithLeftoverIncompletePart() throws Excep storageService.append(info, new ByteArrayInputStream(new byte[50])); } + @Test + public void testFinalizeCompletedUploadWithInconsistentLeftoverPartPurged() throws Exception { + UploadInfo info = new UploadInfo(); + UploadId id = new UploadId("inconsistent-part-123"); + info.setId(id); + info.setLength(100L); + info.setOffset(0L); + + String json = UploadInfoJsonSerializer.serialize(info); + + StatObjectResponse leftoverHead = mock(StatObjectResponse.class); + // Leftover is 30 bytes, but total length is 100 bytes (0 parts + 30 != 100) -> Case 3 + when(leftoverHead.size()).thenReturn(30L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".part")) { + return leftoverHead; + } + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + storageService.append(info, new ByteArrayInputStream(new byte[100])); + + // Verify inconsistent .part buffer was deleted + verify(minioClient, atLeastOnce()) + .removeObject( + argThat( + (RemoveObjectArgs args) -> + args.object().equals("metadata/inconsistent-part-123.part"))); + } + + @Test(expected = IOException.class) + public void testFinalizeCompletedUploadLeftoverPartPromotionExceptionThrowsIOException() + throws Exception { + UploadInfo info = new UploadInfo(); + UploadId id = new UploadId("leftover-promo-err-123"); + info.setId(id); + info.setLength(50L); + info.setOffset(0L); + + String json = UploadInfoJsonSerializer.serialize(info); + + StatObjectResponse leftoverHead = mock(StatObjectResponse.class); + when(leftoverHead.size()).thenReturn(50L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".part")) { + return leftoverHead; + } + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + throw new IOException("Simulated network failure streaming leftover part"); + }); + + storageService.append(info, new ByteArrayInputStream(new byte[50])); + } + @Test public void testFinalizeCompletedUploadZeroLength() throws Exception { UploadInfo info = new UploadInfo(); @@ -1796,6 +1903,320 @@ public void testCleanupExpiredUploadsPrunesStaleTempFilesAndChecksumIndices() th } } + @Test + public void testCloudUploadThreadPoolSizeConfiguration() { + assertEquals( + "Default thread pool size is 10", 10, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(25); + assertEquals( + "Updated thread pool size is 25", 25, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(4); + assertEquals("Reduced thread pool size is 4", 4, storageService.getCloudUploadThreadPoolSize()); + + try { + storageService.setCloudUploadThreadPoolSize(0); + fail("Should reject 0 pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + + try { + storageService.setCloudUploadThreadPoolSize(-5); + fail("Should reject negative pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + } + + @Test + public void testCloseGracefulShutdown() throws Exception { + // Verify closing storage service cleanly terminates background upload executor without error + storageService.close(); + // KISS: verifying method executes cleanly without throwing an exception + + // Verify close handles thread interruption gracefully + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + } + + @Test + public void testFinalizeUploadWithStaleIncompletePartIgnored() throws Exception { + UploadId uploadId = new UploadId("stale-part-test"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setOffset(0L); + info.setLength(3425070L); // 3.42 MB total upload length + + String infoJson = UploadInfoJsonSerializer.serialize(info); + String partKey1 = "metadata/stale-part-test.part.00001"; + String stalePartKey = "metadata/stale-part-test.part"; + + Item part1Item = mock(Item.class); + when(part1Item.objectName()).thenReturn(partKey1); + + StatObjectResponse part1Stat = mock(StatObjectResponse.class); + when(part1Stat.size()).thenReturn(3425070L); // Numbered part already covers full 3.42 MB + + StatObjectResponse stalePartStat = mock(StatObjectResponse.class); + when(stalePartStat.size()).thenReturn(1277952L); // Stale leftover from prior pause + + StatObjectResponse objectNotExists = mock(StatObjectResponse.class); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenAnswer( + invocation -> { + ListObjectsArgs args = invocation.getArgument(0); + if (args.prefix().startsWith("metadata/stale-part-test.part.")) { + return Collections.singletonList(new Result<>(part1Item)); + } + return Collections.emptyList(); + }); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals(partKey1)) { + return part1Stat; + } else if (args.object().equals(stalePartKey)) { + return stalePartStat; + } else if (args.object().equals("uploads/stale-part-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return objectNotExists; + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(infoJson.getBytes()); + } + return mockGetObjectResponse(new byte[0]); + }); + + // Append 0 bytes to trigger finalization check + ByteArrayInputStream emptyStream = new ByteArrayInputStream(new byte[0]); + UploadInfo result = storageService.append(info, emptyStream); + + assertNotNull(result); + // Verify that the stale .part buffer was deleted via removeObject and NOT promoted to + // part.00002 + verify(minioClient, atLeastOnce()) + .removeObject( + argThat( + (RemoveObjectArgs args) -> args.object().equals("metadata/stale-part-test.part"))); + } + + @Test + public void testFinalizeUploadWithLegitimateNewIncompletePartPromoted() throws Exception { + UploadId uploadId = new UploadId("legit-part-test"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setOffset(0L); + info.setLength(10000000L); // 10 MB total length + + String infoJson = UploadInfoJsonSerializer.serialize(info); + String partKey1 = "metadata/legit-part-test.part.00001"; + String legitPartKey = "metadata/legit-part-test.part"; + + Item part1Item = mock(Item.class); + when(part1Item.objectName()).thenReturn(partKey1); + + StatObjectResponse part1Stat = mock(StatObjectResponse.class); + when(part1Stat.size()).thenReturn(8000000L); // 8 MB numbered part + + StatObjectResponse legitPartStat = mock(StatObjectResponse.class); + when(legitPartStat.size()).thenReturn(2000000L); // 2 MB tail; 8 MB + 2 MB == 10 MB exact match! + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenAnswer( + invocation -> { + ListObjectsArgs args = invocation.getArgument(0); + if (args.prefix().startsWith("metadata/legit-part-test.part.")) { + return Collections.singletonList(new Result<>(part1Item)); + } + return Collections.emptyList(); + }); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals(partKey1)) { + return part1Stat; + } else if (args.object().equals(legitPartKey)) { + return legitPartStat; + } else if (args.object().equals("uploads/legit-part-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return mock(StatObjectResponse.class); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(infoJson.getBytes()); + } else if (args.object().equals(legitPartKey)) { + return mockGetObjectResponse(new byte[2000000]); + } + return mockGetObjectResponse(new byte[0]); + }); + + ByteArrayInputStream emptyStream = new ByteArrayInputStream(new byte[0]); + UploadInfo result = storageService.append(info, emptyStream); + + assertNotNull(result); + // Verify that the legitimate 2 MB tail was promoted to part.00002 + verify(minioClient) + .putObject( + argThat( + (PutObjectArgs args) -> + args.object().equals("metadata/legit-part-test.part.00002"))); + } + + @Test + public void testCalculateCurrentOffsetDoesNotDoubleCountOrExceedLength() throws Exception { + UploadId uploadId = new UploadId("calc-offset-test"); + String infoJson = "{\"id\":\"calc-offset-test\",\"length\":3425070,\"offset\":null}"; + + String partKey1 = "metadata/calc-offset-test.part.00001"; + String stalePartKey = "metadata/calc-offset-test.part"; + + Item part1Item = mock(Item.class); + when(part1Item.objectName()).thenReturn(partKey1); + + StatObjectResponse part1Stat = mock(StatObjectResponse.class); + when(part1Stat.size()).thenReturn(3425070L); // 3.42 MB numbered part + + StatObjectResponse stalePartStat = mock(StatObjectResponse.class); + when(stalePartStat.size()).thenReturn(1277952L); // Stale 1.27 MB leftover + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(infoJson.getBytes()); + } + return mockGetObjectResponse(new byte[0]); + }); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenAnswer( + invocation -> { + ListObjectsArgs args = invocation.getArgument(0); + if (args.prefix().startsWith("metadata/calc-offset-test.part.")) { + return Collections.singletonList(new Result<>(part1Item)); + } + return Collections.emptyList(); + }); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals(partKey1)) { + return part1Stat; + } else if (args.object().equals(stalePartKey)) { + return stalePartStat; + } else if (args.object().equals("uploads/calc-offset-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return mock(StatObjectResponse.class); + }); + + UploadInfo fetched = storageService.getUploadInfo(uploadId); + assertNotNull(fetched); + // Must be exactly 3,425,070 bytes (not 4,703,022 bytes with double-counted stale part!) + assertEquals(Long.valueOf(3425070L), fetched.getOffset()); + + // Verify stale .part buffer was deleted on the fly + verify(minioClient, atLeastOnce()) + .removeObject( + argThat( + (RemoveObjectArgs args) -> args.object().equals("metadata/calc-offset-test.part"))); + } + + @Test + public void testCalcOptimalPartSizeCalculations() { + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(null)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(0L)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(100L)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(100L * 1024 * 1024)); + assertEquals( + 8 * 1024 * 1024L, storageService.calcOptimalPartSize(10_000L * 8 * 1024 * 1024L - 1)); + + long largeLength = 10_000L * 16 * 1024 * 1024L; + assertEquals((largeLength / 10_000L) + 1, storageService.calcOptimalPartSize(largeLength)); + + // For 1 TB, part size auto-scales up so upload fits within 10,000 parts + long oneTb = 1024L * 1024 * 1024 * 1024L; + assertEquals((oneTb / 10_000L) + 1, storageService.calcOptimalPartSize(oneTb)); + + // For 5 TB (S3 max limit), part size scales up to ~524.3 MB + long fiveTb = 5L * 1024 * 1024 * 1024 * 1024L; + assertEquals((fiveTb / 10_000L) + 1, storageService.calcOptimalPartSize(fiveTb)); + } + + @Test + public void testSetAndGetPreferredPartSize() { + assertEquals(8 * 1024 * 1024L, storageService.getPreferredPartSize()); + + storageService.setPreferredPartSize(16 * 1024 * 1024L); + assertEquals(16 * 1024 * 1024L, storageService.getPreferredPartSize()); + + try { + storageService.setPreferredPartSize(4 * 1024 * 1024L); // Below 5MB limit + fail("Should reject part size below 5MB"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("Preferred part size must be between")); + } + + try { + storageService.setPreferredPartSize(6L * 1024 * 1024 * 1024L); // Above 5GB limit + fail("Should reject part size above 5GB"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("Preferred part size must be between")); + } + } + + @Test + public void testValidateRemainingPartBudget() throws Exception { + UploadInfo info = new UploadInfo(); + storageService.validateRemainingPartBudget(info, 9999); + // KISS: 9999 parts is within budget, verifies method completes cleanly + + try { + storageService.validateRemainingPartBudget(info, 10000); + fail("Should throw MaxAppendSizeExceededException at 10000 parts"); + } catch (MaxAppendSizeExceededException expected) { + assertTrue(expected.getMessage().contains("maximum allowed S3 limit of 10000 parts")); + } + + try { + storageService.validateRemainingPartBudget(info, 10005); + fail("Should throw MaxAppendSizeExceededException above 10000 parts"); + } catch (MaxAppendSizeExceededException expected) { + assertTrue(expected.getMessage().contains("maximum allowed S3 limit of 10000 parts")); + } + } + private GetObjectResponse mockGetObjectResponse(byte[] bytes) { return new GetObjectResponse( null, "test-bucket", "eu-central-1", "object-key", new ByteArrayInputStream(bytes)); diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java new file mode 100644 index 00000000..53a0de9f --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -0,0 +1,576 @@ +package me.desair.tus.server.upload.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests verifying bounded 3-slot asynchronous chunk uploading, backpressure, exception + * translation, timeout aborts, and CallerRunsPolicy degradation in {@link AsyncChunkUploader}. + */ +public class AsyncChunkUploaderTest { + + private ExecutorService executor; + private File tempDir; + + @Before + public void setUp() throws Exception { + executor = Executors.newFixedThreadPool(4); + tempDir = Files.createTempDirectory("tus-async-uploader-test").toFile(); + } + + @After + public void tearDown() throws Exception { + if (executor != null) { + executor.shutdownNow(); + } + if (tempDir != null && tempDir.exists()) { + org.apache.commons.io.FileUtils.deleteDirectory(tempDir); + } + } + + @Test + public void testFirstChunkSubmissionDirectToSlot3() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + Files.write(chunk1.toPath(), new byte[] {1, 2, 3}); + + CountDownLatch uploadFinished = new CountDownLatch(1); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 3, + "part-1", + () -> { + uploadFinished.countDown(); + }); + + assertTrue( + "Upload task should execute in background", uploadFinished.await(3, TimeUnit.SECONDS)); + int confirmed = uploader.drainAndComplete(3000); + assertEquals("One chunk confirmed uploaded", 1, confirmed); + } + + // Verify temp file is deleted by the background worker in finally block + assertFalse("Chunk file should be cleaned up after successful upload", chunk1.exists()); + } + + @Test + public void testPipelinedSubmissionThroughSlot2AndSlot3() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + File chunk3 = new File(tempDir, "chunk3.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + Files.write(chunk3.toPath(), new byte[] {3}); + + List uploadedOrder = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch unblockSlot3 = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + // Chunk 1 occupies Slot 3 + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + unblockSlot3.await(3, TimeUnit.SECONDS); + uploadedOrder.add("part-1"); + }); + + // Chunk 2 placed in Slot 2 (waiting) while Chunk 1 is running + uploader.submitChunk( + chunk2, + 1, + "part-2", + () -> { + uploadedOrder.add("part-2"); + }); + + // Unblock Chunk 1 so it finishes + unblockSlot3.countDown(); + + // Chunk 3 submitted: blocks until Chunk 1 finishes, promotes Chunk 2 to Slot 3, stores Chunk + // 3 in Slot 2 + uploader.submitChunk( + chunk3, + 1, + "part-3", + () -> { + uploadedOrder.add("part-3"); + }); + + int confirmed = uploader.drainAndComplete(3000); + assertEquals("All 3 chunks confirmed uploaded", 3, confirmed); + assertEquals( + "Parts uploaded in sequential order", + List.of("part-1", "part-2", "part-3"), + uploadedOrder); + } + + assertFalse("Chunk 1 should be deleted", chunk1.exists()); + assertFalse("Chunk 2 should be deleted", chunk2.exists()); + assertFalse("Chunk 3 should be deleted", chunk3.exists()); + } + + @Test + public void testBackpressureBlocksReaderWhenBothSlotsFull() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + File chunk3 = new File(tempDir, "chunk3.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + Files.write(chunk3.toPath(), new byte[] {3}); + + CountDownLatch chunk1Hold = new CountDownLatch(1); + AtomicBoolean chunk3Submitted = new AtomicBoolean(false); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + chunk1Hold.await(3, TimeUnit.SECONDS); + }); + + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + + // Launch thread trying to submit chunk 3: must block because Slot 3 is busy and Slot 2 is + // occupied + Thread submitThread = + new Thread( + () -> { + try { + uploader.submitChunk(chunk3, 1, "part-3", () -> {}); + chunk3Submitted.set(true); + } catch (IOException ignored) { + } + }); + submitThread.start(); + + // Give thread 150ms to attempt submit; it must be blocked waiting on Slot 3 + Thread.sleep(150); + assertFalse("Submit of chunk 3 must block due to backpressure", chunk3Submitted.get()); + + // Release chunk 1 + chunk1Hold.countDown(); + submitThread.join(3000); + + assertTrue("Submit of chunk 3 unblocked after chunk 1 finished", chunk3Submitted.get()); + int confirmed = uploader.drainAndComplete(3000); + assertEquals("All 3 chunks confirmed", 3, confirmed); + } + } + + @Test + public void testUploadActionFailurePropagatesIOExceptionAndAborts() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + throw new IOException("Simulated S3 network failure"); + }); + + // Submitting next chunk or draining must detect previous failure and throw IOException + try { + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + uploader.drainAndComplete(3000); + fail("Expected IOException from failed chunk upload"); + } catch (IOException e) { + assertTrue( + "Exception message should reflect upload failure", + e.getMessage().contains("Simulated S3 network failure")); + } + } + + assertFalse("Chunk 2 should be cleaned up by abort()", chunk2.exists()); + } + + @Test + public void testDrainTimeoutAbortsInFlightUpload() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + + CountDownLatch hangLatch = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + hangLatch.await(10, TimeUnit.SECONDS); + }); + + try { + // Enforce short 200ms drain timeout + uploader.drainAndComplete(200); + fail("Expected timeout IOException during drain"); + } catch (IOException e) { + assertTrue("Should report timeout error", e.getMessage().contains("Timed out after")); + assertTrue("Should report key", e.getMessage().contains("part-1")); + } + } finally { + hangLatch.countDown(); + } + } + + @Test + public void testCloseAutomaticallyAbortsIfUncompleted() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + CountDownLatch latch = new CountDownLatch(1); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + latch.await(3, TimeUnit.SECONDS); + }); + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + // Exiting without calling drainAndComplete() triggers close() which calls abort() + } finally { + latch.countDown(); + } + + assertFalse( + "Waiting chunk file in Slot 2 should be deleted by abort on close", chunk2.exists()); + } + + @Test + public void testSynchronousQueueCallerRunsPolicyDegradation() throws Exception { + // ThreadPool with exactly 1 thread and SynchronousQueue + ThreadPoolExecutor singlePool = + new ThreadPoolExecutor( + 1, + 1, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + CountDownLatch poolThreadBusy = new CountDownLatch(1); + CountDownLatch releaseBusy = new CountDownLatch(1); + + // Occupy the only thread in singlePool + singlePool.submit( + () -> { + poolThreadBusy.countDown(); + try { + releaseBusy.await(3, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue("Single pool thread is now occupied", poolThreadBusy.await(3, TimeUnit.SECONDS)); + + String mainThreadName = Thread.currentThread().getName(); + List executionThreads = Collections.synchronizedList(new ArrayList<>()); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(singlePool)) { + // Since pool is saturated and queue is SynchronousQueue, CallerRunsPolicy executes task + // directly on caller thread + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + executionThreads.add(Thread.currentThread().getName()); + }); + + int confirmed = uploader.drainAndComplete(3000); + assertEquals("One chunk confirmed", 1, confirmed); + assertEquals( + "Executed directly on caller thread without queue stall", + mainThreadName, + executionThreads.get(0)); + } finally { + releaseBusy.countDown(); + singlePool.shutdownNow(); + } + } + + @Test + public void testHighConcurrencyMultiUploaderSimulation() throws Exception { + int concurrentUploaders = 15; + ExecutorService sharedPool = + new ThreadPoolExecutor( + 4, + 4, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + CountDownLatch allDone = new CountDownLatch(concurrentUploaders); + AtomicInteger totalUploadedChunks = new AtomicInteger(0); + + for (int i = 0; i < concurrentUploaders; i++) { + final int uploaderId = i; + new Thread( + () -> { + try { + File f1 = new File(tempDir, "c-" + uploaderId + "-1.tmp"); + File f2 = new File(tempDir, "c-" + uploaderId + "-2.tmp"); + Files.write(f1.toPath(), new byte[] {1, 2}); + Files.write(f2.toPath(), new byte[] {3, 4}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(sharedPool)) { + uploader.submitChunk( + f1, + 2, + "u" + uploaderId + "-p1", + () -> { + Thread.sleep(20); + }); + uploader.submitChunk( + f2, + 2, + "u" + uploaderId + "-p2", + () -> { + Thread.sleep(20); + }); + int confirmed = uploader.drainAndComplete(5000); + totalUploadedChunks.addAndGet(confirmed); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + allDone.countDown(); + } + }) + .start(); + } + + assertTrue( + "All concurrent uploaders must complete within 10s", allDone.await(10, TimeUnit.SECONDS)); + assertEquals( + "All 30 chunks (15 uploaders x 2 chunks) must complete", 30, totalUploadedChunks.get()); + + sharedPool.shutdownNow(); + } + + @Test + public void testGetConfirmedCount() throws Exception { + File f1 = new File(tempDir, "count1.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + assertEquals(0, uploader.getConfirmedCount()); + uploader.submitChunk(f1, 1, "k1", () -> {}); + uploader.drainAndComplete(3000); + assertEquals(1, uploader.getConfirmedCount()); + } + } + + @Test + public void testSubmitChunkDiscoversPreviousDoneUploadFailureAndCleansUp() throws Exception { + File f1 = new File(tempDir, "err1.tmp"); + File f2 = new File(tempDir, "err2.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + + CountDownLatch chunk1Done = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k1", + () -> { + chunk1Done.countDown(); + throw new IOException("Simulated disk read error on cloud worker"); + }); + + assertTrue(chunk1Done.await(3, TimeUnit.SECONDS)); + // Give worker thread a brief moment to update Future state to done + Thread.sleep(50); + + try { + uploader.submitChunk(f2, 1, "k2", () -> {}); + fail("Should have thrown IOException when discovering chunk 1 failure"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Simulated disk read error on cloud worker")); + } + } + + assertFalse("Chunk 2 must be cleaned up on failure", f2.exists()); + } + + @Test + public void testUploadActionThrowsRuntimeExceptionTranslatedToIOException() throws Exception { + File f1 = new File(tempDir, "runtime_err.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "runtime-key", + () -> { + throw new IllegalStateException("Boom!"); + }); + + try { + uploader.drainAndComplete(3000); + fail("Should have thrown IOException wrapping RuntimeException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Upload failed for key runtime-key: Boom!")); + assertTrue(e.getCause() instanceof IllegalStateException); + } + } + } + + @Test + public void testUploadActionThrowsErrorTranslatedToIOException() throws Exception { + File f1 = new File(tempDir, "error_key.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "error-key", + () -> { + throw new AssertionError("Simulated fatal assertion error"); + }); + + try { + uploader.drainAndComplete(3000); + fail("Should have thrown IOException wrapping AssertionError"); + } catch (IOException e) { + assertTrue( + e.getMessage().contains("Unexpected error during chunk upload for key error-key")); + } + } + } + + @Test + public void testDrainInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "interrupted.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + + AtomicBoolean gotInterruptedIOException = new AtomicBoolean(false); + + Thread drainThread = + new Thread( + () -> { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k-interrupt", + () -> { + started.countDown(); + unblock.await(5, TimeUnit.SECONDS); + }); + uploader.drainAndComplete(5000); + } catch (IOException e) { + if (e.getMessage().contains("Interrupted waiting for chunk upload")) { + gotInterruptedIOException.set(true); + } + } catch (Exception ignored) { + } + }); + + drainThread.start(); + assertTrue(started.await(3, TimeUnit.SECONDS)); + drainThread.interrupt(); + drainThread.join(3000); + unblock.countDown(); + + assertTrue( + "Interrupted thread should throw IOException mentioning Interrupted", + gotInterruptedIOException.get()); + } + + @Test + public void testSubmitChunkBackpressureInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "bp-int1.tmp"); + File f2 = new File(tempDir, "bp-int2.tmp"); + File f3 = new File(tempDir, "bp-int3.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + Files.write(f3.toPath(), new byte[] {3}); + + CountDownLatch f1Started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + AtomicBoolean gotBackpressureInterruptedException = new AtomicBoolean(false); + + Thread submitThread = + new Thread( + () -> { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k1", + () -> { + f1Started.countDown(); + unblock.await(5, TimeUnit.SECONDS); + }); + uploader.submitChunk(f2, 1, "k2", () -> {}); + // Third chunk triggers waitForInFlight() backpressure + uploader.submitChunk(f3, 1, "k3", () -> {}); + } catch (IOException e) { + if (e.getMessage().contains("Interrupted while checking in-flight chunk upload")) { + gotBackpressureInterruptedException.set(true); + } + } catch (Exception ignored) { + } + }); + + submitThread.start(); + assertTrue(f1Started.await(3, TimeUnit.SECONDS)); + // Brief sleep to let submitThread block in waitForInFlight() + try { + Thread.sleep(50); + } catch (InterruptedException ignored) { + } + submitThread.interrupt(); + submitThread.join(3000); + unblock.countDown(); + + assertTrue( + "Interrupted backpressure wait should throw IOException", + gotBackpressureInterruptedException.get()); + } +} From c2e7020cf21b6f89716cdf6e85e6038a3ff6c5a6 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Mon, 28 Sep 2026 22:24:23 +0200 Subject: [PATCH 02/15] fix(rufh): do not invoke append on creation requests without content --- .../RufhAppendPatchRequestHandler.java | 9 ++-- .../RufhCreationPostRequestHandler.java | 26 ++++++++-- .../RufhCreationPostRequestHandlerTest.java | 52 +++++++++++++++++++ 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index 53974029..d929d8fa 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -55,10 +55,13 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); - long cl = servletRequest.getContentLengthLong(); - if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && cl >= 0) { + // Per RUFH §4.2.1: If upload length was deferred and the client completes the upload + // via Upload-Complete: ?1, derive and set the total length from current offset + + // Content-Length. + long contentLength = servletRequest.getContentLengthLong(); + if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && contentLength >= 0) { long currentOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; - uploadInfo.setLength(currentOffset + cl); + uploadInfo.setLength(currentOffset + contentLength); uploadStorageService.update(uploadInfo); } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 4ff1232d..2702c23e 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -16,6 +16,7 @@ import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; import me.desair.tus.server.util.Utils; +import org.apache.commons.lang3.Strings; /** * Request handler for upload creation requests via HTTP POST, PUT, or PATCH. @@ -66,10 +67,13 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); - long cl = servletRequest.getContentLengthLong(); + // Per RUFH §4.2.1: If upload length is deferred upon creation and the client completes + // the upload in the creation request via Upload-Complete: ?1, the total length is derived + // from Content-Length. + long contentLength = servletRequest.getContentLengthLong(); Long announcedLength = uploadLength; - if (announcedLength == null && Boolean.TRUE.equals(uploadComplete) && cl >= 0) { - announcedLength = cl; + if (announcedLength == null && Boolean.TRUE.equals(uploadComplete) && contentLength >= 0) { + announcedLength = contentLength; } UploadInfo uploadInfo; @@ -90,8 +94,22 @@ public HttpProblemDetails process( String uploadUri = Utils.getUploadUriOnCreation(uploadInfo, servletRequest, uploadStorageService); + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + // An empty creation request carries no body (contentLength <= 0 without chunked encoding). + // In servlet requests without a body, getContentLengthLong() returns -1. We must verify + // that actual payload content is present before invoking storageService.append(); otherwise, + // sending a 0-byte stream to backends with minAppendSize configured (e.g. S3 or Azure Blob) + // would trigger MinAppendSizeNotMetException on an empty creation request. + boolean hasContent = + contentLength > 0 + || (contentLength < 0 + && servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING) != null + && Strings.CI.contains( + servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked")); + InputStream is = servletRequest.getContentInputStream(); - if (is != null && servletRequest.getContentLengthLong() != 0) { + if (is != null && hasContent) { if (uploadLockingService != null) { InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(uploadUri, interruptibleStream); diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index e68c1776..e4e9b8c4 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; @@ -515,4 +516,55 @@ public void testProcessCreationWithUploadCompleteAndContentLengthSetsAnnouncedLe assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + @Test + public void testProcessCreationWithChunkedTransferEncodingCallsAppend() throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.addHeader(HttpHeader.TRANSFER_ENCODING, "chunked"); + byte[] content = "chunked stream data".getBytes(); + // Do not set content via setContent to keep Content-Length at -1, provide via InputStream + request.setContent(content); + + // Custom request to simulate chunked request without Content-Length header (cl < 0) + TusServletRequest tusRequest = + new TusServletRequest(request) { + @Override + public long getContentLengthLong() { + return -1L; + } + + @Override + public InputStream getContentInputStream() { + return new java.io.ByteArrayInputStream(content); + } + }; + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("chunked-id")); + createdInfo.setOffset(0L); + + UploadInfo appendedInfo = new UploadInfo(); + appendedInfo.setId(new UploadId("chunked-id")); + appendedInfo.setOffset((long) content.length); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(appendedInfo); + + handler.process( + HttpMethod.POST, + tusRequest, + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).append(eq(createdInfo), any(InputStream.class)); + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } } From fe63589bdf8ce508c8f9ef546ab631523d78879f Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Mon, 28 Sep 2026 23:04:13 +0200 Subject: [PATCH 03/15] perf(s3): optimize S3 lock acquisition and multi-object release - Skip redundant isLockExpired pre-check in S3LockingService for optimistic conditional PutObject - Batch delete .lock and .stop keys via S3 Multi-Object Delete in S3UploadLock - Clean up unused deleteS3LockObjectIfOwner and deleteS3ObjectQuietly methods - Remove createMinioContainer alias from TestUtils and clarify RustFS Testcontainers setup - Update CHANGELOG.md --- CHANGELOG.md | 1 + .../server/upload/s3/S3LockingService.java | 10 +- .../tus/server/upload/s3/S3UploadLock.java | 72 +++-- .../java/me/desair/tus/server/TestUtils.java | 13 +- .../server/upload/s3/ITS3LockingService.java | 14 +- .../server/upload/s3/ITS3RufhProtocol.java | 14 +- .../server/upload/s3/ITS3StorageService.java | 14 +- .../upload/s3/ITS3TusFileUploadService.java | 14 +- .../upload/s3/S3LockingServiceTest.java | 51 ++-- .../server/upload/s3/S3UploadLockTest.java | 259 +++++++++--------- 10 files changed, 246 insertions(+), 216 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7a9c6cd..846a9adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file. - **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. - **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. - **Lock-Holding Stream Lifecycle**: `TusFileUploadService.getUploadedBytes(...)` now retains the upload lock until the returned stream is closed, preventing concurrent modifications or deletions from corrupting data mid-stream. +- **S3 Locking Optimization**: Optimized `S3LockingService` and `S3UploadLock` by skipping the redundant `isLockExpired` pre-check on happy-path acquisitions (saving 1 remote GET round-trip) and using S3 Multi-Object Delete to delete `.lock` and `.stop` objects in a single batch round-trip on lock release. ### Fixed - **RFC 9110 Media Type Matching & MIME Parameter Tolerance**: Enhanced `Content-Type` header validation in `ContentTypeValidator`, `PostContentTypeValidator`, and `RufhAppendValidator` using RFC 9110 §8.3 compliant media-type parsing (`Utils.isMediaType`). Media type matching now tolerates MIME parameters (such as `;charset=UTF-8` automatically appended by Spring Boot `CharacterEncodingFilter`, servlet wrappers, proxies, or HTTP clients), whitespace variations, and case-insensitivity without incorrectly rejecting valid requests with `406 Not Acceptable`. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 02b721aa..ab5f4cf5 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -153,9 +153,13 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { String lockKey = buildLockKey(uploadId); String stopKey = buildStopKey(uploadId); - if (!isLockExpired(lockKey)) { - return null; - } + // Optimistic conditional write: + // Skip redundant isLockExpired() pre-check. In >99.9% of requests, no lock exists, + // so an initial isLockExpired() issues an expensive S3 GET that 404s (~320ms penalty). + // By issuing putObject with "If-None-Match: *" directly, happy-path acquisition + // takes only 1 network call. If a lock already exists, S3 atomically returns 412 + // Precondition Failed, causing this method to return null. The caller + // (acquireOrEvictExpiredLock) will then inspect isLockExpired() and evict if expired. try { leaseData.setLockPath(lockKey); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index 5b1c8899..5b3a18b3 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -3,12 +3,19 @@ import io.minio.GetObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteRequest; +import io.minio.messages.DeleteResult; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; +import java.util.stream.Stream; import me.desair.tus.server.upload.AbstractLeaseLock; import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadLock; @@ -104,24 +111,54 @@ protected void doRenewLease() throws Exception { @Override protected void releaseLockResource() { - // Owner-Safe Lock Release: only delete .lock lease if it is still owned by this holder - deleteS3LockObjectIfOwner(lockKey); - deleteS3ObjectQuietly(stopKey); + // Owner-Safe Lock Release: only delete .lock lease if it is still owned by this holder. + // If ownership was stolen by another node (e.g. heartbeat lost or execution ran too long), + // skip deletion entirely so we do not delete another node's lock or any stop signal + // potentially intended for that new holder. + if (lockKey != null && !doesLockOwnershipMatch(lockKey)) { + log.info( + "Skipping deletion of S3 lock key {}: lock is currently held by another node", lockKey); + return; + } + + // Batch deletion of .lock and .stop keys via S3 Multi-Object Delete. + // AWS S3 and MinIO execute multi-object delete in a single network round trip (POST /?delete), + // eliminating an extra round trip on every lock release. + deleteS3Objects(lockKey, stopKey); } - void deleteS3LockObjectIfOwner(String key) { - if (key == null || minioClient == null || bucket == null) { + void deleteS3Objects(String firstKey, String secondKey) { + if (minioClient == null || bucket == null) { + return; + } + List objects = + Stream.of(firstKey, secondKey) + .filter(Objects::nonNull) + .map(DeleteRequest.Object::new) + .collect(Collectors.toList()); + if (objects.isEmpty()) { return; } try { - if (!doesLockOwnershipMatch(key)) { - log.info( - "Skipping deletion of S3 lock key {}: lock is currently held by another node", key); - return; + Iterable> results = + minioClient.removeObjects( + RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); + if (results != null) { + for (Result result : results) { + try { + if (result != null) { + DeleteResult.Error error = result.get(); + if (error != null) { + log.debug("Failed to delete S3 object {}: {}", error.objectName(), error.message()); + } + } + } catch (Exception e) { + log.debug("Failed to process batch delete result", e); + } + } } - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); } catch (Exception e) { - log.debug("Failed to delete S3 lock object {}", key, e); + log.debug("Failed to batch delete S3 objects {} and {}", firstKey, secondKey, e); } } @@ -149,15 +186,4 @@ boolean doesLockOwnershipMatch(String key) { } return true; } - - private void deleteS3ObjectQuietly(String key) { - if (key == null || minioClient == null || bucket == null) { - return; - } - try { - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); - } catch (Exception e) { - log.debug("Failed to delete S3 object {}", key, e); - } - } } diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index 32433879..55f1f6fe 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -7,8 +7,8 @@ import org.testcontainers.containers.GenericContainer; /** - * Helper utility class for S3 integration tests running against Testcontainers MinIO using the - * MinIO Java SDK. Supports both Docker and Podman container engines automatically. + * Helper utility class for integration tests running against Testcontainers RustFS (S3) and Azurite + * (Azure Blob). Supports both Docker and Podman container engines automatically. */ public final class TestUtils { @@ -77,15 +77,6 @@ public static GenericContainer createRustFsContainer() { .withEnv("RUSTFS_SECRET_KEY", "rustfsadmin"); } - /** - * Alias for {@link #createRustFsContainer()} for compatibility. - * - * @return A configured GenericContainer instance (not started yet) - */ - public static GenericContainer createMinioContainer() { - return createRustFsContainer(); - } - /** * Create a {@link MinioClient} configured to connect to the given S3/RustFS container. * diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java index 58248985..916ff8e2 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java @@ -17,7 +17,7 @@ public class ITS3LockingService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-locking-service-bucket"; @@ -26,20 +26,20 @@ public class ITS3LockingService { @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java index f281926c..60b21270 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java @@ -14,27 +14,27 @@ */ public class ITS3RufhProtocol extends AbstractITRufhProtocol { - private static org.testcontainers.containers.GenericContainer minio; + private static org.testcontainers.containers.GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-rufh-s3-bucket"; @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java index 544147a0..c2f10768 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java @@ -21,7 +21,7 @@ public class ITS3StorageService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-storage-service-bucket"; @@ -30,20 +30,20 @@ public class ITS3StorageService { @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java index af3c0436..c3a0a008 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java @@ -16,27 +16,27 @@ */ public class ITS3TusFileUploadService extends AbstractITTusFileUploadService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-service-s3-bucket"; @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 368d2684..41a9e90d 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -90,6 +90,26 @@ public void setUp() throws Exception { }) .when(minioClient) .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); + + Mockito.doAnswer( + invocation -> { + io.minio.RemoveObjectsArgs args = invocation.getArgument(0); + if (args != null && args.objects() != null) { + for (io.minio.messages.DeleteRequest.Object obj : args.objects()) { + try { + java.lang.reflect.Field field = + io.minio.messages.DeleteRequest.Object.class.getDeclaredField("name"); + field.setAccessible(true); + String name = (String) field.get(obj); + s3StorageMap.remove(name); + } catch (Exception ignored) { + } + } + } + return java.util.Collections.emptyList(); + }) + .when(minioClient) + .removeObjects(Mockito.any(io.minio.RemoveObjectsArgs.class)); } @Test @@ -558,30 +578,17 @@ public void testLockUploadByUriWithContentionOnPostPutVerificationThrows() throw "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.stop"); String rivalJson = me.desair.tus.server.util.LeaseDataJsonSerializer.serialize(rivalLock); - // Initial check sees expired/missing, but post-put verification sees rival holder - java.util.concurrent.atomic.AtomicInteger getCallCount = - new java.util.concurrent.atomic.AtomicInteger(0); + // PutObject succeeds, but post-put read-after-write verification sees rival holder Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))) - .thenAnswer( - inv -> { - if (getCallCount.incrementAndGet() == 1) { - // First call: check if expired (missing -> not locked) - ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); - Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); - throw new io.minio.errors.ErrorResponseException(errorResponse, null, null); - } - // Second call: read-after-write verification sees rivalLock - return new GetObjectResponse( - null, - "test-bucket", - "eu-central-1", - "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", - new ByteArrayInputStream(rivalJson.getBytes(StandardCharsets.UTF_8))); - }); + .thenReturn( + new GetObjectResponse( + null, + "test-bucket", + "eu-central-1", + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + new ByteArrayInputStream(rivalJson.getBytes(StandardCharsets.UTF_8)))); - UploadLock lock = - lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); - assertNull(lock); + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java index 8b7e66b4..4ea4d024 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java @@ -9,8 +9,13 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; +import io.minio.messages.DeleteResult; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import me.desair.tus.server.upload.LeaseData; @@ -87,7 +92,7 @@ public void testRenewLeaseExceptionHandling() throws Exception { public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Exception { Mockito.doThrow(new RuntimeException("Remove failed")) .when(minioClient) - .removeObject(any(RemoveObjectArgs.class)); + .removeObjects(any(RemoveObjectsArgs.class)); LeaseData leaseData = createLeaseData("holder-123", "/files/upload-1"); S3UploadLock lockWithNullKeys = @@ -109,8 +114,28 @@ public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Excep } @Test - public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Exception { - // Simulate remote lock owned by another holder + public void testCloseHeartbeatExecutorShutdownException() throws Exception { + java.util.concurrent.ScheduledExecutorService mockExecutor = + mock(java.util.concurrent.ScheduledExecutorService.class); + Mockito.doThrow(new RuntimeException("Shutdown error")).when(mockExecutor).shutdownNow(); + + LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap, + mockExecutor); + + lock.close(); + assertEquals("holder-123", lock.getHolderId()); + } + + @Test + public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { LeaseData otherLock = new LeaseData( "other-holder", @@ -134,6 +159,10 @@ public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Except Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + InterruptibleInputStream stream = + new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); + inputStreamMap.put("/files/upload-1", stream); + S3UploadLock lock = new S3UploadLock( myLeaseData, @@ -143,18 +172,19 @@ public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Except "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); + lock.renewLease(); + + // On lease ownership loss, renewLease must abort the active input stream immediately + // to prevent writing un-locked bytes to S3. + assertTrue(stream.isInterrupted()); - // Must NOT remove object because it belongs to other-holder - Mockito.verify(minioClient, Mockito.never()) - .removeObject( - Mockito.argThat( - args -> args != null && "tus-locks/upload-1.lock".equals(args.object()))); + // Must NOT call putObject because lock is now held by other-holder + Mockito.verify(minioClient, Mockito.never()).putObject(any(PutObjectArgs.class)); + lock.close(); } @Test - public void testDeleteS3LockObjectIfOwnerDeletesWhenHolderMatches() throws Exception { - // Simulate remote lock owned by this holder + public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { LeaseData myLock = new LeaseData( "my-holder", @@ -187,25 +217,15 @@ public void testDeleteS3LockObjectIfOwnerDeletesWhenHolderMatches() throws Excep "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); + lock.renewLease(); - // Must remove object because it matches my-holder - Mockito.verify(minioClient) - .removeObject( - Mockito.argThat( - args -> args != null && "tus-locks/upload-1.lock".equals(args.object()))); + // Must call putObject because holder matches + Mockito.verify(minioClient).putObject(any(PutObjectArgs.class)); + lock.close(); } @Test - public void testDeleteS3LockObjectIfOwnerHandlesNoSuchKey() throws Exception { - io.minio.messages.ErrorResponse errorResponse = - Mockito.mock(io.minio.messages.ErrorResponse.class); - Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); - io.minio.errors.ErrorResponseException ex = - new io.minio.errors.ErrorResponseException(errorResponse, null, null); - - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenThrow(ex); - + public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -216,56 +236,36 @@ public void testDeleteS3LockObjectIfOwnerHandlesNoSuchKey() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); - assertEquals("my-holder", lock.getHolderId()); - } - - @Test - public void testCloseHeartbeatExecutorShutdownException() throws Exception { - java.util.concurrent.ScheduledExecutorService mockExecutor = - mock(java.util.concurrent.ScheduledExecutorService.class); - Mockito.doThrow(new RuntimeException("Shutdown error")).when(mockExecutor).shutdownNow(); + // Null key returns false + org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch(null)); - LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); - S3UploadLock lock = + // Null minioClient returns false + S3UploadLock nullClientLock = new S3UploadLock( - myLeaseData, - minioClient, - "test-bucket", - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop", - inputStreamMap, - mockExecutor); - - lock.close(); - assertEquals("holder-123", lock.getHolderId()); - } + myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + org.junit.Assert.assertFalse(nullClientLock.doesLockOwnershipMatch("key")); - @Test - public void testDeleteS3LockObjectIfOwnerNullChecksAndExceptionHandling() throws Exception { - LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); - S3UploadLock lock = + // Null bucket returns false + S3UploadLock nullBucketLock = new S3UploadLock( - myLeaseData, - minioClient, - "test-bucket", - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop", - inputStreamMap); + myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + org.junit.Assert.assertFalse(nullBucketLock.doesLockOwnershipMatch("key")); - // Null key check - lock.deleteS3LockObjectIfOwner(null); + // General exception (non-ErrorResponseException) returns true to allow proceed + Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) + .thenThrow(new RuntimeException("Transient S3 error")); + org.junit.Assert.assertTrue(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); - // Exception during removeObject - Mockito.doThrow(new RuntimeException("Remove failed")) - .when(minioClient) - .removeObject(any(RemoveObjectArgs.class)); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); - assertEquals("holder-123", lock.getHolderId()); + // Renew lease with null lockKey or null minioClient + nullClientLock.doRenewLease(); + S3UploadLock nullKeyLock = + new S3UploadLock(myLeaseData, minioClient, "test-bucket", null, null, inputStreamMap); + nullKeyLock.doRenewLease(); } @Test - public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { + public void testReleaseLockResourceWhenOwnershipMismatch() throws Exception { + // Simulate remote lock owned by another holder LeaseData otherLock = new LeaseData( "other-holder", @@ -283,16 +283,11 @@ public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { "test-bucket", "eu-central-1", "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + new ByteArrayInputStream(json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); - InterruptibleInputStream stream = - new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); - inputStreamMap.put("/files/upload-1", stream); - S3UploadLock lock = new S3UploadLock( myLeaseData, @@ -302,41 +297,16 @@ public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.renewLease(); - - // On lease ownership loss, renewLease must abort the active input stream immediately - // to prevent writing un-locked bytes to S3. - assertTrue(stream.isInterrupted()); - - // Must NOT call putObject because lock is now held by other-holder - Mockito.verify(minioClient, Mockito.never()).putObject(any(PutObjectArgs.class)); lock.close(); + + // Must NOT call removeObjects or removeObject because lock was stolen by another node + // and any stop signal might have been set by a third thread for the new holder + Mockito.verify(minioClient, Mockito.never()).removeObjects(any(RemoveObjectsArgs.class)); + Mockito.verify(minioClient, Mockito.never()).removeObject(any(RemoveObjectArgs.class)); } @Test - public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { - LeaseData myLock = - new LeaseData( - "my-holder", - "/files/upload-1", - 60000L, - System.currentTimeMillis() + 60000L, - System.currentTimeMillis(), - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop"); - String json = LeaseDataJsonSerializer.serialize(myLock); - - io.minio.GetObjectResponse response = - new io.minio.GetObjectResponse( - null, - "test-bucket", - "eu-central-1", - "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); - - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); - + public void testDeleteS3ObjectsNullChecksAndEmpty() { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -347,15 +317,29 @@ public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.renewLease(); + // Null client + S3UploadLock nullClientLock = + new S3UploadLock( + myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + nullClientLock.deleteS3Objects("key1", "key2"); - // Must call putObject because holder matches - Mockito.verify(minioClient).putObject(any(PutObjectArgs.class)); - lock.close(); + // Null bucket + S3UploadLock nullBucketLock = + new S3UploadLock( + myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + nullBucketLock.deleteS3Objects("key1", "key2"); + + // Both keys null -> empty objects + lock.deleteS3Objects(null, null); + + Mockito.verify(minioClient, Mockito.never()).removeObjects(any(RemoveObjectsArgs.class)); + assertEquals("test-bucket", lock.getBucket()); + assertEquals("tus-locks/upload-1.lock", lock.getLockKey()); + assertEquals("tus-locks/upload-1.stop", lock.getStopKey()); } @Test - public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception { + public void testDeleteS3ObjectsWithErrorsAndExceptions() throws Exception { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -366,30 +350,47 @@ public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception "tus-locks/upload-1.stop", inputStreamMap); - // Null key returns false - org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch(null)); + DeleteResult.Error deleteError = mock(DeleteResult.Error.class); + Mockito.when(deleteError.objectName()).thenReturn("tus-locks/upload-1.lock"); + Mockito.when(deleteError.message()).thenReturn("Access Denied"); - // Null minioClient returns false - S3UploadLock nullClientLock = - new S3UploadLock( - myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); - org.junit.Assert.assertFalse(nullClientLock.doesLockOwnershipMatch("key")); + @SuppressWarnings("unchecked") + Result errorResult = mock(Result.class); + Mockito.when(errorResult.get()).thenReturn(deleteError); - // Null bucket returns false - S3UploadLock nullBucketLock = + @SuppressWarnings("unchecked") + Result throwingResult = mock(Result.class); + Mockito.when(throwingResult.get()).thenThrow(new RuntimeException("Result parsing error")); + + List> resultList = Arrays.asList(errorResult, throwingResult, null); + + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))).thenReturn(resultList); + + // Call deleteS3Objects with only firstKey, then with both keys + lock.deleteS3Objects("tus-locks/upload-1.lock", null); + lock.deleteS3Objects(null, "tus-locks/upload-1.stop"); + + Mockito.verify(minioClient, Mockito.times(2)).removeObjects(any(RemoveObjectsArgs.class)); + assertEquals("my-holder", lock.getHolderId()); + } + + @Test + public void testDeleteS3ObjectsWhenMinioClientThrows() { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + S3UploadLock lock = new S3UploadLock( - myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); - org.junit.Assert.assertFalse(nullBucketLock.doesLockOwnershipMatch("key")); + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap); - // General exception (non-ErrorResponseException) returns true to allow proceed - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) - .thenThrow(new RuntimeException("Transient S3 error")); - org.junit.Assert.assertTrue(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenThrow(new RuntimeException("S3 connection error")); - // Renew lease with null lockKey or null minioClient - nullClientLock.doRenewLease(); - S3UploadLock nullKeyLock = - new S3UploadLock(myLeaseData, minioClient, "test-bucket", null, null, inputStreamMap); - nullKeyLock.doRenewLease(); + // Should catch exception quietly without propagating + lock.deleteS3Objects("tus-locks/upload-1.lock", "tus-locks/upload-1.stop"); + assertEquals("my-holder", lock.getHolderId()); } } From 89da876c418a53559801b1b09f0f9957f0b62edd Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Tue, 29 Sep 2026 09:20:29 +0200 Subject: [PATCH 04/15] feat(s3): configurable locking jitter, HTTP 501 non-CAS fallback, and streamlined constructors - Added configurable jitter bounds (withJitter) in AbstractLeaseLockingService and S3LockingService defaulting to 20-60 ms, with zero-jitter support (0, 0) for pure AWS S3 / Cloudflare R2 deployments. - Implemented automatic non-CAS fallback in S3LockingService when If-None-Match: * returns HTTP 501 / NotImplemented (supporting Backblaze B2, Ceph RGW) along with withS3ConditionalWritesSupported(boolean) override. - Streamlined S3LockingService constructors to minimal required set (basic 2-arg and full 6-arg). - Clarified locking documentation and inline comments regarding lock arbitration on non-CAS backends. --- CHANGELOG.md | 2 +- docs/LOCKING.md | 2 +- docs/S3_STORAGE.md | 39 ++++- .../upload/AbstractLeaseLockingService.java | 65 +++++++- .../server/upload/s3/S3LockingService.java | 143 +++++++++++++----- .../server/upload/s3/S3StorageService.java | 3 +- .../desair/tus/server/upload/s3/S3Utils.java | 4 +- .../AbstractLeaseLockingServiceTest.java | 61 +++++++- .../upload/s3/S3LockingServiceTest.java | 136 ++++++++++++++++- .../tus/server/upload/s3/S3UtilsTest.java | 18 +++ 10 files changed, 413 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 846a9adc..6058f827 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ All notable changes to this project will be documented in this file. - **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. - **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. - **Lock-Holding Stream Lifecycle**: `TusFileUploadService.getUploadedBytes(...)` now retains the upload lock until the returned stream is closed, preventing concurrent modifications or deletions from corrupting data mid-stream. -- **S3 Locking Optimization**: Optimized `S3LockingService` and `S3UploadLock` by skipping the redundant `isLockExpired` pre-check on happy-path acquisitions (saving 1 remote GET round-trip) and using S3 Multi-Object Delete to delete `.lock` and `.stop` objects in a single batch round-trip on lock release. +- **S3 Locking Optimization & Non-CAS Fallback**: Optimized `S3LockingService` and `S3UploadLock` by skipping the redundant `isLockExpired` pre-check on happy-path acquisitions (saving 1 remote GET round-trip) and using S3 Multi-Object Delete to delete `.lock` and `.stop` objects in a single batch round-trip on lock release. Made read-after-write verification jitter bounds configurable via `AbstractLeaseLockingService.setJitter(minMs, maxMs)` and `S3LockingService.withJitter(minMs, maxMs)`. Added automatic non-CAS fallback when `If-None-Match: *` returns `HTTP 501 Not Implemented` (enabling seamless out-of-the-box support for Backblaze B2 and Ceph RGW) along with `withS3ConditionalWritesSupported(boolean)` override. Streamlined `S3LockingService` constructors to minimal required set and documented zero-jitter throughput optimization for pure AWS S3 / Cloudflare R2 deployments in `docs/S3_STORAGE.md`. ### Fixed - **RFC 9110 Media Type Matching & MIME Parameter Tolerance**: Enhanced `Content-Type` header validation in `ContentTypeValidator`, `PostContentTypeValidator`, and `RufhAppendValidator` using RFC 9110 §8.3 compliant media-type parsing (`Utils.isMediaType`). Media type matching now tolerates MIME parameters (such as `;charset=UTF-8` automatically appended by Spring Boot `CharacterEncodingFilter`, servlet wrappers, proxies, or HTTP clients), whitespace variations, and case-insensitivity without incorrectly rejecting valid requests with `406 Not Acceptable`. diff --git a/docs/LOCKING.md b/docs/LOCKING.md index 0b946612..6416525e 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -86,7 +86,7 @@ public interface UploadLock extends Closeable { |---|---|---|---| | **Disk & Network Filesystems (Default)** | `LeaseFileLockingService` | Atomic sibling mutex directory (`.mutex/`), in-place expired lock takeover, TTL-based JSON lease files with heartbeat renewal, ownership fencing, and `.stop` signal files. Fully safe on NFSv3/v4, AWS EFS, SMB/CIFS, Kubernetes containers, and local disks. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | | **Local File System (Legacy Opt-Out)** | `DiskLockingService` | OS kernel-level exclusive POSIX `FileLock` (`fcntl`) with JVM shutdown hooks and `.stop` signal files. Best for single-node deployments on local disk. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | -| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), conditional writes (`If-None-Match: *`), jittered read-after-write verification, heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/S3_STORAGE.md) | +| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), atomic conditional writes (`If-None-Match: *`), non-CAS lock arbitration with configurable jitter backoff (`withJitter`) for backends lacking CAS (B2, Ceph, Wasabi), heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/S3_STORAGE.md) | | **Azure Blob Storage** | `AzureBlobLockingService` | Native Azure Blob Storage exclusive 30-second leases (`BlobLeaseClient`), background daemon renewal, and `.stop` signal blob polling watchdog. | [`docs/AZURE_BLOB_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/AZURE_BLOB_STORAGE.md) | --- diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 14eff747..ac44c643 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -211,7 +211,7 @@ Understanding the architectural distinction between disk/file locking and S3 dis - Removing renewal with a short TTL would cause locks to expire mid-upload during long transfers, leading to race conditions and data corruption. - Removing renewal with an infinite/static lock would mean a single pod crash (`kill -9`, node OOM) leaves behind an orphaned `.lock` object in S3, permanently deadlocking that upload ID. -### TOCTOU (Time-of-Check to Time-of-Use) Mitigation in S3 +### Lock Arbitration & TOCTOU Mitigation in S3 In stateless distributed object storage, acquiring a lock via standard `GetObject` followed by `PutObject` is vulnerable to a classic **Time-of-Check to Time-of-Use (TOCTOU)** race condition: 1. **Time of Check (TOC)**: Two contender nodes (Node A and Node B) concurrently check if a `.lock` object exists or is expired. Both observe it as available. @@ -222,15 +222,46 @@ To guarantee waterproof single-winner lock exclusivity across cloud providers an 1. **Conditional Writes (`If-None-Match: *`)**: - `PutObject` requests include the `If-None-Match: *` header. - - On Amazon S3 and compliant object storage engines, S3 atomically rejects the second write with `HTTP 412 Precondition Failed` (`PreconditionFailed`), immediately preventing concurrent overwrites. -2. **Jittered Read-After-Write Verification**: - - For S3-compatible emulators or endpoints that do not strictly enforce conditional writes on `PutObject`, the acquiring node sleeps for a brief randomized jitter (20–60ms) to allow in-flight writes to settle, then re-reads `GetObject`. + - On Amazon S3 and compliant object storage engines, S3 atomically rejects the second write with `HTTP 412 Precondition Failed` (`PreconditionFailed`), immediately preventing concurrent overwrites in a single network round-trip. +2. **Lock Arbitration on Backends Without Atomic Conditional Writes (Jittered Read-After-Write Verification)**: + - Several major S3-compatible engines do not support atomic conditional writes: + | Backend | Conditional Write (`If-None-Match: *`) Behavior | `S3LockingService` Handling | + | :--- | :--- | :--- | + | **AWS S3, Cloudflare R2, LocalStack, RustFS** | Returns `HTTP 412 Precondition Failed` | Optimistic CAS (1 network call) | + | **Backblaze B2, Ceph RGW** | Returns `HTTP 501 Not Implemented` | Auto-downgrades to non-CAS arbitration | + | **Wasabi** | Silently ignored (last-write-wins overwrite) | Layer 2 jitter arbitration or `withS3ConditionalWritesSupported(false)` | + - On backends without conditional write support, contender nodes sleep for a brief randomized jitter (default: 20–60ms) to allow competing writes to settle, then re-read `GetObject`. - If the remote `holderId` does not match its own, the node detects that it was overtaken, rejects the acquisition, and leaves the winner's lock intact. 3. **Safe Expired Lock Eviction**: - When evicting an expired lock, the lock object's expiration is verified again immediately before deletion to prevent evicting a fresh lock created by a winning peer. 4. **Owner-Safe Lock Release**: - In `S3UploadLock.close()`, the node verifies that the remote lock is still owned by its own `holderId` before deleting it. If its lease expired while the process was paused and another node took over ownership, the previous node will never delete the new owner's active lock. +### Jitter Configuration & Zero-Jitter Performance Tuning + +The jitter window can be customized via `.withJitter(minMs, maxMs)` on `S3LockingService`: + +```java +// Option A: Zero-Jitter for AWS S3 & Cloudflare R2 (Maximum Throughput) +// On backends with atomic conditional writes, jitter is not needed. +// Passing (0L, 0L) completely bypasses Thread.sleep for fastest lock acquisition: +S3LockingService s3LockingService = + new S3LockingService(minioClient, bucketName) + .withJitter(0L, 0L); + +// Option B: High-Latency or Distributed Backends (Ceph, multi-datacenter MinIO) +// High-latency backends or geo-replicated clusters may need a wider window to settle writes: +S3LockingService s3LockingService = + new S3LockingService(minioClient, bucketName) + .withJitter(50L, 200L); + +// Option C: Explicit Non-CAS Mode (e.g. Wasabi) +// Pre-checks existing locks before writing unconditionally: +S3LockingService s3LockingService = + new S3LockingService(minioClient, bucketName) + .withS3ConditionalWritesSupported(false); +``` + ### Clock Synchronization & NTP Requirement Distributed TTL lease evaluation relies on wall-clock timestamps (`expiresAt`). While `S3LockingService` incorporates a 2-second safety buffer (`CLOCK_SKEW_SAFETY_MARGIN_MS`) to absorb minor time deviations between pods, all cluster nodes and containers running `tus-java-server` MUST synchronize their system clocks using NTP (Network Time Protocol) or Amazon Time Sync Service (`chrony`). Avoid clock skew exceeding $\pm 1$ second between cluster nodes. diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java index a1f87707..93c8023e 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java @@ -37,9 +37,14 @@ public abstract class AbstractLeaseLockingService extends AbstractCloseableResou */ public static final long CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L; + public static final long DEFAULT_JITTER_MIN_MS = 20L; + public static final long DEFAULT_JITTER_MAX_MS = 60L; + protected final long leaseDurationMs; protected final long pollIntervalMs; protected UploadIdFactory idFactory; + protected long jitterMinMs = DEFAULT_JITTER_MIN_MS; + protected long jitterMaxMs = DEFAULT_JITTER_MAX_MS; protected final Map activeInputStreams = new ConcurrentHashMap<>(); protected final ScheduledExecutorService watchdogExecutor; @@ -251,17 +256,71 @@ public void checkStopSignals() { } /** - * Applies randomized jitter backoff to settle concurrent in-flight writes. + * Applies randomized jitter backoff to settle concurrent in-flight writes and arbitrate lock + * acquisition races on storage backends lacking atomic conditional writes. + * + *

If {@code maxMs <= 0}, this method returns immediately without sleeping, allowing callers to + * bypass jitter on backends with guaranteed atomic conditional writes (e.g. AWS S3). * * @param minMs Minimum jitter duration in milliseconds * @param maxMs Maximum jitter duration in milliseconds */ protected void applyJitter(long minMs, long maxMs) { + if (maxMs <= 0) { + return; + } try { - long jitterMs = minMs + (long) (Math.random() * (maxMs - minMs)); - Thread.sleep(jitterMs); + long jitterMs = (minMs >= maxMs) ? minMs : minMs + (long) (Math.random() * (maxMs - minMs)); + if (jitterMs > 0) { + Thread.sleep(jitterMs); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } + + /** + * Applies randomized jitter backoff using the configured {@link #getJitterMinMs()} and {@link + * #getJitterMaxMs()} bounds. + */ + protected void applyJitter() { + applyJitter(jitterMinMs, jitterMaxMs); + } + + /** + * Returns the configured minimum jitter duration in milliseconds. + * + * @return Minimum jitter duration in milliseconds + */ + public long getJitterMinMs() { + return jitterMinMs; + } + + /** + * Returns the configured maximum jitter duration in milliseconds. + * + * @return Maximum jitter duration in milliseconds + */ + public long getJitterMaxMs() { + return jitterMaxMs; + } + + /** + * Configures the jitter bounds used during lock acquisition read-after-write verification. + * + * @param minMs Minimum jitter duration in milliseconds (must be >= 0) + * @param maxMs Maximum jitter duration in milliseconds (must be >= minMs) + * @throws IllegalArgumentException If minMs is negative or maxMs is less than minMs + */ + public void setJitter(long minMs, long maxMs) { + if (minMs < 0) { + throw new IllegalArgumentException("jitterMinMs cannot be negative: " + minMs); + } + if (maxMs < minMs) { + throw new IllegalArgumentException( + "jitterMaxMs (" + maxMs + ") cannot be less than jitterMinMs (" + minMs + ")"); + } + this.jitterMinMs = minMs; + this.jitterMaxMs = maxMs; + } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index ab5f4cf5..f37c1278 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -61,6 +61,7 @@ public class S3LockingService extends AbstractLeaseLockingService { private final MinioClient minioClient; private final String bucket; private final String locksPrefix; + private volatile boolean s3ConditionalWritesSupported = true; /** * Basic constructor using default lock prefix ("locks/"), 30s lease duration, and 2s polling @@ -75,30 +76,7 @@ public S3LockingService(MinioClient minioClient, String bucket) { bucket, DEFAULT_LOCKS_PREFIX, DEFAULT_LEASE_DURATION_MS, - DEFAULT_POLL_INTERVAL_MS); - } - - /** - * Full constructor allowing custom configuration for all locking parameters. - * - * @param minioClient Pre-configured MinIO Client - * @param bucket Target S3 bucket name - * @param locksPrefix Object key prefix for locks and stop signals - * @param leaseDurationMs Lock lease duration in milliseconds - * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals - */ - public S3LockingService( - MinioClient minioClient, - String bucket, - String locksPrefix, - long leaseDurationMs, - long pollIntervalMs) { - this( - minioClient, - bucket, - locksPrefix, - leaseDurationMs, - pollIntervalMs, + DEFAULT_POLL_INTERVAL_MS, new UuidUploadIdFactory()); } @@ -125,6 +103,51 @@ public S3LockingService( this.locksPrefix = sanitizePrefix(locksPrefix); } + /** + * Configures custom jitter bounds used during lock acquisition read-after-write verification. + * + *

On S3-compatible backends lacking atomic conditional writes (e.g., Wasabi, SeaweedFS, Ceph, + * Backblaze B2, older MinIO), randomized jitter backoff resolves last-write-wins races. For + * high-latency or cross-region backends, configure higher bounds (e.g. 50–200 ms). For pure AWS + * S3 or Cloudflare R2 deployments with strong conditional write enforcement, jitter can be + * disabled by passing {@code 0, 0} to maximize throughput. + * + * @param minMs Minimum jitter duration in milliseconds (must be >= 0) + * @param maxMs Maximum jitter duration in milliseconds (must be >= minMs) + * @return This service instance for fluent chaining + */ + public S3LockingService withJitter(long minMs, long maxMs) { + setJitter(minMs, maxMs); + return this; + } + + /** + * Configures whether the underlying S3 endpoint supports atomic conditional writes via {@code + * If-None-Match: *}. + * + *

Defaults to {@code true} (optimistic). If the endpoint returns HTTP 501 Not Implemented, + * this is automatically downgraded to {@code false}. For endpoints known to silently ignore + * {@code If-None-Match: *} (such as Wasabi), setting this to {@code false} ensures the service + * pre-checks existing lock status before writing. + * + * @param supported Whether conditional writes are supported + * @return This service instance for fluent chaining + */ + public S3LockingService withS3ConditionalWritesSupported(boolean supported) { + this.s3ConditionalWritesSupported = supported; + return this; + } + + /** + * Returns whether the underlying S3 endpoint currently supports atomic conditional writes. + * + * @return true if conditional writes are supported or assumed supported; false if downgraded or + * disabled + */ + public boolean isS3ConditionalWritesSupported() { + return s3ConditionalWritesSupported; + } + @Override public void cleanupStaleLocks() throws IOException { try { @@ -167,21 +190,54 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { byte[] lockContentBytes = LeaseDataJsonSerializer.serializeToBytes(leaseData); - // Layer 1: Conditional PutObject with "If-None-Match: *" - // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object already - // exists - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucket) - .object(lockKey) - .extraHeaders(Collections.singletonMap("If-None-Match", "*")) - .stream( - new ByteArrayInputStream(lockContentBytes), (long) lockContentBytes.length, -1L) - .build()); + if (s3ConditionalWritesSupported) { + // Layer 1: Optimistic Conditional PutObject with "If-None-Match: *" + // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object + // already + // exists + try { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(lockKey) + .extraHeaders(Collections.singletonMap("If-None-Match", "*")) + .stream( + new ByteArrayInputStream(lockContentBytes), + (long) lockContentBytes.length, + -1L) + .build()); + } catch (ErrorResponseException e) { + S3ErrorType errorType = S3Utils.parseErrorResponse(e); + if (errorType == S3ErrorType.API_NOT_IMPLEMENTED) { + // Backend (e.g. Backblaze B2, Ceph RGW) does not support conditional writes. + // Downgrade to non-CAS arbitration mode and proceed with safe pre-check + unconditional + // write. + log.info( + "S3 endpoint does not support conditional writes (If-None-Match: *). " + + "Downgrading to non-CAS lock arbitration for key {}", + lockKey); + s3ConditionalWritesSupported = false; + if (!isLockExpired(lockKey)) { + return null; + } + writeUnconditionalLockObject(lockKey, lockContentBytes); + } else { + throw e; + } + } + } else { + // Non-CAS mode (e.g., Wasabi, Ceph RGW, Backblaze B2): + // Verify no active unexpired lock exists before performing unconditional write + if (!isLockExpired(lockKey)) { + return null; + } + writeUnconditionalLockObject(lockKey, lockContentBytes); + } // Layer 2: Jittered Read-After-Write Verification - // For emulators or S3 backends where If-None-Match is not strictly enforced, - // pause for a small randomized jitter (20-60ms) and verify our holderId is still the owner + // For non-CAS backends or backends where If-None-Match is not strictly enforced, + // pause for a randomized jitter duration to allow competing writes to settle, + // then verify our holderId is still the owner applyJitter(); if (!verifyLockOwnership(lockKey, leaseData.getHolderId())) { return null; @@ -202,6 +258,14 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { } } + private void writeUnconditionalLockObject(String lockKey, byte[] lockContentBytes) + throws Exception { + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(lockKey).stream( + new ByteArrayInputStream(lockContentBytes), (long) lockContentBytes.length, -1L) + .build()); + } + @Override protected boolean isLockExpired(UploadId uploadId) { if (uploadId == null) { @@ -311,8 +375,9 @@ private void deleteObjectQuietly(String key) { } } - void applyJitter() { - applyJitter(20L, 60L); + @Override + protected void applyJitter() { + super.applyJitter(); } private String sanitizePrefix(String prefix) { diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 8b059137..6e9c2502 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -352,7 +352,8 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) processPayloadChunks(info, preparedStream, info.getId(), partObjectKey); // Step 4: Recalculate total uploaded byte offset across all uploaded part objects in S3. - // S3 listObjects can exhibit eventual consistency; take the maximum of remote parts query + // S3 listObjects in distributed clusters (e.g. multi-site Ceph, cluster rebalancing, or + // async replication) can exhibit listing delays; take the maximum of remote parts query // and locally verified stream progression to ensure newOffset accurately reflects bytes // successfully written. long calculatedOffset = diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java index 618f41c4..2d0c2eaa 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java @@ -43,7 +43,9 @@ public static S3ErrorType parseErrorResponse(ErrorResponseException exception) { return S3ErrorType.ACCESS_DENIED; } - if ("APINotImplemented".equalsIgnoreCase(code) || "NotImplemented".equalsIgnoreCase(code)) { + if ("APINotImplemented".equalsIgnoreCase(code) + || "NotImplemented".equalsIgnoreCase(code) + || (exception.response() != null && exception.response().code() == 501)) { return S3ErrorType.API_NOT_IMPLEMENTED; } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java index d547ef9c..09aa108e 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java @@ -4,6 +4,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -77,6 +78,10 @@ public void testJitter(long min, long max) { applyJitter(min, max); } + public void testJitterDefault() { + applyJitter(); + } + public void testCheckStopSignals() { checkStopSignals(); } @@ -228,8 +233,60 @@ public void testCheckStopSignalsIteratesActiveStreams() { } @Test - public void testApplyJitter() { - service.testJitter(1L, 5L); + public void testDefaultJitterBounds() { + assertEquals(20L, service.getJitterMinMs()); + assertEquals(60L, service.getJitterMaxMs()); + } + + @Test + public void testSetJitterValidBounds() { + service.setJitter(10L, 50L); + assertEquals(10L, service.getJitterMinMs()); + assertEquals(50L, service.getJitterMaxMs()); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetJitterNegativeMinThrows() { + service.setJitter(-1L, 50L); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetJitterMaxLessThanMinThrows() { + service.setJitter(50L, 10L); + } + + @Test + public void testApplyJitterWithZeroOrNegativeMaxMsReturnsImmediately() { + long start = System.currentTimeMillis(); + service.testJitter(0L, 0L); + service.testJitter(0L, -5L); + long elapsed = System.currentTimeMillis() - start; + // KISS: zero/negative maxMs returns immediately without sleep + assertTrue(elapsed < 100L); + } + + @Test + public void testApplyJitterWhenMinGreaterThanOrEqualMax() { + long start = System.currentTimeMillis(); + service.testJitter(2L, 2L); + service.testJitter(3L, 2L); + long elapsed = System.currentTimeMillis() - start; + assertTrue(elapsed >= 2L); + } + + @Test + public void testApplyJitterDefault() { + service.setJitter(1L, 3L); + service.testJitterDefault(); + assertEquals(1L, service.getJitterMinMs()); + assertEquals(3L, service.getJitterMaxMs()); + } + + @Test + public void testApplyJitterPreservesInterrupt() { + Thread.currentThread().interrupt(); + service.testJitter(10L, 20L); + assertTrue(Thread.interrupted()); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 41a9e90d..832220d6 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -4,7 +4,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import io.minio.GetObjectArgs; import io.minio.GetObjectResponse; @@ -28,6 +30,7 @@ import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadLock; +import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.InterruptibleInputStream; import org.junit.Before; import org.junit.Test; @@ -333,11 +336,12 @@ public void testInterruptStreamStandardStreamCloseException() throws Exception { @Test public void testSanitizePrefixNullOrEmpty() throws Exception { S3LockingService serviceWithEmptyPrefix = - new S3LockingService(minioClient, "test-bucket", "", 30000L, 0L); + new S3LockingService(minioClient, "test-bucket", "", 30000L, 0L, new UuidUploadIdFactory()); assertNotNull(serviceWithEmptyPrefix); S3LockingService serviceWithNullPrefix = - new S3LockingService(minioClient, "test-bucket", null, 30000L, 0L); + new S3LockingService( + minioClient, "test-bucket", null, 30000L, 0L, new UuidUploadIdFactory()); assertNotNull(serviceWithNullPrefix); } @@ -376,12 +380,12 @@ public void testCheckStopSignalForEntryHappyPath() throws Exception { io.minio.StatObjectResponse mockStat = Mockito.mock(io.minio.StatObjectResponse.class); Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))).thenReturn(mockStat); - S3LockingService service = - new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); - service.setIdFactory(idFactory); + + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L, idFactory); ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); InterruptibleInputStream stream = new InterruptibleInputStream(bais); @@ -411,12 +415,12 @@ public void testCheckStopSignalForEntryNoSuchKey() throws Exception { Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))) .thenThrow(noSuchKeyException); - S3LockingService service = - new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); - service.setIdFactory(idFactory); + + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L, idFactory); ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); InterruptibleInputStream stream = new InterruptibleInputStream(bais); @@ -862,4 +866,120 @@ public void testEvictExpiredLockHandlesRemoveObjectException() throws Exception boolean evicted = lockingService.evictExpiredLock(uploadId); assertFalse(evicted); } + + @Test + public void testWithJitterConfiguresBoundsAndReturnsSelf() { + S3LockingService returned = lockingService.withJitter(15L, 75L); + assertSame(lockingService, returned); + assertEquals(15L, lockingService.getJitterMinMs()); + assertEquals(75L, lockingService.getJitterMaxMs()); + } + + @Test + public void testLockAcquisitionWithJitterDisabled() throws Exception { + lockingService.withJitter(0L, 0L); + assertEquals(0L, lockingService.getJitterMinMs()); + assertEquals(0L, lockingService.getJitterMaxMs()); + + UploadLock lock = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock); + lock.close(); + } + + @Test + public void testWithS3ConditionalWritesSupported() { + assertTrue(lockingService.isS3ConditionalWritesSupported()); + S3LockingService returned = lockingService.withS3ConditionalWritesSupported(false); + assertSame(lockingService, returned); + assertFalse(lockingService.isS3ConditionalWritesSupported()); + } + + @Test + public void testConditionalWriteNotImplementedDowngradesToNonCasModeAndSucceeds() + throws Exception { + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NotImplemented"); + ErrorResponseException notImplementedEx = + new ErrorResponseException(errorResponse, null, "NotImplemented"); + + // The first PutObject with If-None-Match: * throws NotImplemented (e.g. Backblaze B2, Ceph RGW) + Mockito.doAnswer( + invocation -> { + PutObjectArgs args = invocation.getArgument(0); + if (args.extraHeaders() != null && args.extraHeaders().containsKey("If-None-Match")) { + throw notImplementedEx; + } + // Unconditional fallback write stores bytes in our in-memory map + java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int read; + java.io.InputStream is = args.stream(); + while ((read = is.read(buf)) != -1) { + baos.write(buf, 0, read); + } + s3StorageMap.put(args.object(), baos.toByteArray()); + return null; + }) + .when(minioClient) + .putObject(Mockito.any(PutObjectArgs.class)); + + assertTrue(lockingService.isS3ConditionalWritesSupported()); + + UploadLock lock = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock); + // Verified that service automatically downgraded to non-CAS mode + assertFalse(lockingService.isS3ConditionalWritesSupported()); + lock.close(); + } + + @Test(expected = UploadAlreadyLockedException.class) + public void testConditionalWriteNotImplementedFailsWhenActiveLockExists() throws Exception { + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NotImplemented"); + ErrorResponseException notImplementedEx = + new ErrorResponseException(errorResponse, null, "NotImplemented"); + + // PutObject throws NotImplemented when attempting conditional write + Mockito.doThrow(notImplementedEx).when(minioClient).putObject(Mockito.any(PutObjectArgs.class)); + + // Existing lock in S3 is actively held by another contender + LeaseData rivalLock = + new LeaseData( + "rival-holder", + "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e", + 30000L, + System.currentTimeMillis() + 30000L, + System.currentTimeMillis(), + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.stop"); + s3StorageMap.put( + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(rivalLock)); + + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + } + + @Test + public void testExplicitNonCasModeChecksLockExpirationBeforeUnconditionalWrite() + throws Exception { + lockingService.withS3ConditionalWritesSupported(false); + assertFalse(lockingService.isS3ConditionalWritesSupported()); + + // First lock acquisition succeeds + UploadLock lock1 = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock1); + + // Second lock acquisition while first is active fails because isLockExpired returns false + try { + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + fail("Expected UploadAlreadyLockedException when trying to acquire an actively locked URI"); + } catch (UploadAlreadyLockedException expected) { + assertNotNull(expected.getMessage()); + } + + lock1.close(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java index 53ee607d..5dc3b3c9 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java @@ -35,10 +35,28 @@ public void testParseErrorResponseCodes() throws Exception { assertEquals( S3ErrorType.API_NOT_IMPLEMENTED, S3Utils.parseErrorResponse(createExceptionWithCode("APINotImplemented"))); + assertEquals( + S3ErrorType.API_NOT_IMPLEMENTED, + S3Utils.parseErrorResponse(createExceptionWithCode("NotImplemented"))); assertEquals( S3ErrorType.UNKNOWN, S3Utils.parseErrorResponse(createExceptionWithCode("InternalError"))); } + @Test + public void testParseErrorResponseHttp501() { + okhttp3.Response httpResponse = + new okhttp3.Response.Builder() + .request(new okhttp3.Request.Builder().url("https://example.com").build()) + .protocol(okhttp3.Protocol.HTTP_1_1) + .code(501) + .message("Not Implemented") + .build(); + ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); + org.mockito.Mockito.when(errorResponse.code()).thenReturn(""); + ErrorResponseException ex = new ErrorResponseException(errorResponse, httpResponse, null); + assertEquals(S3ErrorType.API_NOT_IMPLEMENTED, S3Utils.parseErrorResponse(ex)); + } + private ErrorResponseException createExceptionWithCode(String code) { ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); org.mockito.Mockito.when(errorResponse.code()).thenReturn(code); From a4ec6b82e80f51429ead32dcc5f40d46e492d5f1 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 3 Oct 2026 16:20:25 +0200 Subject: [PATCH 05/15] feat(s3): add native S3 multipart copy helper and connection parameter constructors --- CHANGELOG.md | 4 + docs/S3_STORAGE.md | 9 +- scripts/rufh_conformity_test.py | 39 ++ scripts/tus_conformity_test.py | 58 +++ .../server/core/CorePatchRequestHandler.java | 23 +- .../CreationWithUploadPostRequestHandler.java | 27 +- .../RufhAppendPatchRequestHandler.java | 33 +- .../RufhCreationPostRequestHandler.java | 34 +- .../upload/azure/AzureBlobStorageService.java | 39 +- .../upload/s3/S3ConcatenationService.java | 35 +- .../server/upload/s3/S3LockingService.java | 60 +++ .../upload/s3/S3ServerSideComposeHelper.java | 303 +++++++++++++++ .../server/upload/s3/S3StorageService.java | 190 +++++++-- .../upload/util/AsyncChunkUploader.java | 13 + .../core/CorePatchRequestHandlerTest.java | 74 ++++ ...ationWithUploadPostRequestHandlerTest.java | 64 ++++ .../RufhAppendPatchRequestHandlerTest.java | 79 ++++ .../RufhCreationPostRequestHandlerTest.java | 78 ++++ .../azure/ITAzureBlobStorageService.java | 34 ++ .../upload/s3/S3ConcatenationServiceTest.java | 10 +- .../upload/s3/S3LockingServiceTest.java | 18 + .../s3/S3ServerSideComposeHelperTest.java | 362 ++++++++++++++++++ .../upload/s3/S3StorageServiceTest.java | 132 +++++++ .../upload/util/AsyncChunkUploaderTest.java | 14 + 24 files changed, 1667 insertions(+), 65 deletions(-) create mode 100644 src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java create mode 100644 src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6058f827..c906a2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ All notable changes to this project will be documented in this file. - **Clear Content-Length on Error Responses**: Cleared `Content-Length` response header prior to invoking `HttpServletResponse.sendError(...)` during exception handling, resolving buffer conflicts and exceptions in Undertow and other servlet containers ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). - **Prevent Disk Truncate Underflow**: Guarded file truncate logic in `DiskStorageService` against underflow when removing bytes (`Math.max(0L, file.size() - byteCount)`). - **File Channel Leak Prevention in FileBasedLock**: Guaranteed `FileChannel` is closed immediately upon lock acquisition errors to avoid file descriptor leaks. +- **Cloud Upload Pause & Drain Timeout Resilience**: Extended `AsyncChunkUploader` default drain timeout from 4 seconds to 60 seconds and wrapped chunk draining in error-resilient recovery logic across `AzureBlobStorageService` and `S3StorageService`. Confirmed uploaded chunks and staged Azure blocks are committed and metadata (`UploadInfo` offset) is persisted to storage before throwing stream or drain exceptions, ensuring paused uploads (such as Uppy client pause/resume) preserve their progress and resume from the exact byte offset instead of restarting from 0. +- **Azure Blob Storage Upload Cancellation Lease Safety**: Checked lock blob lease state before calling `deleteIfExists()` in `AzureBlobStorageService.terminateUpload()`. When an active lease is held on the lock blob by an ongoing request (e.g. `DELETE` cancellation), attempting deletion without specifying the lease ID triggered an Azure SDK error log (`HTTP 412 LeaseIdMissing`). Skipping deletion for actively leased blobs prevents this error and allows normal lease release upon request completion. +- **Lock Contention Stream Interruption Handling**: Handled `IOException` from `InterruptibleInputStream` across upload request handlers (`CorePatchRequestHandler`, `RufhAppendPatchRequestHandler`, `RufhCreationPostRequestHandler`, `CreationWithUploadPostRequestHandler`). When an upload stream is interrupted by the locking service watchdog during lock contention (such as concurrent `HEAD` progress checks or `DELETE` cancellation requests), the storage backend commits all buffered bytes received so far and updates the offset in storage. Request handlers now reload the refreshed `UploadInfo` and return a clean HTTP 204/201 response with the updated offset rather than bubbling an unhandled `IOException` / HTTP 500 to the servlet container. +- **S3 Server-Side Part Composition & Native Multipart Copy**: Implemented `S3ServerSideComposeHelper` to resolve AWS S3 header rejection during server-side part composition. MinIO Java SDK 9.0.3's `composeObject` delegates to `UploadPartCopy` with an empty body placeholder that automatically injects `Content-MD5` and `Content-Type` headers, which Amazon AWS S3 strictly forbids on part copy requests and rejects with HTTP 400 (`The specified header is not valid in this context`). `S3ServerSideComposeHelper` executes native S3 multipart copy requests directly with SigV4 signing omitting `Content-MD5`, allowing fast zero-bandwidth server-side composition on both AWS S3 and MinIO/Ceph clusters without external AWS SDK dependencies. Added reflection-free constructors to `S3StorageService` and `S3LockingService` accepting connection parameters directly, with `eu-central-1` as the default fallback region, while preserving multi-tier streaming concatenation fallbacks. ### Breaking - **Downloads**: In order to support both the Tus protocol and RUFH protocol, the unofficial download extension will not return a HTTP status code `204` for uploads that are still in progress and will not contain the response header `Tus-Resumable`. Removed the `UploadInProgressException` class. diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index ac44c643..4328c739 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -46,8 +46,13 @@ MinioClient minioClient = MinioClient.builder() .build(); // 2. Instantiate S3 Storage and Distributed Locking services -S3StorageService s3StorageService = new S3StorageService(minioClient, bucketName); -S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); +// Option A: Direct connection parameters (recommended, builds internal client with server-side compose helper) +S3StorageService s3StorageService = new S3StorageService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); +S3LockingService s3LockingService = new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); + +// Option B: Using a pre-configured MinIO Client +// S3StorageService s3StorageService = new S3StorageService(minioClient, endpoint, "eu-central-1", accessKey, secretKey, bucketName); +// S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); // 3. Configure TusFileUploadService with S3 storage and locking // Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. diff --git a/scripts/rufh_conformity_test.py b/scripts/rufh_conformity_test.py index 6e4c0319..d973a588 100755 --- a/scripts/rufh_conformity_test.py +++ b/scripts/rufh_conformity_test.py @@ -1114,6 +1114,45 @@ def test_options_with_upload_complete_header(self, target_url, request): status, resp_headers, _, _ = http_request("OPTIONS", target_url, headers=headers, test_name=request.node.name) assert status in (200, 204), "OPTIONS request with Upload-Complete MUST succeed" + def test_paused_upload_head_offset_and_resume(self, target_url, request): + """ + §5 & §10.1: Resuming an Incomplete Upload After Pause. + Quote: "The client can pause the upload by withholding the rest of the content..." + Verifies that after uploading a chunk and pausing, querying HEAD returns the persisted + offset without resetting to 0, and the upload completes successfully when resumed from + that offset. + """ + upload_uri = create_partial_upload(target_url, test_name=request.node.name) + status1, resp1, _, _ = http_request( + "PATCH", + upload_uri, + headers={UPLOAD_OFFSET: "0", UPLOAD_COMPLETE: FALSE, CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD}, + body=b"0123456789" * 4, # 40 bytes + test_name=request.node.name, + ) + assert status1 in (200, 204) + assert resp1.get(UPLOAD_OFFSET) == "40" + + # Simulate pause delay + time.sleep(0.5) + + # Verify offset via HEAD after pause + status_h, h_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name) + assert status_h in (200, 204) + assert h_headers.get(UPLOAD_OFFSET) == "40", "HEAD after pause must report 40 bytes" + + # Resume and complete upload + status2, resp2, _, _ = http_request( + "PATCH", + upload_uri, + headers={UPLOAD_OFFSET: "40", UPLOAD_COMPLETE: TRUE, CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD}, + body=b"abcdefghijklmnopqrstuvwxyz" * 2, # 52 bytes + test_name=request.node.name, + ) + assert status2 in (200, 201, 204) + assert resp2.get(UPLOAD_OFFSET) == "92" + assert resp2.get(UPLOAD_COMPLETE) == TRUE + class TestHttpDigests: """Tests for HTTP Digests (RFC 9530).""" diff --git a/scripts/tus_conformity_test.py b/scripts/tus_conformity_test.py index 78fb3b9c..5e09fa7c 100755 --- a/scripts/tus_conformity_test.py +++ b/scripts/tus_conformity_test.py @@ -1150,6 +1150,64 @@ def test_multi_chunk_resumable_upload_workflow(self, target_url): assert head_final.get(UPLOAD_OFFSET) == "120" assert head_final.get(UPLOAD_LENGTH) == "120" + def test_paused_upload_head_offset_and_resume(self, target_url): + """ + §6 PATCH: Upload Pause and Resume Workflow. + Quote: "If the connection is interrupted during a PATCH request, the Client SHOULD issue + a HEAD request to determine the current offset before resuming the upload." + Verifies that when an upload is paused midway, issuing a HEAD request accurately returns + the persisted offset without resetting to 0, and resuming with a subsequent PATCH from + that persisted offset successfully completes the upload. + """ + total_data = b"0123456789abcdefghijklmnopqrstuvwxyz" * 4 # 144 bytes + part1 = total_data[:60] + part2 = total_data[60:] + + upload_url, _ = create_upload(target_url, upload_length=str(len(total_data))) + + # 1. Send first chunk (simulating upload progress before user clicks pause) + headers1 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st1, resp_headers1, _ = http_request("PATCH", upload_url, headers=headers1, body=part1) + assert st1 == 204, f"Expected 204 No Content for first chunk, got {st1}" + assert resp_headers1.get(UPLOAD_OFFSET) == "60" + + # 2. Simulate client pause: query offset via HEAD to verify saved position + st_head1, head1, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_head1 in (200, 204) + assert head1.get(UPLOAD_OFFSET) == "60", ( + f"Expected Upload-Offset 60 after pause, got {head1.get(UPLOAD_OFFSET)}" + ) + + # 3. Simulate pause delay + time.sleep(0.5) + + # 4. Verify offset remained persisted and did not reset to 0 + st_head2, head2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_head2 in (200, 204) + assert head2.get(UPLOAD_OFFSET) == "60", ( + f"Expected Upload-Offset to remain 60, got {head2.get(UPLOAD_OFFSET)}" + ) + + # 5. Resume upload from the persisted offset + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "60", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st2, resp_headers2, _ = http_request("PATCH", upload_url, headers=headers2, body=part2) + assert st2 == 204, f"Expected 204 No Content for resumed chunk, got {st2}" + assert resp_headers2.get(UPLOAD_OFFSET) == str(len(total_data)) + + # 6. Final verification + st_final, head_final, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_final in (200, 204) + assert head_final.get(UPLOAD_OFFSET) == str(len(total_data)) + assert head_final.get(UPLOAD_LENGTH) == str(len(total_data)) + # CLI Entry Point & Custom Formatted Summary Reporter if __name__ == "__main__": diff --git a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java index 8ed264c8..11aca9d6 100644 --- a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java @@ -66,13 +66,14 @@ public HttpProblemDetails process( if (uploadInfo == null) { found = false; } else if (uploadInfo.isUploadInProgress()) { + InterruptibleInputStream interruptibleStream = null; try { InputStream stream = servletRequest.getContentInputStream(); // If a locking service is provided, wrap the input stream in an InterruptibleInputStream // and register it with the locking service to allow for interruption of the upload. if (lockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); + interruptibleStream = new InterruptibleInputStream(stream); lockingService.registerInputStream(servletRequest.getRequestURI(), interruptibleStream); stream = interruptibleStream; } @@ -94,6 +95,26 @@ public HttpProblemDetails process( } } catch (UploadNotFoundException e) { found = false; + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // PATCH with 204 No Content and the new Upload-Offset rather than throwing a 500 error. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "Upload PATCH request for URI {} was interrupted by locking service contention; " + + "saved partial upload up to offset {}", + servletRequest.getRequestURI(), + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = + uploadStorageService.getUploadInfo(servletRequest.getRequestURI(), ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java index 6594e191..7f8a8a94 100644 --- a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java @@ -46,13 +46,36 @@ public HttpProblemDetails process( UploadInfo uploadInfo = uploadStorageService.getUploadInfo(location, ownerKey); if (uploadInfo != null && uploadInfo.isUploadInProgress()) { InputStream stream = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); + interruptibleStream = new InterruptibleInputStream(stream); uploadLockingService.registerInputStream(location, interruptibleStream); stream = interruptibleStream; } - uploadInfo = uploadStorageService.append(uploadInfo, stream); + try { + uploadInfo = uploadStorageService.append(uploadInfo, stream); + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and + // updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // upload rather than propagating an unhandled error to the servlet container. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "Upload creation-with-upload POST request for URI {} was interrupted by locking" + + " service contention; saved partial upload up to offset {}", + location, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } + } servletResponse.setHeader( HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index d929d8fa..787089a7 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -14,6 +14,8 @@ import me.desair.tus.server.util.StructuredHeaderUtil; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Request handler for data append requests via HTTP PATCH. @@ -27,6 +29,8 @@ */ public class RufhAppendPatchRequestHandler extends AbstractRequestHandler { + private static final Logger log = LoggerFactory.getLogger(RufhAppendPatchRequestHandler.class); + @Override public boolean supports(HttpMethod method) { return HttpMethod.PATCH.equals(method); @@ -66,15 +70,36 @@ public HttpProblemDetails process( } InputStream is = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (is != null) { if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); + interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(requestUri, interruptibleStream); is = interruptibleStream; } - UploadInfo appended = uploadStorageService.append(uploadInfo, is); - if (appended != null) { - uploadInfo = appended; + try { + UploadInfo appended = uploadStorageService.append(uploadInfo, is); + if (appended != null) { + uploadInfo = appended; + } + } catch (IOException e) { + // When an append stream is interrupted by the locking service watchdog or a concurrent + // release request, the storage backend commits all bytes received so far and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // append rather than propagating an unhandled error. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "RUFH append request for URI {} was interrupted by locking service contention; " + + "saved partial upload up to offset {}", + requestUri, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = uploadStorageService.getUploadInfo(requestUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 2702c23e..e4eb2712 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -17,6 +17,8 @@ import me.desair.tus.server.util.TusServletResponse; import me.desair.tus.server.util.Utils; import org.apache.commons.lang3.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Request handler for upload creation requests via HTTP POST, PUT, or PATCH. @@ -29,6 +31,8 @@ */ public class RufhCreationPostRequestHandler extends AbstractRequestHandler { + private static final Logger log = LoggerFactory.getLogger(RufhCreationPostRequestHandler.class); + public RufhCreationPostRequestHandler() { // Default constructor } @@ -109,15 +113,37 @@ public HttpProblemDetails process( servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked")); InputStream is = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (is != null && hasContent) { if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); + interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(uploadUri, interruptibleStream); is = interruptibleStream; } - UploadInfo appended = uploadStorageService.append(uploadInfo, is); - if (appended != null) { - uploadInfo = appended; + try { + UploadInfo appended = uploadStorageService.append(uploadInfo, is); + if (appended != null) { + uploadInfo = appended; + } + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // upload rather than propagating an unhandled error to the servlet container. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + log.info( + "RUFH creation request with body for URI {} was interrupted by locking service" + + " contention; saved partial upload up to offset {}", + uploadUri, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 686797a2..ec0f6872 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -4,6 +4,7 @@ import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobProperties; import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.Block; import com.azure.storage.blob.models.BlockList; @@ -47,6 +48,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -233,6 +235,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) int initialBlockCount = blockIds.size(); long totalAppended = 0L; IOException streamException = null; + IOException drainException = null; TusException pendingTusException = null; List plannedBlockIds = new ArrayList<>(); @@ -301,9 +304,18 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } - int confirmedCount = uploader.drainAndComplete(4000); - for (int i = 0; i < confirmedCount; i++) { - blockIds.add(plannedBlockIds.get(i)); + // Drain remaining staged chunks with 60s timeout. + // Catch drainException so any chunks confirmed uploaded before timeout or error + // are committed and the metadata offset is preserved without loss. + try { + uploader.drainAndComplete(); + } catch (IOException e) { + drainException = e; + } finally { + int confirmed = Math.min(uploader.getConfirmedCount(), plannedBlockIds.size()); + for (int i = 0; i < confirmed; i++) { + blockIds.add(plannedBlockIds.get(i)); + } } } @@ -317,7 +329,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) // Batching block commits into a single call at the end eliminates redundant network // round-trips for every chunk, drastically improving performance. In addition, committing // here before throwing any pending stream or limit exception guarantees zero data loss - // if network drops or limits are hit midway. + // if client pauses, network drops, or limits are hit midway. if (blockIds.size() > initialBlockCount) { try { blockBlobClient.commitBlockList(blockIds, true); @@ -343,6 +355,10 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } + if (drainException != null) { + throw drainException; + } + if (streamException != null) { throw streamException; } @@ -492,16 +508,25 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio containerClient.getBlobClient(checksumKey).deleteIfExists(); } - // 4. Delete lock target and stop signal blobs (handling active lease exceptions gracefully) + // 4. Delete lock target and stop signal blobs try { containerClient.getBlobClient(locksPrefix + id + ".stop").deleteIfExists(); } catch (Exception ignored) { } try { - containerClient.getBlobClient(locksPrefix + id + ".lock").deleteIfExists(); + BlobClient lockBlob = containerClient.getBlobClient(locksPrefix + id + ".lock"); + // Check if lock blob is actively leased by the current request before attempting deletion. + // Attempting deleteIfExists() on a leased blob without the lease ID causes Azure Blob + // Storage to reject the request with HTTP 412 (LeaseIdMissing) and logs an SDK error. + if (Boolean.TRUE.equals(lockBlob.exists())) { + BlobProperties props = lockBlob.getProperties(); + if (props.getLeaseState() != null + && !Strings.CS.equals(props.getLeaseState().toString(), "leased")) { + lockBlob.deleteIfExists(); + } + } } catch (Exception ignored) { - // Lock blob may be actively leased by current request lock (Azure 412 LeaseIdMissing) } log.debug("Terminated upload with ID {}", id); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index 640cf20a..5b128453 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -1,9 +1,7 @@ package me.desair.tus.server.upload.s3; -import io.minio.ComposeObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.SourceObject; import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; @@ -48,6 +46,7 @@ public class S3ConcatenationService implements UploadConcatenationService { private final long minPartSize; private final Path temporaryDirectory; private UploadStorageService uploadStorageService; + private S3ServerSideComposeHelper s3ComposeHelper; /** * Basic constructor using default object prefix ("uploads/") and Java temp directory. @@ -112,6 +111,7 @@ public S3ConcatenationService( ? temporaryDirectory : java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); this.minPartSize = minPartSize; + this.s3ComposeHelper = new S3ServerSideComposeHelper(this.minioClient); } public void setUploadStorageService(UploadStorageService uploadStorageService) { @@ -162,7 +162,7 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep if (canUseServerSideCopy) { // Fast path: Compose S3 objects on cluster server-side without downloading data - mergeUsingServerSideCopy(targetObjectKey, partialUploads); + mergeUsingServerSideCopy(targetObjectKey, partialUploads, totalLength); } else { // Fallback path: Sequential stream re-upload for sub-5MB parts mergeUsingStreamingReupload(targetObjectKey, partialUploads, totalLength); @@ -246,23 +246,34 @@ public List getPartialUploads(UploadInfo info) return output; } - private void mergeUsingServerSideCopy(String targetKey, List partialUploads) - throws IOException { + private void mergeUsingServerSideCopy( + String targetKey, List partialUploads, long totalLength) throws IOException { try { - List sources = new ArrayList<>(); + List partKeys = new ArrayList<>(); for (UploadInfo partial : partialUploads) { - String partKey = partial.getStorageUploadId(); - sources.add(SourceObject.builder().bucket(bucket).object(partKey).build()); + partKeys.add(partial.getStorageUploadId()); } - // Execute S3 server-side object composition - minioClient.composeObject( - ComposeObjectArgs.builder().bucket(bucket).object(targetKey).sources(sources).build()); + // Execute S3 server-side object composition via native S3 multipart copy helper. + // S3ServerSideComposeHelper executes an UploadPartCopy sequence without Content-MD5, + // avoiding MinIO SDK's internal EMPTY_BODY issue on Amazon AWS S3. + s3ComposeHelper.compose(bucket, targetKey, partKeys); } catch (Exception e) { - throw new IOException("Failed server-side S3 composeObject merge for key " + targetKey, e); + // If server-side composition fails for any reason (e.g. S3 permissions, regional policy, + // or unhandled multipart copy restriction), fall back to streaming re-upload as a safety net. + // This ensures concatenated uploads always succeed even if server-side copy is denied. + log.warn( + "Server-side S3 composition failed for target key {}; falling back to streaming re-upload: {}", + targetKey, + e.getMessage()); + mergeUsingStreamingReupload(targetKey, partialUploads, totalLength); } } + void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) { + this.s3ComposeHelper = s3ComposeHelper; + } + private void mergeUsingStreamingReupload( String targetKey, List partialUploads, long totalLength) throws IOException { try (InputStream combinedStream = diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index f37c1278..e9bd22f2 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -80,6 +80,56 @@ public S3LockingService(MinioClient minioClient, String bucket) { new UuidUploadIdFactory()); } + /** + * Constructor accepting explicit connection parameters without requiring a pre-existing + * MinioClient. + * + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "eu-central-1", "us-east-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket Target S3 bucket name + */ + public S3LockingService( + String endpoint, String region, String accessKey, String secretKey, String bucket) { + this( + buildMinioClient(endpoint, region, accessKey, secretKey), + bucket, + DEFAULT_LOCKS_PREFIX, + DEFAULT_LEASE_DURATION_MS, + DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); + } + + /** + * Constructor accepting a pre-configured {@link MinioClient} along with explicit connection + * parameters. + * + * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL + * @param region S3 region name + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket Target S3 bucket name + */ + public S3LockingService( + MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, + String bucket) { + this( + minioClient != null + ? minioClient + : buildMinioClient(endpoint, region, accessKey, secretKey), + bucket, + DEFAULT_LOCKS_PREFIX, + DEFAULT_LEASE_DURATION_MS, + DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); + } + /** * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. * @@ -395,4 +445,14 @@ private String buildLockKey(UploadId uploadId) { private String buildStopKey(UploadId uploadId) { return locksPrefix + uploadId.toString() + ".stop"; } + + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); + } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java new file mode 100644 index 00000000..8a645173 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java @@ -0,0 +1,303 @@ +package me.desair.tus.server.upload.s3; + +import io.minio.AbortMultipartUploadArgs; +import io.minio.Checksum; +import io.minio.CompleteMultipartUploadArgs; +import io.minio.ComposeObjectArgs; +import io.minio.CreateMultipartUploadArgs; +import io.minio.CreateMultipartUploadResponse; +import io.minio.Http; +import io.minio.MinioAsyncClient; +import io.minio.MinioClient; +import io.minio.Signer; +import io.minio.SourceObject; +import io.minio.Time; +import io.minio.Utils; +import io.minio.credentials.Credentials; +import io.minio.credentials.Provider; +import io.minio.credentials.StaticProvider; +import io.minio.messages.Part; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper executing native S3 Multipart Copy operations to combine part objects server-side. + * + *

Why this helper exists: MinIO Java SDK's {@code composeObject()} implementation delegates to + * {@code UploadPartCopy} with an internal {@code EMPTY_BODY} placeholder. In MinIO SDK's {@code + * Http.toRequest()}, that placeholder automatically attaches {@code Content-MD5} and {@code + * Content-Type} headers to the HTTP request. While MinIO server ignores those extra headers, Amazon + * AWS S3 strictly forbids the {@code Content-MD5} header on {@code UploadPartCopy} requests, + * returning HTTP 400 InvalidArgument ("The specified header is not valid in this context"). + * Attempting to remove the header via an OkHttp network interceptor also fails because AWS SigV4 + * signature calculation includes all present headers, resulting in a SignatureDoesNotMatch error. + * + *

Workaround: This helper executes standard S3 Multipart Upload Copy operations directly via + * OkHttp and AWS SigV4 signing without adding {@code Content-MD5}. This allows fast zero-bandwidth + * server-side composition on both Amazon AWS S3 and MinIO/Ceph clusters without needing heavy + * external AWS SDK dependencies. + * + *

Safety Fallback: If native server-side compose fails for any reason (e.g. S3 permissions, + * regional restrictions, or client reflection unavailability), callers (such as {@link + * S3StorageService} and {@link S3ConcatenationService}) catch the exception and fall back to + * sequential streaming concatenation (re-uploading the combined parts). + */ +public class S3ServerSideComposeHelper { + + private static final Logger log = LoggerFactory.getLogger(S3ServerSideComposeHelper.class); + + private final MinioClient minioClient; + private final MinioAsyncClient asyncClient; + private final Http.BaseUrl baseUrl; + private final Provider provider; + private final OkHttpClient httpClient; + private final String explicitRegion; + + /** + * Constructs an instance wrapping the given {@link MinioClient} without reflection. + * + *

If explicit connection parameters are not provided, this helper falls back to calling {@link + * MinioClient#composeObject(ComposeObjectArgs)} directly. + * + * @param minioClient The MinIO client instance + */ + public S3ServerSideComposeHelper(MinioClient minioClient) { + this.minioClient = minioClient; + this.asyncClient = null; + this.baseUrl = null; + this.provider = null; + this.httpClient = null; + this.explicitRegion = null; + } + + /** + * Constructs an instance with an existing {@link MinioClient} and explicit connection parameters. + * + *

This allows native S3 multipart copy without using reflection on {@link MinioClient}. + * + * @param minioClient The existing MinIO client instance + * @param endpoint The S3 endpoint URL (e.g. "https://s3.amazonaws.com" or + * "http://localhost:9000") + * @param region S3 region name (optional, defaults to "eu-central-1" if null or empty) + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + */ + public S3ServerSideComposeHelper( + MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey) { + this.minioClient = minioClient; + this.explicitRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + + Http.BaseUrl base = null; + Provider prov = null; + OkHttpClient client = null; + MinioAsyncClient async = null; + + if (endpoint != null && !endpoint.isEmpty()) { + try { + base = new Http.BaseUrl(endpoint); + if (region != null && !region.isEmpty()) { + base.setRegion(region); + } + prov = new StaticProvider(accessKey, secretKey, null); + client = new OkHttpClient(); + + MinioAsyncClient.Builder asyncBuilder = + MinioAsyncClient.builder().endpoint(endpoint).credentials(accessKey, secretKey); + if (region != null && !region.isEmpty()) { + asyncBuilder.region(region); + } + async = asyncBuilder.build(); + } catch (Exception e) { + log.warn( + "Failed to initialize native S3 connection from endpoint {}: {}", + endpoint, + e.getMessage()); + } + } + + this.baseUrl = base; + this.provider = prov; + this.httpClient = client; + this.asyncClient = async; + } + + /** + * Returns whether native S3 multipart copy is available with the extracted client properties. + * + * @return true if all required fields are accessible; false otherwise + */ + public boolean isAvailable() { + return asyncClient != null && baseUrl != null && httpClient != null; + } + + /** + * Composes the given part keys into the target object key entirely server-side on S3. + * + * @param bucket The S3 bucket name + * @param targetKey The destination object key + * @param partKeys The source part object keys to combine in order + * @throws Exception If composition fails + */ + public void compose(String bucket, String targetKey, List partKeys) throws Exception { + if (!isAvailable()) { + // Fallback for mocked MinioClient instances in tests + List sources = new ArrayList<>(); + for (String pk : partKeys) { + sources.add(SourceObject.builder().bucket(bucket).object(pk).build()); + } + minioClient.composeObject( + ComposeObjectArgs.builder().bucket(bucket).object(targetKey).sources(sources).build()); + return; + } + + String region = baseUrl.region(); + if (region == null || region.isEmpty()) { + region = explicitRegion; + } + if (region == null || region.isEmpty()) { + region = "eu-central-1"; + } + + Credentials credentials = (provider != null) ? provider.fetch() : null; + + // 1. Create Multipart Upload + CreateMultipartUploadArgs createArgs = + CreateMultipartUploadArgs.builder().bucket(bucket).object(targetKey).build(); + CreateMultipartUploadResponse createResponse = + asyncClient.createMultipartUpload(createArgs).join(); + String uploadId = createResponse.result().uploadId(); + log.info( + "Started server-side multipart copy upload (ID: {}) for target {} in bucket {} with {} parts", + uploadId, + targetKey, + bucket, + partKeys.size()); + + List parts = new ArrayList<>(); + try { + // 2. UploadPartCopy for each part chunk without Content-MD5 + for (int i = 0; i < partKeys.size(); i++) { + int partNumber = i + 1; + String partKey = partKeys.get(i); + + Http.QueryParameters queryParams = + new Http.QueryParameters( + "partNumber", Integer.toString(partNumber), "uploadId", uploadId); + HttpUrl url = baseUrl.buildUrl(Http.Method.PUT, bucket, targetKey, region, queryParams); + + // x-amz-copy-source must be URL-encoded "/bucket/key" + String copySource = "/" + bucket + "/" + Utils.encodePath(partKey); + String now = ZonedDateTime.now().format(Time.AMZ_DATE_FORMAT); + + Request.Builder requestBuilder = + new Request.Builder() + .url(url) + .put(RequestBody.create(new byte[0], null)) + .header("Host", Utils.getHostHeader(url)) + .header("x-amz-date", now) + .header("x-amz-copy-source", copySource) + .header("x-amz-content-sha256", Checksum.ZERO_SHA256_HASH); + + if (credentials != null + && credentials.sessionToken() != null + && !credentials.sessionToken().isEmpty()) { + requestBuilder.header("x-amz-security-token", credentials.sessionToken()); + } + + Request httpRequest = requestBuilder.build(); + if (credentials != null) { + httpRequest = + Signer.signV4S3( + httpRequest, + region, + credentials.accessKey(), + credentials.secretKey(), + Checksum.ZERO_SHA256_HASH); + } + + try (Response response = httpClient.newCall(httpRequest).execute()) { + if (!response.isSuccessful()) { + String errorBody = response.body() != null ? response.body().string() : ""; + throw new IOException( + "S3 UploadPartCopy failed for part " + + partNumber + + " (" + + partKey + + "): HTTP " + + response.code() + + " " + + errorBody); + } + String responseBody = response.body() != null ? response.body().string() : ""; + String etag = extractEtagFromXml(responseBody); + if (etag == null || etag.isEmpty()) { + etag = response.header("ETag"); + } + if (etag == null || etag.isEmpty()) { + throw new IOException("Missing ETag in UploadPartCopy response for part " + partNumber); + } + parts.add(new Part(partNumber, etag)); + } + } + + // 3. Complete Multipart Upload + CompleteMultipartUploadArgs completeArgs = + CompleteMultipartUploadArgs.builder() + .bucket(bucket) + .object(targetKey) + .uploadId(uploadId) + .parts(parts.toArray(new Part[0])) + .build(); + asyncClient.completeMultipartUpload(completeArgs).join(); + log.info( + "Successfully completed server-side multipart copy upload (ID: {}) for target {} in bucket {}", + uploadId, + targetKey, + bucket); + + } catch (Exception e) { + try { + asyncClient + .abortMultipartUpload( + AbortMultipartUploadArgs.builder() + .bucket(bucket) + .object(targetKey) + .uploadId(uploadId) + .build()) + .join(); + } catch (Exception abortEx) { + log.warn("Failed to abort multipart upload {} for target {}", uploadId, targetKey, abortEx); + } + throw e; + } + } + + /** + * Extracts the ETag value from an S3 {@code CopyPartResult} XML string. + * + * @param xml The XML response from S3 UploadPartCopy + * @return Extracted ETag string, or null if not found + */ + static String extractEtagFromXml(String xml) { + if (xml == null) { + return null; + } + int start = xml.indexOf(""); + int end = xml.indexOf(""); + if (start != -1 && end != -1 && end > start + 6) { + String etag = xml.substring(start + 6, end).trim(); + etag = etag.replace(""", "\""); + return etag; + } + return null; + } +} diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 6e9c2502..dc89b998 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -1,13 +1,11 @@ package me.desair.tus.server.upload.s3; -import io.minio.ComposeObjectArgs; import io.minio.GetObjectArgs; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; import io.minio.Result; -import io.minio.SourceObject; import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; @@ -121,6 +119,8 @@ public class S3StorageService implements UploadStorageService { private UploadIdFactory idFactory = new UuidUploadIdFactory(); private UploadConcatenationService concatenationService; + private volatile boolean s3ComposeObjectSupported = true; + private S3ServerSideComposeHelper s3ComposeHelper; /** * Basic constructor using default object key prefixes and standard system temp directory. @@ -139,6 +139,64 @@ public S3StorageService(MinioClient minioClient, String bucket) { Paths.get(System.getProperty("java.io.tmpdir"))); } + /** + * Constructor accepting explicit connection parameters without requiring a pre-existing + * MinioClient. + * + *

Builds both {@link MinioClient} and the native {@link S3ServerSideComposeHelper} directly + * using the provided connection properties, avoiding reflection. + * + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + */ + public S3StorageService( + String endpoint, String region, String accessKey, String secretKey, String bucket) { + this( + buildMinioClient(endpoint, region, accessKey, secretKey), + bucket, + DEFAULT_OBJECT_PREFIX, + DEFAULT_METADATA_PREFIX, + DEFAULT_CHECKSUMS_PREFIX, + DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir")), + new S3ServerSideComposeHelper(null, endpoint, region, accessKey, secretKey)); + } + + /** + * Constructor accepting a pre-configured {@link MinioClient} along with the explicit connection + * parameters required by {@link S3ServerSideComposeHelper}. + * + *

This provides full control over {@link MinioClient} configuration while cleanly initializing + * server-side multipart copy without reflection. + * + * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + */ + public S3StorageService( + MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, + String bucket) { + this( + minioClient, + bucket, + DEFAULT_OBJECT_PREFIX, + DEFAULT_METADATA_PREFIX, + DEFAULT_CHECKSUMS_PREFIX, + DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir")), + new S3ServerSideComposeHelper(minioClient, endpoint, region, accessKey, secretKey)); + } + /** * Full constructor allowing full customization of object prefixes and local disk buffer path. * @@ -158,6 +216,27 @@ public S3StorageService( String checksumsPrefix, String locksPrefix, Path temporaryDirectory) { + this( + minioClient, + bucket, + objectPrefix, + metadataPrefix, + checksumsPrefix, + locksPrefix, + temporaryDirectory, + new S3ServerSideComposeHelper(minioClient)); + } + + /** Internal constructor accepting an initialized {@link S3ServerSideComposeHelper}. */ + private S3StorageService( + MinioClient minioClient, + String bucket, + String objectPrefix, + String metadataPrefix, + String checksumsPrefix, + String locksPrefix, + Path temporaryDirectory, + S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); this.objectPrefix = sanitizePrefix(objectPrefix); @@ -195,6 +274,31 @@ public S3StorageService( this.concatenationService = new S3ConcatenationService( this.minioClient, this.bucket, this.objectPrefix, this, this.temporaryDirectory); + this.s3ComposeHelper = + s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); + if (this.concatenationService instanceof S3ConcatenationService) { + ((S3ConcatenationService) this.concatenationService) + .setS3ServerSideComposeHelper(this.s3ComposeHelper); + } + } + + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); + } + + /** + * Returns the underlying {@link MinioClient} configured for this storage service. + * + * @return The MinIO client instance + */ + public MinioClient getMinioClient() { + return this.minioClient; } /** @@ -724,6 +828,22 @@ public UploadConcatenationService getUploadConcatenationService() { return concatenationService; } + boolean isS3ComposeObjectSupported() { + return s3ComposeObjectSupported; + } + + void setS3ComposeObjectSupported(boolean s3ComposeObjectSupported) { + this.s3ComposeObjectSupported = s3ComposeObjectSupported; + } + + void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) { + this.s3ComposeHelper = s3ComposeHelper; + } + + S3ServerSideComposeHelper getS3ServerSideComposeHelper() { + return s3ComposeHelper; + } + @Override public void setIdFactory(UploadIdFactory idFactory) { if (idFactory != null) { @@ -993,7 +1113,7 @@ private AppendResult processPayloadChunks( + MAX_PARTS_PER_UPLOAD + " parts."); if (chunkBytesWritten > 0) { - int confirmedCount = uploader.drainAndComplete(4000); + int confirmedCount = uploader.drainAndComplete(); allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); handedOff = true; @@ -1020,7 +1140,7 @@ private AppendResult processPayloadChunks( } else if (streamFinished && isUploadComplete) { // Sub-5MB final chunk that completes the overall upload: // First drain all preceding parts in the pipeline - int confirmedCount = uploader.drainAndComplete(4000); + int confirmedCount = uploader.drainAndComplete(); allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); String chunkKey = buildChunkPartKey(id, nextPartNumber++); @@ -1031,7 +1151,7 @@ private AppendResult processPayloadChunks( } else { // Sub-5MB incomplete chunk (e.g. upload paused midway or interrupted): // First drain all preceding parts in the pipeline - int confirmedCount = uploader.drainAndComplete(4000); + int confirmedCount = uploader.drainAndComplete(); allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); @@ -1045,13 +1165,20 @@ private AppendResult processPayloadChunks( } } - // Drain any remaining in-flight chunks in the pipeline - int confirmedCount = uploader.drainAndComplete(4000); - int previouslyConfirmed = allPartKeys.size() - preparedStream.remainingPartKeys.size(); - if (confirmedCount > previouslyConfirmed) { - allPartKeys.clear(); - allPartKeys.addAll(preparedStream.remainingPartKeys); - allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + // Drain any remaining in-flight chunks in the pipeline with 60s timeout. + // Catch/finally ensures all confirmed parts are retained even if a subsequent chunk times + // out. + int confirmedCount = 0; + try { + confirmedCount = uploader.drainAndComplete(); + } finally { + int confirmed = Math.max(confirmedCount, uploader.getConfirmedCount()); + int previouslyConfirmed = allPartKeys.size() - preparedStream.remainingPartKeys.size(); + if (confirmed > previouslyConfirmed) { + allPartKeys.clear(); + allPartKeys.addAll(preparedStream.remainingPartKeys); + allPartKeys.addAll(plannedPartKeys.subList(0, confirmed)); + } } } @@ -1195,32 +1322,33 @@ private void finalizeCompletedUploadIfFinished( } } - if (canUseServerSideCompose) { + if (canUseServerSideCompose && s3ComposeObjectSupported) { try { - List sources = new ArrayList<>(); - for (String pk : partKeys) { - sources.add(SourceObject.builder().bucket(bucket).object(pk).build()); - } - - // Perform S3 server-side object composition (composeObject) - minioClient.composeObject( - ComposeObjectArgs.builder() - .bucket(bucket) - .object(objectKey) - .sources(sources) - .build()); + // Perform S3 server-side object composition via native S3 multipart copy + s3ComposeHelper.compose(bucket, objectKey, partKeys); } catch (Exception e) { - log.warn( - "S3 composeObject failed for object {}, falling back to streaming concatenation:" - + " {}", + // MinIO Java SDK's composeObject implementation delegates to UploadPartCopy with an + // EMPTY_BODY, which automatically attaches Content-MD5 and Content-Type headers. + // AWS S3 strictly forbids Content-MD5 on UploadPartCopy and rejects it with 400 + // InvalidArgument ("The specified header is not valid in this context"). + // When server-side compose fails on this S3 endpoint, disable it dynamically + // to avoid redundant failing S3 API roundtrips on subsequent uploads, log at INFO, + // and seamlessly merge chunks via streaming part composition. + s3ComposeObjectSupported = false; + log.info( + "S3 server-side compose failed for object {}, falling back to streaming part" + + " composition: {}", objectKey, e.getMessage()); mergeUsingStreamingReupload(objectKey, partKeys, newOffset); } } else { - log.info( - "Detected sub-5MB non-final parts for ID {}. Using streaming concatenation fallback.", - id); + if (!canUseServerSideCompose) { + log.info( + "Detected sub-5MB non-final parts for ID {}. Using streaming part composition" + + " fallback.", + id); + } mergeUsingStreamingReupload(objectKey, partKeys, newOffset); } diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java index f523e40c..c091bc1f 100644 --- a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -35,6 +35,8 @@ public class AsyncChunkUploader implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(AsyncChunkUploader.class); + public static final long DEFAULT_DRAIN_TIMEOUT_MS = 300_000L; + @FunctionalInterface public interface ChunkUploadAction { /** @@ -141,6 +143,17 @@ public void submitChunk( } } + /** + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) using the default timeout + * ({@link #DEFAULT_DRAIN_TIMEOUT_MS}, 60 seconds). + * + * @return The total number of confirmed successfully uploaded chunks + * @throws IOException If any chunk upload fails or times out + */ + public int drainAndComplete() throws IOException { + return drainAndComplete(DEFAULT_DRAIN_TIMEOUT_MS); + } + /** * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) up to the specified timeout. * diff --git a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java index 05576a70..1656ae53 100644 --- a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.when; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; import java.io.InputStream; import java.util.UUID; import me.desair.tus.server.HttpHeader; @@ -226,4 +227,77 @@ public void testProcess7ParamsBranchCoverage() throws Exception { "owner", null); } + + @Test + public void testProcessInterruptedByLockingServiceContentionReturns204WithUpdatedOffset() + throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + + servletRequest.setRequestURI("/test/upload/123"); + servletRequest.setContent("partial data".getBytes()); + + UploadInfo initialInfo = new UploadInfo(); + initialInfo.setId(new UploadId("123")); + initialInfo.setOffset(0L); + initialInfo.setLength(100L); + + UploadInfo refreshedInfo = new UploadInfo(); + refreshedInfo.setId(new UploadId("123")); + refreshedInfo.setOffset(50L); + refreshedInfo.setLength(100L); + + when(uploadStorageService.getUploadInfo("/test/upload/123", "owner")) + .thenReturn(initialInfo) + .thenReturn(refreshedInfo); + + when(uploadStorageService.append(any(UploadInfo.class), any(InputStream.class))) + .thenAnswer( + invocation -> { + InputStream stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + "owner", + null); + + assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); + assertThat(servletResponse.getHeader(HttpHeader.UPLOAD_OFFSET), is("50")); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + + servletRequest.setRequestURI("/test/upload/123"); + servletRequest.setContent("data".getBytes()); + + UploadInfo initialInfo = new UploadInfo(); + initialInfo.setId(new UploadId("123")); + initialInfo.setOffset(0L); + initialInfo.setLength(100L); + + when(uploadStorageService.getUploadInfo("/test/upload/123", "owner")).thenReturn(initialInfo); + + when(uploadStorageService.append(any(UploadInfo.class), any(InputStream.class))) + .thenThrow(new IOException("Disk failure")); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + "owner", + null); + } } diff --git a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java index 9001c2c0..bf7848a9 100644 --- a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java @@ -10,6 +10,7 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -95,4 +96,67 @@ public void testProcessWithoutLockingService() throws Exception { verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "5"); } + + @Test + public void testProcessInterruptedByLockingServiceContention() throws Exception { + TusServletRequest request = mock(TusServletRequest.class); + TusServletResponse response = mock(TusServletResponse.class); + UploadStorageService storageService = mock(UploadStorageService.class); + UploadLockingService lockingService = mock(UploadLockingService.class); + + when(request.getHeader(HttpHeader.CONTENT_LENGTH)).thenReturn("100"); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/interrupted-cwu"); + when(request.getContentInputStream()).thenReturn(new ByteArrayInputStream("data".getBytes())); + + UploadInfo uploadInfo = new UploadInfo(); + uploadInfo.setLength(100L); + uploadInfo.setOffset(0L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setLength(100L); + refreshed.setOffset(50L); + + when(storageService.getUploadInfo("/files/interrupted-cwu", "owner")) + .thenReturn(uploadInfo) + .thenReturn(refreshed); + + when(storageService.append(eq(uploadInfo), any(InputStream.class))) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + + verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "50"); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + TusServletRequest request = mock(TusServletRequest.class); + TusServletResponse response = mock(TusServletResponse.class); + UploadStorageService storageService = mock(UploadStorageService.class); + UploadLockingService lockingService = mock(UploadLockingService.class); + + when(request.getHeader(HttpHeader.CONTENT_LENGTH)).thenReturn("100"); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/cwu-error"); + when(request.getContentInputStream()).thenReturn(new ByteArrayInputStream("data".getBytes())); + + UploadInfo uploadInfo = new UploadInfo(); + uploadInfo.setLength(100L); + uploadInfo.setOffset(0L); + + when(storageService.getUploadInfo("/files/cwu-error", "owner")).thenReturn(uploadInfo); + when(storageService.append(eq(uploadInfo), any(InputStream.class))) + .thenThrow(new IOException("S3 network error")); + + handler.process( + HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index 609f1a96..ff2fd00e 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -10,6 +10,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; @@ -305,4 +306,82 @@ public void testProcessCompletingAppendOnDeferredLengthUploadWithNullOffsetSetsL assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + @Test + public void testProcessInterruptedByLockingServiceContention() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + byte[] content = "partial-data".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(0L); + info.setLength(1000L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("append-id")); + refreshed.setOffset(500L); + refreshed.setLength(1000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")) + .thenReturn(info) + .thenReturn(refreshed); + + when(storageService.append(any(UploadInfo.class), any())) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(204)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("500")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + byte[] content = "data".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(0L); + info.setLength(1000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())) + .thenThrow(new IOException("Storage failure")); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index e4e9b8c4..3863559f 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -567,4 +568,81 @@ public InputStream getContentInputStream() { assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); } + + @Test + public void testProcessCreationWithBodyInterrupted() throws Exception { + byte[] content = "partial-creation-data".getBytes(); + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.setContent(content); + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("interrupted-create-id")); + createdInfo.setOffset(0L); + createdInfo.setLength(1000L); + + UploadInfo refreshedInfo = new UploadInfo(); + refreshedInfo.setId(new UploadId("interrupted-create-id")); + refreshedInfo.setOffset(500L); + refreshedInfo.setLength(1000L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + when(storageService.getUploadInfo("/files/interrupted-create-id", "owner")) + .thenReturn(refreshedInfo); + + when(storageService.append(any(UploadInfo.class), any())) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("500")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } + + @Test(expected = IOException.class) + public void testProcessCreationWithBodyUninterruptedIoException() throws Exception { + byte[] content = "creation-data".getBytes(); + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.setContent(content); + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("fail-create-id")); + createdInfo.setOffset(0L); + createdInfo.setLength(1000L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + + when(storageService.append(any(UploadInfo.class), any())) + .thenThrow(new IOException("Disk write failure")); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 44505494..1273e67e 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -258,6 +258,40 @@ public void terminateUploadShouldDeleteBlobs() throws Exception { assertNull(storageService.getUploadInfo(created.getId())); } + @Test + public void terminateUploadShouldHandleLeasedAndUnleasedLockBlobs() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertNotNull(storageService.getUploadInfo(created.getId())); + + // 1. Verify unleased lock blob deletion: create an unleased lock blob and verify it is removed + BlobClient lockBlob = containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + storageService.terminateUpload(created); + assertNull(storageService.getUploadInfo(created.getId())); + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + + // 2. Verify leased lock blob: create upload, hold active lease, and verify terminateUpload + // skips deletion so Azure HTTP 412 (LeaseIdMissing) error is never triggered or logged + UploadInfo info2 = new UploadInfo(); + info2.setLength(10L); + UploadInfo created2 = storageService.create(info2, "owner1"); + + AzureBlobLockingService lockingService = new AzureBlobLockingService(containerClient); + UploadLock lock = lockingService.lockUploadByUri("/test/upload/" + created2.getId()); + try { + storageService.terminateUpload(created2); + assertNull(storageService.getUploadInfo(created2.getId())); + } finally { + lock.release(); + } + } + @Test public void removeLastNumberOfBytesPartBlobOnly() throws Exception { storageService.setPreferredBlockSize(4L * 1024 * 1024); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index f9970131..7f3d3470 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -263,8 +263,8 @@ public void testMergeMultiplePartsWithSub5MbNonFinalPartFallsBackToStreaming() t Mockito.verify(minioClient).putObject(Mockito.any(PutObjectArgs.class)); } - @Test(expected = IOException.class) - public void testMergeServerSideCopyFails() throws Exception { + @Test + public void testMergeServerSideCopyFailsFallsBackToStreaming() throws Exception { UploadInfo p1 = new UploadInfo(); p1.setId(new UploadId("part-1")); p1.setOwnerKey("owner-1"); @@ -273,6 +273,8 @@ public void testMergeServerSideCopyFails() throws Exception { p1.setStorageUploadId("uploads/part-1"); Mockito.when(storageService.getUploadInfo("/part-1", "owner-1")).thenReturn(p1); + Mockito.when(storageService.getUploadedBytes(new UploadId("part-1"))) + .thenReturn(new ByteArrayInputStream(new byte[10])); Mockito.when(minioClient.composeObject(Mockito.any(ComposeObjectArgs.class))) .thenThrow(new RuntimeException("Compose error")); @@ -282,6 +284,10 @@ public void testMergeServerSideCopyFails() throws Exception { finalUpload.setConcatenationPartIds(Arrays.asList("/part-1")); concatenationService.merge(finalUpload); + + // Verify fallback to streaming putObject + Mockito.verify(minioClient).putObject(Mockito.any(PutObjectArgs.class)); + assertEquals(Long.valueOf(10L * 1024 * 1024), finalUpload.getLength()); } @Test(expected = IOException.class) diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 832220d6..cff254f3 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -137,6 +137,24 @@ public void testLockUploadByUriInvalidUri() throws Exception { assertNull(lock); } + @Test + public void testExplicitConnectionParametersConstructors() { + S3LockingService serviceWithParams = + new S3LockingService( + "https://s3.amazonaws.com", "eu-central-1", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithParams); + + S3LockingService serviceWithClientAndParams = + new S3LockingService( + minioClient, + "https://s3.amazonaws.com", + "eu-central-1", + "accessKey", + "secretKey", + "test-bucket"); + assertNotNull(serviceWithClientAndParams); + } + @Test public void testIsLocked() throws Exception { assertFalse(lockingService.isLocked((UploadId) null)); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java new file mode 100644 index 00000000..c0afd7ae --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java @@ -0,0 +1,362 @@ +package me.desair.tus.server.upload.s3; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.minio.AbortMultipartUploadArgs; +import io.minio.CompleteMultipartUploadArgs; +import io.minio.ComposeObjectArgs; +import io.minio.CreateMultipartUploadArgs; +import io.minio.CreateMultipartUploadResponse; +import io.minio.Http; +import io.minio.MinioAsyncClient; +import io.minio.MinioClient; +import io.minio.ObjectWriteResponse; +import io.minio.credentials.Credentials; +import io.minio.credentials.Provider; +import io.minio.messages.InitiateMultipartUploadResult; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.Test; + +/** Unit tests for {@link S3ServerSideComposeHelper}. */ +public class S3ServerSideComposeHelperTest { + + @Test + public void testExtractEtagFromXml() { + // Standard XML response + String xml = + "" + + "2026-10-03T12:00:00.000Z" + + "\"1234567890abcdef\"" + + ""; + assertEquals("\"1234567890abcdef\"", S3ServerSideComposeHelper.extractEtagFromXml(xml)); + + // Entity escaped XML quotes + String escapedXml = + ""9876543210fedcba""; + assertEquals("\"9876543210fedcba\"", S3ServerSideComposeHelper.extractEtagFromXml(escapedXml)); + + // Whitespace inside tag + String whitespaceXml = " \"xyz123\" "; + assertEquals("\"xyz123\"", S3ServerSideComposeHelper.extractEtagFromXml(whitespaceXml)); + + // Null or missing tags + assertNull(S3ServerSideComposeHelper.extractEtagFromXml(null)); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + } + + @Test + public void testFallbackWhenNotAvailable() throws Exception { + MinioClient mockMinioClient = mock(MinioClient.class); + // Since mock MinioClient does not have initialized asyncClient/baseUrl/httpClient fields, + // isAvailable() will return false, triggering the composeObject() fallback. + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(mockMinioClient); + assertFalse(helper.isAvailable()); + + List partKeys = Arrays.asList("upload/part-1", "upload/part-2"); + helper.compose("my-bucket", "upload/target", partKeys); + + verify(mockMinioClient).composeObject(any(ComposeObjectArgs.class)); + } + + @Test + public void testNullMinioClient() { + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(null); + assertFalse(helper.isAvailable()); + } + + @Test + public void testNativeComposeSuccess() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", "testSessionToken", null); + when(provider.fetch()).thenReturn(creds); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.us-west-2.amazonaws.com")); + + // Prepare createMultipartUpload response + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse( + null, "my-bucket", "us-west-2", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + // Prepare completeMultipartUpload response + ObjectWriteResponse completeResponse = + new ObjectWriteResponse( + null, "my-bucket", "us-west-2", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + // Mock OkHttp execution for uploadPartCopy + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + String partXml = "\"part-etag-1\""; + ResponseBody body = ResponseBody.create(partXml, MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.us-west-2.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + // Assemble helper with reflection fields populated + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + + assertTrue(helper.isAvailable()); + + List partKeys = Collections.singletonList("upload/part-1"); + helper.compose("my-bucket", "upload/target", partKeys); + + verify(asyncClient).createMultipartUpload(any(CreateMultipartUploadArgs.class)); + verify(httpClient).newCall(any(Request.class)); + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + + @Test + public void testNativeComposeHttpErrorAborts() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", null, null); + when(provider.fetch()).thenReturn(creds); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + // Prepare createMultipartUpload response + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + when(asyncClient.abortMultipartUpload(any(AbortMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + + // Mock OkHttp returning HTTP 400 + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + ResponseBody body = + ResponseBody.create( + "Test Error", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(400) + .message("Bad Request") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + + try { + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + fail("Expected IOException to be thrown"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("HTTP 400")); + } + + verify(asyncClient).abortMultipartUpload(any(AbortMultipartUploadArgs.class)); + } + + @Test + public void testNativeComposeMissingEtagHeaderFallback() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + ObjectWriteResponse completeResponse = + new ObjectWriteResponse(null, "my-bucket", "", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + // Response body has no XML ETag tag, but response header has ETag + ResponseBody body = ResponseBody.create("", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .header("ETag", "\"header-etag\"") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + + @Test + public void testExplicitConnectionParametersConstructor() { + MinioClient realClient = + MinioClient.builder() + .endpoint("https://s3.amazonaws.com") + .credentials("testKey", "testSecret") + .region("us-east-1") + .build(); + + S3ServerSideComposeHelper helper = + new S3ServerSideComposeHelper( + realClient, "https://s3.amazonaws.com", "us-east-1", "testKey", "testSecret"); + assertTrue(helper.isAvailable()); + + S3ServerSideComposeHelper minioOnlyHelper = new S3ServerSideComposeHelper(realClient); + assertFalse(minioOnlyHelper.isAvailable()); + } + + @Test(expected = IOException.class) + public void testNativeComposeMissingEtagThrowsException() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + // Response body has no XML ETag tag, and response header also has no ETag + ResponseBody body = ResponseBody.create("", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + } + + @Test + public void testNativeComposeAbortFailureHandledQuietly() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + // Abort itself throws an exception to test the log.warn catch block + CompletableFuture failedAbort = + new CompletableFuture<>(); + failedAbort.completeExceptionally(new RuntimeException("Abort S3 error")); + when(asyncClient.abortMultipartUpload(any(AbortMultipartUploadArgs.class))) + .thenReturn(failedAbort); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(500) + .message("Internal Server Error") + .body(ResponseBody.create("error", MediaType.parse("text/plain"))) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + try { + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + fail("Expected IOException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("HTTP 500")); + } + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field field = S3ServerSideComposeHelper.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 1b43b0d8..d795775c 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -79,6 +79,26 @@ public void testNullTemporaryDirectoryConstructor() { assertNotNull(serviceWithNullTmp); } + @Test + public void testExplicitConnectionParametersConstructors() { + // 1. Constructor taking connection parameters directly (Option A) + S3StorageService serviceWithParams = + new S3StorageService( + "https://s3.amazonaws.com", "us-east-1", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithParams); + + // 2. Constructor taking MinioClient and connection parameters (Option B) + S3StorageService serviceWithClientAndParams = + new S3StorageService( + minioClient, + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket"); + assertNotNull(serviceWithClientAndParams); + } + @Test public void testGetS3ObjectKeyByUri() throws Exception { UploadInfo info = new UploadInfo(); @@ -1696,6 +1716,118 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA .composeObject(any(ComposeObjectArgs.class)); } + @Test + public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAndDisablesCompose() + throws Exception { + assertTrue(storageService.isS3ComposeObjectSupported()); + + UploadInfo info = new UploadInfo(); + UploadId id = new UploadId("compose-fail-test-123"); + long fiveMb = 5L * 1024L * 1024L; + info.setId(id); + info.setLength(fiveMb + 100L); + info.setOffset(0L); + + String json = UploadInfoJsonSerializer.serialize(info); + + Item item1 = mock(Item.class); + when(item1.objectName()).thenReturn("uploads/compose-fail-test-123.part.00001"); + when(item1.size()).thenReturn(fiveMb); + + Item item2 = mock(Item.class); + when(item2.objectName()).thenReturn("uploads/compose-fail-test-123.part.00002"); + when(item2.size()).thenReturn(100L); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); + + StatObjectResponse stat1 = mock(StatObjectResponse.class); + when(stat1.size()).thenReturn(fiveMb); + StatObjectResponse stat2 = mock(StatObjectResponse.class); + when(stat2.size()).thenReturn(100L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".part.00001")) { + return stat1; + } else if (args.object().endsWith(".part.00002")) { + return stat2; + } + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + when(minioClient.composeObject(any(ComposeObjectArgs.class))) + .thenThrow(new RuntimeException("The specified header is not valid in this context")); + + UploadInfo result = + storageService.append(info, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + assertNotNull(result); + + // Verify composeObject was attempted and failed + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .composeObject(any(ComposeObjectArgs.class)); + // Verify fallback to streaming putObject was executed + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.atLeastOnce()) + .putObject(any(PutObjectArgs.class)); + // Verify s3ComposeObjectSupported is now disabled + assertFalse(storageService.isS3ComposeObjectSupported()); + + // Subsequent upload should directly use streaming without calling composeObject + UploadInfo info2 = new UploadInfo(); + UploadId id2 = new UploadId("compose-skip-test-456"); + info2.setId(id2); + info2.setLength(fiveMb + 100L); + info2.setOffset(0L); + + String json2 = UploadInfoJsonSerializer.serialize(info2); + Item item21 = mock(Item.class); + when(item21.objectName()).thenReturn("uploads/compose-skip-test-456.part.00001"); + when(item21.size()).thenReturn(fiveMb); + + Item item22 = mock(Item.class); + when(item22.objectName()).thenReturn("uploads/compose-skip-test-456.part.00002"); + when(item22.size()).thenReturn(100L); + + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenReturn(Arrays.asList(new Result<>(item21), new Result<>(item22))); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json2.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + UploadInfo result2 = + storageService.append(info2, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + assertNotNull(result2); + + // composeObject count should still be 1 (never called for the second upload) + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .composeObject(any(ComposeObjectArgs.class)); + + // Reset flag for other tests + storageService.setS3ComposeObjectSupported(true); + assertTrue(storageService.isS3ComposeObjectSupported()); + } + @Test public void testPrepareStreamRollsBackSub5MbNumberedPart() throws Exception { UploadInfo info = new UploadInfo(); diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java index 53a0de9f..86a430b5 100644 --- a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -573,4 +573,18 @@ public void testSubmitChunkBackpressureInterruptedThrowsIOException() throws Exc "Interrupted backpressure wait should throw IOException", gotBackpressureInterruptedException.get()); } + + @Test + public void testDrainAndCompleteDefaultTimeout() throws Exception { + File chunk = new File(tempDir, "chunk-default-drain.tmp"); + Files.write(chunk.toPath(), "test-data".getBytes()); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk(chunk, 9, "part-default", () -> {}); + int confirmed = uploader.drainAndComplete(); + assertEquals("One chunk confirmed uploaded using default timeout", 1, confirmed); + } + + assertFalse("Chunk file should be cleaned up", chunk.exists()); + } } From 762f39c1ba22835d5088eaf10aa6ca9f1f45b8a8 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 3 Oct 2026 17:19:25 +0200 Subject: [PATCH 06/15] feat: unify lock wait timeout and cloud chunk drain timeout - Add withLockWaitTimeout(Duration) and getLockWaitTimeout() to TusFileUploadService (default 60s) - Derive max lock retries (timeout / 200ms, default 300) and cloud upload chunk drain timeout (timeout - 5s, default 55s) - Update AsyncChunkUploader, UploadStorageService, S3StorageService, AzureBlobStorageService, DiskStorageService, and ThreadLocalCachedStorageAndLockingService to configure and propagate drain timeout - Update documentation in README.md, CHANGELOG.md, docs/LOCKING.md, docs/S3_STORAGE.md, and docs/AZURE_BLOB_STORAGE.md with lock wait terminology - Add unit tests for timeout derivation, validation, storage synchronization, and cache delegation --- CHANGELOG.md | 5 +- README.md | 2 +- docs/AZURE_BLOB_STORAGE.md | 7 +- docs/LOCKING.md | 12 ++-- docs/S3_STORAGE.md | 8 +++ .../tus/server/TusFileUploadService.java | 67 +++++++++++++++---- .../server/upload/UploadStorageService.java | 20 ++++++ .../upload/azure/AzureBlobStorageService.java | 19 +++++- ...adLocalCachedStorageAndLockingService.java | 11 +++ .../upload/disk/DiskStorageService.java | 14 ++++ .../server/upload/s3/S3LockingService.java | 2 +- .../server/upload/s3/S3StorageService.java | 20 +++++- .../upload/util/AsyncChunkUploader.java | 31 +++++++-- .../tus/server/TusFileUploadServiceTest.java | 56 +++++++++++++--- .../upload/UploadStorageServiceTest.java | 4 ++ .../azure/AzureBlobStorageServiceTest.java | 22 ++++++ ...calCachedStorageAndLockingServiceTest.java | 7 ++ .../upload/disk/DiskStorageServiceTest.java | 12 ++++ .../upload/s3/S3StorageServiceTest.java | 22 ++++++ .../upload/util/AsyncChunkUploaderTest.java | 15 +++++ 20 files changed, 313 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c906a2f2..108ff851 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,12 @@ All notable changes to this project will be documented in this file. - **Dedicated Compliance Test Suites**: Added comprehensive, spec-quoted end-to-end tests using a dedicated Python script `scripts/rufh_conformity_test.py` with documentation on how to run the tests in `docs/CONFORMITY_TESTING.md`. - **User Migration & Interim Responses Documentation**: Added `docs/MIGRATION.md` and `docs/INTERIM_RESPONSES.md` detailing migration strategies, HTTP 104 status frames under IETF RUFH, Tomcat/Servlet container limitations, cached reflection optimizations, and Spring Boot Tomcat Valve integration. - **Server-Side Upload Completion Listeners (`UploadCompletionListener`)**: Added a functional interface callback mechanism allowing developers to register post-upload listeners via `withUploadCompletionListener(UploadCompletionListener)` or `addUploadCompletionListener(UploadCompletionListener)`. Listeners receive the completed `UploadInfo` and `TusFileUploadService` instance after lock release, allowing immediate byte streaming and deletion without contention. Added helper overloads `TusFileUploadService.getUploadedBytes(UploadInfo)` and `TusFileUploadService.deleteUpload(UploadInfo)`. +- **Unified Lock Wait & Cloud Drain Timeout**: Added `TusFileUploadService.withLockWaitTimeout(Duration)` (default 60 seconds) to configure the maximum wait duration for requests contending for an active upload lock. Automatically derives the background cloud upload chunk drain timeout (`lockWaitTimeout - 5 seconds`, default 55s) and the retry polling budget (`lockWaitTimeout / 200ms`, default 300 retries). - **JSON Serialization**: Support storing `UploadInfo` objects as JSON files in the storage backend using `TusFileUploadService.withJsonSerialization(true)`. ### Changed - **Default Disk-Based Locking**: `TusFileUploadService.withStoragePath(String)` now defaults to `LeaseFileLockingService` instead of `DiskLockingService` for out-of-the-box Kubernetes, container, and shared network storage compatibility. See `docs/DISK_BASED_LOCKING.md` for legacy opt-out instructions. -- **Calibrated Retry Budget**: Extended `TusFileUploadService` lock acquisition retry budget to 8.0 seconds (40 retries x 200ms) to ensure reliable contention resolution over network storage. +- **Unified Lock Wait Budget & Cloud Chunk Drain Calibration**: Replaced unreleased `withMaxLockRetries` with `withLockWaitTimeout(Duration)` (default 60s / 300 retries), synchronizing the lock wait timeout budget across storage backends and automatically deriving the cloud chunk drain timeout (`lockWaitTimeout - 5s`, default 55s) to guarantee in-flight chunks are cleanly flushed to cloud storage before releasing upload locks. - **Absolute Base URL & Location Header Support**: Extended `withUploadUri(String)` to accept absolute base URLs (e.g. `https://upload.example.com/files`), returning full URLs in `Location` response headers for upload creation across both Tus 1.0.0 and RUFH protocols while preserving backward compatibility for relative paths. - **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. - **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. @@ -32,7 +33,7 @@ All notable changes to this project will be documented in this file. - **Clear Content-Length on Error Responses**: Cleared `Content-Length` response header prior to invoking `HttpServletResponse.sendError(...)` during exception handling, resolving buffer conflicts and exceptions in Undertow and other servlet containers ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). - **Prevent Disk Truncate Underflow**: Guarded file truncate logic in `DiskStorageService` against underflow when removing bytes (`Math.max(0L, file.size() - byteCount)`). - **File Channel Leak Prevention in FileBasedLock**: Guaranteed `FileChannel` is closed immediately upon lock acquisition errors to avoid file descriptor leaks. -- **Cloud Upload Pause & Drain Timeout Resilience**: Extended `AsyncChunkUploader` default drain timeout from 4 seconds to 60 seconds and wrapped chunk draining in error-resilient recovery logic across `AzureBlobStorageService` and `S3StorageService`. Confirmed uploaded chunks and staged Azure blocks are committed and metadata (`UploadInfo` offset) is persisted to storage before throwing stream or drain exceptions, ensuring paused uploads (such as Uppy client pause/resume) preserve their progress and resume from the exact byte offset instead of restarting from 0. +- **Cloud Upload Pause & Drain Timeout Resilience**: Integrated `AsyncChunkUploader` default drain timeout (55 seconds, calibrated to `lockWaitTimeout - 5s`) with error-resilient recovery logic across `AzureBlobStorageService` and `S3StorageService`. Confirmed uploaded chunks and staged Azure blocks are committed and metadata (`UploadInfo` offset) is persisted to storage before throwing stream or drain exceptions, ensuring paused uploads (such as Uppy client pause/resume) preserve their progress and resume from the exact byte offset instead of restarting from 0. - **Azure Blob Storage Upload Cancellation Lease Safety**: Checked lock blob lease state before calling `deleteIfExists()` in `AzureBlobStorageService.terminateUpload()`. When an active lease is held on the lock blob by an ongoing request (e.g. `DELETE` cancellation), attempting deletion without specifying the lease ID triggered an Azure SDK error log (`HTTP 412 LeaseIdMissing`). Skipping deletion for actively leased blobs prevents this error and allows normal lease release upon request completion. - **Lock Contention Stream Interruption Handling**: Handled `IOException` from `InterruptibleInputStream` across upload request handlers (`CorePatchRequestHandler`, `RufhAppendPatchRequestHandler`, `RufhCreationPostRequestHandler`, `CreationWithUploadPostRequestHandler`). When an upload stream is interrupted by the locking service watchdog during lock contention (such as concurrent `HEAD` progress checks or `DELETE` cancellation requests), the storage backend commits all buffered bytes received so far and updates the offset in storage. Request handlers now reload the refreshed `UploadInfo` and return a clean HTTP 204/201 response with the updated offset rather than bubbling an unhandled `IOException` / HTTP 500 to the servlet container. - **S3 Server-Side Part Composition & Native Multipart Copy**: Implemented `S3ServerSideComposeHelper` to resolve AWS S3 header rejection during server-side part composition. MinIO Java SDK 9.0.3's `composeObject` delegates to `UploadPartCopy` with an empty body placeholder that automatically injects `Content-MD5` and `Content-Type` headers, which Amazon AWS S3 strictly forbids on part copy requests and rejects with HTTP 400 (`The specified header is not valid in this context`). `S3ServerSideComposeHelper` executes native S3 multipart copy requests directly with SigV4 signing omitting `Content-MD5`, allowing fast zero-bandwidth server-side composition on both AWS S3 and MinIO/Ceph clusters without external AWS SDK dependencies. Added reflection-free constructors to `S3StorageService` and `S3LockingService` accepting connection parameters directly, with `eu-central-1` as the default fallback region, while preserving multi-tier streaming concatenation fallbacks. diff --git a/README.md b/README.md index 2fe648d2..b240e1fd 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ After creating the object, you can configure it using the following methods: | `withStoragePath(String)` | `${java.io.tmpdir}/tus` | Path on the filesystem or shared drive where uploaded bytes and metadata are stored when using `DiskStorageService`. | | `withSupportedProtocolVersions(ProtocolVersion)` | `ProtocolVersion.AUTO` | Configures protocol handling: `AUTO` (header-based auto-detection), `TUS_1_0_0` (Tus 1.0.0 only), or `RUFH` (IETF draft-12 only). | | `withMaxUploadSize(Long)` | `Long.MAX_VALUE` | Maximum allowed total upload size in bytes per upload resource. | -| `withMaxLockRetries(int)` | `40` | Maximum lock acquisition retries during lock contention resolution (200ms sleep, resulting in an 8.0s timeout budget). | +| `withLockWaitTimeout(Duration)` | `Duration.ofSeconds(60)` | Maximum duration a request waits to acquire an upload lock held by an in-flight transfer (retrying every 200ms, resulting in 300 retries). Automatically configures cloud background chunk drain timeout to 5s less than this value (default 55s). | | `withChunkedTransferDecoding(Boolean)` | `false` | Enables manual chunked HTTP decoding for servlet containers that do not decode chunked requests natively. | | `withThreadLocalCache(Boolean)` | `false` | Enables in-memory thread-local caching of upload request data to reduce storage backend I/O load. | | `withUploadExpirationPeriod(Long)` | `null` (disabled) | Expiration period in milliseconds after which incomplete/expired uploads become eligible for cleanup. | diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index 7a1fb930..f3ba0ec2 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -182,11 +182,16 @@ Block sizes auto-calibrate based on total upload size: ### Lease Renewal Rationale `AzureBlobLockingService` uses native Azure Blob Leases (30-second duration) for distributed locking. Because large file uploads can stream over several minutes or hours, `AzureBlobUploadLock` runs a background daemon thread that renews the lease every 10 seconds. If an application server crashes unexpectedly, the lease auto-expires after 30 seconds without requiring manual lock cleanup sweeps. -### Lock Contention Resolution +### Lock Wait & Contention Resolution Lock contention resolution operates on two levels: 1. **JVM-local**: Active `InterruptibleInputStream` instances are registered in a concurrent map and interrupted directly if a concurrent lock request arrives in the same JVM. 2. **Cross-replica**: A `.stop` signal blob (`locks/.stop`) is written to Azure Storage. A background watchdog thread polls for `.stop` blobs and interrupts active streams on other cluster nodes. +When a client resumes an interrupted transfer by sending a `HEAD` request (or cancels it with `DELETE`), the request enters a **lock wait** retry loop in `TusFileUploadService`: +- **Lock Wait Timeout (`withLockWaitTimeout(Duration)`)**: Governs how long the incoming request waits (polling every 200ms; default 60 seconds / 300 retries). +- **Background Chunk Draining (`drainTimeout`)**: The interrupted upload is automatically allotted up to `lockWaitTimeout - 5 seconds` (default 55 seconds) to drain and stage in-flight blocks to Azure Blob Storage via `AsyncChunkUploader` before committing the block list and persisting the updated offset. +- **Lock Handover**: Because the drain timeout leaves a 5-second buffer before the lock wait budget expires, the active upload cleanly finishes staging blocks and releases the Azure Blob Lease within the caller's wait window, enabling seamless resumption without `UploadAlreadyLockedException`. + --- ## 8. Troubleshooting Guide diff --git a/docs/LOCKING.md b/docs/LOCKING.md index 6416525e..6d73d8ab 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -1,6 +1,6 @@ -# Upload Locking & Lock Contention Resolution +# Upload Locking, Lock Wait & Contention Resolution -This document describes why locking is necessary in the `tus-java-server` library, how the core `UploadLockingService` interface is structured, how lock contention resolution works across replicas, and where to find detailed documentation for each concrete locking mechanism implementation. +This document describes why locking is necessary in the `tus-java-server` library, how the core `UploadLockingService` interface is structured, how lock wait and contention resolution works across replicas, and where to find detailed documentation for each concrete locking mechanism implementation. --- @@ -8,14 +8,14 @@ This document describes why locking is necessary in the `tus-java-server` librar In the `tus` protocol (and IETF Resumable Uploads for HTTP specification), client uploads can be interrupted and resumed across multiple HTTP requests. Multiple concurrent requests targeting the same upload resource must be strictly prevented to avoid data corruption (such as out-of-order byte writes or overlapping file offsets). -### Stalled Uploads & Lock Contention Handling +### Stalled Uploads, Lock Wait & Contention Handling 1. **Active Streaming**: When a client sends upload bytes via a `PATCH` (or RUFH `POST`/`PATCH`) request, the server acquires an exclusive lock on that upload. 2. **Network Interruption**: If the client's network drops, the original `PATCH` connection may remain open on the server in a "half-open" state (a stalled socket read waiting for client bytes). 3. **Resume Attempt**: The client, recognizing the disconnect, attempts to resume by sending a `HEAD` request to query the current offset (or a `DELETE` request to terminate the upload). -4. **Lock Conflict**: The stalled `PATCH` request is still running on the server and holding the lock, which would block the client's `HEAD` or `DELETE` request indefinitely if not resolved. - -To solve this, `tus-java-server` includes a **lock contention resolution mechanism** where an incoming `HEAD` or `DELETE` request signals the server to interrupt the stalled `PATCH` byte stream cleanly, releasing the lock for immediate resumption. +4. **Lock Conflict & Lock Wait**: The stalled `PATCH` request is still running on the server and holding the lock. Rather than failing immediately, the server enters a **lock wait** retry loop (retrying every 200ms up to `withLockWaitTimeout(Duration)`, default 60s / 300 retries). +5. **Contention Resolution**: The waiting request calls `lockingService.requestLockRelease(requestUri)`, which signals the active upload to interrupt its input stream. +6. **Drain & Handover**: For cloud backends (S3, Azure), the interrupted upload is allotted a background drain timeout (`lockWaitTimeout - 5 seconds`, default 55s) to safely flush in-flight chunks and persist updated byte offsets before releasing the lock. Once freed, the waiting request acquires the lock and immediately returns the accurate offset to the client. --- diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 4328c739..3e4b46f2 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -204,6 +204,14 @@ S3StorageService s3Storage = new S3StorageService( - If lock contention occurs across replicas, `S3LockingService` writes a `.stop` signal object in S3, signaling the active request on another pod to interrupt its input stream cleanly. - No external database or Redis cache is required for distributed locking. +### Lock Wait & Background Chunk Draining + +When a client resumes an interrupted transfer by sending a `HEAD` request (or cancels it with `DELETE`), the request enters a **lock wait** retry loop in `TusFileUploadService`: +- **Lock Wait Timeout (`withLockWaitTimeout(Duration)`)**: Governs how long the incoming request waits (polling every 200ms; default 60 seconds / 300 retries). +- **Contention Interruption**: The waiting pod writes a `.stop` object in S3. A background watchdog on the pod running the active `PATCH` detects this within 2 seconds and cleanly interrupts the stream. +- **Background Chunk Draining (`drainTimeout`)**: The active upload is automatically allotted up to `lockWaitTimeout - 5 seconds` (default 55 seconds) to drain and persist any in-flight 5MB+ chunks to S3 via `AsyncChunkUploader` before committing the new offset. +- **Lock Handover**: Because the drain timeout leaves a 5-second buffer before the lock wait budget expires, the active upload finishes persisting data and releases the lock well within the caller's wait window, enabling clean resumption without race conditions. + ### Why Lease Renewal is Required (`S3Lock` vs `FileBasedLock`) Understanding the architectural distinction between disk/file locking and S3 distributed locking is essential: diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 8f203f23..159d7a7a 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -7,6 +7,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -58,7 +59,10 @@ public class TusFileUploadService implements Closeable { private static final Logger log = LoggerFactory.getLogger(TusFileUploadService.class); - public static final int DEFAULT_MAX_LOCK_RETRIES = 40; + public static final Duration DEFAULT_LOCK_WAIT_TIMEOUT = Duration.ofSeconds(60); + public static final long LOCK_RETRY_INTERVAL_MS = 200L; + public static final int DEFAULT_MAX_LOCK_RETRIES = + (int) (DEFAULT_LOCK_WAIT_TIMEOUT.toMillis() / LOCK_RETRY_INTERVAL_MS); private UploadStorageService uploadStorageService; private UploadLockingService uploadLockingService; @@ -68,6 +72,7 @@ public class TusFileUploadService implements Closeable { private boolean isThreadLocalCacheEnabled = false; private boolean isChunkedTransferDecodingEnabled = false; private ProtocolVersion supportedProtocolVersion = ProtocolVersion.AUTO; + private Duration lockWaitTimeout = DEFAULT_LOCK_WAIT_TIMEOUT; private int maxLockRetries = DEFAULT_MAX_LOCK_RETRIES; private final List uploadCompletionListeners = new CopyOnWriteArrayList<>(); @@ -77,9 +82,18 @@ public TusFileUploadService() { String storagePath = FileUtils.getTempDirectoryPath() + File.separator + "tus"; this.uploadStorageService = new DiskStorageService(idFactory, storagePath); this.uploadLockingService = new LeaseFileLockingService(idFactory, storagePath); + this.uploadStorageService.setDrainTimeout(calculateDrainTimeout(DEFAULT_LOCK_WAIT_TIMEOUT)); initFeatures(); } + private static Duration calculateDrainTimeout(Duration timeout) { + return timeout.minus(Duration.ofSeconds(5)); + } + + private static int calculateMaxLockRetries(Duration timeout) { + return (int) (timeout.toMillis() / LOCK_RETRY_INTERVAL_MS); + } + protected void initFeatures() { // The order of the features is important addTusExtension(new CoreProtocol()); @@ -290,6 +304,7 @@ public TusFileUploadService withUploadStorageService(UploadStorageService upload this.uploadStorageService.isUploadDeduplicationEnabled()); uploadStorageService.setJsonSerializationEnabled( this.uploadStorageService.isJsonSerializationEnabled()); + uploadStorageService.setDrainTimeout(this.uploadStorageService.getDrainTimeout()); uploadStorageService.setIdFactory(this.idFactory); // Update the upload storage service this.uploadStorageService = uploadStorageService; @@ -345,21 +360,47 @@ public TusFileUploadService withUploadLockingService(UploadLockingService upload } /** - * Specify the maximum number of retries the service will attempt to acquire an upload lock before - * failing with an {@link UploadAlreadyLockedException} during lock contention resolution (e.g. - * for HEAD or DELETE requests). Default is {@value #DEFAULT_MAX_LOCK_RETRIES} retries. + * Specify the maximum duration a request will wait to acquire an upload lock held by another + * in-flight request before failing with an {@link + * me.desair.tus.server.exception.UploadAlreadyLockedException} during lock wait and contention + * resolution (e.g. for HEAD or DELETE requests). + * + *

Configuring this setting automatically derives: + * + *

    + *
  • The background cloud upload chunk drain timeout for cloud storage backends (S3, Azure), + * set to {@code lockWaitTimeout - 5 seconds}. + *
  • The maximum number of lock acquisition retries (polling every {@value + * #LOCK_RETRY_INTERVAL_MS}ms). + *
+ * + *

The timeout must be greater than 5 seconds to provide a positive drain window for in-flight + * chunks. Default is 60 seconds (yielding a 55-second drain timeout and 300 retries). * - * @param maxLockRetries The maximum number of lock acquisition retries (must be 0 or greater) + * @param lockWaitTimeout The maximum duration to wait for an upload lock (must be > 5 seconds) * @return The current service */ - public TusFileUploadService withMaxLockRetries(int maxLockRetries) { - Validate.isTrue(maxLockRetries >= 0, "The max lock retries must be 0 or greater"); - this.maxLockRetries = maxLockRetries; + public TusFileUploadService withLockWaitTimeout(Duration lockWaitTimeout) { + Validate.notNull(lockWaitTimeout, "The lock wait timeout cannot be null"); + Validate.isTrue( + lockWaitTimeout.toMillis() > 5000L, "The lock wait timeout must be greater than 5 seconds"); + this.lockWaitTimeout = lockWaitTimeout; + this.maxLockRetries = calculateMaxLockRetries(lockWaitTimeout); + this.uploadStorageService.setDrainTimeout(calculateDrainTimeout(lockWaitTimeout)); return this; } /** - * Get the maximum number of lock acquisition retries. + * Get the maximum duration a request will wait to acquire an upload lock. + * + * @return The current lock wait timeout + */ + public Duration getLockWaitTimeout() { + return lockWaitTimeout; + } + + /** + * Get the maximum number of lock acquisition retries, derived from {@link #getLockWaitTimeout()}. * * @return The maximum number of lock acquisition retries */ @@ -571,9 +612,9 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) throws TusException, IOException { UploadLock lock = null; int retries = 0; - // Retry budget calibrated by default to 40 retries x 200ms = 8.0 seconds to accommodate - // NFS/network storage attribute cache propagation (actimeo=3s), watchdog polling - // interval (1.5s), and socket stream interruption and cleanup overhead. + // Lock wait loop: retries every LOCK_RETRY_INTERVAL_MS ms up to the configured lockWaitTimeout + // budget (default 60s / 300 retries). When an in-flight upload is active, requestLockRelease + // signals it to interrupt and cleanly drain in-flight chunks before releasing the lock. while (retries < maxLockRetries) { try { lock = uploadLockingService.lockUploadByUri(requestUri); @@ -583,7 +624,7 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) uploadLockingService.requestLockRelease(requestUri); retries++; try { - Thread.sleep(200L); + Thread.sleep(LOCK_RETRY_INTERVAL_MS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Lock acquisition retry interrupted", ie); diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index bdce0298..5f168ba4 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; @@ -297,6 +298,25 @@ default int getCloudUploadThreadPoolSize() { return 10; } + /** + * Set the timeout for draining in-flight chunks during upload completion or interruption. + * + * @param drainTimeout Drain timeout duration + */ + default void setDrainTimeout(Duration drainTimeout) { + // Default no-op for non-cloud implementations + } + + /** + * Get the timeout for draining in-flight chunks during upload completion or interruption. + * Defaults to 55 seconds (derived from the 60-second default lock wait timeout minus 5 seconds). + * + * @return Drain timeout duration, defaults to 55 seconds + */ + default Duration getDrainTimeout() { + return Duration.ofSeconds(55); + } + /** * Closes any underlying storage resources. * diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index ec0f6872..2f62320e 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -22,6 +22,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -94,6 +95,7 @@ public class AzureBlobStorageService implements UploadStorageService { private long preferredBlockSize = DEFAULT_PREFERRED_BLOCK_SIZE; private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); private final ThreadPoolExecutor uploadExecutor; private Long maxUploadSize; @@ -241,7 +243,8 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) List plannedBlockIds = new ArrayList<>(); List plannedChunkSizes = new ArrayList<>(); - try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis())) { // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob while (true) { File chunkFile = null; @@ -304,7 +307,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } - // Drain remaining staged chunks with 60s timeout. + // Drain remaining staged chunks with configured timeout (defaults to 55s, lock wait - 5s). // Catch drainException so any chunks confirmed uploaded before timeout or error // are committed and the metadata offset is preserved without loss. try { @@ -759,6 +762,18 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + @Override public void close() throws IOException { uploadExecutor.shutdown(); diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index ee83f592..5658f808 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.ref.WeakReference; +import java.time.Duration; import java.util.Objects; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; @@ -243,6 +244,16 @@ public int getCloudUploadThreadPoolSize() { return storageServiceDelegate.getCloudUploadThreadPoolSize(); } + @Override + public void setDrainTimeout(Duration drainTimeout) { + storageServiceDelegate.setDrainTimeout(drainTimeout); + } + + @Override + public Duration getDrainTimeout() { + return storageServiceDelegate.getDrainTimeout(); + } + @Override public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { UploadLock uploadLock = lockingServiceDelegate.lockUploadByUri(requestUri); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 5a428699..ca5c853e 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -15,6 +15,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -52,6 +53,7 @@ public class DiskStorageService extends AbstractDiskBasedService implements Uplo private Long minAppendSize = null; private Long minSize = null; private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); private Long uploadExpirationPeriod = null; private UploadIdFactory idFactory; private UploadConcatenationService uploadConcatenationService; @@ -130,6 +132,18 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + @Override public void setUploadDeduplicationEnabled(boolean enabled) { this.isUploadDeduplicationEnabled = enabled; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index e9bd22f2..84245e97 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -297,7 +297,7 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { } catch (ErrorResponseException e) { S3ErrorType errorType = S3Utils.parseErrorResponse(e); if (errorType == S3ErrorType.PRECONDITION_FAILED || errorType == S3ErrorType.CONFLICT) { - log.info("Lock contention for key {}: S3 conditional write precondition failed", lockKey); + log.debug("Lock contention for key {}: S3 conditional write precondition failed", lockKey); return null; } log.warn("Unexpected S3 error response acquiring lock for key {}", lockKey, e); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index dc89b998..01766dcb 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -22,6 +22,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Enumeration; @@ -108,6 +109,7 @@ public class S3StorageService implements UploadStorageService { private long preferredPartSize = DEFAULT_PREFERRED_PART_SIZE; private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); private final ThreadPoolExecutor uploadExecutor; private Long maxUploadSize; @@ -872,6 +874,18 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + /** * Set the preferred chunk part size in bytes used when buffering and uploading parts to S3. * @@ -1042,7 +1056,8 @@ private AppendResult processPayloadChunks( List plannedPartKeys = new ArrayList<>(); - try (AsyncChunkUploader uploader = new AsyncChunkUploader(uploadExecutor)) { + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis())) { while (!streamFinished) { File tempChunkFile = Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); @@ -1165,7 +1180,8 @@ private AppendResult processPayloadChunks( } } - // Drain any remaining in-flight chunks in the pipeline with 60s timeout. + // Drain any remaining in-flight chunks in the pipeline with configured drain timeout + // (defaults to 55s, lock wait - 5s). // Catch/finally ensures all confirmed parts are retained even if a subsequent chunk times // out. int confirmedCount = 0; diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java index c091bc1f..b143fded 100644 --- a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -35,7 +35,7 @@ public class AsyncChunkUploader implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(AsyncChunkUploader.class); - public static final long DEFAULT_DRAIN_TIMEOUT_MS = 300_000L; + public static final long DEFAULT_DRAIN_TIMEOUT_MS = 55_000L; @FunctionalInterface public interface ChunkUploadAction { @@ -48,6 +48,7 @@ public interface ChunkUploadAction { } private final ExecutorService executor; + private final long drainTimeoutMs; // Slot 3: In-flight upload task and its associated local file and key private Future inFlightUpload; @@ -67,12 +68,23 @@ public interface ChunkUploadAction { private boolean completed; /** - * Constructs an uploader using the given shared executor. + * Constructs an uploader using the given shared executor and default 55-second drain timeout. * * @param executor Shared thread pool executor for background chunk uploads */ public AsyncChunkUploader(ExecutorService executor) { + this(executor, DEFAULT_DRAIN_TIMEOUT_MS); + } + + /** + * Constructs an uploader using the given shared executor and explicit drain timeout. + * + * @param executor Shared thread pool executor for background chunk uploads + * @param drainTimeoutMs Maximum duration in milliseconds to drain in-flight chunks + */ + public AsyncChunkUploader(ExecutorService executor, long drainTimeoutMs) { this.executor = Objects.requireNonNull(executor, "ExecutorService must not be null"); + this.drainTimeoutMs = drainTimeoutMs; } /** @@ -144,14 +156,23 @@ public void submitChunk( } /** - * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) using the default timeout - * ({@link #DEFAULT_DRAIN_TIMEOUT_MS}, 60 seconds). + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) using the configured drain + * timeout (defaults to {@link #DEFAULT_DRAIN_TIMEOUT_MS}, 55 seconds). * * @return The total number of confirmed successfully uploaded chunks * @throws IOException If any chunk upload fails or times out */ public int drainAndComplete() throws IOException { - return drainAndComplete(DEFAULT_DRAIN_TIMEOUT_MS); + return drainAndComplete(drainTimeoutMs); + } + + /** + * Returns the configured drain timeout in milliseconds. + * + * @return Drain timeout in milliseconds + */ + public long getDrainTimeoutMs() { + return drainTimeoutMs; } /** diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 981ebace..33cf0a29 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -14,6 +14,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import me.desair.tus.server.exception.UploadAlreadyLockedException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -62,7 +63,7 @@ public void testAcquireUploadLockFallback() throws Exception { TusFileUploadService service = new TusFileUploadService() .withUploadLockingService(mockLockingService) - .withMaxLockRetries(5); + .withLockWaitTimeout(Duration.ofSeconds(6)); UploadLock lock = service.acquireUploadLock(HttpMethod.HEAD, "/files/test"); assertNotNull(lock); @@ -343,6 +344,7 @@ public void testWithUploadStorageServicePreservesConfiguration() { verify(newStorage).setMinSize(2048L); verify(newStorage).setCloudUploadThreadPoolSize(18); verify(newStorage).setUploadDeduplicationEnabled(true); + verify(newStorage).setDrainTimeout(Duration.ofSeconds(55)); } @Test @@ -357,6 +359,7 @@ public void testThreadLocalCacheDelegatesAppendAndMinSizes() { assertThat(service.getUploadStorageService().getMaxAppendSize(), is(1024L)); assertThat(service.getUploadStorageService().getMinAppendSize(), is(512L)); assertThat(service.getUploadStorageService().getMinSize(), is(2048L)); + assertThat(service.getUploadStorageService().getDrainTimeout(), is(Duration.ofSeconds(55))); assertThat(service.getUploadStorageService().getCloudUploadThreadPoolSize(), is(16)); } @@ -629,20 +632,53 @@ public void testDeleteUploadSingleArg() throws Exception { } @Test - public void testWithMaxLockRetries() { + public void testWithLockWaitTimeout() { TusFileUploadService service = new TusFileUploadService(); - assertEquals(40, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(60), service.getLockWaitTimeout()); + assertEquals(300, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(55), service.getUploadStorageService().getDrainTimeout()); - service.withMaxLockRetries(5); - assertEquals(5, service.getMaxLockRetries()); + service.withLockWaitTimeout(Duration.ofSeconds(30)); + assertEquals(Duration.ofSeconds(30), service.getLockWaitTimeout()); + assertEquals(150, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(25), service.getUploadStorageService().getDrainTimeout()); - service.withMaxLockRetries(0); - assertEquals(0, service.getMaxLockRetries()); + service.withLockWaitTimeout(Duration.ofMillis(6000)); + assertEquals(Duration.ofMillis(6000), service.getLockWaitTimeout()); + assertEquals(30, service.getMaxLockRetries()); + assertEquals(Duration.ofMillis(1000), service.getUploadStorageService().getDrainTimeout()); + } + + @Test(expected = NullPointerException.class) + public void testWithLockWaitTimeoutNullThrows() { + new TusFileUploadService().withLockWaitTimeout(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithLockWaitTimeoutFiveSecondsOrLessThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(5)); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithLockWaitTimeoutZeroThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ZERO); } @Test(expected = IllegalArgumentException.class) - public void testWithMaxLockRetriesNegativeThrows() { - new TusFileUploadService().withMaxLockRetries(-1); + public void testWithLockWaitTimeoutNegativeThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(-1)); + } + + @Test + public void testWithUploadStorageServiceCopiesDrainTimeout() { + TusFileUploadService service = + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(20)); + + UploadStorageService newStorageService = mock(UploadStorageService.class); + when(newStorageService.isUploadDeduplicationEnabled()).thenReturn(false); + + service.withUploadStorageService(newStorageService); + verify(newStorageService).setDrainTimeout(Duration.ofSeconds(15)); } @Test @@ -659,7 +695,7 @@ public void testAcquireUploadLockWithConfiguredRetries() throws Exception { TusFileUploadService service = new TusFileUploadService() .withUploadLockingService(mockLockingService) - .withMaxLockRetries(2); + .withLockWaitTimeout(Duration.ofMillis(5200)); UploadLock lock = service.acquireUploadLock(HttpMethod.HEAD, "/files/test"); assertNotNull(lock); diff --git a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java index 1933b822..151db73e 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; @@ -145,6 +146,9 @@ public void testDefaultAppendAndSizeConfigurationMethods() throws Exception { dummyStorageService.setCloudUploadThreadPoolSize(15); assertThat(dummyStorageService.getCloudUploadThreadPoolSize(), is(10)); + dummyStorageService.setDrainTimeout(Duration.ofSeconds(20)); + assertThat(dummyStorageService.getDrainTimeout(), is(Duration.ofSeconds(55))); + // Default close is a no-op dummyStorageService.close(); } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index 9f4c052e..8c2f33da 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -11,6 +11,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.upload.TimeBasedUploadIdFactory; @@ -261,6 +262,27 @@ public void testCloudUploadThreadPoolSizeConfiguration() { } } + @Test + public void testDrainTimeoutConfiguration() { + assertEquals( + "Default drain timeout is 55 seconds", + Duration.ofSeconds(55), + storageService.getDrainTimeout()); + + storageService.setDrainTimeout(Duration.ofSeconds(25)); + assertEquals( + "Updated drain timeout is 25 seconds", + Duration.ofSeconds(25), + storageService.getDrainTimeout()); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertEquals( + "Null drain timeout preserves previous value", + Duration.ofSeconds(25), + storageService.getDrainTimeout()); + } + @Test public void testCloseGracefulShutdown() throws Exception { // Verify closing storage service cleanly terminates background upload executor without error diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index d27e22cb..be72ffec 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import java.util.UUID; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.upload.UploadId; @@ -185,6 +186,12 @@ public void testDelegateMethods() throws Exception { service.getCloudUploadThreadPoolSize(); verify(mockStorage, times(1)).getCloudUploadThreadPoolSize(); + service.setDrainTimeout(Duration.ofSeconds(25)); + verify(mockStorage, times(1)).setDrainTimeout(Duration.ofSeconds(25)); + + service.getDrainTimeout(); + verify(mockStorage, times(1)).getDrainTimeout(); + when(mockStorage.create(info, "owner")).thenReturn(info); assertEquals(info, service.create(info, "owner")); verify(mockStorage, times(1)).create(info, "owner"); diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index 486be745..d24195ed 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -1123,4 +1124,15 @@ public void testSetCloudUploadThreadPoolSizeZeroThrowsIllegalArgumentException() public void testSetCloudUploadThreadPoolSizeNegativeThrowsIllegalArgumentException() { storageService.setCloudUploadThreadPoolSize(-1); } + + @Test + public void testDefaultDrainTimeoutOnDiskStorageService() { + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(55))); + storageService.setDrainTimeout(Duration.ofSeconds(20)); + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(20))); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(20))); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index d795775c..78008706 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -32,6 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import me.desair.tus.server.checksum.ChecksumAlgorithm; @@ -2062,6 +2063,27 @@ public void testCloudUploadThreadPoolSizeConfiguration() { } } + @Test + public void testDrainTimeoutConfiguration() { + assertEquals( + "Default drain timeout is 55 seconds", + Duration.ofSeconds(55), + storageService.getDrainTimeout()); + + storageService.setDrainTimeout(Duration.ofSeconds(20)); + assertEquals( + "Updated drain timeout is 20 seconds", + Duration.ofSeconds(20), + storageService.getDrainTimeout()); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertEquals( + "Null drain timeout preserves previous value", + Duration.ofSeconds(20), + storageService.getDrainTimeout()); + } + @Test public void testCloseGracefulShutdown() throws Exception { // Verify closing storage service cleanly terminates background upload executor without error diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java index 86a430b5..85cc42ce 100644 --- a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -587,4 +587,19 @@ public void testDrainAndCompleteDefaultTimeout() throws Exception { assertFalse("Chunk file should be cleaned up", chunk.exists()); } + + @Test + public void testCustomDrainTimeoutConstructorAndGetter() { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor, 12_345L)) { + assertEquals(12_345L, uploader.getDrainTimeoutMs()); + } + } + + @Test + public void testDefaultDrainTimeoutConstant() { + assertEquals(55_000L, AsyncChunkUploader.DEFAULT_DRAIN_TIMEOUT_MS); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + assertEquals(55_000L, uploader.getDrainTimeoutMs()); + } + } } From 7b843fdb48020327387d423e3f8d0363331f7df1 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 4 Oct 2026 18:50:03 +0200 Subject: [PATCH 07/15] feat(s3): decouple S3 storage, locking, and concatenation services from MinioClient --- CHANGELOG.md | 2 +- docs/S3_STORAGE.md | 84 +++++++---- .../upload/s3/S3ConcatenationService.java | 132 +++++++++++------- .../server/upload/s3/S3LockingService.java | 70 +++++----- .../upload/s3/S3ServerSideComposeHelper.java | 43 +++--- .../server/upload/s3/S3StorageService.java | 107 +++++--------- .../tus/server/upload/s3/S3UploadLock.java | 2 +- .../java/me/desair/tus/server/TestUtils.java | 12 +- .../server/upload/s3/ITS3LockingService.java | 3 +- .../server/upload/s3/ITS3RufhProtocol.java | 10 +- .../server/upload/s3/ITS3StorageService.java | 3 +- .../upload/s3/ITS3TusFileUploadService.java | 10 +- .../upload/s3/S3ConcatenationServiceTest.java | 81 ++++++++--- .../upload/s3/S3LockingServiceTest.java | 47 +++++-- .../s3/S3ServerSideComposeHelperTest.java | 10 +- .../upload/s3/S3StorageServiceTest.java | 72 ++++++++-- 16 files changed, 428 insertions(+), 260 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 108ff851..fa9dee4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### New - **NFS- & SMB-Safe Lease Locking (`LeaseFileLockingService` & `LeaseFileMutex`)**: Added distributed, container-safe filesystem locking using atomic sibling mutex directories (`.mutex/`), in-place expired lock takeover, and TTL-based JSON lease files with background heartbeat renewal and ownership fencing. Operates reliably across multi-server replicas on NFS (v3/v4), AWS EFS, Azure Files, Windows SMB/CIFS, and local disks without requiring Redis, ZooKeeper, or OS-level `FileLock` daemons. Comprehensive guide and legacy opt-out instructions available in `docs/DISK_BASED_LOCKING.md`. -- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService` (MinIO SDK), distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, and complete documentation in `docs/S3_STORAGE.md`. +- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService`, distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, configured entirely via standard S3 connection parameters, and complete documentation in `docs/S3_STORAGE.md`. - **Azure Blob Storage & Distributed Leases**: Added native Azure Blob Storage support via `AzureBlobStorageService` (Block Blob staging with streaming appends, sub-threshold buffering, truncation, and deduplication), distributed locking via `AzureBlobLockingService` (Azure Blob Leases with auto-renewal, JVM interruption, cross-replica `.stop` signals, and clean shutdown), zero-copy server-side concatenation via `AzureBlobConcatenationService` (`stageBlockFromUrl`), and comprehensive documentation in `docs/AZURE_BLOB_STORAGE.md`. - **IETF Resumable Uploads for HTTP (RUFH) Protocol**: Implemented full support for the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`). - **Dual Protocol Auto-Detection**: Added transparent protocol routing in `TusFileUploadService` supporting both legacy `TUS_1_0_0` (`Tus-Resumable: 1.0.0`) and `RUFH` (`ProtocolVersion.RUFH`) clients concurrently on the same endpoint. diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 3e4b46f2..c87c67e5 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -29,38 +29,35 @@ Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) ### Step 2: Configure `TusFileUploadService` ```java -import io.minio.MinioClient; import me.desair.tus.server.TusFileUploadService; import me.desair.tus.server.upload.s3.S3StorageService; import me.desair.tus.server.upload.s3.S3LockingService; -// 1. Production Configuration: Load S3 parameters securely from environment variables +// Option A: Local S3-compatible backend (e.g. MinIO, RustFS, Ceph) +// The region parameter can be omitted (defaults to "local"); local backends ignore region in SigV4 validation. +S3StorageService localS3Storage = new S3StorageService( + "http://localhost:9000", "minioadmin", "minioadmin", "my-upload-bucket"); +S3LockingService localS3Locking = new S3LockingService( + "http://localhost:9000", "minioadmin", "minioadmin", "my-upload-bucket"); + +// Option B: AWS S3 with explicit region +// AWS strictly enforces region validation for bucket routing and SigV4 authentication. String endpoint = System.getenv().getOrDefault("S3_ENDPOINT", "https://s3.eu-central-1.amazonaws.com"); String bucketName = System.getenv().getOrDefault("S3_BUCKET_NAME", "my-upload-bucket"); String accessKey = System.getenv("AWS_ACCESS_KEY_ID"); String secretKey = System.getenv("AWS_SECRET_ACCESS_KEY"); -MinioClient minioClient = MinioClient.builder() - .endpoint(endpoint) - .credentials(accessKey, secretKey) - .build(); - -// 2. Instantiate S3 Storage and Distributed Locking services -// Option A: Direct connection parameters (recommended, builds internal client with server-side compose helper) -S3StorageService s3StorageService = new S3StorageService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); -S3LockingService s3LockingService = new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName); - -// Option B: Using a pre-configured MinIO Client -// S3StorageService s3StorageService = new S3StorageService(minioClient, endpoint, "eu-central-1", accessKey, secretKey, bucketName); -// S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); +S3StorageService awsS3Storage = new S3StorageService( + endpoint, "eu-central-1", accessKey, secretKey, bucketName); +S3LockingService awsS3Locking = new S3LockingService( + endpoint, "eu-central-1", accessKey, secretKey, bucketName); -// 3. Configure TusFileUploadService with S3 storage and locking +// Configure TusFileUploadService with S3 storage and locking // Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. -// Manual call to tusService.close() or s3LockingService.close() is optional for custom container lifecycles. TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files/upload") - .withUploadStorageService(s3StorageService) - .withUploadLockingService(s3LockingService); + .withUploadStorageService(awsS3Storage) + .withUploadLockingService(awsS3Locking); ``` --- @@ -144,19 +141,41 @@ try (InputStream stream = minioClient.getObject( --- -## 5. Configuring Custom S3 Endpoints (MinIO, RustFS, R2, Ceph, GCS) +## 5. Configuring Custom S3 Endpoints & Regions (AWS vs. Local S3 Backends) + +`S3StorageService`, `S3LockingService`, and `S3ConcatenationService` accept any S3 endpoint and handle regional authentication flexibly: + +- **Local S3-Compatible Backends (MinIO, RustFS, Ceph, SeaweedFS)**: + Self-hosted and local S3-compatible storage engines do not strictly validate AWS regions during Signature Version 4 (SigV4) authentication. You can omit the `region` parameter completely; it automatically defaults to `"local"`. +- **AWS S3 & Cloud Providers (AWS S3, Cloudflare R2, Google Cloud Storage)**: + AWS strictly validates the region parameter against bucket locations during SigV4 authentication and request routing. Always supply the target region (e.g., `"eu-central-1"`, `"us-east-1"`, or `"auto"` for Cloudflare R2). + +### Example 1: Local S3-Compatible Backend (Without Region) + +```java +// Omit region parameter; defaults to "local" +S3StorageService localS3Storage = + new S3StorageService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket"); + +S3LockingService localS3Locking = + new S3LockingService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket"); + +S3ConcatenationService localS3Concat = + new S3ConcatenationService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket", localS3Storage); +``` -`S3StorageService` accepts any pre-configured `MinioClient`. To connect to an S3-compatible backend (such as local MinIO, RustFS, or Cloudflare R2), override the endpoint when building the `MinioClient`: +### Example 2: AWS S3 (With Explicit Region) ```java -import io.minio.MinioClient; +// Provide explicit AWS region (e.g. "eu-central-1", "us-east-1") +S3StorageService awsS3Storage = + new S3StorageService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket"); -MinioClient minioClient = MinioClient.builder() - .endpoint("http://minio.local:9000") - .credentials("minioadmin", "minioadmin") - .build(); +S3LockingService awsS3Locking = + new S3LockingService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket"); -S3StorageService s3Storage = new S3StorageService(minioClient, "my-bucket"); +S3ConcatenationService awsS3Concat = + new S3ConcatenationService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket", awsS3Storage); ``` --- @@ -184,7 +203,10 @@ S3 requires every part chunk of a multipart upload to be at least 5 MB (except t Path customTempDir = Paths.get("/var/tmp/tus-buffer"); S3StorageService s3Storage = new S3StorageService( - minioClient, + endpoint, + "eu-central-1", + accessKey, + secretKey, "my-bucket", "uploads/", "metadata/", @@ -259,19 +281,19 @@ The jitter window can be customized via `.withJitter(minMs, maxMs)` on `S3Lockin // On backends with atomic conditional writes, jitter is not needed. // Passing (0L, 0L) completely bypasses Thread.sleep for fastest lock acquisition: S3LockingService s3LockingService = - new S3LockingService(minioClient, bucketName) + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) .withJitter(0L, 0L); // Option B: High-Latency or Distributed Backends (Ceph, multi-datacenter MinIO) // High-latency backends or geo-replicated clusters may need a wider window to settle writes: S3LockingService s3LockingService = - new S3LockingService(minioClient, bucketName) + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) .withJitter(50L, 200L); // Option C: Explicit Non-CAS Mode (e.g. Wasabi) // Pre-checks existing locks before writing unconditionally: S3LockingService s3LockingService = - new S3LockingService(minioClient, bucketName) + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) .withS3ConditionalWritesSupported(false); ``` diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index 5b128453..b6207758 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -45,77 +45,126 @@ public class S3ConcatenationService implements UploadConcatenationService { private final String objectPrefix; private final long minPartSize; private final Path temporaryDirectory; - private UploadStorageService uploadStorageService; + private final UploadStorageService uploadStorageService; private S3ServerSideComposeHelper s3ComposeHelper; /** - * Basic constructor using default object prefix ("uploads/") and Java temp directory. + * Convenience constructor for local S3-compatible backends taking backing {@link + * UploadStorageService}. Defaults the region to "local". * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + * @param uploadStorageService Underlying storage service */ - public S3ConcatenationService(MinioClient minioClient, String bucket) { - this(minioClient, bucket, "uploads/", null, null); + public S3ConcatenationService( + String endpoint, + String accessKey, + String secretKey, + String bucket, + UploadStorageService uploadStorageService) { + this(endpoint, "local", accessKey, secretKey, bucket, uploadStorageService); } /** - * Convenient constructor taking MinioClient, bucket, and UploadStorageService. + * Convenient constructor taking connection parameters and backing {@link UploadStorageService}. * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name * @param uploadStorageService Underlying storage service */ public S3ConcatenationService( - MinioClient minioClient, String bucket, UploadStorageService uploadStorageService) { - this(minioClient, bucket, "uploads/", uploadStorageService, null); + String endpoint, + String region, + String accessKey, + String secretKey, + String bucket, + UploadStorageService uploadStorageService) { + this( + endpoint, + region, + accessKey, + secretKey, + bucket, + "uploads/", + uploadStorageService, + null, + DEFAULT_MIN_PART_SIZE); } /** - * Constructs an S3ConcatenationService. + * Full constructor allowing custom object prefix, temporary directory, and minimum part size. + * + *

Delegates to the internal package-private constructor accepting {@link MinioClient}. * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL + * @param region S3 region name + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name * @param objectPrefix Key prefix for data objects * @param uploadStorageService Underlying storage service * @param temporaryDirectory Directory for temporary buffer files + * @param minPartSize Minimum part chunk size for server-side composition */ public S3ConcatenationService( - MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, String bucket, String objectPrefix, UploadStorageService uploadStorageService, - Path temporaryDirectory) { + Path temporaryDirectory, + long minPartSize) { this( - minioClient, + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, objectPrefix, uploadStorageService, temporaryDirectory, - DEFAULT_MIN_PART_SIZE); + minPartSize, + new S3ServerSideComposeHelper(endpoint, region, accessKey, secretKey)); } - /** Full constructor allowing custom minimum part size. */ - public S3ConcatenationService( + /** + * Internal package-private constructor accepting {@link MinioClient} where all parameter + * configuration is concentrated. + */ + S3ConcatenationService( MinioClient minioClient, String bucket, String objectPrefix, UploadStorageService uploadStorageService, Path temporaryDirectory, - long minPartSize) { + long minPartSize, + S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); this.objectPrefix = objectPrefix != null ? objectPrefix : ""; - this.uploadStorageService = uploadStorageService; + this.uploadStorageService = + Objects.requireNonNull(uploadStorageService, "UploadStorageService must not be null"); this.temporaryDirectory = temporaryDirectory != null ? temporaryDirectory : java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); this.minPartSize = minPartSize; - this.s3ComposeHelper = new S3ServerSideComposeHelper(this.minioClient); + this.s3ComposeHelper = + s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); } - public void setUploadStorageService(UploadStorageService uploadStorageService) { - this.uploadStorageService = uploadStorageService; + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); } @Override @@ -126,8 +175,7 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep return; } - Long expirationPeriod = - uploadStorageService != null ? uploadStorageService.getUploadExpirationPeriod() : null; + Long expirationPeriod = uploadStorageService.getUploadExpirationPeriod(); List partialUploads = getPartialUploads(uploadInfo); Long totalLength = calculateTotalLength(partialUploads); @@ -175,12 +223,10 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep } uploadInfo.setStorageUploadId(targetObjectKey); - if (uploadStorageService != null) { - try { - uploadStorageService.update(uploadInfo); - } catch (UploadNotFoundException e) { - log.warn("Failed to update concatenated upload info for " + uploadInfo.getId(), e); - } + try { + uploadStorageService.update(uploadInfo); + } catch (UploadNotFoundException e) { + log.warn("Failed to update concatenated upload info for " + uploadInfo.getId(), e); } } } @@ -192,11 +238,6 @@ public InputStream getConcatenatedBytes(UploadInfo uploadInfo) return null; } - if (uploadStorageService == null) { - throw new IOException( - "UploadStorageService must be configured to retrieve concatenated upload bytes"); - } - if (uploadInfo.isUploadInProgress()) { merge(uploadInfo); } @@ -219,10 +260,7 @@ public List getPartialUploads(UploadInfo info) List output = new ArrayList<>(concatenationParts.size()); for (String childUri : concatenationParts) { - UploadInfo childInfo = - uploadStorageService != null - ? uploadStorageService.getUploadInfo(childUri, info.getOwnerKey()) - : null; + UploadInfo childInfo = uploadStorageService.getUploadInfo(childUri, info.getOwnerKey()); if (childInfo == null) { throw new UploadNotFoundException( "Upload with URI " + childUri + " was not found for owner " + info.getOwnerKey()); @@ -308,12 +346,10 @@ private boolean checkAllCompleted(Long expirationPeriod, List partia completed = false; } else if (expirationPeriod != null) { childInfo.updateExpiration(expirationPeriod); - if (uploadStorageService != null) { - try { - uploadStorageService.update(childInfo); - } catch (UploadNotFoundException e) { - log.debug("Failed to update child upload expiration for " + childInfo.getId(), e); - } + try { + uploadStorageService.update(childInfo); + } catch (UploadNotFoundException e) { + log.debug("Failed to update child upload expiration for " + childInfo.getId(), e); } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 84245e97..1def9dc0 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -64,25 +64,21 @@ public class S3LockingService extends AbstractLeaseLockingService { private volatile boolean s3ConditionalWritesSupported = true; /** - * Basic constructor using default lock prefix ("locks/"), 30s lease duration, and 2s polling - * interval. + * Convenience constructor for local S3-compatible backends where region is omitted. Defaults the + * region to "local". * - * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket Target S3 bucket name */ - public S3LockingService(MinioClient minioClient, String bucket) { - this( - minioClient, - bucket, - DEFAULT_LOCKS_PREFIX, - DEFAULT_LEASE_DURATION_MS, - DEFAULT_POLL_INTERVAL_MS, - new UuidUploadIdFactory()); + public S3LockingService(String endpoint, String accessKey, String secretKey, String bucket) { + this(endpoint, "local", accessKey, secretKey, bucket); } /** - * Constructor accepting explicit connection parameters without requiring a pre-existing - * MinioClient. + * Basic constructor accepting explicit connection parameters without exposing underlying client + * libraries. * * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") * @param region S3 region name (e.g. "eu-central-1", "us-east-1") @@ -93,7 +89,10 @@ public S3LockingService(MinioClient minioClient, String bucket) { public S3LockingService( String endpoint, String region, String accessKey, String secretKey, String bucket) { this( - buildMinioClient(endpoint, region, accessKey, secretKey), + endpoint, + region, + accessKey, + secretKey, bucket, DEFAULT_LOCKS_PREFIX, DEFAULT_LEASE_DURATION_MS, @@ -102,45 +101,44 @@ public S3LockingService( } /** - * Constructor accepting a pre-configured {@link MinioClient} along with explicit connection - * parameters. + * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. + * + *

Delegates to the internal package-private constructor that accepts {@link MinioClient}. * - * @param minioClient Pre-configured MinIO Client * @param endpoint S3 endpoint URL * @param region S3 region name * @param accessKey S3 access key / username * @param secretKey S3 secret key / password * @param bucket Target S3 bucket name + * @param locksPrefix Object key prefix for locks and stop signals + * @param leaseDurationMs Lock lease duration in milliseconds + * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals + * @param idFactory Custom {@link UploadIdFactory} */ public S3LockingService( - MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey, - String bucket) { + String bucket, + String locksPrefix, + long leaseDurationMs, + long pollIntervalMs, + UploadIdFactory idFactory) { this( - minioClient != null - ? minioClient - : buildMinioClient(endpoint, region, accessKey, secretKey), + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, - DEFAULT_LOCKS_PREFIX, - DEFAULT_LEASE_DURATION_MS, - DEFAULT_POLL_INTERVAL_MS, - new UuidUploadIdFactory()); + locksPrefix, + leaseDurationMs, + pollIntervalMs, + idFactory); } /** - * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. - * - * @param minioClient Pre-configured MinIO Client - * @param bucket Target S3 bucket name - * @param locksPrefix Object key prefix for locks and stop signals - * @param leaseDurationMs Lock lease duration in milliseconds - * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals - * @param idFactory Custom {@link UploadIdFactory} + * Package-private constructor accepting {@link MinioClient} where all parameter configuration is + * concentrated. */ - public S3LockingService( + S3LockingService( MinioClient minioClient, String bucket, String locksPrefix, @@ -448,7 +446,7 @@ private String buildStopKey(UploadId uploadId) { private static MinioClient buildMinioClient( String endpoint, String region, String accessKey, String secretKey) { - String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; return MinioClient.builder() .endpoint(endpoint) .credentials(accessKey, secretKey) diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java index 8a645173..a805391d 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java @@ -63,38 +63,35 @@ public class S3ServerSideComposeHelper { private final String explicitRegion; /** - * Constructs an instance wrapping the given {@link MinioClient} without reflection. - * - *

If explicit connection parameters are not provided, this helper falls back to calling {@link - * MinioClient#composeObject(ComposeObjectArgs)} directly. + * Constructs an instance with explicit connection parameters without requiring a {@link + * MinioClient}. * - * @param minioClient The MinIO client instance + * @param endpoint The S3 endpoint URL (e.g. "https://s3.amazonaws.com" or + * "http://localhost:9000") + * @param region S3 region name (optional, defaults to "local" if null or empty) + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password */ - public S3ServerSideComposeHelper(MinioClient minioClient) { - this.minioClient = minioClient; - this.asyncClient = null; - this.baseUrl = null; - this.provider = null; - this.httpClient = null; - this.explicitRegion = null; + public S3ServerSideComposeHelper( + String endpoint, String region, String accessKey, String secretKey) { + this(null, endpoint, region, accessKey, secretKey); } /** - * Constructs an instance with an existing {@link MinioClient} and explicit connection parameters. + * Constructs an instance wrapping the given {@link MinioClient} without reflection. * - *

This allows native S3 multipart copy without using reflection on {@link MinioClient}. + *

Package-private constructor for internal testing. * - * @param minioClient The existing MinIO client instance - * @param endpoint The S3 endpoint URL (e.g. "https://s3.amazonaws.com" or - * "http://localhost:9000") - * @param region S3 region name (optional, defaults to "eu-central-1" if null or empty) - * @param accessKey S3 access key / username - * @param secretKey S3 secret key / password + * @param minioClient The MinIO client instance */ - public S3ServerSideComposeHelper( + S3ServerSideComposeHelper(MinioClient minioClient) { + this(minioClient, null, null, null, null); + } + + private S3ServerSideComposeHelper( MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey) { this.minioClient = minioClient; - this.explicitRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + this.explicitRegion = (region != null && !region.isEmpty()) ? region : "local"; Http.BaseUrl base = null; Provider prov = null; @@ -164,7 +161,7 @@ public void compose(String bucket, String targetKey, List partKeys) thro region = explicitRegion; } if (region == null || region.isEmpty()) { - region = "eu-central-1"; + region = "local"; } Credentials credentials = (provider != null) ? provider.fetch() : null; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 01766dcb..1f1513d1 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -125,28 +125,21 @@ public class S3StorageService implements UploadStorageService { private S3ServerSideComposeHelper s3ComposeHelper; /** - * Basic constructor using default object key prefixes and standard system temp directory. + * Convenience constructor for local S3-compatible backends (e.g., MinIO, RustFS, Ceph) where + * region is omitted. Defaults the region to "local". * - * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket S3 bucket name */ - public S3StorageService(MinioClient minioClient, String bucket) { - this( - minioClient, - bucket, - DEFAULT_OBJECT_PREFIX, - DEFAULT_METADATA_PREFIX, - DEFAULT_CHECKSUMS_PREFIX, - DEFAULT_LOCKS_PREFIX, - Paths.get(System.getProperty("java.io.tmpdir"))); + public S3StorageService(String endpoint, String accessKey, String secretKey, String bucket) { + this(endpoint, "local", accessKey, secretKey, bucket); } /** - * Constructor accepting explicit connection parameters without requiring a pre-existing - * MinioClient. - * - *

Builds both {@link MinioClient} and the native {@link S3ServerSideComposeHelper} directly - * using the provided connection properties, avoiding reflection. + * Basic constructor accepting explicit connection parameters without exposing underlying client + * libraries. * * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") * @param region S3 region name (e.g. "us-east-1", "eu-central-1") @@ -157,53 +150,28 @@ public S3StorageService(MinioClient minioClient, String bucket) { public S3StorageService( String endpoint, String region, String accessKey, String secretKey, String bucket) { this( - buildMinioClient(endpoint, region, accessKey, secretKey), + endpoint, + region, + accessKey, + secretKey, bucket, DEFAULT_OBJECT_PREFIX, DEFAULT_METADATA_PREFIX, DEFAULT_CHECKSUMS_PREFIX, DEFAULT_LOCKS_PREFIX, - Paths.get(System.getProperty("java.io.tmpdir")), - new S3ServerSideComposeHelper(null, endpoint, region, accessKey, secretKey)); + Paths.get(System.getProperty("java.io.tmpdir"))); } /** - * Constructor accepting a pre-configured {@link MinioClient} along with the explicit connection - * parameters required by {@link S3ServerSideComposeHelper}. + * Full constructor accepting explicit connection parameters and prefix/buffer customization. * - *

This provides full control over {@link MinioClient} configuration while cleanly initializing - * server-side multipart copy without reflection. + *

Delegates to the internal package-private constructor accepting {@link MinioClient}. * - * @param minioClient Pre-configured MinIO Client * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") * @param region S3 region name (e.g. "us-east-1", "eu-central-1") * @param accessKey S3 access key / username * @param secretKey S3 secret key / password * @param bucket S3 bucket name - */ - public S3StorageService( - MinioClient minioClient, - String endpoint, - String region, - String accessKey, - String secretKey, - String bucket) { - this( - minioClient, - bucket, - DEFAULT_OBJECT_PREFIX, - DEFAULT_METADATA_PREFIX, - DEFAULT_CHECKSUMS_PREFIX, - DEFAULT_LOCKS_PREFIX, - Paths.get(System.getProperty("java.io.tmpdir")), - new S3ServerSideComposeHelper(minioClient, endpoint, region, accessKey, secretKey)); - } - - /** - * Full constructor allowing full customization of object prefixes and local disk buffer path. - * - * @param minioClient Pre-configured MinIO Client - * @param bucket S3 bucket name * @param objectPrefix Key prefix for final completed file objects * @param metadataPrefix Key prefix for metadata (.info JSON and .part buffer) objects * @param checksumsPrefix Key prefix for checksum deduplication index objects @@ -211,7 +179,10 @@ public S3StorageService( * @param temporaryDirectory Local directory path for staging chunks before S3 upload */ public S3StorageService( - MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, String bucket, String objectPrefix, String metadataPrefix, @@ -219,18 +190,21 @@ public S3StorageService( String locksPrefix, Path temporaryDirectory) { this( - minioClient, + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, objectPrefix, metadataPrefix, checksumsPrefix, locksPrefix, temporaryDirectory, - new S3ServerSideComposeHelper(minioClient)); + new S3ServerSideComposeHelper(endpoint, region, accessKey, secretKey)); } - /** Internal constructor accepting an initialized {@link S3ServerSideComposeHelper}. */ - private S3StorageService( + /** + * Internal package-private constructor accepting {@link MinioClient} where all parameter + * configuration and initialization logic is concentrated. + */ + S3StorageService( MinioClient minioClient, String bucket, String objectPrefix, @@ -273,20 +247,22 @@ private S3StorageService( }, new ThreadPoolExecutor.CallerRunsPolicy()); - this.concatenationService = - new S3ConcatenationService( - this.minioClient, this.bucket, this.objectPrefix, this, this.temporaryDirectory); this.s3ComposeHelper = s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); - if (this.concatenationService instanceof S3ConcatenationService) { - ((S3ConcatenationService) this.concatenationService) - .setS3ServerSideComposeHelper(this.s3ComposeHelper); - } + this.concatenationService = + new S3ConcatenationService( + this.minioClient, + this.bucket, + this.objectPrefix, + this, + this.temporaryDirectory, + DEFAULT_MIN_PART_SIZE, + this.s3ComposeHelper); } private static MinioClient buildMinioClient( String endpoint, String region, String accessKey, String secretKey) { - String effectiveRegion = (region != null && !region.isEmpty()) ? region : "eu-central-1"; + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; return MinioClient.builder() .endpoint(endpoint) .credentials(accessKey, secretKey) @@ -294,15 +270,6 @@ private static MinioClient buildMinioClient( .build(); } - /** - * Returns the underlying {@link MinioClient} configured for this storage service. - * - * @return The MinIO client instance - */ - public MinioClient getMinioClient() { - return this.minioClient; - } - /** * Returns the S3 object key for the completed upload data of the given upload info. If the upload * was deduplicated, this returns the parent upload's physical S3 object key. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index 5b3a18b3..a0519af7 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -51,7 +51,7 @@ public class S3UploadLock extends AbstractLeaseLock { * @param stopKey The S3 object key for the interrupt stop signal * @param inputStreamMap Map of active request input streams */ - public S3UploadLock( + S3UploadLock( LeaseData leaseData, MinioClient minioClient, String bucket, diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index 55f1f6fe..a8358eea 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -77,6 +77,16 @@ public static GenericContainer createRustFsContainer() { .withEnv("RUSTFS_SECRET_KEY", "rustfsadmin"); } + /** + * Returns the S3 endpoint URL for the active S3 Testcontainer (RustFS). + * + * @param s3Container The active S3 Testcontainer (RustFS) + * @return S3 endpoint URL + */ + public static String getS3Endpoint(GenericContainer s3Container) { + return "http://" + s3Container.getHost() + ":" + s3Container.getMappedPort(9000); + } + /** * Create a {@link MinioClient} configured to connect to the given S3/RustFS container. * @@ -84,7 +94,7 @@ public static GenericContainer createRustFsContainer() { * @return Pre-configured MinioClient */ public static MinioClient createMinioClient(GenericContainer s3Container) { - String s3Url = "http://" + s3Container.getHost() + ":" + s3Container.getMappedPort(9000); + String s3Url = getS3Endpoint(s3Container); return MinioClient.builder().endpoint(s3Url).credentials("rustfsadmin", "rustfsadmin").build(); } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java index 916ff8e2..02724796 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java @@ -46,7 +46,8 @@ public static void tearDownClass() { @Before public void setUp() { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - lockingService = new S3LockingService(minioClient, BUCKET); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + lockingService = new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java index 60b21270..97872e3c 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java @@ -47,9 +47,13 @@ protected TusFileUploadService createTusFileUploadService() { protected TusFileUploadService createTusFileUploadService(String uploadUri) { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - S3StorageService s3Storage = new S3StorageService(minioClient, BUCKET); - S3LockingService s3Locking = new S3LockingService(minioClient, BUCKET); - S3ConcatenationService s3Concat = new S3ConcatenationService(minioClient, BUCKET, s3Storage); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + S3StorageService s3Storage = + new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3LockingService s3Locking = + new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3ConcatenationService s3Concat = + new S3ConcatenationService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET, s3Storage); s3Storage.setUploadConcatenationService(s3Concat); return new TusFileUploadService() diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java index c2f10768..b230dca9 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java @@ -50,7 +50,8 @@ public static void tearDownClass() { @Before public void setUp() { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - storageService = new S3StorageService(minioClient, BUCKET); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + storageService = new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java index c3a0a008..6d8da536 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java @@ -49,9 +49,13 @@ protected TusFileUploadService createTusFileUploadService() { protected TusFileUploadService createTusFileUploadService(String uploadUri) { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - S3StorageService s3Storage = new S3StorageService(minioClient, BUCKET); - S3LockingService s3Locking = new S3LockingService(minioClient, BUCKET); - S3ConcatenationService s3Concat = new S3ConcatenationService(minioClient, BUCKET, s3Storage); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + S3StorageService s3Storage = + new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3LockingService s3Locking = + new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3ConcatenationService s3Concat = + new S3ConcatenationService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET, s3Storage); s3Storage.setUploadConcatenationService(s3Concat); return new TusFileUploadService() diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index 7f3d3470..df8255c6 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -39,18 +39,63 @@ public void setUp() { "test-bucket", "uploads/", storageService, - Paths.get(System.getProperty("java.io.tmpdir"))); + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); } @Test - public void testConstructorsAndSetters() { - S3ConcatenationService service1 = new S3ConcatenationService(minioClient, "test-bucket"); - S3ConcatenationService service2 = - new S3ConcatenationService(minioClient, "test-bucket", storageService); - service1.setUploadStorageService(storageService); - - assertNotNull(service1); - assertNotNull(service2); + public void testConstructors() { + S3ConcatenationService service = + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); + assertNotNull(service); + + // Public connection parameter constructors without region (defaults to "local") + S3ConcatenationService serviceParamsNoRegionWithStorage = + new S3ConcatenationService( + "https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket", storageService); + assertNotNull(serviceParamsNoRegionWithStorage); + + // Public connection parameter constructors with region + S3ConcatenationService serviceParamsWithRegion = + new S3ConcatenationService( + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket", + storageService); + assertNotNull(serviceParamsWithRegion); + + S3ConcatenationService serviceParamsFull = + new S3ConcatenationService( + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L); + assertNotNull(serviceParamsFull); + + S3ConcatenationService defaultRegionService = + new S3ConcatenationService( + "https://s3.amazonaws.com", + null, + "accessKey", + "secretKey", + "test-bucket", + storageService); + assertNotNull(defaultRegionService); } @Test @@ -354,14 +399,16 @@ public void testGetConcatenatedBytesTriggersMergeWhenStorageUploadIdIsNull() thr assertNotNull(result); } - @Test(expected = IOException.class) - public void testGetConcatenatedBytesWithoutStorageService() throws Exception { - S3ConcatenationService standalone = new S3ConcatenationService(minioClient, "test-bucket"); - UploadInfo info = new UploadInfo(); - info.setId(new UploadId("concat-1")); - info.setStorageUploadId("uploads/concat-1"); - - standalone.getConcatenatedBytes(info); + @Test(expected = NullPointerException.class) + public void testConstructorNullStorageServiceThrowsException() { + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + null, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index cff254f3..f067aa38 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -45,7 +45,14 @@ public class S3LockingServiceTest { @Before public void setUp() throws Exception { minioClient = Mockito.mock(MinioClient.class); - lockingService = new S3LockingService(minioClient, "test-bucket"); + lockingService = + new S3LockingService( + minioClient, + "test-bucket", + S3LockingService.DEFAULT_LOCKS_PREFIX, + S3LockingService.DEFAULT_LEASE_DURATION_MS, + S3LockingService.DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); s3StorageMap.clear(); Mockito.when(minioClient.putObject(Mockito.any(PutObjectArgs.class))) @@ -139,20 +146,32 @@ public void testLockUploadByUriInvalidUri() throws Exception { @Test public void testExplicitConnectionParametersConstructors() { + S3LockingService serviceWithoutRegion = + new S3LockingService("https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithoutRegion); + S3LockingService serviceWithParams = new S3LockingService( "https://s3.amazonaws.com", "eu-central-1", "accessKey", "secretKey", "test-bucket"); assertNotNull(serviceWithParams); - S3LockingService serviceWithClientAndParams = + S3LockingService fullServiceWithParams = new S3LockingService( - minioClient, "https://s3.amazonaws.com", "eu-central-1", "accessKey", "secretKey", - "test-bucket"); - assertNotNull(serviceWithClientAndParams); + "test-bucket", + "locks/", + 30000L, + 2000L, + new me.desair.tus.server.upload.UuidUploadIdFactory()); + assertNotNull(fullServiceWithParams); + + S3LockingService defaultRegionService = + new S3LockingService( + "https://s3.amazonaws.com", null, "accessKey", "secretKey", "test-bucket"); + assertNotNull(defaultRegionService); } @Test @@ -368,6 +387,16 @@ public void testClose() throws Exception { lockingService.close(); } + private S3LockingService createLockingService(MinioClient client, String bucket) { + return new S3LockingService( + client, + bucket, + S3LockingService.DEFAULT_LOCKS_PREFIX, + S3LockingService.DEFAULT_LEASE_DURATION_MS, + S3LockingService.DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); + } + @Test public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { lockingService.setIdFactory(new me.desair.tus.server.upload.TimeBasedUploadIdFactory()); @@ -378,7 +407,7 @@ public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { .when(mockClient) .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); @@ -512,7 +541,7 @@ public void testCleanupStaleLocksWithExpiredAndNonExpiredLocks() throws Exceptio .thenReturn(expiredStream) .thenReturn(validStream); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); service.cleanupStaleLocks(); // Expired lock object is deleted @@ -531,7 +560,7 @@ public void testWriteStopSignalException() throws Exception { Mockito.when(mockClient.putObject(Mockito.any(PutObjectArgs.class))) .thenThrow(new RuntimeException("S3 Put error")); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); @@ -544,7 +573,7 @@ public void testWriteStopSignalException() throws Exception { @Test public void testCloseInterruptsActiveStreams() throws Exception { MinioClient mockClient = Mockito.mock(MinioClient.class); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); ByteArrayInputStream bis = new ByteArrayInputStream(new byte[] {1, 2, 3}); InterruptibleInputStream iis = new InterruptibleInputStream(bis); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java index c0afd7ae..3dff378f 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java @@ -259,10 +259,14 @@ public void testExplicitConnectionParametersConstructor() { .region("us-east-1") .build(); - S3ServerSideComposeHelper helper = + S3ServerSideComposeHelper helperWithoutClient = new S3ServerSideComposeHelper( - realClient, "https://s3.amazonaws.com", "us-east-1", "testKey", "testSecret"); - assertTrue(helper.isAvailable()); + "https://s3.amazonaws.com", "us-east-1", "testKey", "testSecret"); + assertTrue(helperWithoutClient.isAvailable()); + + S3ServerSideComposeHelper helperWithNullRegion = + new S3ServerSideComposeHelper("https://s3.amazonaws.com", null, "testKey", "testSecret"); + assertTrue(helperWithNullRegion.isAvailable()); S3ServerSideComposeHelper minioOnlyHelper = new S3ServerSideComposeHelper(realClient); assertFalse(minioOnlyHelper.isAvailable()); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 78008706..f0d5dc29 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -32,6 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Paths; import java.time.Duration; import java.util.Arrays; import java.util.Collections; @@ -53,7 +54,16 @@ public class S3StorageServiceTest { @Before public void setUp() { minioClient = mock(MinioClient.class); - storageService = new S3StorageService(minioClient, "test-bucket"); + storageService = + new S3StorageService( + minioClient, + "test-bucket", + S3StorageService.DEFAULT_OBJECT_PREFIX, + S3StorageService.DEFAULT_METADATA_PREFIX, + S3StorageService.DEFAULT_CHECKSUMS_PREFIX, + S3StorageService.DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir")), + null); } @Test @@ -76,28 +86,51 @@ public void testNullTemporaryDirectoryConstructor() { java.nio.file.Path nullPath = null; S3StorageService serviceWithNullTmp = new S3StorageService( - minioClient, "test-bucket", "uploads/", "uploads/", "checksums/", "locks/", nullPath); + minioClient, + "test-bucket", + "uploads/", + "uploads/", + "checksums/", + "locks/", + nullPath, + null); assertNotNull(serviceWithNullTmp); } @Test public void testExplicitConnectionParametersConstructors() { - // 1. Constructor taking connection parameters directly (Option A) + // 1. Constructor taking connection parameters directly without region (defaults to "local") + S3StorageService serviceWithoutRegion = + new S3StorageService("https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithoutRegion); + + // 2. Constructor taking connection parameters directly with region S3StorageService serviceWithParams = new S3StorageService( "https://s3.amazonaws.com", "us-east-1", "accessKey", "secretKey", "test-bucket"); assertNotNull(serviceWithParams); - // 2. Constructor taking MinioClient and connection parameters (Option B) - S3StorageService serviceWithClientAndParams = + // 3. Full constructor taking connection parameters with prefix and temporary directory + // configuration + S3StorageService fullServiceWithParams = new S3StorageService( - minioClient, "https://s3.amazonaws.com", "us-east-1", "accessKey", "secretKey", - "test-bucket"); - assertNotNull(serviceWithClientAndParams); + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + java.nio.file.Paths.get(System.getProperty("java.io.tmpdir"))); + assertNotNull(fullServiceWithParams); + + // 4. Null/empty region defaults cleanly + S3StorageService defaultRegionService = + new S3StorageService( + "https://s3.amazonaws.com", null, "accessKey", "secretKey", "test-bucket"); + assertNotNull(defaultRegionService); } @Test @@ -769,7 +802,15 @@ public void testConfigurationSettersAndGetters() { storageService.setIdFactory(new me.desair.tus.server.upload.UuidUploadIdFactory()); - S3ConcatenationService concat = new S3ConcatenationService(minioClient, "test-bucket"); + S3ConcatenationService concat = + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); storageService.setUploadConcatenationService(concat); assertEquals(concat, storageService.getUploadConcatenationService()); @@ -1591,11 +1632,11 @@ public void testDeleteObjectQuietlyNullAndException() throws Exception { public void testSanitizePrefixNullOrEmptyInS3StorageService() throws Exception { java.nio.file.Path tmpDir = java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); - S3StorageService s1 = new S3StorageService(minioClient, "bucket", "", "", "", "", tmpDir); + S3StorageService s1 = new S3StorageService(minioClient, "bucket", "", "", "", "", tmpDir, null); assertNotNull(s1); S3StorageService s2 = - new S3StorageService(minioClient, "bucket", null, null, null, null, tmpDir); + new S3StorageService(minioClient, "bucket", null, null, null, null, tmpDir, null); assertNotNull(s2); } @@ -1998,7 +2039,14 @@ public void testCleanupExpiredUploadsPrunesStaleTempFilesAndChecksumIndices() th S3StorageService customService = new S3StorageService( - minioClient, "test-bucket", "uploads/", "metadata/", "checksums/", "locks/", tempDir); + minioClient, + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + tempDir, + null); customService.setUploadDeduplicationEnabled(true); Item checksumItem = mock(Item.class); From e17625d86694b46169fd6164b3d1621cef7ead63 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 10 Oct 2026 09:43:35 +0200 Subject: [PATCH 08/15] fix(cloud): improve pause/resume byte integrity, lock contention resolution, and eliminate volatile variables --- AGENTS.md | 7 + CHANGELOG.md | 6 + docs/S3_STORAGE.md | 3 + .../server/core/CorePatchRequestHandler.java | 12 +- .../CreationWithUploadPostRequestHandler.java | 11 +- .../RufhAppendPatchRequestHandler.java | 10 +- .../RufhCreationPostRequestHandler.java | 10 +- .../AbstractCloseableResourceService.java | 24 +- .../tus/server/upload/AbstractLeaseLock.java | 50 +- .../tus/server/upload/UploadIdFactory.java | 18 +- .../desair/tus/server/upload/UploadInfo.java | 21 + .../upload/azure/AzureBlobStorageService.java | 29 +- .../upload/azure/AzureBlobUploadLock.java | 130 ++- .../server/upload/s3/S3LockingService.java | 57 +- .../server/upload/s3/S3StorageService.java | 874 ++++++++++------- .../tus/server/upload/s3/S3UploadLock.java | 58 +- .../upload/util/AsyncChunkUploader.java | 54 +- .../server/util/InterruptibleInputStream.java | 21 +- .../java/me/desair/tus/server/util/Utils.java | 18 + .../AbstractITTusFileUploadService.java | 4 +- .../RufhAppendPatchRequestHandlerTest.java | 47 + .../RufhCreationPostRequestHandlerTest.java | 44 + .../server/upload/AbstractLeaseLockTest.java | 59 +- .../tus/server/upload/UploadInfoTest.java | 19 + .../azure/AzureBlobStorageServiceTest.java | 16 + .../upload/azure/AzureBlobUploadLockTest.java | 161 ++- .../upload/s3/S3LockingServiceTest.java | 28 + .../upload/s3/S3StorageServiceTest.java | 924 +++++++++++------- .../server/upload/s3/S3UploadLockTest.java | 61 ++ .../upload/util/AsyncChunkUploaderTest.java | 181 +++- .../me/desair/tus/server/util/UtilsTest.java | 15 + 31 files changed, 2139 insertions(+), 833 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9a35c4e1..f33fa5bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,6 +180,13 @@ To maintain a clear separation between fast, offline unit tests and containerize - Do NOT use legacy `java.io.File.createTempFile(...)`. On POSIX systems, it creates files with overly permissive default umask permissions (often world-readable or group-readable, CWE-378 / SonarQube java:S5443). - Always use `java.nio.file.Files.createTempFile(...)` (or `Utils.createTempSiblingPath(...)` / `Utils.createTempSibling(...)`), which creates temporary files with restricted owner-only permissions (`rw-------` / `0600`) by default. +### 23. Thread Safety: Prevent Volatile Class Variables & Use ReentrantReadWriteLock +- Do NOT use `volatile` class variables for mutable state, lazy compilation, or feature flags. `volatile` references on complex objects or mutable fields can trigger static analysis warnings (e.g. SonarQube java:S3077) and cannot guard multi-step compound actions, non-atomic transitions, or lazy initializations. +- Guard concurrent mutable state using a `ReadWriteLock` (`ReentrantReadWriteLock`). +- Consistently name the lock field `private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock();` in every class for pattern consistency across the codebase. +- Use `thisObjectLock.readLock()` for fast, concurrent reads, and `thisObjectLock.writeLock()` for state mutations, lazy pattern compilation, or feature capability degradation. +- Always acquire and release locks using `try ... finally` blocks (e.g. `thisObjectLock.readLock().lock(); try { return field; } finally { thisObjectLock.readLock().unlock(); }`). + ## IETF Resumable Uploads for HTTP (RUFH) Spec Maintenance & Update Playbook diff --git a/CHANGELOG.md b/CHANGELOG.md index fa9dee4f..fa7b4fb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,12 @@ All notable changes to this project will be documented in this file. - **Azure Blob Storage Upload Cancellation Lease Safety**: Checked lock blob lease state before calling `deleteIfExists()` in `AzureBlobStorageService.terminateUpload()`. When an active lease is held on the lock blob by an ongoing request (e.g. `DELETE` cancellation), attempting deletion without specifying the lease ID triggered an Azure SDK error log (`HTTP 412 LeaseIdMissing`). Skipping deletion for actively leased blobs prevents this error and allows normal lease release upon request completion. - **Lock Contention Stream Interruption Handling**: Handled `IOException` from `InterruptibleInputStream` across upload request handlers (`CorePatchRequestHandler`, `RufhAppendPatchRequestHandler`, `RufhCreationPostRequestHandler`, `CreationWithUploadPostRequestHandler`). When an upload stream is interrupted by the locking service watchdog during lock contention (such as concurrent `HEAD` progress checks or `DELETE` cancellation requests), the storage backend commits all buffered bytes received so far and updates the offset in storage. Request handlers now reload the refreshed `UploadInfo` and return a clean HTTP 204/201 response with the updated offset rather than bubbling an unhandled `IOException` / HTTP 500 to the servlet container. - **S3 Server-Side Part Composition & Native Multipart Copy**: Implemented `S3ServerSideComposeHelper` to resolve AWS S3 header rejection during server-side part composition. MinIO Java SDK 9.0.3's `composeObject` delegates to `UploadPartCopy` with an empty body placeholder that automatically injects `Content-MD5` and `Content-Type` headers, which Amazon AWS S3 strictly forbids on part copy requests and rejects with HTTP 400 (`The specified header is not valid in this context`). `S3ServerSideComposeHelper` executes native S3 multipart copy requests directly with SigV4 signing omitting `Content-MD5`, allowing fast zero-bandwidth server-side composition on both AWS S3 and MinIO/Ceph clusters without external AWS SDK dependencies. Added reflection-free constructors to `S3StorageService` and `S3LockingService` accepting connection parameters directly, with `eu-central-1` as the default fallback region, while preserving multi-tier streaming concatenation fallbacks. +- **S3 & Azure Pause/Resume Lock Contention & Byte Integrity**: Resolved lock contention and data corruption issues when uploads are repeatedly paused and resumed: + - **Lock Release on Interruption**: Updated `AsyncChunkUploader` with periodic contention polling (`waitForInFlight`) and bounded worker aborts (`500ms`), ensuring background upload threads cleanly terminate when locks are released and prevented locks from lingering until TTL expiry. + - **Manifest-Tracked Parts (`uploadPartKeys`)**: Replaced non-deterministic S3 prefix listing with an authoritative `uploadPartKeys` manifest in `UploadInfo`. Stale tail buffers are preserved in S3 until new manifests are committed, preventing byte loss or duplicate byte prepends (+8 MB offsets or drops to 0) during multiple pause/resume cycles. + - **Multi-Object Delete with Resilient Fallback**: Implemented batch Multi-Object Delete (`POST /?delete`) in `S3UploadLock` and `S3StorageService` with automatic per-object `DELETE` fallback when batch deletes are restricted by bucket IAM policies or unsupported by S3-compatible providers. + - **Transient Network Tolerance in Lease Renewals**: Tolerated transient network blips in `AbstractLeaseLock` and `AzureBlobUploadLock`, preventing unexpired valid locks from being prematurely aborted during heartbeat retries. + - **Graceful Lock Eviction**: Handled `NO_SUCH_KEY` cleanly in `S3LockingService.evictExpiredLock()` without generating false eviction warnings when locks were already released by their owners. ### Breaking - **Downloads**: In order to support both the Tus protocol and RUFH protocol, the unofficial download extension will not return a HTTP status code `204` for uploads that are still in progress and will not contain the response header `Tus-Resumable`. Removed the `UploadInProgressException` class. diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index c87c67e5..05f77181 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -339,6 +339,9 @@ The following minimal AWS IAM policy permissions are required for `S3StorageServ > [!IMPORTANT] > Make sure to change "my-upload-bucket" to the correct bucket name that you created for your uploads. +> [!NOTE] +> `S3UploadLock` and `S3StorageService` attempt batch Multi-Object Delete (`POST /?delete`) first to delete temporary chunk parts, `.lock`, and `.stop` files in a single round-trip. If batch deletion is not permitted by your IAM policy or unsupported by an S3-compatible backend, the service automatically falls back to individual single-object `DELETE` (`s3:DeleteObject`) requests. + --- ## 9. Developer Instructions: Running Local S3 Integration Tests diff --git a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java index 11aca9d6..d234c881 100644 --- a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java @@ -102,16 +102,18 @@ public HttpProblemDetails process( // the offset in storage. We reload the updated UploadInfo and acknowledge the partial // PATCH with 204 No Content and the new Upload-Offset rather than throwing a 500 error. if (interruptibleStream != null && interruptibleStream.isInterrupted()) { - log.info( - "Upload PATCH request for URI {} was interrupted by locking service contention; " - + "saved partial upload up to offset {}", - servletRequest.getRequestURI(), - uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + // Refresh UploadInfo first so the log statement and response reflect the true committed + // byte offset persisted by the storage backend up to the interruption point. UploadInfo refreshed = uploadStorageService.getUploadInfo(servletRequest.getRequestURI(), ownerKey); if (refreshed != null) { uploadInfo = refreshed; } + log.info( + "Upload PATCH request for URI {} was interrupted by locking service contention; " + + "saved partial upload up to offset {}", + servletRequest.getRequestURI(), + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); } else { throw e; } diff --git a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java index 7f8a8a94..61b0bb39 100644 --- a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java @@ -63,15 +63,18 @@ public HttpProblemDetails process( // the offset in storage. We reload the updated UploadInfo and acknowledge the partial // upload rather than propagating an unhandled error to the servlet container. if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Refresh UploadInfo first so the log statement and response reflect the true + // committed + // byte offset persisted by the storage backend up to the interruption point. + UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } log.info( "Upload creation-with-upload POST request for URI {} was interrupted by locking" + " service contention; saved partial upload up to offset {}", location, uploadInfo != null ? uploadInfo.getOffset() : "unknown"); - UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); - if (refreshed != null) { - uploadInfo = refreshed; - } } else { throw e; } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index 787089a7..0d889b2d 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -88,15 +88,17 @@ public HttpProblemDetails process( // the offset in storage. We reload the updated UploadInfo and acknowledge the partial // append rather than propagating an unhandled error. if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Refresh UploadInfo first so the log statement and response reflect the true committed + // byte offset persisted by the storage backend up to the interruption point. + UploadInfo refreshed = uploadStorageService.getUploadInfo(requestUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } log.info( "RUFH append request for URI {} was interrupted by locking service contention; " + "saved partial upload up to offset {}", requestUri, uploadInfo != null ? uploadInfo.getOffset() : "unknown"); - UploadInfo refreshed = uploadStorageService.getUploadInfo(requestUri, ownerKey); - if (refreshed != null) { - uploadInfo = refreshed; - } } else { throw e; } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index e4eb2712..aa46f7db 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -132,15 +132,17 @@ public HttpProblemDetails process( // the offset in storage. We reload the updated UploadInfo and acknowledge the partial // upload rather than propagating an unhandled error to the servlet container. if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Refresh UploadInfo first so the log statement and response reflect the true committed + // byte offset persisted by the storage backend up to the interruption point. + UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } log.info( "RUFH creation request with body for URI {} was interrupted by locking service" + " contention; saved partial upload up to offset {}", uploadUri, uploadInfo != null ? uploadInfo.getOffset() : "unknown"); - UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey); - if (refreshed != null) { - uploadInfo = refreshed; - } } else { throw e; } diff --git a/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java b/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java index 8560af5a..c0b4b022 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java @@ -2,6 +2,8 @@ import java.io.Closeable; import java.io.IOException; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Abstract base class for services that manage closeable resources and optional JVM shutdown hooks. @@ -9,7 +11,8 @@ public abstract class AbstractCloseableResourceService implements Closeable { private final Thread shutdownHook; - private volatile boolean closed = false; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean closed = false; /** Constructs a closeable resource service without a JVM shutdown hook. */ protected AbstractCloseableResourceService() { @@ -36,7 +39,12 @@ protected AbstractCloseableResourceService(String shutdownHookName) { * @return true if the service is closed, false otherwise */ public boolean isClosed() { - return closed; + thisObjectLock.readLock().lock(); + try { + return closed; + } finally { + thisObjectLock.readLock().unlock(); + } } /** @@ -47,11 +55,17 @@ public boolean isClosed() { */ @Override public void close() throws IOException { - if (!closed) { + thisObjectLock.writeLock().lock(); + try { + if (closed) { + return; + } closed = true; - deregisterShutdownHook(); - cleanupOnClose(); + } finally { + thisObjectLock.writeLock().unlock(); } + deregisterShutdownHook(); + cleanupOnClose(); } /** diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java index 8beec793..ee24adfe 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java @@ -126,25 +126,51 @@ public void close() { * Renew the lock lease by advancing the expiration timestamp and persisting the updated metadata. */ public void renewLease() { + long oldExpiresAt = leaseData.getExpiresAt(); + long candidateExpiresAt = System.currentTimeMillis() + leaseData.getLeaseDurationMs(); try { - leaseData.setExpiresAt(System.currentTimeMillis() + leaseData.getLeaseDurationMs()); + leaseData.setExpiresAt(candidateExpiresAt); doRenewLease(); + } catch (IllegalStateException e) { + leaseData.setExpiresAt(oldExpiresAt); + // Ownership lost (e.g. lease taken over by another node): abort immediately + log.info( + "Aborting active input stream for upload URI {} as lock was taken over: {}", + getRequestUri(), + e.getMessage()); + abortActiveStreamAndHeartbeat(); } catch (Exception e) { - // If lease renewal fails, abort any active input stream immediately - // to prevent the upload thread from continuing to write un-locked bytes. - if (activeInputStreams != null && getRequestUri() != null) { - InputStream stream = activeInputStreams.get(getRequestUri()); - if (stream != null) { - log.info( - "Aborting active input stream for upload URI {} due to lease renewal failure", - getRequestUri()); - Utils.interruptStream(stream); - } + leaseData.setExpiresAt(oldExpiresAt); + // Transient failure (e.g. network timeout). Only abort if the previous lease actually + // expired. + if (System.currentTimeMillis() >= oldExpiresAt) { + log.warn( + "Lease for upload URI {} expired after renewal failure; aborting active stream", + getRequestUri(), + e); + abortActiveStreamAndHeartbeat(); + } else { + log.warn( + "Transient failure renewing lease for upload URI {} (will retry on next tick): {}", + getRequestUri(), + e.getMessage()); } - Utils.shutdownExecutor(heartbeatExecutor); } } + private void abortActiveStreamAndHeartbeat() { + if (activeInputStreams != null && getRequestUri() != null) { + InputStream stream = activeInputStreams.get(getRequestUri()); + if (stream != null) { + log.info( + "Aborting active input stream for upload URI {} due to lease renewal failure", + getRequestUri()); + Utils.interruptStream(stream); + } + } + Utils.shutdownExecutor(heartbeatExecutor); + } + /** * Subclass hook to persist updated lease metadata during heartbeat renewal. * diff --git a/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java b/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java index 274dd735..717ffb01 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java +++ b/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java @@ -16,7 +16,7 @@ */ public abstract class UploadIdFactory { - private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); private String uploadUri = "/"; // Read and write operations on uploadUri and uploadUriPattern are guarded by a // ReentrantReadWriteLock. @@ -41,12 +41,12 @@ public void setUploadUri(String uploadUri) { "The upload URI should start with /, http://, or https://"); Validate.isTrue(!Strings.CS.endsWith(uploadUri, "$"), "The upload URI should not end with $"); - lock.writeLock().lock(); + thisObjectLock.writeLock().lock(); try { this.uploadUri = uploadUri; this.uploadUriPattern = null; } finally { - lock.writeLock().unlock(); + thisObjectLock.writeLock().unlock(); } } @@ -57,11 +57,11 @@ public void setUploadUri(String uploadUri) { * @return The URI of the main tus upload endpoint. */ public String getUploadUri() { - lock.readLock().lock(); + thisObjectLock.readLock().lock(); try { return uploadUri; } finally { - lock.readLock().unlock(); + thisObjectLock.readLock().unlock(); } } @@ -107,16 +107,16 @@ public UploadId readUploadId(String url) { * @return A (cached) Pattern to match upload URI's */ protected Pattern getUploadUriPattern() { - lock.readLock().lock(); + thisObjectLock.readLock().lock(); try { if (uploadUriPattern != null) { return uploadUriPattern; } } finally { - lock.readLock().unlock(); + thisObjectLock.readLock().unlock(); } - lock.writeLock().lock(); + thisObjectLock.writeLock().lock(); try { if (uploadUriPattern == null) { // Extract upload IDs by removing upload URI from start of request URI. @@ -127,7 +127,7 @@ protected Pattern getUploadUriPattern() { } return uploadUriPattern; } finally { - lock.writeLock().unlock(); + thisObjectLock.writeLock().unlock(); } } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadInfo.java b/src/main/java/me/desair/tus/server/upload/UploadInfo.java index a843940a..56f6356f 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadInfo.java +++ b/src/main/java/me/desair/tus/server/upload/UploadInfo.java @@ -43,6 +43,7 @@ public class UploadInfo implements Serializable { private String representationDigest; private String requestedRepresentationDigests; private String storageUploadId; + private List uploadPartKeys; /** Default constructor to use if an upload is created without HTTP request. */ public UploadInfo() { @@ -460,6 +461,24 @@ public void setStorageUploadId(String storageUploadId) { this.storageUploadId = storageUploadId; } + /** + * Get the ordered list of storage part keys or block IDs committed for this upload. + * + * @return The list of part keys or block IDs, or null if uninitialized/unsupported + */ + public List getUploadPartKeys() { + return uploadPartKeys; + } + + /** + * Set the ordered list of storage part keys or block IDs committed for this upload. + * + * @param uploadPartKeys The list of part keys or block IDs + */ + public void setUploadPartKeys(List uploadPartKeys) { + this.uploadPartKeys = uploadPartKeys; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -489,6 +508,7 @@ public boolean equals(Object o) { .append(getRepresentationDigest(), that.getRepresentationDigest()) .append(getRequestedRepresentationDigests(), that.getRequestedRepresentationDigests()) .append(getStorageUploadId(), that.getStorageUploadId()) + .append(getUploadPartKeys(), that.getUploadPartKeys()) .isEquals(); } @@ -511,6 +531,7 @@ public int hashCode() { .append(getRepresentationDigest()) .append(getRequestedRepresentationDigests()) .append(getStorageUploadId()) + .append(getUploadPartKeys()) .toHashCode(); } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 2f62320e..63c038d7 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -27,6 +27,7 @@ import java.util.Base64; import java.util.List; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -44,6 +45,7 @@ import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; import me.desair.tus.server.upload.util.AsyncChunkUploader; +import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.IOUtils; @@ -226,6 +228,14 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) containerClient.getBlobClient(getAzureBlobName(upload)).getBlockBlobClient(); List blockIds = getCommittedBlockIds(blockBlobClient); + // If metadata records committed uploadPartKeys, ensure the blob's committed blocks do not + // exceed the authoritative metadata offset (e.g. if a prior commitBlockList succeeded + // but updating UploadInfo metadata failed or was interrupted). + if (upload.getUploadPartKeys() != null && blockIds.size() > upload.getUploadPartKeys().size()) { + blockIds = new ArrayList<>(upload.getUploadPartKeys()); + blockBlobClient.commitBlockList(blockIds, true); + } + // 2. Validate block budget: Azure Block Blobs support up to 50,000 blocks validateRemainingBlockBudget(upload, blockIds.size()); @@ -243,8 +253,9 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) List plannedBlockIds = new ArrayList<>(); List plannedChunkSizes = new ArrayList<>(); + InterruptibleInputStream interruptibleStream = Utils.toInterruptibleStream(inputStream); try (AsyncChunkUploader uploader = - new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis())) { + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis(), interruptibleStream)) { // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob while (true) { File chunkFile = null; @@ -337,6 +348,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) try { blockBlobClient.commitBlockList(blockIds, true); upload.setOffset(initialOffset + confirmedAppended); + upload.setUploadPartKeys(new ArrayList<>(blockIds)); if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { upload.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); } @@ -965,9 +977,18 @@ private void commitEmptyDataBlob(UploadId id) { blockBlobClient.commitBlockList(new ArrayList<>(), true); } - /** Generates Base64 encoded block ID matching Azure Block Blob standards. */ - private String generateBlockId(int index) { - String idString = String.format("block-%06d", index); + /** + * Generates a Base64-encoded block ID matching Azure Block Blob standards. + * + *

Incorporates an 8-character random UUID suffix to guarantee that late-landing blocks from + * previous aborted or timed-out worker threads never collide with or overwrite blocks in + * + * @param index The block index + * @return The Base64-encoded block ID + */ + String generateBlockId(int index) { + String idString = + String.format("blk-%06d-%s", index, UUID.randomUUID().toString().substring(0, 8)); return Base64.getEncoder().encodeToString(idString.getBytes(StandardCharsets.UTF_8)); } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java index c0daa6b5..86b30243 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java @@ -1,6 +1,7 @@ package me.desair.tus.server.upload.azure; import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.specialized.BlobLeaseClient; import java.io.IOException; import java.lang.ref.WeakReference; @@ -8,6 +9,8 @@ import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.Utils; @@ -25,7 +28,6 @@ public class AzureBlobUploadLock implements UploadLock { private static final Logger log = LoggerFactory.getLogger(AzureBlobUploadLock.class); - private static final long RENEWAL_INTERVAL_SECONDS = 10L; private final BlobLeaseClient leaseClient; @@ -34,7 +36,8 @@ public class AzureBlobUploadLock implements UploadLock { private final String uploadId; private final Map> activeStreams; private final ScheduledExecutorService renewalExecutor; - private volatile boolean released = false; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean released = false; /** * Constructs an {@link AzureBlobUploadLock} wrapping an acquired Azure Blob lease. @@ -114,32 +117,89 @@ public AzureBlobUploadLock( } } + private static final long LEASE_DURATION_MS = 60000L; + private long leaseExpiresAt = System.currentTimeMillis() + LEASE_DURATION_MS; + BlobLeaseClient getLeaseClient() { return leaseClient; } + void setLeaseExpiresAt(long leaseExpiresAt) { + thisObjectLock.writeLock().lock(); + try { + this.leaseExpiresAt = leaseExpiresAt; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + /** Attempts to renew the lease with Azure Blob Storage. */ void renewLease() { - if (released) { - return; + thisObjectLock.readLock().lock(); + try { + if (released) { + return; + } + } finally { + thisObjectLock.readLock().unlock(); } try { executeRenew(); + thisObjectLock.writeLock().lock(); + try { + leaseExpiresAt = System.currentTimeMillis() + LEASE_DURATION_MS; + } finally { + thisObjectLock.writeLock().unlock(); + } log.trace("Successfully renewed Azure blob lease for upload URI {}", uploadUri); } catch (Exception e) { - log.warn("Failed to renew Azure blob lease for upload URI {}: {}", uploadUri, e.getMessage()); - released = true; - shutdownExecutor(); - // If lease renewal fails, abort any active input stream immediately - // so the upload thread does not continue writing un-locked data to Azure. - if (activeStreams != null && uploadId != null) { - WeakReference streamRef = activeStreams.remove(uploadId); - if (streamRef != null) { - log.info( - "Aborting active stream for upload ID {} due to lease renewal failure", uploadId); - Utils.interruptStream(streamRef.get()); + boolean leaseLost = false; + if (e instanceof BlobStorageException) { + AzureErrorType errorType = AzureUtils.parseErrorResponse((BlobStorageException) e); + if (errorType == AzureErrorType.LEASE_ALREADY_PRESENT + || errorType == AzureErrorType.LEASE_NOT_PRESENT + || errorType == AzureErrorType.CONFLICT) { + leaseLost = true; } } + + boolean expired; + thisObjectLock.readLock().lock(); + try { + expired = System.currentTimeMillis() >= leaseExpiresAt; + } finally { + thisObjectLock.readLock().unlock(); + } + + if (leaseLost || expired) { + log.warn( + "Azure blob lease definitively lost/expired for upload URI {}: {}", + uploadUri, + e.getMessage()); + thisObjectLock.writeLock().lock(); + try { + released = true; + } finally { + thisObjectLock.writeLock().unlock(); + } + shutdownExecutor(); + // If lease renewal fails definitively, abort any active input stream immediately + // so the upload thread does not continue writing un-locked data to Azure. + if (activeStreams != null && uploadId != null) { + WeakReference streamRef = activeStreams.remove(uploadId); + if (streamRef != null) { + log.info( + "Aborting active stream for upload ID {} due to lease renewal failure", uploadId); + Utils.interruptStream(streamRef.get()); + } + } + } else { + log.warn( + "Transient failure renewing Azure blob lease for upload URI {} (will retry on next tick):" + + " {}", + uploadUri, + e.getMessage()); + } } } @@ -152,24 +212,30 @@ void executeRenew() { @Override public void release() { - if (!released) { - released = true; - shutdownExecutor(); - // Remove active stream registration from JVM map upon release - // to avoid stale references lingering in heap. - if (activeStreams != null && uploadId != null) { - activeStreams.remove(uploadId); + thisObjectLock.writeLock().lock(); + try { + if (released) { + return; } - if (leaseClient != null) { - try { - leaseClient.releaseLease(); - log.trace("Released Azure blob lease for upload URI {}", uploadUri); - } catch (Exception e) { - log.debug( - "Azure blob lease release failed (may have already expired/broken) for URI {}: {}", - uploadUri, - e.getMessage()); - } + released = true; + } finally { + thisObjectLock.writeLock().unlock(); + } + shutdownExecutor(); + // Remove active stream registration from JVM map upon release + // to avoid stale references lingering in heap. + if (activeStreams != null && uploadId != null) { + activeStreams.remove(uploadId); + } + if (leaseClient != null) { + try { + leaseClient.releaseLease(); + log.trace("Released Azure blob lease for upload URI {}", uploadUri); + } catch (Exception e) { + log.debug( + "Azure blob lease release failed (may have already expired/broken) for URI {}: {}", + uploadUri, + e.getMessage()); } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 1def9dc0..22ef8c5b 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -14,6 +14,8 @@ import java.io.InputStream; import java.util.Collections; import java.util.Objects; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.upload.AbstractLeaseLockingService; import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadId; @@ -61,7 +63,8 @@ public class S3LockingService extends AbstractLeaseLockingService { private final MinioClient minioClient; private final String bucket; private final String locksPrefix; - private volatile boolean s3ConditionalWritesSupported = true; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean s3ConditionalWritesSupported = true; /** * Convenience constructor for local S3-compatible backends where region is omitted. Defaults the @@ -182,7 +185,12 @@ public S3LockingService withJitter(long minMs, long maxMs) { * @return This service instance for fluent chaining */ public S3LockingService withS3ConditionalWritesSupported(boolean supported) { - this.s3ConditionalWritesSupported = supported; + thisObjectLock.writeLock().lock(); + try { + this.s3ConditionalWritesSupported = supported; + } finally { + thisObjectLock.writeLock().unlock(); + } return this; } @@ -193,7 +201,12 @@ public S3LockingService withS3ConditionalWritesSupported(boolean supported) { * disabled */ public boolean isS3ConditionalWritesSupported() { - return s3ConditionalWritesSupported; + thisObjectLock.readLock().lock(); + try { + return s3ConditionalWritesSupported; + } finally { + thisObjectLock.readLock().unlock(); + } } @Override @@ -238,7 +251,15 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { byte[] lockContentBytes = LeaseDataJsonSerializer.serializeToBytes(leaseData); - if (s3ConditionalWritesSupported) { + boolean conditionalSupported; + thisObjectLock.readLock().lock(); + try { + conditionalSupported = s3ConditionalWritesSupported; + } finally { + thisObjectLock.readLock().unlock(); + } + + if (conditionalSupported) { // Layer 1: Optimistic Conditional PutObject with "If-None-Match: *" // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object // already @@ -264,7 +285,12 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { "S3 endpoint does not support conditional writes (If-None-Match: *). " + "Downgrading to non-CAS lock arbitration for key {}", lockKey); - s3ConditionalWritesSupported = false; + thisObjectLock.writeLock().lock(); + try { + s3ConditionalWritesSupported = false; + } finally { + thisObjectLock.writeLock().unlock(); + } if (!isLockExpired(lockKey)) { return null; } @@ -328,15 +354,24 @@ protected boolean evictExpiredLock(UploadId uploadId) { return false; } String lockKey = buildLockKey(uploadId); - if (!isLockExpired(lockKey)) { - return false; - } - // Delete the expired .lock object from S3 so the subsequent tryAcquireLock() conditional - // write with "If-None-Match: *" can succeed and take over the abandoned lock. - try { + try (InputStream stream = + minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(lockKey).build())) { + LeaseData lock = LeaseDataJsonSerializer.deserialize(stream); + if (!isLeaseExpired(lock, System.currentTimeMillis())) { + return false; + } + // Actual expired lock found in S3: delete it so next tryAcquireLock() can succeed minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(lockKey).build()); log.info("Evicted expired S3 lock for key {}", lockKey); return true; + } catch (ErrorResponseException e) { + if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { + // Lock key was already released normally and removed from S3. + // Return true to indicate the resource is clear without emitting false eviction logs. + return true; + } + log.warn("Failed checking S3 lock expiration for key {}", lockKey, e); + return false; } catch (Exception e) { log.warn("Failed to evict expired S3 lock for key {}", lockKey, e); return false; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 1f1513d1..3a35d073 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -5,10 +5,13 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; import io.minio.Result; import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteRequest; +import io.minio.messages.DeleteResult; import io.minio.messages.Item; import java.io.ByteArrayInputStream; import java.io.File; @@ -25,14 +28,19 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MaxUploadLengthExceededException; @@ -49,6 +57,7 @@ import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; import me.desair.tus.server.upload.util.AsyncChunkUploader; +import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.FileUtils; @@ -121,7 +130,9 @@ public class S3StorageService implements UploadStorageService { private UploadIdFactory idFactory = new UuidUploadIdFactory(); private UploadConcatenationService concatenationService; - private volatile boolean s3ComposeObjectSupported = true; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean s3ComposeObjectSupported = true; + private boolean supportsBatchDelete = true; private S3ServerSideComposeHelper s3ComposeHelper; /** @@ -388,6 +399,12 @@ public UploadInfo create(UploadInfo info, String ownerKey) throws IOException { } info.setOwnerKey(ownerKey); info.setStorageUploadId(buildObjectKey(info.getId())); + if (info.getUploadPartKeys() == null) { + info.setUploadPartKeys(new ArrayList<>()); + } + if (info.getOffset() == null) { + info.setOffset(0L); + } // Persist initial UploadInfo metadata object (.info) to S3 try { @@ -407,69 +424,83 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) info.setLength(upload.getLength()); } String objectKey = getS3ObjectKey(info); - String partObjectKey = buildIncompletePartKey(info.getId()); + InterruptibleInputStream interruptibleStream = Utils.toInterruptibleStream(inputStream); - // Step 2: If a previous sub-5MB .part buffer exists in S3, download & prepend it to incoming - // stream. Also roll back any existing sub-5MB numbered parts (e.g. from prior aborted - // sessions). + // Step 2: If the last part is sub-5MB, download & prepend its bytes to the incoming stream. + // The previous tail part is intentionally kept in S3 as a staleTailKey and is ONLY deleted + // after the new manifest is successfully committed to S3. This ensures zero data loss if + // this request is paused or interrupted immediately. PreparedStream preparedStream = - prepareStreamWithExistingIncompletePart(info.getId(), partObjectKey, inputStream); + prepareStreamWithExistingIncompletePart(info, interruptibleStream); // Validate that the upload has not exceeded S3's 10,000 multipart parts ceiling validateRemainingPartBudget(info, preparedStream.remainingPartKeys.size()); - // Step 3: Process payload stream in optimal chunk parts and upload to S3 - boolean successfullyFinished = false; - try { - AppendResult appendResult = - processPayloadChunks(info, preparedStream, info.getId(), partObjectKey); - - // Step 4: Recalculate total uploaded byte offset across all uploaded part objects in S3. - // S3 listObjects in distributed clusters (e.g. multi-site Ceph, cluster rebalancing, or - // async replication) can exhibit listing delays; take the maximum of remote parts query - // and locally verified stream progression to ensure newOffset accurately reflects bytes - // successfully written. - long calculatedOffset = - calculateCurrentOffset(objectKey, info.getId(), partObjectKey, info.getLength()); - long currentTotalOffset = - (info.getOffset() != null ? info.getOffset() : 0L) - + appendResult.totalBytesAppended - - preparedStream.prependedBytes; - long newOffset = Math.max(calculatedOffset, currentTotalOffset); + // Clean up any unmanifested orphan part objects left in S3 from earlier aborted attempts. + // Pruning unmanifested parts under the upload lock prevents listing delays and ensures + // that only parts explicitly tracked in the manifest are considered valid. + pruneOrphanParts(info.getId(), preparedStream.remainingPartKeys, preparedStream.staleTailKeys); + + // Step 3: Process payload stream in optimal chunk parts and upload to S3. + AppendResult appendResult = processPayloadChunks(info, preparedStream, info.getId()); + + // Step 4: Persist any newly confirmed chunk parts and update the byte offset. + // Drained chunks and trailing parts are committed to the manifest before rethrowing any + // stream interruption exception, ensuring maximum byte persistence. + boolean hasNewParts = + appendResult.allPartKeys.size() > preparedStream.remainingPartKeys.size() + || appendResult.confirmedBytesAppended > 0; + + if (hasNewParts) { + long currentOffsetBeforeStream = + Math.max( + info.getOffset() != null ? info.getOffset() : 0L, + preparedStream.existingPartsTotalSize + preparedStream.prependedBytes); + long baseOffset = currentOffsetBeforeStream - preparedStream.prependedBytes; + long newOffset = baseOffset + appendResult.confirmedBytesAppended; info.setOffset(newOffset); upload.setOffset(newOffset); - - // Step 5: Validate minimum append size constraints if configured - // Subtract prependedBytes so minAppendSize accurately measures the payload transferred - // in THIS request rather than earlier buffered bytes. - // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, - // or to requests completing the upload by including the Upload-Complete: ?1 header field." - boolean isCompletingOrEmpty = - !info.isUploadInProgress() || (info.getLength() != null && newOffset >= info.getLength()); - long requestPayloadAppended = - Math.max(0L, appendResult.totalBytesAppended - preparedStream.prependedBytes); - if (minAppendSize != null && !isCompletingOrEmpty && requestPayloadAppended < minAppendSize) { - throw new MinAppendSizeNotMetException( - "Append payload size " - + requestPayloadAppended - + " is below minimum limit " - + minAppendSize); - } - - // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object - finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), appendResult, newOffset); + info.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); update(info); - successfullyFinished = true; - return info; - } finally { - if (!successfullyFinished) { - long newOffset = - calculateCurrentOffset(objectKey, info.getId(), partObjectKey, info.getLength()); - info.setOffset(newOffset); - upload.setOffset(newOffset); - update(info); + + // Safe transactional cleanup: Delete old tail parts only AFTER the new manifest is saved. + // If the upload was interrupted earlier, staleTailKeys were untouched, preserving bytes. + for (String staleKey : preparedStream.staleTailKeys) { + deleteObjectQuietly(staleKey); } } + + // Rethrow any stream reading or size limit exceptions after metadata has been safely saved + if (appendResult.readException != null) { + throw appendResult.readException; + } + if (appendResult.maxAppendSizeException != null) { + throw appendResult.maxAppendSizeException; + } + + // Step 5: Validate minimum append size constraints if configured. + // Subtract prependedBytes so minAppendSize accurately measures payload transferred in THIS + // request. Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no + // content, or to requests completing the upload by including the Upload-Complete: ?1 header + // field." + long currentOffset = info.getOffset() != null ? info.getOffset() : 0L; + boolean isCompletingOrEmpty = + !info.isUploadInProgress() + || (info.getLength() != null && currentOffset >= info.getLength()); + long requestPayloadAppended = + Math.max(0L, appendResult.confirmedBytesAppended - preparedStream.prependedBytes); + if (minAppendSize != null && !isCompletingOrEmpty && requestPayloadAppended < minAppendSize) { + throw new MinAppendSizeNotMetException( + "Append payload size " + + requestPayloadAppended + + " is below minimum limit " + + minAppendSize); + } + + // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object + finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), currentOffset); + update(info); + return info; } @Override @@ -592,20 +623,24 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) return; } String objectKey = getS3ObjectKey(uploadInfo); - String partKey = buildIncompletePartKey(uploadInfo.getId()); - long newOffset = Math.max(0L, uploadInfo.getOffset() - byteCount); uploadInfo.setOffset(newOffset); - update(uploadInfo); // If final completed object exists in S3, truncate it if (objectExists(objectKey)) { - truncateFromCompletedObject(objectKey, partKey, newOffset); + truncateCompletedObject(uploadInfo, objectKey, newOffset); + update(uploadInfo); + return; + } + + // If upload has manifest parts in uploadPartKeys, truncate backwards from parts + if (uploadInfo.getUploadPartKeys() != null && !uploadInfo.getUploadPartKeys().isEmpty()) { + truncateManifestParts(uploadInfo, byteCount); + update(uploadInfo); return; } - // Otherwise truncate from incomplete .part object - truncateFromIncompletePart(partKey, byteCount); + update(uploadInfo); } @Override @@ -615,12 +650,10 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio } String objectKey = getS3ObjectKey(uploadInfo); String metadataKey = buildMetadataKey(uploadInfo.getId()); - String partKey = buildIncompletePartKey(uploadInfo.getId()); - // Delete final object, metadata object, and incomplete part object from S3 + // Delete final object and metadata object from S3 deleteObjectQuietly(objectKey); deleteObjectQuietly(metadataKey); - deleteObjectQuietly(partKey); // Delete all temporary part chunk objects (e.g. metadata/.part.00001) deleteAllPartObjectsQuietly(uploadInfo.getId()); @@ -798,11 +831,21 @@ public UploadConcatenationService getUploadConcatenationService() { } boolean isS3ComposeObjectSupported() { - return s3ComposeObjectSupported; + thisObjectLock.readLock().lock(); + try { + return s3ComposeObjectSupported; + } finally { + thisObjectLock.readLock().unlock(); + } } void setS3ComposeObjectSupported(boolean s3ComposeObjectSupported) { - this.s3ComposeObjectSupported = s3ComposeObjectSupported; + thisObjectLock.writeLock().lock(); + try { + this.s3ComposeObjectSupported = s3ComposeObjectSupported; + } finally { + thisObjectLock.writeLock().unlock(); + } } void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) { @@ -879,6 +922,24 @@ public long getPreferredPartSize() { return preferredPartSize; } + boolean isSupportsBatchDelete() { + thisObjectLock.readLock().lock(); + try { + return supportsBatchDelete; + } finally { + thisObjectLock.readLock().unlock(); + } + } + + void setSupportsBatchDelete(boolean supportsBatchDelete) { + thisObjectLock.writeLock().lock(); + try { + this.supportsBatchDelete = supportsBatchDelete; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + @Override public void close() throws IOException { uploadExecutor.shutdown(); @@ -918,52 +979,34 @@ private void validateUploadLimits(UploadInfo info) throws TusException { } /** - * Checks if an incomplete sub-5MB {@code .part} buffer object exists in S3 from a previous - * interrupted or paused request. If present, downloads its bytes and prepends them to the - * incoming stream. + * Prepares the incoming input stream by rolling back any prior sub-5MB incomplete tail part into + * the stream buffer so it can continue growing. * - *

Additionally, if prior paused sessions promoted a sub-5MB chunk to a numbered part (e.g. - * {@code .part.00001}), this method dynamically rolls back that sub-5MB chunk from S3 into the - * stream buffer so that subsequent appends can grow the chunk past AWS S3's 5 MB minimum part - * size. + *

The previous tail part is intentionally kept in S3 as a {@code staleTailKey} and is ONLY + * deleted after the new manifest is successfully committed to S3. This ensures zero data loss if + * this request is paused or interrupted immediately. * - * @param id The upload identifier - * @param partObjectKey The S3 object key for the incomplete .part buffer - * @param inputStream The incoming request input stream - * @return {@link PreparedStream} containing the combined stream and prepended byte count + * @param info The upload metadata + * @param interruptibleStream The incoming request interruptible input stream + * @return {@link PreparedStream} containing the combined stream, prepended bytes, remaining + * parts, and stale tail keys * @throws IOException If reading or downloading from S3 fails */ private PreparedStream prepareStreamWithExistingIncompletePart( - UploadId id, String partObjectKey, InputStream inputStream) throws IOException { + UploadInfo info, InterruptibleInputStream interruptibleStream) throws IOException { long prependedBytes = 0L; - InputStream combinedStream = inputStream; + InputStream combinedStream = interruptibleStream; + List staleTailKeys = new ArrayList<>(); - // 1. Check for leftover incomplete sub-5MB .part buffer object - try { - StatObjectResponse partHead = - minioClient.statObject( - StatObjectArgs.builder().bucket(bucket).object(partObjectKey).build()); - if (partHead != null && partHead.size() > 0) { - try (InputStream partStream = - minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(partObjectKey).build())) { - byte[] partBytes = IOUtils.toByteArray(partStream); - prependedBytes += partBytes.length; - combinedStream = - new SequenceInputStream(new ByteArrayInputStream(partBytes), combinedStream); - } - deleteObjectQuietly(partObjectKey); - } - } catch (ErrorResponseException e) { - if (S3Utils.parseErrorResponse(e) != S3ErrorType.NO_SUCH_KEY) { - log.debug("Error checking incomplete part object {}: {}", partObjectKey, e.getMessage()); - } - } catch (Exception e) { - log.debug("Unexpected error reading incomplete part {}: {}", partObjectKey, e.getMessage()); - } + // 1. Fetch current committed part keys from manifest + List existingPartKeys = + info.getUploadPartKeys() != null + ? new ArrayList<>(info.getUploadPartKeys()) + : new ArrayList<>(); + + long existingPartsTotalSize = calculateTotalPartsSize(existingPartKeys); - // 2. Roll back any existing sub-5MB numbered parts (e.g. from prior aborted or paused uploads) - List existingPartKeys = fetchExistingPartKeys(id); + // 2. Roll back any existing sub-5MB numbered parts into the stream buffer so they can grow if (!existingPartKeys.isEmpty()) { String lastPartKey = existingPartKeys.get(existingPartKeys.size() - 1); try { @@ -972,7 +1015,7 @@ private PreparedStream prepareStreamWithExistingIncompletePart( StatObjectArgs.builder().bucket(bucket).object(lastPartKey).build()); if (stat.size() < minPartSize) { log.info( - "Rolling back sub-5MB numbered part {} (size: {}) into stream buffer", + "Rolling back sub-5MB part {} (size: {}) into stream buffer", lastPartKey, stat.size()); try (InputStream lastPartStream = @@ -982,73 +1025,104 @@ private PreparedStream prepareStreamWithExistingIncompletePart( prependedBytes += lastPartBytes.length; combinedStream = new SequenceInputStream(new ByteArrayInputStream(lastPartBytes), combinedStream); + staleTailKeys.add(lastPartKey); + existingPartsTotalSize -= lastPartBytes.length; + existingPartKeys.remove(existingPartKeys.size() - 1); } - deleteObjectQuietly(lastPartKey); - existingPartKeys.remove(existingPartKeys.size() - 1); } } catch (Exception e) { log.debug("Error inspecting last part key {}: {}", lastPartKey, e.getMessage()); } } - return new PreparedStream(combinedStream, prependedBytes, existingPartKeys); + return new PreparedStream( + combinedStream, + interruptibleStream, + prependedBytes, + existingPartsTotalSize, + existingPartKeys, + staleTailKeys); + } + + private long calculateTotalPartsSize(List partKeys) { + long totalSize = 0L; + for (String key : partKeys) { + try { + StatObjectResponse stat = + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(key).build()); + if (stat != null) { + totalSize += stat.size(); + } + } catch (Exception ignored) { + } + } + return totalSize; + } + + /** + * Prunes unmanifested orphaned part objects left in S3 from earlier aborted parallel attempts. + * + * @param id The upload identifier + * @param manifestedParts Parts actively tracked in the manifest + * @param staleTailKeys Stale tail parts pending replacement in this request + */ + private void pruneOrphanParts( + UploadId id, List manifestedParts, List staleTailKeys) { + List s3Parts = fetchExistingPartKeys(id); + for (String s3Part : s3Parts) { + if (!manifestedParts.contains(s3Part) && !staleTailKeys.contains(s3Part)) { + log.debug("Pruning unmanifested orphan S3 part {}", s3Part); + deleteObjectQuietly(s3Part); + } + } } /** * Reads bytes from the incoming stream into temporary local files of optimal part size (default * 8MB). Parts ≥ 5MB are uploaded asynchronously to S3 via {@link AsyncChunkUploader}, - * overlapping client stream reading with cloud upload. Any trailing chunk under 5MB is saved as a - * temporary .part object unless it completes the overall upload. + * overlapping client stream reading with cloud upload. Any trailing chunk under 5MB is uploaded + * directly to S3 and tracked in the manifest. */ private AppendResult processPayloadChunks( - UploadInfo info, PreparedStream preparedStream, UploadId id, String partObjectKey) - throws IOException, MaxAppendSizeExceededException { + UploadInfo info, PreparedStream preparedStream, UploadId id) { List allPartKeys = new ArrayList<>(preparedStream.remainingPartKeys); int nextPartNumber = allPartKeys.size() + 1; - InputStream streamToRead = preparedStream.stream; long optimalPartSize = calcOptimalPartSize(info.getLength()); byte[] buffer = new byte[8192]; long totalBytesAppended = 0; - - // Calibrate base offset: info.getOffset() already includes prependedBytes from prior writes. - // Subtracting prependedBytes ensures currentTotalOffset accurately tracks progress from - // baseOffset. - long baseOffset = info.getOffset() - preparedStream.prependedBytes; + long confirmedBytesAppended = 0; + long currentOffsetBeforeStream = + Math.max( + info.getOffset() != null ? info.getOffset() : 0L, + preparedStream.existingPartsTotalSize + preparedStream.prependedBytes); + long baseOffset = currentOffsetBeforeStream - preparedStream.prependedBytes; boolean streamFinished = false; MaxAppendSizeExceededException maxAppendSizeException = null; IOException readException = null; List plannedPartKeys = new ArrayList<>(); + List plannedChunkSizes = new ArrayList<>(); try (AsyncChunkUploader uploader = - new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis())) { + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis(), preparedStream)) { while (!streamFinished) { - File tempChunkFile = - Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); - // Do not call tempChunkFile.deleteOnExit() here. In high-throughput long-running services, - // deleteOnExit() registers entries in a static JVM set that cannot be garbage collected, - // creating an unbounded memory leak. Temp files are deleted in try-finally blocks below. - + File tempChunkFile = null; long chunkBytesWritten = 0; boolean handedOff = false; try { + tempChunkFile = + Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); + try (FileOutputStream fos = new FileOutputStream(tempChunkFile)) { int bytesRead; while (chunkBytesWritten < optimalPartSize - && (bytesRead = streamToRead.read(buffer)) != -1) { + && (bytesRead = preparedStream.read(buffer)) != -1) { long requestBytesSoFar = (totalBytesAppended + bytesRead) - preparedStream.prependedBytes; if (maxAppendSize != null && requestBytesSoFar > maxAppendSize) { - // If maxAppendSize is exceeded, do not discard tempChunkFile immediately. - // If preparedStream had prepended bytes from a previous incomplete .part, - // discarding - // tempChunkFile would permanently lose those bytes. Instead, stop reading and - // record - // maxAppendSizeException so the bytes currently in tempChunkFile are flushed to S3 - // and UploadInfo offset is accurately preserved. maxAppendSizeException = new MaxAppendSizeExceededException( "Append payload exceeded limit of " + maxAppendSize); @@ -1072,20 +1146,6 @@ private AppendResult processPayloadChunks( break; } - // Base offset plus total bytes appended accurately measures uploaded progress without - // double-counting - long currentTotalOffset = baseOffset + totalBytesAppended; - - // Interruption Guard: If an IOException or limit exception occurred (e.g. client pause or - // connection drop), - // the chunk must NEVER be considered complete, preventing sub-5MB chunks from being - // promoted. - boolean isUploadComplete = - readException == null - && maxAppendSizeException == null - && info.getLength() != null - && currentTotalOffset >= info.getLength(); - // Validate remaining part capacity before allocating next S3 part number if (allPartKeys.size() + plannedPartKeys.size() >= MAX_PARTS_PER_UPLOAD || nextPartNumber > MAX_PARTS_PER_UPLOAD) { @@ -1096,83 +1156,105 @@ private AppendResult processPayloadChunks( + " parts."); if (chunkBytesWritten > 0) { int confirmedCount = uploader.drainAndComplete(); - allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); - storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + int confirmed = Math.min(confirmedCount, plannedPartKeys.size()); + for (int i = 0; i < confirmed; i++) { + allPartKeys.add(plannedPartKeys.get(i)); + confirmedBytesAppended += plannedChunkSizes.get(i); + } + plannedPartKeys.clear(); + plannedChunkSizes.clear(); + + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); + allPartKeys.add(chunkKey); + confirmedBytesAppended += chunkBytesWritten; handedOff = true; } streamFinished = true; break; } + long currentTotalOffset = baseOffset + totalBytesAppended; + boolean isUploadComplete = + readException == null + && maxAppendSizeException == null + && info.getLength() != null + && currentTotalOffset >= info.getLength(); + // AWS S3 / MinIO Rule: Parts must be >= 5 MB unless it's the final part completing the - // upload + // upload or an incomplete sub-5MB tail chunk if (chunkBytesWritten >= minPartSize) { // Full part chunk (>= 5 MB): Submit to AsyncChunkUploader pipeline for background // upload String chunkKey = buildChunkPartKey(id, nextPartNumber++); plannedPartKeys.add(chunkKey); + plannedChunkSizes.add(chunkBytesWritten); + File fileToUpload = tempChunkFile; long bytesToUpload = chunkBytesWritten; uploader.submitChunk( tempChunkFile, bytesToUpload, chunkKey, - () -> uploadChunkToS3(chunkKey, tempChunkFile, bytesToUpload)); + () -> uploadChunkToS3(chunkKey, fileToUpload, bytesToUpload)); handedOff = true; - } else if (streamFinished && isUploadComplete) { - // Sub-5MB final chunk that completes the overall upload: + } else { + // Sub-5MB chunk (e.g. final completing chunk, or partial chunk when + // paused/interrupted): // First drain all preceding parts in the pipeline int confirmedCount = uploader.drainAndComplete(); - allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); + int confirmed = Math.min(confirmedCount, plannedPartKeys.size()); + for (int i = 0; i < confirmed; i++) { + allPartKeys.add(plannedPartKeys.get(i)); + confirmedBytesAppended += plannedChunkSizes.get(i); + } + plannedPartKeys.clear(); + plannedChunkSizes.clear(); String chunkKey = buildChunkPartKey(id, nextPartNumber++); uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); allPartKeys.add(chunkKey); - handedOff = true; - - } else { - // Sub-5MB incomplete chunk (e.g. upload paused midway or interrupted): - // First drain all preceding parts in the pipeline - int confirmedCount = uploader.drainAndComplete(); - allPartKeys.addAll(plannedPartKeys.subList(0, confirmedCount)); - - storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + confirmedBytesAppended += chunkBytesWritten; handedOff = true; } + } catch (Exception e) { + if (e instanceof IOException && readException == null) { + readException = (IOException) e; + } + streamFinished = true; } finally { - if (!handedOff) { + if (!handedOff && tempChunkFile != null) { FileUtils.deleteQuietly(tempChunkFile); } } } // Drain any remaining in-flight chunks in the pipeline with configured drain timeout - // (defaults to 55s, lock wait - 5s). - // Catch/finally ensures all confirmed parts are retained even if a subsequent chunk times - // out. int confirmedCount = 0; try { confirmedCount = uploader.drainAndComplete(); + } catch (IOException e) { + if (readException == null) { + readException = e; + } } finally { - int confirmed = Math.max(confirmedCount, uploader.getConfirmedCount()); - int previouslyConfirmed = allPartKeys.size() - preparedStream.remainingPartKeys.size(); - if (confirmed > previouslyConfirmed) { - allPartKeys.clear(); - allPartKeys.addAll(preparedStream.remainingPartKeys); - allPartKeys.addAll(plannedPartKeys.subList(0, confirmed)); + int confirmed = + Math.min( + Math.max(confirmedCount, uploader.getConfirmedCount()), plannedPartKeys.size()); + for (int i = 0; i < confirmed; i++) { + allPartKeys.add(plannedPartKeys.get(i)); + confirmedBytesAppended += plannedChunkSizes.get(i); } } + } catch (Exception e) { + if (e instanceof IOException && readException == null) { + readException = (IOException) e; + } } - if (readException != null) { - throw readException; - } - if (maxAppendSizeException != null) { - throw maxAppendSizeException; - } - - return new AppendResult(totalBytesAppended, allPartKeys); + return new AppendResult( + confirmedBytesAppended, allPartKeys, readException, maxAppendSizeException); } private void uploadChunkToS3(String chunkKey, File tempChunkFile, long chunkLength) @@ -1210,78 +1292,26 @@ private void storeIncompletePartToS3(String partObjectKey, File tempChunkFile, l * #mergeUsingStreamingReupload(String, List, long)}, guaranteeing successful upload finalization. */ private void finalizeCompletedUploadIfFinished( - UploadInfo info, String objectKey, UploadId id, AppendResult appendResult, long newOffset) - throws IOException { + UploadInfo info, String objectKey, UploadId id, long newOffset) throws IOException { if (info.getLength() != null && newOffset >= info.getLength()) { - List partKeys = fetchExistingPartKeys(id); - - long existingPartsTotalSize = 0L; - for (String pk : partKeys) { - try { - StatObjectResponse stat = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(pk).build()); - existingPartsTotalSize += stat.size(); - } catch (Exception ignored) { - } - } - - // If leftover sub-5MB .part exists, apply arithmetic budget invariants - String leftoverPartKey = buildIncompletePartKey(id); - if (objectExists(leftoverPartKey)) { - long leftoverSize = 0L; - try { - StatObjectResponse partHead = - minioClient.statObject( - StatObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build()); - if (partHead != null) { - leftoverSize = partHead.size(); - } - } catch (Exception ignored) { - } - - // Arithmetic Decision Matrix: - // Case 1: Numbered parts already satisfy the entire upload length. - // The .part buffer is an orphaned/stale duplicate (e.g. from prior pause). - // MUST DELETE to prevent byte duplication. - if (existingPartsTotalSize >= info.getLength()) { - log.info( - "Purging stale incomplete part {} (size: {}) as numbered parts already cover upload length ({})", - leftoverPartKey, - leftoverSize, - info.getLength()); - deleteObjectQuietly(leftoverPartKey); - - } else if (existingPartsTotalSize + leftoverSize == info.getLength()) { - // Case 2: Numbered parts plus this leftover buffer match the expected length exactly. - // This is a legitimate new tail written during this request. - // MUST PROMOTE to the final numbered part. - int nextPartNum = partKeys.size() + 1; - String finalChunkKey = buildChunkPartKey(id, nextPartNum); - try (InputStream stream = - minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build())) { - byte[] bytes = IOUtils.toByteArray(stream); - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(finalChunkKey).stream( - new ByteArrayInputStream(bytes), (long) bytes.length, -1L) - .build()); - partKeys.add(finalChunkKey); - } catch (Exception e) { - throw new IOException("Failed to finalize incomplete part for ID " + id, e); - } - deleteObjectQuietly(leftoverPartKey); - - } else { - // Case 3: Oversized or inconsistent leftover buffer. - log.warn( - "Discarding inconsistent incomplete part {} (size: {}, numbered parts: {}, total length: {})", - leftoverPartKey, - leftoverSize, - existingPartsTotalSize, - info.getLength()); - deleteObjectQuietly(leftoverPartKey); - } + List partKeys = + info.getUploadPartKeys() != null + ? new ArrayList<>(info.getUploadPartKeys()) + : new ArrayList<>(); + + long sumPartSizes = calculateTotalPartsSize(partKeys); + + // Guard: Enforce strict byte length equality before final object composition. + // This invariant prevents corrupted uploads where duplicate parts or missed chunk offsets + // would cause the final object to exceed or fall short of the declared length. + if (info.getLength() > 0 && sumPartSizes > 0 && sumPartSizes != info.getLength()) { + throw new IOException( + "Upload finalization aborted: sum of part sizes (" + + sumPartSizes + + ") does not match expected length (" + + info.getLength() + + ")"); } if (!partKeys.isEmpty()) { @@ -1305,7 +1335,7 @@ private void finalizeCompletedUploadIfFinished( } } - if (canUseServerSideCompose && s3ComposeObjectSupported) { + if (canUseServerSideCompose && isS3ComposeObjectSupported()) { try { // Perform S3 server-side object composition via native S3 multipart copy s3ComposeHelper.compose(bucket, objectKey, partKeys); @@ -1317,7 +1347,7 @@ private void finalizeCompletedUploadIfFinished( // When server-side compose fails on this S3 endpoint, disable it dynamically // to avoid redundant failing S3 API roundtrips on subsequent uploads, log at INFO, // and seamlessly merge chunks via streaming part composition. - s3ComposeObjectSupported = false; + setS3ComposeObjectSupported(false); log.info( "S3 server-side compose failed for object {}, falling back to streaming part" + " composition: {}", @@ -1335,10 +1365,10 @@ private void finalizeCompletedUploadIfFinished( mergeUsingStreamingReupload(objectKey, partKeys, newOffset); } - // Clean up temporary part chunk objects in S3 - for (String pk : partKeys) { - deleteObjectQuietly(pk); - } + // Clean up temporary part chunk objects in S3 using multi-delete with fallback + deleteS3ObjectsQuietly(partKeys); + info.setUploadPartKeys(null); + } else if (info.getLength() == 0L) { // Zero-byte upload: create the empty destination object in S3 try { @@ -1346,6 +1376,7 @@ private void finalizeCompletedUploadIfFinished( PutObjectArgs.builder().bucket(bucket).object(objectKey).stream( new ByteArrayInputStream(new byte[0]), 0L, -1L) .build()); + info.setUploadPartKeys(null); } catch (Exception e) { throw new IOException("Failed to create empty completed object for ID " + id, e); } @@ -1389,23 +1420,23 @@ private InputStream fetchS3ByteStream(UploadId id, UploadInfo info) GetObjectArgs.builder().bucket(bucket).object(objectKey).build()); } catch (ErrorResponseException e) { if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { - // Step 2: Fallback to reading from incomplete .part object if upload is in-progress - String partKey = buildIncompletePartKey(id); - try { - return minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(partKey).build()); - } catch (ErrorResponseException ex) { - // If the incomplete .part object is also missing (NoSuchKey) and the offset is - // zero or null, it means no bytes have been uploaded yet (e.g. immediately after - // creation). In this case, we return an empty stream rather than throwing an exception. - if (S3Utils.parseErrorResponse(ex) == S3ErrorType.NO_SUCH_KEY) { - if (info != null && (info.getOffset() == null || info.getOffset() == 0L)) { - return new ByteArrayInputStream(new byte[0]); - } + // Step 2: Fallback to reading from manifest part objects if upload is in-progress + List partKeys = + info != null && info.getUploadPartKeys() != null + ? info.getUploadPartKeys() + : Collections.emptyList(); + if (!partKeys.isEmpty()) { + try { + Enumeration inputStreams = + new S3PartInputStreamEnumeration(minioClient, bucket, partKeys); + return new SequenceInputStream(inputStreams); + } catch (Exception ex) { + log.debug("Failed to stream manifest parts for upload ID {}", id, ex); } - log.debug("Failed to read incomplete .part object {}", partKey, ex); - } catch (Exception exception) { - log.debug("Failed to read incomplete .part object {}", partKey, exception); + } + + if (info != null && (info.getOffset() == null || info.getOffset() == 0L)) { + return new ByteArrayInputStream(new byte[0]); } } throw new UploadNotFoundException("Uploaded bytes object not found for ID " + id); @@ -1414,108 +1445,125 @@ private InputStream fetchS3ByteStream(UploadId id, UploadInfo info) } } - private void truncateFromCompletedObject(String objectKey, String partKey, long newOffset) + /** + * Truncates bytes backwards across committed manifest parts in reverse chronological order. + * + *

AWS S3 objects are immutable, so truncation cannot truncate in-place without replacing the + * affected object. If a part's total size is less than or equal to the bytes to remove, the + * entire part object is deleted from S3 and removed from the manifest. If a part's size exceeds + * the bytes to remove, {@link #truncateSinglePart(String, long, long)} rewrites the remaining + * prefix bytes to S3. + * + * @param uploadInfo The upload metadata containing the list of part keys + * @param bytesToRemove The total number of bytes to strip from the tail + */ + private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) { + List partKeys = new ArrayList<>(uploadInfo.getUploadPartKeys()); + long bytesToRemoveRemaining = bytesToRemove; + + while (!partKeys.isEmpty() && bytesToRemoveRemaining > 0) { + String lastKey = partKeys.get(partKeys.size() - 1); + try { + StatObjectResponse stat = + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(lastKey).build()); + long partSize = stat.size(); + + if (bytesToRemoveRemaining >= partSize) { + // The entire part is subsumed by the truncation; delete it and adjust remaining budget + deleteObjectQuietly(lastKey); + partKeys.remove(partKeys.size() - 1); + bytesToRemoveRemaining -= partSize; + } else { + // Only a fraction of this part is being removed; rewrite the remaining prefix + truncateSinglePart(lastKey, partSize, bytesToRemoveRemaining); + bytesToRemoveRemaining = 0; + } + } catch (Exception e) { + log.debug("Error truncating manifest part {}", lastKey, e); + break; + } + } + uploadInfo.setUploadPartKeys(partKeys); + } + + /** + * Partially truncates an individual S3 part object by downloading its content and overwriting it + * in S3 with only the remaining byte prefix. + * + * @param partKey S3 object key of the part to partially truncate + * @param partSize Current total size of the part object + * @param bytesToRemove Number of tail bytes to remove from this specific part + * @throws IOException If reading or re-uploading the truncated part fails + */ + private void truncateSinglePart(String partKey, long partSize, long bytesToRemove) + throws IOException { + try (InputStream partStream = + minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(partKey).build())) { + byte[] bytes = IOUtils.toByteArray(partStream); + int newLength = (int) (partSize - bytesToRemove); + byte[] remaining = Arrays.copyOf(bytes, newLength); + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(partKey).stream( + new ByteArrayInputStream(remaining), (long) remaining.length, -1L) + .build()); + } catch (Exception e) { + throw new IOException("Failed to partially truncate S3 part object " + partKey, e); + } + } + + /** + * Truncates a final completed S3 object by converting the remaining byte prefix into a manifest + * part and deleting the completed object. + * + * @param uploadInfo The upload metadata to update with the new manifest part + * @param objectKey S3 object key of the completed object + * @param newOffset The truncated target byte length + * @throws IOException If downloading or rewriting the object fails + */ + private void truncateCompletedObject(UploadInfo uploadInfo, String objectKey, long newOffset) throws IOException { if (newOffset > 0) { try (InputStream objStream = minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(objectKey).build())) { byte[] remainingBytes = new byte[(int) newOffset]; IOUtils.readFully(objStream, remainingBytes); + String chunkKey = buildChunkPartKey(uploadInfo.getId(), 1); minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(partKey).stream( + PutObjectArgs.builder().bucket(bucket).object(chunkKey).stream( new ByteArrayInputStream(remainingBytes), (long) remainingBytes.length, -1L) .build()); + uploadInfo.setUploadPartKeys(new ArrayList<>(Collections.singletonList(chunkKey))); } catch (Exception e) { throw new IOException("Failed to truncate completed object key " + objectKey, e); } + } else { + uploadInfo.setUploadPartKeys(new ArrayList<>()); } deleteObjectQuietly(objectKey); } - private void truncateFromIncompletePart(String partKey, long byteCount) { - try { - StatObjectResponse head = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(partKey).build()); - long partSize = head.size(); - - if (byteCount >= partSize) { - deleteObjectQuietly(partKey); - } else { - try (InputStream partStream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(partKey).build())) { - byte[] bytes = IOUtils.toByteArray(partStream); - int newLength = (int) (bytes.length - byteCount); - byte[] remaining = Arrays.copyOf(bytes, newLength); - - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(partKey).stream( - new ByteArrayInputStream(remaining), (long) remaining.length, -1L) - .build()); - } - } - } catch (ErrorResponseException ignored) { - } catch (Exception e) { - log.debug("Error truncating incomplete part object {}", partKey, e); - } - } - private void calculateAndSetOffset(UploadInfo info) { if (info == null || info.getId() == null) { return; } String objectKey = getS3ObjectKey(info); - String partKey = buildIncompletePartKey(info.getId()); - - long offset = calculateCurrentOffset(objectKey, info.getId(), partKey, info.getLength()); - info.setOffset(offset); - } - - private long calculateCurrentOffset( - String objectKey, UploadId id, String partKey, Long expectedLength) { - long offset = 0; - if (objectExists(objectKey)) { try { StatObjectResponse head = minioClient.statObject( StatObjectArgs.builder().bucket(bucket).object(objectKey).build()); - offset += head.size(); - } catch (Exception ignored) { - } - } - - List partKeys = fetchExistingPartKeys(id); - long numberedPartsSize = 0L; - for (String pk : partKeys) { - try { - StatObjectResponse stat = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(pk).build()); - numberedPartsSize += stat.size(); - } catch (Exception ignored) { - } - } - offset += numberedPartsSize; - - if (!partKeys.contains(partKey) && objectExists(partKey)) { - try { - StatObjectResponse partHead = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(partKey).build()); - if (partHead != null) { - long partSize = partHead.size(); - if (expectedLength == null || numberedPartsSize + partSize <= expectedLength) { - offset += partSize; - } else if (numberedPartsSize >= expectedLength) { - // Numbered parts already cover the file; partKey is a stale orphan - deleteObjectQuietly(partKey); - } + if (head != null) { + info.setOffset(head.size()); + return; } - } catch (ErrorResponseException ignored) { - } catch (Exception e) { - log.debug("Error reading head for incomplete part object {}", partKey, e); + } catch (Exception ignored) { } } - return offset; + List partKeys = + info.getUploadPartKeys() != null ? info.getUploadPartKeys() : Collections.emptyList(); + long offset = calculateTotalPartsSize(partKeys); + info.setOffset(offset); } private List fetchExistingPartKeys(UploadId id) { @@ -1529,13 +1577,86 @@ private List fetchExistingPartKeys(UploadId id) { } } catch (Exception ignored) { } + Collections.sort(partKeys); return partKeys; } private void deleteAllPartObjectsQuietly(UploadId id) { List partKeys = fetchExistingPartKeys(id); - for (String pk : partKeys) { - deleteObjectQuietly(pk); + deleteS3ObjectsQuietly(partKeys); + } + + /** + * Deletes multiple S3 objects efficiently using batch Multi-Object Delete (POST /?delete), with + * automatic fallback to individual single-object DELETE requests if batch delete is unsupported + * or rejected by bucket IAM permissions. + * + *

If batch delete fails once, it is permanently disabled for this storage service instance to + * avoid unnecessary failing S3 API round-trips. + * + * @param keys The S3 object keys to delete + */ + void deleteS3ObjectsQuietly(Collection keys) { + if (keys == null || keys.isEmpty()) { + return; + } + + if (!isSupportsBatchDelete()) { + deleteObjectsIndividually(keys); + return; + } + + List objects = toDeleteRequestObjects(keys); + if (objects.isEmpty()) { + return; + } + + boolean batchFailed = false; + try { + Iterable> results = + minioClient.removeObjects( + RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); + if (results != null) { + for (Result res : results) { + if (res != null) { + DeleteResult.Error error = res.get(); + if (error != null) { + log.debug( + "Batch delete reported error for key {}: {}; switching to individual delete", + error.objectName(), + error.message()); + batchFailed = true; + break; + } + } + } + } + } catch (Exception e) { + log.info( + "Batch multi-delete failed; permanently switching to individual delete: {}", + e.getMessage()); + batchFailed = true; + } + + if (batchFailed) { + setSupportsBatchDelete(false); + deleteObjectsIndividually(keys); + } + } + + private List toDeleteRequestObjects(Collection keys) { + List objects = new ArrayList<>(); + for (String key : keys) { + if (key != null) { + objects.add(new DeleteRequest.Object(key)); + } + } + return objects; + } + + private void deleteObjectsIndividually(Collection keys) { + for (String key : keys) { + deleteObjectQuietly(key); } } @@ -1631,12 +1752,11 @@ private String buildMetadataKey(UploadId id) { return metadataPrefix + Objects.toString(id) + ".info"; } - private String buildIncompletePartKey(UploadId id) { - return metadataPrefix + Objects.toString(id) + ".part"; - } - private String buildChunkPartKey(UploadId id, int partNumber) { - return metadataPrefix + Objects.toString(id) + ".part." + String.format("%05d", partNumber); + return metadataPrefix + + Objects.toString(id) + + ".part." + + String.format("%05d-%s", partNumber, UUID.randomUUID().toString().substring(0, 8)); } private String buildChecksumKey(String checksum, ChecksumAlgorithm algorithm) { @@ -1653,28 +1773,60 @@ private String buildStopKey(UploadId id) { } private static class AppendResult { - final long totalBytesAppended; + final long confirmedBytesAppended; final List allPartKeys; - - AppendResult(long totalBytesAppended, List allPartKeys) { - this.totalBytesAppended = totalBytesAppended; + final IOException readException; + final MaxAppendSizeExceededException maxAppendSizeException; + + AppendResult( + long confirmedBytesAppended, + List allPartKeys, + IOException readException, + MaxAppendSizeExceededException maxAppendSizeException) { + this.confirmedBytesAppended = confirmedBytesAppended; this.allPartKeys = allPartKeys; + this.readException = readException; + this.maxAppendSizeException = maxAppendSizeException; } } /** - * Helper value object encapsulating an input stream prepended with prior incomplete chunk bytes, - * the number of prepended bytes (for base offset calibration), and the remaining part keys. + * Helper stream wrapper encapsulating an input stream prepended with prior incomplete chunk + * bytes, delegating interruption state to the original incoming stream. */ - private static class PreparedStream { - final InputStream stream; + private static class PreparedStream extends InterruptibleInputStream { + private final InterruptibleInputStream originalStream; final long prependedBytes; + final long existingPartsTotalSize; final List remainingPartKeys; - - PreparedStream(InputStream stream, long prependedBytes, List remainingPartKeys) { - this.stream = stream; + final List staleTailKeys; + + PreparedStream( + InputStream combinedStream, + InterruptibleInputStream originalStream, + long prependedBytes, + long existingPartsTotalSize, + List remainingPartKeys, + List staleTailKeys) { + super(combinedStream); + this.originalStream = originalStream; this.prependedBytes = prependedBytes; + this.existingPartsTotalSize = existingPartsTotalSize; this.remainingPartKeys = remainingPartKeys; + this.staleTailKeys = staleTailKeys; + } + + @Override + public boolean isInterrupted() { + return super.isInterrupted() || (originalStream != null && originalStream.isInterrupted()); + } + + @Override + public void interrupt() { + super.interrupt(); + if (originalStream != null) { + originalStream.interrupt(); + } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index a0519af7..318a7a0b 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -3,6 +3,7 @@ import io.minio.GetObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; import io.minio.RemoveObjectsArgs; import io.minio.Result; import io.minio.errors.ErrorResponseException; @@ -131,15 +132,19 @@ void deleteS3Objects(String firstKey, String secondKey) { if (minioClient == null || bucket == null) { return; } - List objects = - Stream.of(firstKey, secondKey) - .filter(Objects::nonNull) - .map(DeleteRequest.Object::new) - .collect(Collectors.toList()); - if (objects.isEmpty()) { + List keysToDelete = + Stream.of(firstKey, secondKey).filter(Objects::nonNull).collect(Collectors.toList()); + if (keysToDelete.isEmpty()) { return; } + + boolean batchFailed = false; + List objects = + keysToDelete.stream().map(DeleteRequest.Object::new).collect(Collectors.toList()); + try { + // First attempt fast S3 Multi-Object Delete (POST /?delete) in a single round trip. + // Supported by MinIO and compliant S3 endpoints where bucket policies permit multi-delete. Iterable> results = minioClient.removeObjects( RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); @@ -149,16 +154,51 @@ void deleteS3Objects(String firstKey, String secondKey) { if (result != null) { DeleteResult.Error error = result.get(); if (error != null) { - log.debug("Failed to delete S3 object {}: {}", error.objectName(), error.message()); + // S3 returned an error for an object (e.g. AccessDenied on multi-delete POST + // /?delete) + log.debug( + "Batch delete reported error for S3 object {}: {}; falling back to individual" + + " delete", + error.objectName(), + error.message()); + batchFailed = true; + break; } } } catch (Exception e) { - log.debug("Failed to process batch delete result", e); + log.debug( + "Exception iterating batch delete results; falling back to individual delete", e); + batchFailed = true; + break; } } } } catch (Exception e) { - log.debug("Failed to batch delete S3 objects {} and {}", firstKey, secondKey, e); + log.debug( + "Batch multi-delete failed for keys {}; falling back to individual single-object DELETE", + keysToDelete, + e); + batchFailed = true; + } + + // Fall back to resilient individual single-object DELETE calls (DELETE /bucket/key). + // Standard AWS IAM policies often grant s3:DeleteObject exclusively on arn:aws:s3:::bucket/*, + // which allows individual object deletions while rejecting bucket-level multi-object delete. + if (batchFailed) { + for (String key : keysToDelete) { + deleteSingleObject(key); + } + } + } + + void deleteSingleObject(String key) { + if (minioClient == null || bucket == null || key == null) { + return; + } + try { + minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); + } catch (Exception e) { + log.warn("Failed to delete S3 lock object key {} in bucket {}", key, bucket, e); } } diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java index b143fded..14c191be 100644 --- a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -8,6 +8,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import me.desair.tus.server.util.InterruptibleInputStream; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,6 +65,8 @@ public interface ChunkUploadAction { // Number of chunks confirmed uploaded to the cloud private int confirmedCount; + private final InterruptibleInputStream inputStream; + // Set to true once drainAndComplete() has successfully finished private boolean completed; @@ -73,7 +76,7 @@ public interface ChunkUploadAction { * @param executor Shared thread pool executor for background chunk uploads */ public AsyncChunkUploader(ExecutorService executor) { - this(executor, DEFAULT_DRAIN_TIMEOUT_MS); + this(executor, DEFAULT_DRAIN_TIMEOUT_MS, null); } /** @@ -83,8 +86,22 @@ public AsyncChunkUploader(ExecutorService executor) { * @param drainTimeoutMs Maximum duration in milliseconds to drain in-flight chunks */ public AsyncChunkUploader(ExecutorService executor, long drainTimeoutMs) { + this(executor, drainTimeoutMs, null); + } + + /** + * Constructs an uploader using the given shared executor, drain timeout, and interruptible input + * stream. + * + * @param executor Shared thread pool executor for background chunk uploads + * @param drainTimeoutMs Maximum duration in milliseconds to drain in-flight chunks + * @param inputStream The interruptible upload stream to monitor for contention signals + */ + public AsyncChunkUploader( + ExecutorService executor, long drainTimeoutMs, InterruptibleInputStream inputStream) { this.executor = Objects.requireNonNull(executor, "ExecutorService must not be null"); this.drainTimeoutMs = drainTimeoutMs; + this.inputStream = inputStream; } /** @@ -222,6 +239,11 @@ public int drainAndComplete(long timeoutMs) throws IOException { public void abort() { if (inFlightUpload != null && !inFlightUpload.isDone()) { inFlightUpload.cancel(true); + try { + // Wait briefly for worker thread to exit so no background thread outlives the lock + inFlightUpload.get(500, TimeUnit.MILLISECONDS); + } catch (Exception ignored) { + } } if (inFlightFile != null) { FileUtils.deleteQuietly(inFlightFile); @@ -290,7 +312,35 @@ private void checkAndConfirmInFlight() throws IOException { } private void waitForInFlight() throws IOException { - checkAndConfirmInFlight(); + // When waiting under backpressure, poll with contention checks so that if another + // thread signals lock contention (e.g. HEAD or DELETE request), the servlet thread wakes up + // quickly rather than remaining blocked indefinitely on in-flight chunk upload. + while (inFlightUpload != null && !inFlightUpload.isDone()) { + if (isContentionRequested()) { + throw new IOException("Upload was interrupted by locking service contention"); + } + try { + inFlightUpload.get(50, TimeUnit.MILLISECONDS); + break; + } catch (TimeoutException e) { + // Poll tick: loop and re-evaluate contention + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException( + "Interrupted waiting for in-flight chunk upload for " + inFlightKey, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, inFlightKey); + } + } + if (inFlightUpload != null && inFlightUpload.isDone()) { + checkAndConfirmInFlight(); + } + } + + private boolean isContentionRequested() { + return inputStream != null && inputStream.isInterrupted(); } private void awaitFuture(Future future, long timeoutMs, String key) throws IOException { diff --git a/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java b/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java index 7fe2257a..5a39e302 100644 --- a/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java +++ b/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java @@ -2,6 +2,8 @@ import java.io.IOException; import java.io.InputStream; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * An InputStream wrapper that can be interrupted by another thread. When interrupted, it throws an @@ -10,7 +12,8 @@ public class InterruptibleInputStream extends InputStream { private final InputStream delegate; - private volatile boolean interrupted = false; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean interrupted = false; /** * Constructs an interruptible input stream wrapping the given delegate stream. @@ -26,7 +29,7 @@ public InterruptibleInputStream(InputStream delegate) { } private void checkInterrupted() throws IOException { - if (interrupted) { + if (isInterrupted()) { throw new IOException("Stream was interrupted by the upload locking service watchdog"); } } @@ -43,7 +46,12 @@ private void checkInterrupted() throws IOException { * HEAD requests) wait for all in-flight bytes to be committed before reading the upload offset. */ public void interrupt() { - interrupted = true; + thisObjectLock.writeLock().lock(); + try { + interrupted = true; + } finally { + thisObjectLock.writeLock().unlock(); + } try { delegate.close(); } catch (IOException e) { @@ -52,7 +60,12 @@ public void interrupt() { } public boolean isInterrupted() { - return interrupted; + thisObjectLock.readLock().lock(); + try { + return interrupted; + } finally { + thisObjectLock.readLock().unlock(); + } } @Override diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index e884f3c7..906cc477 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -605,6 +605,24 @@ public static void shutdownExecutor(ScheduledExecutorService executor) { } } + /** + * Ensures the given input stream is an {@link InterruptibleInputStream}. If the input stream is + * already an {@code InterruptibleInputStream}, it is cast and returned; otherwise, it is wrapped + * in a new {@code InterruptibleInputStream}. Returns null if the input stream is null. + * + * @param inputStream The input stream to check or wrap + * @return An InterruptibleInputStream wrapping or casting the input stream, or null if null + */ + public static InterruptibleInputStream toInterruptibleStream(InputStream inputStream) { + if (inputStream == null) { + return null; + } + if (inputStream instanceof InterruptibleInputStream) { + return (InterruptibleInputStream) inputStream; + } + return new InterruptibleInputStream(inputStream); + } + /** * Safely interrupts an input stream if it is an instance of {@link InterruptibleInputStream}, or * closes it quietly if it is a standard input stream. Any exceptions encountered during diff --git a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java index 13812ae1..40ce876b 100644 --- a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java @@ -1583,7 +1583,7 @@ public void testLockContentionAndHeadRelease() throws Exception { InputStream blockingStream = new InputStream() { - private volatile boolean closed = false; + private boolean closed = false; @Override public int read() throws IOException { @@ -1727,7 +1727,7 @@ public void testInterruptedUploadPersistsBytesBeforeHeadResponds() throws Except InputStream partialBlockingStream = new InputStream() { private final InputStream byteStream = new ByteArrayInputStream(payload); - private volatile boolean closed = false; + private boolean closed = false; @Override public int read() throws IOException { diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index ff2fd00e..27441b1e 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; +import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; @@ -384,4 +385,50 @@ public void testProcessUninterruptedIoExceptionRethrown() throws Exception { "owner", null); } + + @Test + public void testProcessInterruptedByLockContention() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.setContent("hello world".getBytes()); + + UploadInfo initial = new UploadInfo(); + initial.setId(new UploadId("append-id")); + initial.setOffset(1000L); + initial.setLength(2000L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("append-id")); + refreshed.setOffset(1005L); + refreshed.setLength(2000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")) + .thenReturn(initial) + .thenReturn(refreshed); + + when(storageService.append(eq(initial), any(InputStream.class))) + .thenAnswer( + inv -> { + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Stream interrupted by locking service contention"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(204)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("1005")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index 3863559f..34ad2bf4 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -645,4 +645,48 @@ public void testProcessCreationWithBodyUninterruptedIoException() throws Excepti "owner", null); } + + @Test + public void testProcessCreationInterruptedByLockContention() throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.setContent("creation body".getBytes()); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("creation-id")); + info.setLength(1000L); + info.setOffset(0L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("creation-id")); + refreshed.setLength(1000L); + refreshed.setOffset(13L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))).thenReturn(info); + when(storageService.getUploadInfo("/files/creation-id", "owner")).thenReturn(refreshed); + + when(storageService.append(eq(info), any(InputStream.class))) + .thenAnswer( + inv -> { + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Stream interrupted by locking service contention"); + }); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("13")); + } } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java index 38098a52..bc7715c3 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java @@ -150,7 +150,7 @@ public void testRenewLeaseFailureInterruptsActiveStreamAndShutsDownExecutor() { LeaseData leaseData = new LeaseData( - "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() + 10000L); + "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() - 1000L); TestLeaseLock lock = new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { @@ -167,4 +167,61 @@ protected void doRenewLease() { assertTrue(stream.isInterrupted()); verify(mockExecutor).shutdownNow(); } + + @Test + public void testRenewLeaseTransientFailureDoesNotInterruptActiveStreamIfLeaseValid() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + Map activeStreams = new ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new ByteArrayInputStream("test".getBytes())); + activeStreams.put("/files/upload-failing", stream); + + LeaseData leaseData = + new LeaseData( + "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() + 10000L); + + TestLeaseLock lock = + new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { + @Override + protected void doRenewLease() { + throw new RuntimeException("Simulated transient network glitch"); + } + }; + + // Trigger renewLease and verify that transient glitch does not abort while lease is still valid + lock.renewLease(); + + org.junit.Assert.assertFalse(stream.isInterrupted()); + org.mockito.Mockito.verify(mockExecutor, org.mockito.Mockito.never()).shutdownNow(); + } + + @Test + public void testRenewLeaseOwnershipLostInterruptsActiveStreamAndShutsDownExecutor() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + Map activeStreams = new ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new ByteArrayInputStream("test".getBytes())); + activeStreams.put("/files/upload-lost", stream); + + LeaseData leaseData = + new LeaseData( + "holder-lost", "/files/upload-lost", 10000L, System.currentTimeMillis() + 10000L); + + TestLeaseLock lock = + new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { + @Override + protected void doRenewLease() { + throw new IllegalStateException("Lease taken over by rival holder"); + } + }; + + lock.renewLease(); + + assertTrue(stream.isInterrupted()); + verify(mockExecutor).shutdownNow(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java b/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java index a209f3fb..9d7371f5 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java @@ -282,4 +282,23 @@ public void testGetCreatorIpAddressesWithXforwardedFor() throws Exception { UploadInfo info = new UploadInfo(servletRequest); assertThat(info.getCreatorIpAddresses(), is("24.23.22.21, 192.168.1.1, 10.11.12.13")); } + + @Test + public void testUploadPartKeysEqualsAndHashCode() { + UploadInfo info1 = new UploadInfo(); + UploadInfo info2 = new UploadInfo(); + + assertThat(info1.getUploadPartKeys(), nullValue()); + assertEquals(info1, info2); + assertEquals(info1.hashCode(), info2.hashCode()); + + java.util.List parts = java.util.Arrays.asList("part-1", "part-2"); + info1.setUploadPartKeys(parts); + assertNotEquals(info1, info2); + + info2.setUploadPartKeys(new java.util.ArrayList<>(parts)); + assertEquals(info1, info2); + assertEquals(info1.hashCode(), info2.hashCode()); + assertEquals(parts, info1.getUploadPartKeys()); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index 8c2f33da..66aaec08 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -297,4 +297,20 @@ public void testCloseGracefulShutdown() throws Exception { Thread.interrupted(); // Clear interrupted status } } + + @Test + public void testGenerateBlockIdEntropyAndFormat() { + String blockId0 = storageService.generateBlockId(0); + String blockId1 = storageService.generateBlockId(0); + + assertNotNull(blockId0); + assertNotNull(blockId1); + // Two block IDs generated for the same index must have different UUID entropy + org.junit.Assert.assertNotEquals(blockId0, blockId1); + + byte[] decoded0 = java.util.Base64.getDecoder().decode(blockId0); + String text0 = new String(decoded0, java.nio.charset.StandardCharsets.UTF_8); + assertTrue(text0.startsWith("blk-000000-")); + assertEquals(19, text0.length()); // "blk-000000-" (11) + 8 hex chars = 19 + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java index 7ee5988f..ddef9786 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java @@ -5,9 +5,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.specialized.BlobLeaseClient; import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import java.util.concurrent.ScheduledExecutorService; @@ -139,11 +143,166 @@ void executeRenew() { } }; - // Verify that failed renewal immediately aborts the active stream + // When the lease has already expired, renewal failure immediately aborts + lock.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); lock.renewLease(); org.junit.Assert.assertTrue(stream.isInterrupted()); org.junit.Assert.assertFalse(activeStreams.containsKey("upload-failing")); verify(mockExecutor).shutdownNow(); } + + @Test + public void testRenewLeaseTransientFailureTolerated() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new java.io.ByteArrayInputStream("test".getBytes())); + activeStreams.put("upload-transient", new java.lang.ref.WeakReference<>(stream)); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, + null, + "/test/upload/upload-transient", + mockExecutor, + "upload-transient", + activeStreams) { + @Override + void executeRenew() { + throw new RuntimeException("Simulated transient network glitch"); + } + }; + + // Lease valid for another 30 seconds: transient blip must NOT abort + lock.setLeaseExpiresAt(System.currentTimeMillis() + 30000L); + lock.renewLease(); + + org.junit.Assert.assertFalse(stream.isInterrupted()); + org.junit.Assert.assertTrue(activeStreams.containsKey("upload-transient")); + org.mockito.Mockito.verify(mockExecutor, org.mockito.Mockito.never()).shutdownNow(); + } + + @Test + public void testRenewLeaseLostInterruptsActiveStream() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new java.io.ByteArrayInputStream("test".getBytes())); + activeStreams.put("upload-lost", new java.lang.ref.WeakReference<>(stream)); + + HttpResponse response = mock(HttpResponse.class); + org.mockito.Mockito.when(response.getStatusCode()).thenReturn(409); + HttpHeaders headers = new HttpHeaders(); + headers.set("x-ms-error-code", BlobErrorCode.LEASE_NOT_PRESENT_WITH_LEASE_OPERATION.toString()); + org.mockito.Mockito.when(response.getHeaders()).thenReturn(headers); + + BlobStorageException leaseLostException = + new BlobStorageException( + "Lease lost", response, BlobErrorCode.LEASE_NOT_PRESENT_WITH_LEASE_OPERATION); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, null, "/test/upload/upload-lost", mockExecutor, "upload-lost", activeStreams) { + @Override + void executeRenew() { + throw leaseLostException; + } + }; + + // Even if lease was theoretically valid, leaseLost aborts immediately + lock.setLeaseExpiresAt(System.currentTimeMillis() + 30000L); + lock.renewLease(); + + org.junit.Assert.assertTrue(stream.isInterrupted()); + org.junit.Assert.assertFalse(activeStreams.containsKey("upload-lost")); + verify(mockExecutor).shutdownNow(); + } + + @Test + public void testRenewLeaseConflictErrorInterruptsStream() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new java.io.ByteArrayInputStream("test".getBytes())); + activeStreams.put("upload-conflict", new java.lang.ref.WeakReference<>(stream)); + + HttpResponse response = mock(HttpResponse.class); + org.mockito.Mockito.when(response.getStatusCode()).thenReturn(409); + HttpHeaders headers = new HttpHeaders(); + headers.set("x-ms-error-code", "Conflict"); + org.mockito.Mockito.when(response.getHeaders()).thenReturn(headers); + + BlobStorageException conflictException = new BlobStorageException("Conflict", response, null); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, + null, + "/test/upload/upload-conflict", + mockExecutor, + "upload-conflict", + activeStreams) { + @Override + void executeRenew() { + throw conflictException; + } + }; + + lock.setLeaseExpiresAt(System.currentTimeMillis() + 30000L); + lock.renewLease(); + + org.junit.Assert.assertTrue(stream.isInterrupted()); + org.junit.Assert.assertFalse(activeStreams.containsKey("upload-conflict")); + verify(mockExecutor).shutdownNow(); + } + + @Test + public void testRenewLeaseLostWithoutActiveStreamHandlesCleanly() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + + // activeStreams and uploadId both null + AzureBlobUploadLock lockWithoutStreams = + new AzureBlobUploadLock(null, null, "/test/upload/upload-null", mockExecutor, null, null) { + @Override + void executeRenew() { + throw new RuntimeException("Renewal failed"); + } + }; + + lockWithoutStreams.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); + lockWithoutStreams.renewLease(); + verify(mockExecutor).shutdownNow(); + + // activeStreams present but no stream registered for uploadId + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + ScheduledExecutorService mockExecutor2 = mock(ScheduledExecutorService.class); + + AzureBlobUploadLock lockWithEmptyStreams = + new AzureBlobUploadLock( + null, null, "/test/upload/upload-empty", mockExecutor2, "upload-empty", activeStreams) { + @Override + void executeRenew() { + throw new RuntimeException("Renewal failed"); + } + }; + + lockWithEmptyStreams.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); + lockWithEmptyStreams.renewLease(); + verify(mockExecutor2).shutdownNow(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index f067aa38..a94fde12 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -914,6 +914,34 @@ public void testEvictExpiredLockHandlesRemoveObjectException() throws Exception assertFalse(evicted); } + @Test + public void testEvictExpiredLockHandlesErrorResponseExceptionNonNoSuchKey() throws Exception { + UploadId uploadId = new UploadId("test-access-denied"); + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("AccessDenied"); + ErrorResponseException accessDeniedEx = + new ErrorResponseException(errorResponse, null, "AccessDenied"); + + Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))).thenThrow(accessDeniedEx); + + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertFalse(evicted); + } + + @Test + public void testEvictExpiredLockHandlesNoSuchKeyReturnsTrue() throws Exception { + UploadId uploadId = new UploadId("test-no-such-key"); + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); + ErrorResponseException noSuchKeyEx = + new ErrorResponseException(errorResponse, null, "NoSuchKey"); + + Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))).thenThrow(noSuchKeyEx); + + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertTrue(evicted); + } + @Test public void testWithJitterConfiguresBoundsAndReturnsSelf() { S3LockingService returned = lockingService.withJitter(15L, 75L); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index f0d5dc29..7d3ea5a4 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -22,6 +22,7 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; import io.minio.Result; import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; @@ -34,8 +35,10 @@ import java.io.InputStream; import java.nio.file.Paths; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinUploadLengthNotReachedException; @@ -273,14 +276,15 @@ public void testAppendExceedsMaxAppendSizePreservesIncompletePartAndUpdatesOffse info.setId(uploadId); info.setLength(10000L); info.setOffset(20L); + String partKey1 = "metadata/" + uploadId + ".part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(partKey1))); String json = UploadInfoJsonSerializer.serialize(info); when(minioClient.getObject(any(GetObjectArgs.class))) .thenAnswer( invocation -> { GetObjectArgs args = invocation.getArgument(0); - // Simulate existing .part buffer object of 20 bytes - if (args.object().endsWith(".part")) { + if (args.object().equals(partKey1)) { return mockGetObjectResponse(new byte[20]); } return mockGetObjectResponse(json.getBytes()); @@ -292,7 +296,7 @@ public void testAppendExceedsMaxAppendSizePreservesIncompletePartAndUpdatesOffse .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part")) { + if (args.object().equals(partKey1)) { return partStat; } throw new ErrorResponseException( @@ -310,8 +314,9 @@ public void testAppendExceedsMaxAppendSizePreservesIncompletePartAndUpdatesOffse } // Verify that when maxAppendSize was exceeded, the prepended bytes were NOT lost - // and were flushed back to S3 as an incomplete .part object. - verify(minioClient, atLeastOnce()).putObject(argThat(args -> args.object().endsWith(".part"))); + // and were flushed back to S3 as a part chunk object. + verify(minioClient, atLeastOnce()) + .putObject(argThat(args -> args != null && args.object().contains(".part."))); } @Test(expected = MinUploadLengthNotReachedException.class) @@ -418,6 +423,7 @@ public void testAppendInterruptedMultiPart() throws Exception { info.setId(new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e")); info.setLength(200000000L); info.setOffset(0L); + info.setUploadPartKeys(new ArrayList<>()); String json = UploadInfoJsonSerializer.serialize(info); when(minioClient.getObject(any(GetObjectArgs.class))) @@ -974,129 +980,6 @@ public void testFinalizeCompletedUploadWithMultipleParts() throws Exception { storageService.append(info, new ByteArrayInputStream(new byte[100])); } - @Test - public void testFinalizeCompletedUploadWithLeftoverIncompletePart() throws Exception { - UploadInfo info = new UploadInfo(); - UploadId id = new UploadId("leftover-part-123"); - info.setId(id); - info.setLength(50L); - info.setOffset(0L); - - String json = UploadInfoJsonSerializer.serialize(info); - - StatObjectResponse leftoverHead = mock(StatObjectResponse.class); - when(leftoverHead.size()).thenReturn(50L); - - when(minioClient.statObject(any(StatObjectArgs.class))) - .thenAnswer( - invocation -> { - StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part")) { - return leftoverHead; - } - ErrorResponse err = mock(ErrorResponse.class); - when(err.code()).thenReturn("NoSuchKey"); - throw new ErrorResponseException(err, null, null); - }); - - when(minioClient.getObject(any(GetObjectArgs.class))) - .thenAnswer( - invocation -> { - GetObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".info")) { - return mockGetObjectResponse(json.getBytes()); - } - return mockGetObjectResponse(new byte[50]); - }); - - storageService.append(info, new ByteArrayInputStream(new byte[50])); - } - - @Test - public void testFinalizeCompletedUploadWithInconsistentLeftoverPartPurged() throws Exception { - UploadInfo info = new UploadInfo(); - UploadId id = new UploadId("inconsistent-part-123"); - info.setId(id); - info.setLength(100L); - info.setOffset(0L); - - String json = UploadInfoJsonSerializer.serialize(info); - - StatObjectResponse leftoverHead = mock(StatObjectResponse.class); - // Leftover is 30 bytes, but total length is 100 bytes (0 parts + 30 != 100) -> Case 3 - when(leftoverHead.size()).thenReturn(30L); - - when(minioClient.statObject(any(StatObjectArgs.class))) - .thenAnswer( - invocation -> { - StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part")) { - return leftoverHead; - } - ErrorResponse err = mock(ErrorResponse.class); - when(err.code()).thenReturn("NoSuchKey"); - throw new ErrorResponseException(err, null, null); - }); - - when(minioClient.getObject(any(GetObjectArgs.class))) - .thenAnswer( - invocation -> { - GetObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".info")) { - return mockGetObjectResponse(json.getBytes()); - } - return mockGetObjectResponse(new byte[100]); - }); - - storageService.append(info, new ByteArrayInputStream(new byte[100])); - - // Verify inconsistent .part buffer was deleted - verify(minioClient, atLeastOnce()) - .removeObject( - argThat( - (RemoveObjectArgs args) -> - args.object().equals("metadata/inconsistent-part-123.part"))); - } - - @Test(expected = IOException.class) - public void testFinalizeCompletedUploadLeftoverPartPromotionExceptionThrowsIOException() - throws Exception { - UploadInfo info = new UploadInfo(); - UploadId id = new UploadId("leftover-promo-err-123"); - info.setId(id); - info.setLength(50L); - info.setOffset(0L); - - String json = UploadInfoJsonSerializer.serialize(info); - - StatObjectResponse leftoverHead = mock(StatObjectResponse.class); - when(leftoverHead.size()).thenReturn(50L); - - when(minioClient.statObject(any(StatObjectArgs.class))) - .thenAnswer( - invocation -> { - StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part")) { - return leftoverHead; - } - ErrorResponse err = mock(ErrorResponse.class); - when(err.code()).thenReturn("NoSuchKey"); - throw new ErrorResponseException(err, null, null); - }); - - when(minioClient.getObject(any(GetObjectArgs.class))) - .thenAnswer( - invocation -> { - GetObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".info")) { - return mockGetObjectResponse(json.getBytes()); - } - throw new IOException("Simulated network failure streaming leftover part"); - }); - - storageService.append(info, new ByteArrayInputStream(new byte[50])); - } - @Test public void testFinalizeCompletedUploadZeroLength() throws Exception { UploadInfo info = new UploadInfo(); @@ -1212,6 +1095,8 @@ public void testFetchS3ByteStreamIncompletePartFallback() throws Exception { UploadId id = new UploadId("fallback-123"); info.setId(id); info.setOffset(50L); + String partKey1 = "metadata/fallback-123.part.00001"; + info.setUploadPartKeys(Collections.singletonList(partKey1)); ErrorResponse noSuchKeyErr = mock(ErrorResponse.class); when(noSuchKeyErr.code()).thenReturn("NoSuchKey"); @@ -1224,7 +1109,7 @@ public void testFetchS3ByteStreamIncompletePartFallback() throws Exception { if (args.object().endsWith(".info")) { return mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes()); } - if (args.object().endsWith(".part")) { + if (args.object().equals(partKey1)) { return mockGetObjectResponse("part-data".getBytes()); } throw noSuchKeyEx; @@ -1647,21 +1532,12 @@ public void testFinalizeCompletedUploadStreamingReuploadWhenSub5MbIntermediatePa UploadId id = new UploadId("sub5mb-test-123"); info.setId(id); info.setLength(2000L); - info.setOffset(0L); + info.setOffset(2000L); + info.setUploadPartKeys( + Arrays.asList("uploads/sub5mb-test-123.part.00001", "uploads/sub5mb-test-123.part.00002")); String json = UploadInfoJsonSerializer.serialize(info); - Item item1 = mock(Item.class); - when(item1.objectName()).thenReturn("uploads/sub5mb-test-123.part.00001"); - when(item1.size()).thenReturn(1000L); - - Item item2 = mock(Item.class); - when(item2.objectName()).thenReturn("uploads/sub5mb-test-123.part.00002"); - when(item2.size()).thenReturn(1000L); - - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); - StatObjectResponse stat1 = mock(StatObjectResponse.class); when(stat1.size()).thenReturn(1000L); StatObjectResponse stat2 = mock(StatObjectResponse.class); @@ -1671,9 +1547,9 @@ public void testFinalizeCompletedUploadStreamingReuploadWhenSub5MbIntermediatePa .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part.00001")) { + if (args.object().contains(".part.00001")) { return stat1; - } else if (args.object().endsWith(".part.00002")) { + } else if (args.object().contains(".part.00002")) { return stat2; } ErrorResponse err = mock(ErrorResponse.class); @@ -1691,7 +1567,7 @@ public void testFinalizeCompletedUploadStreamingReuploadWhenSub5MbIntermediatePa return mockGetObjectResponse(new byte[1000]); }); - UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[2000])); + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); assertNotNull(result); org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.never()) .composeObject(any(ComposeObjectArgs.class)); @@ -1707,21 +1583,13 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA long fiveMb = 5L * 1024L * 1024L; info.setId(id); info.setLength(fiveMb + 100L); - info.setOffset(0L); + info.setOffset(fiveMb + 100L); + info.setUploadPartKeys( + Arrays.asList( + "uploads/compose-test-123.part.00001", "uploads/compose-test-123.part.00002")); String json = UploadInfoJsonSerializer.serialize(info); - Item item1 = mock(Item.class); - when(item1.objectName()).thenReturn("uploads/compose-test-123.part.00001"); - when(item1.size()).thenReturn(fiveMb); - - Item item2 = mock(Item.class); - when(item2.objectName()).thenReturn("uploads/compose-test-123.part.00002"); - when(item2.size()).thenReturn(100L); - - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); - StatObjectResponse stat1 = mock(StatObjectResponse.class); when(stat1.size()).thenReturn(fiveMb); StatObjectResponse stat2 = mock(StatObjectResponse.class); @@ -1731,9 +1599,9 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part.00001")) { + if (args.object().contains(".part.00001")) { return stat1; - } else if (args.object().endsWith(".part.00002")) { + } else if (args.object().contains(".part.00002")) { return stat2; } ErrorResponse err = mock(ErrorResponse.class); @@ -1751,8 +1619,7 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA return mockGetObjectResponse(new byte[100]); }); - UploadInfo result = - storageService.append(info, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); assertNotNull(result); org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) .composeObject(any(ComposeObjectArgs.class)); @@ -1768,21 +1635,14 @@ public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAnd long fiveMb = 5L * 1024L * 1024L; info.setId(id); info.setLength(fiveMb + 100L); - info.setOffset(0L); + info.setOffset(fiveMb + 100L); + info.setUploadPartKeys( + Arrays.asList( + "uploads/compose-fail-test-123.part.00001", + "uploads/compose-fail-test-123.part.00002")); String json = UploadInfoJsonSerializer.serialize(info); - Item item1 = mock(Item.class); - when(item1.objectName()).thenReturn("uploads/compose-fail-test-123.part.00001"); - when(item1.size()).thenReturn(fiveMb); - - Item item2 = mock(Item.class); - when(item2.objectName()).thenReturn("uploads/compose-fail-test-123.part.00002"); - when(item2.size()).thenReturn(100L); - - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); - StatObjectResponse stat1 = mock(StatObjectResponse.class); when(stat1.size()).thenReturn(fiveMb); StatObjectResponse stat2 = mock(StatObjectResponse.class); @@ -1792,9 +1652,9 @@ public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAnd .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part.00001")) { + if (args.object().contains(".part.00001")) { return stat1; - } else if (args.object().endsWith(".part.00002")) { + } else if (args.object().contains(".part.00002")) { return stat2; } ErrorResponse err = mock(ErrorResponse.class); @@ -1815,8 +1675,7 @@ public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAnd when(minioClient.composeObject(any(ComposeObjectArgs.class))) .thenThrow(new RuntimeException("The specified header is not valid in this context")); - UploadInfo result = - storageService.append(info, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); assertNotNull(result); // Verify composeObject was attempted and failed @@ -1833,19 +1692,13 @@ public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAnd UploadId id2 = new UploadId("compose-skip-test-456"); info2.setId(id2); info2.setLength(fiveMb + 100L); - info2.setOffset(0L); + info2.setOffset(fiveMb + 100L); + info2.setUploadPartKeys( + Arrays.asList( + "uploads/compose-skip-test-456.part.00001", + "uploads/compose-skip-test-456.part.00002")); String json2 = UploadInfoJsonSerializer.serialize(info2); - Item item21 = mock(Item.class); - when(item21.objectName()).thenReturn("uploads/compose-skip-test-456.part.00001"); - when(item21.size()).thenReturn(fiveMb); - - Item item22 = mock(Item.class); - when(item22.objectName()).thenReturn("uploads/compose-skip-test-456.part.00002"); - when(item22.size()).thenReturn(100L); - - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenReturn(Arrays.asList(new Result<>(item21), new Result<>(item22))); when(minioClient.getObject(any(GetObjectArgs.class))) .thenAnswer( @@ -1857,8 +1710,7 @@ public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAnd return mockGetObjectResponse(new byte[100]); }); - UploadInfo result2 = - storageService.append(info2, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + UploadInfo result2 = storageService.append(info2, new ByteArrayInputStream(new byte[0])); assertNotNull(result2); // composeObject count should still be 1 (never called for the second upload) @@ -1867,7 +1719,6 @@ public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAnd // Reset flag for other tests storageService.setS3ComposeObjectSupported(true); - assertTrue(storageService.isS3ComposeObjectSupported()); } @Test @@ -1903,6 +1754,18 @@ public void testPrepareStreamRollsBackSub5MbNumberedPart() throws Exception { return java.util.Collections.emptyList(); }); + java.util.concurrent.atomic.AtomicBoolean partSaved = + new java.util.concurrent.atomic.AtomicBoolean(false); + when(minioClient.putObject(any(PutObjectArgs.class))) + .thenAnswer( + inv -> { + PutObjectArgs args = inv.getArgument(0); + if (args.object().endsWith(".part")) { + partSaved.set(true); + } + return null; + }); + StatObjectResponse stat1 = mock(StatObjectResponse.class); when(stat1.size()).thenReturn(500L); @@ -1913,9 +1776,9 @@ public void testPrepareStreamRollsBackSub5MbNumberedPart() throws Exception { .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part.00001")) { + if (args.object().contains(".part.00001")) { return stat1; - } else if (args.object().endsWith(".part")) { + } else if (args.object().endsWith(".part") && partSaved.get()) { return statPart; } ErrorResponse err = mock(ErrorResponse.class); @@ -2147,166 +2010,17 @@ public void testCloseGracefulShutdown() throws Exception { } } - @Test - public void testFinalizeUploadWithStaleIncompletePartIgnored() throws Exception { - UploadId uploadId = new UploadId("stale-part-test"); - UploadInfo info = new UploadInfo(); - info.setId(uploadId); - info.setOffset(0L); - info.setLength(3425070L); // 3.42 MB total upload length - - String infoJson = UploadInfoJsonSerializer.serialize(info); - String partKey1 = "metadata/stale-part-test.part.00001"; - String stalePartKey = "metadata/stale-part-test.part"; - - Item part1Item = mock(Item.class); - when(part1Item.objectName()).thenReturn(partKey1); - - StatObjectResponse part1Stat = mock(StatObjectResponse.class); - when(part1Stat.size()).thenReturn(3425070L); // Numbered part already covers full 3.42 MB - - StatObjectResponse stalePartStat = mock(StatObjectResponse.class); - when(stalePartStat.size()).thenReturn(1277952L); // Stale leftover from prior pause - - StatObjectResponse objectNotExists = mock(StatObjectResponse.class); - - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenAnswer( - invocation -> { - ListObjectsArgs args = invocation.getArgument(0); - if (args.prefix().startsWith("metadata/stale-part-test.part.")) { - return Collections.singletonList(new Result<>(part1Item)); - } - return Collections.emptyList(); - }); - - when(minioClient.statObject(any(StatObjectArgs.class))) - .thenAnswer( - invocation -> { - StatObjectArgs args = invocation.getArgument(0); - if (args.object().equals(partKey1)) { - return part1Stat; - } else if (args.object().equals(stalePartKey)) { - return stalePartStat; - } else if (args.object().equals("uploads/stale-part-test")) { - ErrorResponse err = mock(ErrorResponse.class); - when(err.code()).thenReturn("NoSuchKey"); - throw new ErrorResponseException(err, null, null); - } - return objectNotExists; - }); - - when(minioClient.getObject(any(GetObjectArgs.class))) - .thenAnswer( - invocation -> { - GetObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".info")) { - return mockGetObjectResponse(infoJson.getBytes()); - } - return mockGetObjectResponse(new byte[0]); - }); - - // Append 0 bytes to trigger finalization check - ByteArrayInputStream emptyStream = new ByteArrayInputStream(new byte[0]); - UploadInfo result = storageService.append(info, emptyStream); - - assertNotNull(result); - // Verify that the stale .part buffer was deleted via removeObject and NOT promoted to - // part.00002 - verify(minioClient, atLeastOnce()) - .removeObject( - argThat( - (RemoveObjectArgs args) -> args.object().equals("metadata/stale-part-test.part"))); - } - - @Test - public void testFinalizeUploadWithLegitimateNewIncompletePartPromoted() throws Exception { - UploadId uploadId = new UploadId("legit-part-test"); - UploadInfo info = new UploadInfo(); - info.setId(uploadId); - info.setOffset(0L); - info.setLength(10000000L); // 10 MB total length - - String infoJson = UploadInfoJsonSerializer.serialize(info); - String partKey1 = "metadata/legit-part-test.part.00001"; - String legitPartKey = "metadata/legit-part-test.part"; - - Item part1Item = mock(Item.class); - when(part1Item.objectName()).thenReturn(partKey1); - - StatObjectResponse part1Stat = mock(StatObjectResponse.class); - when(part1Stat.size()).thenReturn(8000000L); // 8 MB numbered part - - StatObjectResponse legitPartStat = mock(StatObjectResponse.class); - when(legitPartStat.size()).thenReturn(2000000L); // 2 MB tail; 8 MB + 2 MB == 10 MB exact match! - - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenAnswer( - invocation -> { - ListObjectsArgs args = invocation.getArgument(0); - if (args.prefix().startsWith("metadata/legit-part-test.part.")) { - return Collections.singletonList(new Result<>(part1Item)); - } - return Collections.emptyList(); - }); - - when(minioClient.statObject(any(StatObjectArgs.class))) - .thenAnswer( - invocation -> { - StatObjectArgs args = invocation.getArgument(0); - if (args.object().equals(partKey1)) { - return part1Stat; - } else if (args.object().equals(legitPartKey)) { - return legitPartStat; - } else if (args.object().equals("uploads/legit-part-test")) { - ErrorResponse err = mock(ErrorResponse.class); - when(err.code()).thenReturn("NoSuchKey"); - throw new ErrorResponseException(err, null, null); - } - return mock(StatObjectResponse.class); - }); - - when(minioClient.getObject(any(GetObjectArgs.class))) - .thenAnswer( - invocation -> { - GetObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".info")) { - return mockGetObjectResponse(infoJson.getBytes()); - } else if (args.object().equals(legitPartKey)) { - return mockGetObjectResponse(new byte[2000000]); - } - return mockGetObjectResponse(new byte[0]); - }); - - ByteArrayInputStream emptyStream = new ByteArrayInputStream(new byte[0]); - UploadInfo result = storageService.append(info, emptyStream); - - assertNotNull(result); - // Verify that the legitimate 2 MB tail was promoted to part.00002 - verify(minioClient) - .putObject( - argThat( - (PutObjectArgs args) -> - args.object().equals("metadata/legit-part-test.part.00002"))); - } - @Test public void testCalculateCurrentOffsetDoesNotDoubleCountOrExceedLength() throws Exception { UploadId uploadId = new UploadId("calc-offset-test"); - String infoJson = "{\"id\":\"calc-offset-test\",\"length\":3425070,\"offset\":null}"; + String infoJson = + "{\"id\":\"calc-offset-test\",\"length\":3425070,\"offset\":null,\"uploadPartKeys\":[\"metadata/calc-offset-test.part.00001\"]}"; String partKey1 = "metadata/calc-offset-test.part.00001"; - String stalePartKey = "metadata/calc-offset-test.part"; - - Item part1Item = mock(Item.class); - when(part1Item.objectName()).thenReturn(partKey1); StatObjectResponse part1Stat = mock(StatObjectResponse.class); when(part1Stat.size()).thenReturn(3425070L); // 3.42 MB numbered part - StatObjectResponse stalePartStat = mock(StatObjectResponse.class); - when(stalePartStat.size()).thenReturn(1277952L); // Stale 1.27 MB leftover - when(minioClient.getObject(any(GetObjectArgs.class))) .thenAnswer( invocation -> { @@ -2317,24 +2031,12 @@ public void testCalculateCurrentOffsetDoesNotDoubleCountOrExceedLength() throws return mockGetObjectResponse(new byte[0]); }); - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenAnswer( - invocation -> { - ListObjectsArgs args = invocation.getArgument(0); - if (args.prefix().startsWith("metadata/calc-offset-test.part.")) { - return Collections.singletonList(new Result<>(part1Item)); - } - return Collections.emptyList(); - }); - when(minioClient.statObject(any(StatObjectArgs.class))) .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); if (args.object().equals(partKey1)) { return part1Stat; - } else if (args.object().equals(stalePartKey)) { - return stalePartStat; } else if (args.object().equals("uploads/calc-offset-test")) { ErrorResponse err = mock(ErrorResponse.class); when(err.code()).thenReturn("NoSuchKey"); @@ -2345,14 +2047,7 @@ public void testCalculateCurrentOffsetDoesNotDoubleCountOrExceedLength() throws UploadInfo fetched = storageService.getUploadInfo(uploadId); assertNotNull(fetched); - // Must be exactly 3,425,070 bytes (not 4,703,022 bytes with double-counted stale part!) assertEquals(Long.valueOf(3425070L), fetched.getOffset()); - - // Verify stale .part buffer was deleted on the fly - verify(minioClient, atLeastOnce()) - .removeObject( - argThat( - (RemoveObjectArgs args) -> args.object().equals("metadata/calc-offset-test.part"))); } @Test @@ -2419,6 +2114,511 @@ public void testValidateRemainingPartBudget() throws Exception { } } + @Test + public void testRemoveLastNumberOfBytesWithManifestParts() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("trunc-manifest-test")); + info.setLength(300L); + info.setOffset(300L); + info.setUploadPartKeys( + new ArrayList<>(Arrays.asList("metadata/trunc.part.00001", "metadata/trunc.part.00002"))); + + String json = UploadInfoJsonSerializer.serialize(info); + + StatObjectResponse stat1 = mock(StatObjectResponse.class); + when(stat1.size()).thenReturn(150L); + StatObjectResponse stat2 = mock(StatObjectResponse.class); + when(stat2.size()).thenReturn(150L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals("metadata/trunc.part.00002")) { + return stat2; + } else if (args.object().equals("metadata/trunc.part.00001")) { + return stat1; + } else if (args.object().equals("uploads/trunc-manifest-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return mock(StatObjectResponse.class); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + return mockGetObjectResponse(new byte[150]); + }); + + // Remove 200 bytes: all 150 bytes of part 2, plus 50 bytes of part 1 + storageService.removeLastNumberOfBytes(info, 200L); + + assertEquals(Long.valueOf(100L), info.getOffset()); + assertEquals(1, info.getUploadPartKeys().size()); + assertEquals("metadata/trunc.part.00001", info.getUploadPartKeys().get(0)); + + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/trunc.part.00002"))); + verify(minioClient) + .putObject( + argThat((PutObjectArgs args) -> args.object().equals("metadata/trunc.part.00001"))); + } + + @Test + public void testDeleteS3ObjectsBatchFailureFallback() throws Exception { + assertTrue(storageService.isSupportsBatchDelete()); + UploadId uploadId = new UploadId("batch-del-fallback"); + List keys = Arrays.asList("metadata/batch.part.00001", "metadata/batch.part.00002"); + + io.minio.messages.DeleteResult.Error delErr = mock(io.minio.messages.DeleteResult.Error.class); + when(delErr.objectName()).thenReturn("metadata/batch.part.00001"); + when(delErr.message()).thenReturn("AccessDenied"); + + @SuppressWarnings("unchecked") + Result errResult = mock(Result.class); + when(errResult.get()).thenReturn(delErr); + + when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenReturn(Collections.singletonList(errResult)); + + storageService.deleteS3ObjectsQuietly(keys); + + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00001"))); + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00002"))); + assertFalse(storageService.isSupportsBatchDelete()); + + // Subsequent batch delete should bypass removeObjects directly + storageService.deleteS3ObjectsQuietly(Collections.singletonList("metadata/batch.part.00003")); + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00003"))); + // removeObjects was only called once (for the first attempt) + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .removeObjects(any(RemoveObjectsArgs.class)); + + // Reset flag for subsequent tests + storageService.setSupportsBatchDelete(true); + } + + @Test + public void testDeleteS3ObjectsThrowsExceptionFallsBackToIndividualDelete() throws Exception { + assertTrue(storageService.isSupportsBatchDelete()); + UploadId uploadId = new UploadId("batch-del-ex"); + List keys = Collections.singletonList("metadata/batch-ex.part.00001"); + + when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenThrow(new RuntimeException("MinIO network failure")); + + storageService.deleteS3ObjectsQuietly(keys); + + verify(minioClient) + .removeObject( + argThat( + (RemoveObjectArgs args) -> args.object().equals("metadata/batch-ex.part.00001"))); + assertFalse(storageService.isSupportsBatchDelete()); + + // Null or empty handling + storageService.deleteS3ObjectsQuietly(null); + storageService.deleteS3ObjectsQuietly(Collections.emptyList()); + storageService.deleteS3ObjectsQuietly(Collections.singletonList(null)); + + // Reset flag + storageService.setSupportsBatchDelete(true); + } + + @Test + public void testGetUploadedBytesFromManifestPartsWhenCompletedObjectMissing() throws Exception { + UploadId uploadId = new UploadId("manifest-stream-test"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(12L); + info.setOffset(12L); + info.setUploadPartKeys( + Arrays.asList( + "metadata/manifest-stream-test.part.00001", + "metadata/manifest-stream-test.part.00002")); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/manifest-stream-test.info".equals(a.object())))) + .thenReturn( + mockGetObjectResponse( + UploadInfoJsonSerializer.serialize(info) + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + ErrorResponse err = new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "uploads/manifest-stream-test".equals(a.object())))) + .thenThrow(new ErrorResponseException(err, null, null)); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/manifest-stream-test.part.00001".equals(a.object())))) + .thenReturn( + mockGetObjectResponse("Hello ".getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/manifest-stream-test.part.00002".equals(a.object())))) + .thenReturn( + mockGetObjectResponse("World!".getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + try (InputStream stream = storageService.getUploadedBytes(uploadId)) { + assertNotNull(stream); + byte[] readBytes = org.apache.commons.io.IOUtils.toByteArray(stream); + assertEquals("Hello World!", new String(readBytes, java.nio.charset.StandardCharsets.UTF_8)); + } + } + + @Test + public void testAppendWithInterruptibleInputStream() throws Exception { + UploadId uploadId = new UploadId("interrupt-stream-append"); + UploadInfo initial = new UploadInfo(); + initial.setId(uploadId); + initial.setLength(10L); + initial.setOffset(null); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/interrupt-stream-append.info".equals(a.object())))) + .thenAnswer( + inv -> + mockGetObjectResponse( + UploadInfoJsonSerializer.serialize(initial) + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + UploadInfo created = storageService.create(initial, "owner"); + assertNotNull(created.getOffset()); + assertEquals(0L, created.getOffset().longValue()); + + byte[] data = "12345".getBytes(java.nio.charset.StandardCharsets.UTF_8); + try (me.desair.tus.server.util.InterruptibleInputStream is = + new me.desair.tus.server.util.InterruptibleInputStream(new ByteArrayInputStream(data))) { + UploadInfo updated = storageService.append(created, is); + assertEquals(5L, updated.getOffset().longValue()); + } + } + + @Test + public void testRemoveLastNumberOfBytesManifestPartPutObjectException() throws Exception { + UploadId uploadId = new UploadId("trunc-put-err"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(10L); + info.setUploadPartKeys( + new ArrayList<>(Collections.singletonList("metadata/trunc-put-err.part.00001"))); + + ErrorResponse noSuchKeyErr = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.statObject( + argThat( + (StatObjectArgs args) -> + args != null && "uploads/trunc-put-err".equals(args.object())))) + .thenThrow(new ErrorResponseException(noSuchKeyErr, null, null)); + + StatObjectResponse stat = mock(StatObjectResponse.class); + when(stat.size()).thenReturn(10L); + when(minioClient.statObject( + argThat( + (StatObjectArgs args) -> + args != null && "metadata/trunc-put-err.part.00001".equals(args.object())))) + .thenReturn(stat); + + byte[] partData = new byte[10]; + when(minioClient.getObject( + argThat( + (GetObjectArgs args) -> + args != null && "metadata/trunc-put-err.part.00001".equals(args.object())))) + .thenReturn(mockGetObjectResponse(partData)); + + doThrow(new RuntimeException("Simulated PutObject failure")) + .when(minioClient) + .putObject( + argThat( + (PutObjectArgs args) -> + args != null && "metadata/trunc-put-err.part.00001".equals(args.object()))); + + storageService.removeLastNumberOfBytes(info, 3L); + assertEquals(7L, info.getOffset().longValue()); + } + + @Test + public void testSupportsBatchDeleteConfiguration() { + assertTrue(storageService.isSupportsBatchDelete()); + storageService.setSupportsBatchDelete(false); + assertFalse(storageService.isSupportsBatchDelete()); + storageService.setSupportsBatchDelete(true); + assertTrue(storageService.isSupportsBatchDelete()); + } + + @Test + public void testCloseInterruptedGracefully() throws Exception { + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testRemoveLastNumberOfBytesTruncatesCompletedObject() throws Exception { + UploadId uploadId = new UploadId("trunc-completed-1"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + + String completedKey = "uploads/trunc-completed-1"; + StatObjectResponse headStat = mock(StatObjectResponse.class); + when(headStat.size()).thenReturn(100L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(headStat); + + byte[] fullBytes = new byte[100]; + for (int i = 0; i < 100; i++) { + fullBytes[i] = (byte) i; + } + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(mockGetObjectResponse(fullBytes)); + + storageService.removeLastNumberOfBytes(info, 20L); + + // 80 bytes should be rewritten as part 1, and completedKey should be deleted + verify(minioClient) + .putObject( + argThat( + (PutObjectArgs a) -> + a != null && a.object().startsWith("metadata/trunc-completed-1.part.00001-"))); + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs a) -> a != null && completedKey.equals(a.object()))); + assertEquals(1, info.getUploadPartKeys().size()); + assertTrue( + info.getUploadPartKeys().get(0).startsWith("metadata/trunc-completed-1.part.00001-")); + } + + @Test + public void testRemoveLastNumberOfBytesTruncatesCompletedObjectToZero() throws Exception { + UploadId uploadId = new UploadId("trunc-completed-zero"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + + String completedKey = "uploads/trunc-completed-zero"; + StatObjectResponse headStat = mock(StatObjectResponse.class); + when(headStat.size()).thenReturn(100L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(headStat); + + storageService.removeLastNumberOfBytes(info, 100L); + + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs a) -> a != null && completedKey.equals(a.object()))); + assertTrue(info.getUploadPartKeys().isEmpty()); + } + + @Test(expected = IOException.class) + public void testRemoveLastNumberOfBytesTruncatesCompletedObjectThrowsIOExceptionOnReadFailure() + throws Exception { + UploadId uploadId = new UploadId("trunc-completed-err"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + + String completedKey = "uploads/trunc-completed-err"; + StatObjectResponse headStat = mock(StatObjectResponse.class); + when(headStat.size()).thenReturn(100L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(headStat); + + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenThrow(new RuntimeException("Simulated read failure")); + + storageService.removeLastNumberOfBytes(info, 20L); + } + + @Test + public void testRemoveLastNumberOfBytesTruncatesSinglePartRewrite() throws Exception { + UploadId uploadId = new UploadId("trunc-single-part"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + String part1Key = "uploads/trunc-single-part.part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(part1Key))); + + // completedKey does not exist + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.statObject( + argThat( + (StatObjectArgs a) -> a != null && "uploads/trunc-single-part".equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + StatObjectResponse partStat = mock(StatObjectResponse.class); + when(partStat.size()).thenReturn(100L); + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(partStat); + + byte[] partData = new byte[100]; + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(mockGetObjectResponse(partData)); + + storageService.removeLastNumberOfBytes(info, 30L); + + // Part should be rewritten with 70 bytes + verify(minioClient) + .putObject(argThat((PutObjectArgs a) -> a != null && part1Key.equals(a.object()))); + assertEquals(1, info.getUploadPartKeys().size()); + } + + @Test + public void testRemoveLastNumberOfBytesDeletesEntirePartWhenSubsumed() throws Exception { + UploadId uploadId = new UploadId("trunc-subsumed"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + String part1Key = "uploads/trunc-subsumed.part.00001"; + String part2Key = "uploads/trunc-subsumed.part.00002"; + info.setUploadPartKeys(new ArrayList<>(Arrays.asList(part1Key, part2Key))); + + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.statObject( + argThat( + (StatObjectArgs a) -> a != null && "uploads/trunc-subsumed".equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + StatObjectResponse statPart1 = mock(StatObjectResponse.class); + when(statPart1.size()).thenReturn(50L); + StatObjectResponse statPart2 = mock(StatObjectResponse.class); + when(statPart2.size()).thenReturn(50L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(statPart1); + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && part2Key.equals(a.object())))) + .thenReturn(statPart2); + + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(mockGetObjectResponse(new byte[50])); + + // Removing 70 bytes should completely remove part2 (50 bytes) and truncate part1 by 20 bytes + storageService.removeLastNumberOfBytes(info, 70L); + + verify(minioClient) + .removeObject(argThat((RemoveObjectArgs a) -> a != null && part2Key.equals(a.object()))); + verify(minioClient) + .putObject(argThat((PutObjectArgs a) -> a != null && part1Key.equals(a.object()))); + assertEquals(1, info.getUploadPartKeys().size()); + assertEquals(part1Key, info.getUploadPartKeys().get(0)); + } + + @Test + public void testFetchS3ByteStreamFallbackToManifestParts() throws Exception { + UploadId uploadId = new UploadId("stream-fallback-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(50L); + String part1Key = "uploads/stream-fallback-id.part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(part1Key))); + + String completedKey = "uploads/stream-fallback-id"; + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(mockGetObjectResponse("streamed-bytes".getBytes())); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/stream-fallback-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + InputStream stream = storageService.getUploadedBytes(uploadId); + assertNotNull(stream); + byte[] content = stream.readAllBytes(); + assertEquals("streamed-bytes", new String(content)); + } + + @Test + public void testPruneOrphanPartsDeletesUnmanifestedS3Part() throws Exception { + UploadId uploadId = new UploadId("orphan-test-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000L); + info.setOffset(0L); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/orphan-test-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + Item manifestedItem = mock(Item.class); + when(manifestedItem.objectName()).thenReturn("metadata/orphan-test-id.part.00001"); + Item orphanItem = mock(Item.class); + when(orphanItem.objectName()).thenReturn("metadata/orphan-test-id.part.99999"); + + Result res1 = new Result<>(manifestedItem); + Result res2 = new Result<>(orphanItem); + + when(minioClient.listObjects( + argThat( + (ListObjectsArgs a) -> + a != null && "metadata/orphan-test-id.part.".equals(a.prefix())))) + .thenReturn(Arrays.asList(res1, res2)); + + storageService.append(info, new ByteArrayInputStream(new byte[0])); + + verify(minioClient) + .removeObject( + argThat( + (RemoveObjectArgs a) -> + a != null && "metadata/orphan-test-id.part.99999".equals(a.object()))); + } + private GetObjectResponse mockGetObjectResponse(byte[] bytes) { return new GetObjectResponse( null, "test-bucket", "eu-central-1", "object-key", new ByteArrayInputStream(bytes)); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java index 4ea4d024..63d5b831 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java @@ -393,4 +393,65 @@ public void testDeleteS3ObjectsWhenMinioClientThrows() { lock.deleteS3Objects("tus-locks/upload-1.lock", "tus-locks/upload-1.stop"); assertEquals("my-holder", lock.getHolderId()); } + + @Test + public void testDeleteSingleObjectEdgeCases() throws Exception { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap); + + lock.deleteSingleObject(null); + + S3UploadLock nullClientLock = + new S3UploadLock(myLeaseData, null, "test-bucket", "k1", "k2", inputStreamMap); + nullClientLock.deleteSingleObject("k1"); + + S3UploadLock nullBucketLock = + new S3UploadLock(myLeaseData, minioClient, null, "k1", "k2", inputStreamMap); + nullBucketLock.deleteSingleObject("k1"); + + Mockito.doThrow(new RuntimeException("removeObject error")) + .when(minioClient) + .removeObject(any(RemoveObjectArgs.class)); + lock.deleteSingleObject("tus-locks/upload-1.lock"); + assertEquals("my-holder", lock.getHolderId()); + } + + @Test + public void testDeleteS3ObjectsResultIteratorExceptionFallback() throws Exception { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap); + + @SuppressWarnings("unchecked") + io.minio.Result mockResult = mock(io.minio.Result.class); + Mockito.when(mockResult.get()) + .thenThrow(new RuntimeException("Simulated error while reading delete result")); + + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenReturn(java.util.Collections.singletonList(mockResult)); + + lock.deleteS3Objects("tus-locks/upload-1.lock", "tus-locks/upload-1.stop"); + + Mockito.verify(minioClient) + .removeObject( + org.mockito.ArgumentMatchers.argThat( + (RemoveObjectArgs args) -> args.object().equals("tus-locks/upload-1.lock"))); + Mockito.verify(minioClient) + .removeObject( + org.mockito.ArgumentMatchers.argThat( + (RemoveObjectArgs args) -> args.object().equals("tus-locks/upload-1.stop"))); + } } diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java index 85cc42ce..71c2354d 100644 --- a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -5,6 +5,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -19,6 +20,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import me.desair.tus.server.util.InterruptibleInputStream; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -551,7 +553,8 @@ public void testSubmitChunkBackpressureInterruptedThrowsIOException() throws Exc // Third chunk triggers waitForInFlight() backpressure uploader.submitChunk(f3, 1, "k3", () -> {}); } catch (IOException e) { - if (e.getMessage().contains("Interrupted while checking in-flight chunk upload")) { + if (e.getMessage().contains("Interrupted") + && e.getMessage().contains("in-flight chunk upload")) { gotBackpressureInterruptedException.set(true); } } catch (Exception ignored) { @@ -602,4 +605,180 @@ public void testDefaultDrainTimeoutConstant() { assertEquals(55_000L, uploader.getDrainTimeoutMs()); } } + + @Test + public void testSubmitChunkBackpressureContentionSignalThrowsIOException() throws Exception { + File f1 = new File(tempDir, "contention-c1.tmp"); + File f2 = new File(tempDir, "contention-c2.tmp"); + File f3 = new File(tempDir, "contention-c3.tmp"); + Files.write(f1.toPath(), "data-1".getBytes()); + Files.write(f2.toPath(), "data-2".getBytes()); + Files.write(f3.toPath(), "data-3".getBytes()); + + CountDownLatch f1Started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(bais); + + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(executor, 5000L, interruptibleStream)) { + uploader.submitChunk( + f1, + 6, + "k1", + () -> { + f1Started.countDown(); + try { + unblock.await(3, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + uploader.submitChunk(f2, 6, "k2", () -> {}); + assertTrue(f1Started.await(3, TimeUnit.SECONDS)); + + // Signal contention before 3rd submitChunk which requires waiting + interruptibleStream.interrupt(); + + try { + uploader.submitChunk(f3, 6, "k3", () -> {}); + fail("Expected IOException due to lock contention"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("contention")); + } + } finally { + unblock.countDown(); + } + } + + @Test + public void testAbortWithActiveInFlightUpload() throws Exception { + File chunk1 = new File(tempDir, "abort-active-c1.tmp"); + Files.write(chunk1.toPath(), "data-abort".getBytes()); + + CountDownLatch uploadStarted = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + + AsyncChunkUploader uploader = new AsyncChunkUploader(executor); + uploader.submitChunk( + chunk1, + 10, + "k-abort", + () -> { + uploadStarted.countDown(); + try { + unblock.await(5, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue(uploadStarted.await(3, TimeUnit.SECONDS)); + // Calling abort while worker is running cancels inFlightUpload and waits up to 500ms + uploader.abort(); + unblock.countDown(); + uploader.close(); + } + + @Test + public void testWaitForInFlightExecutionExceptionPropagated() throws Exception { + File f1 = new File(tempDir, "exec-err-1.tmp"); + File f2 = new File(tempDir, "exec-err-2.tmp"); + File f3 = new File(tempDir, "exec-err-3.tmp"); + Files.write(f1.toPath(), "err1".getBytes()); + Files.write(f2.toPath(), "err2".getBytes()); + Files.write(f3.toPath(), "err3".getBytes()); + + CountDownLatch f1Started = new CountDownLatch(1); + CountDownLatch triggerFailure = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 4, + "k-fail-1", + () -> { + f1Started.countDown(); + try { + triggerFailure.await(5, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + throw new RuntimeException("S3 simulated upload failure"); + }); + + assertTrue(f1Started.await(3, TimeUnit.SECONDS)); + // Submit slot 2 while slot 1 is blocked in latch + uploader.submitChunk(f2, 4, "k-fail-2", () -> {}); + + // In a background thread release triggerFailure after 100ms so failure happens while + // waitForInFlight polls + Executors.newSingleThreadScheduledExecutor() + .schedule(triggerFailure::countDown, 100, TimeUnit.MILLISECONDS); + + // Submitting 3rd chunk triggers waitForInFlight() for slot 1 which throws ExecutionException + try { + uploader.submitChunk(f3, 4, "k-fail-3", () -> {}); + fail("Expected IOException when in-flight upload threw ExecutionException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("S3 simulated upload failure")); + } + } + } + + @Test + public void testAbortWithActiveInFlightUploadWaitsAndCancels() throws Exception { + File f1 = new File(tempDir, "abort-f1.tmp"); + Files.write(f1.toPath(), new byte[] {1, 2, 3}); + + CountDownLatch taskStarted = new CountDownLatch(1); + CountDownLatch hangLatch = new CountDownLatch(1); + + AsyncChunkUploader uploader = new AsyncChunkUploader(executor); + uploader.submitChunk( + f1, + 3, + "k-abort", + () -> { + taskStarted.countDown(); + try { + hangLatch.await(10, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue(taskStarted.await(3, TimeUnit.SECONDS)); + uploader.abort(); + + hangLatch.countDown(); + assertFalse("File should be cleaned up by abort", f1.exists()); + } + + @Test + public void testWaitForInFlightUploadTimeoutPolling() throws Exception { + File f1 = new File(tempDir, "poll-f1.tmp"); + File f2 = new File(tempDir, "poll-f2.tmp"); + File f3 = new File(tempDir, "poll-f3.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + Files.write(f3.toPath(), new byte[] {3}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k-poll-1", + () -> { + try { + Thread.sleep(120); + } catch (InterruptedException ignored) { + } + }); + + uploader.submitChunk(f2, 1, "k-poll-2", () -> {}); + + // Submitting 3rd chunk triggers waitForInFlightUpload(), which will encounter + // TimeoutException + // on the 50ms tick before succeeding on the 3rd tick. + uploader.submitChunk(f3, 1, "k-poll-3", () -> {}); + uploader.drainAndComplete(3000); + } + } } diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index 1443dab7..a8d36f72 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -4,10 +4,12 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import jakarta.servlet.http.HttpServletRequest; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.Serializable; import java.nio.channels.FileChannel; @@ -971,6 +973,19 @@ public void testExtractMediaType() { is("application/offset+octet-stream")); } + @Test + public void testToInterruptibleStream() throws Exception { + assertThat(Utils.toInterruptibleStream(null), is(nullValue())); + + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[] {1, 2, 3}); + InterruptibleInputStream wrapped = Utils.toInterruptibleStream(bais); + assertThat(wrapped, is(notNullValue())); + assertThat(wrapped.read(), is(1)); + + InterruptibleInputStream same = Utils.toInterruptibleStream(wrapped); + assertThat(same, is(sameInstance(wrapped))); + } + /** Simple serializable class for testing. */ public static class TestSerializable implements Serializable { private static final long serialVersionUID = 1L; From 80e1988466096f682c7eac9d9e5f4ce5b921a522 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 10 Oct 2026 11:55:33 +0200 Subject: [PATCH 09/15] test: increase test coverage for S3 and Azure storage services --- .../upload/azure/AzureBlobLockingService.java | 7 +- .../upload/azure/AzureBlobStorageService.java | 6 +- .../server/upload/s3/S3StorageService.java | 6 +- .../upload/util/AsyncChunkUploader.java | 2 +- .../AzureBlobConcatenationServiceTest.java | 20 ++ .../azure/AzureBlobStorageServiceTest.java | 62 +++- .../server/upload/azure/AzureUtilsTest.java | 8 + .../azure/ITAzureBlobLockingService.java | 26 ++ .../azure/ITAzureBlobStorageService.java | 47 ++++ .../upload/s3/S3ConcatenationServiceTest.java | 7 + .../s3/S3ServerSideComposeHelperTest.java | 63 +++++ .../upload/s3/S3StorageServiceTest.java | 264 ++++++++++++++++++ .../upload/util/AsyncChunkUploaderTest.java | 36 +++ 13 files changed, 541 insertions(+), 13 deletions(-) diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index c14c5bc7..9a62c624 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -150,7 +150,8 @@ public UploadLock lockUploadByUri(String requestUri) throws TusException, IOExce AzureErrorType errorType = AzureUtils.parseErrorResponse(e); if (errorType == AzureErrorType.LEASE_ALREADY_PRESENT || errorType == AzureErrorType.CONFLICT) { - log.info("Lock contention for upload URI {}: Azure blob lease is already held", requestUri); + log.debug( + "Lock contention for upload URI {}: Azure blob lease is already held", requestUri); throw new UploadAlreadyLockedException( "Upload with URI " + requestUri + " is currently locked"); } @@ -210,7 +211,7 @@ private void interruptLocalStream(String idStr) { } /** Creates a .stop signal blob to request remote pods to halt active streaming appends. */ - private void createStopSignalBlob(String idStr) { + void createStopSignalBlob(String idStr) { try { BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); stopBlob.upload(BinaryData.fromString("stop"), true); @@ -220,7 +221,7 @@ private void createStopSignalBlob(String idStr) { } /** Deletes the .stop signal blob after lock acquisition. */ - private void deleteStopSignalBlob(String idStr) { + void deleteStopSignalBlob(String idStr) { try { BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); stopBlob.deleteIfExists(); diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 63c038d7..63f35521 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -893,7 +893,7 @@ private List getCommittedBlockIds(BlockBlobClient blockBlobClient) { } /** Validates effective max append size limit. */ - private void validateMaxAppendSize(long totalAppended, Long effectiveMaxAppendSize) + void validateMaxAppendSize(long totalAppended, Long effectiveMaxAppendSize) throws MaxAppendSizeExceededException { if (effectiveMaxAppendSize != null && totalAppended > effectiveMaxAppendSize) { throw new MaxAppendSizeExceededException( @@ -905,7 +905,7 @@ private void validateMaxAppendSize(long totalAppended, Long effectiveMaxAppendSi } /** Validates min append size limit. */ - private void validateMinAppendSize(long totalAppended) throws MinAppendSizeNotMetException { + void validateMinAppendSize(long totalAppended) throws MinAppendSizeNotMetException { if (minAppendSize != null && totalAppended < minAppendSize) { throw new MinAppendSizeNotMetException( "Append size " + totalAppended + " is less than minimum allowed of " + minAppendSize); @@ -1037,7 +1037,7 @@ private String sanitizePrefix(String prefix) { } /** Deletes local file suppressing exceptions. */ - private void deleteFileQuietly(File file) { + void deleteFileQuietly(File file) { if (file != null && file.exists()) { try { Files.delete(file.toPath()); diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 3a35d073..1a92e11f 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -1542,7 +1542,7 @@ private void truncateCompletedObject(UploadInfo uploadInfo, String objectKey, lo deleteObjectQuietly(objectKey); } - private void calculateAndSetOffset(UploadInfo info) { + void calculateAndSetOffset(UploadInfo info) { if (info == null || info.getId() == null) { return; } @@ -1794,7 +1794,7 @@ private static class AppendResult { * Helper stream wrapper encapsulating an input stream prepended with prior incomplete chunk * bytes, delegating interruption state to the original incoming stream. */ - private static class PreparedStream extends InterruptibleInputStream { + static class PreparedStream extends InterruptibleInputStream { private final InterruptibleInputStream originalStream; final long prependedBytes; final long existingPartsTotalSize; @@ -1834,7 +1834,7 @@ public void interrupt() { * Enumeration that lazily queries and opens S3 input streams for a list of part keys, enabling * streaming re-upload concatenation without holding all parts in memory. */ - private static class S3PartInputStreamEnumeration implements Enumeration { + static class S3PartInputStreamEnumeration implements Enumeration { private final MinioClient minioClient; private final String bucket; private final List partKeys; diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java index 14c191be..6f6aa912 100644 --- a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -293,7 +293,7 @@ private void submitToSlot3( }); } - private void checkAndConfirmInFlight() throws IOException { + void checkAndConfirmInFlight() throws IOException { try { inFlightUpload.get(); confirmedCount++; diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java index 464f526f..875edeaa 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java @@ -277,4 +277,24 @@ public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasGenerationThrows() String result = concatenationService.getAuthorizedBlobUrl(mockBlob); assertEquals("https://account.blob.core.windows.net/container/blob", result); } + + @Test + public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasIsEmpty() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())).thenReturn(""); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals("https://account.blob.core.windows.net/container/blob", result); + } + + @Test + public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasIsNull() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())).thenReturn(null); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals("https://account.blob.core.windows.net/container/blob", result); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index 66aaec08..27dfdb53 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -1,6 +1,7 @@ package me.desair.tus.server.upload.azure; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -8,6 +9,7 @@ import com.azure.storage.blob.BlobContainerClientBuilder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -287,12 +289,12 @@ public void testDrainTimeoutConfiguration() { public void testCloseGracefulShutdown() throws Exception { // Verify closing storage service cleanly terminates background upload executor without error storageService.close(); - // KISS: verifying method executes cleanly without throwing an exception - // Verify close handles thread interruption gracefully + // Verify close handles thread interruption gracefully on an active executor + AzureBlobStorageService interruptedService = new AzureBlobStorageService(containerClient); Thread.currentThread().interrupt(); try { - storageService.close(); + interruptedService.close(); } finally { Thread.interrupted(); // Clear interrupted status } @@ -313,4 +315,58 @@ public void testGenerateBlockIdEntropyAndFormat() { assertTrue(text0.startsWith("blk-000000-")); assertEquals(19, text0.length()); // "blk-000000-" (11) + 8 hex chars = 19 } + + @Test(expected = me.desair.tus.server.exception.MaxAppendSizeExceededException.class) + public void testValidateMaxAppendSizeThrowsWhenExceeded() throws Exception { + storageService.validateMaxAppendSize(100L, 50L); + } + + @Test + public void testValidateMaxAppendSizePassesWhenWithinLimit() throws Exception { + storageService.validateMaxAppendSize(50L, 100L); + storageService.validateMaxAppendSize(50L, null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test(expected = me.desair.tus.server.exception.MinAppendSizeNotMetException.class) + public void testValidateMinAppendSizeThrowsWhenNotMet() throws Exception { + storageService.setMinAppendSize(100L); + storageService.validateMinAppendSize(50L); + } + + @Test + public void testValidateMinAppendSizePassesWhenMet() throws Exception { + storageService.setMinAppendSize(100L); + storageService.validateMinAppendSize(150L); + storageService.setMinAppendSize(null); + storageService.validateMinAppendSize(10L); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testDeleteFileQuietly() throws Exception { + // Null and non-existent file are no-ops + storageService.deleteFileQuietly(null); + storageService.deleteFileQuietly(new File("/path/to/non/existent/file/tus.tmp")); + + // Normal file deletion + Path tempFile = Files.createTempFile("tus-delete-quietly", ".tmp"); + assertTrue(Files.exists(tempFile)); + storageService.deleteFileQuietly(tempFile.toFile()); + assertFalse(Files.exists(tempFile)); + + // Non-empty directory causes Files.delete to throw DirectoryNotEmptyException, caught quietly + Path tempDir = Files.createTempDirectory("tus-delete-quietly-dir"); + Path childFile = Files.createTempFile(tempDir, "child", ".tmp"); + try { + storageService.deleteFileQuietly(tempDir.toFile()); + assertTrue( + "Directory should still exist because deletion failed quietly", Files.exists(tempDir)); + } finally { + Files.deleteIfExists(childFile); + Files.deleteIfExists(tempDir); + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java index 067e4be0..6102383d 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java @@ -93,4 +93,12 @@ public void testParseErrorResponseErrorCodeStringMatching() { BlobStorageException exLeaseMissing = createException(409, BlobErrorCode.LEASE_ID_MISSING); assertEquals(AzureErrorType.LEASE_NOT_PRESENT, AzureUtils.parseErrorResponse(exLeaseMissing)); } + + @Test + public void testParseErrorResponseErrorCodeThrowsExceptionHandled() { + HttpResponse response = mock(HttpResponse.class); + when(response.getHeaders()).thenThrow(new RuntimeException("Simulated error code failure")); + BlobStorageException ex = new BlobStorageException("Test exception", response, null); + assertEquals(AzureErrorType.UNKNOWN, AzureUtils.parseErrorResponse(ex)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java index d85d88d6..3c79345a 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java @@ -203,4 +203,30 @@ public void lockUploadByUriShouldThrowIOExceptionOnStorageException() throws Exc service.lockUploadByUri("/test/upload/12345"); } + + @Test + public void testCreateAndDeleteStopSignalBlob() { + lockingService.createStopSignalBlob("test-stop-id"); + com.azure.storage.blob.BlobClient stopBlob = + containerClient.getBlobClient("locks/test-stop-id.stop"); + assertTrue(Boolean.TRUE.equals(stopBlob.exists())); + + lockingService.deleteStopSignalBlob("test-stop-id"); + assertFalse(Boolean.TRUE.equals(stopBlob.exists())); + } + + @Test + public void testStopSignalBlobCatchBlocksHandledGracefully() { + com.azure.storage.blob.BlobServiceClient serviceClient = containerClient.getServiceClient(); + BlobContainerClient nonExistentContainer = + serviceClient.getBlobContainerClient("non-existent-" + System.nanoTime()); + AzureBlobLockingService failingService = new AzureBlobLockingService(nonExistentContainer); + + // Verified against Azurite that non-existent container calls fail fast and are handled quietly + failingService.createStopSignalBlob("fail-stop"); + failingService.deleteStopSignalBlob("fail-stop"); + failingService.ensureLockBlobExists(nonExistentContainer.getBlobClient("fail.lock")); + // KISS: verifying methods execute cleanly without throwing uncaught exceptions + assertTrue(true); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 1273e67e..2ea74777 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -898,4 +898,51 @@ public void testAppendRecoversPartialBlocksOnMaxAppendSizeExceeded() throws Exce UploadInfo reloaded = storageService.getUploadInfo(created.getId()); assertNotNull(reloaded); } + + @Test + public void testAppendTruncatesCommittedBlocksWhenExceedingAuthoritativeMetadata() + throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(500L); + UploadInfo created = storageService.create(info, "owner1"); + + // Upload first 100 bytes + storageService.append(created, new ByteArrayInputStream(new byte[100])); + // Upload second 100 bytes + storageService.append(created, new ByteArrayInputStream(new byte[100])); + + UploadInfo reloaded = storageService.getUploadInfo(created.getId()); + assertNotNull(reloaded.getUploadPartKeys()); + assertEquals(2, reloaded.getUploadPartKeys().size()); + assertEquals(Long.valueOf(200L), reloaded.getOffset()); + + // Simulate scenario where authoritative metadata records only 1 committed block + reloaded.getUploadPartKeys().remove(1); + reloaded.setOffset(100L); + storageService.update(reloaded); + + // Now call append again: blockIds in Azure blob (2) > uploadPartKeys in metadata (1) + // This exercises reconciliation logic where committed blocks are aligned with authoritative + // metadata + UploadInfo appended = storageService.append(reloaded, new ByteArrayInputStream(new byte[50])); + + assertEquals(Long.valueOf(150L), appended.getOffset()); + assertEquals(2, appended.getUploadPartKeys().size()); + } + + @Test + public void testTerminateUploadWithUnleasedLockBlob() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(100L); + UploadInfo created = storageService.create(info, "owner1"); + + // Create an unleased lock blob to verify safe cleanup of unleased lock blobs + BlobClient lockBlob = containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(BinaryData.fromString("lock-content"), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + storageService.terminateUpload(created); + + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index df8255c6..4d518f5e 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -543,4 +543,11 @@ public void testMergeBothNullOwnerKeys() throws Exception { assertEquals(Long.valueOf(10L * 1024 * 1024), parent.getLength()); assertEquals(Long.valueOf(10L * 1024 * 1024), parent.getOffset()); } + + @Test + public void testSetS3ServerSideComposeHelper() { + S3ServerSideComposeHelper mockHelper = Mockito.mock(S3ServerSideComposeHelper.class); + concatenationService.setS3ServerSideComposeHelper(mockHelper); + assertNotNull(concatenationService); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java index 3dff378f..552c7b0f 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java @@ -358,6 +358,69 @@ public void testNativeComposeAbortFailureHandledQuietly() throws Exception { } } + @Test + public void testExplicitConnectionParametersInitializationFailureHandledGracefully() { + // When endpoint contains invalid syntax that triggers an exception during MinioAsyncClient + // creation, the constructor catches the exception and logs a warning gracefully without + // throwing. + S3ServerSideComposeHelper helper = + new S3ServerSideComposeHelper( + "http://:::invalid:::", "us-east-1", "accessKey", "secretKey"); + assertFalse(helper.isAvailable()); + } + + @Test + public void testNativeComposeRegionFallbackToLocal() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", null, null); + when(provider.fetch()).thenReturn(creds); + + // Endpoint without AWS region domain (e.g. localhost) so baseUrl.region() is empty + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("http://localhost:9000")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + ObjectWriteResponse completeResponse = + new ObjectWriteResponse(null, "my-bucket", "", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + String partXml = "\"part-etag-1\""; + ResponseBody body = ResponseBody.create(partXml, MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("http://localhost:9000").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + setField(helper, "explicitRegion", null); + + // Verify composition executes with fallback region "local" without errors + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + private static void setField(Object target, String fieldName, Object value) throws Exception { Field field = S3ServerSideComposeHelper.class.getDeclaredField(fieldName); field.setAccessible(true); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 7d3ea5a4..b7e77cb9 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -42,6 +42,7 @@ import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinUploadLengthNotReachedException; +import me.desair.tus.server.exception.UploadNotFoundException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; @@ -2619,6 +2620,269 @@ public void testPruneOrphanPartsDeletesUnmanifestedS3Part() throws Exception { a != null && "metadata/orphan-test-id.part.99999".equals(a.object()))); } + @Test + public void testConstructorUnableToEnsureTempDirLogsDebug() throws Exception { + java.nio.file.Path tempFile = java.nio.file.Files.createTempFile("tus-existing-file", ".tmp"); + try { + S3StorageService service = + new S3StorageService( + minioClient, + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + tempFile, + null); + assertNotNull(service); + } finally { + java.nio.file.Files.deleteIfExists(tempFile); + } + } + + @Test + public void testAppendSetsLengthWhenInfoLengthIsNull() throws Exception { + UploadId uploadId = new UploadId("deferred-len-id"); + UploadInfo infoInS3 = new UploadInfo(); + infoInS3.setId(uploadId); + infoInS3.setLength(null); + infoInS3.setOffset(0L); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/deferred-len-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(infoInS3).getBytes())); + + UploadInfo appendUpload = new UploadInfo(); + appendUpload.setId(uploadId); + appendUpload.setLength(500L); + + UploadInfo result = storageService.append(appendUpload, new ByteArrayInputStream(new byte[0])); + assertNotNull(result); + assertEquals(Long.valueOf(500L), result.getLength()); + } + + @Test + public void testGetUploadInfoByChecksumNoSuchKeyReturnsNull() throws Exception { + storageService.setUploadDeduplicationEnabled(true); + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && a.object().startsWith("checksums/")))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + UploadInfo info = storageService.getUploadInfoByChecksum("hash-123", ChecksumAlgorithm.SHA1); + assertNull(info); + } + + @Test + public void testPruneOrphanedChecksumIndicesListObjectsExceptionHandled() throws Exception { + storageService.setUploadDeduplicationEnabled(true); + when(minioClient.listObjects( + argThat((ListObjectsArgs a) -> a != null && a.prefix().startsWith("checksums/")))) + .thenThrow(new RuntimeException("Simulated S3 listing error")); + + storageService.cleanupExpiredUploads(null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testIsJsonSerializationEnabled() { + assertTrue(storageService.isJsonSerializationEnabled()); + } + + @Test + public void testSetAndGetS3ServerSideComposeHelper() { + S3ServerSideComposeHelper mockHelper = mock(S3ServerSideComposeHelper.class); + storageService.setS3ServerSideComposeHelper(mockHelper); + assertEquals(mockHelper, storageService.getS3ServerSideComposeHelper()); + } + + @Test + public void testCloseInterruptedHandledGracefully() throws Exception { + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testPrepareStreamWithExistingIncompletePartInspectionExceptionHandled() + throws Exception { + UploadId uploadId = new UploadId("part-inspect-err-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000L); + info.setOffset(50L); + String partKey = "metadata/part-inspect-err-id.part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(partKey))); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/part-inspect-err-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + // StatObject on last part throws Exception to test lines 1033-1034 catch block + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && partKey.equals(a.object())))) + .thenThrow(new RuntimeException("Part inspection failed")); + + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); + assertNotNull(result); + } + + @Test + public void testCalculateAndSetOffsetNullAndStatObjectException() throws Exception { + // Null info / null ID does nothing + storageService.calculateAndSetOffset(null); + UploadInfo nullIdInfo = new UploadInfo(); + storageService.calculateAndSetOffset(nullIdInfo); + + // Object exists but statObject throws Exception + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("stat-fail-id")); + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && "uploads/stat-fail-id".equals(a.object())))) + .thenThrow(new RuntimeException("StatObject failed")); + + storageService.calculateAndSetOffset(info); + assertEquals(Long.valueOf(0L), info.getOffset()); + } + + @Test + public void testDeleteObjectsBatchEmptyListAndErrorReported() throws Exception { + // Empty list returns immediately + storageService.deleteS3ObjectsQuietly(Collections.emptyList()); + + // Batch delete reports error, falling back to individual delete + @SuppressWarnings("unchecked") + Result mockRes = mock(Result.class); + io.minio.messages.DeleteResult.Error mockError = + mock(io.minio.messages.DeleteResult.Error.class); + when(mockError.objectName()).thenReturn("key-1"); + when(mockError.message()).thenReturn("Access Denied"); + when(mockRes.get()).thenReturn(mockError); + + when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenReturn(Collections.singletonList(mockRes)); + + storageService.deleteS3ObjectsQuietly(Arrays.asList("key-1", "key-2")); + verify(minioClient) + .removeObject(argThat((RemoveObjectArgs a) -> a != null && "key-1".equals(a.object()))); + verify(minioClient) + .removeObject(argThat((RemoveObjectArgs a) -> a != null && "key-2".equals(a.object()))); + } + + @Test + public void testPreparedStreamInterruptDelegation() { + me.desair.tus.server.util.InterruptibleInputStream origStream = + new me.desair.tus.server.util.InterruptibleInputStream( + new ByteArrayInputStream(new byte[10])); + S3StorageService.PreparedStream prepStream = + new S3StorageService.PreparedStream( + new ByteArrayInputStream(new byte[10]), + origStream, + 0L, + 0L, + Collections.emptyList(), + Collections.emptyList()); + + assertFalse(prepStream.isInterrupted()); + prepStream.interrupt(); + assertTrue(prepStream.isInterrupted()); + assertTrue(origStream.isInterrupted()); + } + + @Test(expected = java.util.NoSuchElementException.class) + public void testS3PartInputStreamEnumerationThrowsNoSuchElementException() { + S3StorageService.S3PartInputStreamEnumeration enumeration = + new S3StorageService.S3PartInputStreamEnumeration( + minioClient, "test-bucket", Collections.emptyList()); + assertFalse(enumeration.hasMoreElements()); + enumeration.nextElement(); + } + + @Test(expected = UploadNotFoundException.class) + public void testGetUploadedBytesStreamFailureThrowsUploadNotFoundException() throws Exception { + UploadId uploadId = new UploadId("err-bytes-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setOffset(100L); + info.setUploadPartKeys(Collections.singletonList("metadata/err-bytes-id.part.00001")); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> a != null && "metadata/err-bytes-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && "uploads/err-bytes-id".equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/err-bytes-id.part.00001".equals(a.object())))) + .thenThrow(new RuntimeException("S3 read part error")); + + InputStream is = storageService.getUploadedBytes(uploadId); + if (is != null) { + is.read(); + } + } + + @Test(expected = MaxAppendSizeExceededException.class) + public void testAppendWithMaxPartsReachedFlushesChunkAndThrows() throws Exception { + UploadId uploadId = new UploadId("max-parts-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000000L); + info.setOffset(50000L); + List tenThousandKeys = new ArrayList<>(Collections.nCopies(10000, "metadata/part.key")); + info.setUploadPartKeys(tenThousandKeys); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> a != null && "metadata/max-parts-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + storageService.append(info, new ByteArrayInputStream(new byte[50])); + } + + @Test + public void testAppendDrainExceptionRecordedAndThrowsIOException() throws Exception { + UploadId uploadId = new UploadId("drain-err-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000L); + info.setOffset(0L); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> a != null && "metadata/drain-err-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + doThrow(new RuntimeException(new IOException("S3 chunk put failed"))) + .when(minioClient) + .putObject(argThat((PutObjectArgs a) -> a != null && a.object().contains(".part."))); + + try { + storageService.append(info, new ByteArrayInputStream(new byte[100])); + fail("Expected IOException from drain failure"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("S3 chunk put failed") || e.getCause() != null); + } + } + private GetObjectResponse mockGetObjectResponse(byte[] bytes) { return new GetObjectResponse( null, "test-bucket", "eu-central-1", "object-key", new ByteArrayInputStream(bytes)); diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java index 71c2354d..60f9b921 100644 --- a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -781,4 +781,40 @@ public void testWaitForInFlightUploadTimeoutPolling() throws Exception { uploader.drainAndComplete(3000); } } + + @Test + public void testCheckAndConfirmInFlightInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "interrupt-f1.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + CountDownLatch uploadStarted = new CountDownLatch(1); + CountDownLatch holdUpload = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k-int-1", + () -> { + uploadStarted.countDown(); + try { + holdUpload.await(10, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue(uploadStarted.await(3, TimeUnit.SECONDS)); + + Thread.currentThread().interrupt(); + try { + uploader.checkAndConfirmInFlight(); + fail("Expected IOException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Interrupted while checking in-flight chunk upload")); + } finally { + Thread.interrupted(); // clear + holdUpload.countDown(); + } + } + } } From 506b4743c15fd281d219e3da168ec3c2f84f6930 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sat, 10 Oct 2026 19:18:59 +0200 Subject: [PATCH 10/15] feat(lock): automatically clean up locks on upload completion, termination, and expiration --- CHANGELOG.md | 1 + README.md | 2 +- docs/AZURE_BLOB_STORAGE.md | 9 + docs/LOCKING.md | 13 +- .../tus/server/TusFileUploadService.java | 12 ++ .../expiration/ExpirationRequestHandler.java | 39 +++- .../handler/RufhDeleteRequestHandler.java | 3 + .../TerminationDeleteRequestHandler.java | 23 +- .../upload/AbstractLeaseLockingService.java | 5 + .../server/upload/UploadLockingService.java | 26 +++ .../upload/azure/AzureBlobLockingService.java | 167 ++++++++++++++- .../upload/azure/AzureBlobStorageService.java | 3 + .../upload/azure/AzureBlobUploadLock.java | 79 ++++++- ...adLocalCachedStorageAndLockingService.java | 10 + .../upload/disk/DiskLockingService.java | 26 +++ .../upload/disk/DiskStorageService.java | 3 + .../upload/disk/LeaseFileLockingService.java | 18 ++ .../server/upload/s3/S3LockingService.java | 15 ++ .../server/upload/s3/S3StorageService.java | 4 + .../tus/server/TusFileUploadServiceTest.java | 120 +++++++++++ .../ExpirationRequestHandlerTest.java | 34 +++ .../handler/RufhDeleteRequestHandlerTest.java | 6 +- .../TerminationDeleteRequestHandlerTest.java | 28 +++ .../AbstractLeaseLockingServiceTest.java | 21 ++ .../upload/UploadLockingServiceTest.java | 2 + .../azure/AzureBlobLockingServiceTest.java | 202 +++++++++++++++++- .../upload/azure/AzureBlobUploadLockTest.java | 35 ++- .../azure/ITAzureBlobLockingService.java | 96 +++++++++ .../azure/ITAzureBlobStorageService.java | 7 +- .../ITAzureBlobTusFileUploadService.java | 104 +++++++++ ...calCachedStorageAndLockingServiceTest.java | 6 + .../upload/disk/DiskLockingServiceTest.java | 58 +++++ .../upload/disk/DiskStorageServiceTest.java | 1 + .../disk/LeaseFileLockingServiceTest.java | 69 ++++++ .../upload/s3/S3LockingServiceTest.java | 78 +++++++ .../upload/s3/S3StorageServiceTest.java | 3 +- 36 files changed, 1298 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa7b4fb1..9268bd43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file. - **Dedicated Compliance Test Suites**: Added comprehensive, spec-quoted end-to-end tests using a dedicated Python script `scripts/rufh_conformity_test.py` with documentation on how to run the tests in `docs/CONFORMITY_TESTING.md`. - **User Migration & Interim Responses Documentation**: Added `docs/MIGRATION.md` and `docs/INTERIM_RESPONSES.md` detailing migration strategies, HTTP 104 status frames under IETF RUFH, Tomcat/Servlet container limitations, cached reflection optimizations, and Spring Boot Tomcat Valve integration. - **Server-Side Upload Completion Listeners (`UploadCompletionListener`)**: Added a functional interface callback mechanism allowing developers to register post-upload listeners via `withUploadCompletionListener(UploadCompletionListener)` or `addUploadCompletionListener(UploadCompletionListener)`. Listeners receive the completed `UploadInfo` and `TusFileUploadService` instance after lock release, allowing immediate byte streaming and deletion without contention. Added helper overloads `TusFileUploadService.getUploadedBytes(UploadInfo)` and `TusFileUploadService.deleteUpload(UploadInfo)`. +- **Automatic Lock Cleanup on Completion, Termination & Expiration**: Added `cleanupLock(UploadId)` and `cleanupLock(String uploadUri)` across `UploadLockingService` implementations (`AzureBlobLockingService`, `DiskLockingService`, `LeaseFileLockingService`, `S3LockingService`). Automatically removes lock files, directories, and blobs when an upload completes (`offset == length`), when an upload is deleted via `DELETE` (204 No Content), and when expired uploads are pruned (`cleanupExpiredUploads()`), with a 120-second fallback sweep for abandoned Azure lock blobs. Safeguarded against deleting actively held locks mid-stream. - **Unified Lock Wait & Cloud Drain Timeout**: Added `TusFileUploadService.withLockWaitTimeout(Duration)` (default 60 seconds) to configure the maximum wait duration for requests contending for an active upload lock. Automatically derives the background cloud upload chunk drain timeout (`lockWaitTimeout - 5 seconds`, default 55s) and the retry polling budget (`lockWaitTimeout / 200ms`, default 300 retries). - **JSON Serialization**: Support storing `UploadInfo` objects as JSON files in the storage backend using `TusFileUploadService.withJsonSerialization(true)`. diff --git a/README.md b/README.md index b240e1fd..d08c362d 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ The Javadoc of this library can be found at https://tus.desair.me/. As of versio - **Multi-Replica Support**: Uses distributed S3 object locking and TTL leases, enabling multi-replica container deployments without requiring Redis or external databases. See [S3 Storage Guide](docs/S3_STORAGE.md). 3. **Azure Blob Storage** (`AzureBlobStorageService`, `AzureBlobLockingService`, & `AzureBlobConcatenationService`): - **Microsoft Azure Cloud**: Native Azure Blob Storage using the `azure-storage-blob` SDK. - - **Multi-Replica Support**: Uses native Azure Blob Leases (30s renewable leases) for distributed locking across cluster replicas. See [Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md). + - **Multi-Replica Support**: Uses native Azure Blob Leases (30s renewable leases) for distributed locking across cluster replicas with automatic lock cleanup on upload completion, termination, and expiration. See [Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md). ## How Resumable Uploads Work diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index f3ba0ec2..cc98e7bc 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -103,6 +103,15 @@ TusFileUploadService tusService = new TusFileUploadService() | `checksumsPrefix` | `"checksums/"` | Blob name prefix for deduplication index objects | | `locksPrefix` | `"locks/"` | Blob name prefix for distributed lock lease objects | +### Lock Blob Lifecycle & Automatic Cleanup + +During an in-progress upload, `AzureBlobLockingService` creates a target lock blob (`locks/.lock`) and holds a 30-second native Azure Blob Lease with automatic background renewal. + +1. **Automatic Cleanup on Completion**: Once an upload completes (`offset == length`), `TusFileUploadService` calls `cleanupLock(UploadId)` after releasing the lease, deleting the `.lock` blob and any `.stop` signal blob. +2. **Automatic Cleanup on Termination & Expiration**: When an upload is canceled (`DELETE`) or purged via `cleanupExpiredUploads()`, the corresponding `.lock` and `.stop` blobs are deleted. +3. **Lease Safety Guard**: Deleting an actively leased blob without providing the lease ID triggers HTTP 412 (`LeaseIdMissing`). `AzureBlobLockingService` inspects the blob's lease status and skips deleting blobs that are actively leased by concurrent requests. +4. **Fallback Stale Lock Sweeper (`cleanupStaleLocks`)**: To prevent abandoned lock blobs after ungraceful pod termination or network partitions, `cleanupStaleLocks()` runs as a scheduled fallback. It sweeps unleased `.lock` blobs older than 120 seconds and `.stop` signal blobs older than 30 seconds. Actively leased blobs are never deleted. + --- ## 4. Post-Upload Processing (`getAzureBlobName`) diff --git a/docs/LOCKING.md b/docs/LOCKING.md index 6d73d8ab..dbadfeaf 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -34,9 +34,15 @@ public interface UploadLockingService { // Checks if an upload is currently locked boolean isLocked(UploadId id); - // Cleans up stale or expired locks + // Cleans up stale or expired locks (fallback sweeper) void cleanupStaleLocks() throws IOException; + // Cleans up the lock file or object for a completed, terminated, or expired upload + default void cleanupLock(UploadId id) throws IOException {} + + // Cleans up the lock file or object for a completed, terminated, or expired upload by URI + default void cleanupLock(String uploadUri) throws IOException {} + // Injects the UploadIdFactory instance used to parse upload IDs from request URIs void setIdFactory(UploadIdFactory idFactory); @@ -68,9 +74,12 @@ public interface UploadLock extends Closeable { } ``` -### Request Flow & Contention Resolution +### Request Flow, Lock Cleanup & Contention Resolution - **Stream Registration**: When a request starts streaming payload bytes, its input stream is wrapped in an `InterruptibleInputStream` and registered via `lockingService.registerInputStream(requestUri, inputStream)`. +- **Automatic Lock Cleanup on Completion**: When an upload finishes (`offset == length`), `TusFileUploadService` automatically invokes `lockingService.cleanupLock(uploadId)` to remove the lock file/blob (`.lock`) and any stop signal file/blob (`.stop`). +- **Automatic Lock Cleanup on Termination & Expiration**: When an upload is canceled via `DELETE` (204 No Content) or expired via `cleanupExpiredUploads()`, the associated lock file or blob is also automatically deleted via `lockingService.cleanupLock(uploadId)`. Actively held locks are defended and never deleted mid-stream. +- **Fallback Stale Lock Sweeper**: As a safety net, `cleanupStaleLocks()` periodically sweeps orphaned lock records (e.g. unleased Azure blobs older than 120s or stop signal markers older than 30s) left behind after hard node crashes or power outages. - **Release Request**: When a concurrent `HEAD` or `DELETE` request encounters an active lock, `TusFileUploadService` catches `UploadAlreadyLockedException` and calls `lockingService.requestLockRelease(requestUri)`. - **Stream Interruption**: - If the lock is held in the **same JVM**, `requestLockRelease` interrupts the local stream directly. diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 159d7a7a..a72ae52c 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -602,6 +602,7 @@ public UploadInfo process( } if (wasInProgress && processedUploadInfo != null && !processedUploadInfo.isUploadInProgress()) { + cleanupLock(processedUploadInfo.getId()); notifyUploadCompletionListeners(processedUploadInfo); } @@ -946,6 +947,7 @@ public void deleteUpload(String uploadUri, String ownerKey) throws IOException, UploadInfo uploadInfo = uploadStorageService.getUploadInfo(uploadUri, ownerKey); if (uploadInfo != null) { uploadStorageService.terminateUpload(uploadInfo); + cleanupLock(uploadInfo.getId()); } } } @@ -1041,6 +1043,16 @@ protected void notifyUploadCompletionListeners(UploadInfo uploadInfo) { } } + private void cleanupLock(UploadId id) { + if (id != null && uploadLockingService != null) { + try { + uploadLockingService.cleanupLock(id); + } catch (Exception e) { + log.debug("Error cleaning up lock for upload ID {}: {}", id, e.getMessage()); + } + } + } + public ProtocolVersion detectProtocolVersion(HttpServletRequest request) { return Utils.detectProtocolVersion(request, supportedProtocolVersion); } diff --git a/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java b/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java index 67d86746..84e2230a 100644 --- a/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java +++ b/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java @@ -3,8 +3,11 @@ import java.io.IOException; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.HttpProblemDetails; import me.desair.tus.server.exception.TusException; +import me.desair.tus.server.exception.UploadNotFoundException; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; import me.desair.tus.server.util.TusServletRequest; @@ -27,12 +30,14 @@ public boolean supports(HttpMethod method) { } @Override - public void process( + public HttpProblemDetails process( HttpMethod method, TusServletRequest servletRequest, TusServletResponse servletResponse, UploadStorageService uploadStorageService, - String ownerKey) + UploadLockingService uploadLockingService, + String ownerKey, + TusException exception) throws IOException, TusException { // For post requests, the upload URI is part of the response @@ -50,13 +55,31 @@ public void process( // If the expiration is known at the creation, the Upload-Expires header MUST be included in // the response to the initial POST request. Its value MAY change over time. - if (expirationPeriod != null - && expirationPeriod > 0 - && uploadInfo != null - && !uploadInfo.isExpired()) { - uploadInfo.updateExpiration(expirationPeriod); - uploadStorageService.update(uploadInfo); + if (expirationPeriod != null && expirationPeriod > 0 && uploadInfo != null) { + if (uploadInfo.isExpired()) { + uploadStorageService.terminateUpload(uploadInfo); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(uploadInfo.getId()); + } + throw new UploadNotFoundException( + "The upload for path " + uploadUri + " has expired and was removed."); + } else { + uploadInfo.updateExpiration(expirationPeriod); + uploadStorageService.update(uploadInfo); + } } + return null; + } + + @Override + public void process( + HttpMethod method, + TusServletRequest servletRequest, + TusServletResponse servletResponse, + UploadStorageService uploadStorageService, + String ownerKey) + throws IOException, TusException { + process(method, servletRequest, servletResponse, uploadStorageService, null, ownerKey, null); } @Override diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java index 35b9f2cc..f086036e 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java @@ -46,6 +46,9 @@ public HttpProblemDetails process( } uploadStorageService.terminateUpload(uploadInfo); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(uploadInfo.getId()); + } servletResponse.setStatus(204); return null; } diff --git a/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java b/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java index 7b61176f..25c2a7b0 100644 --- a/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java +++ b/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java @@ -3,8 +3,10 @@ import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.HttpProblemDetails; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; import me.desair.tus.server.util.TusServletRequest; @@ -24,12 +26,14 @@ public boolean supports(HttpMethod method) { } @Override - public void process( + public HttpProblemDetails process( HttpMethod method, TusServletRequest servletRequest, TusServletResponse servletResponse, UploadStorageService uploadStorageService, - String ownerKey) + UploadLockingService uploadLockingService, + String ownerKey, + TusException exception) throws IOException, TusException { UploadInfo uploadInfo = @@ -37,8 +41,23 @@ public void process( if (uploadInfo != null) { uploadStorageService.terminateUpload(uploadInfo); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(uploadInfo.getId()); + } } servletResponse.setStatus(HttpServletResponse.SC_NO_CONTENT); + return null; + } + + @Override + public void process( + HttpMethod method, + TusServletRequest servletRequest, + TusServletResponse servletResponse, + UploadStorageService uploadStorageService, + String ownerKey) + throws IOException, TusException { + process(method, servletRequest, servletResponse, uploadStorageService, null, ownerKey, null); } } diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java index 93c8023e..521c3cc0 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java @@ -323,4 +323,9 @@ public void setJitter(long minMs, long maxMs) { this.jitterMinMs = minMs; this.jitterMaxMs = maxMs; } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + cleanupLock(uploadUri != null && idFactory != null ? idFactory.readUploadId(uploadUri) : null); + } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java index 0a3cc427..d97fb701 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java @@ -72,4 +72,30 @@ default void requestLockRelease(String requestUri) { default void close() throws IOException { // No-op by default for backwards compatibility } + + /** + * Cleans up the lock file or resources associated with a specific completed, terminated, or + * expired upload identifier. + * + *

This method is invoked automatically when an upload completes successfully, is terminated + * via DELETE, or is cleaned up during expiration sweeps, freeing underlying lock storage (such as + * Azure Blob Lease lock blobs or on-disk lock files). + * + * @param id The upload identifier of the completed, terminated, or expired upload + * @throws IOException If cleaning up lock resources fails + */ + default void cleanupLock(UploadId id) throws IOException { + // No-op by default for backwards compatibility + } + + /** + * Cleans up the lock file or resources associated with a specific completed, terminated, or + * expired upload URI. + * + * @param uploadUri The request URI of the completed, terminated, or expired upload + * @throws IOException If cleaning up lock resources fails + */ + default void cleanupLock(String uploadUri) throws IOException { + // No-op by default for backwards compatibility + } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index 9a62c624..059d749c 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -3,14 +3,21 @@ import com.azure.core.util.BinaryData; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobItemProperties; import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.DeleteSnapshotsOptionType; +import com.azure.storage.blob.models.LeaseStateType; +import com.azure.storage.blob.models.ListBlobsOptions; import com.azure.storage.blob.specialized.BlobLeaseClient; import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import java.io.IOException; import java.io.InputStream; import java.lang.ref.WeakReference; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -59,11 +66,14 @@ public class AzureBlobLockingService extends AbstractCloseableResourceService public static final String DEFAULT_LOCKS_PREFIX = "locks/"; private static final int LEASE_DURATION_SECONDS = 30; + private static final long DEFAULT_STALE_LOCK_THRESHOLD_SECONDS = 120L; + private static final long STOP_SIGNAL_EXPIRATION_SECONDS = 30L; private final BlobContainerClient containerClient; private final String locksPrefix; final Map> activeStreams = new ConcurrentHashMap<>(); + final Map activeLocks = new ConcurrentHashMap<>(); private UploadIdFactory idFactory = new UuidUploadIdFactory(); @@ -105,6 +115,7 @@ protected void cleanupOnClose() throws IOException { } } activeStreams.clear(); + activeLocks.clear(); } @Override @@ -145,7 +156,11 @@ public UploadLock lockUploadByUri(String requestUri) throws TusException, IOExce // Lock successfully acquired: clear any lingering .stop signal blob deleteStopSignalBlob(idStr); - return new AzureBlobUploadLock(leaseClient, lockBlob, requestUri, idStr, activeStreams); + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + leaseClient, lockBlob, requestUri, idStr, activeStreams, activeLocks); + activeLocks.put(idStr, lock); + return lock; } catch (BlobStorageException e) { AzureErrorType errorType = AzureUtils.parseErrorResponse(e); if (errorType == AzureErrorType.LEASE_ALREADY_PRESENT @@ -159,9 +174,149 @@ public UploadLock lockUploadByUri(String requestUri) throws TusException, IOExce } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + String idStr = id.toString(); + try { + // 1. Delete any lingering cross-pod .stop signal blob from prior lock contention + deleteStopSignalBlob(idStr); + + // 2. Delete the target .lock blob in the locks prefix + BlobClient lockBlob = getBlobClient(locksPrefix + idStr + ".lock"); + AzureBlobUploadLock activeLock = activeLocks.get(idStr); + if (activeLock != null && activeLock.getLeaseClient() != null) { + // The lock is actively held by this instance (e.g. during DELETE termination). + // Safely delete the blob specifying the active lease ID condition. + deleteLockBlobWithLease(lockBlob, activeLock.getLeaseClient().getLeaseId()); + activeLock.markBlobDeleted(); + } else { + deleteLockBlobIfUnleased(lockBlob); + } + } catch (Exception e) { + log.debug("Failed to clean up Azure lock blob for upload ID {}: {}", idStr, e.getMessage()); + } + } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + cleanupLock(uploadUri != null && idFactory != null ? idFactory.readUploadId(uploadUri) : null); + } + + /** + * Safely deletes an actively leased lock blob using its active lease ID. + * + *

When an upload is terminated (e.g. via {@code DELETE}) while holding the request lock, + * standard unleased deletion cannot be used because Azure returns HTTP 412 (LeaseIdMissing). + * Providing the active lease ID in {@link BlobRequestConditions} allows Azure to delete the + * actively leased lock blob immediately. + * + * @param lockBlob The blob client for the lock blob + * @param leaseId The active lease ID + * @return true if successfully deleted, false otherwise + */ + boolean deleteLockBlobWithLease(BlobClient lockBlob, String leaseId) { + if (lockBlob == null || leaseId == null) { + return false; + } + try { + lockBlob.deleteWithResponse( + DeleteSnapshotsOptionType.INCLUDE, + new BlobRequestConditions().setLeaseId(leaseId), + null, + null); + log.debug("Cleaned up actively leased Azure lock blob {}", lockBlob.getBlobName()); + return true; + } catch (Exception e) { + log.debug("Failed to delete leased lock blob {}: {}", lockBlob.getBlobName(), e.getMessage()); + return false; + } + } + + /** + * Safely deletes an unleased lock blob. Attempting deleteIfExists() on an actively leased blob + * without the lease ID causes Azure Blob Storage to reject the request with HTTP 412 + * (LeaseIdMissing). This helper verifies lease state before deletion. + */ + boolean deleteLockBlobIfUnleased(BlobClient lockBlob) { + if (lockBlob == null) { + return false; + } + try { + if (Boolean.TRUE.equals(lockBlob.exists())) { + BlobProperties props = lockBlob.getProperties(); + // Check lease state before deletion: if actively held by another thread or node, + // do not attempt deletion to avoid lease conflict errors. + if (props.getLeaseState() != null + && !Strings.CS.equals(props.getLeaseState().toString(), "leased")) { + lockBlob.deleteIfExists(); + log.debug("Cleaned up Azure lock blob {}", lockBlob.getBlobName()); + return true; + } + } + } catch (Exception e) { + log.debug( + "Failed to delete unleased lock blob {}: {}", lockBlob.getBlobName(), e.getMessage()); + } + return false; + } + @Override public void cleanupStaleLocks() throws IOException { - // Azure Blob Leases auto-expire after 30s on holder failure; no manual sweeps needed + try { + ListBlobsOptions options = new ListBlobsOptions().setPrefix(locksPrefix); + OffsetDateTime now = OffsetDateTime.now(); + + for (BlobItem item : listBlobs(options)) { + if (item == null || item.getName() == null) { + continue; + } + String blobName = item.getName(); + BlobItemProperties props = item.getProperties(); + if (props == null) { + continue; + } + // Defensively retrieve properties; uninitialized BlobItem instances in Azure SDK v12 + // throw NullPointerException on property access if internalProperties is null. + OffsetDateTime lastModified = null; + LeaseStateType leaseState = null; + try { + lastModified = props.getLastModified(); + leaseState = props.getLeaseState(); + } catch (NullPointerException ignored) { + continue; + } + + // 1. Clean up stale .stop signal blobs older than 30s + if (blobName.endsWith(".stop")) { + if (lastModified != null + && lastModified.isBefore(now.minusSeconds(STOP_SIGNAL_EXPIRATION_SECONDS))) { + getBlobClient(blobName).deleteIfExists(); + log.info("Cleaned up stale Azure stop signal blob: {}", blobName); + } + } + // 2. Clean up un-leased .lock blobs older than 120s + else if (blobName.endsWith(".lock")) { + // Never delete actively leased blobs to prevent interrupting active upload threads + if (leaseState != null && leaseState == LeaseStateType.LEASED) { + continue; + } + // The 120s grace period ensures we do not delete lock blobs for uploads actively + // between chunk requests, while cleaning up abandoned or pre-existing lock blobs. + if (lastModified != null + && lastModified.isBefore(now.minusSeconds(DEFAULT_STALE_LOCK_THRESHOLD_SECONDS))) { + BlobClient lockBlob = getBlobClient(blobName); + if (deleteLockBlobIfUnleased(lockBlob)) { + log.info("Cleaned up stale unleased Azure lock blob: {}", blobName); + } + } + } + } + } catch (Exception e) { + throw new IOException("Failed to cleanup stale Azure locks in container", e); + } } @Override @@ -252,6 +407,14 @@ BlobLeaseClient createBlobLeaseClient(BlobClient lockBlob) { return new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); } + Iterable listBlobs(ListBlobsOptions options) { + return containerClient.listBlobs(options, null); + } + + BlobClient getBlobClient(String blobName) { + return containerClient.getBlobClient(blobName); + } + /** * Periodically polled by {@link #watchdogExecutor} to detect remote .stop signal blobs on Azure * and interrupt local active streams. diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 63f35521..f300ac92 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -652,6 +652,9 @@ public void cleanupExpiredUploads(UploadLockingService lockingService) throws IO } catch (UploadNotFoundException ignored) { // Upload was already terminated or deleted concurrently } + if (lockingService != null) { + lockingService.cleanupLock(id); + } } } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java index 86b30243..e9898f58 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java @@ -35,9 +35,11 @@ public class AzureBlobUploadLock implements UploadLock { private final String uploadUri; private final String uploadId; private final Map> activeStreams; + private final Map activeLocks; private final ScheduledExecutorService renewalExecutor; private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); private boolean released = false; + private boolean blobDeleted = false; /** * Constructs an {@link AzureBlobUploadLock} wrapping an acquired Azure Blob lease. @@ -53,6 +55,7 @@ public AzureBlobUploadLock(BlobLeaseClient leaseClient, BlobClient lockBlob, Str uploadUri, null, null, + null, null); } @@ -77,7 +80,35 @@ public AzureBlobUploadLock( uploadUri, null, uploadId, - activeStreams); + activeStreams, + null); + } + + /** + * Full constructor for active lock with stream registration, lease tracking, and active lock map. + * + * @param leaseClient The pre-acquired {@link BlobLeaseClient} holding the lease + * @param lockBlob The target lock {@link BlobClient} + * @param uploadUri The upload URI associated with this lock + * @param uploadId The upload ID associated with this lock + * @param activeStreams JVM-wide map of active input streams + * @param activeLocks Map of currently active locks in the locking service + */ + public AzureBlobUploadLock( + BlobLeaseClient leaseClient, + BlobClient lockBlob, + String uploadUri, + String uploadId, + Map> activeStreams, + Map activeLocks) { + this( + Objects.requireNonNull(leaseClient, "leaseClient must not be null"), + Objects.requireNonNull(lockBlob, "lockBlob must not be null"), + uploadUri, + null, + uploadId, + activeStreams, + activeLocks); } AzureBlobUploadLock( @@ -85,7 +116,7 @@ public AzureBlobUploadLock( BlobClient lockBlob, String uploadUri, ScheduledExecutorService renewalExecutor) { - this(leaseClient, lockBlob, uploadUri, renewalExecutor, null, null); + this(leaseClient, lockBlob, uploadUri, renewalExecutor, null, null, null); } AzureBlobUploadLock( @@ -95,11 +126,23 @@ public AzureBlobUploadLock( ScheduledExecutorService renewalExecutor, String uploadId, Map> activeStreams) { + this(leaseClient, lockBlob, uploadUri, renewalExecutor, uploadId, activeStreams, null); + } + + AzureBlobUploadLock( + BlobLeaseClient leaseClient, + BlobClient lockBlob, + String uploadUri, + ScheduledExecutorService renewalExecutor, + String uploadId, + Map> activeStreams, + Map activeLocks) { this.leaseClient = leaseClient; this.lockBlob = lockBlob; this.uploadUri = Objects.requireNonNull(uploadUri, "uploadUri must not be null"); this.uploadId = uploadId; this.activeStreams = activeStreams; + this.activeLocks = activeLocks; if (renewalExecutor != null) { this.renewalExecutor = renewalExecutor; @@ -124,6 +167,24 @@ BlobLeaseClient getLeaseClient() { return leaseClient; } + void markBlobDeleted() { + thisObjectLock.writeLock().lock(); + try { + this.blobDeleted = true; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + + boolean isBlobDeleted() { + thisObjectLock.readLock().lock(); + try { + return blobDeleted; + } finally { + thisObjectLock.readLock().unlock(); + } + } + void setLeaseExpiresAt(long leaseExpiresAt) { thisObjectLock.writeLock().lock(); try { @@ -222,12 +283,22 @@ public void release() { thisObjectLock.writeLock().unlock(); } shutdownExecutor(); - // Remove active stream registration from JVM map upon release + // Remove active stream and lock registrations from service maps upon release // to avoid stale references lingering in heap. if (activeStreams != null && uploadId != null) { activeStreams.remove(uploadId); } - if (leaseClient != null) { + if (activeLocks != null && uploadId != null) { + activeLocks.remove(uploadId); + } + boolean alreadyDeleted; + thisObjectLock.readLock().lock(); + try { + alreadyDeleted = blobDeleted; + } finally { + thisObjectLock.readLock().unlock(); + } + if (leaseClient != null && !alreadyDeleted) { try { leaseClient.releaseLease(); log.trace("Released Azure blob lease for upload URI {}", uploadUri); diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index 5658f808..9e8a1280 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -281,6 +281,16 @@ public void requestLockRelease(String requestUri) { lockingServiceDelegate.requestLockRelease(requestUri); } + @Override + public void cleanupLock(UploadId id) throws IOException { + lockingServiceDelegate.cleanupLock(id); + } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + lockingServiceDelegate.cleanupLock(uploadUri); + } + @Override public void close() throws IOException { lockingServiceDelegate.close(); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java index dfb7c696..067fef49 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java @@ -114,6 +114,32 @@ public void cleanupStaleLocks() throws IOException { } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + // Only delete lock file if it exists and is not actively locked by another process/request. + // Notice we check Files.exists first because calling isLocked on a non-existent lock path + // would open a FileChannel with CREATE and instantiate a lock file on disk unnecessarily. + Path lockPath = getLockPath(id); + if (lockPath != null && Files.exists(lockPath)) { + if (!isLocked(id)) { + Files.deleteIfExists(lockPath); + } + } + Path stopPath = getStopPath(id); + if (stopPath != null) { + Files.deleteIfExists(stopPath); + } + } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + UploadId id = idFactory != null ? idFactory.readUploadId(uploadUri) : null; + cleanupLock(id); + } + /** Checks whether the upload is locked by attempting to obtain a short-lived file lock. */ @Override public boolean isLocked(UploadId id) { diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index ca5c853e..234e7dba 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -555,6 +555,9 @@ public void cleanupExpiredUploads(UploadLockingService uploadLockingService) thr for (Path path : expiredUploadsStream) { FileUtils.deleteDirectory(path.toFile()); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(new UploadId(path.getFileName().toString())); + } } } } diff --git a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java index f144f25d..635ce45b 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java @@ -160,6 +160,24 @@ public void cleanupStaleLocks() throws IOException { } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + Path lockDirPath = getLockDirPath(id); + if (lockDirPath != null && Files.exists(lockDirPath)) { + // Safe eviction under sibling mutex to prevent deleting an active successor's lease + if (isLockExpired(id)) { + atomicEvictExpiredLock(lockDirPath); + } + } + Path stopFilePath = getStopFilePath(id); + if (stopFilePath != null) { + Utils.deletePathQuietly(stopFilePath); + } + } + /** * Attempts primary lock acquisition or in-place takeover of an expired lease under the protection * of a {@link LeaseFileMutex}. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 22ef8c5b..eddf2b45 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -229,6 +229,21 @@ public void cleanupStaleLocks() throws IOException { } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + String lockKey = buildLockKey(id); + String stopKey = buildStopKey(id); + + // Only delete the lock object if it is expired or already absent + if (isLockExpired(lockKey)) { + deleteObjectQuietly(lockKey); + } + deleteObjectQuietly(stopKey); + } + @Override protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { if (leaseData == null) { diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 1a92e11f..9c8bb0ca 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -605,6 +605,10 @@ public void cleanupExpiredUploads(UploadLockingService uploadLockingService) thr && info.isExpired() && (uploadLockingService == null || !uploadLockingService.isLocked(id))) { terminateUpload(info); + if (uploadLockingService != null) { + // Clean up any remaining lock files now that the expired upload is terminated + uploadLockingService.cleanupLock(id); + } } } } diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 33cf0a29..b1b8ec93 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.io.InputStream; import java.time.Duration; +import java.util.UUID; import me.desair.tus.server.exception.UploadAlreadyLockedException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -931,4 +932,123 @@ public void testProcessLockedRequestCatchesUnexpectedRuntimeException() throws E assertThat(result, is(nullValue())); verify(mockResp, atLeastOnce()).setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } + + @Test + public void testProcessCleansUpLockOnUploadCompletion() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadLock mockLock = mock(UploadLock.class); + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + + UploadStorageService mockStorage = mock(UploadStorageService.class); + UUID uuid = UUID.randomUUID(); + UploadId testId = new UploadId(uuid); + String uploadUri = "/files/upload/" + uuid; + + UploadInfo inProgressInfo = new UploadInfo(); + inProgressInfo.setId(testId); + inProgressInfo.setOffset(5L); + inProgressInfo.setLength(10L); + + UploadInfo completedInfo = new UploadInfo(); + completedInfo.setId(testId); + completedInfo.setOffset(10L); + completedInfo.setLength(10L); + + when(mockStorage.getUploadInfo(uploadUri, "owner")) + .thenReturn(inProgressInfo) + .thenReturn(inProgressInfo) + .thenReturn(inProgressInfo) + .thenReturn(completedInfo); + when(mockStorage.append(any(), any())).thenReturn(completedInfo); + + jakarta.servlet.http.HttpServletRequest mockReq = + mock(jakarta.servlet.http.HttpServletRequest.class); + jakarta.servlet.http.HttpServletResponse mockResp = + mock(jakarta.servlet.http.HttpServletResponse.class); + when(mockReq.getMethod()).thenReturn("PATCH"); + when(mockReq.getRequestURI()).thenReturn(uploadUri); + when(mockReq.getHeader(anyString())).thenReturn(null); + when(mockReq.getHeader(HttpHeader.TUS_RESUMABLE)).thenReturn("1.0.0"); + when(mockReq.getHeader(HttpHeader.CONTENT_TYPE)).thenReturn("application/offset+octet-stream"); + when(mockReq.getHeader(HttpHeader.UPLOAD_OFFSET)).thenReturn("5"); + when(mockReq.getInputStream()) + .thenReturn( + new me.desair.tus.server.util.MockServletInputStream( + new ByteArrayInputStream(new byte[5]))); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorage); + + service.process(mockReq, mockResp, "owner"); + + verify(mockLockingService).cleanupLock(testId); + } + + @Test + public void testProcessCleansUpLockOnDeleteTermination() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadLock mockLock = mock(UploadLock.class); + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + + UploadStorageService mockStorage = mock(UploadStorageService.class); + UUID uuid = UUID.randomUUID(); + UploadId testId = new UploadId(uuid); + String uploadUri = "/files/upload/" + uuid; + + UploadInfo info = new UploadInfo(); + info.setId(testId); + info.setOffset(5L); + info.setLength(10L); + + when(mockStorage.getUploadInfo(uploadUri, "owner")).thenReturn(info); + + jakarta.servlet.http.HttpServletRequest mockReq = + mock(jakarta.servlet.http.HttpServletRequest.class); + org.springframework.mock.web.MockHttpServletResponse mockResp = + new org.springframework.mock.web.MockHttpServletResponse(); + when(mockReq.getMethod()).thenReturn("DELETE"); + when(mockReq.getRequestURI()).thenReturn(uploadUri); + when(mockReq.getHeader(anyString())).thenReturn(null); + when(mockReq.getHeader(HttpHeader.TUS_RESUMABLE)).thenReturn("1.0.0"); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorage); + + service.process(mockReq, mockResp, "owner"); + + verify(mockLockingService).cleanupLock(testId); + } + + @Test + public void testDeleteUploadCleansUpLock() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadLock mockLock = mock(UploadLock.class); + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + + UploadStorageService mockStorage = mock(UploadStorageService.class); + UUID uuid = UUID.randomUUID(); + UploadId testId = new UploadId(uuid); + String uploadUri = "/files/upload/" + uuid; + + UploadInfo info = new UploadInfo(); + info.setId(testId); + when(mockStorage.getUploadInfo(uploadUri, "owner")).thenReturn(info); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorage); + + service.deleteUpload(uploadUri, "owner"); + + verify(mockStorage).terminateUpload(info); + verify(mockLockingService).cleanupLock(testId); + } } diff --git a/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java b/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java index 7e2652c0..da6944f9 100644 --- a/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java @@ -15,7 +15,10 @@ import java.util.TimeZone; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; @@ -209,6 +212,37 @@ public void testNegativeExpiration() throws Exception { assertThat(tusResponse.getHeader(HttpHeader.UPLOAD_EXPIRES), is(nullValue())); } + @Test(expected = UploadNotFoundException.class) + public void testExpiredUploadTerminatesUploadAndCleansUpLock() throws Exception { + UploadInfo info = createUploadInfo(); + info.setId(new UploadId("expired-upload-id")); + info.setOffset(2L); + info.setLength(10L); + // Set expiration in the past relative to mock time (2018-01-20 10:43:11) + info.setExpirationTimestamp(1000L); + + when(uploadStorageService.getUploadInfo(nullable(String.class), nullable(String.class))) + .thenReturn(info); + when(uploadStorageService.getUploadExpirationPeriod()).thenReturn(172800000L); + + UploadLockingService mockLocking = org.mockito.Mockito.mock(UploadLockingService.class); + TusServletResponse tusResponse = new TusServletResponse(this.servletResponse); + + try { + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + tusResponse, + uploadStorageService, + mockLocking, + null, + null); + } finally { + verify(uploadStorageService, times(1)).terminateUpload(info); + verify(mockLocking, times(1)).cleanupLock(info.getId()); + } + } + private UploadInfo createUploadInfo() { return new UploadInfo() { @Override diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java index 024993b2..e451523d 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java @@ -10,6 +10,7 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; @@ -56,17 +57,20 @@ public void testProcessDeleteRequest() throws Exception { info.setId(new UploadId("delete-id")); when(storageService.getUploadInfo("/files/delete-id", "owner")).thenReturn(info); + UploadLockingService mockLocking = org.mockito.Mockito.mock(UploadLockingService.class); + handler.process( HttpMethod.DELETE, new TusServletRequest(request), new TusServletResponse(response), storageService, - null, + mockLocking, "owner", null); assertThat(response.getStatus(), is(204)); verify(storageService).terminateUpload(info); + verify(mockLocking).cleanupLock(info.getId()); } @Test(expected = me.desair.tus.server.exception.TusException.class) diff --git a/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java b/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java index 54521e97..c4753fd2 100644 --- a/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java @@ -14,6 +14,7 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; @@ -92,4 +93,31 @@ public void testWithExistingUpload() throws Exception { verify(uploadStorageService, times(1)).terminateUpload(info); assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); } + + @Test + public void testWithExistingUploadAndLockingService() throws Exception { + final UploadId id = new UploadId(UUID.randomUUID()); + + UploadInfo info = new UploadInfo(); + info.setId(id); + info.setOffset(2L); + info.setLength(10L); + when(uploadStorageService.getUploadInfo(nullable(String.class), nullable(String.class))) + .thenReturn(info); + + UploadLockingService mockLocking = org.mockito.Mockito.mock(UploadLockingService.class); + + handler.process( + HttpMethod.DELETE, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + null, + null); + + verify(uploadStorageService, times(1)).terminateUpload(info); + verify(mockLocking, times(1)).cleanupLock(id); + assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java index 09aa108e..7b4bbb74 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java @@ -33,11 +33,17 @@ private static class TestLeaseLockingService extends AbstractLeaseLockingService private final List stopSignalsWritten = new ArrayList<>(); private final List stopSignalsChecked = new ArrayList<>(); private final AtomicBoolean cleanedUp = new AtomicBoolean(false); + private final List cleanedUpLocks = new ArrayList<>(); public TestLeaseLockingService() { super(new UuidUploadIdFactory(), 30000L, 1000L, null, "test-lease-locking-service-watchdog"); } + @Override + public void cleanupLock(UploadId id) throws IOException { + cleanedUpLocks.add(id); + } + @Override protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { if (acquireShouldSucceed) { @@ -301,4 +307,19 @@ public void testCleanupOnClose() throws Exception { assertTrue(service.cleanedUp.get()); assertTrue(service.isClosed()); } + + @Test + public void testCleanupLockByUriDelegatesToUploadId() throws Exception { + UUID uuid = UUID.randomUUID(); + UploadId id = new UploadId(uuid); + String uri = "/files/" + uuid; + service.cleanupLock(uri); + + assertEquals(1, service.cleanedUpLocks.size()); + assertEquals(id, service.cleanedUpLocks.get(0)); + + service.cleanupLock((String) null); + assertEquals(2, service.cleanedUpLocks.size()); + assertThat(service.cleanedUpLocks.get(1), is(nullValue())); + } } diff --git a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java index 10b7d436..f5ddb64e 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java @@ -33,6 +33,8 @@ public void setIdFactory(UploadIdFactory idFactory) {} // Test default methods for coverage dummyService.registerInputStream("/test/upload/123", new ByteArrayInputStream(new byte[0])); dummyService.requestLockRelease("/test/upload/123"); + dummyService.cleanupLock(new UploadId("123")); + dummyService.cleanupLock("/test/upload/123"); dummyService.close(); assertFalse(dummyService.isLocked(new UploadId("123"))); diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java index 03897dbb..2da5badb 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java @@ -3,14 +3,25 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.LeaseStateType; +import com.azure.storage.blob.models.ListBlobsOptions; import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.List; import me.desair.tus.server.upload.TimeBasedUploadIdFactory; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.util.InterruptibleInputStream; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; /** * Offline unit tests for {@link AzureBlobLockingService} verifying parameter validation, prefix @@ -87,9 +98,69 @@ public void requestLockReleaseShouldDoNothingOnInvalidUri() { } @Test - public void cleanupStaleLocksShouldNotThrow() throws Exception { - // KISS: verifying method executes cleanly without throwing an exception - lockingService.cleanupStaleLocks(); + public void cleanupLockWithNullOrInvalidShouldDoNothing() throws Exception { + // KISS: verifying null/invalid parameters are handled safely without throwing exceptions + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + lockingService.cleanupLock("invalid-uri-no-id"); + } + + @Test + public void deleteLockBlobIfUnleasedNullReturnsFalse() { + assertFalse(lockingService.deleteLockBlobIfUnleased(null)); + } + + @Test + public void deleteLockBlobWithLeaseNullArguments() { + assertFalse(lockingService.deleteLockBlobWithLease(null, "lease-id")); + BlobClient mockBlob = Mockito.mock(BlobClient.class); + assertFalse(lockingService.deleteLockBlobWithLease(mockBlob, null)); + } + + @Test + public void deleteLockBlobWithLeaseExceptionHandled() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.doThrow(new RuntimeException("Simulated delete lease error")) + .when(mockBlob) + .deleteWithResponse(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + assertFalse(lockingService.deleteLockBlobWithLease(mockBlob, "lease-id")); + } + + @Test + public void cleanupLockWithActiveLockDeletesWithLease() throws Exception { + UploadId testId = new UploadId("test-active-lock-id"); + BlobClient mockBlob = Mockito.mock(BlobClient.class); + com.azure.storage.blob.specialized.BlobLeaseClient realLease = + new com.azure.storage.blob.specialized.BlobLeaseClientBuilder() + .blobClient(mockBlob) + .leaseId("mock-lease-123") + .buildClient(); + + AzureBlobUploadLock mockLock = Mockito.mock(AzureBlobUploadLock.class); + Mockito.when(mockLock.getLeaseClient()).thenReturn(realLease); + + AzureBlobLockingService testService = + new AzureBlobLockingService(containerClient) { + @Override + BlobClient getBlobClient(String blobName) { + return mockBlob; + } + + @Override + void deleteStopSignalBlob(String idStr) { + // KISS: no-op in unit test to avoid network call against dummy endpoint + } + + @Override + boolean deleteLockBlobWithLease(BlobClient lockBlob, String leaseId) { + return true; + } + }; + + testService.activeLocks.put("test-active-lock-id", mockLock); + testService.cleanupLock(testId); + + Mockito.verify(mockLock).markBlobDeleted(); } @Test @@ -105,19 +176,136 @@ public void closeInterruptsActiveWatchdogThread() throws Exception { InterruptibleInputStream stream = new InterruptibleInputStream(bais); lockingService.registerInputStream("/test/upload/88888", stream); - org.junit.Assert.assertFalse(stream.isInterrupted()); + assertFalse(stream.isInterrupted()); lockingService.close(); - org.junit.Assert.assertTrue(stream.isInterrupted()); + assertTrue(stream.isInterrupted()); } @Test public void testPollStopSignalsCleansStaleWeakReferences() { lockingService.activeStreams.put("stale-upload-id", new java.lang.ref.WeakReference<>(null)); - org.junit.Assert.assertTrue(lockingService.activeStreams.containsKey("stale-upload-id")); + assertTrue(lockingService.activeStreams.containsKey("stale-upload-id")); lockingService.pollStopSignals(); - org.junit.Assert.assertFalse(lockingService.activeStreams.containsKey("stale-upload-id")); + assertFalse(lockingService.activeStreams.containsKey("stale-upload-id")); + } + + @Test + public void cleanupLockHandlesExceptionGracefully() throws Exception { + AzureBlobLockingService throwingService = + new AzureBlobLockingService(containerClient) { + @Override + void deleteStopSignalBlob(String idStr) { + throw new RuntimeException("Simulated delete stop failure"); + } + }; + // KISS: verify method catches exception and logs debug without propagating + throwingService.cleanupLock(new UploadId("12345")); + } + + @Test + public void deleteLockBlobIfUnleasedCatchesExceptionAndReturnsFalse() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.when(mockBlob.exists()).thenThrow(new RuntimeException("Simulated exists failure")); + assertFalse(lockingService.deleteLockBlobIfUnleased(mockBlob)); + } + + @Test + public void testCleanupStaleLocksWithVariousBlobItems() throws Exception { + OffsetDateTime oldDate = OffsetDateTime.now().minusSeconds(200); + OffsetDateTime freshDate = OffsetDateTime.now(); + + BlobItem nullPropsItem = createTestBlobItemWithNullProps("locks/test.lock"); + BlobItem staleStopItem = createTestBlobItem("locks/stale.stop", oldDate, null); + BlobItem freshStopItem = createTestBlobItem("locks/fresh.stop", freshDate, null); + BlobItem leasedLockItem = + createTestBlobItem("locks/leased.lock", oldDate, LeaseStateType.LEASED); + BlobItem staleUnleasedLockItem = + createTestBlobItem("locks/stale.lock", oldDate, LeaseStateType.AVAILABLE); + BlobItem otherItem = createTestBlobItem("locks/other.txt", oldDate, null); + + List items = + Arrays.asList( + null, + nullPropsItem, + staleStopItem, + freshStopItem, + leasedLockItem, + staleUnleasedLockItem, + otherItem); + + BlobClient mockStaleBlob = Mockito.mock(BlobClient.class); + + AzureBlobLockingService mockService = + new AzureBlobLockingService(containerClient) { + @Override + Iterable listBlobs(ListBlobsOptions options) { + return items; + } + + @Override + BlobClient getBlobClient(String blobName) { + return mockStaleBlob; + } + + @Override + boolean deleteLockBlobIfUnleased(BlobClient lockBlob) { + return true; + } + }; + + mockService.cleanupStaleLocks(); + Mockito.verify(mockStaleBlob).deleteIfExists(); + } + + @Test(expected = IOException.class) + public void testCleanupStaleLocksThrowsIOExceptionOnListFailure() throws Exception { + AzureBlobLockingService failingService = + new AzureBlobLockingService(containerClient) { + @Override + Iterable listBlobs(ListBlobsOptions options) { + throw new RuntimeException("Simulated list failure"); + } + }; + failingService.cleanupStaleLocks(); + } + + @Test + public void ensureLockBlobExistsCatchesGenericException() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.doThrow(new RuntimeException("Simulated upload error")) + .when(mockBlob) + .upload(Mockito.any(), Mockito.anyLong()); + // KISS: verify method catches generic exception without propagating + lockingService.ensureLockBlobExists(mockBlob); + } + + private BlobItem createTestBlobItem( + String name, OffsetDateTime lastModified, LeaseStateType leaseState) { + com.azure.storage.blob.implementation.models.BlobName blobName = + new com.azure.storage.blob.implementation.models.BlobName() + .setContent(name) + .setEncoded(false); + com.azure.storage.blob.implementation.models.BlobItemPropertiesInternal props = + new com.azure.storage.blob.implementation.models.BlobItemPropertiesInternal() + .setLastModified(lastModified) + .setLeaseState(leaseState); + com.azure.storage.blob.implementation.models.BlobItemInternal itemInternal = + new com.azure.storage.blob.implementation.models.BlobItemInternal() + .setName(blobName) + .setProperties(props); + return com.azure.storage.blob.implementation.util.ModelHelper.populateBlobItem(itemInternal); + } + + private BlobItem createTestBlobItemWithNullProps(String name) { + com.azure.storage.blob.implementation.models.BlobName blobName = + new com.azure.storage.blob.implementation.models.BlobName() + .setContent(name) + .setEncoded(false); + com.azure.storage.blob.implementation.models.BlobItemInternal itemInternal = + new com.azure.storage.blob.implementation.models.BlobItemInternal().setName(blobName); + return com.azure.storage.blob.implementation.util.ModelHelper.populateBlobItem(itemInternal); } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java index ddef9786..10ef2a7f 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java @@ -73,7 +73,8 @@ public void testLockGettersAndReleaseWithExecutor() throws Exception { @Test public void testNullRenewalExecutorAndNullLeaseClient() { AzureBlobUploadLock lock = - new AzureBlobUploadLock(null, null, "/test/upload/null-exec", null, null, null); + new AzureBlobUploadLock( + null, null, "/test/upload/null-exec", (ScheduledExecutorService) null, null, null); org.junit.Assert.assertNull(lock.getLeaseClient()); assertEquals("/test/upload/null-exec", lock.getUploadUri()); lock.executeRenew(); @@ -305,4 +306,36 @@ void executeRenew() { lockWithEmptyStreams.renewLease(); verify(mockExecutor2).shutdownNow(); } + + @Test + public void testReleaseRemovesActiveLockAndSkipsReleaseWhenBlobDeleted() { + BlobLeaseClient realLease = + new BlobLeaseClientBuilder().blobClient(lockBlob).leaseId("mock-lease-123").buildClient(); + java.util.Map activeLocks = + new java.util.concurrent.ConcurrentHashMap<>(); + + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + realLease, + lockBlob, + "/test/upload/lock-deleted-test", + mockExecutor, + "lock-deleted-test", + null, + activeLocks); + + activeLocks.put("lock-deleted-test", lock); + org.junit.Assert.assertFalse(lock.isBlobDeleted()); + + lock.markBlobDeleted(); + org.junit.Assert.assertTrue(lock.isBlobDeleted()); + + // Release skips leaseClient.releaseLease() because blob is marked deleted, + // so no network call is attempted against the unmocked endpoint. + lock.release(); + + // Verify activeLocks is cleared + org.junit.Assert.assertFalse(activeLocks.containsKey("lock-deleted-test")); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java index 3c79345a..fc4c7f36 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java @@ -229,4 +229,100 @@ public void testStopSignalBlobCatchBlocksHandledGracefully() { // KISS: verifying methods execute cleanly without throwing uncaught exceptions assertTrue(true); } + + @Test + public void testCleanupLockDeletesLockAndStopBlobs() throws Exception { + UploadId id = new UploadId(111222L); + UploadLock lock = lockingService.lockUploadByUri("/test/upload/111222"); + assertNotNull(lock); + lockingService.createStopSignalBlob("111222"); + lock.release(); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/111222.lock"); + com.azure.storage.blob.BlobClient stopBlob = containerClient.getBlobClient("locks/111222.stop"); + + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + assertTrue(Boolean.TRUE.equals(stopBlob.exists())); + + lockingService.cleanupLock(id); + + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + assertFalse(Boolean.TRUE.equals(stopBlob.exists())); + } + + @Test + public void testCleanupLockByUri() throws Exception { + UploadLock lock = lockingService.lockUploadByUri("/test/upload/333444"); + assertNotNull(lock); + lock.release(); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/333444.lock"); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + lockingService.cleanupLock("/test/upload/333444"); + + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + } + + @Test + public void testCleanupLockActivelyLeasedBlobByExternalClientIsPreserved() throws Exception { + UploadId id = new UploadId(555666L); + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/555666.lock"); + lockBlob.upload(new ByteArrayInputStream(new byte[0]), 0); + + // External lease client simulates another replica / pod holding the lease + com.azure.storage.blob.specialized.BlobLeaseClient externalLease = + new com.azure.storage.blob.specialized.BlobLeaseClientBuilder() + .blobClient(lockBlob) + .buildClient(); + externalLease.acquireLease(30); + + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + // When actively leased by another node, cleanupLock must NOT delete the lock blob + lockingService.cleanupLock(id); + assertTrue( + "Actively leased blob by another client must not be deleted", + Boolean.TRUE.equals(lockBlob.exists())); + + // After the other client releases the lease, cleanupLock should delete it + externalLease.releaseLease(); + lockingService.cleanupLock(id); + assertFalse("Unleased blob should be deleted", Boolean.TRUE.equals(lockBlob.exists())); + } + + @Test + public void testCleanupLockActivelyLeasedBlobByCurrentServiceIsDeleted() throws Exception { + UploadId id = new UploadId(666777L); + UploadLock lock = lockingService.lockUploadByUri("/test/upload/666777"); + assertNotNull(lock); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/666777.lock"); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + // When actively leased by the current service (e.g. during DELETE termination), + // cleanupLock deletes the blob using the active lease ID + lockingService.cleanupLock(id); + assertFalse( + "Actively leased blob by current service should be deleted", + Boolean.TRUE.equals(lockBlob.exists())); + + // Release after cleanup should execute cleanly without error + lock.release(); + } + + @Test + public void testCleanupStaleLocksPreservesFreshBlobs() throws Exception { + UploadLock lock = lockingService.lockUploadByUri("/test/upload/777888"); + assertNotNull(lock); + lock.release(); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/777888.lock"); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + lockingService.cleanupStaleLocks(); + + // Fresh blob (created just now, < 120s old) must NOT be deleted by cleanupStaleLocks + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 2ea74777..4a99704e 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -373,12 +373,17 @@ public void cleanupExpiredUploadsShouldDeleteExpired() throws Exception { info.setLength(20L); // In-progress upload: 10 bytes uploaded out of 20 UploadInfo created = storageService.create(info, "owner1"); - storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + // Create lock blob for this upload to verify lock is cleaned up too + com.azure.storage.blob.BlobClient lockBlob = + containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(com.azure.core.util.BinaryData.fromString(""), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); Thread.sleep(50L); // Cleanup with expiration period 0 (all uploads expired) storageService.cleanupExpiredUploads(new AzureBlobLockingService(containerClient)); assertNull(storageService.getUploadInfo(created.getId())); + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java index 6a9f9d52..7be2e903 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java @@ -1,12 +1,19 @@ package me.desair.tus.server.upload.azure; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + import com.azure.storage.blob.BlobContainerClient; +import jakarta.servlet.http.HttpServletResponse; import me.desair.tus.server.AbstractITTusFileUploadService; +import me.desair.tus.server.HttpHeader; import me.desair.tus.server.ProtocolVersion; import me.desair.tus.server.TestUtils; import me.desair.tus.server.TusFileUploadService; +import org.apache.commons.lang3.StringUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Test; import org.testcontainers.containers.GenericContainer; /** @@ -65,4 +72,101 @@ protected TusFileUploadService createTusFileUploadService(String uploadUri) { .withDownloadFeature() .withChunkedTransferDecoding(true); } + + /** + * Verifies that when an upload completes via PATCH (offset reaching total length), the lock blob + * in the locks/ directory is automatically cleaned up and deleted. + */ + @Test + public void testUploadCompletionCleansUpLockBlob() throws Exception { + String uploadContent = "Testing Azure lock blob cleanup on upload completion!"; + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, uploadContent.getBytes().length); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_CREATED, servletResponse.getStatus()); + + String location = + UPLOAD_URI + + StringUtils.substringAfter( + servletResponse.getHeader(HttpHeader.LOCATION), UPLOAD_URI); + String uploadIdStr = StringUtils.substringAfterLast(location, "/"); + + // Complete the upload via PATCH + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(location); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, uploadContent.getBytes().length); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, 0); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(uploadContent.getBytes()); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_NO_CONTENT, servletResponse.getStatus()); + + // Verify lock blob is cleaned up on completion + assertFalse( + "Lock blob should be removed upon upload completion", + containerClient.getBlobClient("locks/" + uploadIdStr + ".lock").exists()); + assertFalse( + "Stop signal blob should not exist", + containerClient.getBlobClient("locks/" + uploadIdStr + ".stop").exists()); + } + + /** + * Verifies that when an in-progress upload is terminated via DELETE, the lock blob in the locks/ + * directory is automatically cleaned up and deleted. + */ + @Test + public void testDeleteTerminationCleansUpLockBlob() throws Exception { + String uploadContent = "Incomplete upload to terminate"; + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, uploadContent.getBytes().length * 2); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_CREATED, servletResponse.getStatus()); + + String location = + UPLOAD_URI + + StringUtils.substringAfter( + servletResponse.getHeader(HttpHeader.LOCATION), UPLOAD_URI); + String uploadIdStr = StringUtils.substringAfterLast(location, "/"); + + // Partially upload via PATCH + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(location); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, uploadContent.getBytes().length); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, 0); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(uploadContent.getBytes()); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_NO_CONTENT, servletResponse.getStatus()); + + // Delete the upload + reset(); + servletRequest.setMethod("DELETE"); + servletRequest.setRequestURI(location); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_NO_CONTENT, servletResponse.getStatus()); + + // Verify lock blob is cleaned up on termination + assertFalse( + "Lock blob should be removed upon DELETE termination", + containerClient.getBlobClient("locks/" + uploadIdStr + ".lock").exists()); + assertFalse( + "Stop signal blob should not exist", + containerClient.getBlobClient("locks/" + uploadIdStr + ".stop").exists()); + } } diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index be72ffec..fc5a99d4 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -252,6 +252,12 @@ public void testDelegateMethods() throws Exception { service.requestLockRelease("/files/1"); verify(mockLocking, times(1)).requestLockRelease("/files/1"); + service.cleanupLock(id); + verify(mockLocking, times(1)).cleanupLock(id); + + service.cleanupLock("/files/1"); + verify(mockLocking, times(1)).cleanupLock("/files/1"); + service.close(); verify(mockLocking, times(1)).close(); } diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java index 39464c3e..b9eee747 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java @@ -685,4 +685,62 @@ public void testCloseInterruptsActiveStreams() throws Exception { lockingService.close(); assertTrue(iis.isInterrupted()); } + + @Test + public void testCleanupLockNullAndInvalid() throws Exception { + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testCleanupLockDeletesLockAndStopFiles() throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + Path lockPath = storagePath.resolve("locks").resolve(id.toString()); + Path stopPath = storagePath.resolve("locks").resolve(id.toString() + ".stop"); + Files.createFile(lockPath); + Files.createFile(stopPath); + + assertTrue(Files.exists(lockPath)); + assertTrue(Files.exists(stopPath)); + + lockingService.cleanupLock(id); + + assertFalse(Files.exists(lockPath)); + assertFalse(Files.exists(stopPath)); + } + + @Test + public void testCleanupLockByUri() throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + when(idFactory.readUploadId(UPLOAD_URL + "/" + id)).thenReturn(id); + + Path lockPath = storagePath.resolve("locks").resolve(id.toString()); + Files.createFile(lockPath); + assertTrue(Files.exists(lockPath)); + + lockingService.cleanupLock(UPLOAD_URL + "/" + id); + + assertFalse(Files.exists(lockPath)); + } + + @Test + public void testCleanupLockWhenActivelyLockedDoesNotDeleteLockFile() throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + when(idFactory.readUploadId(UPLOAD_URL + "/" + id)).thenReturn(id); + + UploadLock lock = lockingService.lockUploadByUri(UPLOAD_URL + "/" + id); + assertThat(lock, is(not(nullValue()))); + + Path lockPath = storagePath.resolve("locks").resolve(id.toString()); + assertTrue(Files.exists(lockPath)); + + // When actively locked, cleanupLock must preserve the lock file + lockingService.cleanupLock(id); + assertTrue(Files.exists(lockPath)); + + lock.release(); + assertFalse(Files.exists(lockPath)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index d24195ed..bbfc4d32 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -538,6 +538,7 @@ public void cleanupExpiredUploads() throws Exception { assertFalse(Files.exists(getUploadInfoPath(info.getId()))); assertFalse(Files.exists(getStoragePath(info.getId()))); + org.mockito.Mockito.verify(uploadLockingService).cleanupLock(info.getId()); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java index 9f78d6fa..27d0c822 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java @@ -919,4 +919,73 @@ private LeaseData createExpiredLease( lease.setAcquiredAt(expiresAt - 30_000L); return lease; } + + @Test + public void testCleanupLockNullAndInvalid() throws Exception { + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testCleanupLockDeletesExpiredLockDirAndStopFile() throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + UploadId id = new UploadId(uploadIdStr); + Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); + Path stopFile = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); + Files.createDirectories(lockDir); + Files.write(stopFile, new byte[0]); + + long now = System.currentTimeMillis(); + LeaseData expiredLease = + createExpiredLease( + "old-node", UPLOAD_URL + "/" + uploadIdStr, lockDir.toString(), now - 5000L); + LeaseDataJsonSerializer.serializeToPath(expiredLease, lockDir.resolve("lease.json")); + + assertTrue(Files.exists(lockDir)); + assertTrue(Files.exists(stopFile)); + + lockingService.cleanupLock(id); + + assertFalse(Files.exists(lockDir)); + assertFalse(Files.exists(stopFile)); + } + + @Test + public void testCleanupLockByUri() throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + String uri = UPLOAD_URL + "/" + uploadIdStr; + Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); + Files.createDirectories(lockDir); + + long now = System.currentTimeMillis(); + LeaseData expiredLease = createExpiredLease("old-node", uri, lockDir.toString(), now - 5000L); + LeaseDataJsonSerializer.serializeToPath(expiredLease, lockDir.resolve("lease.json")); + assertTrue(Files.exists(lockDir)); + + lockingService.cleanupLock(uri); + + assertFalse(Files.exists(lockDir)); + } + + @Test + public void testCleanupLockPreservesActivelyHeldLease() throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + String uri = UPLOAD_URL + "/" + uploadIdStr; + UploadId id = new UploadId(uploadIdStr); + + UploadLock lock = lockingService.lockUploadByUri(uri); + assertNotNull(lock); + + Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); + assertTrue(Files.exists(lockDir)); + + // Actively held lock must not be deleted by cleanupLock + lockingService.cleanupLock(id); + assertTrue(Files.exists(lockDir)); + + lock.release(); + assertFalse(Files.exists(lockDir)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index a94fde12..e38b4dcb 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -1057,4 +1057,82 @@ public void testExplicitNonCasModeChecksLockExpirationBeforeUnconditionalWrite() lock1.close(); } + + @Test + public void testCleanupLockNullAndInvalid() throws Exception { + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testCleanupLockDeletesExpiredLockAndStopObjects() throws Exception { + UploadId id = new UploadId(UUID.randomUUID()); + String lockKey = "locks/" + id + ".lock"; + String stopKey = "locks/" + id + ".stop"; + + LeaseData expiredLock = + new LeaseData( + "old-holder", + "/files/upload/" + id, + 30000L, + System.currentTimeMillis() - 5000L, + System.currentTimeMillis() - 35000L, + lockKey, + stopKey); + s3StorageMap.put( + lockKey, me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(expiredLock)); + s3StorageMap.put(stopKey, new byte[0]); + + assertTrue(s3StorageMap.containsKey(lockKey)); + assertTrue(s3StorageMap.containsKey(stopKey)); + + lockingService.cleanupLock(id); + + assertFalse(s3StorageMap.containsKey(lockKey)); + assertFalse(s3StorageMap.containsKey(stopKey)); + } + + @Test + public void testCleanupLockByUri() throws Exception { + UploadId id = new UploadId(UUID.randomUUID()); + String uri = "/files/upload/" + id; + String lockKey = "locks/" + id + ".lock"; + + LeaseData expiredLock = + new LeaseData( + "old-holder", + uri, + 30000L, + System.currentTimeMillis() - 5000L, + System.currentTimeMillis() - 35000L, + lockKey, + null); + s3StorageMap.put( + lockKey, me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(expiredLock)); + assertTrue(s3StorageMap.containsKey(lockKey)); + + lockingService.cleanupLock(uri); + + assertFalse(s3StorageMap.containsKey(lockKey)); + } + + @Test + public void testCleanupLockPreservesActivelyHeldLock() throws Exception { + UploadId id = new UploadId(UUID.randomUUID()); + String uri = "/files/upload/" + id; + String lockKey = "locks/" + id + ".lock"; + + UploadLock lock = lockingService.lockUploadByUri(uri); + assertNotNull(lock); + assertTrue(s3StorageMap.containsKey(lockKey)); + + // Actively held unexpired lock must not be deleted by cleanupLock + lockingService.cleanupLock(id); + assertTrue(s3StorageMap.containsKey(lockKey)); + + lock.close(); + assertFalse(s3StorageMap.containsKey(lockKey)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index b7e77cb9..2a663ae1 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -935,7 +935,7 @@ public void testCleanupExpiredUploads() throws Exception { String json = UploadInfoJsonSerializer.serialize(expiredInfo); Item item = mock(Item.class); - when(item.objectName()).thenReturn("uploads/expired-123.info"); + when(item.objectName()).thenReturn("metadata/expired-123.info"); Result result = new Result<>(item); when(minioClient.listObjects(any(ListObjectsArgs.class))) .thenReturn(java.util.Collections.singletonList(result)); @@ -947,6 +947,7 @@ public void testCleanupExpiredUploads() throws Exception { when(mockLocking.isLocked(expiredId)).thenReturn(false); storageService.cleanupExpiredUploads(mockLocking); + verify(mockLocking).cleanupLock(expiredId); } @Test(expected = IOException.class) From b4498b45b6eff8ad4c812f86e4d362be7dc703bb Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 11 Oct 2026 14:27:30 +0200 Subject: [PATCH 11/15] docs: add architecture guide and overhaul project documentation --- .github/workflows/docs-link-check.yml | 31 ++ AGENTS.md | 11 +- CHANGELOG.md | 71 ++-- CONTRIBUTING.md | 64 ++++ README.md | 524 +++++++++----------------- docs/ARCHITECTURE.md | 309 +++++++++++++++ docs/AZURE_BLOB_STORAGE.md | 27 +- docs/CONFIGURATION.md | 85 +++++ docs/CONFORMITY_TESTING.md | 4 +- docs/DEPLOYMENT.md | 49 +++ docs/DISK_BASED_LOCKING.md | 28 +- docs/INTEGRATION_GUIDE.md | 302 +++++++++++++++ docs/LOCKING.md | 8 +- docs/MIGRATION.md | 43 +-- docs/PROTOCOLS.md | 137 +++++++ docs/README.md | 26 ++ docs/S3_STORAGE.md | 17 +- docs/TESTING.md | 38 +- docs/TROUBLESHOOTING.md | 22 ++ docs/UPGRADING.md | 74 ++++ lychee.toml | 16 + 21 files changed, 1426 insertions(+), 460 deletions(-) create mode 100644 .github/workflows/docs-link-check.yml create mode 100644 CONTRIBUTING.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/CONFIGURATION.md create mode 100644 docs/DEPLOYMENT.md create mode 100644 docs/INTEGRATION_GUIDE.md create mode 100644 docs/PROTOCOLS.md create mode 100644 docs/README.md create mode 100644 docs/TROUBLESHOOTING.md create mode 100644 docs/UPGRADING.md create mode 100644 lychee.toml diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml new file mode 100644 index 00000000..0d2344a4 --- /dev/null +++ b/.github/workflows/docs-link-check.yml @@ -0,0 +1,31 @@ +# Checks that links and anchors in the Markdown documentation are not broken. +# Pull requests only verify local files/anchors (offline, fast and never flaky); +# the weekly run also checks external URLs. + +name: Documentation Link Check + +permissions: + contents: read + +on: + pull_request: + paths: ["**.md", ".github/workflows/docs-link-check.yml", "lychee.toml"] + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +jobs: + link-check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Check Markdown links + uses: lycheeverse/lychee-action@v2 + with: + # Offline (local links and #anchors only) on pull requests, full check otherwise + args: >- + --config lychee.toml + ${{ github.event_name == 'pull_request' && '--offline' || '' }} + README.md CONTRIBUTING.md CHANGELOG.md 'docs/**/*.md' + fail: true diff --git a/AGENTS.md b/AGENTS.md index f33fa5bd..092694c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # Agent Instructions ## Project Context -When working on this project, always read the [`README.md`](README.md) file to obtain full context on project architecture, features, configuration options, and dual protocol version support (Tus 1.0.0 & IETF RUFH). +When working on this project, always read the [`README.md`](README.md) and [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) files to obtain full context on project architecture, extension pipelines, features, configuration options, and dual protocol version support (Tus 1.0.0 & IETF RUFH). ## GitHub CLI (`gh`) & Git Usage - Always run `git` commands (e.g., `git status`, `git diff`, `git add`, `git commit`, `git push`) and `gh` commands unsandboxed (setting `BypassSandbox: true` when calling `run_command`) to ensure git hooks, local tools, and remote repository authentication work without sandbox errors. @@ -23,7 +23,7 @@ Any new feature, bugfix, or improvement must be developed in a separate branch t When a new feature is introduced, the `README.md` file must be updated with information on this new feature (e.g. configuration, usage). ### CHANGELOG.md -For any new feature, big improvements, or fixes, the `CHANGELOG.md` file must be updated to describe the changes added in this version. Use a release version header (e.g., `## [1.0.0-3.2]`) instead of `## [Unreleased]`. Derive this next release version from the SNAPSHOT version declared in the `pom.xml` file by removing the `-SNAPSHOT` suffix. Make sure to not add duplicate headers. +For any new feature, big improvements, or fixes, the `CHANGELOG.md` file must be updated to describe the changes added in this version. Use a release version header (e.g., `## [2.0.0]`) instead of `## [Unreleased]`. Derive this next release version from the SNAPSHOT version declared in the `pom.xml` file by removing the `-SNAPSHOT` suffix. Make sure to not add duplicate headers. ### Release Process When performing a release, please strictly follow the instructions outlined in the [docs/RELEASE.md](docs/RELEASE.md) documentation file. @@ -35,8 +35,9 @@ When performing a release, please strictly follow the instructions outlined in t - **Jakarta EE / Servlets**: Always use `jakarta.servlet.*` package imports instead of the legacy `javax.servlet.*` packages. ### 2. Extension Architecture & Protocol Applicability -- Every protocol extension MUST extend `AbstractTusExtension` and declare its applicability via `isApplicable(HttpMethod, ProtocolVersion)`. -- `TusFileUploadService` MUST NOT contain protocol-specific conditionals, version branching, or hardcoded error handling; all protocol-specific validation and execution logic belongs inside dedicated `RequestValidator` and `RequestHandler` implementations. +- Read the definitive architectural design in [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) before implementing new protocol logic. +- `TusFileUploadService` is the single facade service class orchestrating protocols composed of modular extensions. It MUST NOT contain protocol-specific conditionals, version branching, or hardcoded error handling. +- Every protocol extension MUST extend `AbstractTusExtension` and declare its applicability via `isApplicable(HttpMethod, ProtocolVersion)`. All protocol-specific validation and execution logic belongs inside dedicated `RequestValidator` and `RequestHandler` implementations. ### 3. Explicit Parameter Passing & No Request Attributes - Do NOT use magic string servlet request attributes (such as `"me.desair.tus.uploadLockingService"` or `"me.desair.tus.protocolVersion"`). @@ -111,6 +112,7 @@ Whenever a new setter or configuration property (such as `setMinAppendSize`, `se - A corresponding `with...` builder method (e.g. `withMinAppendSize`, `withMinSize`) MUST be added to `TusFileUploadService` with thorough Javadoc comments. - `TusFileUploadService.withUploadStorageService(...)` MUST be updated to copy the setting from the old `UploadStorageService` instance to the new one. - `ThreadLocalCachedStorageAndLockingService` MUST delegate the setter and getter methods to `storageServiceDelegate`. +- **ThreadLocalCache Wrapper Safety**: When `.withThreadLocalCache(true)` is enabled, `getUploadStorageService()` returns `ThreadLocalCachedStorageAndLockingService` (a wrapper), not concrete classes like `S3StorageService` or `AzureBlobStorageService`. Never cast `getUploadStorageService()` directly in downstream code; always inject or retain the concrete storage service bean directly. ### 15. Typed Exceptions & HttpServletResponse Status Codes - Do NOT throw generic `TusException` directly when throwing protocol errors or request validation failures. @@ -147,6 +149,7 @@ To maintain a clear separation between fast, offline unit tests and containerize - Focus strictly on business processes, end-to-end user flows, protocol interactions, and backend service capability flows (such as `ITAzureBlobStorageService`, `ITAzureBlobLockingService`, `ITAzureBlobConcatenationService`, `ITAzureBlobRufhProtocol`, `ITAzureBlobTusFileUploadService`). - StorageService, LockingService, and ConcatenationService integration tests are maintained because they represent key capability flows that can be combined across different backend types (e.g. Azure storage combined with S3 locking). - Internal helper objects or component handles that do NOT represent an independent business process (such as `UploadLock` handles) MUST NOT have dedicated `IT*` integration test classes (e.g. `ITAzureBlobUploadLock` is omitted in favor of testing `AzureBlobUploadLockTest` offline and testing lock lifecycles end-to-end via `ITAzureBlobLockingService` / `ITAzureBlobTusFileUploadService`). +- **Automated Container Lifecycle (Podman & Docker)**: Integration tests automatically manage container lifecycles via Testcontainers using either **Podman** or **Docker**. Never invoke `docker` CLI directly or instruct developers to manually start containers with `docker run` or `podman run`. - **Naming & Execution Rules**: - Unit test classes MUST end with `Test.java` (executed during `mvn test`). - Integration test classes MUST start with `IT` and MUST NOT end with `Test` or `Test.java` (executed during `mvn verify`). diff --git a/CHANGELOG.md b/CHANGELOG.md index 9268bd43..d063cf3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,49 +4,44 @@ All notable changes to this project will be documented in this file. ## [2.0.0] -### New +Major release adding the IETF Resumable Uploads for HTTP (RUFH) protocol and cloud storage backends. **Upgrading from `1.0.0-3.x`? Read the [Upgrade Guide](docs/UPGRADING.md).** -- **NFS- & SMB-Safe Lease Locking (`LeaseFileLockingService` & `LeaseFileMutex`)**: Added distributed, container-safe filesystem locking using atomic sibling mutex directories (`.mutex/`), in-place expired lock takeover, and TTL-based JSON lease files with background heartbeat renewal and ownership fencing. Operates reliably across multi-server replicas on NFS (v3/v4), AWS EFS, Azure Files, Windows SMB/CIFS, and local disks without requiring Redis, ZooKeeper, or OS-level `FileLock` daemons. Comprehensive guide and legacy opt-out instructions available in `docs/DISK_BASED_LOCKING.md`. -- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService`, distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, configured entirely via standard S3 connection parameters, and complete documentation in `docs/S3_STORAGE.md`. -- **Azure Blob Storage & Distributed Leases**: Added native Azure Blob Storage support via `AzureBlobStorageService` (Block Blob staging with streaming appends, sub-threshold buffering, truncation, and deduplication), distributed locking via `AzureBlobLockingService` (Azure Blob Leases with auto-renewal, JVM interruption, cross-replica `.stop` signals, and clean shutdown), zero-copy server-side concatenation via `AzureBlobConcatenationService` (`stageBlockFromUrl`), and comprehensive documentation in `docs/AZURE_BLOB_STORAGE.md`. -- **IETF Resumable Uploads for HTTP (RUFH) Protocol**: Implemented full support for the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`). - - **Dual Protocol Auto-Detection**: Added transparent protocol routing in `TusFileUploadService` supporting both legacy `TUS_1_0_0` (`Tus-Resumable: 1.0.0`) and `RUFH` (`ProtocolVersion.RUFH`) clients concurrently on the same endpoint. - - **RFC 9651 Structured Header Fields**: Implemented RFC 9651 parsing and serialization for `Upload-Offset`, `Upload-Complete`, `Upload-Length`, and `Upload-Limit` dictionary headers. - - **RFC 7807 Problem Details JSON**: Added support for standard `application/problem+json` error responses (`mismatching-upload-offset`, `completed-upload`, `inconsistent-upload-length`). - - **Dedicated Compliance Test Suites**: Added comprehensive, spec-quoted end-to-end tests using a dedicated Python script `scripts/rufh_conformity_test.py` with documentation on how to run the tests in `docs/CONFORMITY_TESTING.md`. - - **User Migration & Interim Responses Documentation**: Added `docs/MIGRATION.md` and `docs/INTERIM_RESPONSES.md` detailing migration strategies, HTTP 104 status frames under IETF RUFH, Tomcat/Servlet container limitations, cached reflection optimizations, and Spring Boot Tomcat Valve integration. -- **Server-Side Upload Completion Listeners (`UploadCompletionListener`)**: Added a functional interface callback mechanism allowing developers to register post-upload listeners via `withUploadCompletionListener(UploadCompletionListener)` or `addUploadCompletionListener(UploadCompletionListener)`. Listeners receive the completed `UploadInfo` and `TusFileUploadService` instance after lock release, allowing immediate byte streaming and deletion without contention. Added helper overloads `TusFileUploadService.getUploadedBytes(UploadInfo)` and `TusFileUploadService.deleteUpload(UploadInfo)`. -- **Automatic Lock Cleanup on Completion, Termination & Expiration**: Added `cleanupLock(UploadId)` and `cleanupLock(String uploadUri)` across `UploadLockingService` implementations (`AzureBlobLockingService`, `DiskLockingService`, `LeaseFileLockingService`, `S3LockingService`). Automatically removes lock files, directories, and blobs when an upload completes (`offset == length`), when an upload is deleted via `DELETE` (204 No Content), and when expired uploads are pruned (`cleanupExpiredUploads()`), with a 120-second fallback sweep for abandoned Azure lock blobs. Safeguarded against deleting actively held locks mid-stream. -- **Unified Lock Wait & Cloud Drain Timeout**: Added `TusFileUploadService.withLockWaitTimeout(Duration)` (default 60 seconds) to configure the maximum wait duration for requests contending for an active upload lock. Automatically derives the background cloud upload chunk drain timeout (`lockWaitTimeout - 5 seconds`, default 55s) and the retry polling budget (`lockWaitTimeout / 200ms`, default 300 retries). -- **JSON Serialization**: Support storing `UploadInfo` objects as JSON files in the storage backend using `TusFileUploadService.withJsonSerialization(true)`. +### Highlights +- IETF Resumable Uploads for HTTP (`draft-ietf-httpbis-resumable-upload-12`) served next to Tus 1.0.0 on the same endpoint. +- S3-compatible and Azure Blob storage backends with distributed, database-free locking. +- NFS/SMB-safe lease locking is now the default for disk storage. +- Server-side `UploadCompletionListener` and a `process()` method returning the `UploadInfo`. -### Changed -- **Default Disk-Based Locking**: `TusFileUploadService.withStoragePath(String)` now defaults to `LeaseFileLockingService` instead of `DiskLockingService` for out-of-the-box Kubernetes, container, and shared network storage compatibility. See `docs/DISK_BASED_LOCKING.md` for legacy opt-out instructions. -- **Unified Lock Wait Budget & Cloud Chunk Drain Calibration**: Replaced unreleased `withMaxLockRetries` with `withLockWaitTimeout(Duration)` (default 60s / 300 retries), synchronizing the lock wait timeout budget across storage backends and automatically deriving the cloud chunk drain timeout (`lockWaitTimeout - 5s`, default 55s) to guarantee in-flight chunks are cleanly flushed to cloud storage before releasing upload locks. -- **Absolute Base URL & Location Header Support**: Extended `withUploadUri(String)` to accept absolute base URLs (e.g. `https://upload.example.com/files`), returning full URLs in `Location` response headers for upload creation across both Tus 1.0.0 and RUFH protocols while preserving backward compatibility for relative paths. -- **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. -- **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. -- **Lock-Holding Stream Lifecycle**: `TusFileUploadService.getUploadedBytes(...)` now retains the upload lock until the returned stream is closed, preventing concurrent modifications or deletions from corrupting data mid-stream. -- **S3 Locking Optimization & Non-CAS Fallback**: Optimized `S3LockingService` and `S3UploadLock` by skipping the redundant `isLockExpired` pre-check on happy-path acquisitions (saving 1 remote GET round-trip) and using S3 Multi-Object Delete to delete `.lock` and `.stop` objects in a single batch round-trip on lock release. Made read-after-write verification jitter bounds configurable via `AbstractLeaseLockingService.setJitter(minMs, maxMs)` and `S3LockingService.withJitter(minMs, maxMs)`. Added automatic non-CAS fallback when `If-None-Match: *` returns `HTTP 501 Not Implemented` (enabling seamless out-of-the-box support for Backblaze B2 and Ceph RGW) along with `withS3ConditionalWritesSupported(boolean)` override. Streamlined `S3LockingService` constructors to minimal required set and documented zero-jitter throughput optimization for pure AWS S3 / Cloudflare R2 deployments in `docs/S3_STORAGE.md`. +### Upgrade notes (breaking & behavioural changes) +See [docs/UPGRADING.md](docs/UPGRADING.md) for details and code examples. +- **Java 17 and Jakarta EE**: requires Java 17+ and `jakarta.servlet.*` (Tomcat 10+, Jetty 11+, Spring Boot 3+). `javax.servlet` is no longer supported. +- **Default locking**: `withStoragePath(String)` now uses `LeaseFileLockingService` instead of `DiskLockingService`. Opt out with `withUploadLockingService(new DiskLockingService(path))`, see [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). +- **Lock-holding streams**: `getUploadedBytes(...)` keeps the upload locked until the returned stream is closed. Always use try-with-resources. +- **`process(...)` return value**: now returns the created/updated `UploadInfo` on successful `POST`, `PATCH`, and `HEAD` requests (or `null` on errors, `OPTIONS`, or `DELETE` termination). It also returns a value for progress requests, so check the HTTP method when recording a newly created upload. +- **Download extension (breaking)**: no longer returns `204` for uploads still in progress and no longer sets the `Tus-Resumable` header (needed for dual-protocol support). The `UploadInProgressException` class was removed. +- **Jackson bundled**: `jackson-databind`, `jackson-annotations` and `jackson-core` are now compile-scope dependencies; no separate declaration needed. +- **Protocol detection**: `ProtocolVersion.AUTO` is the default. Restrict with `withSupportedProtocolVersions(...)`, see [docs/PROTOCOLS.md](docs/PROTOCOLS.md). + +### New +- **IETF Resumable Uploads for HTTP (RUFH)**: full draft-12 support with automatic protocol detection, RFC 9651 structured headers (`Upload-Offset`, `Upload-Complete`, `Upload-Length`, `Upload-Limit`), RFC 7807 `application/problem+json` errors, RFC 9530 HTTP digests, `104` interim responses and a spec-quoted conformity suite (`scripts/rufh_conformity_test.py`). See [docs/PROTOCOLS.md](docs/PROTOCOLS.md), [docs/MIGRATION.md](docs/MIGRATION.md), [docs/INTERIM_RESPONSES.md](docs/INTERIM_RESPONSES.md) and [docs/CONFORMITY_TESTING.md](docs/CONFORMITY_TESTING.md). +- **NFS- & SMB-safe lease locking (`LeaseFileLockingService`)**: atomic sibling mutex directories, TTL lease files with heartbeat renewal and ownership fencing; safe on NFS v3/v4, AWS EFS, Azure Files, SMB/CIFS and local disks, with no Redis, ZooKeeper or `FileLock` daemons. See [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). +- **S3-compatible storage**: `S3StorageService`, `S3LockingService` (conditional writes, TTL leases, automatic non-CAS fallback for e.g. Backblaze B2 and Ceph RGW, configurable jitter via `withJitter(minMs, maxMs)` and `withS3ConditionalWritesSupported(boolean)`) and `S3ConcatenationService`, including server-side multipart composition that works on AWS S3 and MinIO/Ceph. See [docs/S3_STORAGE.md](docs/S3_STORAGE.md). +- **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService` (Blob Leases with auto-renewal and cross-replica stop signals) and zero-copy `AzureBlobConcatenationService`. See [docs/AZURE_BLOB_STORAGE.md](docs/AZURE_BLOB_STORAGE.md). +- **`UploadCompletionListener`**: register via `withUploadCompletionListener(...)` / `addUploadCompletionListener(...)`. Listeners run after the lock is released, so they can stream and delete the bytes immediately. Added `getUploadedBytes(UploadInfo)` and `deleteUpload(UploadInfo)` overloads. +- **`withLockWaitTimeout(Duration)`** (default 60 s): maximum time a request waits for a contended upload lock; also derives the cloud chunk drain timeout (`lockWaitTimeout - 5 s`) and retry budget. See [docs/LOCKING.md](docs/LOCKING.md). +- **Automatic lock cleanup** on completion, termination (`DELETE`) and expiration across all locking services (`cleanupLock(...)`), plus a fallback sweeper for abandoned locks. +- **JSON serialization**: store `UploadInfo` as JSON with `withJsonSerialization(true)`. +- **Absolute base URLs**: `withUploadUri(String)` accepts e.g. `https://upload.example.com/files`, returning full `Location` URLs for Tus and RUFH; relative paths keep working. ### Fixed -- **RFC 9110 Media Type Matching & MIME Parameter Tolerance**: Enhanced `Content-Type` header validation in `ContentTypeValidator`, `PostContentTypeValidator`, and `RufhAppendValidator` using RFC 9110 §8.3 compliant media-type parsing (`Utils.isMediaType`). Media type matching now tolerates MIME parameters (such as `;charset=UTF-8` automatically appended by Spring Boot `CharacterEncodingFilter`, servlet wrappers, proxies, or HTTP clients), whitespace variations, and case-insensitivity without incorrectly rejecting valid requests with `406 Not Acceptable`. -- **Clear Content-Length on Error Responses**: Cleared `Content-Length` response header prior to invoking `HttpServletResponse.sendError(...)` during exception handling, resolving buffer conflicts and exceptions in Undertow and other servlet containers ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). -- **Prevent Disk Truncate Underflow**: Guarded file truncate logic in `DiskStorageService` against underflow when removing bytes (`Math.max(0L, file.size() - byteCount)`). -- **File Channel Leak Prevention in FileBasedLock**: Guaranteed `FileChannel` is closed immediately upon lock acquisition errors to avoid file descriptor leaks. -- **Cloud Upload Pause & Drain Timeout Resilience**: Integrated `AsyncChunkUploader` default drain timeout (55 seconds, calibrated to `lockWaitTimeout - 5s`) with error-resilient recovery logic across `AzureBlobStorageService` and `S3StorageService`. Confirmed uploaded chunks and staged Azure blocks are committed and metadata (`UploadInfo` offset) is persisted to storage before throwing stream or drain exceptions, ensuring paused uploads (such as Uppy client pause/resume) preserve their progress and resume from the exact byte offset instead of restarting from 0. -- **Azure Blob Storage Upload Cancellation Lease Safety**: Checked lock blob lease state before calling `deleteIfExists()` in `AzureBlobStorageService.terminateUpload()`. When an active lease is held on the lock blob by an ongoing request (e.g. `DELETE` cancellation), attempting deletion without specifying the lease ID triggered an Azure SDK error log (`HTTP 412 LeaseIdMissing`). Skipping deletion for actively leased blobs prevents this error and allows normal lease release upon request completion. -- **Lock Contention Stream Interruption Handling**: Handled `IOException` from `InterruptibleInputStream` across upload request handlers (`CorePatchRequestHandler`, `RufhAppendPatchRequestHandler`, `RufhCreationPostRequestHandler`, `CreationWithUploadPostRequestHandler`). When an upload stream is interrupted by the locking service watchdog during lock contention (such as concurrent `HEAD` progress checks or `DELETE` cancellation requests), the storage backend commits all buffered bytes received so far and updates the offset in storage. Request handlers now reload the refreshed `UploadInfo` and return a clean HTTP 204/201 response with the updated offset rather than bubbling an unhandled `IOException` / HTTP 500 to the servlet container. -- **S3 Server-Side Part Composition & Native Multipart Copy**: Implemented `S3ServerSideComposeHelper` to resolve AWS S3 header rejection during server-side part composition. MinIO Java SDK 9.0.3's `composeObject` delegates to `UploadPartCopy` with an empty body placeholder that automatically injects `Content-MD5` and `Content-Type` headers, which Amazon AWS S3 strictly forbids on part copy requests and rejects with HTTP 400 (`The specified header is not valid in this context`). `S3ServerSideComposeHelper` executes native S3 multipart copy requests directly with SigV4 signing omitting `Content-MD5`, allowing fast zero-bandwidth server-side composition on both AWS S3 and MinIO/Ceph clusters without external AWS SDK dependencies. Added reflection-free constructors to `S3StorageService` and `S3LockingService` accepting connection parameters directly, with `eu-central-1` as the default fallback region, while preserving multi-tier streaming concatenation fallbacks. -- **S3 & Azure Pause/Resume Lock Contention & Byte Integrity**: Resolved lock contention and data corruption issues when uploads are repeatedly paused and resumed: - - **Lock Release on Interruption**: Updated `AsyncChunkUploader` with periodic contention polling (`waitForInFlight`) and bounded worker aborts (`500ms`), ensuring background upload threads cleanly terminate when locks are released and prevented locks from lingering until TTL expiry. - - **Manifest-Tracked Parts (`uploadPartKeys`)**: Replaced non-deterministic S3 prefix listing with an authoritative `uploadPartKeys` manifest in `UploadInfo`. Stale tail buffers are preserved in S3 until new manifests are committed, preventing byte loss or duplicate byte prepends (+8 MB offsets or drops to 0) during multiple pause/resume cycles. - - **Multi-Object Delete with Resilient Fallback**: Implemented batch Multi-Object Delete (`POST /?delete`) in `S3UploadLock` and `S3StorageService` with automatic per-object `DELETE` fallback when batch deletes are restricted by bucket IAM policies or unsupported by S3-compatible providers. - - **Transient Network Tolerance in Lease Renewals**: Tolerated transient network blips in `AbstractLeaseLock` and `AzureBlobUploadLock`, preventing unexpired valid locks from being prematurely aborted during heartbeat retries. - - **Graceful Lock Eviction**: Handled `NO_SUCH_KEY` cleanly in `S3LockingService.evictExpiredLock()` without generating false eviction warnings when locks were already released by their owners. +- **Content-Type matching**: media types are matched per RFC 9110 and tolerate parameters such as `;charset=UTF-8` (added by Spring `CharacterEncodingFilter`, proxies or clients) instead of returning `406`. +- **Undertow & other containers**: `Content-Length` is cleared before `sendError(...)` ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). +- **Disk**: truncate underflow guard in `DiskStorageService`; `FileChannel` leak on lock acquisition errors fixed in `FileBasedLock`. -### Breaking -- **Downloads**: In order to support both the Tus protocol and RUFH protocol, the unofficial download extension will not return a HTTP status code `204` for uploads that are still in progress and will not contain the response header `Tus-Resumable`. Removed the `UploadInProgressException` class. +### Security +- **Path traversal protection**: hardened `LeaseFileLockingService` against directory traversal via crafted upload identifiers ([#141](https://github.com/tomdesair/tus-java-server/pull/141)). +- **CORS origin validation**: prevented arbitrary `Origin` header reflection in CORS preflight responses ([#132](https://github.com/tomdesair/tus-java-server/pull/132)). +- **Secure temporary files**: replaced legacy `File.createTempFile` with `Files.createTempFile` enforcing owner-only POSIX permissions (`0600`, CWE-378 / SonarQube java:S5443) ([#131](https://github.com/tomdesair/tus-java-server/pull/131)). ## [1.0.0-3.3] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..13ae1b11 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,64 @@ +# Contributing + +This library comes without any warranty and is released under a [MIT license](https://github.com/tomdesair/tus-java-server/blob/master/LICENSE). If you encounter any bugs or if you have an idea for a useful improvement you are welcome to [open a new issue](https://github.com/tomdesair/tus-java-server/issues) or to [create a pull request](https://github.com/tomdesair/tus-java-server/pulls) with the proposed implementation. Please note that any contributed code needs to be accompanied by automated unit and/or integration tests and comply with the [defined code-style](#code-style). + +Useful guides: +- [Architecture guide](docs/ARCHITECTURE.md) +- [Testing guide](docs/TESTING.md) and coverage checks +- [Protocol conformity testing](docs/CONFORMITY_TESTING.md) +- [Release process](docs/RELEASE.md) + +## System Architecture & Where to Contribute + +Before proposing design changes or implementing new features, please read the **[Architecture Guide](docs/ARCHITECTURE.md)**. It provides a complete overview of: +- The single facade service class (`TusFileUploadService`) and the dual-protocol extension pipeline. +- How core business logic is encapsulated in `RequestValidator` and `RequestHandler` implementations. +- The pluggable storage architecture (including both lease-based and legacy file-lock-based disk implementations, S3, and Azure Blob). +- The `AbstractLeaseLockingService` framework for distributed lease-based locking and lock contention handling. +- The contributor blueprint mapping common extension scenarios to the right packages and classes. + +## Branching Strategy + +Any new feature, bugfix, or improvement should be developed in a separate branch that starts with either `feature/` or `bugfix/` and has a meaningful, concise name (e.g., `feature/lock-contention-resolution` or `bugfix/fix-upload-timeout`). + +## Code Style + +All pull requests should have the correct formatting according to [Google Java Style](https://github.com/google/google-java-format) code formatting. To verify if the code style is correct run: + +```bash +mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:check +``` + +To reformat your code run: + +```bash +mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:format +``` + +See the [Google Java Style Github page](https://github.com/google/google-java-format) on recommendations on how to configure this in your IDE. Or if you have Python 3, you can also use [pre-commit](https://pre-commit.com) to make your life easier: + +```bash +pip install pre-commit +pre-commit install +``` + +## Running Tests & Verifying Coverage + +### 1. Fast Offline Unit Tests +Unit tests run in isolation without external dependencies or containers: +```bash +mvn test -q +``` + +### 2. Integration Tests (Testcontainers) +Integration tests verify end-to-end storage, distributed locking, and protocol flows against local Testcontainers instances (RustFS for S3, Azurite for Azure Blob): +```bash +mvn verify -q +``` +> **Container Runtime**: Integration tests automatically spin up and manage container lifecycles via Testcontainers using either **Podman** or **Docker**. No manual container startup is required. If no container runtime is running, integration tests are gracefully skipped. + +### 3. Code Coverage Gate +Verify that modified code meets the required unit test coverage threshold (default: 90% per file): +```bash +python3 scripts/check-coverage.py --per-file-limit 90 +``` diff --git a/README.md b/README.md index d08c362d..02886496 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,32 @@ [![Maven Central](https://img.shields.io/maven-central/v/me.desair.tus/tus-java-server.svg)](https://central.sonatype.com/artifact/me.desair.tus/tus-java-server) [![Java 17+](https://img.shields.io/badge/Java-17%2B-blue.svg)](https://adoptium.net) [![Build and Tests](https://github.com/tomdesair/tus-java-server/actions/workflows/build.yml/badge.svg)](https://github.com/tomdesair/tus-java-server/actions?query=branch%3Amaster+) [![Coverage Status](https://coveralls.io/repos/github/tomdesair/tus-java-server/badge.svg?branch=master)](https://coveralls.io/github/tomdesair/tus-java-server?branch=master) [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=me.desair.tus%3Atus-java-server&metric=bugs)](https://sonarcloud.io/dashboard?id=me.desair.tus%3Atus-java-server) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=me.desair.tus%3Atus-java-server&metric=vulnerabilities)](https://sonarcloud.io/dashboard?id=me.desair.tus%3Atus-java-server) [![Duplicated Lines](https://sonarcloud.io/api/project_badges/measure?project=me.desair.tus%3Atus-java-server&metric=duplicated_lines_density)](https://sonarcloud.io/dashboard?id=me.desair.tus%3Atus-java-server) # tus-java-server -This library can be used to enable resumable (and potentially asynchronous) file uploads in any Java web application. This allows the users of your application to upload large files over slow and unreliable internet connections. The ability to pause or resume a file upload (after a connection loss or reset) is achieved by implementing the open file upload protocols tus (https://tus.io/) and Resumable Uploads for HTTP (RUFH). This library implements the server-side of the tus v1.0.0 protocol as well as the official IETF Resumable Uploads for HTTP specification ([draft-ietf-httpbis-resumable-upload](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/)), offering dual protocol version support. -The Javadoc of this library can be found at https://tus.desair.me/. As of version 2.0.0, this library requires Java 17+. +**Resumable (and potentially asynchronous) file uploads for any Java web application, with support for both the [tus 1.0.0](https://tus.io/) protocol and the new IETF [Resumable Uploads for HTTP (RUFH)](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/) standard.** -### Key Features -* ⚡ **Dual Protocol Support**: Seamless interoperability with [Tus 1.0.0](https://tus.io/) and the official [IETF Resumable Uploads for HTTP (draft-12)](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/). -* ☁️ **Pluggable Storage Backends**: Native support for Local Disk Volumes, NFS network shares, S3-compatible Object Storage (AWS S3, MinIO, Cloudflare R2, Ceph, GCS), and Azure Blob Storage. +This library lets the users of your application upload large files over slow and unreliable connections: an upload can be paused or resumed (after a connection loss or reset) without starting over. It implements the server side of the tus v1.0.0 protocol **and** the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload`, draft-12) on the same endpoint, so tus clients (Uppy, `tus-js-client`) and IETF RUFH clients work side by side. + +Javadoc: https://tus.desair.me/ · Requires Java 17+ (Jakarta Servlet 6.0) · Upgrading from 1.x? See the [Upgrade Guide](docs/UPGRADING.md). + +## Key Features +* ⚡ **IETF RUFH Ready**: Implements the official [IETF Resumable Uploads for HTTP (draft-12)](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/) — including `Upload-Complete`, `Upload-Limit`, RFC 7807 problem details, RFC 9530 digests and `104` interim responses — alongside full [Tus 1.0.0](https://tus.io/) support. Protocol is auto-detected per request. *(RUFH is still an IETF draft, so treat that part as experimental until it becomes an RFC.)* +* ☁️ **Pluggable Storage Backends**: Local Disk Volumes, NFS network shares, S3-compatible Object Storage (AWS S3, MinIO, Cloudflare R2, Ceph, GCS), and Azure Blob Storage. * 🔒 **Zero-Database Distributed Locking**: Built-in lease-based locking enabling multi-replica cluster & Kubernetes container deployments without requiring Redis or relational databases. * 🛡️ **Data Integrity & Resiliency**: Built-in HTTP Digests ([RFC 9530](https://www.rfc-editor.org/rfc/rfc9530.html)), checksum verification, and duplicate upload deduplication. * 🚀 **Production-Ready & Lightweight**: Minimal dependencies (Jakarta Servlet API 6.0 & Apache Commons), non-blocking lock contention resolution, and thread-local caching. ## Table of Contents -- [Storage Backend Options](#storage-backend-options) -- [How Resumable Uploads Work](#how-resumable-uploads-work) -- [Quick Start and Examples](#quick-start-and-examples) -- [Usage and Configuration](#usage-and-configuration) - - [1. Setup & Configuration Options](#1-setup) - - [2. Receiving a Resumable Upload](#2-receiving-a-resumable-upload) - - [3. Handling Upload Completion & Retrieving Files](#3-handling-upload-completion--retrieving-files) - - [4. Upload cleanup](#4-upload-cleanup) -- [Protocol Version Support (Tus 1.0.0 & IETF Resumable Uploads)](#protocol-version-support-tus-100--ietf-resumable-uploads) -- [Protocol Extensions](#protocol-extensions) -- [Advanced Usage](#advanced-usage) - - [HTTP Digests (RFC 9530)](#http-digests-rfc-9530) - - [Emitting HTTP 104 Interim Responses in Tomcat / Spring Boot](#emitting-http-104-interim-responses-in-tomcat--spring-boot) -- [Compatible Client Implementations & Conformity Testing](#compatible-client-implementations--conformity-testing) +- [Quick Start](#quick-start) +- [Try It with curl](#try-it-with-curl) +- [How It Works](#how-it-works) +- [Documentation](#documentation) +- [Compatible Clients & Conformity Testing](#compatible-clients--conformity-testing) - [Versioning](#versioning) - [Contributing](#contributing) -## Storage Backend Options - -`tus-java-server` provides pluggable storage architecture supporting multiple backend storage options: - -1. **File Disk & Network Storage** (`DiskStorageService` & `LeaseFileLockingService`): - - **Local File System**: Direct disk storage on application server instance. - - **Shared NFS Network Drives**: Distributed, container-safe lease locking for multi-server setups (NFSv3/v4, AWS EFS, Azure Files, SMB/CIFS). See [Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md). - - **Kubernetes Persistent Volume**: Mounted volume (`ReadWriteMany` / `ReadWriteOnce`) for containerized applications. -2. **S3-Compatible Object Storage** (`S3StorageService`, `S3LockingService`, & `S3ConcatenationService`): - - **Cloud & On-Premise S3**: AWS S3, MinIO, RustFS, Cloudflare R2, Ceph, or Google Cloud Storage. - - **Multi-Replica Support**: Uses distributed S3 object locking and TTL leases, enabling multi-replica container deployments without requiring Redis or external databases. See [S3 Storage Guide](docs/S3_STORAGE.md). -3. **Azure Blob Storage** (`AzureBlobStorageService`, `AzureBlobLockingService`, & `AzureBlobConcatenationService`): - - **Microsoft Azure Cloud**: Native Azure Blob Storage using the `azure-storage-blob` SDK. - - **Multi-Replica Support**: Uses native Azure Blob Leases (30s renewable leases) for distributed locking across cluster replicas with automatic lock cleanup on upload completion, termination, and expiration. See [Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md). - -## How Resumable Uploads Work - -### Tus 1.0.0 Protocol Flow -```mermaid -sequenceDiagram - autonumber - actor Client as Frontend Client (Uppy / tus-js-client) - participant ProtocolEndpoint as Server: Tus Protocol Endpoint (/api/upload) - participant Storage as Storage Backend (Disk / S3 / Azure) - participant AppEndpoint as Application API (/api/documents) - - Note over Client,Storage: 1. Create Resumable Upload Resource - Client->>ProtocolEndpoint: POST /api/upload
(Tus-Resumable: 1.0.0, Upload-Length: 1048576, Upload-Metadata: ...) - ProtocolEndpoint->>Storage: Initialize upload metadata - ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a..., Tus-Resumable: 1.0.0) - - Note over Client,Storage: 2. Stream Data Chunk(s) - Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Content-Type: application/offset+octet-stream) - ProtocolEndpoint->>Storage: Append chunk bytes to storage - ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Tus-Resumable: 1.0.0) - - Note over Client,AppEndpoint: 3. Domain Notification & Consumption - Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") - AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) - AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) - AppEndpoint-->>Client: 200 OK (Processed) -``` - -### IETF Resumable Uploads for HTTP (RUFH) Flow -```mermaid -sequenceDiagram - autonumber - actor Client as Frontend Client (RUFH Client) - participant ProtocolEndpoint as Server: RUFH Protocol Endpoint (/api/upload) - participant Storage as Storage Backend (Disk / S3 / Azure) - participant AppEndpoint as Application API (/api/documents) - - Note over Client,Storage: 1. Create Resumable Upload Resource - Client->>ProtocolEndpoint: POST /api/upload
(Upload-Complete: ?0) - ProtocolEndpoint->>Storage: Initialize upload metadata - ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a...) - - Note over Client,Storage: 2. Stream Data Chunk(s) - Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Upload-Complete: ?1, Content-Type: application/partial-upload) - ProtocolEndpoint->>Storage: Append chunk bytes to storage - ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Upload-Complete: ?1) - - Note over Client,AppEndpoint: 3. Domain Notification & Consumption - Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") - AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) - AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) - AppEndpoint-->>Client: 200 OK (Processed) -``` +## Quick Start -## Quick Start and Examples -The tus-java-server library only depends on Jakarta Servlet API 6.0 and some Apache Commons utility libraries. This -means that (in theory) you can use this library on any modern Java Web Application server like Tomcat, JBoss, Jetty... By default all uploaded data and information is stored on a (shared) file volume of the application server. - -You can add the latest stable version of this library to your application using Maven or Gradle: +### 1. Add the dependency **Maven:** ```xml @@ -124,97 +47,50 @@ implementation 'me.desair.tus:tus-java-server:2.0.0' implementation("me.desair.tus:tus-java-server:2.0.0") ``` -Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are bundled directly with compile scope by `tus-java-server`. When using S3 storage (`S3StorageService`), simply include the MinIO Java SDK dependency matching `pom.xml`: - -```xml - - io.minio - minio - 9.0.3 - -``` - -The main entry point of the library is the `me.desair.tus.server.TusFileUploadService.process(jakarta.servlet.http.HttpServletRequest, jakarta.servlet.http.HttpServletResponse)` method. You can call this method inside a `jakarta.servlet.http.HttpServlet`, a `jakarta.servlet.Filter` or any REST API controller of a framework that gives you access to `HttpServletRequest` and `HttpServletResponse` objects. In the following list, you can find some example implementations: +S3 and Azure backends need an additional SDK dependency, see the [Configuration Reference](docs/CONFIGURATION.md#dependencies). -* [Resumable and asynchronous file upload in Spring Boot REST API with Uppy JavaScript client.](https://github.com/tomdesair/tus-java-server-spring-demo) -* [Detailed blog post by Ralph](https://golb.hplar.ch/2019/06/upload-with-tus.html) on how to use this library in [Spring Boot in combination with the Tus JavaScript client](https://github.com/ralscha/blog2019/tree/master/uploadtus). -* [Resumable and asynchronous file upload using Uppy with form submission in Dropwizard (Jetty)](https://github.com/tomdesair/tus-java-server-dropwizard-demo) -* (more examples to come!) +### 2. Create the service (once) and forward requests to it -#### Frontend Client Example (Uppy / JavaScript) -Connect any standard Tus client (e.g. [Uppy](https://uppy.io/) or `tus-js-client`) to your backend upload endpoint: +The main entry point is `me.desair.tus.server.TusFileUploadService.process(HttpServletRequest, HttpServletResponse)`. Call it from any `HttpServlet`, `Filter` or REST controller that gives you the servlet request and response. Create the service **once** (it is thread-safe) and share it. By default uploads are stored on a (shared) file volume. -```javascript -import Uppy from '@uppy/core'; -import Tus from '@uppy/tus'; +**Spring Boot 3** (`spring-boot-starter-web`, Java 17+). The snippets are complete and compile as-is; adjust the paths: -const uppy = new Uppy().use(Tus, { - endpoint: 'http://localhost:8080/api/upload', - chunkSize: 5 * 1024 * 1024 // 5MB chunk size -}); -``` - -## Usage and Configuration +```java +import me.desair.tus.server.TusFileUploadService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; -### 1. Setup -The first step is to create a `TusFileUploadService` object using its constructor. You can make this object available as a (Spring bean) singleton or create a new instance for each request. For example, in a Spring Boot application: +@Configuration +public class TusConfig { -```java -@Bean -public TusFileUploadService tusFileUploadService() { + @Bean(destroyMethod = "close") + public TusFileUploadService tusFileUploadService() { return new TusFileUploadService() .withStoragePath("/path/to/uploads") .withUploadUri("/api/upload") + .withUploadDeduplication(true) .withThreadLocalCache(true); + } } ``` -See the [tus-java-server-spring-demo](https://github.com/tomdesair/tus-java-server-spring-demo) repository for a complete Spring Boot reference implementation. - -After creating the object, you can configure it using the following methods: - -#### Configuration Options Reference - -| Method | Default | Description | -|---|---|---| -| `withUploadUri(String)` | `null` | Sets relative path (e.g. `/api/upload`) or absolute base URL (e.g. `https://upload.example.com/api/upload`) under which the upload endpoint is exposed. Supports regex parameters (e.g. `/users/[0-9]+/files/upload`). | -| `withStoragePath(String)` | `${java.io.tmpdir}/tus` | Path on the filesystem or shared drive where uploaded bytes and metadata are stored when using `DiskStorageService`. | -| `withSupportedProtocolVersions(ProtocolVersion)` | `ProtocolVersion.AUTO` | Configures protocol handling: `AUTO` (header-based auto-detection), `TUS_1_0_0` (Tus 1.0.0 only), or `RUFH` (IETF draft-12 only). | -| `withMaxUploadSize(Long)` | `Long.MAX_VALUE` | Maximum allowed total upload size in bytes per upload resource. | -| `withLockWaitTimeout(Duration)` | `Duration.ofSeconds(60)` | Maximum duration a request waits to acquire an upload lock held by an in-flight transfer (retrying every 200ms, resulting in 300 retries). Automatically configures cloud background chunk drain timeout to 5s less than this value (default 55s). | -| `withChunkedTransferDecoding(Boolean)` | `false` | Enables manual chunked HTTP decoding for servlet containers that do not decode chunked requests natively. | -| `withThreadLocalCache(Boolean)` | `false` | Enables in-memory thread-local caching of upload request data to reduce storage backend I/O load. | -| `withUploadExpirationPeriod(Long)` | `null` (disabled) | Expiration period in milliseconds after which incomplete/expired uploads become eligible for cleanup. | -| `withDownloadFeature()` | Disabled | Enables the unofficial `download` extension allowing clients to retrieve uploaded bytes via HTTP `GET`. | -| `withUploadDeduplication(Boolean)` | `false` | Enables duplicate file detection by checksum, linking new uploads (`duplicatesUploadId`) and skipping redundant storage writes. | -| `addTusExtension(TusExtension)` | All standard enabled | Adds a custom extension (e.g. application authorization checks). | -| `disableTusExtension(String)` | None | Disables a built-in extension (`creation`, `checksum`, `expiration`, `concatenation`, `termination`, `download`, `cors`). | -| `withUploadIdFactory(UploadIdFactory)` | `UuidUploadIdFactory` | Custom ID generator for upload resources (e.g., `UuidUploadIdFactory` or `TimeBasedUploadIdFactory`). | -| `withUploadCompletionListener(UploadCompletionListener)` | None | Registers a callback invoked immediately when an upload finishes transferring all bytes and is completed. | -| `withCloudUploadThreadPoolSize(int)` | `10` | Maximum number of worker threads used for asynchronous background chunk uploading in cloud storage backends (S3, Azure). | -| `withJsonSerialization()` | Java serialization | Enables JSON serialization for upload metadata (`UploadInfo`). Jackson is bundled by default. | -| `withUploadStorageService(UploadStorageService)` | `DiskStorageService` | Configures custom or cloud storage backend (`DiskStorageService`, `S3StorageService`, `AzureBlobStorageService`). | -| `withUploadLockingService(UploadLockingService)` | `LeaseFileLockingService` | Configures custom or cloud locking backend (`LeaseFileLockingService`, `S3LockingService`, `AzureBlobLockingService`). | - -The library provides filesystem-based storage (`DiskStorageService` / `LeaseFileLockingService`), S3-compatible object storage (`S3StorageService` / `S3LockingService`), and Azure Blob Storage (`AzureBlobStorageService` / `AzureBlobLockingService`). Cloud backends feature an asynchronous 3-slot chunk pipeline (`AsyncChunkUploader`) that overlaps client payload streaming with cloud staging in the background. See the **[Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md)**, **[S3 Storage Guide](docs/S3_STORAGE.md)**, and **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md)** for detailed instructions on multi-replica container deployments in Kubernetes, post-upload processing, and legacy locking opt-out. - -### 2. Receiving a Resumable Upload -To process an upload request you have to pass the current `jakarta.servlet.http.HttpServletRequest` and `jakarta.servlet.http.HttpServletResponse` objects to the `me.desair.tus.server.TusFileUploadService.process()` method. Typical places were you can do this are inside Servlets, Filters or REST API Controllers. - -The `process()` method returns the processed `UploadInfo` object (or `null` for `OPTIONS` preflight requests or when an error response is generated). - -For example, in a Spring MVC REST Controller: - ```java -@Controller -@CrossOrigin(origins = "*") +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@RestController public class FileUploadController { - @Autowired - private TusFileUploadService tusFileUploadService; + private final TusFileUploadService tusFileUploadService; - @Autowired - private UserSessionRepository userSessionRepository; + public FileUploadController(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } @RequestMapping( value = {"/api/upload", "/api/upload/**"}, @@ -227,218 +103,194 @@ public class FileUploadController { RequestMethod.OPTIONS, RequestMethod.GET }) - public void processUpload( - HttpServletRequest request, HttpServletResponse response, @RequestParam String userSessionId) + public void processUpload(HttpServletRequest request, HttpServletResponse response) throws IOException { - - UploadInfo info = tusFileUploadService.process(request, response, userSessionId); - - // Creation vs. Progress Ambiguity: Check if this was a new upload creation request - if (info != null && "POST".equalsIgnoreCase(request.getMethod())) { - userSessionRepository.recordUpload(userSessionId, info.getId()); - } + tusFileUploadService.process(request, response); } } ``` -> [!NOTE] -> **Creation vs. Progress Ambiguity**: The `process()` method returns an `UploadInfo` on both creation (`POST`) and progress updates (`PATCH` / `HEAD`). When associating an upload ID with a user session or database entity on creation, check `info != null && "POST".equalsIgnoreCase(request.getMethod())`. -> -> **Single-Request Upload Completion**: When clients upload the entire file in a single request using the `creation-with-upload` extension or RUFH single-request POST, the upload is already complete when `process()` returns, and any registered `UploadCompletionListener` will have already fired *before* `process()` returns to your controller. - -Optionally you can also pass a `String ownerKey` parameter to `process()`. The `ownerKey` can be used to have a hard separation between uploads of different users, groups or tenants in a multi-tenant setup. Examples of `ownerKey` values are user ID's, group names, client ID's... - -### 3. Handling Upload Completion & Retrieving Files - -You can handle upload completion in two ways: -1. **Server-Side Callback Listener (`UploadCompletionListener`)**: Register a hook directly on `TusFileUploadService` that triggers automatically when the final chunk is uploaded. -2. **Domain API Endpoint**: Have your frontend Tus client notify your application backend domain endpoint (e.g. `POST /api/documents`) once upload completes. - -#### Option A: Server-Side `UploadCompletionListener` -Register one or more completion listeners on your `TusFileUploadService`. When an upload finishes transferring all bytes, the callback executes with the completed `UploadInfo` and the service instance. The upload lock is released *before* the listener is invoked, allowing you to safely stream or delete uploaded bytes immediately: - -```java -@Bean -public TusFileUploadService tusFileUploadService() { - return new TusFileUploadService() - .withStoragePath("/path/to/uploads") - .withUploadUri("/api/upload") - .withUploadCompletionListener((uploadInfo, service) -> { - // Filename needs to be set as metadata by the client - String fileName = uploadInfo.getMetadata().get("filename"); - try (InputStream is = service.getUploadedBytes(uploadInfo)) { - Files.copy(is, Paths.get("/var/data/documents", fileName)); - } catch (Exception e) { - log.error("Failed to process completed upload {}", uploadInfo.getId(), e); - } - // Optionally delete upload temporary files after processing - service.deleteUpload(uploadInfo); - }); -} -``` - -#### Option B: Client-Initiated Domain Notification -When an upload completes, the client receives the final `204 No Content` response from the Tus/RUFH protocol endpoint (`/api/upload/...`). Because Tus and RUFH are decoupled file transport protocols, your frontend application can notify your backend domain API (e.g. `POST /api/documents`) that the upload is complete and pass along the `uploadUrl`. - -> [!NOTE] -> `POST /api/documents` represents your application's domain REST endpoint, not a protocol endpoint. The Tus server itself handles file transfer (`/api/upload`), while your application endpoint coordinates business logic, database persistence, and final file consumption. +**Plain Jakarta Servlet 6 (Tomcat 10+, Jetty 12, Undertow...)** ```java -@RestController -public class DocumentController { - - @Autowired - private TusFileUploadService tusFileUploadService; - - @PostMapping("/api/documents") - public ResponseEntity completeDocumentUpload(@RequestBody DocumentUploadRequest request) - throws IOException { - String uploadUrl = request.getUploadUrl(); +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; + +@WebServlet("/api/upload/*") +public class UploadServlet extends HttpServlet { + + private TusFileUploadService tus; + + @Override + public void init() { + tus = + new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true); + } - // 1. Retrieve upload metadata (filename, original length, custom client metadata) - UploadInfo info = tusFileUploadService.getUploadInfo(uploadUrl); - String originalFileName = info.getMetadata().get("filename"); + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Override service() (not doPost/doGet) so that PATCH, HEAD and OPTIONS reach the library + tus.process(req, resp); + } - // 2. Stream uploaded bytes to permanent storage, database, or virus scanner - try (InputStream is = tusFileUploadService.getUploadedBytes(uploadUrl)) { - Files.copy(is, Paths.get("/var/data/documents", originalFileName)); + @Override + public void destroy() { + try { + tus.close(); + } catch (IOException e) { + log("Failed to close TusFileUploadService", e); } - - // 3. Clean up the temporary upload bytes and locks - tusFileUploadService.deleteUpload(uploadUrl); - - return ResponseEntity.ok().build(); } } ``` -Using the `me.desair.tus.server.TusFileUploadService.getUploadInfo(String uploadUrl)` method you can retrieve metadata about a specific upload process. This includes metadata provided by the client as well as metadata kept by the library like creation timestamp, creator ip-address list, upload length... The method `UploadInfo.getId()` will return the unique identifier of this upload encapsulated in an `UploadId` instance. The original (custom generated) identifier object of this upload can be retrieved using `UploadId.getOriginalObject()`. A URL safe string representation of the identifier is returned by `UploadId.toString()`. It is highly recommended to consult the [JavaDoc of both classes](https://tus.desair.me/). - -#### Cloud Storage Native Processing (S3 & Azure Blob) -When using cloud object storage backends, downstream services can directly obtain the raw cloud object key or blob name to perform zero-download server-side copying, background job processing, or direct cloud SDK operations: +CORS and `OPTIONS` preflight are handled by the library itself. More variants (multi-tenancy, cleanup, security) are in the [Integration Guide](docs/INTEGRATION_GUIDE.md). -* **S3 / MinIO Storage**: Use `((S3StorageService) tusFileUploadService.getUploadStorageService()).getS3ObjectKey(uploadUrl)` to get the full S3 object key (e.g. `uploads/018f3a...`). For complete server-side `copyObject` examples and MinIO SDK usage, see the **[S3 Storage Guide](docs/S3_STORAGE.md#4-post-upload-processing-gets3objectkey)**. -* **Azure Blob Storage**: Use `((AzureBlobStorageService) tusFileUploadService.getUploadStorageService()).getAzureBlobName(uploadUrl)` to get the full blob name. For complete Azure SDK examples with `BlobClient`, see the **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md#4-post-upload-processing-getazureblobname)**. +> [!NOTE] +> When using Spring Security, the upload endpoint must be authenticated like any other endpoint, but `OPTIONS` preflight requests must be allowed through. If your Spring Boot setup uses a `/api` prefix or context path, make sure `withUploadUri(...)` matches the path as seen by the application. -### 4. Upload cleanup -After having processed the uploaded bytes on the server backend (e.g. copy them to their final persistent location), it's important to cleanup the (temporary) uploaded bytes. This can be done by calling the `me.desair.tus.server.TusFileUploadService.deleteUpload(String uploadUri)` method as shown in the example above. This will remove the uploaded bytes and any associated upload information from the storage backend. Alternatively, a client can also remove an (in-progress) upload using the [termination extension](https://tus.io/protocols/resumable-upload.html#termination). +### 3. Handle finished uploads -Next to removing uploads after they have been completed and processed by the backend, it is also recommended to schedule a regular maintenance task to clean up any expired uploads or locks. Cleaning up expired uploads and locks can be achieved using the `me.desair.tus.server.TusFileUploadService.cleanup()` method: +Register an `UploadCompletionListener` on the service from step 2 (Spring Boot version shown, replace the `TusConfig` class): ```java -// Run periodically (e.g., via @Scheduled in Spring) -@Scheduled(fixedDelay = 600000) // Every 10 minutes -public void cleanupExpiredUploads() { - tusFileUploadService.cleanup(); +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import me.desair.tus.server.TusFileUploadService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class TusConfig { + + private static final Logger log = LoggerFactory.getLogger(TusConfig.class); + + @Bean(destroyMethod = "close") + public TusFileUploadService tusFileUploadService() { + return new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true) + .withUploadCompletionListener( + (uploadInfo, service) -> { + // Safely retrieve or guess the file name (preventing path traversal and NPE) + String fileName = Paths.get(uploadInfo.getFileName()).getFileName().toString(); + try { + try (InputStream is = service.getUploadedBytes(uploadInfo)) { + Files.copy(is, Paths.get("/var/data/documents", fileName)); + } // closing the stream releases the lock, which deleteUpload needs + service.deleteUpload(uploadInfo); + } catch (Exception e) { + log.error("Failed to process completed upload {}", uploadInfo.getId(), e); + } + }); + } } ``` -## Protocol Version Support (Tus 1.0.0 & IETF Resumable Uploads) +Alternatively let the client notify your own REST endpoint and call `getUploadInfo`, `getUploadedBytes` and `deleteUpload` there. Also schedule `tusFileUploadService.cleanup()` to remove expired uploads. See [Handling Upload Completion](docs/INTEGRATION_GUIDE.md#2-handling-upload-completion--retrieving-files) and [Cleanup](docs/INTEGRATION_GUIDE.md#3-cleanup). -> [!WARNING] -> **Experimental Feature Disclaimer**: The IETF Resumable Uploads for HTTP (RUFH) specification (`draft-ietf-httpbis-resumable-upload`) is currently an active IETF draft. While this library implements draft-12 compliance, the RUFH protocol support should be considered **experimental** until the specification is published as an official RFC standard. +### 4. Connect a client -`tus-java-server` supports both protocol specifications seamlessly: -1. **Tus 1.0.0**: The widely-adopted [tus protocol standard](https://tus.io/). -2. **IETF Resumable Uploads for HTTP**: The official IETF standardization draft ([draft-ietf-httpbis-resumable-upload](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/)). +```javascript +import Uppy from '@uppy/core'; +import Tus from '@uppy/tus'; -### Configuring Protocol Version -You can configure protocol support via `withSupportedProtocolVersions(ProtocolVersion)`: +const uppy = new Uppy().use(Tus, { + endpoint: 'http://localhost:8080/api/upload', + chunkSize: 5 * 1024 * 1024 // 5MB chunk size +}); +``` -* `ProtocolVersion.AUTO` (Default): Automatically detects protocol version per HTTP request based on request headers (`Tus-Resumable` header triggers Tus 1.0.0; `Upload-Complete` header or `application/partial-upload` content type triggers IETF RUFH). -* `ProtocolVersion.TUS_1_0_0`: Enforces Tus 1.0.0 handling exclusively. -* `ProtocolVersion.RUFH`: Enforces IETF Resumable Uploads for HTTP (RUFH) handling exclusively. +Complete working examples: -### Protocol Comparison & Available Features +* [Resumable and asynchronous file upload in Spring Boot REST API with Uppy JavaScript client.](https://github.com/tomdesair/tus-java-server-spring-demo) +* [Detailed blog post by Ralph](https://golb.hplar.ch/2019/06/upload-with-tus.html) on how to use this library in [Spring Boot in combination with the Tus JavaScript client](https://github.com/ralscha/blog2019/tree/master/uploadtus). +* [Resumable and asynchronous file upload using Uppy with form submission in Dropwizard (Jetty)](https://github.com/tomdesair/tus-java-server-dropwizard-demo) -| Feature / Capability | Tus 1.0.0 | IETF Resumable Uploads | -|---|---|---| -| **Auto-Detection Signal** | `Tus-Resumable: 1.0.0` header | `Upload-Complete` header or `Content-Type: application/partial-upload` | -| **Creation** | `POST` with `Upload-Length` / `Upload-Defer-Length` | `POST` with `Upload-Complete: ?0` / `?1` (or `application/partial-upload`) | -| **Append Chunks** | `PATCH` with `Upload-Offset` | `PATCH` with `Upload-Offset` & `Content-Type: application/partial-upload` | -| **Upload Status Query** | `HEAD` returns `Upload-Offset` & `Upload-Length` | `HEAD` returns `Upload-Offset` & `Upload-Complete` | -| **Offset Mismatch Error** | HTTP 409 Conflict | HTTP 409 Conflict with RFC 7807 `application/problem+json` details | -| **104 Interim Responses** | N/A | Supported (see [docs/INTERIM_RESPONSES.md](docs/INTERIM_RESPONSES.md)) | -| **Upload Cancellation** | `DELETE` with `Tus-Resumable: 1.0.0` | `DELETE` with `Upload-Complete: ?0` | -| **Checksum Validation** | Supported (`Checksum` extension) | Supported (based on HTTP Digests / RFC 9530) | -| **Expiration Handling** | Supported (`Upload-Expires` header) | Supported (`max-age` parameter in `Upload-Limit` header) | -| **Concatenation** | Supported (`Concatenation` extension) | N/A | -| **Download Extension** | Supported (`Download` extension) | Supported (`Download` extension; without it, `GET` requests perform offset retrieval) | +The `scripts/` folder contains runnable Python test clients for both protocols ([Conformity Testing Guide](docs/CONFORMITY_TESTING.md)). -## Protocol Extensions -Besides the [tus core protocol](https://tus.io/protocols/resumable-upload.html#core-protocol), the library has all optional tus protocol extensions enabled by default. This means that the `Tus-Extension` header has value `creation,creation-defer-length,creation-with-upload,checksum,checksum-trailer,termination,expiration,concatenation,concatenation-unfinished`. Optionally you can also enable an unofficial `download` extension (see [configuration section](#usage-and-configuration)). +## Try It with curl -* [creation](https://tus.io/protocols/resumable-upload.html#creation): The creation extension allows you to create new uploads and to retrieve the upload URL for them. -* [creation-defer-length](https://tus.io/protocols/resumable-upload.html#post): You can create a new upload even if you don't know its final length at the time of creation. -* [creation-with-upload](https://tus.io/protocols/resumable-upload.html#creation): The creation-with-upload extension allows you to create the upload resource and upload initial file data in a single POST request. -* [checksum](https://tus.io/protocols/resumable-upload.html#checksum): An extension that allows you to verify data integrity of each upload (PATCH) request. -* [checksum-trailer](https://tus.io/protocols/resumable-upload.html#checksum): If the checksum hash cannot be calculated at the beginning of the upload, it may be included as a trailer HTTP header at the end of the chunked HTTP request. -* [termination](https://tus.io/protocols/resumable-upload.html#termination): Clients can terminate completed or in-progress uploads which allows the tus-java-server library to free up resources on the server. -* [expiration](https://tus.io/protocols/resumable-upload.html#expiration): You can instruct the tus-java-server library to cleanup uploads that are older than a configurable period. The expiration extension applies to both Tus 1.0.0 and IETF Resumable Uploads for HTTP (RUFH) protocols: - * **Tus 1.0.0 Protocol**: Expiration is communicated to clients via the `Upload-Expires` response header field formatted as an HTTP date-time (RFC 7231, e.g. `Upload-Expires: Wed, 25 Jun 2026 16:00:00 GMT`). - * **IETF RUFH Protocol**: Expiration is communicated to clients via the `max-age` parameter in the `Upload-Limit` response header field indicating remaining valid seconds (e.g. `Upload-Limit: max-size=1048576, max-age=3600`). -* [concatenation](https://tus.io/protocols/resumable-upload.html#concatenation): This extension can be used to concatenate multiple uploads into a single final upload enabling clients to perform parallel uploads and to upload non-contiguous chunks. -* [concatenation-unfinished](https://tus.io/protocols/resumable-upload.html#concatenation): The client is allowed send the request to concatenate partial uploads while these partial uploads are still in progress. -* [http-digests](https://datatracker.ietf.org/doc/rfc9530/): An extension implementing RFC 9530 to verify data integrity for the Resumable Uploads for HTTP (RUFH) protocol. Supported headers include `Content-Digest`, `Repr-Digest`, `Want-Content-Digest`, and `Want-Repr-Digest`. -* `download`: The (unofficial) download extension allows clients to download uploaded files using a HTTP `GET` request. You can enable this extension by calling the `withDownloadFeature()` method. -* `cors`: The (unofficial) CORS extension adds native CORS support out-of-the-box, setting CORS headers for all requests and responses, and handling preflight `OPTIONS` requests automatically. It is enabled by default. +Verify your endpoint without a frontend (replace the URL and use the `Location` header returned by the `POST`). -## Advanced Usage +**IETF Resumable Uploads (RUFH)** +```bash +# 1. Create the upload resource (more data will follow) +curl -i -X POST http://localhost:8080/api/upload -H "Upload-Complete: ?0" +# 2. Send the data and mark the upload as complete +curl -i -X PATCH http://localhost:8080/api/upload/ \ + -H "Upload-Offset: 0" -H "Upload-Complete: ?1" \ + -H "Content-Type: application/partial-upload" --data-binary "hello world" +# 3. Query progress at any time +curl -I http://localhost:8080/api/upload/ +``` -### HTTP Digests ([RFC 9530](https://www.rfc-editor.org/rfc/rfc9530.html)) -The `http-digests` extension implements RFC 9530 to support data integrity checks for both individual data chunks (`Content-Digest`) and the entire file (`Repr-Digest`). -* **Performance Disclaimer**: Calculating representation digests (`Repr-Digest`) requires streaming the entire uploaded file from disk. For extremely large files, this can introduce non-trivial I/O performance overhead on the server. To optimize, only request it via `Want-Repr-Digest` when absolutely necessary. +**Tus 1.0.0** +```bash +curl -i -X POST http://localhost:8080/api/upload \ + -H "Tus-Resumable: 1.0.0" -H "Upload-Length: 11" -H "Upload-Metadata: filename aGVsbG8udHh0" +curl -i -X PATCH http://localhost:8080/api/upload/ \ + -H "Tus-Resumable: 1.0.0" -H "Upload-Offset: 0" \ + -H "Content-Type: application/offset+octet-stream" --data-binary "hello world" +curl -I http://localhost:8080/api/upload/ -H "Tus-Resumable: 1.0.0" +``` -### Emitting HTTP 104 Interim Responses in Tomcat / Spring Boot -Because standard Java Servlet API (`HttpServletResponse`) does not natively support emitting 1xx informational responses, applications running on embedded Tomcat (such as Spring Boot) can use a custom Tomcat `Valve` to write the raw HTTP 104 interim response frame directly to Tomcat's underlying TCP socket buffer before servlet execution. +## How It Works -For complete architectural details, Tomcat/Servlet API limitations, and production recommendations, see the dedicated **[HTTP 104 Interim Responses Guide](docs/INTERIM_RESPONSES.md)**. +Your application only forwards HTTP requests to `TusFileUploadService`; the library implements the protocol, stores the bytes in a pluggable backend, locks uploads against concurrent writers and resolves lock contention when a client resumes. Your application consumes the finished upload and deletes it. -A live, production-ready reference implementation using cached reflection can be found in the [tus-java-server-spring-demo](https://github.com/tomdesair/tus-java-server-spring-demo) repository: -* [`TusInterimResponseTomcatValve`](https://github.com/tomdesair/tus-java-server-spring-demo/blob/main/spring-boot-rest/src/main/java/me/desair/spring/tus/TusInterimResponseTomcatValve.java) +```mermaid +sequenceDiagram + autonumber + actor Client as Client (Uppy / tus-js-client / RUFH client) + participant Protocol as Upload endpoint (/api/upload) + participant Storage as Storage (Disk / S3 / Azure) + participant App as Your application API + + Client->>Protocol: POST (create upload) + Protocol->>Storage: Initialize upload + Protocol-->>Client: 201 Created (Location) + Client->>Protocol: PATCH chunk(s), resumable after a failure + Protocol->>Storage: Append bytes + Protocol-->>Client: 204 (Upload-Offset) + Client->>App: "Upload finished" (or UploadCompletionListener) + App->>Storage: getUploadedBytes(...), deleteUpload(...) +``` -To register the Valve in Spring Boot via `TomcatServletWebServerFactory`: +Detailed Tus and RUFH flow diagrams: [Protocols](docs/PROTOCOLS.md). -```java -@Bean -public TomcatServletWebServerFactory tomcatFactory(TusFileUploadService tusFileUploadService) { - TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory(); - factory.addContextValves(new TusInterimResponseTomcatValve(tusFileUploadService)); - return factory; -} -``` +## Documentation -## Compatible Client Implementations & Conformity Testing -This server implementation has been tested with: -- **Tus 1.0.0 Clients & Conformity Tests**: Tested with [Uppy](https://uppy.io/), `tus-js-client`, and our built-in Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) validating all core protocol mechanisms and protocol extensions (creation, creation-with-upload, checksum, termination, concatenation, and expiration) across Disk, S3, and Azure Blob backends. -- **IETF Resumable Uploads Clients & Conformity Tests**: The implementation has been thoroughly tested with our built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH protocol specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). +| Topic | Guide | +|---|---| +| Overview of all documents | [Documentation index](docs/README.md) | +| `process()`, completion, cleanup, multi-tenancy, security, frontend | [Integration Guide](docs/INTEGRATION_GUIDE.md) | +| All `with...()` options, storage backend selection, dependencies | [Configuration Reference](docs/CONFIGURATION.md) | +| Tus vs. IETF RUFH, protocol version selection, extensions, HTTP digests, 104 responses | [Protocols](docs/PROTOCOLS.md), [Migrating clients to RUFH](docs/MIGRATION.md), [Interim responses](docs/INTERIM_RESPONSES.md) | +| Multi-replica, Kubernetes, reverse proxies | [Deployment Guide](docs/DEPLOYMENT.md), [Locking](docs/LOCKING.md) | +| Storage backends | [Disk & network (NFS/EFS/SMB)](docs/DISK_BASED_LOCKING.md), [S3-compatible](docs/S3_STORAGE.md), [Azure Blob](docs/AZURE_BLOB_STORAGE.md) | +| Problems | [Troubleshooting](docs/TROUBLESHOOTING.md) | +| Upgrading from 1.x | [Upgrade Guide](docs/UPGRADING.md), [CHANGELOG](CHANGELOG.md) | -For detailed instructions on running our native conformity test suites across all storage backends (Disk, S3, Azure Blob) and interpreting results, see the **[Conformity Testing Guide (docs/CONFORMITY_TESTING.md)](docs/CONFORMITY_TESTING.md)**. +## Compatible Clients & Conformity Testing -This repository also contains comprehensive automated integration test suites (`ITTusFileUploadService`, `RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`) validating both protocol specifications. +The server is tested with [Uppy](https://uppy.io/), `tus-js-client`, the built-in Tus v1.0.0 and RUFH (draft-12 and RFC 9530) conformity suites in `scripts/` across Disk, S3 and Azure Blob backends, and the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). See [Protocols](docs/PROTOCOLS.md#compatible-clients--conformity-testing) and the [Conformity Testing Guide](docs/CONFORMITY_TESTING.md). ## Versioning This artifact follows `MAJOR.MINOR.PATCH` semantic versioning. Version `2.0.0` introduces major dual-protocol support for both Tus 1.0.0 and the IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload`). Version `1.0.0-3.3` was the last Tus protocol-only version. ## Contributing -This library comes without any warranty and is released under a [MIT license](https://github.com/tomdesair/tus-java-server/blob/master/LICENSE). If you encounter any bugs or if you have an idea for a useful improvement you are welcome to [open a new issue](https://github.com/tomdesair/tus-java-server/issues) or to [create a pull request](https://github.com/tomdesair/tus-java-server/pulls) with the proposed implementation. Please note that any contributed code needs to be accompanied by automated unit and/or integration tests and comply with the [defined code-style](#code-style). - -### Code Style -All pull requests should have the correct formatting according to [Google Java Style](https://github.com/google/google-java-format) code formatting. To verify if the code style is correct run: - -``` -mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:check -``` - -To reformat your code run: - -``` -mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:format -``` - -See the [Google Java Style Github page](https://github.com/google/google-java-format) on recommendations on how to configure this in your IDE. Or if you have Python 3, you can also use [pre-commit](https://pre-commit.com) to make your live easier: - -``` -pip install pre-commit -pre-commit install -``` +This library comes without any warranty and is released under a [MIT license](https://github.com/tomdesair/tus-java-server/blob/master/LICENSE). Bug reports, ideas and pull requests are welcome; contributed code needs automated tests and must follow the Google Java Style. See [CONTRIBUTING.md](CONTRIBUTING.md) for details, code style and testing instructions. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..80ebd339 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,309 @@ +# Architecture Guide: tus-java-server + +This document describes the architectural design of `tus-java-server`. It is written for developers and contributors who want to understand how the library works internally, how requests flow through the system, and where to place new functionality. + +--- + +## 1. Architectural Philosophy + +`tus-java-server` is an embeddable, container-native resumable upload library for **Java 17+** and **Jakarta EE** (`jakarta.servlet.*`). Its design is guided by four principles: + +1. **Single Central Service Class**: A single facade—[`TusFileUploadService`](../src/main/java/me/desair/tus/server/TusFileUploadService.java)—acts as the public API entry point and orchestrator. It manages configuration, concurrency boundaries, and extension execution without protocol-specific branching. +2. **Protocol Composition via Extensions**: Protocols are not hardcoded monoliths. Both supported protocols (**Tus 1.0.0** and **IETF RUFH**) are composed of modular extensions extending [`AbstractTusExtension`](../src/main/java/me/desair/tus/server/util/AbstractTusExtension.java). +3. **Business Logic in Validators & Handlers**: All protocol rules, request checks, and byte operations reside exclusively in dedicated [`RequestValidator`](../src/main/java/me/desair/tus/server/RequestValidator.java) and [`RequestHandler`](../src/main/java/me/desair/tus/server/RequestHandler.java) implementations. +4. **Zero External Coordination Dependencies by Default**: Out-of-the-box distributed locking uses atomic filesystem leases and POSIX sibling mutexes. No Redis, ZooKeeper, or database cluster is required to achieve multi-replica container safety. + +--- + +## 2. High-Level Component Architecture + +The diagram below illustrates how incoming HTTP requests flow through the library down to the physical storage and locking abstractions: + +```mermaid +graph TD + Client["Client (Tus / RUFH / Uppy / curl)"] -->|HTTP Request| ServletContainer["Servlet Container / Spring Boot"] + ServletContainer -->|HttpServletRequest / Response| Service["TusFileUploadService (Central Facade)"] + + subgraph Service Layer ["Facade & Concurrency Boundary"] + Service -->|1. Acquire / Wait Lock| Locking["UploadLockingService (UploadLock)"] + Service -->|2. Detect Version| ProtoDetect["ProtocolVersion.AUTO (Utils.detectProtocolVersion)"] + Service -->|3. Route Pipeline| Extensions["Extension Registry (enabledFeatures)"] + end + + subgraph Extension Pipeline ["Protocol Extensions"] + Extensions --> TusExt["Tus 1.0.0 Extensions
(CoreProtocol, Creation, Checksum, etc.)"] + Extensions --> RufhExt["IETF RUFH Extension
(ResumableUploadsForHttpProtocol)"] + Extensions --> SharedExt["Shared Extensions
(CorsExtension, Expiration, Download, HttpDigests)"] + end + + subgraph Processing Engine ["Business Logic Engine"] + TusExt & RufhExt & SharedExt --> Validators["RequestValidator Pipeline
(Pure pre-condition validation)"] + TusExt & RufhExt & SharedExt --> Handlers["RequestHandler Pipeline
(I/O Streaming, state updates, response headers)"] + end + + subgraph Storage & Concurrency SPI ["Pluggable Infrastructure SPI"] + Handlers --> Storage["UploadStorageService
(Disk / S3 / Azure Blob)"] + Handlers --> Locking + Storage -.-> Caching["ThreadLocalCachedStorageAndLockingService"] + Storage -.-> Concat["UploadConcatenationService"] + end +``` + +--- + +## 3. Dual Protocols Composed of Extensions + +`tus-java-server` serves **two resumable upload protocols** on the exact same endpoint: + +1. **Tus 1.0.0**: The widely-adopted [tus protocol standard](https://tus.io/protocols/resumable-upload.html). +2. **IETF Resumable Uploads for HTTP (RUFH)**: The official standardization draft ([draft-ietf-httpbis-resumable-upload-12](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/)). + +By default, the service operates in `ProtocolVersion.AUTO` mode: it inspects incoming HTTP headers per request (`Tus-Resumable` activates Tus 1.0.0; `Upload-Complete` or `Content-Type: application/partial-upload` activates RUFH). + +### Protocol Composition Matrix + +Each protocol is built from a combination of **mandatory** and **optional** extensions: + +| Protocol | Mandatory Extensions | Optional Extensions | +|---|---|---| +| **Tus 1.0.0** | `CoreProtocol` (core `OPTIONS`, `HEAD`, `PATCH`) | `CreationExtension`, `CreationWithUploadExtension`, `ChecksumExtension`, `TerminationExtension`, `ExpirationExtension`, `ConcatenationExtension`, `CorsExtension`, `DownloadExtension` | +| **IETF RUFH** | `ResumableUploadsForHttpProtocol` (`OPTIONS`, `HEAD`, `GET`, `POST`, `PUT`, `PATCH`, `DELETE`) | `HttpDigestsExtension` (RFC 9530 digests), `ExpirationExtension` (`max-age`), `CorsExtension`, `DownloadExtension` | + +- **Mandatory Extensions**: `CoreProtocol` cannot be disabled; it defines core byte offset tracking and stream resumption for Tus 1.0.0. For RUFH requests, `ResumableUploadsForHttpProtocol` is the mandatory protocol handler. +- **Optional Extensions**: Can be disabled at runtime via `service.disableTusExtension("name")` or enabled on demand (e.g. `service.withDownloadFeature()`). + +--- + +## 4. The Validator & Handler Pipeline + +All protocol business logic is decoupled into two interfaces: + +### 1. `RequestValidator` (Validation Phase) +- **Role**: Pure, side-effect-free pre-condition verification. +- **Contract**: Inspects request headers and parameters. If invalid, throws a typed [`TusException`](../src/main/java/me/desair/tus/server/exception/TusException.java) containing the exact HTTP status code (e.g., `400 Bad Request`, `404 Not Found`, `409 Conflict`, `412 Precondition Failed`, `413 Payload Too Large`). +- **Examples**: `TusResumableValidator`, `UploadOffsetValidator`, `RufhCreationValidator`, `RufhAppendValidator`. + +### 2. `RequestHandler` (Execution & Error Phase) +- **Role**: State mutation, byte payload ingestion, response header generation, and error formatting. +- **Contract**: + - Standard handlers (`isErrorHandler() == false`): Streams upload bytes, updates offsets in storage, sets protocol headers (`Upload-Offset`, `Location`, `Upload-Limit`). + - Error handlers (`isErrorHandler() == true`): Intercepts thrown exceptions to emit protocol-specific error payloads, such as RFC 7807 problem details (`application/problem+json` in `RufhErrorHandler`). +- **Examples**: `CorePatchRequestHandler`, `RufhCreationPostRequestHandler`, `RufhErrorHandler`. + +### Request Lifecycle Sequence + +The sequence below details the execution of `TusFileUploadService.process(...)`: + +```mermaid +sequenceDiagram + autonumber + actor Client + participant Service as TusFileUploadService + participant Locking as UploadLockingService + participant Extension as AbstractTusExtension + participant Validator as RequestValidator + participant Handler as RequestHandler + participant Storage as UploadStorageService + + Client->>Service: HTTP Request (POST / PATCH / HEAD / DELETE) + Service->>Locking: acquireUploadLock(method, uri) (wait / retry up to 60s) + Locking-->>Service: UploadLock handle + Service->>Service: checkWasInProgress(uri) + + rect rgb(240, 248, 255) + note over Service,Validator: Phase 1: Request Validation + Service->>Extension: validate(method, req, storage, locking, version) + Extension->>Validator: validate(method, req, storage) + Validator-->>Extension: OK (or throws typed TusException) + end + + rect rgb(245, 255, 245) + note over Service,Storage: Phase 2: Request Execution + Service->>Extension: process(method, req, resp, storage, locking, version) + Extension->>Handler: process(method, req, resp, storage, locking) + Handler->>Storage: append(stream) / create(info) / update() + Handler-->>Service: Set response status & headers + end + + opt Exception Thrown + Service->>Extension: handleError(method, req, resp, storage, locking, version, exception) + Extension->>Handler: ErrorHandler.process(...) + Handler-->>Service: Write RFC 7807 problem details / error status + end + + opt Upload Finished + Service->>Locking: cleanupLock(uploadId) + Service->>Service: notifyUploadCompletionListeners(uploadInfo) + end + + Service->>Locking: lock.close() (releases lock handle) + Service-->>Client: HTTP Response (201 / 204 / 409 / etc.) +``` + +--- + +## 5. Pluggable Storage Backend Architecture + +File storage and metadata persistence are decoupled behind the [`UploadStorageService`](../src/main/java/me/desair/tus/server/upload/UploadStorageService.java) SPI. + +``` +UploadStorageService +├── DiskStorageService (Filesystem & Network Shares) +├── S3StorageService (AWS S3, MinIO, RustFS, Ceph, Cloudflare R2) +└── AzureBlobStorageService (Azure Blob Storage Block Blobs) +``` + +### Two Disk Storage Implementations + +For local filesystems and shared network volumes, `tus-java-server` provides two distinct concurrency/storage combinations: + +1. **Current / Default: Lease File Based (`DiskStorageService` + `LeaseFileLockingService`)**: + - Configured automatically via `withStoragePath(String)`. + - Replaces OS kernel file locks with atomic sibling mutex directories (`.mutex/`) and JSON lease metadata (`lease.json`). + - Works safely across multi-replica container clusters, Kubernetes pods (`ReadWriteMany`), AWS EFS, Azure Files, SMB, and NFSv3/v4 mounts without requiring `rpc.statd` or external databases. +2. **Legacy Opt-Out: File Lock Based (`DiskStorageService` + `DiskLockingService`)**: + - Opt-out via `withUploadStorageService(new DiskStorageService(...)).withUploadLockingService(new DiskLockingService(...))`. + - Uses OS kernel-level locks (`java.nio.channels.FileLock` via POSIX `fcntl` or Windows `LockFileEx`). + - Suitable strictly for single-node deployments on local disk. + +### Cloud Storage Implementations + +- **`S3StorageService`**: Stores chunks as S3 objects and concatenates them using multipart upload copy or compose helpers (`S3ConcatenationService`). +- **`AzureBlobStorageService`**: Stages chunks as uncommitted block blobs using `.part` temporary buffers and commits block lists atomically upon completion. Zero-copy concatenation is supported via `stageBlockFromUrl`. + +### Storage Extensions & Helpers + +- **Upload Concatenation SPI (`UploadConcatenationService`)**: Supports merging parallel chunk uploads into a single final resource. Implemented natively for Disk (`VirtualConcatenationService`), S3 (`S3ConcatenationService`), and Azure (`AzureBlobConcatenationService`). +- **Upload Deduplication**: Enabled via `.withUploadDeduplication(true)`. Completed uploads are indexed by checksum. Duplicate uploads are linked via `duplicatesUploadId`. Read operations dynamically resolve to the parent upload, while write and truncation operations strictly operate on the target upload to prevent corrupting parent data. +- **Thread-Local Request Caching (`ThreadLocalCachedStorageAndLockingService`)**: Enabled via `.withThreadLocalCache(true)`. Caches `UploadInfo` in memory for the duration of a single HTTP request to eliminate redundant network roundtrips to S3 or Azure Blob Storage. + +--- + +## 6. Distributed Locking & Lock Contention Resolution + +Resumable uploads require exclusive concurrency control per upload resource to prevent race conditions (e.g., overlapping byte appends or simultaneous resume and delete requests). + +All locking backends implement [`UploadLockingService`](../src/main/java/me/desair/tus/server/upload/UploadLockingService.java) and return [`UploadLock`](../src/main/java/me/desair/tus/server/upload/UploadLock.java) handles. + +### Building Lease-Based Lock Providers: `AbstractLeaseLockingService` + +Custom distributed locking services (or extensions for GCP, Redis, DynamoDB, etc.) should extend [`AbstractLeaseLockingService`](../src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java): + +```mermaid +classDiagram + class UploadLockingService { + <> + +lockUploadByUri(uri) UploadLock + +isLocked(id) boolean + +cleanupLock(id) void + +registerInputStream(uri, stream) void + +requestLockRelease(uri) void + } + + class AbstractCloseableResourceService { + <> + #registerShutdownHook() + #cleanupOnClose() + } + + class AbstractLeaseLockingService { + <> + #long leaseDurationMs + #long CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L + #Map activeInputStreams + #ScheduledExecutorService watchdogExecutor + +lockUploadByUri(uri) UploadLock + +requestLockRelease(uri) void + #acquireOrEvictExpiredLock(id, holderId, uri) + #tryAcquireLock(id, leaseData)* UploadLock + #isLockExpired(id)* boolean + #evictExpiredLock(id)* boolean + #writeStopSignal(id)* void + #checkStopSignals()* void + } + + class LeaseFileLockingService { + +tryAcquireLock() + +evictExpiredLock() + } + + class S3LockingService { + +tryAcquireLock() + +evictExpiredLock() + } + + UploadLockingService <|.. AbstractLeaseLockingService + AbstractCloseableResourceService <|-- AbstractLeaseLockingService + AbstractLeaseLockingService <|-- LeaseFileLockingService + AbstractLeaseLockingService <|-- S3LockingService +``` + +**Key Features of `AbstractLeaseLockingService`**: +1. **Clock Skew Safety Margin**: Built-in 2.0-second buffer (`CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L`) absorbs NTP drift between cluster nodes, preventing premature lease eviction. +2. **Stream Interruption Registry**: Tracks active in-flight input streams in `activeInputStreams` and interrupts them cleanly via `Utils.interruptStream(...)`. +3. **Contention Dispatch & Polling Watchdog**: When contention is detected, dispatches a `.stop` signal and runs a background watchdog executor (`Utils.scheduleWatchdog(...)`) to terminate paused streams on remote nodes. +4. **TOCTOU-Safe Eviction with Jitter**: Implements randomized backoff jitter (`jitterMinMs` to `jitterMaxMs`) to prevent thundering herd collisions when multiple nodes contend for an expired lease. + +### Cross-Node Lock Contention Resolution + +When a client loses network connectivity, the server's `PATCH` socket may remain stalled in a half-open state. When the client resumes with a `HEAD` request (or cancels with `DELETE`), contention is resolved across nodes: + +```mermaid +sequenceDiagram + autonumber + actor Client + participant NodeA as Pod A (Stalled PATCH) + participant SharedStorage as Shared Storage / Bucket + participant NodeB as Pod B (Resuming HEAD) + + Note over NodeA: Stalled PATCH stream holds lease lock
(Stream wrapped in InterruptibleInputStream) + Client->>NodeB: HEAD /files/upload/123 + NodeB->>NodeB: acquireUploadLock fails (UploadAlreadyLockedException) + NodeB->>NodeB: Enter lock wait loop (every 200ms up to 60s) + NodeB->>SharedStorage: requestLockRelease writes 123.stop signal + + Note over NodeA: Watchdog polls every 1.5s and detects 123.stop + NodeA->>NodeA: watchdog calls stream.interrupt() + NodeA->>SharedStorage: Drain in-flight cloud chunks (up to 55s) + NodeA->>SharedStorage: Release lease lock & remove 123.stop + + NodeB->>SharedStorage: acquireUploadLock succeeds! + NodeB->>SharedStorage: Read current Upload-Offset + NodeB-->>Client: 204 No Content (Upload-Offset: 524288) +``` + +For more details on locking mechanisms, see [`docs/LOCKING.md`](LOCKING.md) and [`docs/DISK_BASED_LOCKING.md`](DISK_BASED_LOCKING.md). + +--- + +## 7. Three-Tiered Testing Strategy + +The repository maintains high reliability across protocols and storage engines via three testing tiers: + +```mermaid +graph TD + subgraph Testing Pyramid + Top["Conformity Test Suites (Python)
• Socket-level HTTP verification against running servers
• scripts/tus_conformity_test.py (50+ rules)
• scripts/rufh_conformity_test.py (RFC 9530, draft-12)"] + Mid["Integration Tests (Java IT*)
• Real container lifecycles via Testcontainers (Podman / Docker)
• Verified in CI via mvn verify
• Abstract base classes test Disk, S3 (RustFS), and Azure (Azurite)"] + Base["Unit Tests (Java *Test.java)
• Pure offline Mockito isolation (< 2s)
• Verified via mvn test
• Strict code coverage gate (> 90% per file)"] + end + Base --> Mid --> Top +``` + +1. **Unit Tests (`*Test.java`)**: Target specific classes in complete isolation with Mockito. Zero container or network requirements. Executed rapidly via `mvn test -q`. Coverage is verified via `python3 scripts/check-coverage.py --per-file-limit 90`. +2. **Integration Tests (`IT*`)**: End-to-end multi-backend verification running under `mvn verify -q`. Tests spin up dynamic local containers via Testcontainers using **either Podman or Docker**. Base classes (`AbstractITRufhProtocol`, `AbstractITTusFileUploadService`) ensure identical test coverage across Disk, S3, and Azure backends. +3. **Conformity Tests (`scripts/*_conformity_test.py`)**: Black-box Python test suites executed against live server deployments (e.g. `tus-java-server-spring-demo` on port 8080) to validate strict specification compliance. + +For detailed testing instructions, see [`docs/TESTING.md`](TESTING.md) and [`docs/CONFORMITY_TESTING.md`](CONFORMITY_TESTING.md). + +--- + +## 8. Contributor Blueprint: Where to Add New Functionality + +| Task | Where to Implement | Key Guidelines | +|---|---|---| +| **Add a Request Validation Rule** | Create a class implementing `RequestValidator` in the target extension's `validation` package. | Keep validation side-effect-free. Throw typed `TusException` subclasses. Register in `extension.initValidators()`. | +| **Add a Request Processing Action** | Create a class implementing `RequestHandler` (or extending `AbstractRequestHandler`). | Wrap streaming payload inputs in `InterruptibleInputStream`. Set headers on `TusServletResponse`. Register in `extension.initRequestHandlers()`. | +| **Add a New Protocol Extension** | Create a package under `me.desair.tus.server.` and extend `AbstractTusExtension`. | Implement `getName()`, `isApplicable(method, version)`, `initValidators()`, and `initRequestHandlers()`. Register in `TusFileUploadService.initFeatures()`. | +| **Add a Storage Backend** | Implement `UploadStorageService` (and optionally `UploadConcatenationService`). | Support deduplication (`duplicatesUploadId`). Implement offline unit tests (`*Test.java`) and Testcontainers integration tests (`IT*`). | +| **Add a Distributed Locking Provider** | Extend `AbstractLeaseLockingService` and `AbstractLeaseLock`. | Implement the template hooks (`tryAcquireLock`, `isLockExpired`, `evictExpiredLock`, `writeStopSignal`, `checkStopSignals`). Inherit NTP safety margins and stream interruption automatically. | diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index cc98e7bc..b0bb3bfd 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -64,7 +64,9 @@ AzureBlobLockingService azureLockingService = new AzureBlobLockingService(contai TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files/upload") .withUploadStorageService(azureStorageService) - .withUploadLockingService(azureLockingService); + .withUploadLockingService(azureLockingService) + .withUploadDeduplication(true) + .withThreadLocalCache(true); ``` --- @@ -73,11 +75,11 @@ TusFileUploadService tusService = new TusFileUploadService() > [!IMPORTANT] > **Why `ThreadLocalCachedStorageAndLockingService` is Recommended for Azure**: -> By default, `TusFileUploadService` automatically wraps your custom `UploadStorageService` and `UploadLockingService` in a `ThreadLocalCachedStorageAndLockingService`. +> By default, thread-local caching is disabled (`false`). You should explicitly enable it via `.withThreadLocalCache(true)`. > > During a single HTTP request lifecycle (POST, PATCH, HEAD, DELETE), the tus server validates request headers, reads upload state, appends data, and constructs response headers. Without caching, retrieving `UploadInfo` and calculating offsets would require multiple redundant network roundtrips to Azure (`downloadContent` on `.info`, `getProperties`). > -> `ThreadLocalCachedStorageAndLockingService` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces Azure network latency and API call cost per request. +> Enabling `withThreadLocalCache(true)` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces Azure network latency and API call cost per request. --- @@ -122,13 +124,14 @@ After an upload completes, downstream services can obtain the direct Azure blob import com.azure.storage.blob.BlobClient; import me.desair.tus.server.upload.azure.AzureBlobStorageService; -AzureBlobStorageService azureStorage = (AzureBlobStorageService) tusService.getUploadStorageService(); - +// Inject or reference the concrete AzureBlobStorageService bean directly. +// (Avoid casting tusService.getUploadStorageService() when withThreadLocalCache(true) +// is enabled, as it returns ThreadLocalCachedStorageAndLockingService). String uploadUri = "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"; String ownerKey = "user-123"; // 1. Obtain full Azure blob name after upload completion -String blobName = azureStorage.getAzureBlobName(uploadUri, ownerKey); +String blobName = azureStorageService.getAzureBlobName(uploadUri, ownerKey); // e.g. "uploads/24249a5b-01a4-4bf8-b67a-364273bb5a2e" // 2. Direct Azure SDK access for post-upload processing @@ -209,7 +212,7 @@ When a client resumes an interrupted transfer by sending a `HEAD` request (or ca |---|---|---| | **HTTP 409 Conflict** | Another process or cluster pod currently holds an active lease on the lock blob. | Normal behavior during concurrent PATCH/DELETE requests. Retry after lock release. | | **HTTP 404 BlobNotFound** | The upload metadata `.info` blob does not exist or was expired/deleted. | Verify upload ID validity or upload expiration timestamps (`uploadExpirationPeriod`). | -| **Azurite connection refused** | Azurite emulator is not running or listening on port 10000. | Launch Azurite via Docker (`docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite`). | +| **Azurite connection refused** | Container runtime (Podman or Docker) is not running during integration tests. | Start Podman or Docker; Testcontainers automatically manages Azurite startup and teardown during integration tests. | | **`MaxAppendSizeExceededException`** | Incoming PATCH payload chunk exceeded the configured `maxAppendSize`. | Adjust `withMaxAppendSize()` setting on `TusFileUploadService`. | --- @@ -223,11 +226,11 @@ When a client resumes an interrupted transfer by sending a `HEAD` request (or ca | `AzureBlobLockingServiceTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure locking service parameters, prefix normalization, URI parsing, and defensive handling. | | `AzureBlobUploadLockTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure upload lock parameters, getters, and daemon executor lifecycle. | | `AzureBlobConcatenationServiceTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure concatenation service parameters, prefix sanitization, guard clauses, and partial upload handling. | -| `ITAzureBlobStorageService` | Integration | Azurite (Docker) | ~ 3s | Live end-to-end storage integration test against Azurite emulator (Block Blob staging, `.part` buffer commits, truncations, deduplication, and expiration). | -| `ITAzureBlobLockingService` | Integration | Azurite (Docker) | ~ 2s | Distributed locking, Azure Blob Leases, lease renewals, lock contention (409 Conflict), stream interruption, & stop signal integration tests. | -| `ITAzureBlobConcatenationService` | Integration | Azurite (Docker) | ~ 2s | Server-side zero-copy block concatenation (`stageBlockFromUrl`) integration tests. | -| `ITAzureBlobRufhProtocol` | Integration | Azurite (Docker) | ~ 3s | IETF RUFH protocol integration suite for Azure backend. | -| `ITAzureBlobTusFileUploadService` | Integration | Azurite (Docker) | ~ 3s | Tus 1.0.0 protocol integration suite for Azure backend. | +| `ITAzureBlobStorageService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | Live end-to-end storage integration test against Azurite emulator (Block Blob staging, `.part` buffer commits, truncations, deduplication, and expiration). | +| `ITAzureBlobLockingService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 2s | Distributed locking, Azure Blob Leases, lease renewals, lock contention (409 Conflict), stream interruption, & stop signal integration tests. | +| `ITAzureBlobConcatenationService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 2s | Server-side zero-copy block concatenation (`stageBlockFromUrl`) integration tests. | +| `ITAzureBlobRufhProtocol` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | IETF RUFH protocol integration suite for Azure backend. | +| `ITAzureBlobTusFileUploadService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | Tus 1.0.0 protocol integration suite for Azure backend. | --- diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 00000000..a3d5cdb3 --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,85 @@ +# Configuration Reference + +All configuration is done through the fluent `with...()` methods of `TusFileUploadService`. Create **one** instance and share it (e.g. as a Spring singleton bean); the service is thread-safe and holds the storage and locking backends. + +```java +@Bean +public TusFileUploadService tusFileUploadService() { + return new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true); +} +``` + +## Dependencies + +The library only depends on Jakarta Servlet API 6.0 and some Apache Commons utility libraries, so (in theory) you can use it on any modern Java web application server such as Tomcat, JBoss or Jetty. By default all uploaded data and information is stored on a (shared) file volume of the application server. + +Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are bundled directly with compile scope by `tus-java-server`. When using S3 storage (`S3StorageService`), include the MinIO Java SDK dependency matching the version in `pom.xml`: + +```xml + + io.minio + minio + 9.0.3 + +``` + +For Azure Blob Storage dependencies see the [Azure Blob Storage Guide](AZURE_BLOB_STORAGE.md#1-quick-start). + +## Options + +### Endpoint & protocol + +| Method | Default | Description | +|---|---|---| +| `withUploadUri(String)` | `"/"` | Sets relative path (e.g. `/api/upload`) or absolute base URL (e.g. `https://upload.example.com/api/upload`) under which the upload endpoint is exposed. Supports regex parameters (e.g. `/users/[0-9]+/files/upload`). Use the absolute form when running behind a proxy that rewrites the path or host, so that `Location` headers are correct. | +| `withSupportedProtocolVersions(ProtocolVersion)` | `ProtocolVersion.AUTO` | Configures protocol handling: `AUTO` (header-based auto-detection), `TUS_1_0_0` (Tus 1.0.0 only), or `RUFH` (IETF draft-12 only). See [PROTOCOLS.md](PROTOCOLS.md). | +| `withChunkedTransferDecoding(Boolean)` | `false` | Enables manual chunked HTTP decoding for servlet containers that do not decode chunked requests natively. | +| `addTusExtension(TusExtension)` | All standard enabled | Adds a custom extension (e.g. application authorization checks). | +| `disableTusExtension(String)` | None | Disables a built-in extension (`creation`, `creation-with-upload`, `checksum`, `expiration`, `concatenation`, `termination`, `download`, `cors`, `http-digests`, `resumable-uploads-for-http`). | +| `withDownloadFeature()` | Disabled | Enables the unofficial `download` extension allowing clients to retrieve uploaded bytes via HTTP `GET`. | + +### Limits, expiration & integrity + +| Method | Default | Description | +|---|---|---| +| `withMaxUploadSize(Long)` | `Long.MAX_VALUE` | Maximum allowed total upload size in bytes per upload resource. | +| `withMaxAppendSize(Long)` / `withMinAppendSize(Long)` / `withMinSize(Long)` | No limit | IETF RUFH only: maximum size of a single append (`PATCH`) request, minimum size of an append or creation request, and minimum total upload size (all in bytes). | +| `withUploadExpirationPeriod(Long)` | `null` (disabled) | Expiration period in **milliseconds** after which incomplete/expired uploads become eligible for cleanup (run `cleanup()` periodically, see the [Integration Guide](INTEGRATION_GUIDE.md#3-cleanup)). | +| `withUploadDeduplication(Boolean)` | `false` | Enables duplicate file detection by checksum, linking new uploads (`duplicatesUploadId`) and skipping redundant storage writes. | + +### Identity, events & serialization + +| Method | Default | Description | +|---|---|---| +| `withUploadIdFactory(UploadIdFactory)` | `UuidUploadIdFactory` | Custom ID generator for upload resources (e.g., `UuidUploadIdFactory` or `TimeBasedUploadIdFactory`). | +| `withUploadCompletionListener(UploadCompletionListener)` / `addUploadCompletionListener(...)` | None | Registers a callback invoked immediately when an upload finishes transferring all bytes and is completed. Both methods append to the list of active listeners. | +| `withJsonSerialization()` / `withJsonSerialization(boolean)` | Java serialization | Enables or disables JSON serialization for upload metadata (`UploadInfo`). Jackson is bundled by default. | + +### Storage, locking & performance + +| Method | Default | Description | +|---|---|---| +| `withStoragePath(String)` | `${java.io.tmpdir}/tus` | Path on the filesystem or shared drive where uploaded bytes and metadata are stored when using `DiskStorageService`. Also configures `LeaseFileLockingService`. | +| `withUploadStorageService(UploadStorageService)` | `DiskStorageService` | Configures custom or cloud storage backend (`DiskStorageService`, `S3StorageService`, `AzureBlobStorageService`). Automatically copies over previous limits and configuration. | +| `withUploadLockingService(UploadLockingService)` | `LeaseFileLockingService` | Configures custom or cloud locking backend (`LeaseFileLockingService`, `S3LockingService`, `AzureBlobLockingService`). | +| `withLockWaitTimeout(Duration)` | `Duration.ofSeconds(60)` | Maximum duration a request waits to acquire an upload lock held by an in-flight transfer (retrying every 200ms, resulting in 300 retries). Automatically configures cloud background chunk drain timeout to 5s less than this value (default 55s). See [LOCKING.md](LOCKING.md). | +| `withThreadLocalCache(Boolean)` | `false` | Enables in-memory thread-local caching of upload request data to reduce storage backend I/O load. Strongly recommended for cloud backends (S3, Azure) to eliminate redundant remote metadata lookups per request. | +| `withCloudUploadThreadPoolSize(int)` | `10` | Maximum number of worker threads used for asynchronous background chunk uploading in cloud storage backends (S3, Azure). | + +> Backend-specific options (S3 jitter, Azure credentials, key prefixes...) are documented in the respective backend guides below. + +## Storage Backend Options + +`tus-java-server` provides a pluggable storage architecture supporting multiple backends. Use the table to pick one, then follow the linked guide. See also [DEPLOYMENT.md](DEPLOYMENT.md) for multi-replica considerations. + +| Backend | Classes | Best for | Guide | +|---|---|---|---| +| **File Disk & Network Storage** (default) | `DiskStorageService` & `LeaseFileLockingService` | Single node, shared NFS (v3/v4, AWS EFS, Azure Files, SMB/CIFS) or a Kubernetes Persistent Volume (`ReadWriteMany` / `ReadWriteOnce`). | [Disk & Network Storage Locking Guide](DISK_BASED_LOCKING.md) | +| **S3-Compatible Object Storage** | `S3StorageService`, `S3LockingService` & `S3ConcatenationService` | AWS S3, MinIO, RustFS, Cloudflare R2, Ceph or Google Cloud Storage. Uses distributed S3 object locking and TTL leases, so multi-replica container deployments work without Redis or external databases. | [S3 Storage Guide](S3_STORAGE.md) | +| **Azure Blob Storage** | `AzureBlobStorageService`, `AzureBlobLockingService` & `AzureBlobConcatenationService` | Microsoft Azure using the `azure-storage-blob` SDK. Uses native Azure Blob Leases (30s renewable leases) for distributed locking with automatic lock cleanup on upload completion, termination, and expiration. | [Azure Blob Storage Guide](AZURE_BLOB_STORAGE.md) | + +Cloud backends feature an asynchronous 3-slot chunk pipeline (`AsyncChunkUploader`) that overlaps client payload streaming with cloud staging in the background. The guides above cover multi-replica deployments in Kubernetes, post-upload processing, and legacy locking opt-out. diff --git a/docs/CONFORMITY_TESTING.md b/docs/CONFORMITY_TESTING.md index 660b9e4f..ba482c68 100644 --- a/docs/CONFORMITY_TESTING.md +++ b/docs/CONFORMITY_TESTING.md @@ -132,7 +132,7 @@ done --- -## 6. Understanding Test Results & AI Agent Remediation +## 6. Understanding Test Results When executed directly via Python, both conformity test scripts produce a structured summary report detailing: @@ -166,7 +166,7 @@ If a test fails, the runner outputs a structured failure breakdown containing: - Verbatim specification quote defining the required behavior - Exact failure assertion and returned HTTP status or header values -Developers and automated AI agents can use this failure breakdown to immediately diagnose compliance discrepancies and make targeted fixes in the server code. +Use this failure breakdown to immediately diagnose compliance discrepancies and make targeted fixes in the server code. --- diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 00000000..89589bad --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,49 @@ +# Deployment Guide + +Checklist for running `tus-java-server` in production: choosing a backend, running several replicas and configuring proxies. + +## 1. Choose a Backend + +| Situation | Recommended setup | +|---|---| +| Single application instance, local disk | Defaults: `withStoragePath(...)` (`DiskStorageService` + `LeaseFileLockingService`). Optionally the legacy `DiskLockingService`, see [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#5-opt-out--backward-compatibility-guide). | +| Several replicas sharing a volume (NFS, EFS, Azure Files, SMB, Kubernetes `ReadWriteMany`) | Defaults; read the mount recommendations in [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#4-production-network-mount-configuration-guide). | +| Several replicas, no shared volume, AWS / S3-compatible | `S3StorageService` + `S3LockingService`, see [S3_STORAGE.md](S3_STORAGE.md). | +| Several replicas on Azure | `AzureBlobStorageService` + `AzureBlobLockingService`, see [AZURE_BLOB_STORAGE.md](AZURE_BLOB_STORAGE.md). | + +All built-in backends are lease-based and need **no Redis or database**. Custom lock providers can be added, see [LOCKING.md](LOCKING.md#4-implementing-a-custom-uploadlockingservice). + +## 2. Multi-Replica Checklist + +- Every replica must use the **same** storage and locking backend and `withUploadUri(...)`. +- Keep clocks synchronised with NTP (lease expiry depends on it). See the clock notes in [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#35-clock-synchronization--ntp-requirement) and [S3_STORAGE.md](S3_STORAGE.md#3-clock-synchronization--ntp-requirement). +- Requests for one upload may land on different replicas: contention between a stalled `PATCH` and a resuming `HEAD`/`DELETE` is resolved across replicas automatically, see [LOCKING.md](LOCKING.md). +- Run `cleanup()` on a schedule (on one or all replicas, it is safe to run concurrently). +- Call `TusFileUploadService.close()` on shutdown so locks are released. +- Cloud backends: configure bucket / container lifecycle rules for abandoned objects (see [S3 lifecycle rules](S3_STORAGE.md#8-operational-hardening-s3-lifecycle-cleanup-rules) and [Azure lifecycle rules](AZURE_BLOB_STORAGE.md#6-operational-hardening-azure-blob-lifecycle-management-rules)). + +## 3. Reverse Proxies & Load Balancers + +Ensure that reverse proxies (Nginx, HAProxy, AWS ALB, Cloudflare): +1. Forward the protocol headers: Tus (`Tus-Resumable`, `Upload-Offset`, `Upload-Length`, `Upload-Metadata`, `Upload-Checksum`, `Upload-Defer-Length`, `Upload-Concat`) and RUFH (`Upload-Offset`, `Upload-Complete`, `Upload-Length`, `Upload-Limit`, `Upload-Draft`, `Content-Digest`, `Repr-Digest`). +2. Do not strip or alter `Content-Type: application/partial-upload` or `application/offset+octet-stream`. +3. Allow the HTTP methods `PATCH`, `HEAD` and `DELETE`. +4. Do not buffer whole request bodies and allow large bodies and long read timeouts. For Nginx, for example: + +```nginx +location /api/upload { + proxy_pass http://app; + proxy_request_buffering off; # stream chunks straight to the app + client_max_body_size 0; # size limits are enforced by withMaxUploadSize(...) + proxy_read_timeout 300s; + proxy_http_version 1.1; +} +``` + +5. If the proxy changes the public host or path, set an **absolute** `withUploadUri("https://upload.example.com/api/upload")` so that `Location` headers contain the public URL. + +## 4. Servlet Container Notes + +- Containers that do not decode `Transfer-Encoding: chunked` natively need `withChunkedTransferDecoding(true)`. +- HTTP 104 interim responses (RUFH) need container-specific support, see [INTERIM_RESPONSES.md](INTERIM_RESPONSES.md). +- Use `jakarta.servlet` containers (Tomcat 10+, Jetty 11+, Spring Boot 3+). The library no longer supports `javax.servlet`. diff --git a/docs/DISK_BASED_LOCKING.md b/docs/DISK_BASED_LOCKING.md index 9748ce0e..d7072478 100644 --- a/docs/DISK_BASED_LOCKING.md +++ b/docs/DISK_BASED_LOCKING.md @@ -55,7 +55,7 @@ Locks are structured as a dedicated directory containing a JSON lease metadata f ## 3. Distributed Concurrency & Contention Resolution -### 1. Lock Acquisition Flow (`tryAcquireLock`) +### 3.1 Lock Acquisition Flow (`tryAcquireLock`) To acquire or take over a lock: 1. Extract `UploadId` from the request URI. 2. Acquire sibling mutex `/locks/.mutex/` via `Files.createDirectory`. @@ -67,7 +67,7 @@ To acquire or take over a lock: - Start background heartbeat daemon (renews every $\text{leaseDuration} / 3$) and return `LeaseFileUploadLock`. 4. Release `.mutex/`. -### 2. Lock Release Flow (`lock.close()`) +### 3.2 Lock Release Flow (`lock.close()`) When an active upload completes or is aborted: 1. Stop background heartbeat daemon. 2. Acquire sibling mutex `.mutex/`. @@ -75,24 +75,24 @@ When an active upload completes or is aborted: 4. If ownership matches: delete `lease.json` and delete `.lock/`. 5. Release `.mutex/`. -### 3. Heartbeat Lease Auto-Renewal +### 3.3 Heartbeat Lease Auto-Renewal Active streaming uploads periodically renew their lease by updating `expiresAt` in `lease.json` every $\text{leaseDuration} / 3$ (default: every 10 seconds for a 30s lease). The renewal verifies `holderId` ownership under the sibling mutex to ensure it aborts if the lease was taken over after a long pause. -### 4. Lock Contention & `.stop` Signal Files +### 3.4 Lock Contention & `.stop` Signal Files When a client sends a `HEAD` or `DELETE` request to resume or cancel an upload while a stalled `PATCH` stream holds the lock: 1. The resuming server catches `UploadAlreadyLockedException` and calls `requestLockRelease(requestUri)`. 2. It interrupts any JVM-local stream and writes `/locks/.stop`. 3. A background watchdog thread on the holding replica (polling every 1.5 seconds) detects `.stop` and calls `stream.interrupt()`. -4. The stalled `PATCH` stream aborts and releases its lock. The server's 8.0-second retry budget ($40 \times 200\text{ms}$) allows the `HEAD` or `DELETE` request to acquire the lock and succeed seamlessly. +4. The stalled `PATCH` stream aborts and releases its lock. The server's 60-second retry budget ($300 \times 200\text{ms}$, configurable via `withLockWaitTimeout(Duration)`) allows the `HEAD` or `DELETE` request to acquire the lock and succeed seamlessly. -### 5. Clock Synchronization & NTP Requirement +### 3.5 Clock Synchronization & NTP Requirement Distributed lease evaluation across nodes relies on `expiresAt` timestamps written to `lease.json`. When sharing a network volume across multiple pods or virtual machines, all host nodes MUST synchronize system clocks using NTP (Network Time Protocol) or cloud time synchronization daemons (`chrony`). To absorb inevitable minor NTP drift, `LeaseFileLockingService` incorporates a built-in 2.0-second safety margin (`CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L` inherited from `AbstractLeaseLockingService`), ensuring that a node whose clock is slightly ahead will never prematurely evict a live peer's lease. Clock drift between cluster nodes should remain well within $\pm 1$ second to ensure accurate TTL lease expiration and takeover. --- ## 4. Production Network Mount Configuration Guide -### Linux / NFSv4 Mount Recommendations +### 4.1 Linux / NFSv4 Mount Recommendations For multi-replica deployments connecting to shared NFS storage (e.g. AWS EFS, Azure NetApp Files, on-premise NFS): ```bash @@ -103,7 +103,7 @@ mount -t nfs4 -o rw,hard,intr,rsize=1048576,wsize=1048576,actimeo=3 nfs-server:/ - `actimeo=3`: Attribute caching timeout of 3s ensures `.stop` signal files and lease renewals propagate rapidly across pods. - `rsize=1048576,wsize=1048576`: 1MB I/O buffers for maximum streaming append throughput. -### Windows / SMB 3.x Configuration +### 4.2 Windows / SMB 3.x Configuration For Windows Server SMB shares or Azure Files SMB: ```powershell @@ -116,16 +116,18 @@ New-SmbMapping -RemotePath "\\smb-server\uploads" -LocalPath "Z:" -Persistent $T If you are running single-node deployments on local disk and specifically require legacy OS-level `FileLock` (`fcntl` / `LockFileEx`), you can easily opt out of `LeaseFileLockingService` and restore `DiskLockingService`. -### 1. Programmatic Java Configuration (Opt-Out): +### 5.1 Programmatic Java Configuration (Opt-Out): ```java String storagePath = "/var/data/tus-uploads"; TusFileUploadService tus = new TusFileUploadService() .withUploadStorageService(new DiskStorageService(storagePath)) - .withUploadLockingService(new DiskLockingService(storagePath)); // Opt-out to legacy FileLock + .withUploadLockingService(new DiskLockingService(storagePath)) // Opt-out to legacy FileLock + .withUploadDeduplication(true) + .withThreadLocalCache(true); ``` -### 2. Spring Boot Bean Configuration (Opt-Out): +### 5.2 Spring Boot Bean Configuration (Opt-Out): ```java @Configuration public class TusConfig { @@ -135,7 +137,9 @@ public class TusConfig { return new TusFileUploadService() .withUploadUri("/files/upload") .withUploadStorageService(new DiskStorageService(storagePath)) - .withUploadLockingService(new DiskLockingService(storagePath)); // Opt-out to legacy FileLock + .withUploadLockingService(new DiskLockingService(storagePath)) // Opt-out to legacy FileLock + .withUploadDeduplication(true) + .withThreadLocalCache(true); } } ``` diff --git a/docs/INTEGRATION_GUIDE.md b/docs/INTEGRATION_GUIDE.md new file mode 100644 index 00000000..7a0550ff --- /dev/null +++ b/docs/INTEGRATION_GUIDE.md @@ -0,0 +1,302 @@ +# Integration Guide + +How to wire `tus-java-server` into your Java application, handle completed uploads and keep storage clean. For configuration options see the [Configuration Reference](CONFIGURATION.md). + +## Upload Lifecycle + +1. **Create & transfer** — the client talks to the upload endpoint; your code only forwards the request to `TusFileUploadService.process(...)`. +2. **Complete** — when the last byte arrives the upload is complete (an optional `UploadCompletionListener` fires). +3. **Consume** — your application reads the bytes (`getUploadedBytes`) and moves them to their final location. +4. **Clean up** — your application calls `deleteUpload`; a scheduled `cleanup()` removes abandoned or expired uploads. + +## 1. Receiving a Resumable Upload + +To process an upload request you have to pass the current `jakarta.servlet.http.HttpServletRequest` and `jakarta.servlet.http.HttpServletResponse` objects to `me.desair.tus.server.TusFileUploadService.process()`. Typical places where you can do this are Servlets, Filters or REST API Controllers. The same call handles both Tus 1.0.0 and IETF RUFH clients. + +The `process()` method returns the processed `UploadInfo` object on successful `POST`, `PATCH`, and `HEAD` requests (or `null` for `OPTIONS` preflight requests, `DELETE` terminations, or when an error response is generated). + +### Spring Boot 3 controller + +The Spring snippets below are complete and compile as-is (Spring Boot 3, Java 17+, `spring-boot-starter-web`); only the paths need adjusting. + +```java +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class FileUploadController { + + private final TusFileUploadService tusFileUploadService; + + public FileUploadController(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } + + @RequestMapping( + value = {"/api/upload", "/api/upload/**"}, + method = { + RequestMethod.POST, + RequestMethod.PUT, + RequestMethod.PATCH, + RequestMethod.HEAD, + RequestMethod.DELETE, + RequestMethod.OPTIONS, + RequestMethod.GET + }) + public void processUpload(HttpServletRequest request, HttpServletResponse response) + throws IOException { + tusFileUploadService.process(request, response); + } +} +``` + +> CORS headers and `OPTIONS` preflight handling are built in (the `cors` extension), so no `@CrossOrigin` annotation is needed. + +### Plain Jakarta Servlet (Tomcat, Jetty, Undertow, ...) + +```java +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; + +@WebServlet("/api/upload/*") +public class UploadServlet extends HttpServlet { + + private TusFileUploadService tus; + + @Override + public void init() { + tus = + new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true); + } + + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Override service() (not doPost/doGet) so that PATCH, HEAD and OPTIONS reach the library + tus.process(req, resp); + } + + @Override + public void destroy() { + try { + tus.close(); + } catch (IOException e) { + log("Failed to close TusFileUploadService", e); + } + } +} +``` + +Note that overriding `service(...)` (instead of `doGet`/`doPost`) is required so that `PATCH` requests reach the library. + +### Creation vs. progress ambiguity + +`process()` returns an `UploadInfo` on both creation (`POST`) and progress updates (`PATCH` / `HEAD`). When associating an upload ID with a user session or database entity on creation, check the method: + +```java +UploadInfo info = tusFileUploadService.process(request, response, ownerKey); +if (info != null && "POST".equalsIgnoreCase(request.getMethod())) { + userSessionRepository.recordUpload(ownerKey, info.getId()); +} +``` + +**Single-request upload completion**: when clients upload the entire file in a single request using the `creation-with-upload` extension or a RUFH single-request `POST`, the upload is already complete when `process()` returns, and any registered `UploadCompletionListener` will have already fired *before* `process()` returns to your controller. + +### Multi-tenancy with `ownerKey` + +`process(request, response, String ownerKey)` accepts an optional `ownerKey`. The `ownerKey` provides a hard separation between uploads of different users, groups or tenants: an upload created with one key cannot be read, queried or deleted with another. Examples of `ownerKey` values are user ID's, group names, client ID's... Use the same overloads with `ownerKey` on `getUploadInfo`, `getUploadedBytes` and `deleteUpload` when reading the upload back. + +## 2. Handling Upload Completion & Retrieving Files + +You can handle upload completion in two ways: +1. **Server-Side Callback Listener (`UploadCompletionListener`)**: Register a hook directly on `TusFileUploadService` that triggers automatically when the final chunk is uploaded. +2. **Domain API Endpoint**: Have your frontend notify your application backend domain endpoint (e.g. `POST /api/documents`) once upload completes. + +### Option A: Server-Side `UploadCompletionListener` + +Register one or more completion listeners on your `TusFileUploadService`. When an upload finishes transferring all bytes, the callback executes with the completed `UploadInfo` and the service instance. The upload lock is released *before* the listener is invoked, allowing you to safely stream or delete uploaded bytes immediately: + +```java +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import me.desair.tus.server.TusFileUploadService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class TusConfig { + + private static final Logger log = LoggerFactory.getLogger(TusConfig.class); + + @Bean(destroyMethod = "close") + public TusFileUploadService tusFileUploadService() { + return new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true) + .withUploadCompletionListener( + (uploadInfo, service) -> { + // Safely retrieve or guess the file name (preventing path traversal and NPE) + String fileName = + Paths.get(uploadInfo.getFileName()).getFileName().toString(); + try { + try (InputStream is = service.getUploadedBytes(uploadInfo)) { + Files.copy(is, Paths.get("/var/data/documents", fileName)); + } // closing the stream releases the lock, which deleteUpload needs + service.deleteUpload(uploadInfo); + } catch (Exception e) { + log.error("Failed to process completed upload {}", uploadInfo.getId(), e); + } + }); + } +} +``` + +> [!WARNING] +> Never use a client-supplied `filename` directly as a path component without sanitising it (path traversal). The examples above are simplified. + +### Option B: Client-Initiated Domain Notification + +When an upload completes, the client receives the final `204 No Content` response from the Tus/RUFH protocol endpoint (`/api/upload/...`). Because Tus and RUFH are decoupled file transport protocols, your frontend application can notify your backend domain API (e.g. `POST /api/documents`) that the upload is complete and pass along the `uploadUrl`. + +> [!NOTE] +> `POST /api/documents` represents your application's domain REST endpoint, not a protocol endpoint. The Tus server itself handles file transfer (`/api/upload`), while your application endpoint coordinates business logic, database persistence, and final file consumption. + +```java +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import me.desair.tus.server.TusFileUploadService; +import me.desair.tus.server.exception.TusException; +import me.desair.tus.server.upload.UploadInfo; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class DocumentController { + + /** Request body sent by your frontend once the upload finished. */ + public record DocumentUploadRequest(String uploadUrl) {} + + private final TusFileUploadService tusFileUploadService; + + public DocumentController(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } + + @PostMapping("/api/documents") + public ResponseEntity completeDocumentUpload(@RequestBody DocumentUploadRequest request) + throws IOException, TusException { + String uploadUrl = request.uploadUrl(); + + // 1. Retrieve upload metadata (file name, length, custom client metadata) + UploadInfo info = tusFileUploadService.getUploadInfo(uploadUrl); + String fileName = Paths.get(info.getFileName()).getFileName().toString(); + + // 2. Stream the uploaded bytes to their permanent location + try (InputStream is = tusFileUploadService.getUploadedBytes(uploadUrl)) { + Files.copy(is, Paths.get("/var/data/documents", fileName)); + } + + // 3. Remove the temporary upload + tusFileUploadService.deleteUpload(uploadUrl); + return ResponseEntity.ok().build(); + } +} +``` + +> `getUploadedBytes(...)` returns a stream that holds the upload lock until it is closed. Always use try-with-resources. + +### Upload information + +Using `me.desair.tus.server.TusFileUploadService.getUploadInfo(String uploadUrl)` you can retrieve metadata about a specific upload process. This includes metadata provided by the client as well as metadata kept by the library like creation timestamp, creator ip-address list, upload length... The method `UploadInfo.getId()` returns the unique identifier of this upload encapsulated in an `UploadId` instance. The original (custom generated) identifier object of this upload can be retrieved using `UploadId.getOriginalObject()`. A URL safe string representation of the identifier is returned by `UploadId.toString()`. It is highly recommended to consult the [JavaDoc of both classes](https://tus.desair.me/). + +### Cloud Storage Native Processing (S3 & Azure Blob) + +When using cloud object storage backends, downstream services can directly obtain the raw cloud object key or blob name to perform zero-download server-side copying, background job processing, or direct cloud SDK operations: + +* **S3 / MinIO Storage**: Call `s3StorageService.getS3ObjectKey(uploadUrl)` to get the full S3 object key (e.g. `uploads/018f3a...`). For complete server-side `copyObject` examples and MinIO SDK usage, see the **[S3 Storage Guide](S3_STORAGE.md#4-post-upload-processing-gets3objectkey)**. +* **Azure Blob Storage**: Call `azureBlobStorageService.getAzureBlobName(uploadUrl)` to get the full blob name. For complete Azure SDK examples with `BlobClient`, see the **[Azure Blob Storage Guide](AZURE_BLOB_STORAGE.md#4-post-upload-processing-getazureblobname)**. + +> [!TIP] +> In Spring Boot applications, inject your `S3StorageService` or `AzureBlobStorageService` bean directly into downstream controllers or completion listeners. Avoid casting `(S3StorageService) tusFileUploadService.getUploadStorageService()` because when `withThreadLocalCache(true)` is enabled, `getUploadStorageService()` returns the `ThreadLocalCachedStorageAndLockingService` wrapper, causing a `ClassCastException`. + +## 3. Cleanup + +After having processed the uploaded bytes on the server backend (e.g. copy them to their final persistent location), it's important to clean up the (temporary) uploaded bytes. This can be done by calling `me.desair.tus.server.TusFileUploadService.deleteUpload(String uploadUri)` as shown above. This will remove the uploaded bytes and any associated upload information from the storage backend. Alternatively, a client can also remove an (in-progress) upload using the [termination extension](https://tus.io/protocols/resumable-upload.html#termination). + +Next to removing uploads after they have been completed and processed by the backend, it is also recommended to schedule a regular maintenance task to clean up any expired uploads or locks, using `me.desair.tus.server.TusFileUploadService.cleanup()` (expiration requires `withUploadExpirationPeriod(...)`): + +```java +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +public class CleanupJob { + + private final TusFileUploadService tusFileUploadService; + + public CleanupJob(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } + + @Scheduled(fixedDelay = 600000) // every 10 minutes; requires @EnableScheduling + public void cleanupExpiredUploads() throws IOException { + tusFileUploadService.cleanup(); + } +} +``` + +## 4. Security Checklist + +The library implements the transfer protocols; **authentication and authorization are your responsibility**: + +* **Authenticate the endpoint**: protect `/api/upload/**` with your normal security layer (Spring Security, servlet filter, API gateway) *before* calling `process()`. Remember `OPTIONS` preflight requests carry no credentials. +* **Isolate tenants**: pass a user/tenant `ownerKey` to `process()`, `getUploadInfo`, `getUploadedBytes` and `deleteUpload` so uploads can't be accessed across users. +* **Custom checks**: implement a `TusExtension` (see `addTusExtension`) for rules such as allowed file types or per-user quotas. +* **Limit resource use**: set `withMaxUploadSize(...)` and `withUploadExpirationPeriod(...)`, and schedule `cleanup()`. +* **Treat metadata as untrusted**: `Upload-Metadata` (filename, content type) is client-controlled. Sanitise filenames and verify content before use. +* **Prefer HTTPS** and, for integrity, enable checksums (`Upload-Checksum` for Tus, `Content-Digest` / `Repr-Digest` for RUFH). + +## 5. Frontend Client Example (Uppy / JavaScript) + +Connect any standard Tus client (e.g. [Uppy](https://uppy.io/) or `tus-js-client`) to your backend upload endpoint: + +```javascript +import Uppy from '@uppy/core'; +import Tus from '@uppy/tus'; + +const uppy = new Uppy().use(Tus, { + endpoint: 'http://localhost:8080/api/upload', + chunkSize: 5 * 1024 * 1024 // 5MB chunk size +}); +``` + +Complete working examples: + +* [Resumable and asynchronous file upload in Spring Boot REST API with Uppy JavaScript client.](https://github.com/tomdesair/tus-java-server-spring-demo) +* [Detailed blog post by Ralph](https://golb.hplar.ch/2019/06/upload-with-tus.html) on how to use this library in [Spring Boot in combination with the Tus JavaScript client](https://github.com/ralscha/blog2019/tree/master/uploadtus). +* [Resumable and asynchronous file upload using Uppy with form submission in Dropwizard (Jetty)](https://github.com/tomdesair/tus-java-server-dropwizard-demo) + +For a curl-based walkthrough of both protocols see the [Quick Start in the README](../README.md#try-it-with-curl). diff --git a/docs/LOCKING.md b/docs/LOCKING.md index dbadfeaf..348bbea5 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -93,10 +93,10 @@ public interface UploadLock extends Closeable { | Storage Backend / Environment | Locking Service Class | Key Characteristics & Architecture | Documentation File | |---|---|---|---| -| **Disk & Network Filesystems (Default)** | `LeaseFileLockingService` | Atomic sibling mutex directory (`.mutex/`), in-place expired lock takeover, TTL-based JSON lease files with heartbeat renewal, ownership fencing, and `.stop` signal files. Fully safe on NFSv3/v4, AWS EFS, SMB/CIFS, Kubernetes containers, and local disks. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | -| **Local File System (Legacy Opt-Out)** | `DiskLockingService` | OS kernel-level exclusive POSIX `FileLock` (`fcntl`) with JVM shutdown hooks and `.stop` signal files. Best for single-node deployments on local disk. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | -| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), atomic conditional writes (`If-None-Match: *`), non-CAS lock arbitration with configurable jitter backoff (`withJitter`) for backends lacking CAS (B2, Ceph, Wasabi), heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/S3_STORAGE.md) | -| **Azure Blob Storage** | `AzureBlobLockingService` | Native Azure Blob Storage exclusive 30-second leases (`BlobLeaseClient`), background daemon renewal, and `.stop` signal blob polling watchdog. | [`docs/AZURE_BLOB_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/AZURE_BLOB_STORAGE.md) | +| **Disk & Network Filesystems (Default)** | `LeaseFileLockingService` | Atomic sibling mutex directory (`.mutex/`), in-place expired lock takeover, TTL-based JSON lease files with heartbeat renewal, ownership fencing, and `.stop` signal files. Fully safe on NFSv3/v4, AWS EFS, SMB/CIFS, Kubernetes containers, and local disks. | [`docs/DISK_BASED_LOCKING.md`](DISK_BASED_LOCKING.md) | +| **Local File System (Legacy Opt-Out)** | `DiskLockingService` | OS kernel-level exclusive POSIX `FileLock` (`fcntl`) with JVM shutdown hooks and `.stop` signal files. Best for single-node deployments on local disk. | [`docs/DISK_BASED_LOCKING.md`](DISK_BASED_LOCKING.md) | +| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), atomic conditional writes (`If-None-Match: *`), non-CAS lock arbitration with configurable jitter backoff (`withJitter`) for backends lacking CAS (B2, Ceph, Wasabi), heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](S3_STORAGE.md) | +| **Azure Blob Storage** | `AzureBlobLockingService` | Native Azure Blob Storage exclusive 30-second leases (`BlobLeaseClient`), background daemon renewal, and `.stop` signal blob polling watchdog. | [`docs/AZURE_BLOB_STORAGE.md`](AZURE_BLOB_STORAGE.md) | --- diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 3adbba5b..6c35441b 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -24,6 +24,8 @@ import me.desair.tus.server.TusFileUploadService; TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files") + .withUploadDeduplication(true) + .withThreadLocalCache(true) // Enabled by default: AUTO mode detects TUS_1_0_0 vs RUFH per request .withSupportedProtocolVersions(ProtocolVersion.AUTO); ``` @@ -150,43 +152,6 @@ Ensure that reverse proxies (Nginx, HAProxy, AWS ALB, Cloudflare): --- -## 6. Java Server API Migration Guide (v1.0.0-3.3 to v2.0.0) +## 6. Upgrading the Java Server Library -When upgrading backend Java applications from `1.0.0-3.3` to `2.0.0`, note the following changes and recommendations: - -### 6.1 Java 17 & Jakarta EE Baseline -- **Java 17**: `tus-java-server` 2.0.0 targets Java 17+. -- **Jakarta EE**: The library uses `jakarta.servlet.*` package imports instead of legacy `javax.servlet.*` packages, aligning with Spring Boot 3.x, Tomcat 10+, and Jetty 11+. - -### 6.2 Default Distributed Lease Locking -- In `1.0.0-3.3`, `TusFileUploadService.withStoragePath(path)` defaulted to `DiskLockingService`, which relied on OS kernel file locks (`java.nio.channels.FileLock`). -- In `2.0.0`, `withStoragePath(path)` defaults to `LeaseFileLockingService`, which uses atomic sibling mutex directories (`.mutex/`) and JSON lease files with auto-renewal. This provides container-safe, distributed locking across multi-replica pods on Kubernetes, NFS (v3/v4), AWS EFS, Azure Files, and SMB without requiring OS-level lock daemons or external coordination. -- **Opting Out**: If your application runs exclusively on local disks and you prefer kernel file locks, explicitly configure: - ```java - tusService.withUploadLockingService(new DiskLockingService(storagePath)); - ``` - -### 6.3 Jackson Dependencies Bundled in Compile Scope -- Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are now included with compile scope in `tus-java-server`. You no longer need to declare Jackson separately in your application `pom.xml` when using JSON metadata serialization (`withJsonSerialization()`) or cloud storage. - -### 6.4 Safe Lock-Holding InputStream Lifecycle -- In 2.0.0, `tusService.getUploadedBytes(...)` returns a stream that retains an exclusive upload lock until closed. This prevents concurrent `DELETE` or `PATCH` requests from corrupting data mid-stream. -- **Best Practice**: Always consume uploaded bytes within a `try-with-resources` block: - ```java - try (InputStream stream = tusService.getUploadedBytes(uploadUri)) { - // Process file data safely under lock protection - } // Lock is automatically released on close - ``` - -### 6.5 Pluggable Cloud Storage Backends -- `2.0.0` introduces native cloud object storage backends: - - **S3 Storage**: `S3StorageService`, `S3LockingService`, and `S3ConcatenationService` (AWS S3, MinIO, RustFS, Cloudflare R2, Ceph, GCS). See [docs/S3_STORAGE.md](S3_STORAGE.md). - - **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService`, and `AzureBlobConcatenationService`. See [docs/AZURE_BLOB_STORAGE.md](AZURE_BLOB_STORAGE.md). - -### 6.6 Post-Upload Listeners (`UploadCompletionListener`) -- You can now register callbacks that fire immediately upon upload completion: - ```java - tusService.withUploadCompletionListener((uploadInfo, service) -> { - log.info("Upload completed: {}", uploadInfo.getId()); - }); - ``` +Looking for the Java API changes between `1.0.0-3.3` and `2.0.0` (Java 17, Jakarta, default locking, lock-holding streams, storage backends, listeners)? See the **[Upgrade Guide](UPGRADING.md)**. diff --git a/docs/PROTOCOLS.md b/docs/PROTOCOLS.md new file mode 100644 index 00000000..707a3966 --- /dev/null +++ b/docs/PROTOCOLS.md @@ -0,0 +1,137 @@ +# Protocols: Tus 1.0.0 & IETF Resumable Uploads for HTTP (RUFH) + +`tus-java-server` serves **both** resumable upload protocols on the same endpoint: + +1. **Tus 1.0.0**: the widely-adopted [tus protocol standard](https://tus.io/). +2. **IETF Resumable Uploads for HTTP (RUFH)**: the official IETF standardization draft ([draft-ietf-httpbis-resumable-upload](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/), implemented at **draft-12**). + +> [!WARNING] +> **Experimental Feature Disclaimer**: The IETF Resumable Uploads for HTTP (RUFH) specification (`draft-ietf-httpbis-resumable-upload`) is currently an active IETF draft. While this library implements draft-12 compliance, the RUFH protocol support should be considered **experimental** until the specification is published as an official RFC standard. + +Migrating clients from Tus to RUFH? See the [RUFH Migration Guide](MIGRATION.md). Upgrading the Java library? See the [Upgrade Guide](UPGRADING.md). + +## Configuring the Protocol Version + +Configure protocol support via `withSupportedProtocolVersions(ProtocolVersion)`: + +* `ProtocolVersion.AUTO` (Default): Automatically detects the protocol version per HTTP request based on request headers (`Tus-Resumable` header triggers Tus 1.0.0; `Upload-Complete` header or `application/partial-upload` content type triggers IETF RUFH). +* `ProtocolVersion.TUS_1_0_0`: Enforces Tus 1.0.0 handling exclusively. +* `ProtocolVersion.RUFH`: Enforces IETF Resumable Uploads for HTTP (RUFH) handling exclusively. + +## How the Protocols Work + +### Tus 1.0.0 Protocol Flow +```mermaid +sequenceDiagram + autonumber + actor Client as Frontend Client (Uppy / tus-js-client) + participant ProtocolEndpoint as Server: Tus Protocol Endpoint (/api/upload) + participant Storage as Storage Backend (Disk / S3 / Azure) + participant AppEndpoint as Application API (/api/documents) + + Note over Client,Storage: 1. Create Resumable Upload Resource + Client->>ProtocolEndpoint: POST /api/upload
(Tus-Resumable: 1.0.0, Upload-Length: 1048576, Upload-Metadata: ...) + ProtocolEndpoint->>Storage: Initialize upload metadata + ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a..., Tus-Resumable: 1.0.0) + + Note over Client,Storage: 2. Stream Data Chunk(s) + Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Content-Type: application/offset+octet-stream) + ProtocolEndpoint->>Storage: Append chunk bytes to storage + ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Tus-Resumable: 1.0.0) + + Note over Client,AppEndpoint: 3. Domain Notification & Consumption + Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") + AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) + AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) + AppEndpoint-->>Client: 200 OK (Processed) +``` + +### IETF Resumable Uploads for HTTP (RUFH) Flow +```mermaid +sequenceDiagram + autonumber + actor Client as Frontend Client (RUFH Client) + participant ProtocolEndpoint as Server: RUFH Protocol Endpoint (/api/upload) + participant Storage as Storage Backend (Disk / S3 / Azure) + participant AppEndpoint as Application API (/api/documents) + + Note over Client,Storage: 1. Create Resumable Upload Resource + Client->>ProtocolEndpoint: POST /api/upload
(Upload-Complete: ?0) + ProtocolEndpoint->>Storage: Initialize upload metadata + ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a...) + + Note over Client,Storage: 2. Stream Data Chunk(s) + Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Upload-Complete: ?1, Content-Type: application/partial-upload) + ProtocolEndpoint->>Storage: Append chunk bytes to storage + ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Upload-Complete: ?1) + + Note over Client,AppEndpoint: 3. Domain Notification & Consumption + Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") + AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) + AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) + AppEndpoint-->>Client: 200 OK (Processed) +``` + +## Protocol Comparison & Available Features + +| Feature / Capability | Tus 1.0.0 | IETF Resumable Uploads | +|---|---|---| +| **Auto-Detection Signal** | `Tus-Resumable: 1.0.0` header | `Upload-Complete` header or `Content-Type: application/partial-upload` | +| **Creation** | `POST` with `Upload-Length` / `Upload-Defer-Length` | `POST` with `Upload-Complete: ?0` / `?1` (or `application/partial-upload`) | +| **Append Chunks** | `PATCH` with `Upload-Offset` | `PATCH` with `Upload-Offset` & `Content-Type: application/partial-upload` | +| **Upload Status Query** | `HEAD` returns `Upload-Offset` & `Upload-Length` | `HEAD` returns `Upload-Offset` & `Upload-Complete` | +| **Offset Mismatch Error** | HTTP 409 Conflict | HTTP 409 Conflict with RFC 7807 `application/problem+json` details | +| **104 Interim Responses** | N/A | Supported (see [INTERIM_RESPONSES.md](INTERIM_RESPONSES.md)) | +| **Upload Cancellation** | `DELETE` with `Tus-Resumable: 1.0.0` | `DELETE` with `Upload-Complete: ?0` | +| **Checksum Validation** | Supported (`Checksum` extension) | Supported (based on HTTP Digests / RFC 9530) | +| **Expiration Handling** | Supported (`Upload-Expires` header) | Supported (`max-age` parameter in `Upload-Limit` header) | +| **Concatenation** | Supported (`Concatenation` extension) | N/A | +| **Download Extension** | Supported (`Download` extension) | Supported (`Download` extension; without it, `GET` requests perform offset retrieval) | + +## Protocol Extensions + +Besides the [tus core protocol](https://tus.io/protocols/resumable-upload.html#core-protocol), the library has all optional tus protocol extensions enabled by default. This means that the `Tus-Extension` header has value `creation,creation-defer-length,creation-with-upload,checksum,checksum-trailer,termination,expiration,concatenation,concatenation-unfinished`. Optionally you can also enable an unofficial `download` extension (see the [Configuration Reference](CONFIGURATION.md)). + +* [creation](https://tus.io/protocols/resumable-upload.html#creation): The creation extension allows you to create new uploads and to retrieve the upload URL for them. +* [creation-defer-length](https://tus.io/protocols/resumable-upload.html#post): You can create a new upload even if you don't know its final length at the time of creation. +* [creation-with-upload](https://tus.io/protocols/resumable-upload.html#creation): The creation-with-upload extension allows you to create the upload resource and upload initial file data in a single POST request. +* [checksum](https://tus.io/protocols/resumable-upload.html#checksum): An extension that allows you to verify data integrity of each upload (PATCH) request. +* [checksum-trailer](https://tus.io/protocols/resumable-upload.html#checksum): If the checksum hash cannot be calculated at the beginning of the upload, it may be included as a trailer HTTP header at the end of the chunked HTTP request. +* [termination](https://tus.io/protocols/resumable-upload.html#termination): Clients can terminate completed or in-progress uploads which allows the tus-java-server library to free up resources on the server. +* [expiration](https://tus.io/protocols/resumable-upload.html#expiration): You can instruct the tus-java-server library to cleanup uploads that are older than a configurable period. The expiration extension applies to both Tus 1.0.0 and IETF Resumable Uploads for HTTP (RUFH) protocols: + * **Tus 1.0.0 Protocol**: Expiration is communicated to clients via the `Upload-Expires` response header field formatted as an HTTP date-time (RFC 7231, e.g. `Upload-Expires: Wed, 25 Jun 2026 16:00:00 GMT`). + * **IETF RUFH Protocol**: Expiration is communicated to clients via the `max-age` parameter in the `Upload-Limit` response header field indicating remaining valid seconds (e.g. `Upload-Limit: max-size=1048576, max-age=3600`). +* [concatenation](https://tus.io/protocols/resumable-upload.html#concatenation): This extension can be used to concatenate multiple uploads into a single final upload enabling clients to perform parallel uploads and to upload non-contiguous chunks. +* [concatenation-unfinished](https://tus.io/protocols/resumable-upload.html#concatenation): The client is allowed send the request to concatenate partial uploads while these partial uploads are still in progress. +* [http-digests](https://datatracker.ietf.org/doc/rfc9530/): An extension implementing RFC 9530 to verify data integrity for the Resumable Uploads for HTTP (RUFH) protocol. Supported headers include `Content-Digest`, `Repr-Digest`, `Want-Content-Digest`, and `Want-Repr-Digest`. +* `download`: The (unofficial) download extension allows clients to download uploaded files using a HTTP `GET` request. You can enable this extension by calling the `withDownloadFeature()` method. +* `cors`: The (unofficial) CORS extension adds native CORS support out-of-the-box, setting CORS headers for all requests and responses, and handling preflight `OPTIONS` requests automatically. It is enabled by default, so you do **not** need `@CrossOrigin` or a separate CORS filter on your upload endpoint. Disable it with `disableTusExtension("cors")` if you manage CORS yourself. + +## HTTP Digests ([RFC 9530](https://www.rfc-editor.org/rfc/rfc9530.html)) + +The `http-digests` extension implements RFC 9530 to support data integrity checks for both individual data chunks (`Content-Digest`) and the entire file (`Repr-Digest`). + +* **Performance Disclaimer**: Calculating representation digests (`Repr-Digest`) requires streaming the entire uploaded file from disk. For extremely large files, this can introduce non-trivial I/O performance overhead on the server. To optimize, only request it via `Want-Repr-Digest` when absolutely necessary. + +## HTTP 104 Interim Responses + +The IETF draft allows the server to send `104 Upload Resumption Supported` interim responses. The Servlet API cannot emit 1xx responses natively; see the [HTTP 104 Interim Responses Guide](INTERIM_RESPONSES.md) for a Tomcat `Valve` (including Spring Boot registration). A live reference implementation using cached reflection is available in the [tus-java-server-spring-demo](https://github.com/tomdesair/tus-java-server-spring-demo) repository: [`TusInterimResponseTomcatValve`](https://github.com/tomdesair/tus-java-server-spring-demo/blob/main/spring-boot-rest/src/main/java/me/desair/spring/tus/TusInterimResponseTomcatValve.java). + +To register the Valve in Spring Boot via `TomcatServletWebServerFactory`: + +```java +@Bean +public TomcatServletWebServerFactory tomcatFactory(TusFileUploadService tusFileUploadService) { + TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory(); + factory.addContextValves(new TusInterimResponseTomcatValve(tusFileUploadService)); + return factory; +} +``` + +## Compatible Clients & Conformity Testing + +This server implementation has been tested with: +- **Tus 1.0.0 Clients & Conformity Tests**: [Uppy](https://uppy.io/), `tus-js-client`, and the built-in Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) validating all core protocol mechanisms and protocol extensions (creation, creation-with-upload, checksum, termination, concatenation, and expiration) across Disk, S3, and Azure Blob backends. +- **IETF Resumable Uploads Clients & Conformity Tests**: The built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). + +See the [Conformity Testing Guide](CONFORMITY_TESTING.md) for how to run the suites. The repository also contains automated integration test suites (`ITTusFileUploadService`, `RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`) validating both protocol specifications. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..f1a1b1a0 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,26 @@ +# Documentation Index + +## I want to use the library +1. [README](../README.md): what it is, quick start (Spring Boot, plain Servlet, curl) +2. [Integration Guide](INTEGRATION_GUIDE.md): `process()`, completion handling, cleanup, security, frontend clients +3. [Configuration Reference](CONFIGURATION.md): all options and storage backend choice +4. [Deployment Guide](DEPLOYMENT.md): multi-replica, reverse proxies, containers +5. [Troubleshooting](TROUBLESHOOTING.md) +6. [Architecture Guide](ARCHITECTURE.md): system structure, protocol extensions, storage SPI, and distributed locking + +## Protocols +- [Protocols](PROTOCOLS.md): Tus 1.0.0 & IETF RUFH flows, comparison, extensions, digests +- [Migrating clients to RUFH](MIGRATION.md): header mapping and request examples +- [HTTP 104 Interim Responses](INTERIM_RESPONSES.md) + +## Upgrading +- [Upgrade Guide 1.x → 2.0](UPGRADING.md) and the [CHANGELOG](../CHANGELOG.md) + +## Storage & locking +- [Locking concepts & custom lock services](LOCKING.md) +- [Disk & network storage (NFS, EFS, SMB)](DISK_BASED_LOCKING.md) +- [S3-compatible storage](S3_STORAGE.md) +- [Azure Blob Storage](AZURE_BLOB_STORAGE.md) + +## Contributors & maintainers +- [Architecture Guide](ARCHITECTURE.md), [CONTRIBUTING](../CONTRIBUTING.md), [Testing guide](TESTING.md), [Conformity testing](CONFORMITY_TESTING.md), [Release process](RELEASE.md) diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 05f77181..8dc3222b 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -52,12 +52,14 @@ S3StorageService awsS3Storage = new S3StorageService( S3LockingService awsS3Locking = new S3LockingService( endpoint, "eu-central-1", accessKey, secretKey, bucketName); -// Configure TusFileUploadService with S3 storage and locking +// Configure TusFileUploadService with S3 storage, locking, and request caching // Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files/upload") .withUploadStorageService(awsS3Storage) - .withUploadLockingService(awsS3Locking); + .withUploadLockingService(awsS3Locking) + .withUploadDeduplication(true) + .withThreadLocalCache(true); ``` --- @@ -66,11 +68,11 @@ TusFileUploadService tusService = new TusFileUploadService() > [!IMPORTANT] > **Why `ThreadLocalCachedStorageAndLockingService` is Recommended for S3**: -> By default, `TusFileUploadService` automatically wraps your custom `UploadStorageService` and `UploadLockingService` in a `ThreadLocalCachedStorageAndLockingService`. +> By default, thread-local caching is disabled (`false`). You should explicitly enable it via `.withThreadLocalCache(true)`. > > During a single HTTP request lifecycle (POST, PATCH, HEAD, DELETE), the tus server validates request headers, reads upload state, appends data, and constructs response headers. Without caching, retrieving `UploadInfo` and calculating offsets would require multiple redundant network roundtrips to S3 (`GetObject` on `.info`, `ListObjects`, `StatObject`). > -> `ThreadLocalCachedStorageAndLockingService` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces S3 network latency and cost per request. +> Enabling `withThreadLocalCache(true)` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces S3 network latency and cost per request. --- @@ -108,13 +110,14 @@ import io.minio.CopySource; import io.minio.GetObjectArgs; import java.io.InputStream; -S3StorageService s3Storage = (S3StorageService) tusService.getUploadStorageService(); - +// Inject or reference the concrete S3StorageService bean directly. +// (Avoid casting tusService.getUploadStorageService() when withThreadLocalCache(true) +// is enabled, as it returns ThreadLocalCachedStorageAndLockingService). String uploadUri = "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"; String ownerKey = "user-123"; // 1. Obtain full S3 key after upload completion using uploadUri and ownerKey -String s3ObjectKey = s3Storage.getS3ObjectKey(uploadUri, ownerKey); +String s3ObjectKey = awsS3Storage.getS3ObjectKey(uploadUri, ownerKey); // e.g. "uploads/24249a5b-01a4-4bf8-b67a-364273bb5a2e" // 2. Example: Storage-side processing using MinioClient (server-side object copy) diff --git a/docs/TESTING.md b/docs/TESTING.md index c664614c..bf55f415 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1,6 +1,32 @@ -# Manual Testing +# Testing Guide -## Setup & Running the Server +This document describes the testing strategy for `tus-java-server`, how to execute each testing tier, and step-by-step instructions for manual lock contention testing. + +--- + +## 1. The Three-Tiered Testing Strategy + +The repository follows a three-tiered testing hierarchy (see [Architecture Guide](ARCHITECTURE.md#7-three-tiered-testing-strategy) for architectural details): + +1. **Unit Tests (`*Test.java`)**: Pure offline tests using Mockito. Fast (< 2s), zero containers or network dependencies. Run with: + ```bash + mvn test -q + ``` +2. **Integration Tests (`IT*`)**: End-to-end multi-backend validation using Testcontainers (Podman or Docker). Tested across Disk, S3 (RustFS), and Azure (Azurite) in CI. Run with: + ```bash + mvn verify -q + ``` +3. **Conformity Tests (`scripts/*_conformity_test.py`)**: Socket-level Python protocol verification suites executed against running server deployments. See the [Conformity Testing Guide](CONFORMITY_TESTING.md) for full instructions: + ```bash + python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload + ``` + +--- + +## 2. Manual Testing & Contention Verification + +### Setup & Running the Server Before performing the manual verification, you need to compile the library and start the Spring Boot demo server. @@ -16,7 +42,7 @@ Open the `spring-boot-rest/pom.xml` file in the sibling `tus-java-server-spring- me.desair.tus tus-java-server - 2.0.0 + 2.0.0-SNAPSHOT ``` @@ -27,7 +53,7 @@ cd ../tus-java-server-spring-demo mvn clean package java -jar spring-boot-rest/target/spring-boot-rest-0.0.1-SNAPSHOT.jar ``` -*Note:* The demo server runs locally on port `8080` and exposes the upload endpoint at `http://localhost:8080/api/upload`. +*Note:* The demo server runs locally on port `8080` and exposes the upload endpoint at `http://localhost:8080/test/api/upload`. --- @@ -70,7 +96,7 @@ curl -X POST \ -H "Upload-Length: 10000000" \ -I http://localhost:8080/test/api/upload ``` -Look for the `Location` header in the response and record the upload resource URL (e.g., `http://localhost:8080/api/upload/000003f1-a850-49de-af03-997272d834c9`). We will refer to this as `${UPLOAD_URL}`. +Look for the `Location` header in the response and record the upload resource URL (e.g., `http://localhost:8080/test/api/upload/000003f1-a850-49de-af03-997272d834c9`). We will refer to this as `${UPLOAD_URL}`. ``` export UPLOAD_URL='http://localhost:8080/test/api/upload/0330595f-1c0d-49f5-9e78-c40b1845157d' @@ -104,7 +130,7 @@ curl -X HEAD -H "Tus-Resumable: 1.0.0" -I ${UPLOAD_URL} - **First Terminal (PATCH)**: - The stalled/throttled `PATCH` upload should immediately abort with an I/O error or exit because the server terminated its input stream in response to the lock release request. - **Second Terminal (HEAD)**: - - The `HEAD` request should succeed with status `204 No Content` after a short delay (the server retries up to 40 times at 200ms intervals, an 8.0-second retry budget, while releasing the lock). + - The `HEAD` request should succeed with status `204 No Content` after a short delay (the server retries up to 300 times at 200ms intervals, a 60-second retry budget, configurable via `withLockWaitTimeout(Duration)`, while releasing the lock). - The response will contain the `Upload-Offset` header reflecting the number of bytes successfully written to disk before the stream was interrupted (e.g., `Upload-Offset: 153600`). - **Subsequent Uploads**: - You can immediately resume the upload using a new `PATCH` request starting from the offset returned in the `HEAD` response. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md new file mode 100644 index 00000000..394a3ee9 --- /dev/null +++ b/docs/TROUBLESHOOTING.md @@ -0,0 +1,22 @@ +# Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `404` for every request, or `process()` does nothing | `withUploadUri(...)` doesn't match the path the request arrives on (context path, proxy rewrite). | Make the URI match the servlet path, or use an absolute URI behind a proxy. | +| `PATCH` returns `405` / never reaches the library | Servlet overrides `doPost`/`doGet` only, or controller mapping lacks `PATCH`. | Override `service(...)` or map all of `POST, PUT, PATCH, HEAD, DELETE, OPTIONS, GET`. | +| Browser blocked by CORS | A second CORS filter conflicts, or `cors` extension disabled. | Rely on the built-in `cors` extension, or disable it (`disableTusExtension("cors")`) and configure CORS yourself. Preflight `OPTIONS` must reach `process()` unauthenticated. | +| `406 Not Acceptable` | Unexpected `Content-Type`. | Tus `PATCH` needs `application/offset+octet-stream`; RUFH needs `application/partial-upload`. Parameters such as `;charset=UTF-8` are tolerated. | +| `409 Conflict` on `PATCH` | Client `Upload-Offset` differs from the server offset (e.g. a retry after a lost response). | The client should send `HEAD`, read the current offset and resume from there. RUFH returns `application/problem+json` with `expected-offset` / `provided-offset`. | +| `413 Payload Too Large` when creating | `Upload-Length` exceeds `withMaxUploadSize(...)`. | Raise the limit or reduce the file size. | +| `423 Locked` / slow `HEAD` after pause | Another request still holds the upload lock (a stalled `PATCH`). | The library asks the lock holder to release and waits up to `withLockWaitTimeout(...)` (default 60 s). Increase the timeout if you have very slow storage. See [LOCKING.md](LOCKING.md). | +| Locks never expire across replicas | Clock skew between nodes, or storage not shared. | Fix NTP; ensure all replicas use the same backend. | +| `HTTP 500` / error in Undertow when an error is sent | Fixed in 2.0.0 (`Content-Length` conflict). | Upgrade. | +| `NoClassDefFoundError` for Jackson | Old version without bundled Jackson. | Upgrade to 2.0.0 (Jackson is bundled). | +| `ClassNotFoundException: javax.servlet...` | 2.x requires `jakarta.servlet`. | Use Tomcat 10+/Jetty 11+/Spring Boot 3+. See [UPGRADING.md](UPGRADING.md). | +| Uploaded file can't be deleted/read: "locked" | A `getUploadedBytes(...)` stream wasn't closed. | Use try-with-resources. | +| Uploads accumulate on disk or in the bucket | `cleanup()` not scheduled or no expiration period. | Set `withUploadExpirationPeriod(...)` and run `cleanup()` periodically. | +| Upload restarts from 0 behind a proxy | Proxy buffers or times out the request body. | See [DEPLOYMENT.md](DEPLOYMENT.md#3-reverse-proxies--load-balancers). | + +Backend-specific problems: [S3 troubleshooting](S3_STORAGE.md#10-troubleshooting-guide) and [Azure troubleshooting](AZURE_BLOB_STORAGE.md#8-troubleshooting-guide). + +Still stuck? Run the [conformity tests](CONFORMITY_TESTING.md) against your endpoint or [open an issue](https://github.com/tomdesair/tus-java-server/issues) with the request/response headers. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md new file mode 100644 index 00000000..bb0a05a2 --- /dev/null +++ b/docs/UPGRADING.md @@ -0,0 +1,74 @@ +# Upgrade Guide: `1.0.0-3.x` → `2.0.0` + +Everything a Java developer needs when upgrading an application from `tus-java-server` `1.0.0-3.3` (or earlier) to `2.0.0`. For the full list of changes see the [CHANGELOG](../CHANGELOG.md). To upgrade your *clients* to the IETF Resumable Uploads protocol see [MIGRATION.md](MIGRATION.md). + +## Upgrade Checklist + +| # | Change | Action required | +|---|---|---| +| 1 | Java 17+ and `jakarta.servlet` | Run on Java 17+ and a Jakarta container (Tomcat 10+, Jetty 11+, Spring Boot 3+). | +| 2 | Default locking is now `LeaseFileLockingService` | None for most; opt out explicitly if you want kernel file locks (§2). | +| 3 | `getUploadedBytes(...)` holds the lock until the stream is closed | Always use try-with-resources (§4). | +| 4 | `process(...)` now returns `UploadInfo` | None; source compatible. Use the return value if useful. | +| 5 | Download extension no longer returns `204` for unfinished uploads and omits `Tus-Resumable` | Check clients that rely on the old behaviour (§7). | +| 6 | Jackson now bundled (compile scope) | Remove redundant Jackson declarations if they conflict. | +| 7 | Dual protocol (`ProtocolVersion.AUTO` by default) | None for Tus clients; see §8 to restrict. | +| 8 | `UploadInProgressException` removed | Remove any references. | +| 9 | Absolute URLs accepted in `withUploadUri(...)` | Optional. | +| 10 | Security hardening (path traversal, CORS, permissions) | Transparent; verify custom URI handling if applicable (§9). | + +## 1. Java 17 & Jakarta EE Baseline +- **Java 17**: `tus-java-server` 2.0.0 targets Java 17+. +- **Jakarta EE**: The library uses `jakarta.servlet.*` package imports instead of legacy `javax.servlet.*` packages, aligning with Spring Boot 3.x, Tomcat 10+, and Jetty 11+. Applications still on `javax.servlet` must stay on `1.0.0-3.3` or migrate. + +## 2. Default Distributed Lease Locking +- In `1.0.0-3.3`, `TusFileUploadService.withStoragePath(path)` defaulted to `DiskLockingService`, which relied on OS kernel file locks (`java.nio.channels.FileLock`). +- In `2.0.0`, `withStoragePath(path)` defaults to `LeaseFileLockingService`, which uses atomic sibling mutex directories (`.mutex/`) and JSON lease files with auto-renewal. This provides container-safe, distributed locking across multi-replica pods on Kubernetes, NFS (v3/v4), AWS EFS, Azure Files, and SMB without requiring OS-level lock daemons or external coordination. See [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md). +- **Opting Out**: If your application runs exclusively on local disks and you prefer kernel file locks, explicitly configure: + ```java + tusService.withUploadLockingService(new DiskLockingService(storagePath)); + ``` +- Existing uploads on disk remain readable: the stored `UploadInfo` format is unchanged (the `serialVersionUID` is preserved). + +## 3. Jackson Dependencies Bundled in Compile Scope +- Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are now included with compile scope in `tus-java-server`. You no longer need to declare Jackson separately in your application `pom.xml` when using JSON metadata serialization (`withJsonSerialization()`) or cloud storage. + +## 4. Safe Lock-Holding InputStream Lifecycle +- In 2.0.0, `tusService.getUploadedBytes(...)` returns a stream that retains an exclusive upload lock until closed. This prevents concurrent `DELETE` or `PATCH` requests from corrupting data mid-stream. +- **Best Practice**: Always consume uploaded bytes within a `try-with-resources` block: + ```java + try (InputStream stream = tusService.getUploadedBytes(uploadUri)) { + // Process file data safely under lock protection + } // Lock is automatically released on close + ``` +- A stream that is never closed leaves the upload locked until the lease expires. + +## 5. `process()` Return Value +`TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track upload IDs. Note that it returns a value for both creation and progress requests, see [Creation vs. progress ambiguity](INTEGRATION_GUIDE.md#creation-vs-progress-ambiguity). + +## 6. New Features You May Want to Adopt +- **Pluggable Cloud Storage Backends** + - **S3 Storage**: `S3StorageService`, `S3LockingService`, and `S3ConcatenationService` (AWS S3, MinIO, RustFS, Cloudflare R2, Ceph, GCS). See [S3_STORAGE.md](S3_STORAGE.md). + - **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService`, and `AzureBlobConcatenationService`. See [AZURE_BLOB_STORAGE.md](AZURE_BLOB_STORAGE.md). + - **Thread-Local Caching**: Enable `.withThreadLocalCache(true)` when using cloud backends to eliminate redundant remote metadata roundtrips per HTTP request. When enabled, `tusService.getUploadStorageService()` returns a `ThreadLocalCachedStorageAndLockingService` wrapper; inject your concrete storage service bean directly rather than casting the wrapper if you need backend-specific methods like `getS3ObjectKey()` or `getAzureBlobName()`. +- **Post-Upload Listeners (`UploadCompletionListener`)**: register callbacks that fire immediately upon upload completion: + ```java + tusService.withUploadCompletionListener((uploadInfo, service) -> { + log.info("Upload completed: {}", uploadInfo.getId()); + }); + ``` +- **`withLockWaitTimeout(Duration)`**: configures how long requests wait for a contended upload lock (default 60 s). See [LOCKING.md](LOCKING.md). +- **`withJsonSerialization()`**: store `UploadInfo` as JSON instead of Java serialization. +- **Absolute upload URIs**: `withUploadUri("https://upload.example.com/files")` returns full `Location` URLs; relative paths keep working. +- **Automatic lock cleanup** on completion, termination and expiration (no action needed). + +## 7. Download Extension Behaviour Change +To support both protocols, the unofficial `download` extension no longer returns HTTP `204` for uploads that are still in progress and no longer includes the `Tus-Resumable` response header. The `UploadInProgressException` class was removed. + +## 8. Dual Protocol Support +By default `ProtocolVersion.AUTO` detects Tus 1.0.0 vs. IETF RUFH per request, so existing Tus clients keep working unchanged. Restrict it with `withSupportedProtocolVersions(...)`, see [PROTOCOLS.md](PROTOCOLS.md). + +## 9. Security Enhancements +- **Path Traversal Protection**: Request URIs and upload IDs are strictly validated to prevent directory traversal attacks. +- **CORS Origin Validation**: Prevents arbitrary Origin reflection when CORS is enabled. +- **Secure File Permissions**: Temporary files are created with restricted owner-only permissions (`0600`) via `Files.createTempFile` instead of legacy world-readable temp files. diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 00000000..dbc900a5 --- /dev/null +++ b/lychee.toml @@ -0,0 +1,16 @@ +# Lychee link checker configuration, see https://lychee.cli.rs/ +# Also verify that #anchors exist in the linked Markdown files +include_fragments = true + +# Treat rate limiting as success; retry transient failures +accept = ["200..=299", "429"] +max_retries = 2 +timeout = 20 + +# Local examples and placeholders that are not meant to resolve +exclude = [ + '^https?://localhost', + '^https?://127\.0\.0\.1', + '^https?://upload\.example\.com', + '^https?://example\.com', +] From a34a8997a6647604224f4f2f020d8ab2e1d0b100 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 11 Oct 2026 20:30:37 +0200 Subject: [PATCH 12/15] ci: fix include_fragments string enum in lychee.toml --- lychee.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lychee.toml b/lychee.toml index dbc900a5..62c1a95e 100644 --- a/lychee.toml +++ b/lychee.toml @@ -1,6 +1,6 @@ # Lychee link checker configuration, see https://lychee.cli.rs/ # Also verify that #anchors exist in the linked Markdown files -include_fragments = true +include_fragments = "full" # Treat rate limiting as success; retry transient failures accept = ["200..=299", "429"] From 4e49a925263eb232cdba8812d52dd3ff36859d3a Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 11 Oct 2026 20:33:28 +0200 Subject: [PATCH 13/15] docs: fix broken section anchor links in DEPLOYMENT.md --- docs/DEPLOYMENT.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 89589bad..4ed5bccc 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -16,11 +16,11 @@ All built-in backends are lease-based and need **no Redis or database**. Custom ## 2. Multi-Replica Checklist - Every replica must use the **same** storage and locking backend and `withUploadUri(...)`. -- Keep clocks synchronised with NTP (lease expiry depends on it). See the clock notes in [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#35-clock-synchronization--ntp-requirement) and [S3_STORAGE.md](S3_STORAGE.md#3-clock-synchronization--ntp-requirement). +- Keep clocks synchronised with NTP (lease expiry depends on it). See the clock notes in [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#35-clock-synchronization--ntp-requirement) and [S3_STORAGE.md](S3_STORAGE.md#clock-synchronization--ntp-requirement). - Requests for one upload may land on different replicas: contention between a stalled `PATCH` and a resuming `HEAD`/`DELETE` is resolved across replicas automatically, see [LOCKING.md](LOCKING.md). - Run `cleanup()` on a schedule (on one or all replicas, it is safe to run concurrently). - Call `TusFileUploadService.close()` on shutdown so locks are released. -- Cloud backends: configure bucket / container lifecycle rules for abandoned objects (see [S3 lifecycle rules](S3_STORAGE.md#8-operational-hardening-s3-lifecycle-cleanup-rules) and [Azure lifecycle rules](AZURE_BLOB_STORAGE.md#6-operational-hardening-azure-blob-lifecycle-management-rules)). +- Cloud backends: configure bucket / container lifecycle rules for abandoned objects (see [S3 lifecycle rules](S3_STORAGE.md#s3-bucket-lifecycle-rules) and [Azure lifecycle rules](AZURE_BLOB_STORAGE.md#11-operational-hardening--cost-optimization-guidance)). ## 3. Reverse Proxies & Load Balancers From fae60b7d32b84398d79bc38638357414409ab715 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 11 Oct 2026 20:42:04 +0200 Subject: [PATCH 14/15] ci: add lychee markdown link check pre-commit hook --- .pre-commit-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7bdc277a..acc734d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,3 +17,10 @@ repos: language: system files: \.java$ pass_filenames: false + - repo: https://github.com/lycheeverse/lychee + rev: lychee-v0.24.2 + hooks: + - id: lychee + name: Lychee Markdown Link Check + args: ["--config", "lychee.toml", "--offline", "--no-progress"] + types: [markdown] From 4b0d73bfb82350fbdb17305d6221c9f152d0b684 Mon Sep 17 00:00:00 2001 From: Tom Desair Date: Sun, 11 Oct 2026 21:05:58 +0200 Subject: [PATCH 15/15] fix(s3): remove unused storeIncompletePartToS3 and cover mid-stream max parts limit --- .../server/upload/s3/S3StorageService.java | 45 +++++--------- .../upload/s3/S3StorageServiceTest.java | 62 ++++++++++++++++++- 2 files changed, 74 insertions(+), 33 deletions(-) diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index 9c8bb0ca..1d0f50e7 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -461,6 +461,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) info.setOffset(newOffset); upload.setOffset(newOffset); info.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); + upload.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); update(info); // Safe transactional cleanup: Delete old tail parts only AFTER the new manifest is saved. @@ -1151,29 +1152,26 @@ private AppendResult processPayloadChunks( } // Validate remaining part capacity before allocating next S3 part number - if (allPartKeys.size() + plannedPartKeys.size() >= MAX_PARTS_PER_UPLOAD - || nextPartNumber > MAX_PARTS_PER_UPLOAD) { + if (allPartKeys.size() + plannedPartKeys.size() >= MAX_PARTS_PER_UPLOAD) { maxAppendSizeException = new MaxAppendSizeExceededException( "Upload has reached the maximum allowed S3 limit of " + MAX_PARTS_PER_UPLOAD + " parts."); - if (chunkBytesWritten > 0) { - int confirmedCount = uploader.drainAndComplete(); - int confirmed = Math.min(confirmedCount, plannedPartKeys.size()); - for (int i = 0; i < confirmed; i++) { - allPartKeys.add(plannedPartKeys.get(i)); - confirmedBytesAppended += plannedChunkSizes.get(i); - } - plannedPartKeys.clear(); - plannedChunkSizes.clear(); - - String chunkKey = buildChunkPartKey(id, nextPartNumber++); - uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); - allPartKeys.add(chunkKey); - confirmedBytesAppended += chunkBytesWritten; - handedOff = true; + int confirmedCount = uploader.drainAndComplete(); + int confirmed = Math.min(confirmedCount, plannedPartKeys.size()); + for (int i = 0; i < confirmed; i++) { + allPartKeys.add(plannedPartKeys.get(i)); + confirmedBytesAppended += plannedChunkSizes.get(i); } + plannedPartKeys.clear(); + plannedChunkSizes.clear(); + + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); + allPartKeys.add(chunkKey); + confirmedBytesAppended += chunkBytesWritten; + handedOff = true; streamFinished = true; break; } @@ -1274,19 +1272,6 @@ private void uploadChunkToS3(String chunkKey, File tempChunkFile, long chunkLeng } } - private void storeIncompletePartToS3(String partObjectKey, File tempChunkFile, long chunkLength) - throws IOException { - try (FileInputStream fis = new FileInputStream(tempChunkFile)) { - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(partObjectKey).stream(fis, chunkLength, -1L) - .build()); - } catch (Exception e) { - throw new IOException("Failed to write incomplete part object to S3 key " + partObjectKey, e); - } finally { - FileUtils.deleteQuietly(tempChunkFile); - } - } - /** * When all expected bytes have been received, this method combines all part chunk objects in S3 * into the final destination object key using MinIO's {@code composeObject} API. diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 2a663ae1..b2a5b259 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -2842,8 +2842,8 @@ public void testGetUploadedBytesStreamFailureThrowsUploadNotFoundException() thr } @Test(expected = MaxAppendSizeExceededException.class) - public void testAppendWithMaxPartsReachedFlushesChunkAndThrows() throws Exception { - UploadId uploadId = new UploadId("max-parts-id"); + public void testAppendWhenInitialPartsBudgetExceededThrows() throws Exception { + UploadId uploadId = new UploadId("max-parts-init-id"); UploadInfo info = new UploadInfo(); info.setId(uploadId); info.setLength(1000000L); @@ -2853,12 +2853,68 @@ public void testAppendWithMaxPartsReachedFlushesChunkAndThrows() throws Exceptio when(minioClient.getObject( argThat( - (GetObjectArgs a) -> a != null && "metadata/max-parts-id.info".equals(a.object())))) + (GetObjectArgs a) -> + a != null && "metadata/max-parts-init-id.info".equals(a.object())))) .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); storageService.append(info, new ByteArrayInputStream(new byte[50])); } + @Test + public void testAppendWithMaxPartsReachedFlushesChunkAndThrows() throws Exception { + storageService.setPreferredPartSize(5 * 1024 * 1024L); + UploadId uploadId = new UploadId("max-parts-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(20_000_000L); + info.setOffset(0L); + List existingKeys = + new ArrayList<>(Collections.nCopies(9999, "uploads/max-parts-id.part.0000001")); + info.setUploadPartKeys(existingKeys); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> a != null && "metadata/max-parts-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + StatObjectResponse stat5Mb = mock(StatObjectResponse.class); + when(stat5Mb.size()).thenReturn(5L * 1024 * 1024); + when(minioClient.statObject(any(StatObjectArgs.class))).thenReturn(stat5Mb); + + // Provide 5MB (which fills part 10,000) followed by 100 bytes (which hits the 10,000 max parts + // limit) + InputStream stream = + new InputStream() { + private long remaining = 5L * 1024 * 1024 + 100; + + @Override + public int read() { + return remaining-- > 0 ? 0 : -1; + } + + @Override + public int read(byte[] b, int off, int len) { + if (remaining <= 0) { + return -1; + } + int toRead = (int) Math.min(len, remaining); + Arrays.fill(b, off, off + toRead, (byte) 0); + remaining -= toRead; + return toRead; + } + }; + + try { + storageService.append(info, stream); + fail("Expected MaxAppendSizeExceededException when exceeding 10,000 parts limit"); + } catch (MaxAppendSizeExceededException e) { + assertTrue(e.getMessage().contains("10000 parts")); + // Verify that the trailing chunk was flushed and the offset was updated before throwing + assertEquals(10001, info.getUploadPartKeys().size()); + assertEquals(10000L * 5 * 1024 * 1024L + 100L, info.getOffset().longValue()); + } + } + @Test public void testAppendDrainExceptionRecordedAndThrowsIOException() throws Exception { UploadId uploadId = new UploadId("drain-err-id");