diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml new file mode 100644 index 00000000..0d2344a4 --- /dev/null +++ b/.github/workflows/docs-link-check.yml @@ -0,0 +1,31 @@ +# Checks that links and anchors in the Markdown documentation are not broken. +# Pull requests only verify local files/anchors (offline, fast and never flaky); +# the weekly run also checks external URLs. + +name: Documentation Link Check + +permissions: + contents: read + +on: + pull_request: + paths: ["**.md", ".github/workflows/docs-link-check.yml", "lychee.toml"] + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +jobs: + link-check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Check Markdown links + uses: lycheeverse/lychee-action@v2 + with: + # Offline (local links and #anchors only) on pull requests, full check otherwise + args: >- + --config lychee.toml + ${{ github.event_name == 'pull_request' && '--offline' || '' }} + README.md CONTRIBUTING.md CHANGELOG.md 'docs/**/*.md' + fail: true diff --git a/.gitignore b/.gitignore index 5a5a0675..5ffdb6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,4 @@ SFTP_STORAGE_ANALYSIS.md NFS_LOCKING.md RELEASE-REVIEW.md ASYNC_REVIEW.md +ASYNC_REVIEW_2.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7bdc277a..acc734d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,3 +17,10 @@ repos: language: system files: \.java$ pass_filenames: false + - repo: https://github.com/lycheeverse/lychee + rev: lychee-v0.24.2 + hooks: + - id: lychee + name: Lychee Markdown Link Check + args: ["--config", "lychee.toml", "--offline", "--no-progress"] + types: [markdown] diff --git a/AGENTS.md b/AGENTS.md index 6ff7fb7d..092694c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # Agent Instructions ## Project Context -When working on this project, always read the [`README.md`](README.md) file to obtain full context on project architecture, features, configuration options, and dual protocol version support (Tus 1.0.0 & IETF RUFH). +When working on this project, always read the [`README.md`](README.md) and [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) files to obtain full context on project architecture, extension pipelines, features, configuration options, and dual protocol version support (Tus 1.0.0 & IETF RUFH). ## GitHub CLI (`gh`) & Git Usage - Always run `git` commands (e.g., `git status`, `git diff`, `git add`, `git commit`, `git push`) and `gh` commands unsandboxed (setting `BypassSandbox: true` when calling `run_command`) to ensure git hooks, local tools, and remote repository authentication work without sandbox errors. @@ -23,7 +23,7 @@ Any new feature, bugfix, or improvement must be developed in a separate branch t When a new feature is introduced, the `README.md` file must be updated with information on this new feature (e.g. configuration, usage). ### CHANGELOG.md -For any new feature, big improvements, or fixes, the `CHANGELOG.md` file must be updated to describe the changes added in this version. Use a release version header (e.g., `## [1.0.0-3.2]`) instead of `## [Unreleased]`. Derive this next release version from the SNAPSHOT version declared in the `pom.xml` file by removing the `-SNAPSHOT` suffix. Make sure to not add duplicate headers. +For any new feature, big improvements, or fixes, the `CHANGELOG.md` file must be updated to describe the changes added in this version. Use a release version header (e.g., `## [2.0.0]`) instead of `## [Unreleased]`. Derive this next release version from the SNAPSHOT version declared in the `pom.xml` file by removing the `-SNAPSHOT` suffix. Make sure to not add duplicate headers. ### Release Process When performing a release, please strictly follow the instructions outlined in the [docs/RELEASE.md](docs/RELEASE.md) documentation file. @@ -35,8 +35,9 @@ When performing a release, please strictly follow the instructions outlined in t - **Jakarta EE / Servlets**: Always use `jakarta.servlet.*` package imports instead of the legacy `javax.servlet.*` packages. ### 2. Extension Architecture & Protocol Applicability -- Every protocol extension MUST extend `AbstractTusExtension` and declare its applicability via `isApplicable(HttpMethod, ProtocolVersion)`. -- `TusFileUploadService` MUST NOT contain protocol-specific conditionals, version branching, or hardcoded error handling; all protocol-specific validation and execution logic belongs inside dedicated `RequestValidator` and `RequestHandler` implementations. +- Read the definitive architectural design in [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) before implementing new protocol logic. +- `TusFileUploadService` is the single facade service class orchestrating protocols composed of modular extensions. It MUST NOT contain protocol-specific conditionals, version branching, or hardcoded error handling. +- Every protocol extension MUST extend `AbstractTusExtension` and declare its applicability via `isApplicable(HttpMethod, ProtocolVersion)`. All protocol-specific validation and execution logic belongs inside dedicated `RequestValidator` and `RequestHandler` implementations. ### 3. Explicit Parameter Passing & No Request Attributes - Do NOT use magic string servlet request attributes (such as `"me.desair.tus.uploadLockingService"` or `"me.desair.tus.protocolVersion"`). @@ -111,6 +112,7 @@ Whenever a new setter or configuration property (such as `setMinAppendSize`, `se - A corresponding `with...` builder method (e.g. `withMinAppendSize`, `withMinSize`) MUST be added to `TusFileUploadService` with thorough Javadoc comments. - `TusFileUploadService.withUploadStorageService(...)` MUST be updated to copy the setting from the old `UploadStorageService` instance to the new one. - `ThreadLocalCachedStorageAndLockingService` MUST delegate the setter and getter methods to `storageServiceDelegate`. +- **ThreadLocalCache Wrapper Safety**: When `.withThreadLocalCache(true)` is enabled, `getUploadStorageService()` returns `ThreadLocalCachedStorageAndLockingService` (a wrapper), not concrete classes like `S3StorageService` or `AzureBlobStorageService`. Never cast `getUploadStorageService()` directly in downstream code; always inject or retain the concrete storage service bean directly. ### 15. Typed Exceptions & HttpServletResponse Status Codes - Do NOT throw generic `TusException` directly when throwing protocol errors or request validation failures. @@ -147,6 +149,7 @@ To maintain a clear separation between fast, offline unit tests and containerize - Focus strictly on business processes, end-to-end user flows, protocol interactions, and backend service capability flows (such as `ITAzureBlobStorageService`, `ITAzureBlobLockingService`, `ITAzureBlobConcatenationService`, `ITAzureBlobRufhProtocol`, `ITAzureBlobTusFileUploadService`). - StorageService, LockingService, and ConcatenationService integration tests are maintained because they represent key capability flows that can be combined across different backend types (e.g. Azure storage combined with S3 locking). - Internal helper objects or component handles that do NOT represent an independent business process (such as `UploadLock` handles) MUST NOT have dedicated `IT*` integration test classes (e.g. `ITAzureBlobUploadLock` is omitted in favor of testing `AzureBlobUploadLockTest` offline and testing lock lifecycles end-to-end via `ITAzureBlobLockingService` / `ITAzureBlobTusFileUploadService`). +- **Automated Container Lifecycle (Podman & Docker)**: Integration tests automatically manage container lifecycles via Testcontainers using either **Podman** or **Docker**. Never invoke `docker` CLI directly or instruct developers to manually start containers with `docker run` or `podman run`. - **Naming & Execution Rules**: - Unit test classes MUST end with `Test.java` (executed during `mvn test`). - Integration test classes MUST start with `IT` and MUST NOT end with `Test` or `Test.java` (executed during `mvn verify`). @@ -180,6 +183,13 @@ To maintain a clear separation between fast, offline unit tests and containerize - Do NOT use legacy `java.io.File.createTempFile(...)`. On POSIX systems, it creates files with overly permissive default umask permissions (often world-readable or group-readable, CWE-378 / SonarQube java:S5443). - Always use `java.nio.file.Files.createTempFile(...)` (or `Utils.createTempSiblingPath(...)` / `Utils.createTempSibling(...)`), which creates temporary files with restricted owner-only permissions (`rw-------` / `0600`) by default. +### 23. Thread Safety: Prevent Volatile Class Variables & Use ReentrantReadWriteLock +- Do NOT use `volatile` class variables for mutable state, lazy compilation, or feature flags. `volatile` references on complex objects or mutable fields can trigger static analysis warnings (e.g. SonarQube java:S3077) and cannot guard multi-step compound actions, non-atomic transitions, or lazy initializations. +- Guard concurrent mutable state using a `ReadWriteLock` (`ReentrantReadWriteLock`). +- Consistently name the lock field `private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock();` in every class for pattern consistency across the codebase. +- Use `thisObjectLock.readLock()` for fast, concurrent reads, and `thisObjectLock.writeLock()` for state mutations, lazy pattern compilation, or feature capability degradation. +- Always acquire and release locks using `try ... finally` blocks (e.g. `thisObjectLock.readLock().lock(); try { return field; } finally { thisObjectLock.readLock().unlock(); }`). + ## IETF Resumable Uploads for HTTP (RUFH) Spec Maintenance & Update Playbook @@ -211,6 +221,7 @@ When updating the IETF protocol implementation for a new draft revision, follow ### 4. Conformity Test Suite Maintenance & Subagent Isolation Whenever a new draft revision of the RUFH specification is published, the repository's Python conformity test suite (`scripts/rufh_conformity_test.py`) MUST be reviewed and updated by a separate, dedicated subagent. - **Strict Isolation Rule**: The subagent tasked with updating `scripts/rufh_conformity_test.py` MUST ONLY consult the official IETF specification document (and RFC 9530) and MUST NOT inspect the Java server implementation code under `src/main/java/`. This ensures the conformity test suite remains an independent, unbiased specification benchmark. +- **Default Multi-Backend Execution**: When executing the conformity test suite (`scripts/rufh_conformity_test.py`), tests MUST be run by default against all three supported storage backend types (Disk: `/test/api/upload`, S3: `/test-s3/api/upload`, and Azure Blob: `/test-azure/api/upload`) as documented in [`docs/CONFORMITY_TESTING.md`](docs/CONFORMITY_TESTING.md). ### 5. Conformity Test Suite Audit — Repeatable Procedure Use this procedure to audit `scripts/rufh_conformity_test.py` against the current (or a new) specification revision. The goal is to identify untested MUST/SHOULD/MAY requirements and produce an actionable improvement report. diff --git a/CHANGELOG.md b/CHANGELOG.md index f7a9c6cd..d063cf3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,36 +4,44 @@ All notable changes to this project will be documented in this file. ## [2.0.0] -### New +Major release adding the IETF Resumable Uploads for HTTP (RUFH) protocol and cloud storage backends. **Upgrading from `1.0.0-3.x`? Read the [Upgrade Guide](docs/UPGRADING.md).** -- **NFS- & SMB-Safe Lease Locking (`LeaseFileLockingService` & `LeaseFileMutex`)**: Added distributed, container-safe filesystem locking using atomic sibling mutex directories (`.mutex/`), in-place expired lock takeover, and TTL-based JSON lease files with background heartbeat renewal and ownership fencing. Operates reliably across multi-server replicas on NFS (v3/v4), AWS EFS, Azure Files, Windows SMB/CIFS, and local disks without requiring Redis, ZooKeeper, or OS-level `FileLock` daemons. Comprehensive guide and legacy opt-out instructions available in `docs/DISK_BASED_LOCKING.md`. -- **S3-Compatible Storage & Distributed Locking**: Added native S3 storage support via `S3StorageService` (MinIO SDK), distributed locking via `S3LockingService` (S3 conditional writes with TTL leases and interrupt signals for multi-replica container deployments), S3-native concatenation via `S3ConcatenationService`, and complete documentation in `docs/S3_STORAGE.md`. -- **Azure Blob Storage & Distributed Leases**: Added native Azure Blob Storage support via `AzureBlobStorageService` (Block Blob staging with streaming appends, sub-threshold buffering, truncation, and deduplication), distributed locking via `AzureBlobLockingService` (Azure Blob Leases with auto-renewal, JVM interruption, cross-replica `.stop` signals, and clean shutdown), zero-copy server-side concatenation via `AzureBlobConcatenationService` (`stageBlockFromUrl`), and comprehensive documentation in `docs/AZURE_BLOB_STORAGE.md`. -- **IETF Resumable Uploads for HTTP (RUFH) Protocol**: Implemented full support for the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`). - - **Dual Protocol Auto-Detection**: Added transparent protocol routing in `TusFileUploadService` supporting both legacy `TUS_1_0_0` (`Tus-Resumable: 1.0.0`) and `RUFH` (`ProtocolVersion.RUFH`) clients concurrently on the same endpoint. - - **RFC 9651 Structured Header Fields**: Implemented RFC 9651 parsing and serialization for `Upload-Offset`, `Upload-Complete`, `Upload-Length`, and `Upload-Limit` dictionary headers. - - **RFC 7807 Problem Details JSON**: Added support for standard `application/problem+json` error responses (`mismatching-upload-offset`, `completed-upload`, `inconsistent-upload-length`). - - **Dedicated Compliance Test Suites**: Added comprehensive, spec-quoted end-to-end tests using a dedicated Python script `scripts/rufh_conformity_test.py` with documentation on how to run the tests in `docs/CONFORMITY_TESTING.md`. - - **User Migration & Interim Responses Documentation**: Added `docs/MIGRATION.md` and `docs/INTERIM_RESPONSES.md` detailing migration strategies, HTTP 104 status frames under IETF RUFH, Tomcat/Servlet container limitations, cached reflection optimizations, and Spring Boot Tomcat Valve integration. -- **Server-Side Upload Completion Listeners (`UploadCompletionListener`)**: Added a functional interface callback mechanism allowing developers to register post-upload listeners via `withUploadCompletionListener(UploadCompletionListener)` or `addUploadCompletionListener(UploadCompletionListener)`. Listeners receive the completed `UploadInfo` and `TusFileUploadService` instance after lock release, allowing immediate byte streaming and deletion without contention. Added helper overloads `TusFileUploadService.getUploadedBytes(UploadInfo)` and `TusFileUploadService.deleteUpload(UploadInfo)`. -- **JSON Serialization**: Support storing `UploadInfo` objects as JSON files in the storage backend using `TusFileUploadService.withJsonSerialization(true)`. +### Highlights +- IETF Resumable Uploads for HTTP (`draft-ietf-httpbis-resumable-upload-12`) served next to Tus 1.0.0 on the same endpoint. +- S3-compatible and Azure Blob storage backends with distributed, database-free locking. +- NFS/SMB-safe lease locking is now the default for disk storage. +- Server-side `UploadCompletionListener` and a `process()` method returning the `UploadInfo`. -### Changed -- **Default Disk-Based Locking**: `TusFileUploadService.withStoragePath(String)` now defaults to `LeaseFileLockingService` instead of `DiskLockingService` for out-of-the-box Kubernetes, container, and shared network storage compatibility. See `docs/DISK_BASED_LOCKING.md` for legacy opt-out instructions. -- **Calibrated Retry Budget**: Extended `TusFileUploadService` lock acquisition retry budget to 8.0 seconds (40 retries x 200ms) to ensure reliable contention resolution over network storage. -- **Absolute Base URL & Location Header Support**: Extended `withUploadUri(String)` to accept absolute base URLs (e.g. `https://upload.example.com/files`), returning full URLs in `Location` response headers for upload creation across both Tus 1.0.0 and RUFH protocols while preserving backward compatibility for relative paths. -- **`process()` Return Value (`UploadInfo`)**: `TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` instance (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track and store upload IDs directly into user sessions or database repositories. -- **Jackson Bundled in Compile Scope**: Promoted Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) to `compile` scope, eliminating `NoClassDefFoundError` when enabling JSON serialization or using cloud storage. -- **Lock-Holding Stream Lifecycle**: `TusFileUploadService.getUploadedBytes(...)` now retains the upload lock until the returned stream is closed, preventing concurrent modifications or deletions from corrupting data mid-stream. +### Upgrade notes (breaking & behavioural changes) +See [docs/UPGRADING.md](docs/UPGRADING.md) for details and code examples. +- **Java 17 and Jakarta EE**: requires Java 17+ and `jakarta.servlet.*` (Tomcat 10+, Jetty 11+, Spring Boot 3+). `javax.servlet` is no longer supported. +- **Default locking**: `withStoragePath(String)` now uses `LeaseFileLockingService` instead of `DiskLockingService`. Opt out with `withUploadLockingService(new DiskLockingService(path))`, see [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). +- **Lock-holding streams**: `getUploadedBytes(...)` keeps the upload locked until the returned stream is closed. Always use try-with-resources. +- **`process(...)` return value**: now returns the created/updated `UploadInfo` on successful `POST`, `PATCH`, and `HEAD` requests (or `null` on errors, `OPTIONS`, or `DELETE` termination). It also returns a value for progress requests, so check the HTTP method when recording a newly created upload. +- **Download extension (breaking)**: no longer returns `204` for uploads still in progress and no longer sets the `Tus-Resumable` header (needed for dual-protocol support). The `UploadInProgressException` class was removed. +- **Jackson bundled**: `jackson-databind`, `jackson-annotations` and `jackson-core` are now compile-scope dependencies; no separate declaration needed. +- **Protocol detection**: `ProtocolVersion.AUTO` is the default. Restrict with `withSupportedProtocolVersions(...)`, see [docs/PROTOCOLS.md](docs/PROTOCOLS.md). + +### New +- **IETF Resumable Uploads for HTTP (RUFH)**: full draft-12 support with automatic protocol detection, RFC 9651 structured headers (`Upload-Offset`, `Upload-Complete`, `Upload-Length`, `Upload-Limit`), RFC 7807 `application/problem+json` errors, RFC 9530 HTTP digests, `104` interim responses and a spec-quoted conformity suite (`scripts/rufh_conformity_test.py`). See [docs/PROTOCOLS.md](docs/PROTOCOLS.md), [docs/MIGRATION.md](docs/MIGRATION.md), [docs/INTERIM_RESPONSES.md](docs/INTERIM_RESPONSES.md) and [docs/CONFORMITY_TESTING.md](docs/CONFORMITY_TESTING.md). +- **NFS- & SMB-safe lease locking (`LeaseFileLockingService`)**: atomic sibling mutex directories, TTL lease files with heartbeat renewal and ownership fencing; safe on NFS v3/v4, AWS EFS, Azure Files, SMB/CIFS and local disks, with no Redis, ZooKeeper or `FileLock` daemons. See [docs/DISK_BASED_LOCKING.md](docs/DISK_BASED_LOCKING.md). +- **S3-compatible storage**: `S3StorageService`, `S3LockingService` (conditional writes, TTL leases, automatic non-CAS fallback for e.g. Backblaze B2 and Ceph RGW, configurable jitter via `withJitter(minMs, maxMs)` and `withS3ConditionalWritesSupported(boolean)`) and `S3ConcatenationService`, including server-side multipart composition that works on AWS S3 and MinIO/Ceph. See [docs/S3_STORAGE.md](docs/S3_STORAGE.md). +- **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService` (Blob Leases with auto-renewal and cross-replica stop signals) and zero-copy `AzureBlobConcatenationService`. See [docs/AZURE_BLOB_STORAGE.md](docs/AZURE_BLOB_STORAGE.md). +- **`UploadCompletionListener`**: register via `withUploadCompletionListener(...)` / `addUploadCompletionListener(...)`. Listeners run after the lock is released, so they can stream and delete the bytes immediately. Added `getUploadedBytes(UploadInfo)` and `deleteUpload(UploadInfo)` overloads. +- **`withLockWaitTimeout(Duration)`** (default 60 s): maximum time a request waits for a contended upload lock; also derives the cloud chunk drain timeout (`lockWaitTimeout - 5 s`) and retry budget. See [docs/LOCKING.md](docs/LOCKING.md). +- **Automatic lock cleanup** on completion, termination (`DELETE`) and expiration across all locking services (`cleanupLock(...)`), plus a fallback sweeper for abandoned locks. +- **JSON serialization**: store `UploadInfo` as JSON with `withJsonSerialization(true)`. +- **Absolute base URLs**: `withUploadUri(String)` accepts e.g. `https://upload.example.com/files`, returning full `Location` URLs for Tus and RUFH; relative paths keep working. ### Fixed -- **RFC 9110 Media Type Matching & MIME Parameter Tolerance**: Enhanced `Content-Type` header validation in `ContentTypeValidator`, `PostContentTypeValidator`, and `RufhAppendValidator` using RFC 9110 §8.3 compliant media-type parsing (`Utils.isMediaType`). Media type matching now tolerates MIME parameters (such as `;charset=UTF-8` automatically appended by Spring Boot `CharacterEncodingFilter`, servlet wrappers, proxies, or HTTP clients), whitespace variations, and case-insensitivity without incorrectly rejecting valid requests with `406 Not Acceptable`. -- **Clear Content-Length on Error Responses**: Cleared `Content-Length` response header prior to invoking `HttpServletResponse.sendError(...)` during exception handling, resolving buffer conflicts and exceptions in Undertow and other servlet containers ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). -- **Prevent Disk Truncate Underflow**: Guarded file truncate logic in `DiskStorageService` against underflow when removing bytes (`Math.max(0L, file.size() - byteCount)`). -- **File Channel Leak Prevention in FileBasedLock**: Guaranteed `FileChannel` is closed immediately upon lock acquisition errors to avoid file descriptor leaks. +- **Content-Type matching**: media types are matched per RFC 9110 and tolerate parameters such as `;charset=UTF-8` (added by Spring `CharacterEncodingFilter`, proxies or clients) instead of returning `406`. +- **Undertow & other containers**: `Content-Length` is cleared before `sendError(...)` ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). +- **Disk**: truncate underflow guard in `DiskStorageService`; `FileChannel` leak on lock acquisition errors fixed in `FileBasedLock`. -### Breaking -- **Downloads**: In order to support both the Tus protocol and RUFH protocol, the unofficial download extension will not return a HTTP status code `204` for uploads that are still in progress and will not contain the response header `Tus-Resumable`. Removed the `UploadInProgressException` class. +### Security +- **Path traversal protection**: hardened `LeaseFileLockingService` against directory traversal via crafted upload identifiers ([#141](https://github.com/tomdesair/tus-java-server/pull/141)). +- **CORS origin validation**: prevented arbitrary `Origin` header reflection in CORS preflight responses ([#132](https://github.com/tomdesair/tus-java-server/pull/132)). +- **Secure temporary files**: replaced legacy `File.createTempFile` with `Files.createTempFile` enforcing owner-only POSIX permissions (`0600`, CWE-378 / SonarQube java:S5443) ([#131](https://github.com/tomdesair/tus-java-server/pull/131)). ## [1.0.0-3.3] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..13ae1b11 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,64 @@ +# Contributing + +This library comes without any warranty and is released under a [MIT license](https://github.com/tomdesair/tus-java-server/blob/master/LICENSE). If you encounter any bugs or if you have an idea for a useful improvement you are welcome to [open a new issue](https://github.com/tomdesair/tus-java-server/issues) or to [create a pull request](https://github.com/tomdesair/tus-java-server/pulls) with the proposed implementation. Please note that any contributed code needs to be accompanied by automated unit and/or integration tests and comply with the [defined code-style](#code-style). + +Useful guides: +- [Architecture guide](docs/ARCHITECTURE.md) +- [Testing guide](docs/TESTING.md) and coverage checks +- [Protocol conformity testing](docs/CONFORMITY_TESTING.md) +- [Release process](docs/RELEASE.md) + +## System Architecture & Where to Contribute + +Before proposing design changes or implementing new features, please read the **[Architecture Guide](docs/ARCHITECTURE.md)**. It provides a complete overview of: +- The single facade service class (`TusFileUploadService`) and the dual-protocol extension pipeline. +- How core business logic is encapsulated in `RequestValidator` and `RequestHandler` implementations. +- The pluggable storage architecture (including both lease-based and legacy file-lock-based disk implementations, S3, and Azure Blob). +- The `AbstractLeaseLockingService` framework for distributed lease-based locking and lock contention handling. +- The contributor blueprint mapping common extension scenarios to the right packages and classes. + +## Branching Strategy + +Any new feature, bugfix, or improvement should be developed in a separate branch that starts with either `feature/` or `bugfix/` and has a meaningful, concise name (e.g., `feature/lock-contention-resolution` or `bugfix/fix-upload-timeout`). + +## Code Style + +All pull requests should have the correct formatting according to [Google Java Style](https://github.com/google/google-java-format) code formatting. To verify if the code style is correct run: + +```bash +mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:check +``` + +To reformat your code run: + +```bash +mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:format +``` + +See the [Google Java Style Github page](https://github.com/google/google-java-format) on recommendations on how to configure this in your IDE. Or if you have Python 3, you can also use [pre-commit](https://pre-commit.com) to make your life easier: + +```bash +pip install pre-commit +pre-commit install +``` + +## Running Tests & Verifying Coverage + +### 1. Fast Offline Unit Tests +Unit tests run in isolation without external dependencies or containers: +```bash +mvn test -q +``` + +### 2. Integration Tests (Testcontainers) +Integration tests verify end-to-end storage, distributed locking, and protocol flows against local Testcontainers instances (RustFS for S3, Azurite for Azure Blob): +```bash +mvn verify -q +``` +> **Container Runtime**: Integration tests automatically spin up and manage container lifecycles via Testcontainers using either **Podman** or **Docker**. No manual container startup is required. If no container runtime is running, integration tests are gracefully skipped. + +### 3. Code Coverage Gate +Verify that modified code meets the required unit test coverage threshold (default: 90% per file): +```bash +python3 scripts/check-coverage.py --per-file-limit 90 +``` diff --git a/README.md b/README.md index bc082a23..02886496 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,32 @@ [![Maven Central](https://img.shields.io/maven-central/v/me.desair.tus/tus-java-server.svg)](https://central.sonatype.com/artifact/me.desair.tus/tus-java-server) [![Java 17+](https://img.shields.io/badge/Java-17%2B-blue.svg)](https://adoptium.net) [![Build and Tests](https://github.com/tomdesair/tus-java-server/actions/workflows/build.yml/badge.svg)](https://github.com/tomdesair/tus-java-server/actions?query=branch%3Amaster+) [![Coverage Status](https://coveralls.io/repos/github/tomdesair/tus-java-server/badge.svg?branch=master)](https://coveralls.io/github/tomdesair/tus-java-server?branch=master) [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=me.desair.tus%3Atus-java-server&metric=bugs)](https://sonarcloud.io/dashboard?id=me.desair.tus%3Atus-java-server) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=me.desair.tus%3Atus-java-server&metric=vulnerabilities)](https://sonarcloud.io/dashboard?id=me.desair.tus%3Atus-java-server) [![Duplicated Lines](https://sonarcloud.io/api/project_badges/measure?project=me.desair.tus%3Atus-java-server&metric=duplicated_lines_density)](https://sonarcloud.io/dashboard?id=me.desair.tus%3Atus-java-server) # tus-java-server -This library can be used to enable resumable (and potentially asynchronous) file uploads in any Java web application. This allows the users of your application to upload large files over slow and unreliable internet connections. The ability to pause or resume a file upload (after a connection loss or reset) is achieved by implementing the open file upload protocols tus (https://tus.io/) and Resumable Uploads for HTTP (RUFH). This library implements the server-side of the tus v1.0.0 protocol as well as the official IETF Resumable Uploads for HTTP specification ([draft-ietf-httpbis-resumable-upload](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/)), offering dual protocol version support. -The Javadoc of this library can be found at https://tus.desair.me/. As of version 2.0.0, this library requires Java 17+. +**Resumable (and potentially asynchronous) file uploads for any Java web application, with support for both the [tus 1.0.0](https://tus.io/) protocol and the new IETF [Resumable Uploads for HTTP (RUFH)](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/) standard.** -### Key Features -* ⚡ **Dual Protocol Support**: Seamless interoperability with [Tus 1.0.0](https://tus.io/) and the official [IETF Resumable Uploads for HTTP (draft-12)](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/). -* ☁️ **Pluggable Storage Backends**: Native support for Local Disk Volumes, NFS network shares, S3-compatible Object Storage (AWS S3, MinIO, Cloudflare R2, Ceph, GCS), and Azure Blob Storage. +This library lets the users of your application upload large files over slow and unreliable connections: an upload can be paused or resumed (after a connection loss or reset) without starting over. It implements the server side of the tus v1.0.0 protocol **and** the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload`, draft-12) on the same endpoint, so tus clients (Uppy, `tus-js-client`) and IETF RUFH clients work side by side. + +Javadoc: https://tus.desair.me/ · Requires Java 17+ (Jakarta Servlet 6.0) · Upgrading from 1.x? See the [Upgrade Guide](docs/UPGRADING.md). + +## Key Features +* ⚡ **IETF RUFH Ready**: Implements the official [IETF Resumable Uploads for HTTP (draft-12)](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/) — including `Upload-Complete`, `Upload-Limit`, RFC 7807 problem details, RFC 9530 digests and `104` interim responses — alongside full [Tus 1.0.0](https://tus.io/) support. Protocol is auto-detected per request. *(RUFH is still an IETF draft, so treat that part as experimental until it becomes an RFC.)* +* ☁️ **Pluggable Storage Backends**: Local Disk Volumes, NFS network shares, S3-compatible Object Storage (AWS S3, MinIO, Cloudflare R2, Ceph, GCS), and Azure Blob Storage. * 🔒 **Zero-Database Distributed Locking**: Built-in lease-based locking enabling multi-replica cluster & Kubernetes container deployments without requiring Redis or relational databases. * 🛡️ **Data Integrity & Resiliency**: Built-in HTTP Digests ([RFC 9530](https://www.rfc-editor.org/rfc/rfc9530.html)), checksum verification, and duplicate upload deduplication. * 🚀 **Production-Ready & Lightweight**: Minimal dependencies (Jakarta Servlet API 6.0 & Apache Commons), non-blocking lock contention resolution, and thread-local caching. ## Table of Contents -- [Storage Backend Options](#storage-backend-options) -- [How Resumable Uploads Work](#how-resumable-uploads-work) -- [Quick Start and Examples](#quick-start-and-examples) -- [Usage and Configuration](#usage-and-configuration) - - [1. Setup & Configuration Options](#1-setup) - - [2. Receiving a Resumable Upload](#2-receiving-a-resumable-upload) - - [3. Handling Upload Completion & Retrieving Files](#3-handling-upload-completion--retrieving-files) - - [4. Upload cleanup](#4-upload-cleanup) -- [Protocol Version Support (Tus 1.0.0 & IETF Resumable Uploads)](#protocol-version-support-tus-100--ietf-resumable-uploads) -- [Protocol Extensions](#protocol-extensions) -- [Advanced Usage](#advanced-usage) - - [HTTP Digests (RFC 9530)](#http-digests-rfc-9530) - - [Emitting HTTP 104 Interim Responses in Tomcat / Spring Boot](#emitting-http-104-interim-responses-in-tomcat--spring-boot) -- [Compatible Client Implementations & Conformity Testing](#compatible-client-implementations--conformity-testing) +- [Quick Start](#quick-start) +- [Try It with curl](#try-it-with-curl) +- [How It Works](#how-it-works) +- [Documentation](#documentation) +- [Compatible Clients & Conformity Testing](#compatible-clients--conformity-testing) - [Versioning](#versioning) - [Contributing](#contributing) -## Storage Backend Options - -`tus-java-server` provides pluggable storage architecture supporting multiple backend storage options: - -1. **File Disk & Network Storage** (`DiskStorageService` & `LeaseFileLockingService`): - - **Local File System**: Direct disk storage on application server instance. - - **Shared NFS Network Drives**: Distributed, container-safe lease locking for multi-server setups (NFSv3/v4, AWS EFS, Azure Files, SMB/CIFS). See [Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md). - - **Kubernetes Persistent Volume**: Mounted volume (`ReadWriteMany` / `ReadWriteOnce`) for containerized applications. -2. **S3-Compatible Object Storage** (`S3StorageService`, `S3LockingService`, & `S3ConcatenationService`): - - **Cloud & On-Premise S3**: AWS S3, MinIO, RustFS, Cloudflare R2, Ceph, or Google Cloud Storage. - - **Multi-Replica Support**: Uses distributed S3 object locking and TTL leases, enabling multi-replica container deployments without requiring Redis or external databases. See [S3 Storage Guide](docs/S3_STORAGE.md). -3. **Azure Blob Storage** (`AzureBlobStorageService`, `AzureBlobLockingService`, & `AzureBlobConcatenationService`): - - **Microsoft Azure Cloud**: Native Azure Blob Storage using the `azure-storage-blob` SDK. - - **Multi-Replica Support**: Uses native Azure Blob Leases (30s renewable leases) for distributed locking across cluster replicas. See [Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md). - -## How Resumable Uploads Work - -### Tus 1.0.0 Protocol Flow -```mermaid -sequenceDiagram - autonumber - actor Client as Frontend Client (Uppy / tus-js-client) - participant ProtocolEndpoint as Server: Tus Protocol Endpoint (/api/upload) - participant Storage as Storage Backend (Disk / S3 / Azure) - participant AppEndpoint as Application API (/api/documents) - - Note over Client,Storage: 1. Create Resumable Upload Resource - Client->>ProtocolEndpoint: POST /api/upload
(Tus-Resumable: 1.0.0, Upload-Length: 1048576, Upload-Metadata: ...) - ProtocolEndpoint->>Storage: Initialize upload metadata - ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a..., Tus-Resumable: 1.0.0) - - Note over Client,Storage: 2. Stream Data Chunk(s) - Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Content-Type: application/offset+octet-stream) - ProtocolEndpoint->>Storage: Append chunk bytes to storage - ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Tus-Resumable: 1.0.0) - - Note over Client,AppEndpoint: 3. Domain Notification & Consumption - Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") - AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) - AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) - AppEndpoint-->>Client: 200 OK (Processed) -``` - -### IETF Resumable Uploads for HTTP (RUFH) Flow -```mermaid -sequenceDiagram - autonumber - actor Client as Frontend Client (RUFH Client) - participant ProtocolEndpoint as Server: RUFH Protocol Endpoint (/api/upload) - participant Storage as Storage Backend (Disk / S3 / Azure) - participant AppEndpoint as Application API (/api/documents) - - Note over Client,Storage: 1. Create Resumable Upload Resource - Client->>ProtocolEndpoint: POST /api/upload
(Upload-Complete: ?0) - ProtocolEndpoint->>Storage: Initialize upload metadata - ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a...) - - Note over Client,Storage: 2. Stream Data Chunk(s) - Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Upload-Complete: ?1, Content-Type: application/partial-upload) - ProtocolEndpoint->>Storage: Append chunk bytes to storage - ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Upload-Complete: ?1) - - Note over Client,AppEndpoint: 3. Domain Notification & Consumption - Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") - AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) - AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) - AppEndpoint-->>Client: 200 OK (Processed) -``` +## Quick Start -## Quick Start and Examples -The tus-java-server library only depends on Jakarta Servlet API 6.0 and some Apache Commons utility libraries. This -means that (in theory) you can use this library on any modern Java Web Application server like Tomcat, JBoss, Jetty... By default all uploaded data and information is stored on a (shared) file volume of the application server. - -You can add the latest stable version of this library to your application using Maven or Gradle: +### 1. Add the dependency **Maven:** ```xml @@ -124,96 +47,50 @@ implementation 'me.desair.tus:tus-java-server:2.0.0' implementation("me.desair.tus:tus-java-server:2.0.0") ``` -Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are bundled directly with compile scope by `tus-java-server`. When using S3 storage (`S3StorageService`), simply include the MinIO Java SDK dependency matching `pom.xml`: - -```xml - - io.minio - minio - 9.0.3 - -``` - -The main entry point of the library is the `me.desair.tus.server.TusFileUploadService.process(jakarta.servlet.http.HttpServletRequest, jakarta.servlet.http.HttpServletResponse)` method. You can call this method inside a `jakarta.servlet.http.HttpServlet`, a `jakarta.servlet.Filter` or any REST API controller of a framework that gives you access to `HttpServletRequest` and `HttpServletResponse` objects. In the following list, you can find some example implementations: +S3 and Azure backends need an additional SDK dependency, see the [Configuration Reference](docs/CONFIGURATION.md#dependencies). -* [Resumable and asynchronous file upload in Spring Boot REST API with Uppy JavaScript client.](https://github.com/tomdesair/tus-java-server-spring-demo) -* [Detailed blog post by Ralph](https://golb.hplar.ch/2019/06/upload-with-tus.html) on how to use this library in [Spring Boot in combination with the Tus JavaScript client](https://github.com/ralscha/blog2019/tree/master/uploadtus). -* [Resumable and asynchronous file upload using Uppy with form submission in Dropwizard (Jetty)](https://github.com/tomdesair/tus-java-server-dropwizard-demo) -* (more examples to come!) +### 2. Create the service (once) and forward requests to it -#### Frontend Client Example (Uppy / JavaScript) -Connect any standard Tus client (e.g. [Uppy](https://uppy.io/) or `tus-js-client`) to your backend upload endpoint: +The main entry point is `me.desair.tus.server.TusFileUploadService.process(HttpServletRequest, HttpServletResponse)`. Call it from any `HttpServlet`, `Filter` or REST controller that gives you the servlet request and response. Create the service **once** (it is thread-safe) and share it. By default uploads are stored on a (shared) file volume. -```javascript -import Uppy from '@uppy/core'; -import Tus from '@uppy/tus'; +**Spring Boot 3** (`spring-boot-starter-web`, Java 17+). The snippets are complete and compile as-is; adjust the paths: -const uppy = new Uppy().use(Tus, { - endpoint: 'http://localhost:8080/api/upload', - chunkSize: 5 * 1024 * 1024 // 5MB chunk size -}); -``` - -## Usage and Configuration +```java +import me.desair.tus.server.TusFileUploadService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; -### 1. Setup -The first step is to create a `TusFileUploadService` object using its constructor. You can make this object available as a (Spring bean) singleton or create a new instance for each request. For example, in a Spring Boot application: +@Configuration +public class TusConfig { -```java -@Bean -public TusFileUploadService tusFileUploadService() { + @Bean(destroyMethod = "close") + public TusFileUploadService tusFileUploadService() { return new TusFileUploadService() .withStoragePath("/path/to/uploads") .withUploadUri("/api/upload") + .withUploadDeduplication(true) .withThreadLocalCache(true); + } } ``` -See the [tus-java-server-spring-demo](https://github.com/tomdesair/tus-java-server-spring-demo) repository for a complete Spring Boot reference implementation. - -After creating the object, you can configure it using the following methods: - -#### Configuration Options Reference - -| Method | Default | Description | -|---|---|---| -| `withUploadUri(String)` | `null` | Sets relative path (e.g. `/api/upload`) or absolute base URL (e.g. `https://upload.example.com/api/upload`) under which the upload endpoint is exposed. Supports regex parameters (e.g. `/users/[0-9]+/files/upload`). | -| `withStoragePath(String)` | `${java.io.tmpdir}/tus` | Path on the filesystem or shared drive where uploaded bytes and metadata are stored when using `DiskStorageService`. | -| `withSupportedProtocolVersions(ProtocolVersion)` | `ProtocolVersion.AUTO` | Configures protocol handling: `AUTO` (header-based auto-detection), `TUS_1_0_0` (Tus 1.0.0 only), or `RUFH` (IETF draft-12 only). | -| `withMaxUploadSize(Long)` | `Long.MAX_VALUE` | Maximum allowed total upload size in bytes per upload resource. | -| `withMaxLockRetries(int)` | `40` | Maximum lock acquisition retries during lock contention resolution (200ms sleep, resulting in an 8.0s timeout budget). | -| `withChunkedTransferDecoding(Boolean)` | `false` | Enables manual chunked HTTP decoding for servlet containers that do not decode chunked requests natively. | -| `withThreadLocalCache(Boolean)` | `false` | Enables in-memory thread-local caching of upload request data to reduce storage backend I/O load. | -| `withUploadExpirationPeriod(Long)` | `null` (disabled) | Expiration period in milliseconds after which incomplete/expired uploads become eligible for cleanup. | -| `withDownloadFeature()` | Disabled | Enables the unofficial `download` extension allowing clients to retrieve uploaded bytes via HTTP `GET`. | -| `withUploadDeduplication(Boolean)` | `false` | Enables duplicate file detection by checksum, linking new uploads (`duplicatesUploadId`) and skipping redundant storage writes. | -| `addTusExtension(TusExtension)` | All standard enabled | Adds a custom extension (e.g. application authorization checks). | -| `disableTusExtension(String)` | None | Disables a built-in extension (`creation`, `checksum`, `expiration`, `concatenation`, `termination`, `download`, `cors`). | -| `withUploadIdFactory(UploadIdFactory)` | `UuidUploadIdFactory` | Custom ID generator for upload resources (e.g., `UuidUploadIdFactory` or `TimeBasedUploadIdFactory`). | -| `withUploadCompletionListener(UploadCompletionListener)` | None | Registers a callback invoked immediately when an upload finishes transferring all bytes and is completed. | -| `withJsonSerialization()` | Java serialization | Enables JSON serialization for upload metadata (`UploadInfo`). Jackson is bundled by default. | -| `withUploadStorageService(UploadStorageService)` | `DiskStorageService` | Configures custom or cloud storage backend (`DiskStorageService`, `S3StorageService`, `AzureBlobStorageService`). | -| `withUploadLockingService(UploadLockingService)` | `LeaseFileLockingService` | Configures custom or cloud locking backend (`LeaseFileLockingService`, `S3LockingService`, `AzureBlobLockingService`). | - -The library provides filesystem-based storage (`DiskStorageService` / `LeaseFileLockingService`), S3-compatible object storage (`S3StorageService` / `S3LockingService`), and Azure Blob Storage (`AzureBlobStorageService` / `AzureBlobLockingService`). See the **[Disk & Network Storage Locking Guide](docs/DISK_BASED_LOCKING.md)**, **[S3 Storage Guide](docs/S3_STORAGE.md)**, and **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md)** for detailed instructions on multi-replica container deployments in Kubernetes, post-upload processing, and legacy locking opt-out. - -### 2. Receiving a Resumable Upload -To process an upload request you have to pass the current `jakarta.servlet.http.HttpServletRequest` and `jakarta.servlet.http.HttpServletResponse` objects to the `me.desair.tus.server.TusFileUploadService.process()` method. Typical places were you can do this are inside Servlets, Filters or REST API Controllers. - -The `process()` method returns the processed `UploadInfo` object (or `null` for `OPTIONS` preflight requests or when an error response is generated). - -For example, in a Spring MVC REST Controller: - ```java -@Controller -@CrossOrigin(origins = "*") +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@RestController public class FileUploadController { - @Autowired - private TusFileUploadService tusFileUploadService; + private final TusFileUploadService tusFileUploadService; - @Autowired - private UserSessionRepository userSessionRepository; + public FileUploadController(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } @RequestMapping( value = {"/api/upload", "/api/upload/**"}, @@ -226,218 +103,194 @@ public class FileUploadController { RequestMethod.OPTIONS, RequestMethod.GET }) - public void processUpload( - HttpServletRequest request, HttpServletResponse response, @RequestParam String userSessionId) + public void processUpload(HttpServletRequest request, HttpServletResponse response) throws IOException { - - UploadInfo info = tusFileUploadService.process(request, response, userSessionId); - - // Creation vs. Progress Ambiguity: Check if this was a new upload creation request - if (info != null && "POST".equalsIgnoreCase(request.getMethod())) { - userSessionRepository.recordUpload(userSessionId, info.getId()); - } + tusFileUploadService.process(request, response); } } ``` -> [!NOTE] -> **Creation vs. Progress Ambiguity**: The `process()` method returns an `UploadInfo` on both creation (`POST`) and progress updates (`PATCH` / `HEAD`). When associating an upload ID with a user session or database entity on creation, check `info != null && "POST".equalsIgnoreCase(request.getMethod())`. -> -> **Single-Request Upload Completion**: When clients upload the entire file in a single request using the `creation-with-upload` extension or RUFH single-request POST, the upload is already complete when `process()` returns, and any registered `UploadCompletionListener` will have already fired *before* `process()` returns to your controller. - -Optionally you can also pass a `String ownerKey` parameter to `process()`. The `ownerKey` can be used to have a hard separation between uploads of different users, groups or tenants in a multi-tenant setup. Examples of `ownerKey` values are user ID's, group names, client ID's... - -### 3. Handling Upload Completion & Retrieving Files - -You can handle upload completion in two ways: -1. **Server-Side Callback Listener (`UploadCompletionListener`)**: Register a hook directly on `TusFileUploadService` that triggers automatically when the final chunk is uploaded. -2. **Domain API Endpoint**: Have your frontend Tus client notify your application backend domain endpoint (e.g. `POST /api/documents`) once upload completes. - -#### Option A: Server-Side `UploadCompletionListener` -Register one or more completion listeners on your `TusFileUploadService`. When an upload finishes transferring all bytes, the callback executes with the completed `UploadInfo` and the service instance. The upload lock is released *before* the listener is invoked, allowing you to safely stream or delete uploaded bytes immediately: - -```java -@Bean -public TusFileUploadService tusFileUploadService() { - return new TusFileUploadService() - .withStoragePath("/path/to/uploads") - .withUploadUri("/api/upload") - .withUploadCompletionListener((uploadInfo, service) -> { - // Filename needs to be set as metadata by the client - String fileName = uploadInfo.getMetadata().get("filename"); - try (InputStream is = service.getUploadedBytes(uploadInfo)) { - Files.copy(is, Paths.get("/var/data/documents", fileName)); - } catch (Exception e) { - log.error("Failed to process completed upload {}", uploadInfo.getId(), e); - } - // Optionally delete upload temporary files after processing - service.deleteUpload(uploadInfo); - }); -} -``` - -#### Option B: Client-Initiated Domain Notification -When an upload completes, the client receives the final `204 No Content` response from the Tus/RUFH protocol endpoint (`/api/upload/...`). Because Tus and RUFH are decoupled file transport protocols, your frontend application can notify your backend domain API (e.g. `POST /api/documents`) that the upload is complete and pass along the `uploadUrl`. - -> [!NOTE] -> `POST /api/documents` represents your application's domain REST endpoint, not a protocol endpoint. The Tus server itself handles file transfer (`/api/upload`), while your application endpoint coordinates business logic, database persistence, and final file consumption. +**Plain Jakarta Servlet 6 (Tomcat 10+, Jetty 12, Undertow...)** ```java -@RestController -public class DocumentController { - - @Autowired - private TusFileUploadService tusFileUploadService; - - @PostMapping("/api/documents") - public ResponseEntity completeDocumentUpload(@RequestBody DocumentUploadRequest request) - throws IOException { - String uploadUrl = request.getUploadUrl(); +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; + +@WebServlet("/api/upload/*") +public class UploadServlet extends HttpServlet { + + private TusFileUploadService tus; + + @Override + public void init() { + tus = + new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true); + } - // 1. Retrieve upload metadata (filename, original length, custom client metadata) - UploadInfo info = tusFileUploadService.getUploadInfo(uploadUrl); - String originalFileName = info.getMetadata().get("filename"); + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Override service() (not doPost/doGet) so that PATCH, HEAD and OPTIONS reach the library + tus.process(req, resp); + } - // 2. Stream uploaded bytes to permanent storage, database, or virus scanner - try (InputStream is = tusFileUploadService.getUploadedBytes(uploadUrl)) { - Files.copy(is, Paths.get("/var/data/documents", originalFileName)); + @Override + public void destroy() { + try { + tus.close(); + } catch (IOException e) { + log("Failed to close TusFileUploadService", e); } - - // 3. Clean up the temporary upload bytes and locks - tusFileUploadService.deleteUpload(uploadUrl); - - return ResponseEntity.ok().build(); } } ``` -Using the `me.desair.tus.server.TusFileUploadService.getUploadInfo(String uploadUrl)` method you can retrieve metadata about a specific upload process. This includes metadata provided by the client as well as metadata kept by the library like creation timestamp, creator ip-address list, upload length... The method `UploadInfo.getId()` will return the unique identifier of this upload encapsulated in an `UploadId` instance. The original (custom generated) identifier object of this upload can be retrieved using `UploadId.getOriginalObject()`. A URL safe string representation of the identifier is returned by `UploadId.toString()`. It is highly recommended to consult the [JavaDoc of both classes](https://tus.desair.me/). - -#### Cloud Storage Native Processing (S3 & Azure Blob) -When using cloud object storage backends, downstream services can directly obtain the raw cloud object key or blob name to perform zero-download server-side copying, background job processing, or direct cloud SDK operations: +CORS and `OPTIONS` preflight are handled by the library itself. More variants (multi-tenancy, cleanup, security) are in the [Integration Guide](docs/INTEGRATION_GUIDE.md). -* **S3 / MinIO Storage**: Use `((S3StorageService) tusFileUploadService.getUploadStorageService()).getS3ObjectKey(uploadUrl)` to get the full S3 object key (e.g. `uploads/018f3a...`). For complete server-side `copyObject` examples and MinIO SDK usage, see the **[S3 Storage Guide](docs/S3_STORAGE.md#4-post-upload-processing-gets3objectkey)**. -* **Azure Blob Storage**: Use `((AzureBlobStorageService) tusFileUploadService.getUploadStorageService()).getAzureBlobName(uploadUrl)` to get the full blob name. For complete Azure SDK examples with `BlobClient`, see the **[Azure Blob Storage Guide](docs/AZURE_BLOB_STORAGE.md#4-post-upload-processing-getazureblobname)**. +> [!NOTE] +> When using Spring Security, the upload endpoint must be authenticated like any other endpoint, but `OPTIONS` preflight requests must be allowed through. If your Spring Boot setup uses a `/api` prefix or context path, make sure `withUploadUri(...)` matches the path as seen by the application. -### 4. Upload cleanup -After having processed the uploaded bytes on the server backend (e.g. copy them to their final persistent location), it's important to cleanup the (temporary) uploaded bytes. This can be done by calling the `me.desair.tus.server.TusFileUploadService.deleteUpload(String uploadUri)` method as shown in the example above. This will remove the uploaded bytes and any associated upload information from the storage backend. Alternatively, a client can also remove an (in-progress) upload using the [termination extension](https://tus.io/protocols/resumable-upload.html#termination). +### 3. Handle finished uploads -Next to removing uploads after they have been completed and processed by the backend, it is also recommended to schedule a regular maintenance task to clean up any expired uploads or locks. Cleaning up expired uploads and locks can be achieved using the `me.desair.tus.server.TusFileUploadService.cleanup()` method: +Register an `UploadCompletionListener` on the service from step 2 (Spring Boot version shown, replace the `TusConfig` class): ```java -// Run periodically (e.g., via @Scheduled in Spring) -@Scheduled(fixedDelay = 600000) // Every 10 minutes -public void cleanupExpiredUploads() { - tusFileUploadService.cleanup(); +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import me.desair.tus.server.TusFileUploadService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class TusConfig { + + private static final Logger log = LoggerFactory.getLogger(TusConfig.class); + + @Bean(destroyMethod = "close") + public TusFileUploadService tusFileUploadService() { + return new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true) + .withUploadCompletionListener( + (uploadInfo, service) -> { + // Safely retrieve or guess the file name (preventing path traversal and NPE) + String fileName = Paths.get(uploadInfo.getFileName()).getFileName().toString(); + try { + try (InputStream is = service.getUploadedBytes(uploadInfo)) { + Files.copy(is, Paths.get("/var/data/documents", fileName)); + } // closing the stream releases the lock, which deleteUpload needs + service.deleteUpload(uploadInfo); + } catch (Exception e) { + log.error("Failed to process completed upload {}", uploadInfo.getId(), e); + } + }); + } } ``` -## Protocol Version Support (Tus 1.0.0 & IETF Resumable Uploads) +Alternatively let the client notify your own REST endpoint and call `getUploadInfo`, `getUploadedBytes` and `deleteUpload` there. Also schedule `tusFileUploadService.cleanup()` to remove expired uploads. See [Handling Upload Completion](docs/INTEGRATION_GUIDE.md#2-handling-upload-completion--retrieving-files) and [Cleanup](docs/INTEGRATION_GUIDE.md#3-cleanup). -> [!WARNING] -> **Experimental Feature Disclaimer**: The IETF Resumable Uploads for HTTP (RUFH) specification (`draft-ietf-httpbis-resumable-upload`) is currently an active IETF draft. While this library implements draft-12 compliance, the RUFH protocol support should be considered **experimental** until the specification is published as an official RFC standard. +### 4. Connect a client -`tus-java-server` supports both protocol specifications seamlessly: -1. **Tus 1.0.0**: The widely-adopted [tus protocol standard](https://tus.io/). -2. **IETF Resumable Uploads for HTTP**: The official IETF standardization draft ([draft-ietf-httpbis-resumable-upload](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/)). +```javascript +import Uppy from '@uppy/core'; +import Tus from '@uppy/tus'; -### Configuring Protocol Version -You can configure protocol support via `withSupportedProtocolVersions(ProtocolVersion)`: +const uppy = new Uppy().use(Tus, { + endpoint: 'http://localhost:8080/api/upload', + chunkSize: 5 * 1024 * 1024 // 5MB chunk size +}); +``` -* `ProtocolVersion.AUTO` (Default): Automatically detects protocol version per HTTP request based on request headers (`Tus-Resumable` header triggers Tus 1.0.0; `Upload-Complete` header or `application/partial-upload` content type triggers IETF RUFH). -* `ProtocolVersion.TUS_1_0_0`: Enforces Tus 1.0.0 handling exclusively. -* `ProtocolVersion.RUFH`: Enforces IETF Resumable Uploads for HTTP (RUFH) handling exclusively. +Complete working examples: -### Protocol Comparison & Available Features +* [Resumable and asynchronous file upload in Spring Boot REST API with Uppy JavaScript client.](https://github.com/tomdesair/tus-java-server-spring-demo) +* [Detailed blog post by Ralph](https://golb.hplar.ch/2019/06/upload-with-tus.html) on how to use this library in [Spring Boot in combination with the Tus JavaScript client](https://github.com/ralscha/blog2019/tree/master/uploadtus). +* [Resumable and asynchronous file upload using Uppy with form submission in Dropwizard (Jetty)](https://github.com/tomdesair/tus-java-server-dropwizard-demo) -| Feature / Capability | Tus 1.0.0 | IETF Resumable Uploads | -|---|---|---| -| **Auto-Detection Signal** | `Tus-Resumable: 1.0.0` header | `Upload-Complete` header or `Content-Type: application/partial-upload` | -| **Creation** | `POST` with `Upload-Length` / `Upload-Defer-Length` | `POST` with `Upload-Complete: ?0` / `?1` (or `application/partial-upload`) | -| **Append Chunks** | `PATCH` with `Upload-Offset` | `PATCH` with `Upload-Offset` & `Content-Type: application/partial-upload` | -| **Upload Status Query** | `HEAD` returns `Upload-Offset` & `Upload-Length` | `HEAD` returns `Upload-Offset` & `Upload-Complete` | -| **Offset Mismatch Error** | HTTP 409 Conflict | HTTP 409 Conflict with RFC 7807 `application/problem+json` details | -| **104 Interim Responses** | N/A | Supported (see [docs/INTERIM_RESPONSES.md](docs/INTERIM_RESPONSES.md)) | -| **Upload Cancellation** | `DELETE` with `Tus-Resumable: 1.0.0` | `DELETE` with `Upload-Complete: ?0` | -| **Checksum Validation** | Supported (`Checksum` extension) | Supported (based on HTTP Digests / RFC 9530) | -| **Expiration Handling** | Supported (`Upload-Expires` header) | Supported (`max-age` parameter in `Upload-Limit` header) | -| **Concatenation** | Supported (`Concatenation` extension) | N/A | -| **Download Extension** | Supported (`Download` extension) | Supported (`Download` extension; without it, `GET` requests perform offset retrieval) | +The `scripts/` folder contains runnable Python test clients for both protocols ([Conformity Testing Guide](docs/CONFORMITY_TESTING.md)). -## Protocol Extensions -Besides the [tus core protocol](https://tus.io/protocols/resumable-upload.html#core-protocol), the library has all optional tus protocol extensions enabled by default. This means that the `Tus-Extension` header has value `creation,creation-defer-length,creation-with-upload,checksum,checksum-trailer,termination,expiration,concatenation,concatenation-unfinished`. Optionally you can also enable an unofficial `download` extension (see [configuration section](#usage-and-configuration)). +## Try It with curl -* [creation](https://tus.io/protocols/resumable-upload.html#creation): The creation extension allows you to create new uploads and to retrieve the upload URL for them. -* [creation-defer-length](https://tus.io/protocols/resumable-upload.html#post): You can create a new upload even if you don't know its final length at the time of creation. -* [creation-with-upload](https://tus.io/protocols/resumable-upload.html#creation): The creation-with-upload extension allows you to create the upload resource and upload initial file data in a single POST request. -* [checksum](https://tus.io/protocols/resumable-upload.html#checksum): An extension that allows you to verify data integrity of each upload (PATCH) request. -* [checksum-trailer](https://tus.io/protocols/resumable-upload.html#checksum): If the checksum hash cannot be calculated at the beginning of the upload, it may be included as a trailer HTTP header at the end of the chunked HTTP request. -* [termination](https://tus.io/protocols/resumable-upload.html#termination): Clients can terminate completed or in-progress uploads which allows the tus-java-server library to free up resources on the server. -* [expiration](https://tus.io/protocols/resumable-upload.html#expiration): You can instruct the tus-java-server library to cleanup uploads that are older than a configurable period. The expiration extension applies to both Tus 1.0.0 and IETF Resumable Uploads for HTTP (RUFH) protocols: - * **Tus 1.0.0 Protocol**: Expiration is communicated to clients via the `Upload-Expires` response header field formatted as an HTTP date-time (RFC 7231, e.g. `Upload-Expires: Wed, 25 Jun 2026 16:00:00 GMT`). - * **IETF RUFH Protocol**: Expiration is communicated to clients via the `max-age` parameter in the `Upload-Limit` response header field indicating remaining valid seconds (e.g. `Upload-Limit: max-size=1048576, max-age=3600`). -* [concatenation](https://tus.io/protocols/resumable-upload.html#concatenation): This extension can be used to concatenate multiple uploads into a single final upload enabling clients to perform parallel uploads and to upload non-contiguous chunks. -* [concatenation-unfinished](https://tus.io/protocols/resumable-upload.html#concatenation): The client is allowed send the request to concatenate partial uploads while these partial uploads are still in progress. -* [http-digests](https://datatracker.ietf.org/doc/rfc9530/): An extension implementing RFC 9530 to verify data integrity for the Resumable Uploads for HTTP (RUFH) protocol. Supported headers include `Content-Digest`, `Repr-Digest`, `Want-Content-Digest`, and `Want-Repr-Digest`. -* `download`: The (unofficial) download extension allows clients to download uploaded files using a HTTP `GET` request. You can enable this extension by calling the `withDownloadFeature()` method. -* `cors`: The (unofficial) CORS extension adds native CORS support out-of-the-box, setting CORS headers for all requests and responses, and handling preflight `OPTIONS` requests automatically. It is enabled by default. +Verify your endpoint without a frontend (replace the URL and use the `Location` header returned by the `POST`). -## Advanced Usage +**IETF Resumable Uploads (RUFH)** +```bash +# 1. Create the upload resource (more data will follow) +curl -i -X POST http://localhost:8080/api/upload -H "Upload-Complete: ?0" +# 2. Send the data and mark the upload as complete +curl -i -X PATCH http://localhost:8080/api/upload/ \ + -H "Upload-Offset: 0" -H "Upload-Complete: ?1" \ + -H "Content-Type: application/partial-upload" --data-binary "hello world" +# 3. Query progress at any time +curl -I http://localhost:8080/api/upload/ +``` -### HTTP Digests ([RFC 9530](https://www.rfc-editor.org/rfc/rfc9530.html)) -The `http-digests` extension implements RFC 9530 to support data integrity checks for both individual data chunks (`Content-Digest`) and the entire file (`Repr-Digest`). -* **Performance Disclaimer**: Calculating representation digests (`Repr-Digest`) requires streaming the entire uploaded file from disk. For extremely large files, this can introduce non-trivial I/O performance overhead on the server. To optimize, only request it via `Want-Repr-Digest` when absolutely necessary. +**Tus 1.0.0** +```bash +curl -i -X POST http://localhost:8080/api/upload \ + -H "Tus-Resumable: 1.0.0" -H "Upload-Length: 11" -H "Upload-Metadata: filename aGVsbG8udHh0" +curl -i -X PATCH http://localhost:8080/api/upload/ \ + -H "Tus-Resumable: 1.0.0" -H "Upload-Offset: 0" \ + -H "Content-Type: application/offset+octet-stream" --data-binary "hello world" +curl -I http://localhost:8080/api/upload/ -H "Tus-Resumable: 1.0.0" +``` -### Emitting HTTP 104 Interim Responses in Tomcat / Spring Boot -Because standard Java Servlet API (`HttpServletResponse`) does not natively support emitting 1xx informational responses, applications running on embedded Tomcat (such as Spring Boot) can use a custom Tomcat `Valve` to write the raw HTTP 104 interim response frame directly to Tomcat's underlying TCP socket buffer before servlet execution. +## How It Works -For complete architectural details, Tomcat/Servlet API limitations, and production recommendations, see the dedicated **[HTTP 104 Interim Responses Guide](docs/INTERIM_RESPONSES.md)**. +Your application only forwards HTTP requests to `TusFileUploadService`; the library implements the protocol, stores the bytes in a pluggable backend, locks uploads against concurrent writers and resolves lock contention when a client resumes. Your application consumes the finished upload and deletes it. -A live, production-ready reference implementation using cached reflection can be found in the [tus-java-server-spring-demo](https://github.com/tomdesair/tus-java-server-spring-demo) repository: -* [`TusInterimResponseTomcatValve`](https://github.com/tomdesair/tus-java-server-spring-demo/blob/main/spring-boot-rest/src/main/java/me/desair/spring/tus/TusInterimResponseTomcatValve.java) +```mermaid +sequenceDiagram + autonumber + actor Client as Client (Uppy / tus-js-client / RUFH client) + participant Protocol as Upload endpoint (/api/upload) + participant Storage as Storage (Disk / S3 / Azure) + participant App as Your application API + + Client->>Protocol: POST (create upload) + Protocol->>Storage: Initialize upload + Protocol-->>Client: 201 Created (Location) + Client->>Protocol: PATCH chunk(s), resumable after a failure + Protocol->>Storage: Append bytes + Protocol-->>Client: 204 (Upload-Offset) + Client->>App: "Upload finished" (or UploadCompletionListener) + App->>Storage: getUploadedBytes(...), deleteUpload(...) +``` -To register the Valve in Spring Boot via `TomcatServletWebServerFactory`: +Detailed Tus and RUFH flow diagrams: [Protocols](docs/PROTOCOLS.md). -```java -@Bean -public TomcatServletWebServerFactory tomcatFactory(TusFileUploadService tusFileUploadService) { - TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory(); - factory.addContextValves(new TusInterimResponseTomcatValve(tusFileUploadService)); - return factory; -} -``` +## Documentation -## Compatible Client Implementations & Conformity Testing -This server implementation has been tested with: -- **Tus 1.0.0 Clients**: Tested with [Uppy](https://uppy.io/) and `tus-js-client`. -- **IETF Resumable Uploads Clients & Conformity Tests**: The implementation has been thoroughly tested with our own built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH protocol specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). +| Topic | Guide | +|---|---| +| Overview of all documents | [Documentation index](docs/README.md) | +| `process()`, completion, cleanup, multi-tenancy, security, frontend | [Integration Guide](docs/INTEGRATION_GUIDE.md) | +| All `with...()` options, storage backend selection, dependencies | [Configuration Reference](docs/CONFIGURATION.md) | +| Tus vs. IETF RUFH, protocol version selection, extensions, HTTP digests, 104 responses | [Protocols](docs/PROTOCOLS.md), [Migrating clients to RUFH](docs/MIGRATION.md), [Interim responses](docs/INTERIM_RESPONSES.md) | +| Multi-replica, Kubernetes, reverse proxies | [Deployment Guide](docs/DEPLOYMENT.md), [Locking](docs/LOCKING.md) | +| Storage backends | [Disk & network (NFS/EFS/SMB)](docs/DISK_BASED_LOCKING.md), [S3-compatible](docs/S3_STORAGE.md), [Azure Blob](docs/AZURE_BLOB_STORAGE.md) | +| Problems | [Troubleshooting](docs/TROUBLESHOOTING.md) | +| Upgrading from 1.x | [Upgrade Guide](docs/UPGRADING.md), [CHANGELOG](CHANGELOG.md) | -For detailed instructions on running our native conformity test suite and interpreting results, see the **[Conformity Testing Guide (docs/CONFORMITY_TESTING.md)](docs/CONFORMITY_TESTING.md)**. +## Compatible Clients & Conformity Testing -This repository also contains comprehensive automated integration test suites (`ITTusFileUploadService`, `RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`) validating both protocol specifications. +The server is tested with [Uppy](https://uppy.io/), `tus-js-client`, the built-in Tus v1.0.0 and RUFH (draft-12 and RFC 9530) conformity suites in `scripts/` across Disk, S3 and Azure Blob backends, and the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). See [Protocols](docs/PROTOCOLS.md#compatible-clients--conformity-testing) and the [Conformity Testing Guide](docs/CONFORMITY_TESTING.md). ## Versioning This artifact follows `MAJOR.MINOR.PATCH` semantic versioning. Version `2.0.0` introduces major dual-protocol support for both Tus 1.0.0 and the IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload`). Version `1.0.0-3.3` was the last Tus protocol-only version. ## Contributing -This library comes without any warranty and is released under a [MIT license](https://github.com/tomdesair/tus-java-server/blob/master/LICENSE). If you encounter any bugs or if you have an idea for a useful improvement you are welcome to [open a new issue](https://github.com/tomdesair/tus-java-server/issues) or to [create a pull request](https://github.com/tomdesair/tus-java-server/pulls) with the proposed implementation. Please note that any contributed code needs to be accompanied by automated unit and/or integration tests and comply with the [defined code-style](#code-style). - -### Code Style -All pull requests should have the correct formatting according to [Google Java Style](https://github.com/google/google-java-format) code formatting. To verify if the code style is correct run: - -``` -mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:check -``` - -To reformat your code run: - -``` -mvn -P codestyle com.spotify.fmt:fmt-maven-plugin:format -``` - -See the [Google Java Style Github page](https://github.com/google/google-java-format) on recommendations on how to configure this in your IDE. Or if you have Python 3, you can also use [pre-commit](https://pre-commit.com) to make your live easier: - -``` -pip install pre-commit -pre-commit install -``` +This library comes without any warranty and is released under a [MIT license](https://github.com/tomdesair/tus-java-server/blob/master/LICENSE). Bug reports, ideas and pull requests are welcome; contributed code needs automated tests and must follow the Google Java Style. See [CONTRIBUTING.md](CONTRIBUTING.md) for details, code style and testing instructions. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..80ebd339 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,309 @@ +# Architecture Guide: tus-java-server + +This document describes the architectural design of `tus-java-server`. It is written for developers and contributors who want to understand how the library works internally, how requests flow through the system, and where to place new functionality. + +--- + +## 1. Architectural Philosophy + +`tus-java-server` is an embeddable, container-native resumable upload library for **Java 17+** and **Jakarta EE** (`jakarta.servlet.*`). Its design is guided by four principles: + +1. **Single Central Service Class**: A single facade—[`TusFileUploadService`](../src/main/java/me/desair/tus/server/TusFileUploadService.java)—acts as the public API entry point and orchestrator. It manages configuration, concurrency boundaries, and extension execution without protocol-specific branching. +2. **Protocol Composition via Extensions**: Protocols are not hardcoded monoliths. Both supported protocols (**Tus 1.0.0** and **IETF RUFH**) are composed of modular extensions extending [`AbstractTusExtension`](../src/main/java/me/desair/tus/server/util/AbstractTusExtension.java). +3. **Business Logic in Validators & Handlers**: All protocol rules, request checks, and byte operations reside exclusively in dedicated [`RequestValidator`](../src/main/java/me/desair/tus/server/RequestValidator.java) and [`RequestHandler`](../src/main/java/me/desair/tus/server/RequestHandler.java) implementations. +4. **Zero External Coordination Dependencies by Default**: Out-of-the-box distributed locking uses atomic filesystem leases and POSIX sibling mutexes. No Redis, ZooKeeper, or database cluster is required to achieve multi-replica container safety. + +--- + +## 2. High-Level Component Architecture + +The diagram below illustrates how incoming HTTP requests flow through the library down to the physical storage and locking abstractions: + +```mermaid +graph TD + Client["Client (Tus / RUFH / Uppy / curl)"] -->|HTTP Request| ServletContainer["Servlet Container / Spring Boot"] + ServletContainer -->|HttpServletRequest / Response| Service["TusFileUploadService (Central Facade)"] + + subgraph Service Layer ["Facade & Concurrency Boundary"] + Service -->|1. Acquire / Wait Lock| Locking["UploadLockingService (UploadLock)"] + Service -->|2. Detect Version| ProtoDetect["ProtocolVersion.AUTO (Utils.detectProtocolVersion)"] + Service -->|3. Route Pipeline| Extensions["Extension Registry (enabledFeatures)"] + end + + subgraph Extension Pipeline ["Protocol Extensions"] + Extensions --> TusExt["Tus 1.0.0 Extensions
(CoreProtocol, Creation, Checksum, etc.)"] + Extensions --> RufhExt["IETF RUFH Extension
(ResumableUploadsForHttpProtocol)"] + Extensions --> SharedExt["Shared Extensions
(CorsExtension, Expiration, Download, HttpDigests)"] + end + + subgraph Processing Engine ["Business Logic Engine"] + TusExt & RufhExt & SharedExt --> Validators["RequestValidator Pipeline
(Pure pre-condition validation)"] + TusExt & RufhExt & SharedExt --> Handlers["RequestHandler Pipeline
(I/O Streaming, state updates, response headers)"] + end + + subgraph Storage & Concurrency SPI ["Pluggable Infrastructure SPI"] + Handlers --> Storage["UploadStorageService
(Disk / S3 / Azure Blob)"] + Handlers --> Locking + Storage -.-> Caching["ThreadLocalCachedStorageAndLockingService"] + Storage -.-> Concat["UploadConcatenationService"] + end +``` + +--- + +## 3. Dual Protocols Composed of Extensions + +`tus-java-server` serves **two resumable upload protocols** on the exact same endpoint: + +1. **Tus 1.0.0**: The widely-adopted [tus protocol standard](https://tus.io/protocols/resumable-upload.html). +2. **IETF Resumable Uploads for HTTP (RUFH)**: The official standardization draft ([draft-ietf-httpbis-resumable-upload-12](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/)). + +By default, the service operates in `ProtocolVersion.AUTO` mode: it inspects incoming HTTP headers per request (`Tus-Resumable` activates Tus 1.0.0; `Upload-Complete` or `Content-Type: application/partial-upload` activates RUFH). + +### Protocol Composition Matrix + +Each protocol is built from a combination of **mandatory** and **optional** extensions: + +| Protocol | Mandatory Extensions | Optional Extensions | +|---|---|---| +| **Tus 1.0.0** | `CoreProtocol` (core `OPTIONS`, `HEAD`, `PATCH`) | `CreationExtension`, `CreationWithUploadExtension`, `ChecksumExtension`, `TerminationExtension`, `ExpirationExtension`, `ConcatenationExtension`, `CorsExtension`, `DownloadExtension` | +| **IETF RUFH** | `ResumableUploadsForHttpProtocol` (`OPTIONS`, `HEAD`, `GET`, `POST`, `PUT`, `PATCH`, `DELETE`) | `HttpDigestsExtension` (RFC 9530 digests), `ExpirationExtension` (`max-age`), `CorsExtension`, `DownloadExtension` | + +- **Mandatory Extensions**: `CoreProtocol` cannot be disabled; it defines core byte offset tracking and stream resumption for Tus 1.0.0. For RUFH requests, `ResumableUploadsForHttpProtocol` is the mandatory protocol handler. +- **Optional Extensions**: Can be disabled at runtime via `service.disableTusExtension("name")` or enabled on demand (e.g. `service.withDownloadFeature()`). + +--- + +## 4. The Validator & Handler Pipeline + +All protocol business logic is decoupled into two interfaces: + +### 1. `RequestValidator` (Validation Phase) +- **Role**: Pure, side-effect-free pre-condition verification. +- **Contract**: Inspects request headers and parameters. If invalid, throws a typed [`TusException`](../src/main/java/me/desair/tus/server/exception/TusException.java) containing the exact HTTP status code (e.g., `400 Bad Request`, `404 Not Found`, `409 Conflict`, `412 Precondition Failed`, `413 Payload Too Large`). +- **Examples**: `TusResumableValidator`, `UploadOffsetValidator`, `RufhCreationValidator`, `RufhAppendValidator`. + +### 2. `RequestHandler` (Execution & Error Phase) +- **Role**: State mutation, byte payload ingestion, response header generation, and error formatting. +- **Contract**: + - Standard handlers (`isErrorHandler() == false`): Streams upload bytes, updates offsets in storage, sets protocol headers (`Upload-Offset`, `Location`, `Upload-Limit`). + - Error handlers (`isErrorHandler() == true`): Intercepts thrown exceptions to emit protocol-specific error payloads, such as RFC 7807 problem details (`application/problem+json` in `RufhErrorHandler`). +- **Examples**: `CorePatchRequestHandler`, `RufhCreationPostRequestHandler`, `RufhErrorHandler`. + +### Request Lifecycle Sequence + +The sequence below details the execution of `TusFileUploadService.process(...)`: + +```mermaid +sequenceDiagram + autonumber + actor Client + participant Service as TusFileUploadService + participant Locking as UploadLockingService + participant Extension as AbstractTusExtension + participant Validator as RequestValidator + participant Handler as RequestHandler + participant Storage as UploadStorageService + + Client->>Service: HTTP Request (POST / PATCH / HEAD / DELETE) + Service->>Locking: acquireUploadLock(method, uri) (wait / retry up to 60s) + Locking-->>Service: UploadLock handle + Service->>Service: checkWasInProgress(uri) + + rect rgb(240, 248, 255) + note over Service,Validator: Phase 1: Request Validation + Service->>Extension: validate(method, req, storage, locking, version) + Extension->>Validator: validate(method, req, storage) + Validator-->>Extension: OK (or throws typed TusException) + end + + rect rgb(245, 255, 245) + note over Service,Storage: Phase 2: Request Execution + Service->>Extension: process(method, req, resp, storage, locking, version) + Extension->>Handler: process(method, req, resp, storage, locking) + Handler->>Storage: append(stream) / create(info) / update() + Handler-->>Service: Set response status & headers + end + + opt Exception Thrown + Service->>Extension: handleError(method, req, resp, storage, locking, version, exception) + Extension->>Handler: ErrorHandler.process(...) + Handler-->>Service: Write RFC 7807 problem details / error status + end + + opt Upload Finished + Service->>Locking: cleanupLock(uploadId) + Service->>Service: notifyUploadCompletionListeners(uploadInfo) + end + + Service->>Locking: lock.close() (releases lock handle) + Service-->>Client: HTTP Response (201 / 204 / 409 / etc.) +``` + +--- + +## 5. Pluggable Storage Backend Architecture + +File storage and metadata persistence are decoupled behind the [`UploadStorageService`](../src/main/java/me/desair/tus/server/upload/UploadStorageService.java) SPI. + +``` +UploadStorageService +├── DiskStorageService (Filesystem & Network Shares) +├── S3StorageService (AWS S3, MinIO, RustFS, Ceph, Cloudflare R2) +└── AzureBlobStorageService (Azure Blob Storage Block Blobs) +``` + +### Two Disk Storage Implementations + +For local filesystems and shared network volumes, `tus-java-server` provides two distinct concurrency/storage combinations: + +1. **Current / Default: Lease File Based (`DiskStorageService` + `LeaseFileLockingService`)**: + - Configured automatically via `withStoragePath(String)`. + - Replaces OS kernel file locks with atomic sibling mutex directories (`.mutex/`) and JSON lease metadata (`lease.json`). + - Works safely across multi-replica container clusters, Kubernetes pods (`ReadWriteMany`), AWS EFS, Azure Files, SMB, and NFSv3/v4 mounts without requiring `rpc.statd` or external databases. +2. **Legacy Opt-Out: File Lock Based (`DiskStorageService` + `DiskLockingService`)**: + - Opt-out via `withUploadStorageService(new DiskStorageService(...)).withUploadLockingService(new DiskLockingService(...))`. + - Uses OS kernel-level locks (`java.nio.channels.FileLock` via POSIX `fcntl` or Windows `LockFileEx`). + - Suitable strictly for single-node deployments on local disk. + +### Cloud Storage Implementations + +- **`S3StorageService`**: Stores chunks as S3 objects and concatenates them using multipart upload copy or compose helpers (`S3ConcatenationService`). +- **`AzureBlobStorageService`**: Stages chunks as uncommitted block blobs using `.part` temporary buffers and commits block lists atomically upon completion. Zero-copy concatenation is supported via `stageBlockFromUrl`. + +### Storage Extensions & Helpers + +- **Upload Concatenation SPI (`UploadConcatenationService`)**: Supports merging parallel chunk uploads into a single final resource. Implemented natively for Disk (`VirtualConcatenationService`), S3 (`S3ConcatenationService`), and Azure (`AzureBlobConcatenationService`). +- **Upload Deduplication**: Enabled via `.withUploadDeduplication(true)`. Completed uploads are indexed by checksum. Duplicate uploads are linked via `duplicatesUploadId`. Read operations dynamically resolve to the parent upload, while write and truncation operations strictly operate on the target upload to prevent corrupting parent data. +- **Thread-Local Request Caching (`ThreadLocalCachedStorageAndLockingService`)**: Enabled via `.withThreadLocalCache(true)`. Caches `UploadInfo` in memory for the duration of a single HTTP request to eliminate redundant network roundtrips to S3 or Azure Blob Storage. + +--- + +## 6. Distributed Locking & Lock Contention Resolution + +Resumable uploads require exclusive concurrency control per upload resource to prevent race conditions (e.g., overlapping byte appends or simultaneous resume and delete requests). + +All locking backends implement [`UploadLockingService`](../src/main/java/me/desair/tus/server/upload/UploadLockingService.java) and return [`UploadLock`](../src/main/java/me/desair/tus/server/upload/UploadLock.java) handles. + +### Building Lease-Based Lock Providers: `AbstractLeaseLockingService` + +Custom distributed locking services (or extensions for GCP, Redis, DynamoDB, etc.) should extend [`AbstractLeaseLockingService`](../src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java): + +```mermaid +classDiagram + class UploadLockingService { + <> + +lockUploadByUri(uri) UploadLock + +isLocked(id) boolean + +cleanupLock(id) void + +registerInputStream(uri, stream) void + +requestLockRelease(uri) void + } + + class AbstractCloseableResourceService { + <> + #registerShutdownHook() + #cleanupOnClose() + } + + class AbstractLeaseLockingService { + <> + #long leaseDurationMs + #long CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L + #Map activeInputStreams + #ScheduledExecutorService watchdogExecutor + +lockUploadByUri(uri) UploadLock + +requestLockRelease(uri) void + #acquireOrEvictExpiredLock(id, holderId, uri) + #tryAcquireLock(id, leaseData)* UploadLock + #isLockExpired(id)* boolean + #evictExpiredLock(id)* boolean + #writeStopSignal(id)* void + #checkStopSignals()* void + } + + class LeaseFileLockingService { + +tryAcquireLock() + +evictExpiredLock() + } + + class S3LockingService { + +tryAcquireLock() + +evictExpiredLock() + } + + UploadLockingService <|.. AbstractLeaseLockingService + AbstractCloseableResourceService <|-- AbstractLeaseLockingService + AbstractLeaseLockingService <|-- LeaseFileLockingService + AbstractLeaseLockingService <|-- S3LockingService +``` + +**Key Features of `AbstractLeaseLockingService`**: +1. **Clock Skew Safety Margin**: Built-in 2.0-second buffer (`CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L`) absorbs NTP drift between cluster nodes, preventing premature lease eviction. +2. **Stream Interruption Registry**: Tracks active in-flight input streams in `activeInputStreams` and interrupts them cleanly via `Utils.interruptStream(...)`. +3. **Contention Dispatch & Polling Watchdog**: When contention is detected, dispatches a `.stop` signal and runs a background watchdog executor (`Utils.scheduleWatchdog(...)`) to terminate paused streams on remote nodes. +4. **TOCTOU-Safe Eviction with Jitter**: Implements randomized backoff jitter (`jitterMinMs` to `jitterMaxMs`) to prevent thundering herd collisions when multiple nodes contend for an expired lease. + +### Cross-Node Lock Contention Resolution + +When a client loses network connectivity, the server's `PATCH` socket may remain stalled in a half-open state. When the client resumes with a `HEAD` request (or cancels with `DELETE`), contention is resolved across nodes: + +```mermaid +sequenceDiagram + autonumber + actor Client + participant NodeA as Pod A (Stalled PATCH) + participant SharedStorage as Shared Storage / Bucket + participant NodeB as Pod B (Resuming HEAD) + + Note over NodeA: Stalled PATCH stream holds lease lock
(Stream wrapped in InterruptibleInputStream) + Client->>NodeB: HEAD /files/upload/123 + NodeB->>NodeB: acquireUploadLock fails (UploadAlreadyLockedException) + NodeB->>NodeB: Enter lock wait loop (every 200ms up to 60s) + NodeB->>SharedStorage: requestLockRelease writes 123.stop signal + + Note over NodeA: Watchdog polls every 1.5s and detects 123.stop + NodeA->>NodeA: watchdog calls stream.interrupt() + NodeA->>SharedStorage: Drain in-flight cloud chunks (up to 55s) + NodeA->>SharedStorage: Release lease lock & remove 123.stop + + NodeB->>SharedStorage: acquireUploadLock succeeds! + NodeB->>SharedStorage: Read current Upload-Offset + NodeB-->>Client: 204 No Content (Upload-Offset: 524288) +``` + +For more details on locking mechanisms, see [`docs/LOCKING.md`](LOCKING.md) and [`docs/DISK_BASED_LOCKING.md`](DISK_BASED_LOCKING.md). + +--- + +## 7. Three-Tiered Testing Strategy + +The repository maintains high reliability across protocols and storage engines via three testing tiers: + +```mermaid +graph TD + subgraph Testing Pyramid + Top["Conformity Test Suites (Python)
• Socket-level HTTP verification against running servers
• scripts/tus_conformity_test.py (50+ rules)
• scripts/rufh_conformity_test.py (RFC 9530, draft-12)"] + Mid["Integration Tests (Java IT*)
• Real container lifecycles via Testcontainers (Podman / Docker)
• Verified in CI via mvn verify
• Abstract base classes test Disk, S3 (RustFS), and Azure (Azurite)"] + Base["Unit Tests (Java *Test.java)
• Pure offline Mockito isolation (< 2s)
• Verified via mvn test
• Strict code coverage gate (> 90% per file)"] + end + Base --> Mid --> Top +``` + +1. **Unit Tests (`*Test.java`)**: Target specific classes in complete isolation with Mockito. Zero container or network requirements. Executed rapidly via `mvn test -q`. Coverage is verified via `python3 scripts/check-coverage.py --per-file-limit 90`. +2. **Integration Tests (`IT*`)**: End-to-end multi-backend verification running under `mvn verify -q`. Tests spin up dynamic local containers via Testcontainers using **either Podman or Docker**. Base classes (`AbstractITRufhProtocol`, `AbstractITTusFileUploadService`) ensure identical test coverage across Disk, S3, and Azure backends. +3. **Conformity Tests (`scripts/*_conformity_test.py`)**: Black-box Python test suites executed against live server deployments (e.g. `tus-java-server-spring-demo` on port 8080) to validate strict specification compliance. + +For detailed testing instructions, see [`docs/TESTING.md`](TESTING.md) and [`docs/CONFORMITY_TESTING.md`](CONFORMITY_TESTING.md). + +--- + +## 8. Contributor Blueprint: Where to Add New Functionality + +| Task | Where to Implement | Key Guidelines | +|---|---|---| +| **Add a Request Validation Rule** | Create a class implementing `RequestValidator` in the target extension's `validation` package. | Keep validation side-effect-free. Throw typed `TusException` subclasses. Register in `extension.initValidators()`. | +| **Add a Request Processing Action** | Create a class implementing `RequestHandler` (or extending `AbstractRequestHandler`). | Wrap streaming payload inputs in `InterruptibleInputStream`. Set headers on `TusServletResponse`. Register in `extension.initRequestHandlers()`. | +| **Add a New Protocol Extension** | Create a package under `me.desair.tus.server.` and extend `AbstractTusExtension`. | Implement `getName()`, `isApplicable(method, version)`, `initValidators()`, and `initRequestHandlers()`. Register in `TusFileUploadService.initFeatures()`. | +| **Add a Storage Backend** | Implement `UploadStorageService` (and optionally `UploadConcatenationService`). | Support deduplication (`duplicatesUploadId`). Implement offline unit tests (`*Test.java`) and Testcontainers integration tests (`IT*`). | +| **Add a Distributed Locking Provider** | Extend `AbstractLeaseLockingService` and `AbstractLeaseLock`. | Implement the template hooks (`tryAcquireLock`, `isLockExpired`, `evictExpiredLock`, `writeStopSignal`, `checkStopSignals`). Inherit NTP safety margins and stream interruption automatically. | diff --git a/docs/AZURE_BLOB_STORAGE.md b/docs/AZURE_BLOB_STORAGE.md index 99101956..b0bb3bfd 100644 --- a/docs/AZURE_BLOB_STORAGE.md +++ b/docs/AZURE_BLOB_STORAGE.md @@ -64,7 +64,9 @@ AzureBlobLockingService azureLockingService = new AzureBlobLockingService(contai TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files/upload") .withUploadStorageService(azureStorageService) - .withUploadLockingService(azureLockingService); + .withUploadLockingService(azureLockingService) + .withUploadDeduplication(true) + .withThreadLocalCache(true); ``` --- @@ -73,11 +75,11 @@ TusFileUploadService tusService = new TusFileUploadService() > [!IMPORTANT] > **Why `ThreadLocalCachedStorageAndLockingService` is Recommended for Azure**: -> By default, `TusFileUploadService` automatically wraps your custom `UploadStorageService` and `UploadLockingService` in a `ThreadLocalCachedStorageAndLockingService`. +> By default, thread-local caching is disabled (`false`). You should explicitly enable it via `.withThreadLocalCache(true)`. > > During a single HTTP request lifecycle (POST, PATCH, HEAD, DELETE), the tus server validates request headers, reads upload state, appends data, and constructs response headers. Without caching, retrieving `UploadInfo` and calculating offsets would require multiple redundant network roundtrips to Azure (`downloadContent` on `.info`, `getProperties`). > -> `ThreadLocalCachedStorageAndLockingService` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces Azure network latency and API call cost per request. +> Enabling `withThreadLocalCache(true)` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces Azure network latency and API call cost per request. --- @@ -103,6 +105,15 @@ TusFileUploadService tusService = new TusFileUploadService() | `checksumsPrefix` | `"checksums/"` | Blob name prefix for deduplication index objects | | `locksPrefix` | `"locks/"` | Blob name prefix for distributed lock lease objects | +### Lock Blob Lifecycle & Automatic Cleanup + +During an in-progress upload, `AzureBlobLockingService` creates a target lock blob (`locks/.lock`) and holds a 30-second native Azure Blob Lease with automatic background renewal. + +1. **Automatic Cleanup on Completion**: Once an upload completes (`offset == length`), `TusFileUploadService` calls `cleanupLock(UploadId)` after releasing the lease, deleting the `.lock` blob and any `.stop` signal blob. +2. **Automatic Cleanup on Termination & Expiration**: When an upload is canceled (`DELETE`) or purged via `cleanupExpiredUploads()`, the corresponding `.lock` and `.stop` blobs are deleted. +3. **Lease Safety Guard**: Deleting an actively leased blob without providing the lease ID triggers HTTP 412 (`LeaseIdMissing`). `AzureBlobLockingService` inspects the blob's lease status and skips deleting blobs that are actively leased by concurrent requests. +4. **Fallback Stale Lock Sweeper (`cleanupStaleLocks`)**: To prevent abandoned lock blobs after ungraceful pod termination or network partitions, `cleanupStaleLocks()` runs as a scheduled fallback. It sweeps unleased `.lock` blobs older than 120 seconds and `.stop` signal blobs older than 30 seconds. Actively leased blobs are never deleted. + --- ## 4. Post-Upload Processing (`getAzureBlobName`) @@ -113,13 +124,14 @@ After an upload completes, downstream services can obtain the direct Azure blob import com.azure.storage.blob.BlobClient; import me.desair.tus.server.upload.azure.AzureBlobStorageService; -AzureBlobStorageService azureStorage = (AzureBlobStorageService) tusService.getUploadStorageService(); - +// Inject or reference the concrete AzureBlobStorageService bean directly. +// (Avoid casting tusService.getUploadStorageService() when withThreadLocalCache(true) +// is enabled, as it returns ThreadLocalCachedStorageAndLockingService). String uploadUri = "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"; String ownerKey = "user-123"; // 1. Obtain full Azure blob name after upload completion -String blobName = azureStorage.getAzureBlobName(uploadUri, ownerKey); +String blobName = azureStorageService.getAzureBlobName(uploadUri, ownerKey); // e.g. "uploads/24249a5b-01a4-4bf8-b67a-364273bb5a2e" // 2. Direct Azure SDK access for post-upload processing @@ -157,15 +169,23 @@ BlobContainerClient containerClient = new BlobContainerClientBuilder() --- -## 6. Local Disk Buffer & Block Size Auto-Calibration +## 6. Local Disk Buffer & Asynchronous Chunk Pipelining -`AzureBlobStorageService` streams incoming PATCH payloads in chunks of `optimalBlockSize` into temporary files, staging each block to Azure as it completes. Peak disk usage per upload is capped at `1 × optimalBlockSize` (e.g. 8 MB). +`AzureBlobStorageService` streams incoming PATCH payloads in chunks of `optimalBlockSize` into temporary files, pipelining block staging to Azure Blob Storage asynchronously via `AsyncChunkUploader`: +- **Asynchronous 3-Slot Pipeline**: + - **Slot 1 (Receiving)**: Streaming incoming bytes from the client into a local temporary file. + - **Slot 2 (Waiting)**: Holds one ready chunk on disk buffer. + - **Slot 3 (Uploading)**: Actively staging the block (`stageBlock`) to Azure Blob Storage on a background daemon worker thread. + - Falls back to synchronous caller execution (`CallerRunsPolicy` on a `SynchronousQueue`) when the thread pool is saturated, preventing worker queue latency and unbounded buffering. +- **Thread Pool Sizing**: Worker threads are managed via a shared executor, default 10 threads, configurable on `TusFileUploadService` via `.withCloudUploadThreadPoolSize(int)` or directly on `AzureBlobStorageService.setCloudUploadThreadPoolSize(int)`. +- **Bounded Disk Footprint**: Peak disk usage per active upload is bounded to at most $2 \times \text{optimalBlockSize}$ (one receiving slot + one waiting slot). Block sizes auto-calibrate based on total upload size: -- **Baseline Preferred Size**: 8 MB (configurable via constructor) +- **Baseline Preferred Size**: 8 MB (configurable via constructor or `setPreferredBlockSize(long)`) - **Minimum Block Size**: 4 MB - **Maximum Block Size**: 4000 MiB (Azure limit) - **Maximum Blocks per Blob**: 50,000 (Azure limit) +- **Multi-TB Support & Auto-Calibration**: When upload length exceeds 400 GB ($50,000 \times 8\text{ MB}$), block size automatically scales up proportionally (e.g. ~22 MB for 1 TB, ~110 MB for 5 TB, supporting blobs up to 190 TB) to ensure the upload finishes within Azure's 50,000 blocks ceiling. --- @@ -174,11 +194,16 @@ Block sizes auto-calibrate based on total upload size: ### Lease Renewal Rationale `AzureBlobLockingService` uses native Azure Blob Leases (30-second duration) for distributed locking. Because large file uploads can stream over several minutes or hours, `AzureBlobUploadLock` runs a background daemon thread that renews the lease every 10 seconds. If an application server crashes unexpectedly, the lease auto-expires after 30 seconds without requiring manual lock cleanup sweeps. -### Lock Contention Resolution +### Lock Wait & Contention Resolution Lock contention resolution operates on two levels: 1. **JVM-local**: Active `InterruptibleInputStream` instances are registered in a concurrent map and interrupted directly if a concurrent lock request arrives in the same JVM. 2. **Cross-replica**: A `.stop` signal blob (`locks/.stop`) is written to Azure Storage. A background watchdog thread polls for `.stop` blobs and interrupts active streams on other cluster nodes. +When a client resumes an interrupted transfer by sending a `HEAD` request (or cancels it with `DELETE`), the request enters a **lock wait** retry loop in `TusFileUploadService`: +- **Lock Wait Timeout (`withLockWaitTimeout(Duration)`)**: Governs how long the incoming request waits (polling every 200ms; default 60 seconds / 300 retries). +- **Background Chunk Draining (`drainTimeout`)**: The interrupted upload is automatically allotted up to `lockWaitTimeout - 5 seconds` (default 55 seconds) to drain and stage in-flight blocks to Azure Blob Storage via `AsyncChunkUploader` before committing the block list and persisting the updated offset. +- **Lock Handover**: Because the drain timeout leaves a 5-second buffer before the lock wait budget expires, the active upload cleanly finishes staging blocks and releases the Azure Blob Lease within the caller's wait window, enabling seamless resumption without `UploadAlreadyLockedException`. + --- ## 8. Troubleshooting Guide @@ -187,7 +212,7 @@ Lock contention resolution operates on two levels: |---|---|---| | **HTTP 409 Conflict** | Another process or cluster pod currently holds an active lease on the lock blob. | Normal behavior during concurrent PATCH/DELETE requests. Retry after lock release. | | **HTTP 404 BlobNotFound** | The upload metadata `.info` blob does not exist or was expired/deleted. | Verify upload ID validity or upload expiration timestamps (`uploadExpirationPeriod`). | -| **Azurite connection refused** | Azurite emulator is not running or listening on port 10000. | Launch Azurite via Docker (`docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite`). | +| **Azurite connection refused** | Container runtime (Podman or Docker) is not running during integration tests. | Start Podman or Docker; Testcontainers automatically manages Azurite startup and teardown during integration tests. | | **`MaxAppendSizeExceededException`** | Incoming PATCH payload chunk exceeded the configured `maxAppendSize`. | Adjust `withMaxAppendSize()` setting on `TusFileUploadService`. | --- @@ -201,11 +226,11 @@ Lock contention resolution operates on two levels: | `AzureBlobLockingServiceTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure locking service parameters, prefix normalization, URI parsing, and defensive handling. | | `AzureBlobUploadLockTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure upload lock parameters, getters, and daemon executor lifecycle. | | `AzureBlobConcatenationServiceTest` | Unit Test | Offline | < 1s | Pure offline unit tests for Azure concatenation service parameters, prefix sanitization, guard clauses, and partial upload handling. | -| `ITAzureBlobStorageService` | Integration | Azurite (Docker) | ~ 3s | Live end-to-end storage integration test against Azurite emulator (Block Blob staging, `.part` buffer commits, truncations, deduplication, and expiration). | -| `ITAzureBlobLockingService` | Integration | Azurite (Docker) | ~ 2s | Distributed locking, Azure Blob Leases, lease renewals, lock contention (409 Conflict), stream interruption, & stop signal integration tests. | -| `ITAzureBlobConcatenationService` | Integration | Azurite (Docker) | ~ 2s | Server-side zero-copy block concatenation (`stageBlockFromUrl`) integration tests. | -| `ITAzureBlobRufhProtocol` | Integration | Azurite (Docker) | ~ 3s | IETF RUFH protocol integration suite for Azure backend. | -| `ITAzureBlobTusFileUploadService` | Integration | Azurite (Docker) | ~ 3s | Tus 1.0.0 protocol integration suite for Azure backend. | +| `ITAzureBlobStorageService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | Live end-to-end storage integration test against Azurite emulator (Block Blob staging, `.part` buffer commits, truncations, deduplication, and expiration). | +| `ITAzureBlobLockingService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 2s | Distributed locking, Azure Blob Leases, lease renewals, lock contention (409 Conflict), stream interruption, & stop signal integration tests. | +| `ITAzureBlobConcatenationService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 2s | Server-side zero-copy block concatenation (`stageBlockFromUrl`) integration tests. | +| `ITAzureBlobRufhProtocol` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | IETF RUFH protocol integration suite for Azure backend. | +| `ITAzureBlobTusFileUploadService` | Integration | Azurite (Testcontainers / Podman or Docker) | ~ 3s | Tus 1.0.0 protocol integration suite for Azure backend. | --- diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 00000000..a3d5cdb3 --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,85 @@ +# Configuration Reference + +All configuration is done through the fluent `with...()` methods of `TusFileUploadService`. Create **one** instance and share it (e.g. as a Spring singleton bean); the service is thread-safe and holds the storage and locking backends. + +```java +@Bean +public TusFileUploadService tusFileUploadService() { + return new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true); +} +``` + +## Dependencies + +The library only depends on Jakarta Servlet API 6.0 and some Apache Commons utility libraries, so (in theory) you can use it on any modern Java web application server such as Tomcat, JBoss or Jetty. By default all uploaded data and information is stored on a (shared) file volume of the application server. + +Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are bundled directly with compile scope by `tus-java-server`. When using S3 storage (`S3StorageService`), include the MinIO Java SDK dependency matching the version in `pom.xml`: + +```xml + + io.minio + minio + 9.0.3 + +``` + +For Azure Blob Storage dependencies see the [Azure Blob Storage Guide](AZURE_BLOB_STORAGE.md#1-quick-start). + +## Options + +### Endpoint & protocol + +| Method | Default | Description | +|---|---|---| +| `withUploadUri(String)` | `"/"` | Sets relative path (e.g. `/api/upload`) or absolute base URL (e.g. `https://upload.example.com/api/upload`) under which the upload endpoint is exposed. Supports regex parameters (e.g. `/users/[0-9]+/files/upload`). Use the absolute form when running behind a proxy that rewrites the path or host, so that `Location` headers are correct. | +| `withSupportedProtocolVersions(ProtocolVersion)` | `ProtocolVersion.AUTO` | Configures protocol handling: `AUTO` (header-based auto-detection), `TUS_1_0_0` (Tus 1.0.0 only), or `RUFH` (IETF draft-12 only). See [PROTOCOLS.md](PROTOCOLS.md). | +| `withChunkedTransferDecoding(Boolean)` | `false` | Enables manual chunked HTTP decoding for servlet containers that do not decode chunked requests natively. | +| `addTusExtension(TusExtension)` | All standard enabled | Adds a custom extension (e.g. application authorization checks). | +| `disableTusExtension(String)` | None | Disables a built-in extension (`creation`, `creation-with-upload`, `checksum`, `expiration`, `concatenation`, `termination`, `download`, `cors`, `http-digests`, `resumable-uploads-for-http`). | +| `withDownloadFeature()` | Disabled | Enables the unofficial `download` extension allowing clients to retrieve uploaded bytes via HTTP `GET`. | + +### Limits, expiration & integrity + +| Method | Default | Description | +|---|---|---| +| `withMaxUploadSize(Long)` | `Long.MAX_VALUE` | Maximum allowed total upload size in bytes per upload resource. | +| `withMaxAppendSize(Long)` / `withMinAppendSize(Long)` / `withMinSize(Long)` | No limit | IETF RUFH only: maximum size of a single append (`PATCH`) request, minimum size of an append or creation request, and minimum total upload size (all in bytes). | +| `withUploadExpirationPeriod(Long)` | `null` (disabled) | Expiration period in **milliseconds** after which incomplete/expired uploads become eligible for cleanup (run `cleanup()` periodically, see the [Integration Guide](INTEGRATION_GUIDE.md#3-cleanup)). | +| `withUploadDeduplication(Boolean)` | `false` | Enables duplicate file detection by checksum, linking new uploads (`duplicatesUploadId`) and skipping redundant storage writes. | + +### Identity, events & serialization + +| Method | Default | Description | +|---|---|---| +| `withUploadIdFactory(UploadIdFactory)` | `UuidUploadIdFactory` | Custom ID generator for upload resources (e.g., `UuidUploadIdFactory` or `TimeBasedUploadIdFactory`). | +| `withUploadCompletionListener(UploadCompletionListener)` / `addUploadCompletionListener(...)` | None | Registers a callback invoked immediately when an upload finishes transferring all bytes and is completed. Both methods append to the list of active listeners. | +| `withJsonSerialization()` / `withJsonSerialization(boolean)` | Java serialization | Enables or disables JSON serialization for upload metadata (`UploadInfo`). Jackson is bundled by default. | + +### Storage, locking & performance + +| Method | Default | Description | +|---|---|---| +| `withStoragePath(String)` | `${java.io.tmpdir}/tus` | Path on the filesystem or shared drive where uploaded bytes and metadata are stored when using `DiskStorageService`. Also configures `LeaseFileLockingService`. | +| `withUploadStorageService(UploadStorageService)` | `DiskStorageService` | Configures custom or cloud storage backend (`DiskStorageService`, `S3StorageService`, `AzureBlobStorageService`). Automatically copies over previous limits and configuration. | +| `withUploadLockingService(UploadLockingService)` | `LeaseFileLockingService` | Configures custom or cloud locking backend (`LeaseFileLockingService`, `S3LockingService`, `AzureBlobLockingService`). | +| `withLockWaitTimeout(Duration)` | `Duration.ofSeconds(60)` | Maximum duration a request waits to acquire an upload lock held by an in-flight transfer (retrying every 200ms, resulting in 300 retries). Automatically configures cloud background chunk drain timeout to 5s less than this value (default 55s). See [LOCKING.md](LOCKING.md). | +| `withThreadLocalCache(Boolean)` | `false` | Enables in-memory thread-local caching of upload request data to reduce storage backend I/O load. Strongly recommended for cloud backends (S3, Azure) to eliminate redundant remote metadata lookups per request. | +| `withCloudUploadThreadPoolSize(int)` | `10` | Maximum number of worker threads used for asynchronous background chunk uploading in cloud storage backends (S3, Azure). | + +> Backend-specific options (S3 jitter, Azure credentials, key prefixes...) are documented in the respective backend guides below. + +## Storage Backend Options + +`tus-java-server` provides a pluggable storage architecture supporting multiple backends. Use the table to pick one, then follow the linked guide. See also [DEPLOYMENT.md](DEPLOYMENT.md) for multi-replica considerations. + +| Backend | Classes | Best for | Guide | +|---|---|---|---| +| **File Disk & Network Storage** (default) | `DiskStorageService` & `LeaseFileLockingService` | Single node, shared NFS (v3/v4, AWS EFS, Azure Files, SMB/CIFS) or a Kubernetes Persistent Volume (`ReadWriteMany` / `ReadWriteOnce`). | [Disk & Network Storage Locking Guide](DISK_BASED_LOCKING.md) | +| **S3-Compatible Object Storage** | `S3StorageService`, `S3LockingService` & `S3ConcatenationService` | AWS S3, MinIO, RustFS, Cloudflare R2, Ceph or Google Cloud Storage. Uses distributed S3 object locking and TTL leases, so multi-replica container deployments work without Redis or external databases. | [S3 Storage Guide](S3_STORAGE.md) | +| **Azure Blob Storage** | `AzureBlobStorageService`, `AzureBlobLockingService` & `AzureBlobConcatenationService` | Microsoft Azure using the `azure-storage-blob` SDK. Uses native Azure Blob Leases (30s renewable leases) for distributed locking with automatic lock cleanup on upload completion, termination, and expiration. | [Azure Blob Storage Guide](AZURE_BLOB_STORAGE.md) | + +Cloud backends feature an asynchronous 3-slot chunk pipeline (`AsyncChunkUploader`) that overlaps client payload streaming with cloud staging in the background. The guides above cover multi-replica deployments in Kubernetes, post-upload processing, and legacy locking opt-out. diff --git a/docs/CONFORMITY_TESTING.md b/docs/CONFORMITY_TESTING.md index 0255a0d8..ba482c68 100644 --- a/docs/CONFORMITY_TESTING.md +++ b/docs/CONFORMITY_TESTING.md @@ -1,17 +1,19 @@ -# Conformity Testing Guide (IETF Resumable Uploads for HTTP) +# Protocol Conformity Testing Guide -This guide describes how to execute the RUFH (Resumable Uploads for HTTP) conformity tests against a locally running instance of the Spring Boot demo server or any RUFH compliant server endpoint. +This guide describes how to execute the automated conformity test suites for both supported resumable upload protocols: +1. **Tus v1.0.0 Protocol**: [`scripts/tus_conformity_test.py`](../scripts/tus_conformity_test.py) validating the official [Tus v1.0.0 Resumable Upload Protocol](https://tus.io/protocols/resumable-upload). +2. **IETF Resumable Uploads for HTTP (RUFH)**: [`scripts/rufh_conformity_test.py`](../scripts/rufh_conformity_test.py) validating [draft-ietf-httpbis-resumable-upload-12](https://www.ietf.org/archive/id/draft-ietf-httpbis-resumable-upload-12.txt) and [RFC 9530 HTTP Digests](https://www.rfc-editor.org/rfc/rfc9530.html). -The test suite validates compliance with [draft-ietf-httpbis-resumable-upload-12](https://www.ietf.org/archive/id/draft-ietf-httpbis-resumable-upload-12.txt) and [RFC 9530 HTTP Digests](https://www.rfc-editor.org/rfc/rfc9530.html). +Both conformity test suites perform end-to-end, socket-level protocol verification against a running server, ensuring strict adherence to HTTP status codes, structured response headers, error semantics, concurrency guarantees, and storage engine consistency. --- ## 1. Build and Install the Server Library -First, compile and install the core `tus-java-server` library to your local Maven repository: +First, compile and install the core `tus-java-server` library into your local Maven repository: ```bash -# In the root of the tus-java-server repository +# In the root directory of the tus-java-server repository mvn clean install -DskipTests ``` @@ -19,7 +21,7 @@ mvn clean install -DskipTests ## 2. Start the Demo Server -1. Verify the dependency in `tus-java-server-spring-demo` project's `spring-boot-rest/pom.xml` points to the snapshot version: +1. Verify that the dependency in `tus-java-server-spring-demo` project's `spring-boot-rest/pom.xml` references the snapshot version: ```xml me.desair.tus @@ -28,64 +30,157 @@ mvn clean install -DskipTests ``` -2. Build and start the Spring Boot REST demo server with a `1 KB` maximum upload size parameter (`--tus.server.max-upload-size=1024`) to enable full limit discovery & limit enforcement verification: +2. Build and start the Spring Boot REST demo server with a `1 KB` maximum upload size limit (`--tus.server.max-upload-size=1024`) to enable full limit discovery & limit enforcement verification: ```bash cd ../tus-java-server-spring-demo mvn clean package -DskipTests java -jar spring-boot-rest/target/spring-boot-rest-0.0.1-SNAPSHOT.jar --tus.server.max-upload-size=1024 ``` - The server will start on port `8080` with the upload endpoint exposed at: - `http://localhost:8080/test/api/upload` + The server will start on port `8080`, exposing upload endpoints for all three storage backends: + - **Disk Storage (Default)**: `http://localhost:8080/test/api/upload` + - **S3 Object Storage**: `http://localhost:8080/test-s3/api/upload` + - **Azure Blob Storage**: `http://localhost:8080/test-azure/api/upload` --- -## 3. Run the Built-In RUFH Conformity Test Suite +## 3. Python Prerequisites -The repository includes its own native Python conformity test suite located at `scripts/rufh_conformity_test.py`. It requires `pytest` and `requests`. - -### Prerequisites -Install Python dependencies if not already installed: +Install required Python test packages if not already available in your environment: ```bash pip install pytest requests ``` -### Running the Test Suite +--- + +## 4. Running the Tus v1.0.0 Conformity Test Suite + +The Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) verifies 50 specification rules covering: +- **Core Protocol (§5 & §6)**: OPTIONS feature discovery, Tus-Resumable version handshake (412 Precondition Failed), HEAD offset retrieval, cache control, PATCH data append, Content-Type enforcement, offset mismatch prevention (409 Conflict), length bounds validation, and `X-HTTP-Method-Override`. +- **Creation Extension (§7.1)**: Upload-Length validation, Upload-Defer-Length handling, defer-to-known length transition, and Upload-Metadata base64 decoding. +- **Creation With Upload Extension (§7.2)**: Single-request POST creations with payload body, partial upload chunking, and content-type enforcement. +- **Expiration Extension (§7.3)**: Upload-Expires header validation formatted in RFC 9110 HTTP datetime format. +- **Checksum Extension (§7.4)**: Upload-Checksum verification (SHA-1), 460 Checksum Mismatch handling and payload discarding, and unsupported checksum algorithm rejection (400 Bad Request). +- **Termination Extension (§7.5)**: DELETE cancellation of in-progress and completed uploads and 404/410 verification. +- **Concatenation Extension (§7.6)**: Partial upload creation, final upload creation by merging partials, offset summation, and 403 Forbidden enforcement on final upload PATCH requests. +- **Concurrency & Workflows**: Concurrent PATCH race contention prevention at identical offset and end-to-end multi-chunk upload flows. + +### Running Against All Three Storage Backends + +The test suite **MUST be run against all three supported storage backend types** (Disk, S3, and Azure Blob) to ensure cross-engine protocol compliance: + +#### Option A: Standalone Python Runner (Custom Formatted Summary) + +```bash +# 1. Disk Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + +# 2. S3 Object Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test-s3/api/upload + +# 3. Azure Blob Storage Backend +python3 scripts/tus_conformity_test.py --url http://localhost:8080/test-azure/api/upload +``` + +Or execute all three backends in a single command loop: +```bash +for endpoint in /test/api/upload /test-s3/api/upload /test-azure/api/upload; do + echo "======================================================================" + echo " Running Tus v1.0.0 Conformity Tests: http://localhost:8080$endpoint" + echo "======================================================================" + python3 scripts/tus_conformity_test.py --url "http://localhost:8080$endpoint" +done +``` + +#### Option B: PyTest Runner + +```bash +# Execute via pytest +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test-s3/api/upload +pytest scripts/tus_conformity_test.py --url http://localhost:8080/test-azure/api/upload +``` + +--- + +## 5. Running the IETF RUFH Conformity Test Suite + +The RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validates compliance with the official IETF Resumable Uploads for HTTP specification (`draft-ietf-httpbis-resumable-upload-12`) and RFC 9530 HTTP Digests. -You can execute the test suite using Python directly or via PyTest: +### Running Against All Three Storage Backends -#### Option A: Running directly with Python (Recommended for structured AI / Agent reporting) ```bash +# 1. Disk Storage Backend python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload + +# 2. S3 Object Storage Backend +python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test-s3/api/upload + +# 3. Azure Blob Storage Backend +python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test-azure/api/upload ``` -#### Option B: Running with PyTest +Or via PyTest: ```bash -pytest scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload +for endpoint in /test/api/upload /test-s3/api/upload /test-azure/api/upload; do + echo "======================================================================" + echo " Running PyTest RUFH Conformity: http://localhost:8080$endpoint" + echo "======================================================================" + pytest scripts/rufh_conformity_test.py --url "http://localhost:8080$endpoint" +done ``` --- -## 4. Understanding Test Results & AI Agent Remediation +## 6. Understanding Test Results -When executed, the script produces a structured summary report detailing: +When executed directly via Python, both conformity test scripts produce a structured summary report detailing: 1. **Total Tests Executed**: Count of total specification compliance tests run. -2. **Passed Tests**: Number of tests matching draft-12 specification requirements. -3. **Failed Tests**: Detailed list of failing tests including test method names, exact error tracebacks, expected status codes/headers, and corresponding RFC section references. -4. **104 Interim Responses**: Count of tests where `HTTP/1.1 104 Upload Resumption Supported` interim responses were detected from the server socket. +2. **Passed Tests**: Number of tests matching specification requirements. +3. **Failed Tests**: Detailed list of failing tests including test method names, exact error tracebacks, expected status codes/headers, and corresponding specification section references. -AI agents and developers can analyze the detailed failure breakdown in the script's console output to pinpoint specific compliance gaps and adjust server logic accordingly. +### Example Output: +``` +====================================================================== + Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite + Specification: https://tus.io/protocols/resumable-upload + Target Endpoint: http://localhost:8080/test/api/upload +====================================================================== +.................................................. [100%] +50 passed in 0.43s + +====================================================================== + CONFORMITY TEST RESULTS +====================================================================== + Total Tests Executed: 50 + Passed: 50 + Failed: 0 +====================================================================== + +[✓] ALL TUS V1.0.0 CONFORMITY TESTS PASSED SUCCESSFULLY! +``` + +If a test fails, the runner outputs a structured failure breakdown containing: +- Test function name and class +- Verbatim specification quote defining the required behavior +- Exact failure assertion and returned HTTP status or header values + +Use this failure breakdown to immediately diagnose compliance discrepancies and make targeted fixes in the server code. --- -## 5. Running Community (IETF Hackathon) Tests +## 7. Running Community (IETF Hackathon) Tests -Alternatively, you can also run the external community test suite from the `ietf-hackathon` repository: +Alternatively, you can also run the external community test suite from the `ietf-hackathon` repository across all three storage backends: ```bash git clone https://github.com/tus/ietf-hackathon.git cd ietf-hackathon/tests pip install -r requirements.txt + +# Run against Disk, S3, and Azure Blob endpoints pytest --url http://localhost:8080/test/api/upload +pytest --url http://localhost:8080/test-s3/api/upload +pytest --url http://localhost:8080/test-azure/api/upload ``` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 00000000..4ed5bccc --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,49 @@ +# Deployment Guide + +Checklist for running `tus-java-server` in production: choosing a backend, running several replicas and configuring proxies. + +## 1. Choose a Backend + +| Situation | Recommended setup | +|---|---| +| Single application instance, local disk | Defaults: `withStoragePath(...)` (`DiskStorageService` + `LeaseFileLockingService`). Optionally the legacy `DiskLockingService`, see [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#5-opt-out--backward-compatibility-guide). | +| Several replicas sharing a volume (NFS, EFS, Azure Files, SMB, Kubernetes `ReadWriteMany`) | Defaults; read the mount recommendations in [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#4-production-network-mount-configuration-guide). | +| Several replicas, no shared volume, AWS / S3-compatible | `S3StorageService` + `S3LockingService`, see [S3_STORAGE.md](S3_STORAGE.md). | +| Several replicas on Azure | `AzureBlobStorageService` + `AzureBlobLockingService`, see [AZURE_BLOB_STORAGE.md](AZURE_BLOB_STORAGE.md). | + +All built-in backends are lease-based and need **no Redis or database**. Custom lock providers can be added, see [LOCKING.md](LOCKING.md#4-implementing-a-custom-uploadlockingservice). + +## 2. Multi-Replica Checklist + +- Every replica must use the **same** storage and locking backend and `withUploadUri(...)`. +- Keep clocks synchronised with NTP (lease expiry depends on it). See the clock notes in [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md#35-clock-synchronization--ntp-requirement) and [S3_STORAGE.md](S3_STORAGE.md#clock-synchronization--ntp-requirement). +- Requests for one upload may land on different replicas: contention between a stalled `PATCH` and a resuming `HEAD`/`DELETE` is resolved across replicas automatically, see [LOCKING.md](LOCKING.md). +- Run `cleanup()` on a schedule (on one or all replicas, it is safe to run concurrently). +- Call `TusFileUploadService.close()` on shutdown so locks are released. +- Cloud backends: configure bucket / container lifecycle rules for abandoned objects (see [S3 lifecycle rules](S3_STORAGE.md#s3-bucket-lifecycle-rules) and [Azure lifecycle rules](AZURE_BLOB_STORAGE.md#11-operational-hardening--cost-optimization-guidance)). + +## 3. Reverse Proxies & Load Balancers + +Ensure that reverse proxies (Nginx, HAProxy, AWS ALB, Cloudflare): +1. Forward the protocol headers: Tus (`Tus-Resumable`, `Upload-Offset`, `Upload-Length`, `Upload-Metadata`, `Upload-Checksum`, `Upload-Defer-Length`, `Upload-Concat`) and RUFH (`Upload-Offset`, `Upload-Complete`, `Upload-Length`, `Upload-Limit`, `Upload-Draft`, `Content-Digest`, `Repr-Digest`). +2. Do not strip or alter `Content-Type: application/partial-upload` or `application/offset+octet-stream`. +3. Allow the HTTP methods `PATCH`, `HEAD` and `DELETE`. +4. Do not buffer whole request bodies and allow large bodies and long read timeouts. For Nginx, for example: + +```nginx +location /api/upload { + proxy_pass http://app; + proxy_request_buffering off; # stream chunks straight to the app + client_max_body_size 0; # size limits are enforced by withMaxUploadSize(...) + proxy_read_timeout 300s; + proxy_http_version 1.1; +} +``` + +5. If the proxy changes the public host or path, set an **absolute** `withUploadUri("https://upload.example.com/api/upload")` so that `Location` headers contain the public URL. + +## 4. Servlet Container Notes + +- Containers that do not decode `Transfer-Encoding: chunked` natively need `withChunkedTransferDecoding(true)`. +- HTTP 104 interim responses (RUFH) need container-specific support, see [INTERIM_RESPONSES.md](INTERIM_RESPONSES.md). +- Use `jakarta.servlet` containers (Tomcat 10+, Jetty 11+, Spring Boot 3+). The library no longer supports `javax.servlet`. diff --git a/docs/DISK_BASED_LOCKING.md b/docs/DISK_BASED_LOCKING.md index 9748ce0e..d7072478 100644 --- a/docs/DISK_BASED_LOCKING.md +++ b/docs/DISK_BASED_LOCKING.md @@ -55,7 +55,7 @@ Locks are structured as a dedicated directory containing a JSON lease metadata f ## 3. Distributed Concurrency & Contention Resolution -### 1. Lock Acquisition Flow (`tryAcquireLock`) +### 3.1 Lock Acquisition Flow (`tryAcquireLock`) To acquire or take over a lock: 1. Extract `UploadId` from the request URI. 2. Acquire sibling mutex `/locks/.mutex/` via `Files.createDirectory`. @@ -67,7 +67,7 @@ To acquire or take over a lock: - Start background heartbeat daemon (renews every $\text{leaseDuration} / 3$) and return `LeaseFileUploadLock`. 4. Release `.mutex/`. -### 2. Lock Release Flow (`lock.close()`) +### 3.2 Lock Release Flow (`lock.close()`) When an active upload completes or is aborted: 1. Stop background heartbeat daemon. 2. Acquire sibling mutex `.mutex/`. @@ -75,24 +75,24 @@ When an active upload completes or is aborted: 4. If ownership matches: delete `lease.json` and delete `.lock/`. 5. Release `.mutex/`. -### 3. Heartbeat Lease Auto-Renewal +### 3.3 Heartbeat Lease Auto-Renewal Active streaming uploads periodically renew their lease by updating `expiresAt` in `lease.json` every $\text{leaseDuration} / 3$ (default: every 10 seconds for a 30s lease). The renewal verifies `holderId` ownership under the sibling mutex to ensure it aborts if the lease was taken over after a long pause. -### 4. Lock Contention & `.stop` Signal Files +### 3.4 Lock Contention & `.stop` Signal Files When a client sends a `HEAD` or `DELETE` request to resume or cancel an upload while a stalled `PATCH` stream holds the lock: 1. The resuming server catches `UploadAlreadyLockedException` and calls `requestLockRelease(requestUri)`. 2. It interrupts any JVM-local stream and writes `/locks/.stop`. 3. A background watchdog thread on the holding replica (polling every 1.5 seconds) detects `.stop` and calls `stream.interrupt()`. -4. The stalled `PATCH` stream aborts and releases its lock. The server's 8.0-second retry budget ($40 \times 200\text{ms}$) allows the `HEAD` or `DELETE` request to acquire the lock and succeed seamlessly. +4. The stalled `PATCH` stream aborts and releases its lock. The server's 60-second retry budget ($300 \times 200\text{ms}$, configurable via `withLockWaitTimeout(Duration)`) allows the `HEAD` or `DELETE` request to acquire the lock and succeed seamlessly. -### 5. Clock Synchronization & NTP Requirement +### 3.5 Clock Synchronization & NTP Requirement Distributed lease evaluation across nodes relies on `expiresAt` timestamps written to `lease.json`. When sharing a network volume across multiple pods or virtual machines, all host nodes MUST synchronize system clocks using NTP (Network Time Protocol) or cloud time synchronization daemons (`chrony`). To absorb inevitable minor NTP drift, `LeaseFileLockingService` incorporates a built-in 2.0-second safety margin (`CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L` inherited from `AbstractLeaseLockingService`), ensuring that a node whose clock is slightly ahead will never prematurely evict a live peer's lease. Clock drift between cluster nodes should remain well within $\pm 1$ second to ensure accurate TTL lease expiration and takeover. --- ## 4. Production Network Mount Configuration Guide -### Linux / NFSv4 Mount Recommendations +### 4.1 Linux / NFSv4 Mount Recommendations For multi-replica deployments connecting to shared NFS storage (e.g. AWS EFS, Azure NetApp Files, on-premise NFS): ```bash @@ -103,7 +103,7 @@ mount -t nfs4 -o rw,hard,intr,rsize=1048576,wsize=1048576,actimeo=3 nfs-server:/ - `actimeo=3`: Attribute caching timeout of 3s ensures `.stop` signal files and lease renewals propagate rapidly across pods. - `rsize=1048576,wsize=1048576`: 1MB I/O buffers for maximum streaming append throughput. -### Windows / SMB 3.x Configuration +### 4.2 Windows / SMB 3.x Configuration For Windows Server SMB shares or Azure Files SMB: ```powershell @@ -116,16 +116,18 @@ New-SmbMapping -RemotePath "\\smb-server\uploads" -LocalPath "Z:" -Persistent $T If you are running single-node deployments on local disk and specifically require legacy OS-level `FileLock` (`fcntl` / `LockFileEx`), you can easily opt out of `LeaseFileLockingService` and restore `DiskLockingService`. -### 1. Programmatic Java Configuration (Opt-Out): +### 5.1 Programmatic Java Configuration (Opt-Out): ```java String storagePath = "/var/data/tus-uploads"; TusFileUploadService tus = new TusFileUploadService() .withUploadStorageService(new DiskStorageService(storagePath)) - .withUploadLockingService(new DiskLockingService(storagePath)); // Opt-out to legacy FileLock + .withUploadLockingService(new DiskLockingService(storagePath)) // Opt-out to legacy FileLock + .withUploadDeduplication(true) + .withThreadLocalCache(true); ``` -### 2. Spring Boot Bean Configuration (Opt-Out): +### 5.2 Spring Boot Bean Configuration (Opt-Out): ```java @Configuration public class TusConfig { @@ -135,7 +137,9 @@ public class TusConfig { return new TusFileUploadService() .withUploadUri("/files/upload") .withUploadStorageService(new DiskStorageService(storagePath)) - .withUploadLockingService(new DiskLockingService(storagePath)); // Opt-out to legacy FileLock + .withUploadLockingService(new DiskLockingService(storagePath)) // Opt-out to legacy FileLock + .withUploadDeduplication(true) + .withThreadLocalCache(true); } } ``` diff --git a/docs/INTEGRATION_GUIDE.md b/docs/INTEGRATION_GUIDE.md new file mode 100644 index 00000000..7a0550ff --- /dev/null +++ b/docs/INTEGRATION_GUIDE.md @@ -0,0 +1,302 @@ +# Integration Guide + +How to wire `tus-java-server` into your Java application, handle completed uploads and keep storage clean. For configuration options see the [Configuration Reference](CONFIGURATION.md). + +## Upload Lifecycle + +1. **Create & transfer** — the client talks to the upload endpoint; your code only forwards the request to `TusFileUploadService.process(...)`. +2. **Complete** — when the last byte arrives the upload is complete (an optional `UploadCompletionListener` fires). +3. **Consume** — your application reads the bytes (`getUploadedBytes`) and moves them to their final location. +4. **Clean up** — your application calls `deleteUpload`; a scheduled `cleanup()` removes abandoned or expired uploads. + +## 1. Receiving a Resumable Upload + +To process an upload request you have to pass the current `jakarta.servlet.http.HttpServletRequest` and `jakarta.servlet.http.HttpServletResponse` objects to `me.desair.tus.server.TusFileUploadService.process()`. Typical places where you can do this are Servlets, Filters or REST API Controllers. The same call handles both Tus 1.0.0 and IETF RUFH clients. + +The `process()` method returns the processed `UploadInfo` object on successful `POST`, `PATCH`, and `HEAD` requests (or `null` for `OPTIONS` preflight requests, `DELETE` terminations, or when an error response is generated). + +### Spring Boot 3 controller + +The Spring snippets below are complete and compile as-is (Spring Boot 3, Java 17+, `spring-boot-starter-web`); only the paths need adjusting. + +```java +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class FileUploadController { + + private final TusFileUploadService tusFileUploadService; + + public FileUploadController(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } + + @RequestMapping( + value = {"/api/upload", "/api/upload/**"}, + method = { + RequestMethod.POST, + RequestMethod.PUT, + RequestMethod.PATCH, + RequestMethod.HEAD, + RequestMethod.DELETE, + RequestMethod.OPTIONS, + RequestMethod.GET + }) + public void processUpload(HttpServletRequest request, HttpServletResponse response) + throws IOException { + tusFileUploadService.process(request, response); + } +} +``` + +> CORS headers and `OPTIONS` preflight handling are built in (the `cors` extension), so no `@CrossOrigin` annotation is needed. + +### Plain Jakarta Servlet (Tomcat, Jetty, Undertow, ...) + +```java +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; + +@WebServlet("/api/upload/*") +public class UploadServlet extends HttpServlet { + + private TusFileUploadService tus; + + @Override + public void init() { + tus = + new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true); + } + + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Override service() (not doPost/doGet) so that PATCH, HEAD and OPTIONS reach the library + tus.process(req, resp); + } + + @Override + public void destroy() { + try { + tus.close(); + } catch (IOException e) { + log("Failed to close TusFileUploadService", e); + } + } +} +``` + +Note that overriding `service(...)` (instead of `doGet`/`doPost`) is required so that `PATCH` requests reach the library. + +### Creation vs. progress ambiguity + +`process()` returns an `UploadInfo` on both creation (`POST`) and progress updates (`PATCH` / `HEAD`). When associating an upload ID with a user session or database entity on creation, check the method: + +```java +UploadInfo info = tusFileUploadService.process(request, response, ownerKey); +if (info != null && "POST".equalsIgnoreCase(request.getMethod())) { + userSessionRepository.recordUpload(ownerKey, info.getId()); +} +``` + +**Single-request upload completion**: when clients upload the entire file in a single request using the `creation-with-upload` extension or a RUFH single-request `POST`, the upload is already complete when `process()` returns, and any registered `UploadCompletionListener` will have already fired *before* `process()` returns to your controller. + +### Multi-tenancy with `ownerKey` + +`process(request, response, String ownerKey)` accepts an optional `ownerKey`. The `ownerKey` provides a hard separation between uploads of different users, groups or tenants: an upload created with one key cannot be read, queried or deleted with another. Examples of `ownerKey` values are user ID's, group names, client ID's... Use the same overloads with `ownerKey` on `getUploadInfo`, `getUploadedBytes` and `deleteUpload` when reading the upload back. + +## 2. Handling Upload Completion & Retrieving Files + +You can handle upload completion in two ways: +1. **Server-Side Callback Listener (`UploadCompletionListener`)**: Register a hook directly on `TusFileUploadService` that triggers automatically when the final chunk is uploaded. +2. **Domain API Endpoint**: Have your frontend notify your application backend domain endpoint (e.g. `POST /api/documents`) once upload completes. + +### Option A: Server-Side `UploadCompletionListener` + +Register one or more completion listeners on your `TusFileUploadService`. When an upload finishes transferring all bytes, the callback executes with the completed `UploadInfo` and the service instance. The upload lock is released *before* the listener is invoked, allowing you to safely stream or delete uploaded bytes immediately: + +```java +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import me.desair.tus.server.TusFileUploadService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class TusConfig { + + private static final Logger log = LoggerFactory.getLogger(TusConfig.class); + + @Bean(destroyMethod = "close") + public TusFileUploadService tusFileUploadService() { + return new TusFileUploadService() + .withStoragePath("/path/to/uploads") + .withUploadUri("/api/upload") + .withUploadDeduplication(true) + .withThreadLocalCache(true) + .withUploadCompletionListener( + (uploadInfo, service) -> { + // Safely retrieve or guess the file name (preventing path traversal and NPE) + String fileName = + Paths.get(uploadInfo.getFileName()).getFileName().toString(); + try { + try (InputStream is = service.getUploadedBytes(uploadInfo)) { + Files.copy(is, Paths.get("/var/data/documents", fileName)); + } // closing the stream releases the lock, which deleteUpload needs + service.deleteUpload(uploadInfo); + } catch (Exception e) { + log.error("Failed to process completed upload {}", uploadInfo.getId(), e); + } + }); + } +} +``` + +> [!WARNING] +> Never use a client-supplied `filename` directly as a path component without sanitising it (path traversal). The examples above are simplified. + +### Option B: Client-Initiated Domain Notification + +When an upload completes, the client receives the final `204 No Content` response from the Tus/RUFH protocol endpoint (`/api/upload/...`). Because Tus and RUFH are decoupled file transport protocols, your frontend application can notify your backend domain API (e.g. `POST /api/documents`) that the upload is complete and pass along the `uploadUrl`. + +> [!NOTE] +> `POST /api/documents` represents your application's domain REST endpoint, not a protocol endpoint. The Tus server itself handles file transfer (`/api/upload`), while your application endpoint coordinates business logic, database persistence, and final file consumption. + +```java +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import me.desair.tus.server.TusFileUploadService; +import me.desair.tus.server.exception.TusException; +import me.desair.tus.server.upload.UploadInfo; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class DocumentController { + + /** Request body sent by your frontend once the upload finished. */ + public record DocumentUploadRequest(String uploadUrl) {} + + private final TusFileUploadService tusFileUploadService; + + public DocumentController(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } + + @PostMapping("/api/documents") + public ResponseEntity completeDocumentUpload(@RequestBody DocumentUploadRequest request) + throws IOException, TusException { + String uploadUrl = request.uploadUrl(); + + // 1. Retrieve upload metadata (file name, length, custom client metadata) + UploadInfo info = tusFileUploadService.getUploadInfo(uploadUrl); + String fileName = Paths.get(info.getFileName()).getFileName().toString(); + + // 2. Stream the uploaded bytes to their permanent location + try (InputStream is = tusFileUploadService.getUploadedBytes(uploadUrl)) { + Files.copy(is, Paths.get("/var/data/documents", fileName)); + } + + // 3. Remove the temporary upload + tusFileUploadService.deleteUpload(uploadUrl); + return ResponseEntity.ok().build(); + } +} +``` + +> `getUploadedBytes(...)` returns a stream that holds the upload lock until it is closed. Always use try-with-resources. + +### Upload information + +Using `me.desair.tus.server.TusFileUploadService.getUploadInfo(String uploadUrl)` you can retrieve metadata about a specific upload process. This includes metadata provided by the client as well as metadata kept by the library like creation timestamp, creator ip-address list, upload length... The method `UploadInfo.getId()` returns the unique identifier of this upload encapsulated in an `UploadId` instance. The original (custom generated) identifier object of this upload can be retrieved using `UploadId.getOriginalObject()`. A URL safe string representation of the identifier is returned by `UploadId.toString()`. It is highly recommended to consult the [JavaDoc of both classes](https://tus.desair.me/). + +### Cloud Storage Native Processing (S3 & Azure Blob) + +When using cloud object storage backends, downstream services can directly obtain the raw cloud object key or blob name to perform zero-download server-side copying, background job processing, or direct cloud SDK operations: + +* **S3 / MinIO Storage**: Call `s3StorageService.getS3ObjectKey(uploadUrl)` to get the full S3 object key (e.g. `uploads/018f3a...`). For complete server-side `copyObject` examples and MinIO SDK usage, see the **[S3 Storage Guide](S3_STORAGE.md#4-post-upload-processing-gets3objectkey)**. +* **Azure Blob Storage**: Call `azureBlobStorageService.getAzureBlobName(uploadUrl)` to get the full blob name. For complete Azure SDK examples with `BlobClient`, see the **[Azure Blob Storage Guide](AZURE_BLOB_STORAGE.md#4-post-upload-processing-getazureblobname)**. + +> [!TIP] +> In Spring Boot applications, inject your `S3StorageService` or `AzureBlobStorageService` bean directly into downstream controllers or completion listeners. Avoid casting `(S3StorageService) tusFileUploadService.getUploadStorageService()` because when `withThreadLocalCache(true)` is enabled, `getUploadStorageService()` returns the `ThreadLocalCachedStorageAndLockingService` wrapper, causing a `ClassCastException`. + +## 3. Cleanup + +After having processed the uploaded bytes on the server backend (e.g. copy them to their final persistent location), it's important to clean up the (temporary) uploaded bytes. This can be done by calling `me.desair.tus.server.TusFileUploadService.deleteUpload(String uploadUri)` as shown above. This will remove the uploaded bytes and any associated upload information from the storage backend. Alternatively, a client can also remove an (in-progress) upload using the [termination extension](https://tus.io/protocols/resumable-upload.html#termination). + +Next to removing uploads after they have been completed and processed by the backend, it is also recommended to schedule a regular maintenance task to clean up any expired uploads or locks, using `me.desair.tus.server.TusFileUploadService.cleanup()` (expiration requires `withUploadExpirationPeriod(...)`): + +```java +import java.io.IOException; +import me.desair.tus.server.TusFileUploadService; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +public class CleanupJob { + + private final TusFileUploadService tusFileUploadService; + + public CleanupJob(TusFileUploadService tusFileUploadService) { + this.tusFileUploadService = tusFileUploadService; + } + + @Scheduled(fixedDelay = 600000) // every 10 minutes; requires @EnableScheduling + public void cleanupExpiredUploads() throws IOException { + tusFileUploadService.cleanup(); + } +} +``` + +## 4. Security Checklist + +The library implements the transfer protocols; **authentication and authorization are your responsibility**: + +* **Authenticate the endpoint**: protect `/api/upload/**` with your normal security layer (Spring Security, servlet filter, API gateway) *before* calling `process()`. Remember `OPTIONS` preflight requests carry no credentials. +* **Isolate tenants**: pass a user/tenant `ownerKey` to `process()`, `getUploadInfo`, `getUploadedBytes` and `deleteUpload` so uploads can't be accessed across users. +* **Custom checks**: implement a `TusExtension` (see `addTusExtension`) for rules such as allowed file types or per-user quotas. +* **Limit resource use**: set `withMaxUploadSize(...)` and `withUploadExpirationPeriod(...)`, and schedule `cleanup()`. +* **Treat metadata as untrusted**: `Upload-Metadata` (filename, content type) is client-controlled. Sanitise filenames and verify content before use. +* **Prefer HTTPS** and, for integrity, enable checksums (`Upload-Checksum` for Tus, `Content-Digest` / `Repr-Digest` for RUFH). + +## 5. Frontend Client Example (Uppy / JavaScript) + +Connect any standard Tus client (e.g. [Uppy](https://uppy.io/) or `tus-js-client`) to your backend upload endpoint: + +```javascript +import Uppy from '@uppy/core'; +import Tus from '@uppy/tus'; + +const uppy = new Uppy().use(Tus, { + endpoint: 'http://localhost:8080/api/upload', + chunkSize: 5 * 1024 * 1024 // 5MB chunk size +}); +``` + +Complete working examples: + +* [Resumable and asynchronous file upload in Spring Boot REST API with Uppy JavaScript client.](https://github.com/tomdesair/tus-java-server-spring-demo) +* [Detailed blog post by Ralph](https://golb.hplar.ch/2019/06/upload-with-tus.html) on how to use this library in [Spring Boot in combination with the Tus JavaScript client](https://github.com/ralscha/blog2019/tree/master/uploadtus). +* [Resumable and asynchronous file upload using Uppy with form submission in Dropwizard (Jetty)](https://github.com/tomdesair/tus-java-server-dropwizard-demo) + +For a curl-based walkthrough of both protocols see the [Quick Start in the README](../README.md#try-it-with-curl). diff --git a/docs/LOCKING.md b/docs/LOCKING.md index 0b946612..348bbea5 100644 --- a/docs/LOCKING.md +++ b/docs/LOCKING.md @@ -1,6 +1,6 @@ -# Upload Locking & Lock Contention Resolution +# Upload Locking, Lock Wait & Contention Resolution -This document describes why locking is necessary in the `tus-java-server` library, how the core `UploadLockingService` interface is structured, how lock contention resolution works across replicas, and where to find detailed documentation for each concrete locking mechanism implementation. +This document describes why locking is necessary in the `tus-java-server` library, how the core `UploadLockingService` interface is structured, how lock wait and contention resolution works across replicas, and where to find detailed documentation for each concrete locking mechanism implementation. --- @@ -8,14 +8,14 @@ This document describes why locking is necessary in the `tus-java-server` librar In the `tus` protocol (and IETF Resumable Uploads for HTTP specification), client uploads can be interrupted and resumed across multiple HTTP requests. Multiple concurrent requests targeting the same upload resource must be strictly prevented to avoid data corruption (such as out-of-order byte writes or overlapping file offsets). -### Stalled Uploads & Lock Contention Handling +### Stalled Uploads, Lock Wait & Contention Handling 1. **Active Streaming**: When a client sends upload bytes via a `PATCH` (or RUFH `POST`/`PATCH`) request, the server acquires an exclusive lock on that upload. 2. **Network Interruption**: If the client's network drops, the original `PATCH` connection may remain open on the server in a "half-open" state (a stalled socket read waiting for client bytes). 3. **Resume Attempt**: The client, recognizing the disconnect, attempts to resume by sending a `HEAD` request to query the current offset (or a `DELETE` request to terminate the upload). -4. **Lock Conflict**: The stalled `PATCH` request is still running on the server and holding the lock, which would block the client's `HEAD` or `DELETE` request indefinitely if not resolved. - -To solve this, `tus-java-server` includes a **lock contention resolution mechanism** where an incoming `HEAD` or `DELETE` request signals the server to interrupt the stalled `PATCH` byte stream cleanly, releasing the lock for immediate resumption. +4. **Lock Conflict & Lock Wait**: The stalled `PATCH` request is still running on the server and holding the lock. Rather than failing immediately, the server enters a **lock wait** retry loop (retrying every 200ms up to `withLockWaitTimeout(Duration)`, default 60s / 300 retries). +5. **Contention Resolution**: The waiting request calls `lockingService.requestLockRelease(requestUri)`, which signals the active upload to interrupt its input stream. +6. **Drain & Handover**: For cloud backends (S3, Azure), the interrupted upload is allotted a background drain timeout (`lockWaitTimeout - 5 seconds`, default 55s) to safely flush in-flight chunks and persist updated byte offsets before releasing the lock. Once freed, the waiting request acquires the lock and immediately returns the accurate offset to the client. --- @@ -34,9 +34,15 @@ public interface UploadLockingService { // Checks if an upload is currently locked boolean isLocked(UploadId id); - // Cleans up stale or expired locks + // Cleans up stale or expired locks (fallback sweeper) void cleanupStaleLocks() throws IOException; + // Cleans up the lock file or object for a completed, terminated, or expired upload + default void cleanupLock(UploadId id) throws IOException {} + + // Cleans up the lock file or object for a completed, terminated, or expired upload by URI + default void cleanupLock(String uploadUri) throws IOException {} + // Injects the UploadIdFactory instance used to parse upload IDs from request URIs void setIdFactory(UploadIdFactory idFactory); @@ -68,9 +74,12 @@ public interface UploadLock extends Closeable { } ``` -### Request Flow & Contention Resolution +### Request Flow, Lock Cleanup & Contention Resolution - **Stream Registration**: When a request starts streaming payload bytes, its input stream is wrapped in an `InterruptibleInputStream` and registered via `lockingService.registerInputStream(requestUri, inputStream)`. +- **Automatic Lock Cleanup on Completion**: When an upload finishes (`offset == length`), `TusFileUploadService` automatically invokes `lockingService.cleanupLock(uploadId)` to remove the lock file/blob (`.lock`) and any stop signal file/blob (`.stop`). +- **Automatic Lock Cleanup on Termination & Expiration**: When an upload is canceled via `DELETE` (204 No Content) or expired via `cleanupExpiredUploads()`, the associated lock file or blob is also automatically deleted via `lockingService.cleanupLock(uploadId)`. Actively held locks are defended and never deleted mid-stream. +- **Fallback Stale Lock Sweeper**: As a safety net, `cleanupStaleLocks()` periodically sweeps orphaned lock records (e.g. unleased Azure blobs older than 120s or stop signal markers older than 30s) left behind after hard node crashes or power outages. - **Release Request**: When a concurrent `HEAD` or `DELETE` request encounters an active lock, `TusFileUploadService` catches `UploadAlreadyLockedException` and calls `lockingService.requestLockRelease(requestUri)`. - **Stream Interruption**: - If the lock is held in the **same JVM**, `requestLockRelease` interrupts the local stream directly. @@ -84,10 +93,10 @@ public interface UploadLock extends Closeable { | Storage Backend / Environment | Locking Service Class | Key Characteristics & Architecture | Documentation File | |---|---|---|---| -| **Disk & Network Filesystems (Default)** | `LeaseFileLockingService` | Atomic sibling mutex directory (`.mutex/`), in-place expired lock takeover, TTL-based JSON lease files with heartbeat renewal, ownership fencing, and `.stop` signal files. Fully safe on NFSv3/v4, AWS EFS, SMB/CIFS, Kubernetes containers, and local disks. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | -| **Local File System (Legacy Opt-Out)** | `DiskLockingService` | OS kernel-level exclusive POSIX `FileLock` (`fcntl`) with JVM shutdown hooks and `.stop` signal files. Best for single-node deployments on local disk. | [`docs/DISK_BASED_LOCKING.md`](file:///Users/tom/projects/tus-java-server/docs/DISK_BASED_LOCKING.md) | -| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), conditional writes (`If-None-Match: *`), jittered read-after-write verification, heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/S3_STORAGE.md) | -| **Azure Blob Storage** | `AzureBlobLockingService` | Native Azure Blob Storage exclusive 30-second leases (`BlobLeaseClient`), background daemon renewal, and `.stop` signal blob polling watchdog. | [`docs/AZURE_BLOB_STORAGE.md`](file:///Users/tom/projects/tus-java-server/docs/AZURE_BLOB_STORAGE.md) | +| **Disk & Network Filesystems (Default)** | `LeaseFileLockingService` | Atomic sibling mutex directory (`.mutex/`), in-place expired lock takeover, TTL-based JSON lease files with heartbeat renewal, ownership fencing, and `.stop` signal files. Fully safe on NFSv3/v4, AWS EFS, SMB/CIFS, Kubernetes containers, and local disks. | [`docs/DISK_BASED_LOCKING.md`](DISK_BASED_LOCKING.md) | +| **Local File System (Legacy Opt-Out)** | `DiskLockingService` | OS kernel-level exclusive POSIX `FileLock` (`fcntl`) with JVM shutdown hooks and `.stop` signal files. Best for single-node deployments on local disk. | [`docs/DISK_BASED_LOCKING.md`](DISK_BASED_LOCKING.md) | +| **Amazon S3 / S3-Compatible** | `S3LockingService` | S3 object-backed TTL lease objects (`.lock`), atomic conditional writes (`If-None-Match: *`), non-CAS lock arbitration with configurable jitter backoff (`withJitter`) for backends lacking CAS (B2, Ceph, Wasabi), heartbeat renewal, and cross-pod `.stop` signal object polling watchdog. | [`docs/S3_STORAGE.md`](S3_STORAGE.md) | +| **Azure Blob Storage** | `AzureBlobLockingService` | Native Azure Blob Storage exclusive 30-second leases (`BlobLeaseClient`), background daemon renewal, and `.stop` signal blob polling watchdog. | [`docs/AZURE_BLOB_STORAGE.md`](AZURE_BLOB_STORAGE.md) | --- diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 3adbba5b..6c35441b 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -24,6 +24,8 @@ import me.desair.tus.server.TusFileUploadService; TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files") + .withUploadDeduplication(true) + .withThreadLocalCache(true) // Enabled by default: AUTO mode detects TUS_1_0_0 vs RUFH per request .withSupportedProtocolVersions(ProtocolVersion.AUTO); ``` @@ -150,43 +152,6 @@ Ensure that reverse proxies (Nginx, HAProxy, AWS ALB, Cloudflare): --- -## 6. Java Server API Migration Guide (v1.0.0-3.3 to v2.0.0) +## 6. Upgrading the Java Server Library -When upgrading backend Java applications from `1.0.0-3.3` to `2.0.0`, note the following changes and recommendations: - -### 6.1 Java 17 & Jakarta EE Baseline -- **Java 17**: `tus-java-server` 2.0.0 targets Java 17+. -- **Jakarta EE**: The library uses `jakarta.servlet.*` package imports instead of legacy `javax.servlet.*` packages, aligning with Spring Boot 3.x, Tomcat 10+, and Jetty 11+. - -### 6.2 Default Distributed Lease Locking -- In `1.0.0-3.3`, `TusFileUploadService.withStoragePath(path)` defaulted to `DiskLockingService`, which relied on OS kernel file locks (`java.nio.channels.FileLock`). -- In `2.0.0`, `withStoragePath(path)` defaults to `LeaseFileLockingService`, which uses atomic sibling mutex directories (`.mutex/`) and JSON lease files with auto-renewal. This provides container-safe, distributed locking across multi-replica pods on Kubernetes, NFS (v3/v4), AWS EFS, Azure Files, and SMB without requiring OS-level lock daemons or external coordination. -- **Opting Out**: If your application runs exclusively on local disks and you prefer kernel file locks, explicitly configure: - ```java - tusService.withUploadLockingService(new DiskLockingService(storagePath)); - ``` - -### 6.3 Jackson Dependencies Bundled in Compile Scope -- Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are now included with compile scope in `tus-java-server`. You no longer need to declare Jackson separately in your application `pom.xml` when using JSON metadata serialization (`withJsonSerialization()`) or cloud storage. - -### 6.4 Safe Lock-Holding InputStream Lifecycle -- In 2.0.0, `tusService.getUploadedBytes(...)` returns a stream that retains an exclusive upload lock until closed. This prevents concurrent `DELETE` or `PATCH` requests from corrupting data mid-stream. -- **Best Practice**: Always consume uploaded bytes within a `try-with-resources` block: - ```java - try (InputStream stream = tusService.getUploadedBytes(uploadUri)) { - // Process file data safely under lock protection - } // Lock is automatically released on close - ``` - -### 6.5 Pluggable Cloud Storage Backends -- `2.0.0` introduces native cloud object storage backends: - - **S3 Storage**: `S3StorageService`, `S3LockingService`, and `S3ConcatenationService` (AWS S3, MinIO, RustFS, Cloudflare R2, Ceph, GCS). See [docs/S3_STORAGE.md](S3_STORAGE.md). - - **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService`, and `AzureBlobConcatenationService`. See [docs/AZURE_BLOB_STORAGE.md](AZURE_BLOB_STORAGE.md). - -### 6.6 Post-Upload Listeners (`UploadCompletionListener`) -- You can now register callbacks that fire immediately upon upload completion: - ```java - tusService.withUploadCompletionListener((uploadInfo, service) -> { - log.info("Upload completed: {}", uploadInfo.getId()); - }); - ``` +Looking for the Java API changes between `1.0.0-3.3` and `2.0.0` (Java 17, Jakarta, default locking, lock-holding streams, storage backends, listeners)? See the **[Upgrade Guide](UPGRADING.md)**. diff --git a/docs/PROTOCOLS.md b/docs/PROTOCOLS.md new file mode 100644 index 00000000..707a3966 --- /dev/null +++ b/docs/PROTOCOLS.md @@ -0,0 +1,137 @@ +# Protocols: Tus 1.0.0 & IETF Resumable Uploads for HTTP (RUFH) + +`tus-java-server` serves **both** resumable upload protocols on the same endpoint: + +1. **Tus 1.0.0**: the widely-adopted [tus protocol standard](https://tus.io/). +2. **IETF Resumable Uploads for HTTP (RUFH)**: the official IETF standardization draft ([draft-ietf-httpbis-resumable-upload](https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/), implemented at **draft-12**). + +> [!WARNING] +> **Experimental Feature Disclaimer**: The IETF Resumable Uploads for HTTP (RUFH) specification (`draft-ietf-httpbis-resumable-upload`) is currently an active IETF draft. While this library implements draft-12 compliance, the RUFH protocol support should be considered **experimental** until the specification is published as an official RFC standard. + +Migrating clients from Tus to RUFH? See the [RUFH Migration Guide](MIGRATION.md). Upgrading the Java library? See the [Upgrade Guide](UPGRADING.md). + +## Configuring the Protocol Version + +Configure protocol support via `withSupportedProtocolVersions(ProtocolVersion)`: + +* `ProtocolVersion.AUTO` (Default): Automatically detects the protocol version per HTTP request based on request headers (`Tus-Resumable` header triggers Tus 1.0.0; `Upload-Complete` header or `application/partial-upload` content type triggers IETF RUFH). +* `ProtocolVersion.TUS_1_0_0`: Enforces Tus 1.0.0 handling exclusively. +* `ProtocolVersion.RUFH`: Enforces IETF Resumable Uploads for HTTP (RUFH) handling exclusively. + +## How the Protocols Work + +### Tus 1.0.0 Protocol Flow +```mermaid +sequenceDiagram + autonumber + actor Client as Frontend Client (Uppy / tus-js-client) + participant ProtocolEndpoint as Server: Tus Protocol Endpoint (/api/upload) + participant Storage as Storage Backend (Disk / S3 / Azure) + participant AppEndpoint as Application API (/api/documents) + + Note over Client,Storage: 1. Create Resumable Upload Resource + Client->>ProtocolEndpoint: POST /api/upload
(Tus-Resumable: 1.0.0, Upload-Length: 1048576, Upload-Metadata: ...) + ProtocolEndpoint->>Storage: Initialize upload metadata + ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a..., Tus-Resumable: 1.0.0) + + Note over Client,Storage: 2. Stream Data Chunk(s) + Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Content-Type: application/offset+octet-stream) + ProtocolEndpoint->>Storage: Append chunk bytes to storage + ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Tus-Resumable: 1.0.0) + + Note over Client,AppEndpoint: 3. Domain Notification & Consumption + Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") + AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) + AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) + AppEndpoint-->>Client: 200 OK (Processed) +``` + +### IETF Resumable Uploads for HTTP (RUFH) Flow +```mermaid +sequenceDiagram + autonumber + actor Client as Frontend Client (RUFH Client) + participant ProtocolEndpoint as Server: RUFH Protocol Endpoint (/api/upload) + participant Storage as Storage Backend (Disk / S3 / Azure) + participant AppEndpoint as Application API (/api/documents) + + Note over Client,Storage: 1. Create Resumable Upload Resource + Client->>ProtocolEndpoint: POST /api/upload
(Upload-Complete: ?0) + ProtocolEndpoint->>Storage: Initialize upload metadata + ProtocolEndpoint-->>Client: 201 Created
(Location: /api/upload/018f3a...) + + Note over Client,Storage: 2. Stream Data Chunk(s) + Client->>ProtocolEndpoint: PATCH /api/upload/018f3a...
(Upload-Offset: 0, Upload-Complete: ?1, Content-Type: application/partial-upload) + ProtocolEndpoint->>Storage: Append chunk bytes to storage + ProtocolEndpoint-->>Client: 204 No Content
(Upload-Offset: 1048576, Upload-Complete: ?1) + + Note over Client,AppEndpoint: 3. Domain Notification & Consumption + Client->>AppEndpoint: POST /api/documents
(uploadUrl: "/api/upload/018f3a...", fileName: "document.pdf") + AppEndpoint->>Storage: tusFileUploadService.getUploadedBytes(uploadUrl) + AppEndpoint->>Storage: tusFileUploadService.deleteUpload(uploadUrl) + AppEndpoint-->>Client: 200 OK (Processed) +``` + +## Protocol Comparison & Available Features + +| Feature / Capability | Tus 1.0.0 | IETF Resumable Uploads | +|---|---|---| +| **Auto-Detection Signal** | `Tus-Resumable: 1.0.0` header | `Upload-Complete` header or `Content-Type: application/partial-upload` | +| **Creation** | `POST` with `Upload-Length` / `Upload-Defer-Length` | `POST` with `Upload-Complete: ?0` / `?1` (or `application/partial-upload`) | +| **Append Chunks** | `PATCH` with `Upload-Offset` | `PATCH` with `Upload-Offset` & `Content-Type: application/partial-upload` | +| **Upload Status Query** | `HEAD` returns `Upload-Offset` & `Upload-Length` | `HEAD` returns `Upload-Offset` & `Upload-Complete` | +| **Offset Mismatch Error** | HTTP 409 Conflict | HTTP 409 Conflict with RFC 7807 `application/problem+json` details | +| **104 Interim Responses** | N/A | Supported (see [INTERIM_RESPONSES.md](INTERIM_RESPONSES.md)) | +| **Upload Cancellation** | `DELETE` with `Tus-Resumable: 1.0.0` | `DELETE` with `Upload-Complete: ?0` | +| **Checksum Validation** | Supported (`Checksum` extension) | Supported (based on HTTP Digests / RFC 9530) | +| **Expiration Handling** | Supported (`Upload-Expires` header) | Supported (`max-age` parameter in `Upload-Limit` header) | +| **Concatenation** | Supported (`Concatenation` extension) | N/A | +| **Download Extension** | Supported (`Download` extension) | Supported (`Download` extension; without it, `GET` requests perform offset retrieval) | + +## Protocol Extensions + +Besides the [tus core protocol](https://tus.io/protocols/resumable-upload.html#core-protocol), the library has all optional tus protocol extensions enabled by default. This means that the `Tus-Extension` header has value `creation,creation-defer-length,creation-with-upload,checksum,checksum-trailer,termination,expiration,concatenation,concatenation-unfinished`. Optionally you can also enable an unofficial `download` extension (see the [Configuration Reference](CONFIGURATION.md)). + +* [creation](https://tus.io/protocols/resumable-upload.html#creation): The creation extension allows you to create new uploads and to retrieve the upload URL for them. +* [creation-defer-length](https://tus.io/protocols/resumable-upload.html#post): You can create a new upload even if you don't know its final length at the time of creation. +* [creation-with-upload](https://tus.io/protocols/resumable-upload.html#creation): The creation-with-upload extension allows you to create the upload resource and upload initial file data in a single POST request. +* [checksum](https://tus.io/protocols/resumable-upload.html#checksum): An extension that allows you to verify data integrity of each upload (PATCH) request. +* [checksum-trailer](https://tus.io/protocols/resumable-upload.html#checksum): If the checksum hash cannot be calculated at the beginning of the upload, it may be included as a trailer HTTP header at the end of the chunked HTTP request. +* [termination](https://tus.io/protocols/resumable-upload.html#termination): Clients can terminate completed or in-progress uploads which allows the tus-java-server library to free up resources on the server. +* [expiration](https://tus.io/protocols/resumable-upload.html#expiration): You can instruct the tus-java-server library to cleanup uploads that are older than a configurable period. The expiration extension applies to both Tus 1.0.0 and IETF Resumable Uploads for HTTP (RUFH) protocols: + * **Tus 1.0.0 Protocol**: Expiration is communicated to clients via the `Upload-Expires` response header field formatted as an HTTP date-time (RFC 7231, e.g. `Upload-Expires: Wed, 25 Jun 2026 16:00:00 GMT`). + * **IETF RUFH Protocol**: Expiration is communicated to clients via the `max-age` parameter in the `Upload-Limit` response header field indicating remaining valid seconds (e.g. `Upload-Limit: max-size=1048576, max-age=3600`). +* [concatenation](https://tus.io/protocols/resumable-upload.html#concatenation): This extension can be used to concatenate multiple uploads into a single final upload enabling clients to perform parallel uploads and to upload non-contiguous chunks. +* [concatenation-unfinished](https://tus.io/protocols/resumable-upload.html#concatenation): The client is allowed send the request to concatenate partial uploads while these partial uploads are still in progress. +* [http-digests](https://datatracker.ietf.org/doc/rfc9530/): An extension implementing RFC 9530 to verify data integrity for the Resumable Uploads for HTTP (RUFH) protocol. Supported headers include `Content-Digest`, `Repr-Digest`, `Want-Content-Digest`, and `Want-Repr-Digest`. +* `download`: The (unofficial) download extension allows clients to download uploaded files using a HTTP `GET` request. You can enable this extension by calling the `withDownloadFeature()` method. +* `cors`: The (unofficial) CORS extension adds native CORS support out-of-the-box, setting CORS headers for all requests and responses, and handling preflight `OPTIONS` requests automatically. It is enabled by default, so you do **not** need `@CrossOrigin` or a separate CORS filter on your upload endpoint. Disable it with `disableTusExtension("cors")` if you manage CORS yourself. + +## HTTP Digests ([RFC 9530](https://www.rfc-editor.org/rfc/rfc9530.html)) + +The `http-digests` extension implements RFC 9530 to support data integrity checks for both individual data chunks (`Content-Digest`) and the entire file (`Repr-Digest`). + +* **Performance Disclaimer**: Calculating representation digests (`Repr-Digest`) requires streaming the entire uploaded file from disk. For extremely large files, this can introduce non-trivial I/O performance overhead on the server. To optimize, only request it via `Want-Repr-Digest` when absolutely necessary. + +## HTTP 104 Interim Responses + +The IETF draft allows the server to send `104 Upload Resumption Supported` interim responses. The Servlet API cannot emit 1xx responses natively; see the [HTTP 104 Interim Responses Guide](INTERIM_RESPONSES.md) for a Tomcat `Valve` (including Spring Boot registration). A live reference implementation using cached reflection is available in the [tus-java-server-spring-demo](https://github.com/tomdesair/tus-java-server-spring-demo) repository: [`TusInterimResponseTomcatValve`](https://github.com/tomdesair/tus-java-server-spring-demo/blob/main/spring-boot-rest/src/main/java/me/desair/spring/tus/TusInterimResponseTomcatValve.java). + +To register the Valve in Spring Boot via `TomcatServletWebServerFactory`: + +```java +@Bean +public TomcatServletWebServerFactory tomcatFactory(TusFileUploadService tusFileUploadService) { + TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory(); + factory.addContextValves(new TusInterimResponseTomcatValve(tusFileUploadService)); + return factory; +} +``` + +## Compatible Clients & Conformity Testing + +This server implementation has been tested with: +- **Tus 1.0.0 Clients & Conformity Tests**: [Uppy](https://uppy.io/), `tus-js-client`, and the built-in Tus v1.0.0 conformity test suite (`scripts/tus_conformity_test.py`) validating all core protocol mechanisms and protocol extensions (creation, creation-with-upload, checksum, termination, concatenation, and expiration) across Disk, S3, and Azure Blob backends. +- **IETF Resumable Uploads Clients & Conformity Tests**: The built-in RUFH conformity test suite (`scripts/rufh_conformity_test.py`) validating compliance with draft-12 of the RUFH specification and RFC 9530 HTTP Digests, as well as the community [RUFH conformity tests from the IETF hackathon](https://github.com/tus/ietf-hackathon). + +See the [Conformity Testing Guide](CONFORMITY_TESTING.md) for how to run the suites. The repository also contains automated integration test suites (`ITTusFileUploadService`, `RufhProtocolCreationTest`, `RufhProtocolAppendTest`, `RufhProtocolHeadTest`, `RufhProtocolCancellationTest`) validating both protocol specifications. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..f1a1b1a0 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,26 @@ +# Documentation Index + +## I want to use the library +1. [README](../README.md): what it is, quick start (Spring Boot, plain Servlet, curl) +2. [Integration Guide](INTEGRATION_GUIDE.md): `process()`, completion handling, cleanup, security, frontend clients +3. [Configuration Reference](CONFIGURATION.md): all options and storage backend choice +4. [Deployment Guide](DEPLOYMENT.md): multi-replica, reverse proxies, containers +5. [Troubleshooting](TROUBLESHOOTING.md) +6. [Architecture Guide](ARCHITECTURE.md): system structure, protocol extensions, storage SPI, and distributed locking + +## Protocols +- [Protocols](PROTOCOLS.md): Tus 1.0.0 & IETF RUFH flows, comparison, extensions, digests +- [Migrating clients to RUFH](MIGRATION.md): header mapping and request examples +- [HTTP 104 Interim Responses](INTERIM_RESPONSES.md) + +## Upgrading +- [Upgrade Guide 1.x → 2.0](UPGRADING.md) and the [CHANGELOG](../CHANGELOG.md) + +## Storage & locking +- [Locking concepts & custom lock services](LOCKING.md) +- [Disk & network storage (NFS, EFS, SMB)](DISK_BASED_LOCKING.md) +- [S3-compatible storage](S3_STORAGE.md) +- [Azure Blob Storage](AZURE_BLOB_STORAGE.md) + +## Contributors & maintainers +- [Architecture Guide](ARCHITECTURE.md), [CONTRIBUTING](../CONTRIBUTING.md), [Testing guide](TESTING.md), [Conformity testing](CONFORMITY_TESTING.md), [Release process](RELEASE.md) diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 87a18626..8dc3222b 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -29,33 +29,37 @@ Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) ### Step 2: Configure `TusFileUploadService` ```java -import io.minio.MinioClient; import me.desair.tus.server.TusFileUploadService; import me.desair.tus.server.upload.s3.S3StorageService; import me.desair.tus.server.upload.s3.S3LockingService; -// 1. Production Configuration: Load S3 parameters securely from environment variables +// Option A: Local S3-compatible backend (e.g. MinIO, RustFS, Ceph) +// The region parameter can be omitted (defaults to "local"); local backends ignore region in SigV4 validation. +S3StorageService localS3Storage = new S3StorageService( + "http://localhost:9000", "minioadmin", "minioadmin", "my-upload-bucket"); +S3LockingService localS3Locking = new S3LockingService( + "http://localhost:9000", "minioadmin", "minioadmin", "my-upload-bucket"); + +// Option B: AWS S3 with explicit region +// AWS strictly enforces region validation for bucket routing and SigV4 authentication. String endpoint = System.getenv().getOrDefault("S3_ENDPOINT", "https://s3.eu-central-1.amazonaws.com"); String bucketName = System.getenv().getOrDefault("S3_BUCKET_NAME", "my-upload-bucket"); String accessKey = System.getenv("AWS_ACCESS_KEY_ID"); String secretKey = System.getenv("AWS_SECRET_ACCESS_KEY"); -MinioClient minioClient = MinioClient.builder() - .endpoint(endpoint) - .credentials(accessKey, secretKey) - .build(); - -// 2. Instantiate S3 Storage and Distributed Locking services -S3StorageService s3StorageService = new S3StorageService(minioClient, bucketName); -S3LockingService s3LockingService = new S3LockingService(minioClient, bucketName); +S3StorageService awsS3Storage = new S3StorageService( + endpoint, "eu-central-1", accessKey, secretKey, bucketName); +S3LockingService awsS3Locking = new S3LockingService( + endpoint, "eu-central-1", accessKey, secretKey, bucketName); -// 3. Configure TusFileUploadService with S3 storage and locking +// Configure TusFileUploadService with S3 storage, locking, and request caching // Note: Automatic JVM shutdown hooks are built-in by default to terminate watchdog threads on pod exit. -// Manual call to tusService.close() or s3LockingService.close() is optional for custom container lifecycles. TusFileUploadService tusService = new TusFileUploadService() .withUploadUri("/files/upload") - .withUploadStorageService(s3StorageService) - .withUploadLockingService(s3LockingService); + .withUploadStorageService(awsS3Storage) + .withUploadLockingService(awsS3Locking) + .withUploadDeduplication(true) + .withThreadLocalCache(true); ``` --- @@ -64,11 +68,11 @@ TusFileUploadService tusService = new TusFileUploadService() > [!IMPORTANT] > **Why `ThreadLocalCachedStorageAndLockingService` is Recommended for S3**: -> By default, `TusFileUploadService` automatically wraps your custom `UploadStorageService` and `UploadLockingService` in a `ThreadLocalCachedStorageAndLockingService`. +> By default, thread-local caching is disabled (`false`). You should explicitly enable it via `.withThreadLocalCache(true)`. > > During a single HTTP request lifecycle (POST, PATCH, HEAD, DELETE), the tus server validates request headers, reads upload state, appends data, and constructs response headers. Without caching, retrieving `UploadInfo` and calculating offsets would require multiple redundant network roundtrips to S3 (`GetObject` on `.info`, `ListObjects`, `StatObject`). > -> `ThreadLocalCachedStorageAndLockingService` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces S3 network latency and cost per request. +> Enabling `withThreadLocalCache(true)` caches the `UploadInfo` in thread-local memory for the duration of a single HTTP request, releasing the cache automatically when the upload lock is closed at the end of the request. This dramatically reduces S3 network latency and cost per request. --- @@ -106,13 +110,14 @@ import io.minio.CopySource; import io.minio.GetObjectArgs; import java.io.InputStream; -S3StorageService s3Storage = (S3StorageService) tusService.getUploadStorageService(); - +// Inject or reference the concrete S3StorageService bean directly. +// (Avoid casting tusService.getUploadStorageService() when withThreadLocalCache(true) +// is enabled, as it returns ThreadLocalCachedStorageAndLockingService). String uploadUri = "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"; String ownerKey = "user-123"; // 1. Obtain full S3 key after upload completion using uploadUri and ownerKey -String s3ObjectKey = s3Storage.getS3ObjectKey(uploadUri, ownerKey); +String s3ObjectKey = awsS3Storage.getS3ObjectKey(uploadUri, ownerKey); // e.g. "uploads/24249a5b-01a4-4bf8-b67a-364273bb5a2e" // 2. Example: Storage-side processing using MinioClient (server-side object copy) @@ -139,19 +144,41 @@ try (InputStream stream = minioClient.getObject( --- -## 5. Configuring Custom S3 Endpoints (MinIO, RustFS, R2, Ceph, GCS) +## 5. Configuring Custom S3 Endpoints & Regions (AWS vs. Local S3 Backends) + +`S3StorageService`, `S3LockingService`, and `S3ConcatenationService` accept any S3 endpoint and handle regional authentication flexibly: + +- **Local S3-Compatible Backends (MinIO, RustFS, Ceph, SeaweedFS)**: + Self-hosted and local S3-compatible storage engines do not strictly validate AWS regions during Signature Version 4 (SigV4) authentication. You can omit the `region` parameter completely; it automatically defaults to `"local"`. +- **AWS S3 & Cloud Providers (AWS S3, Cloudflare R2, Google Cloud Storage)**: + AWS strictly validates the region parameter against bucket locations during SigV4 authentication and request routing. Always supply the target region (e.g., `"eu-central-1"`, `"us-east-1"`, or `"auto"` for Cloudflare R2). + +### Example 1: Local S3-Compatible Backend (Without Region) + +```java +// Omit region parameter; defaults to "local" +S3StorageService localS3Storage = + new S3StorageService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket"); + +S3LockingService localS3Locking = + new S3LockingService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket"); + +S3ConcatenationService localS3Concat = + new S3ConcatenationService("http://minio.local:9000", "minioadmin", "minioadmin", "my-bucket", localS3Storage); +``` -`S3StorageService` accepts any pre-configured `MinioClient`. To connect to an S3-compatible backend (such as local MinIO, RustFS, or Cloudflare R2), override the endpoint when building the `MinioClient`: +### Example 2: AWS S3 (With Explicit Region) ```java -import io.minio.MinioClient; +// Provide explicit AWS region (e.g. "eu-central-1", "us-east-1") +S3StorageService awsS3Storage = + new S3StorageService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket"); -MinioClient minioClient = MinioClient.builder() - .endpoint("http://minio.local:9000") - .credentials("minioadmin", "minioadmin") - .build(); +S3LockingService awsS3Locking = + new S3LockingService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket"); -S3StorageService s3Storage = new S3StorageService(minioClient, "my-bucket"); +S3ConcatenationService awsS3Concat = + new S3ConcatenationService("https://s3.eu-central-1.amazonaws.com", "eu-central-1", accessKey, secretKey, "my-bucket", awsS3Storage); ``` --- @@ -160,15 +187,29 @@ S3StorageService s3Storage = new S3StorageService(minioClient, "my-bucket"); S3 requires every part chunk of a multipart upload to be at least 5 MB (except the final part). -- **Disk Buffering**: `S3StorageService` buffers incoming bytes to local disk in chunks (default 50 MB) before uploading them to S3. -- **Incomplete Parts**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as a `/.part` object in S3. On the next `PATCH` request, this chunk is downloaded, prepended to the incoming stream, and upload proceeds seamlessly. +- **Disk Buffering & Preferred Part Size**: `S3StorageService` buffers incoming bytes to local disk in chunks of **8 MB** (`DEFAULT_PREFERRED_PART_SIZE`), matching Azure Blob Storage's block size. Preferred part size is configurable via `setPreferredPartSize(long)` (between 5 MB and 5 GB). +- **Multi-TB Support & Dynamic Part Size Auto-Calibration**: AWS S3 enforces a maximum ceiling of 10,000 parts per multipart upload. When an upload length exceeds 80 GB ($10,000 \times 8\text{ MB}$), `S3StorageService` automatically scales the part size up proportionally: + - **1 TB upload**: auto-calibrated to ~105 MB per part. + - **5 TB upload** (S3 maximum limit): auto-calibrated to ~525 MB per part. + - Guarantees the entire upload completes within 10,000 parts without exceeding S3's 5 GB maximum part limit. + - Peak local disk buffer usage remains bounded to at most $2 \times \text{optimalPartSize}$ (one receiving slot + one waiting slot). +- **Asynchronous Chunk Pipelining (`AsyncChunkUploader`)**: Instead of blocking the HTTP client thread while uploading chunks to S3, `S3StorageService` pipelines chunks using a bounded 3-slot model: + - **Slot 1 (Receiving)**: Streaming incoming bytes from the client into a local temporary file. + - **Slot 2 (Waiting)**: Holds one ready chunk on local disk buffer. + - **Slot 3 (Uploading)**: Actively uploading a chunk to S3 on a background daemon worker thread. + - When the background thread pool is fully utilized, work seamlessly falls back to the client thread without blocking (`ThreadPoolExecutor.CallerRunsPolicy` on a `SynchronousQueue`), ensuring zero queuing overhead and immediate backpressure. +- **Thread Pool Sizing**: Worker threads are managed via a shared executor, default 10 threads, configurable on `TusFileUploadService` via `.withCloudUploadThreadPoolSize(int)` or directly on `S3StorageService.setCloudUploadThreadPoolSize(int)`. +- **Incomplete Parts & Stale Buffer Protection**: If a client upload stream ends before reaching 5 MB and the upload is not complete, the sub-5MB chunk is saved as a `/.part` object in S3. On subsequent requests, an arithmetic budget guard ($\text{existingPartsTotalSize} + \text{size}(.part) == \text{length}$) verifies whether the `.part` represents a legitimate final part or an obsolete buffer from a previous attempt, preventing file corruption and size inflation. - **Configurable Temp Directory**: The temporary buffer directory can be configured in the constructor or builder: ```java Path customTempDir = Paths.get("/var/tmp/tus-buffer"); S3StorageService s3Storage = new S3StorageService( - minioClient, + endpoint, + "eu-central-1", + accessKey, + secretKey, "my-bucket", "uploads/", "metadata/", @@ -188,6 +229,14 @@ S3StorageService s3Storage = new S3StorageService( - If lock contention occurs across replicas, `S3LockingService` writes a `.stop` signal object in S3, signaling the active request on another pod to interrupt its input stream cleanly. - No external database or Redis cache is required for distributed locking. +### Lock Wait & Background Chunk Draining + +When a client resumes an interrupted transfer by sending a `HEAD` request (or cancels it with `DELETE`), the request enters a **lock wait** retry loop in `TusFileUploadService`: +- **Lock Wait Timeout (`withLockWaitTimeout(Duration)`)**: Governs how long the incoming request waits (polling every 200ms; default 60 seconds / 300 retries). +- **Contention Interruption**: The waiting pod writes a `.stop` object in S3. A background watchdog on the pod running the active `PATCH` detects this within 2 seconds and cleanly interrupts the stream. +- **Background Chunk Draining (`drainTimeout`)**: The active upload is automatically allotted up to `lockWaitTimeout - 5 seconds` (default 55 seconds) to drain and persist any in-flight 5MB+ chunks to S3 via `AsyncChunkUploader` before committing the new offset. +- **Lock Handover**: Because the drain timeout leaves a 5-second buffer before the lock wait budget expires, the active upload finishes persisting data and releases the lock well within the caller's wait window, enabling clean resumption without race conditions. + ### Why Lease Renewal is Required (`S3Lock` vs `FileBasedLock`) Understanding the architectural distinction between disk/file locking and S3 distributed locking is essential: @@ -200,7 +249,7 @@ Understanding the architectural distinction between disk/file locking and S3 dis - Removing renewal with a short TTL would cause locks to expire mid-upload during long transfers, leading to race conditions and data corruption. - Removing renewal with an infinite/static lock would mean a single pod crash (`kill -9`, node OOM) leaves behind an orphaned `.lock` object in S3, permanently deadlocking that upload ID. -### TOCTOU (Time-of-Check to Time-of-Use) Mitigation in S3 +### Lock Arbitration & TOCTOU Mitigation in S3 In stateless distributed object storage, acquiring a lock via standard `GetObject` followed by `PutObject` is vulnerable to a classic **Time-of-Check to Time-of-Use (TOCTOU)** race condition: 1. **Time of Check (TOC)**: Two contender nodes (Node A and Node B) concurrently check if a `.lock` object exists or is expired. Both observe it as available. @@ -211,15 +260,46 @@ To guarantee waterproof single-winner lock exclusivity across cloud providers an 1. **Conditional Writes (`If-None-Match: *`)**: - `PutObject` requests include the `If-None-Match: *` header. - - On Amazon S3 and compliant object storage engines, S3 atomically rejects the second write with `HTTP 412 Precondition Failed` (`PreconditionFailed`), immediately preventing concurrent overwrites. -2. **Jittered Read-After-Write Verification**: - - For S3-compatible emulators or endpoints that do not strictly enforce conditional writes on `PutObject`, the acquiring node sleeps for a brief randomized jitter (20–60ms) to allow in-flight writes to settle, then re-reads `GetObject`. + - On Amazon S3 and compliant object storage engines, S3 atomically rejects the second write with `HTTP 412 Precondition Failed` (`PreconditionFailed`), immediately preventing concurrent overwrites in a single network round-trip. +2. **Lock Arbitration on Backends Without Atomic Conditional Writes (Jittered Read-After-Write Verification)**: + - Several major S3-compatible engines do not support atomic conditional writes: + | Backend | Conditional Write (`If-None-Match: *`) Behavior | `S3LockingService` Handling | + | :--- | :--- | :--- | + | **AWS S3, Cloudflare R2, LocalStack, RustFS** | Returns `HTTP 412 Precondition Failed` | Optimistic CAS (1 network call) | + | **Backblaze B2, Ceph RGW** | Returns `HTTP 501 Not Implemented` | Auto-downgrades to non-CAS arbitration | + | **Wasabi** | Silently ignored (last-write-wins overwrite) | Layer 2 jitter arbitration or `withS3ConditionalWritesSupported(false)` | + - On backends without conditional write support, contender nodes sleep for a brief randomized jitter (default: 20–60ms) to allow competing writes to settle, then re-read `GetObject`. - If the remote `holderId` does not match its own, the node detects that it was overtaken, rejects the acquisition, and leaves the winner's lock intact. 3. **Safe Expired Lock Eviction**: - When evicting an expired lock, the lock object's expiration is verified again immediately before deletion to prevent evicting a fresh lock created by a winning peer. 4. **Owner-Safe Lock Release**: - In `S3UploadLock.close()`, the node verifies that the remote lock is still owned by its own `holderId` before deleting it. If its lease expired while the process was paused and another node took over ownership, the previous node will never delete the new owner's active lock. +### Jitter Configuration & Zero-Jitter Performance Tuning + +The jitter window can be customized via `.withJitter(minMs, maxMs)` on `S3LockingService`: + +```java +// Option A: Zero-Jitter for AWS S3 & Cloudflare R2 (Maximum Throughput) +// On backends with atomic conditional writes, jitter is not needed. +// Passing (0L, 0L) completely bypasses Thread.sleep for fastest lock acquisition: +S3LockingService s3LockingService = + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) + .withJitter(0L, 0L); + +// Option B: High-Latency or Distributed Backends (Ceph, multi-datacenter MinIO) +// High-latency backends or geo-replicated clusters may need a wider window to settle writes: +S3LockingService s3LockingService = + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) + .withJitter(50L, 200L); + +// Option C: Explicit Non-CAS Mode (e.g. Wasabi) +// Pre-checks existing locks before writing unconditionally: +S3LockingService s3LockingService = + new S3LockingService(endpoint, "eu-central-1", accessKey, secretKey, bucketName) + .withS3ConditionalWritesSupported(false); +``` + ### Clock Synchronization & NTP Requirement Distributed TTL lease evaluation relies on wall-clock timestamps (`expiresAt`). While `S3LockingService` incorporates a 2-second safety buffer (`CLOCK_SKEW_SAFETY_MARGIN_MS`) to absorb minor time deviations between pods, all cluster nodes and containers running `tus-java-server` MUST synchronize their system clocks using NTP (Network Time Protocol) or Amazon Time Sync Service (`chrony`). Avoid clock skew exceeding $\pm 1$ second between cluster nodes. @@ -262,6 +342,9 @@ The following minimal AWS IAM policy permissions are required for `S3StorageServ > [!IMPORTANT] > Make sure to change "my-upload-bucket" to the correct bucket name that you created for your uploads. +> [!NOTE] +> `S3UploadLock` and `S3StorageService` attempt batch Multi-Object Delete (`POST /?delete`) first to delete temporary chunk parts, `.lock`, and `.stop` files in a single round-trip. If batch deletion is not permitted by your IAM policy or unsupported by an S3-compatible backend, the service automatically falls back to individual single-object `DELETE` (`s3:DeleteObject`) requests. + --- ## 9. Developer Instructions: Running Local S3 Integration Tests diff --git a/docs/TESTING.md b/docs/TESTING.md index c664614c..bf55f415 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1,6 +1,32 @@ -# Manual Testing +# Testing Guide -## Setup & Running the Server +This document describes the testing strategy for `tus-java-server`, how to execute each testing tier, and step-by-step instructions for manual lock contention testing. + +--- + +## 1. The Three-Tiered Testing Strategy + +The repository follows a three-tiered testing hierarchy (see [Architecture Guide](ARCHITECTURE.md#7-three-tiered-testing-strategy) for architectural details): + +1. **Unit Tests (`*Test.java`)**: Pure offline tests using Mockito. Fast (< 2s), zero containers or network dependencies. Run with: + ```bash + mvn test -q + ``` +2. **Integration Tests (`IT*`)**: End-to-end multi-backend validation using Testcontainers (Podman or Docker). Tested across Disk, S3 (RustFS), and Azure (Azurite) in CI. Run with: + ```bash + mvn verify -q + ``` +3. **Conformity Tests (`scripts/*_conformity_test.py`)**: Socket-level Python protocol verification suites executed against running server deployments. See the [Conformity Testing Guide](CONFORMITY_TESTING.md) for full instructions: + ```bash + python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + python3 scripts/rufh_conformity_test.py --url http://localhost:8080/test/api/upload + ``` + +--- + +## 2. Manual Testing & Contention Verification + +### Setup & Running the Server Before performing the manual verification, you need to compile the library and start the Spring Boot demo server. @@ -16,7 +42,7 @@ Open the `spring-boot-rest/pom.xml` file in the sibling `tus-java-server-spring- me.desair.tus tus-java-server - 2.0.0 + 2.0.0-SNAPSHOT ``` @@ -27,7 +53,7 @@ cd ../tus-java-server-spring-demo mvn clean package java -jar spring-boot-rest/target/spring-boot-rest-0.0.1-SNAPSHOT.jar ``` -*Note:* The demo server runs locally on port `8080` and exposes the upload endpoint at `http://localhost:8080/api/upload`. +*Note:* The demo server runs locally on port `8080` and exposes the upload endpoint at `http://localhost:8080/test/api/upload`. --- @@ -70,7 +96,7 @@ curl -X POST \ -H "Upload-Length: 10000000" \ -I http://localhost:8080/test/api/upload ``` -Look for the `Location` header in the response and record the upload resource URL (e.g., `http://localhost:8080/api/upload/000003f1-a850-49de-af03-997272d834c9`). We will refer to this as `${UPLOAD_URL}`. +Look for the `Location` header in the response and record the upload resource URL (e.g., `http://localhost:8080/test/api/upload/000003f1-a850-49de-af03-997272d834c9`). We will refer to this as `${UPLOAD_URL}`. ``` export UPLOAD_URL='http://localhost:8080/test/api/upload/0330595f-1c0d-49f5-9e78-c40b1845157d' @@ -104,7 +130,7 @@ curl -X HEAD -H "Tus-Resumable: 1.0.0" -I ${UPLOAD_URL} - **First Terminal (PATCH)**: - The stalled/throttled `PATCH` upload should immediately abort with an I/O error or exit because the server terminated its input stream in response to the lock release request. - **Second Terminal (HEAD)**: - - The `HEAD` request should succeed with status `204 No Content` after a short delay (the server retries up to 40 times at 200ms intervals, an 8.0-second retry budget, while releasing the lock). + - The `HEAD` request should succeed with status `204 No Content` after a short delay (the server retries up to 300 times at 200ms intervals, a 60-second retry budget, configurable via `withLockWaitTimeout(Duration)`, while releasing the lock). - The response will contain the `Upload-Offset` header reflecting the number of bytes successfully written to disk before the stream was interrupted (e.g., `Upload-Offset: 153600`). - **Subsequent Uploads**: - You can immediately resume the upload using a new `PATCH` request starting from the offset returned in the `HEAD` response. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md new file mode 100644 index 00000000..394a3ee9 --- /dev/null +++ b/docs/TROUBLESHOOTING.md @@ -0,0 +1,22 @@ +# Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `404` for every request, or `process()` does nothing | `withUploadUri(...)` doesn't match the path the request arrives on (context path, proxy rewrite). | Make the URI match the servlet path, or use an absolute URI behind a proxy. | +| `PATCH` returns `405` / never reaches the library | Servlet overrides `doPost`/`doGet` only, or controller mapping lacks `PATCH`. | Override `service(...)` or map all of `POST, PUT, PATCH, HEAD, DELETE, OPTIONS, GET`. | +| Browser blocked by CORS | A second CORS filter conflicts, or `cors` extension disabled. | Rely on the built-in `cors` extension, or disable it (`disableTusExtension("cors")`) and configure CORS yourself. Preflight `OPTIONS` must reach `process()` unauthenticated. | +| `406 Not Acceptable` | Unexpected `Content-Type`. | Tus `PATCH` needs `application/offset+octet-stream`; RUFH needs `application/partial-upload`. Parameters such as `;charset=UTF-8` are tolerated. | +| `409 Conflict` on `PATCH` | Client `Upload-Offset` differs from the server offset (e.g. a retry after a lost response). | The client should send `HEAD`, read the current offset and resume from there. RUFH returns `application/problem+json` with `expected-offset` / `provided-offset`. | +| `413 Payload Too Large` when creating | `Upload-Length` exceeds `withMaxUploadSize(...)`. | Raise the limit or reduce the file size. | +| `423 Locked` / slow `HEAD` after pause | Another request still holds the upload lock (a stalled `PATCH`). | The library asks the lock holder to release and waits up to `withLockWaitTimeout(...)` (default 60 s). Increase the timeout if you have very slow storage. See [LOCKING.md](LOCKING.md). | +| Locks never expire across replicas | Clock skew between nodes, or storage not shared. | Fix NTP; ensure all replicas use the same backend. | +| `HTTP 500` / error in Undertow when an error is sent | Fixed in 2.0.0 (`Content-Length` conflict). | Upgrade. | +| `NoClassDefFoundError` for Jackson | Old version without bundled Jackson. | Upgrade to 2.0.0 (Jackson is bundled). | +| `ClassNotFoundException: javax.servlet...` | 2.x requires `jakarta.servlet`. | Use Tomcat 10+/Jetty 11+/Spring Boot 3+. See [UPGRADING.md](UPGRADING.md). | +| Uploaded file can't be deleted/read: "locked" | A `getUploadedBytes(...)` stream wasn't closed. | Use try-with-resources. | +| Uploads accumulate on disk or in the bucket | `cleanup()` not scheduled or no expiration period. | Set `withUploadExpirationPeriod(...)` and run `cleanup()` periodically. | +| Upload restarts from 0 behind a proxy | Proxy buffers or times out the request body. | See [DEPLOYMENT.md](DEPLOYMENT.md#3-reverse-proxies--load-balancers). | + +Backend-specific problems: [S3 troubleshooting](S3_STORAGE.md#10-troubleshooting-guide) and [Azure troubleshooting](AZURE_BLOB_STORAGE.md#8-troubleshooting-guide). + +Still stuck? Run the [conformity tests](CONFORMITY_TESTING.md) against your endpoint or [open an issue](https://github.com/tomdesair/tus-java-server/issues) with the request/response headers. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md new file mode 100644 index 00000000..bb0a05a2 --- /dev/null +++ b/docs/UPGRADING.md @@ -0,0 +1,74 @@ +# Upgrade Guide: `1.0.0-3.x` → `2.0.0` + +Everything a Java developer needs when upgrading an application from `tus-java-server` `1.0.0-3.3` (or earlier) to `2.0.0`. For the full list of changes see the [CHANGELOG](../CHANGELOG.md). To upgrade your *clients* to the IETF Resumable Uploads protocol see [MIGRATION.md](MIGRATION.md). + +## Upgrade Checklist + +| # | Change | Action required | +|---|---|---| +| 1 | Java 17+ and `jakarta.servlet` | Run on Java 17+ and a Jakarta container (Tomcat 10+, Jetty 11+, Spring Boot 3+). | +| 2 | Default locking is now `LeaseFileLockingService` | None for most; opt out explicitly if you want kernel file locks (§2). | +| 3 | `getUploadedBytes(...)` holds the lock until the stream is closed | Always use try-with-resources (§4). | +| 4 | `process(...)` now returns `UploadInfo` | None; source compatible. Use the return value if useful. | +| 5 | Download extension no longer returns `204` for unfinished uploads and omits `Tus-Resumable` | Check clients that rely on the old behaviour (§7). | +| 6 | Jackson now bundled (compile scope) | Remove redundant Jackson declarations if they conflict. | +| 7 | Dual protocol (`ProtocolVersion.AUTO` by default) | None for Tus clients; see §8 to restrict. | +| 8 | `UploadInProgressException` removed | Remove any references. | +| 9 | Absolute URLs accepted in `withUploadUri(...)` | Optional. | +| 10 | Security hardening (path traversal, CORS, permissions) | Transparent; verify custom URI handling if applicable (§9). | + +## 1. Java 17 & Jakarta EE Baseline +- **Java 17**: `tus-java-server` 2.0.0 targets Java 17+. +- **Jakarta EE**: The library uses `jakarta.servlet.*` package imports instead of legacy `javax.servlet.*` packages, aligning with Spring Boot 3.x, Tomcat 10+, and Jetty 11+. Applications still on `javax.servlet` must stay on `1.0.0-3.3` or migrate. + +## 2. Default Distributed Lease Locking +- In `1.0.0-3.3`, `TusFileUploadService.withStoragePath(path)` defaulted to `DiskLockingService`, which relied on OS kernel file locks (`java.nio.channels.FileLock`). +- In `2.0.0`, `withStoragePath(path)` defaults to `LeaseFileLockingService`, which uses atomic sibling mutex directories (`.mutex/`) and JSON lease files with auto-renewal. This provides container-safe, distributed locking across multi-replica pods on Kubernetes, NFS (v3/v4), AWS EFS, Azure Files, and SMB without requiring OS-level lock daemons or external coordination. See [DISK_BASED_LOCKING.md](DISK_BASED_LOCKING.md). +- **Opting Out**: If your application runs exclusively on local disks and you prefer kernel file locks, explicitly configure: + ```java + tusService.withUploadLockingService(new DiskLockingService(storagePath)); + ``` +- Existing uploads on disk remain readable: the stored `UploadInfo` format is unchanged (the `serialVersionUID` is preserved). + +## 3. Jackson Dependencies Bundled in Compile Scope +- Jackson dependencies (`jackson-databind`, `jackson-annotations`, `jackson-core`) are now included with compile scope in `tus-java-server`. You no longer need to declare Jackson separately in your application `pom.xml` when using JSON metadata serialization (`withJsonSerialization()`) or cloud storage. + +## 4. Safe Lock-Holding InputStream Lifecycle +- In 2.0.0, `tusService.getUploadedBytes(...)` returns a stream that retains an exclusive upload lock until closed. This prevents concurrent `DELETE` or `PATCH` requests from corrupting data mid-stream. +- **Best Practice**: Always consume uploaded bytes within a `try-with-resources` block: + ```java + try (InputStream stream = tusService.getUploadedBytes(uploadUri)) { + // Process file data safely under lock protection + } // Lock is automatically released on close + ``` +- A stream that is never closed leaves the upload locked until the lease expires. + +## 5. `process()` Return Value +`TusFileUploadService.process(...)` now returns the created or updated `UploadInfo` (or `null` on errors or `OPTIONS` preflight requests), enabling applications to track upload IDs. Note that it returns a value for both creation and progress requests, see [Creation vs. progress ambiguity](INTEGRATION_GUIDE.md#creation-vs-progress-ambiguity). + +## 6. New Features You May Want to Adopt +- **Pluggable Cloud Storage Backends** + - **S3 Storage**: `S3StorageService`, `S3LockingService`, and `S3ConcatenationService` (AWS S3, MinIO, RustFS, Cloudflare R2, Ceph, GCS). See [S3_STORAGE.md](S3_STORAGE.md). + - **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService`, and `AzureBlobConcatenationService`. See [AZURE_BLOB_STORAGE.md](AZURE_BLOB_STORAGE.md). + - **Thread-Local Caching**: Enable `.withThreadLocalCache(true)` when using cloud backends to eliminate redundant remote metadata roundtrips per HTTP request. When enabled, `tusService.getUploadStorageService()` returns a `ThreadLocalCachedStorageAndLockingService` wrapper; inject your concrete storage service bean directly rather than casting the wrapper if you need backend-specific methods like `getS3ObjectKey()` or `getAzureBlobName()`. +- **Post-Upload Listeners (`UploadCompletionListener`)**: register callbacks that fire immediately upon upload completion: + ```java + tusService.withUploadCompletionListener((uploadInfo, service) -> { + log.info("Upload completed: {}", uploadInfo.getId()); + }); + ``` +- **`withLockWaitTimeout(Duration)`**: configures how long requests wait for a contended upload lock (default 60 s). See [LOCKING.md](LOCKING.md). +- **`withJsonSerialization()`**: store `UploadInfo` as JSON instead of Java serialization. +- **Absolute upload URIs**: `withUploadUri("https://upload.example.com/files")` returns full `Location` URLs; relative paths keep working. +- **Automatic lock cleanup** on completion, termination and expiration (no action needed). + +## 7. Download Extension Behaviour Change +To support both protocols, the unofficial `download` extension no longer returns HTTP `204` for uploads that are still in progress and no longer includes the `Tus-Resumable` response header. The `UploadInProgressException` class was removed. + +## 8. Dual Protocol Support +By default `ProtocolVersion.AUTO` detects Tus 1.0.0 vs. IETF RUFH per request, so existing Tus clients keep working unchanged. Restrict it with `withSupportedProtocolVersions(...)`, see [PROTOCOLS.md](PROTOCOLS.md). + +## 9. Security Enhancements +- **Path Traversal Protection**: Request URIs and upload IDs are strictly validated to prevent directory traversal attacks. +- **CORS Origin Validation**: Prevents arbitrary Origin reflection when CORS is enabled. +- **Secure File Permissions**: Temporary files are created with restricted owner-only permissions (`0600`) via `Files.createTempFile` instead of legacy world-readable temp files. diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 00000000..62c1a95e --- /dev/null +++ b/lychee.toml @@ -0,0 +1,16 @@ +# Lychee link checker configuration, see https://lychee.cli.rs/ +# Also verify that #anchors exist in the linked Markdown files +include_fragments = "full" + +# Treat rate limiting as success; retry transient failures +accept = ["200..=299", "429"] +max_retries = 2 +timeout = 20 + +# Local examples and placeholders that are not meant to resolve +exclude = [ + '^https?://localhost', + '^https?://127\.0\.0\.1', + '^https?://upload\.example\.com', + '^https?://example\.com', +] diff --git a/scripts/rufh_conformity_test.py b/scripts/rufh_conformity_test.py index 6e4c0319..d973a588 100755 --- a/scripts/rufh_conformity_test.py +++ b/scripts/rufh_conformity_test.py @@ -1114,6 +1114,45 @@ def test_options_with_upload_complete_header(self, target_url, request): status, resp_headers, _, _ = http_request("OPTIONS", target_url, headers=headers, test_name=request.node.name) assert status in (200, 204), "OPTIONS request with Upload-Complete MUST succeed" + def test_paused_upload_head_offset_and_resume(self, target_url, request): + """ + §5 & §10.1: Resuming an Incomplete Upload After Pause. + Quote: "The client can pause the upload by withholding the rest of the content..." + Verifies that after uploading a chunk and pausing, querying HEAD returns the persisted + offset without resetting to 0, and the upload completes successfully when resumed from + that offset. + """ + upload_uri = create_partial_upload(target_url, test_name=request.node.name) + status1, resp1, _, _ = http_request( + "PATCH", + upload_uri, + headers={UPLOAD_OFFSET: "0", UPLOAD_COMPLETE: FALSE, CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD}, + body=b"0123456789" * 4, # 40 bytes + test_name=request.node.name, + ) + assert status1 in (200, 204) + assert resp1.get(UPLOAD_OFFSET) == "40" + + # Simulate pause delay + time.sleep(0.5) + + # Verify offset via HEAD after pause + status_h, h_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name) + assert status_h in (200, 204) + assert h_headers.get(UPLOAD_OFFSET) == "40", "HEAD after pause must report 40 bytes" + + # Resume and complete upload + status2, resp2, _, _ = http_request( + "PATCH", + upload_uri, + headers={UPLOAD_OFFSET: "40", UPLOAD_COMPLETE: TRUE, CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD}, + body=b"abcdefghijklmnopqrstuvwxyz" * 2, # 52 bytes + test_name=request.node.name, + ) + assert status2 in (200, 201, 204) + assert resp2.get(UPLOAD_OFFSET) == "92" + assert resp2.get(UPLOAD_COMPLETE) == TRUE + class TestHttpDigests: """Tests for HTTP Digests (RFC 9530).""" diff --git a/scripts/tus_conformity_test.py b/scripts/tus_conformity_test.py new file mode 100755 index 00000000..5e09fa7c --- /dev/null +++ b/scripts/tus_conformity_test.py @@ -0,0 +1,1285 @@ +#!/usr/bin/env python3 +""" +Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite + +This conformity test suite validates a server implementation against the +official Tus v1.0.0 Resumable Upload Protocol specification: + https://tus.io/protocols/resumable-upload + +Usage: + pytest scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload + python3 scripts/tus_conformity_test.py --url http://localhost:8080/test/api/upload +""" + +import argparse +import base64 +import email.utils +import hashlib +import os +import socket +import sys +import threading +import time +from urllib.parse import urlparse + +import pytest + +# Protocol Constants +TUS_RESUMABLE = "Tus-Resumable" +TUS_VERSION = "Tus-Version" +TUS_EXTENSION = "Tus-Extension" +TUS_MAX_SIZE = "Tus-Max-Size" +TUS_CHECKSUM_ALGORITHM = "Tus-Checksum-Algorithm" +UPLOAD_OFFSET = "Upload-Offset" +UPLOAD_LENGTH = "Upload-Length" +UPLOAD_DEFER_LENGTH = "Upload-Defer-Length" +UPLOAD_METADATA = "Upload-Metadata" +UPLOAD_CHECKSUM = "Upload-Checksum" +UPLOAD_CONCAT = "Upload-Concat" +UPLOAD_EXPIRES = "Upload-Expires" +LOCATION = "Location" +CONTENT_TYPE = "Content-Type" +CONTENT_LENGTH = "Content-Length" +CACHE_CONTROL = "Cache-Control" +X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override" +APPLICATION_OFFSET_OCTET_STREAM = "application/offset+octet-stream" +TUS_API_VERSION = "1.0.0" + + +def pytest_addoption(parser): + """Add command line options to pytest.""" + parser.addoption( + "--url", + action="store", + default="http://localhost:8080/test/api/upload", + help="Target Tus upload endpoint URL", + ) + + +@pytest.fixture(scope="session") +def target_url(request): + """Fixture providing the target upload URL.""" + try: + return request.config.getoption("--url") + except (ValueError, AttributeError): + return os.environ.get("TUS_URL", "http://localhost:8080/test/api/upload") + + +class CaseInsensitiveDict(dict): + """A case-insensitive dictionary for HTTP headers.""" + + def __init__(self, data=None, **kwargs): + super().__init__() + self._keys = {} + if data: + self.update(data) + if kwargs: + self.update(kwargs) + + def __setitem__(self, key, value): + super().__setitem__(key.lower(), value) + self._keys[key.lower()] = key + + def __getitem__(self, key): + return super().__getitem__(key.lower()) + + def __delitem__(self, key): + super().__delitem__(key.lower()) + del self._keys[key.lower()] + + def __contains__(self, key): + return super().__contains__(key.lower()) + + def get(self, key, default=None): + return super().get(key.lower(), default) + + def update(self, other=None, **kwargs): + if hasattr(other, "items"): + for k, v in other.items(): + self[k] = v + elif other: + for k, v in other: + self[k] = v + for k, v in kwargs.items(): + self[k] = v + + def items(self): + return ((self._keys[k], v) for k, v in super().items()) + + +def parse_headers(lines): + """Parse HTTP header lines into a CaseInsensitiveDict.""" + res_headers = CaseInsensitiveDict() + for line in lines: + if ":" in line: + k, v = line.split(":", 1) + res_headers[k.strip()] = v.strip() + return res_headers + + +def http_request(method, url, headers=None, body=None): + """ + Socket-based HTTP client helper for raw HTTP/1.1 request execution. + Returns (status_code, headers_dict, body_bytes). + """ + if headers is None: + headers = {} + parsed = urlparse(url) + host = parsed.hostname or "localhost" + port = parsed.port or (443 if parsed.scheme == "https" else 80) + path = parsed.path + ("?" + parsed.query if parsed.query else "") + if not path: + path = "/" + + s = socket.create_connection((host, port), timeout=10) + try: + req_headers = CaseInsensitiveDict(headers) + if "Connection" not in req_headers: + req_headers["Connection"] = "close" + + req_lines = [f"{method} {path} HTTP/1.1", f"Host: {host}:{port}"] + for k, v in req_headers.items(): + req_lines.append(f"{k}: {v}") + if body is not None and "Content-Length" not in req_headers: + body_len = len(body) if isinstance(body, bytes) else len(body.encode("utf-8")) + req_lines.append(f"Content-Length: {body_len}") + elif body is None and method in ("POST", "PATCH", "PUT") and "Content-Length" not in req_headers: + req_lines.append("Content-Length: 0") + + req_lines.append("") + req_lines.append("") + req_data = "\r\n".join(req_lines).encode("latin1") + if body: + req_data += body if isinstance(body, bytes) else body.encode("utf-8") + + s.sendall(req_data) + + # Read response data + resp_bytes = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + resp_bytes += chunk + + raw_str = resp_bytes.decode("latin1", errors="replace") + parts = raw_str.split("\r\n\r\n", 1) + headers_part = parts[0] + body_part = parts[1].encode("latin1") if len(parts) > 1 else b"" + + lines = headers_part.split("\r\n") + status_line = lines[0] + status_code = int(status_line.split()[1]) if len(status_line.split()) > 1 else 0 + + res_headers = parse_headers(lines[1:]) + return status_code, res_headers, body_part + except (socket.timeout, ConnectionRefusedError, socket.error) as e: + pytest.fail(f"HTTP request failed: {e}") + finally: + s.close() + + +def create_upload( + target_url, + upload_length="100", + defer_length=False, + metadata=None, + concat=None, + extra_headers=None, + body=None, + content_type=None, +): + """ + Helper to create an upload resource via POST. + Returns (absolute_location_url, response_headers). + """ + headers = {TUS_RESUMABLE: TUS_API_VERSION} + if defer_length: + headers[UPLOAD_DEFER_LENGTH] = "1" + elif upload_length is not None: + headers[UPLOAD_LENGTH] = str(upload_length) + + if metadata: + headers[UPLOAD_METADATA] = metadata + if concat: + headers[UPLOAD_CONCAT] = concat + if content_type: + headers[CONTENT_TYPE] = content_type + if extra_headers: + headers.update(extra_headers) + + status, resp_headers, _ = http_request("POST", target_url, headers=headers, body=body) + assert status == 201, f"Expected 201 Created for upload creation, got {status}" + + loc = resp_headers.get(LOCATION) + assert loc, "Location header MUST be returned upon 201 Created" + if not loc.startswith("http"): + parsed = urlparse(target_url) + loc = f"{parsed.scheme}://{parsed.netloc}{loc}" + return loc, resp_headers + + +class TestCoreProtocol: + """Core Protocol compliance tests (§6).""" + + def test_options_server_configuration(self, target_url): + """ + §6 OPTIONS: Gathering Server Configuration. + Quote: "A successful response indicated by the 204 No Content or 200 OK status MUST contain + the Tus-Version header. It MAY include the Tus-Extension and Tus-Max-Size headers." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204), f"OPTIONS request MUST return 200 or 204, got {status}" + assert TUS_VERSION in resp_headers, "OPTIONS response MUST contain Tus-Version header" + assert TUS_API_VERSION in resp_headers.get(TUS_VERSION, ""), "Tus-Version MUST include 1.0.0" + assert TUS_EXTENSION in resp_headers, "OPTIONS response SHOULD include Tus-Extension header" + + def test_options_ignores_tus_resumable_header(self, target_url): + """ + §6 OPTIONS: Server MUST Ignore Tus-Resumable Header on OPTIONS. + Quote: "The Client SHOULD NOT include the Tus-Resumable header in the request and the + Server MUST ignore the header." + """ + headers = {TUS_RESUMABLE: "unsupported.version.9.9.9"} + status, resp_headers, _ = http_request("OPTIONS", target_url, headers=headers) + assert status in (200, 204), "OPTIONS MUST succeed and ignore Tus-Resumable header" + assert status != 412, "Server MUST NOT respond with 412 on OPTIONS even with invalid Tus-Resumable" + + def test_unsupported_tus_version_returns_412(self, target_url): + """ + §6 Tus-Resumable: Protocol Version Mismatch Returns 412 Precondition Failed. + Quote: "If the version specified by the Client is not supported by the Server, it MUST + respond with the 412 Precondition Failed status and MUST include the Tus-Version header + into the response. In addition, the Server MUST NOT process the request." + """ + headers = { + TUS_RESUMABLE: "0.1.0", + UPLOAD_LENGTH: "100", + } + status, resp_headers, _ = http_request("POST", target_url, headers=headers) + assert status == 412, f"Unsupported version MUST be rejected with 412, got {status}" + assert TUS_VERSION in resp_headers, "412 response MUST include Tus-Version header" + + def test_missing_tus_resumable_header_rejected(self, target_url): + """ + §6 Tus-Resumable: Header Required in Requests. + Quote: "The Tus-Resumable header MUST be included in every request and response except + for OPTIONS requests." + """ + headers = {UPLOAD_LENGTH: "100"} + status, resp_headers, _ = http_request("POST", target_url, headers=headers) + assert status in (400, 412), f"Missing Tus-Resumable header MUST be rejected, got {status}" + + def test_tus_resumable_present_in_responses(self, target_url): + """ + §6 Tus-Resumable: Header Present in Every Response. + Quote: "The Tus-Resumable header MUST be included in every request and response except + for OPTIONS requests. The value MUST be the version of the protocol used by the Client or Server." + """ + upload_url, resp_headers = create_upload(target_url, upload_length="100") + assert resp_headers.get(TUS_RESUMABLE) == TUS_API_VERSION, "POST response MUST contain Tus-Resumable: 1.0.0" + + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(TUS_RESUMABLE) == TUS_API_VERSION, "HEAD response MUST contain Tus-Resumable: 1.0.0" + + def test_head_requires_upload_offset_even_if_zero(self, target_url): + """ + §6 HEAD: Server MUST Always Include Upload-Offset. + Quote: "The Server MUST always include the Upload-Offset header in the response for a HEAD + request, even if the offset is 0, or the upload is already considered completed." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204), f"HEAD request MUST succeed with 200 or 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST be present and 0 initially" + + def test_head_requires_upload_length_when_known(self, target_url): + """ + §6 HEAD: Server MUST Include Upload-Length When Known. + Quote: "If the size of the upload is known, the Server MUST include the Upload-Length + header in the response." + """ + upload_url, _ = create_upload(target_url, upload_length="256") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert resp_headers.get(UPLOAD_LENGTH) == "256", "HEAD response MUST include Upload-Length" + + def test_head_cache_control_no_store(self, target_url): + """ + §6 HEAD: Cache Prevention via Cache-Control: no-store. + Quote: "The Server MUST prevent the client and/or proxies from caching the response by + adding the Cache-Control: no-store header to the response." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + status, resp_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + cache_ctrl = resp_headers.get(CACHE_CONTROL, "") + assert "no-store" in cache_ctrl, f"HEAD response MUST include Cache-Control: no-store, got '{cache_ctrl}'" + + def test_head_non_existent_upload_resource(self, target_url): + """ + §6 HEAD: Non-Existent Resource Returns 404, 410, or 403 Without Upload-Offset. + Quote: "If the resource is not found, the Server SHOULD return either the 404 Not Found, + 410 Gone or 403 Forbidden status without the Upload-Offset header." + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + status, resp_headers, _ = http_request("HEAD", non_existent_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (404, 410, 403), f"HEAD on non-existent resource SHOULD return 404, 410, or 403, got {status}" + assert UPLOAD_OFFSET not in resp_headers, "Non-existent HEAD response MUST NOT include Upload-Offset" + + def test_patch_content_type_required(self, target_url): + """ + §6 PATCH: Content-Type MUST Be application/offset+octet-stream. + Quote: "All PATCH requests MUST use Content-Type: application/offset+octet-stream, + otherwise the server SHOULD return a 415 Unsupported Media Type status." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: "text/plain", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 10) + assert status in (400, 406, 415), f"PATCH with incorrect Content-Type MUST be rejected, got {status}" + + def test_patch_offset_mismatch_returns_409(self, target_url): + """ + §6 PATCH: Mismatching Upload-Offset MUST Return 409 Conflict. + Quote: "The Upload-Offset header’s value MUST be equal to the current offset of the resource... + If the offsets do not match, the Server MUST respond with the 409 Conflict status without + modifying the upload resource." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "50", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 10) + assert status == 409, f"PATCH with mismatching offset MUST return 409 Conflict, got {status}" + + # Verify upload resource was not modified + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be modified after 409 Conflict" + + def test_patch_successful_append(self, target_url): + """ + §6 PATCH: Successful Append Returns 204 No Content With New Upload-Offset. + Quote: "The Server MUST acknowledge successful PATCH requests with the 204 No Content status. + It MUST include the Upload-Offset header containing the new offset. The new offset MUST be + the sum of the offset before the PATCH request and the number of bytes received and processed." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + chunk = b"A" * 40 + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, resp_headers, _ = http_request("PATCH", upload_url, headers=headers, body=chunk) + assert status == 204, f"PATCH MUST return 204 No Content on success, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "40", "Upload-Offset in response MUST be updated to 40" + + # Verify via HEAD + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "40", "HEAD MUST report new offset of 40" + + def test_patch_non_existent_resource(self, target_url): + """ + §6 PATCH: Non-Existent Resource Returns 404 Not Found. + Quote: "If the server receives a PATCH request against a non-existent resource it SHOULD + return a 404 Not Found status." + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", non_existent_url, headers=headers, body=b"A" * 10) + assert status in (404, 410), f"PATCH against non-existent upload SHOULD return 404, got {status}" + + def test_patch_exceeding_upload_length_rejected(self, target_url): + """ + §6 PATCH: Appending Beyond Declared Upload-Length Must Be Rejected. + """ + upload_url, _ = create_upload(target_url, upload_length="30") + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 50) + assert status in (400, 409, 413), f"PATCH exceeding Upload-Length MUST be rejected, got {status}" + + def test_x_http_method_override(self, target_url): + """ + §6 X-HTTP-Method-Override: Request Method Override. + Quote: "The X-HTTP-Method-Override request header MUST be a string which MUST be interpreted + as the request’s method by the Server, if the header is presented. The actual method of the + request MUST be ignored." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + + # Test overriding POST to PATCH + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + X_HTTP_METHOD_OVERRIDE: "PATCH", + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, resp_headers, _ = http_request("POST", upload_url, headers=headers, body=b"A" * 40) + assert status == 204, f"Overridden PATCH request MUST succeed with 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == "40" + + # Test overriding POST to HEAD + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + X_HTTP_METHOD_OVERRIDE: "HEAD", + } + status, head_headers, _ = http_request("POST", upload_url, headers=headers) + assert status in (200, 204), f"Overridden HEAD request MUST succeed with 200 or 204, got {status}" + assert head_headers.get(UPLOAD_OFFSET) == "40" + + +class TestCreationExtension: + """Creation Extension compliance tests (§7.1).""" + + def test_creation_extension_advertised(self, target_url): + """ + §7.1 Creation: Extension Advertisement in OPTIONS. + Quote: "If the Server supports this extension, it MUST add creation to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "creation" in extensions, "Tus-Extension header MUST include 'creation'" + + def test_create_upload_empty_post(self, target_url): + """ + §7.1 Creation: Empty POST Creation Request. + Quote: "An empty POST request is used to create a new upload resource. The Upload-Length + header indicates the size of the entire upload in bytes... The Server MUST acknowledge a + successful upload creation with the 201 Created status. The Server MUST set the Location + header to the URL of the created resource." + """ + upload_url, resp_headers = create_upload(target_url, upload_length="100") + assert upload_url, "Location header pointing to created resource MUST be returned" + assert resp_headers.get(TUS_RESUMABLE) == TUS_API_VERSION + + def test_create_zero_byte_upload(self, target_url): + """ + §7.1 Creation: Zero-Byte Upload Creation. + Quote: "The Upload-Length header MAY be set to 0, indicating that the Client wants to + upload an empty file. Such an upload is immediately complete after its creation without + transferring data using PATCH requests." + """ + upload_url, _ = create_upload(target_url, upload_length="0") + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST be 0 for zero-byte upload" + assert head_headers.get(UPLOAD_LENGTH) == "0", "Upload-Length MUST be 0 for zero-byte upload" + + def test_create_upload_defer_length(self, target_url): + """ + §7.1 Creation: Deferred Length Creation. + Quote: "Upload-Defer-Length: 1 if upload size is not known at the time... As long as the + length of the upload is not known, the Server MUST set Upload-Defer-Length: 1 in all + responses to HEAD requests. If the length was deferred using Upload-Defer-Length: 1, the + Client MUST set the Upload-Length header in the next PATCH request, once the length is known." + """ + upload_url, _ = create_upload(target_url, defer_length=True) + + # Initial HEAD should return Upload-Defer-Length: 1 and no Upload-Length + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_DEFER_LENGTH) == "1", "HEAD MUST include Upload-Defer-Length: 1 when length deferred" + assert UPLOAD_LENGTH not in head_headers, "Upload-Length MUST NOT be present while deferred" + + # PATCH providing length + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + UPLOAD_LENGTH: "80", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"B" * 40) + assert status == 204, f"PATCH providing deferred Upload-Length MUST succeed with 204, got {status}" + + # Subsequent HEAD should now report Upload-Length: 80 and no Upload-Defer-Length + status, head_headers2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers2.get(UPLOAD_LENGTH) == "80", "HEAD MUST now report Upload-Length: 80" + assert UPLOAD_DEFER_LENGTH not in head_headers2, "Upload-Defer-Length MUST be omitted once length is known" + + def test_create_upload_invalid_defer_length_value(self, target_url): + """ + §7.1 Creation: Invalid Upload-Defer-Length Value Returns 400 Bad Request. + Quote: "If the Upload-Defer-Length header contains any other value than 1 the server + should return a 400 Bad Request status." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_DEFER_LENGTH: "2", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Upload-Defer-Length != 1 MUST return 400 Bad Request, got {status}" + + def test_create_upload_missing_both_length_and_defer_length(self, target_url): + """ + §7.1 Creation: POST Missing Both Upload-Length and Upload-Defer-Length. + Quote: "The request MUST include one of the following headers: a) Upload-Length ... + b) Upload-Defer-Length: 1" + """ + headers = {TUS_RESUMABLE: TUS_API_VERSION} + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"POST without length headers MUST return 400 Bad Request, got {status}" + + def test_create_upload_negative_length(self, target_url): + """ + §6 Upload-Length: Value MUST Be Non-Negative Integer. + Quote: "The Upload-Length request and response header indicates the size of the entire + upload in bytes. The value MUST be a non-negative integer." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "-10", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Negative Upload-Length MUST return 400 Bad Request, got {status}" + + def test_create_upload_non_integer_length(self, target_url): + """ + §6 Upload-Length: Non-Integer Value Rejected. + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "not_a_number", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Non-integer Upload-Length MUST return 400 Bad Request, got {status}" + + def test_create_upload_with_metadata(self, target_url): + """ + §7.1 Creation: Upload-Metadata Header Preservation. + Quote: "The Client MAY supply the Upload-Metadata header to add additional metadata to the + upload creation request... If an upload contains additional metadata, responses to HEAD + requests MUST include the Upload-Metadata header and its value as specified by the Client." + """ + meta_filename = base64.b64encode(b"report.pdf").decode("ascii") + meta_author = base64.b64encode(b"Jane Doe").decode("ascii") + metadata = f"filename {meta_filename},author {meta_author}" + + upload_url, _ = create_upload(target_url, upload_length="100", metadata=metadata) + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + resp_meta = head_headers.get(UPLOAD_METADATA, "") + assert f"filename {meta_filename}" in resp_meta, "Upload-Metadata in HEAD MUST contain filename" + assert f"author {meta_author}" in resp_meta, "Upload-Metadata in HEAD MUST contain author" + + def test_create_upload_metadata_empty_value(self, target_url): + """ + §7.1 Creation: Upload-Metadata with Empty Value. + Quote: "The value MAY be empty. In these cases, the space, which would normally separate + the key and the value, MAY be left out." + """ + metadata = "is_confidential" + upload_url, _ = create_upload(target_url, upload_length="100", metadata=metadata) + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert "is_confidential" in head_headers.get(UPLOAD_METADATA, ""), "Upload-Metadata in HEAD MUST contain empty-value key" + + def test_create_upload_exceeding_tus_max_size(self, target_url): + """ + §7.1 Creation: Upload Exceeding Tus-Max-Size Returns 413 Request Entity Too Large. + Quote: "If the length of the upload exceeds the maximum, which MAY be specified using + the Tus-Max-Size header, the Server MUST respond with the 413 Request Entity Too Large status." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + max_size_str = resp_headers.get(TUS_MAX_SIZE) + if not max_size_str: + pytest.skip("Server does not advertise Tus-Max-Size in OPTIONS response") + + max_size = int(max_size_str) + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: str(max_size + 1), + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 413, f"Upload exceeding Tus-Max-Size ({max_size}) MUST return 413, got {status}" + + def test_deferred_length_cannot_be_changed_once_set(self, target_url): + """ + §7.1 Creation: Deferred Length Cannot Be Changed Once Set. + Quote: "Once set the length MUST NOT be changed." + """ + upload_url, _ = create_upload(target_url, defer_length=True) + + # Set length to 100 + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + UPLOAD_LENGTH: "100", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 40) + assert status == 204 + + # Attempt to change length to 120 in subsequent PATCH + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "40", + UPLOAD_LENGTH: "120", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status2, _, _ = http_request("PATCH", upload_url, headers=headers2, body=b"B" * 40) + assert status2 in (400, 409), f"Attempting to modify already set Upload-Length MUST be rejected, got {status2}" + + +class TestCreationWithUploadExtension: + """Creation With Upload Extension compliance tests (§7.2).""" + + def test_creation_with_upload_advertised(self, target_url): + """ + §7.2 Creation With Upload: Extension Advertisement. + Quote: "If the Server supports this extension, it MUST advertise this by including + creation-with-upload in the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "creation-with-upload" in extensions, "Tus-Extension MUST include 'creation-with-upload'" + + def test_creation_with_full_upload(self, target_url): + """ + §7.2 Creation With Upload: Initial POST Containing Entire File Payload. + Quote: "The Client MAY include either the entirety or a chunk of the upload data in the + body of the POST request... The Server SHOULD accept as many bytes as possible and MUST + include the Upload-Offset header in the response and MUST set its value to the offset of + the upload after applying the accepted bytes." + """ + payload = b"Hello, Full Tus Creation With Upload!" + upload_url, resp_headers = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)), "Response MUST return Upload-Offset equal to body length" + + # Verify via HEAD + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == str(len(payload)) + assert head_headers.get(UPLOAD_LENGTH) == str(len(payload)) + + def test_creation_with_partial_upload_then_patch(self, target_url): + """ + §7.2 Creation With Upload: Initial POST With Partial Payload Followed by PATCH. + Quote: "The Server SHOULD accept as many bytes as possible and MUST include the + Upload-Offset header in the response... The Client MUST perform the actual upload + using the core protocol." + """ + part1 = b"Part 1 Data with at least 32 bytes!!" + part2 = b"Part 2 Data with at least 32 bytes!!" + total_length = len(part1) + len(part2) + + upload_url, resp_headers = create_upload( + target_url, + upload_length=str(total_length), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part1, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(part1)) + + # Complete upload using PATCH + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: str(len(part1)), + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, patch_headers, _ = http_request("PATCH", upload_url, headers=headers, body=part2) + assert status == 204 + assert patch_headers.get(UPLOAD_OFFSET) == str(total_length) + + # Final verification via HEAD + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == str(total_length) + + def test_creation_with_upload_and_deferred_length(self, target_url): + """ + §7.2 Creation With Upload: Creation With Upload Combined With Deferred Length. + """ + payload = b"Initial chunk with deferred length" + upload_url, resp_headers = create_upload( + target_url, + defer_length=True, + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + # HEAD verification + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == str(len(payload)) + assert head_headers.get(UPLOAD_DEFER_LENGTH) == "1" + + def test_creation_with_upload_invalid_content_type(self, target_url): + """ + §7.2 Creation With Upload: Non-Empty POST With Invalid Content-Type Must Be Rejected. + Quote: "The Client MUST include the Content-Type: application/offset+octet-stream header." + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "30", + CONTENT_TYPE: "text/plain", + } + status, _, _ = http_request("POST", target_url, headers=headers, body=b"A" * 20) + assert status in (400, 406, 415), f"Non-empty POST with invalid Content-Type MUST be rejected, got {status}" + + def test_creation_with_upload_exceeds_upload_length(self, target_url): + """ + §7.2 Creation With Upload: Body Exceeding Declared Upload-Length Must Be Rejected. + """ + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_LENGTH: "10", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("POST", target_url, headers=headers, body=b"A" * 20) + assert status in (400, 413), f"Body exceeding declared length MUST be rejected, got {status}" + + +class TestChecksumExtension: + """Checksum Extension compliance tests (§7.4).""" + + def test_checksum_extension_advertised(self, target_url): + """ + §7.4 Checksum: Extension and Algorithms Advertisement. + Quote: "If supported, the Server MUST add checksum to the Tus-Extension header. + The Tus-Checksum-Algorithm header MUST be included in the response to an OPTIONS request. + The Server MUST support at least the SHA1 checksum algorithm identified by sha1." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "checksum" in extensions, "Tus-Extension header MUST include 'checksum'" + algorithms = [a.strip() for a in resp_headers.get(TUS_CHECKSUM_ALGORITHM, "").split(",")] + assert "sha1" in algorithms, "Tus-Checksum-Algorithm header MUST support 'sha1'" + + def test_patch_with_valid_sha1_checksum(self, target_url): + """ + §7.4 Checksum: Valid SHA1 Checksum Verification. + Quote: "A Client MAY include the Upload-Checksum header in a PATCH request. Once the entire + request has been received, the Server MUST verify the uploaded chunk against the provided + checksum using the specified algorithm... 3. 204 No Content if the checksums match and + the processing of the data succeeded." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Tus SHA1 checksum verified payload" + digest_bytes = hashlib.sha1(payload).digest() + b64_digest = base64.b64encode(digest_bytes).decode("ascii") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: f"sha1 {b64_digest}", + } + status, resp_headers, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 204, f"PATCH with valid checksum MUST succeed with 204, got {status}" + assert resp_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + def test_patch_with_invalid_checksum_mismatch(self, target_url): + """ + §7.4 Checksum: Checksum Mismatch Returns 460 Checksum Mismatch. + Quote: "2. 460 Checksum Mismatch if the checksums mismatch... In the first two cases the + uploaded chunk MUST be discarded, and the upload and its offset MUST NOT be updated." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Tus SHA1 payload with at least 32 bytes in length" + invalid_digest = base64.b64encode(b"0" * 20).decode("ascii") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: f"sha1 {invalid_digest}", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 460, f"Checksum mismatch MUST return 460 Checksum Mismatch, got {status}" + + # Verify chunk was discarded and offset remains 0 + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be updated after checksum mismatch" + + def test_patch_with_unsupported_checksum_algorithm(self, target_url): + """ + §7.4 Checksum: Unsupported Checksum Algorithm Returns 400 Bad Request. + Quote: "1. 400 Bad Request if the checksum algorithm is not supported by the server... + the uploaded chunk MUST be discarded, and the upload and its offset MUST NOT be updated." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + payload = b"Payload with unknown checksum algorithm at least 32 bytes" + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + UPLOAD_CHECKSUM: "unknown_algo AAAA", + } + status, _, _ = http_request("PATCH", upload_url, headers=headers, body=payload) + assert status == 400, f"Unsupported checksum algorithm MUST return 400 Bad Request, got {status}" + + # Verify chunk was discarded + _, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_headers.get(UPLOAD_OFFSET) == "0", "Upload-Offset MUST NOT be updated after invalid algorithm" + + +class TestTerminationExtension: + """Termination Extension compliance tests (§7.5).""" + + def test_termination_extension_advertised(self, target_url): + """ + §7.5 Termination: Extension Advertisement. + Quote: "If this extension is supported by the Server, it MUST be announced by adding + termination to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "termination" in extensions, "Tus-Extension header MUST include 'termination'" + + def test_delete_existing_upload(self, target_url): + """ + §7.5 Termination: Deletion of Existing Upload. + Quote: "When receiving a DELETE request for an existing upload the Server SHOULD free + associated resources and MUST respond with the 204 No Content status confirming that the + upload was terminated. For all future requests to this URL, the Server SHOULD respond with + the 404 Not Found or 410 Gone status." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + + # Terminate + status, _, _ = http_request("DELETE", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status == 204, f"DELETE on existing upload MUST return 204 No Content, got {status}" + + # Future HEAD request MUST return 404 or 410 + head_status, _, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_status in (404, 410), f"HEAD after DELETE MUST return 404 or 410, got {head_status}" + + # Future PATCH request MUST return 404 or 410 + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + patch_status, _, _ = http_request("PATCH", upload_url, headers=patch_headers, body=b"A" * 10) + assert patch_status in (404, 410), f"PATCH after DELETE MUST return 404 or 410, got {patch_status}" + + def test_delete_non_existent_upload(self, target_url): + """ + §7.5 Termination: DELETE on Non-Existent Resource Returns 404 Not Found. + """ + parsed = urlparse(target_url) + non_existent_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}/definitely-nonexistent-id-99999" + status, _, _ = http_request("DELETE", non_existent_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (404, 410), f"DELETE on non-existent upload SHOULD return 404, got {status}" + + def test_delete_completed_upload(self, target_url): + """ + §7.5 Termination: Termination of Completed Upload. + Quote: "This extension defines a way for the Client to terminate completed and unfinished + uploads allowing the Server to free up used resources." + """ + payload = b"Terminating completed upload test payload" + upload_url, _ = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=payload, + ) + + status, _, _ = http_request("DELETE", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status == 204, f"DELETE on completed upload MUST return 204 No Content, got {status}" + + head_status, _, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_status in (404, 410), f"HEAD on deleted completed upload MUST return 404 or 410, got {head_status}" + + +class TestConcatenationExtension: + """Concatenation Extension compliance tests (§7.6).""" + + def test_concatenation_extension_advertised(self, target_url): + """ + §7.6 Concatenation: Extension Advertisement. + Quote: "If the Server supports this extension, it MUST add concatenation to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "concatenation" in extensions, "Tus-Extension header MUST include 'concatenation'" + + def test_partial_upload_creation_and_head(self, target_url): + """ + §7.6 Concatenation: Partial Upload Creation and HEAD Response. + Quote: "A partial upload represents a chunk of a file. It is constructed by including the + Upload-Concat: partial header while creating a new upload using the Creation extension... + The response to a HEAD request for a partial upload MUST contain the Upload-Offset header. + Response to HEAD request against partial or final upload MUST include the Upload-Concat + header and its value as received in the upload creation request." + """ + upload_url, _ = create_upload(target_url, upload_length="40", concat="partial") + status, head_headers, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == "0" + assert head_headers.get(UPLOAD_CONCAT) == "partial", "HEAD response MUST include Upload-Concat: partial" + + def test_final_upload_concatenation_success(self, target_url): + """ + §7.6 Concatenation: Final Concatenation of Partial Uploads. + Quote: "In order to create a new final upload, the Client MUST add the Upload-Concat header + to the upload creation request. The value MUST be final followed by a semicolon and a + space-separated list of the partial upload URLs that need to be concatenated... The length of + the final upload MUST be the sum of the length of all partial uploads. After successful + concatenation, the Upload-Offset and Upload-Length MUST be set and their values MUST be equal." + """ + # Create and fill partial 1 + part1_data = b"Hello, Concatenated World Part 1!" + part1_url, _ = create_upload( + target_url, + upload_length=str(len(part1_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part1_data, + ) + + # Create and fill partial 2 + part2_data = b"Hello, Concatenated World Part 2!" + part2_url, _ = create_upload( + target_url, + upload_length=str(len(part2_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part2_data, + ) + + total_length = len(part1_data) + len(part2_data) + + # Create final concatenated upload + concat_val = f"final;{part1_url} {part2_url}" + final_url, _ = create_upload(target_url, upload_length=None, concat=concat_val) + + # Verify final upload HEAD + status, head_headers, _ = http_request("HEAD", final_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status in (200, 204) + assert head_headers.get(UPLOAD_OFFSET) == str(total_length), "Upload-Offset MUST equal total combined length" + assert head_headers.get(UPLOAD_LENGTH) == str(total_length), "Upload-Length MUST equal total combined length" + assert head_headers.get(UPLOAD_CONCAT) == concat_val, "Upload-Concat MUST match final concatenation specification" + + def test_final_creation_must_not_include_upload_length(self, target_url): + """ + §7.6 Concatenation: Final Upload Creation MUST NOT Include Upload-Length. + Quote: "The Client MUST NOT include the Upload-Length header in the final upload creation." + """ + # Create a partial upload + part_url, _ = create_upload(target_url, upload_length="20", concat="partial") + + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_CONCAT: f"final;{part_url}", + UPLOAD_LENGTH: "20", + } + status, _, _ = http_request("POST", target_url, headers=headers) + assert status == 400, f"Final creation including Upload-Length MUST be rejected with 400, got {status}" + + def test_patch_against_final_upload_forbidden(self, target_url): + """ + §7.6 Concatenation: PATCH on Final Upload Returns 403 Forbidden. + Quote: "The Server MUST respond with the 403 Forbidden status to PATCH requests against + a final upload URL and MUST NOT modify the final or its partial uploads." + """ + part_data = b"Fixed Part Data with at least 32 bytes!" + part_url, _ = create_upload( + target_url, + upload_length=str(len(part_data)), + concat="partial", + content_type=APPLICATION_OFFSET_OCTET_STREAM, + body=part_data, + ) + final_url, _ = create_upload(target_url, upload_length=None, concat=f"final;{part_url}") + + # Retrieve current offset of final upload via HEAD + _, head_headers, _ = http_request("HEAD", final_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + offset = head_headers.get(UPLOAD_OFFSET, str(len(part_data))) + + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: offset, + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, _, _ = http_request("PATCH", final_url, headers=patch_headers, body=b"") + assert status == 403, f"PATCH against final upload MUST return 403 Forbidden, got {status}" + + +class TestExpirationExtension: + """Expiration Extension compliance tests (§7.3).""" + + def test_expiration_extension_advertised(self, target_url): + """ + §7.3 Expiration: Extension Advertisement. + Quote: "In order to indicate this behavior to the Client, the Server MUST add expiration + to the Tus-Extension header." + """ + status, resp_headers, _ = http_request("OPTIONS", target_url) + assert status in (200, 204) + extensions = [e.strip() for e in resp_headers.get(TUS_EXTENSION, "").split(",")] + assert "expiration" in extensions, "Tus-Extension header MUST include 'expiration'" + + def test_upload_expires_header_format(self, target_url): + """ + §7.3 Expiration: Upload-Expires Header in Datetime Format. + Quote: "The Upload-Expires response header indicates the time after which the unfinished + upload expires... This header MUST be included in every PATCH response if the upload is + going to expire. If the expiration is known at the creation, the Upload-Expires header MUST + be included in the response to the initial POST request... The value of the Upload-Expires + header MUST be in RFC 9110 datetime format." + """ + upload_url, post_headers = create_upload(target_url, upload_length="100") + expires_header = post_headers.get(UPLOAD_EXPIRES) + + if not expires_header: + # Perform a PATCH to observe Upload-Expires on intermediate append + patch_headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + status, patch_resp_headers, _ = http_request("PATCH", upload_url, headers=patch_headers, body=b"A" * 40) + assert status == 204 + expires_header = patch_resp_headers.get(UPLOAD_EXPIRES) + + if expires_header: + dt = email.utils.parsedate_to_datetime(expires_header) + assert dt is not None, f"Upload-Expires '{expires_header}' could not be parsed as valid RFC 9110 datetime" + + +class TestConcurrencyAndWorkflow: + """Concurrency and end-to-end upload workflows.""" + + def test_concurrent_patches_same_offset(self, target_url): + """ + §6 PATCH: Concurrency Contention Prevention. + Quote: "The Upload-Offset header’s value MUST be equal to the current offset of the resource. + If the offsets do not match, the Server MUST respond with the 409 Conflict status without + modifying the upload resource." + """ + upload_url, _ = create_upload(target_url, upload_length="100") + results = [] + + def do_patch(): + headers = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st, _, _ = http_request("PATCH", upload_url, headers=headers, body=b"A" * 40) + results.append(st) + + t1 = threading.Thread(target=do_patch) + t2 = threading.Thread(target=do_patch) + t1.start() + t2.start() + t1.join() + t2.join() + + successes = [r for r in results if r == 204] + assert len(successes) <= 1, f"At most one concurrent PATCH at offset 0 can succeed, got {results}" + + def test_multi_chunk_resumable_upload_workflow(self, target_url): + """ + End-to-End multi-chunk resumable upload workflow. + Create upload -> PATCH chunk 1 -> HEAD verify -> PATCH chunk 2 -> complete. + """ + total_data = b"0123456789" * 12 # 120 bytes + chunk1 = total_data[:40] + chunk2 = total_data[40:80] + chunk3 = total_data[80:] + + upload_url, _ = create_upload(target_url, upload_length=str(len(total_data))) + + # Chunk 1 + headers1 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st1, _, _ = http_request("PATCH", upload_url, headers=headers1, body=chunk1) + assert st1 == 204 + + # HEAD verification + _, head1, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head1.get(UPLOAD_OFFSET) == "40" + + # Chunk 2 + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "40", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st2, _, _ = http_request("PATCH", upload_url, headers=headers2, body=chunk2) + assert st2 == 204 + + # HEAD verification + _, head2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head2.get(UPLOAD_OFFSET) == "80" + + # Chunk 3 (final) + headers3 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "80", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st3, _, _ = http_request("PATCH", upload_url, headers=headers3, body=chunk3) + assert st3 == 204 + + # Final verification + _, head_final, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert head_final.get(UPLOAD_OFFSET) == "120" + assert head_final.get(UPLOAD_LENGTH) == "120" + + def test_paused_upload_head_offset_and_resume(self, target_url): + """ + §6 PATCH: Upload Pause and Resume Workflow. + Quote: "If the connection is interrupted during a PATCH request, the Client SHOULD issue + a HEAD request to determine the current offset before resuming the upload." + Verifies that when an upload is paused midway, issuing a HEAD request accurately returns + the persisted offset without resetting to 0, and resuming with a subsequent PATCH from + that persisted offset successfully completes the upload. + """ + total_data = b"0123456789abcdefghijklmnopqrstuvwxyz" * 4 # 144 bytes + part1 = total_data[:60] + part2 = total_data[60:] + + upload_url, _ = create_upload(target_url, upload_length=str(len(total_data))) + + # 1. Send first chunk (simulating upload progress before user clicks pause) + headers1 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "0", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st1, resp_headers1, _ = http_request("PATCH", upload_url, headers=headers1, body=part1) + assert st1 == 204, f"Expected 204 No Content for first chunk, got {st1}" + assert resp_headers1.get(UPLOAD_OFFSET) == "60" + + # 2. Simulate client pause: query offset via HEAD to verify saved position + st_head1, head1, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_head1 in (200, 204) + assert head1.get(UPLOAD_OFFSET) == "60", ( + f"Expected Upload-Offset 60 after pause, got {head1.get(UPLOAD_OFFSET)}" + ) + + # 3. Simulate pause delay + time.sleep(0.5) + + # 4. Verify offset remained persisted and did not reset to 0 + st_head2, head2, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_head2 in (200, 204) + assert head2.get(UPLOAD_OFFSET) == "60", ( + f"Expected Upload-Offset to remain 60, got {head2.get(UPLOAD_OFFSET)}" + ) + + # 5. Resume upload from the persisted offset + headers2 = { + TUS_RESUMABLE: TUS_API_VERSION, + UPLOAD_OFFSET: "60", + CONTENT_TYPE: APPLICATION_OFFSET_OCTET_STREAM, + } + st2, resp_headers2, _ = http_request("PATCH", upload_url, headers=headers2, body=part2) + assert st2 == 204, f"Expected 204 No Content for resumed chunk, got {st2}" + assert resp_headers2.get(UPLOAD_OFFSET) == str(len(total_data)) + + # 6. Final verification + st_final, head_final, _ = http_request("HEAD", upload_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert st_final in (200, 204) + assert head_final.get(UPLOAD_OFFSET) == str(len(total_data)) + assert head_final.get(UPLOAD_LENGTH) == str(len(total_data)) + + +# CLI Entry Point & Custom Formatted Summary Reporter +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Tus v1.0.0 Conformity Test Runner") + parser.add_argument( + "--url", + default="http://localhost:8080/test/api/upload", + help="Target Tus upload endpoint URL", + ) + args = parser.parse_args() + + os.environ["TUS_URL"] = args.url + + print("=" * 70) + print(" Tus v1.0.0 Resumable Upload Protocol Conformity Test Suite") + print(" Specification: https://tus.io/protocols/resumable-upload") + print(" Target Endpoint:", args.url) + print("=" * 70) + + class CustomReporter: + def __init__(self): + self.passed = [] + self.failed = [] + self.docs = {} + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() + if report.when == "call": + doc = item.obj.__doc__ or "No description provided." + self.docs[report.nodeid] = doc.strip() + if report.passed: + self.passed.append(report.nodeid) + elif report.failed: + if hasattr(report.longrepr, "reprcrash"): + err_text = report.longrepr.reprcrash.message + elif hasattr(report, "longreprtext"): + err_lines = [ + l.strip() + for l in report.longreprtext.splitlines() + if l.strip().startswith("E ") or l.strip().startswith("AssertionError") + ] + err_text = "\n ".join(err_lines) if err_lines else str(report.longrepr) + else: + err_text = str(report.longrepr) + self.failed.append((report.nodeid, err_text)) + + reporter = CustomReporter() + pytest.main([__file__, "-q", f"--url={args.url}"], plugins=[reporter]) + + total_tests = len(reporter.passed) + len(reporter.failed) + + print("\n" + "=" * 70) + print(" CONFORMITY TEST RESULTS") + print("=" * 70) + print(f" Total Tests Executed: {total_tests}") + print(f" Passed: {len(reporter.passed)}") + print(f" Failed: {len(reporter.failed)}") + print("=" * 70) + + if reporter.failed: + print("\n[!] DETAILED FAILURE BREAKDOWN FOR REMEDIATION:") + print("-" * 70) + for idx, (test_id, err_text) in enumerate(reporter.failed, 1): + print(f"\n{idx}. Test: {test_id}") + func_name = test_id.split("::")[-1] + print(f" Function: {func_name}") + print(f" Specification Goal:\n " + reporter.docs.get(test_id, "").replace("\n", "\n ")) + print(f" Failure Reason:\n " + err_text.replace("\n", "\n ")) + print("-" * 70) + else: + print("\n[✓] ALL TUS V1.0.0 CONFORMITY TESTS PASSED SUCCESSFULLY!") + + sys.exit(0 if not reporter.failed else 1) diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 5b5c20fa..a72ae52c 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -7,6 +7,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -58,7 +59,10 @@ public class TusFileUploadService implements Closeable { private static final Logger log = LoggerFactory.getLogger(TusFileUploadService.class); - public static final int DEFAULT_MAX_LOCK_RETRIES = 40; + public static final Duration DEFAULT_LOCK_WAIT_TIMEOUT = Duration.ofSeconds(60); + public static final long LOCK_RETRY_INTERVAL_MS = 200L; + public static final int DEFAULT_MAX_LOCK_RETRIES = + (int) (DEFAULT_LOCK_WAIT_TIMEOUT.toMillis() / LOCK_RETRY_INTERVAL_MS); private UploadStorageService uploadStorageService; private UploadLockingService uploadLockingService; @@ -68,6 +72,7 @@ public class TusFileUploadService implements Closeable { private boolean isThreadLocalCacheEnabled = false; private boolean isChunkedTransferDecodingEnabled = false; private ProtocolVersion supportedProtocolVersion = ProtocolVersion.AUTO; + private Duration lockWaitTimeout = DEFAULT_LOCK_WAIT_TIMEOUT; private int maxLockRetries = DEFAULT_MAX_LOCK_RETRIES; private final List uploadCompletionListeners = new CopyOnWriteArrayList<>(); @@ -77,9 +82,18 @@ public TusFileUploadService() { String storagePath = FileUtils.getTempDirectoryPath() + File.separator + "tus"; this.uploadStorageService = new DiskStorageService(idFactory, storagePath); this.uploadLockingService = new LeaseFileLockingService(idFactory, storagePath); + this.uploadStorageService.setDrainTimeout(calculateDrainTimeout(DEFAULT_LOCK_WAIT_TIMEOUT)); initFeatures(); } + private static Duration calculateDrainTimeout(Duration timeout) { + return timeout.minus(Duration.ofSeconds(5)); + } + + private static int calculateMaxLockRetries(Duration timeout) { + return (int) (timeout.toMillis() / LOCK_RETRY_INTERVAL_MS); + } + protected void initFeatures() { // The order of the features is important addTusExtension(new CoreProtocol()); @@ -235,6 +249,20 @@ public TusFileUploadService withMinSize(Long minSize) { return this; } + /** + * Set the maximum number of worker threads used for asynchronous background chunk uploading in + * cloud storage backends (S3, Azure). Defaults to 10. + * + * @param cloudUploadThreadPoolSize Number of worker threads (must be > 0) + * @return The current service + */ + public TusFileUploadService withCloudUploadThreadPoolSize(int cloudUploadThreadPoolSize) { + Validate.isTrue( + cloudUploadThreadPoolSize > 0, "The cloud upload thread pool size must be greater than 0"); + this.uploadStorageService.setCloudUploadThreadPoolSize(cloudUploadThreadPoolSize); + return this; + } + /** * Provide a custom {@link UploadIdFactory} implementation that should be used to generate * identifiers for the different uploads. Example implementation are {@link @@ -268,12 +296,15 @@ public TusFileUploadService withUploadStorageService(UploadStorageService upload uploadStorageService.setMaxAppendSize(this.uploadStorageService.getMaxAppendSize()); uploadStorageService.setMinAppendSize(this.uploadStorageService.getMinAppendSize()); uploadStorageService.setMinSize(this.uploadStorageService.getMinSize()); + uploadStorageService.setCloudUploadThreadPoolSize( + this.uploadStorageService.getCloudUploadThreadPoolSize()); uploadStorageService.setUploadExpirationPeriod( this.uploadStorageService.getUploadExpirationPeriod()); uploadStorageService.setUploadDeduplicationEnabled( this.uploadStorageService.isUploadDeduplicationEnabled()); uploadStorageService.setJsonSerializationEnabled( this.uploadStorageService.isJsonSerializationEnabled()); + uploadStorageService.setDrainTimeout(this.uploadStorageService.getDrainTimeout()); uploadStorageService.setIdFactory(this.idFactory); // Update the upload storage service this.uploadStorageService = uploadStorageService; @@ -329,21 +360,47 @@ public TusFileUploadService withUploadLockingService(UploadLockingService upload } /** - * Specify the maximum number of retries the service will attempt to acquire an upload lock before - * failing with an {@link UploadAlreadyLockedException} during lock contention resolution (e.g. - * for HEAD or DELETE requests). Default is {@value #DEFAULT_MAX_LOCK_RETRIES} retries. + * Specify the maximum duration a request will wait to acquire an upload lock held by another + * in-flight request before failing with an {@link + * me.desair.tus.server.exception.UploadAlreadyLockedException} during lock wait and contention + * resolution (e.g. for HEAD or DELETE requests). + * + *

Configuring this setting automatically derives: + * + *

    + *
  • The background cloud upload chunk drain timeout for cloud storage backends (S3, Azure), + * set to {@code lockWaitTimeout - 5 seconds}. + *
  • The maximum number of lock acquisition retries (polling every {@value + * #LOCK_RETRY_INTERVAL_MS}ms). + *
* - * @param maxLockRetries The maximum number of lock acquisition retries (must be 0 or greater) + *

The timeout must be greater than 5 seconds to provide a positive drain window for in-flight + * chunks. Default is 60 seconds (yielding a 55-second drain timeout and 300 retries). + * + * @param lockWaitTimeout The maximum duration to wait for an upload lock (must be > 5 seconds) * @return The current service */ - public TusFileUploadService withMaxLockRetries(int maxLockRetries) { - Validate.isTrue(maxLockRetries >= 0, "The max lock retries must be 0 or greater"); - this.maxLockRetries = maxLockRetries; + public TusFileUploadService withLockWaitTimeout(Duration lockWaitTimeout) { + Validate.notNull(lockWaitTimeout, "The lock wait timeout cannot be null"); + Validate.isTrue( + lockWaitTimeout.toMillis() > 5000L, "The lock wait timeout must be greater than 5 seconds"); + this.lockWaitTimeout = lockWaitTimeout; + this.maxLockRetries = calculateMaxLockRetries(lockWaitTimeout); + this.uploadStorageService.setDrainTimeout(calculateDrainTimeout(lockWaitTimeout)); return this; } /** - * Get the maximum number of lock acquisition retries. + * Get the maximum duration a request will wait to acquire an upload lock. + * + * @return The current lock wait timeout + */ + public Duration getLockWaitTimeout() { + return lockWaitTimeout; + } + + /** + * Get the maximum number of lock acquisition retries, derived from {@link #getLockWaitTimeout()}. * * @return The maximum number of lock acquisition retries */ @@ -545,6 +602,7 @@ public UploadInfo process( } if (wasInProgress && processedUploadInfo != null && !processedUploadInfo.isUploadInProgress()) { + cleanupLock(processedUploadInfo.getId()); notifyUploadCompletionListeners(processedUploadInfo); } @@ -555,9 +613,9 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) throws TusException, IOException { UploadLock lock = null; int retries = 0; - // Retry budget calibrated by default to 40 retries x 200ms = 8.0 seconds to accommodate - // NFS/network storage attribute cache propagation (actimeo=3s), watchdog polling - // interval (1.5s), and socket stream interruption and cleanup overhead. + // Lock wait loop: retries every LOCK_RETRY_INTERVAL_MS ms up to the configured lockWaitTimeout + // budget (default 60s / 300 retries). When an in-flight upload is active, requestLockRelease + // signals it to interrupt and cleanly drain in-flight chunks before releasing the lock. while (retries < maxLockRetries) { try { lock = uploadLockingService.lockUploadByUri(requestUri); @@ -567,7 +625,7 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) uploadLockingService.requestLockRelease(requestUri); retries++; try { - Thread.sleep(200L); + Thread.sleep(LOCK_RETRY_INTERVAL_MS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Lock acquisition retry interrupted", ie); @@ -889,6 +947,7 @@ public void deleteUpload(String uploadUri, String ownerKey) throws IOException, UploadInfo uploadInfo = uploadStorageService.getUploadInfo(uploadUri, ownerKey); if (uploadInfo != null) { uploadStorageService.terminateUpload(uploadInfo); + cleanupLock(uploadInfo.getId()); } } } @@ -984,6 +1043,16 @@ protected void notifyUploadCompletionListeners(UploadInfo uploadInfo) { } } + private void cleanupLock(UploadId id) { + if (id != null && uploadLockingService != null) { + try { + uploadLockingService.cleanupLock(id); + } catch (Exception e) { + log.debug("Error cleaning up lock for upload ID {}: {}", id, e.getMessage()); + } + } + } + public ProtocolVersion detectProtocolVersion(HttpServletRequest request) { return Utils.detectProtocolVersion(request, supportedProtocolVersion); } @@ -1085,6 +1154,13 @@ protected void processTusException( response.setStatus(status); if (problemDetails != null) { problemDetails.writeTo(response); + } else if (status == 460) { + // Non-standard HTTP status 460 (Checksum Mismatch) is not recognized by standard + // servlet container error controllers (such as Spring Boot's BasicErrorController), + // which would cause sendError() to fail with an unhandled IllegalArgumentException + // and degrade to HTTP 500. We explicitly set the status and omit sendError(). + response.setHeader(HttpHeader.CONTENT_LENGTH, null); + response.setStatus(status); } else { response.setHeader(HttpHeader.CONTENT_LENGTH, null); response.sendError(status, message); diff --git a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java index 8ed264c8..d234c881 100644 --- a/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/core/CorePatchRequestHandler.java @@ -66,13 +66,14 @@ public HttpProblemDetails process( if (uploadInfo == null) { found = false; } else if (uploadInfo.isUploadInProgress()) { + InterruptibleInputStream interruptibleStream = null; try { InputStream stream = servletRequest.getContentInputStream(); // If a locking service is provided, wrap the input stream in an InterruptibleInputStream // and register it with the locking service to allow for interruption of the upload. if (lockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); + interruptibleStream = new InterruptibleInputStream(stream); lockingService.registerInputStream(servletRequest.getRequestURI(), interruptibleStream); stream = interruptibleStream; } @@ -94,6 +95,28 @@ public HttpProblemDetails process( } } catch (UploadNotFoundException e) { found = false; + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // PATCH with 204 No Content and the new Upload-Offset rather than throwing a 500 error. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Refresh UploadInfo first so the log statement and response reflect the true committed + // byte offset persisted by the storage backend up to the interruption point. + UploadInfo refreshed = + uploadStorageService.getUploadInfo(servletRequest.getRequestURI(), ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + log.info( + "Upload PATCH request for URI {} was interrupted by locking service contention; " + + "saved partial upload up to offset {}", + servletRequest.getRequestURI(), + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java index 6594e191..61b0bb39 100644 --- a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java @@ -46,13 +46,39 @@ public HttpProblemDetails process( UploadInfo uploadInfo = uploadStorageService.getUploadInfo(location, ownerKey); if (uploadInfo != null && uploadInfo.isUploadInProgress()) { InputStream stream = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(stream); + interruptibleStream = new InterruptibleInputStream(stream); uploadLockingService.registerInputStream(location, interruptibleStream); stream = interruptibleStream; } - uploadInfo = uploadStorageService.append(uploadInfo, stream); + try { + uploadInfo = uploadStorageService.append(uploadInfo, stream); + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and + // updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // upload rather than propagating an unhandled error to the servlet container. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Refresh UploadInfo first so the log statement and response reflect the true + // committed + // byte offset persisted by the storage backend up to the interruption point. + UploadInfo refreshed = uploadStorageService.getUploadInfo(location, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + log.info( + "Upload creation-with-upload POST request for URI {} was interrupted by locking" + + " service contention; saved partial upload up to offset {}", + location, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + } else { + throw e; + } + } servletResponse.setHeader( HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); diff --git a/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java b/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java index 67d86746..84e2230a 100644 --- a/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java +++ b/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java @@ -3,8 +3,11 @@ import java.io.IOException; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.HttpProblemDetails; import me.desair.tus.server.exception.TusException; +import me.desair.tus.server.exception.UploadNotFoundException; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; import me.desair.tus.server.util.TusServletRequest; @@ -27,12 +30,14 @@ public boolean supports(HttpMethod method) { } @Override - public void process( + public HttpProblemDetails process( HttpMethod method, TusServletRequest servletRequest, TusServletResponse servletResponse, UploadStorageService uploadStorageService, - String ownerKey) + UploadLockingService uploadLockingService, + String ownerKey, + TusException exception) throws IOException, TusException { // For post requests, the upload URI is part of the response @@ -50,13 +55,31 @@ public void process( // If the expiration is known at the creation, the Upload-Expires header MUST be included in // the response to the initial POST request. Its value MAY change over time. - if (expirationPeriod != null - && expirationPeriod > 0 - && uploadInfo != null - && !uploadInfo.isExpired()) { - uploadInfo.updateExpiration(expirationPeriod); - uploadStorageService.update(uploadInfo); + if (expirationPeriod != null && expirationPeriod > 0 && uploadInfo != null) { + if (uploadInfo.isExpired()) { + uploadStorageService.terminateUpload(uploadInfo); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(uploadInfo.getId()); + } + throw new UploadNotFoundException( + "The upload for path " + uploadUri + " has expired and was removed."); + } else { + uploadInfo.updateExpiration(expirationPeriod); + uploadStorageService.update(uploadInfo); + } } + return null; + } + + @Override + public void process( + HttpMethod method, + TusServletRequest servletRequest, + TusServletResponse servletResponse, + UploadStorageService uploadStorageService, + String ownerKey) + throws IOException, TusException { + process(method, servletRequest, servletResponse, uploadStorageService, null, ownerKey, null); } @Override diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index 5ec06fa4..0d889b2d 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -14,6 +14,8 @@ import me.desair.tus.server.util.StructuredHeaderUtil; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Request handler for data append requests via HTTP PATCH. @@ -27,6 +29,8 @@ */ public class RufhAppendPatchRequestHandler extends AbstractRequestHandler { + private static final Logger log = LoggerFactory.getLogger(RufhAppendPatchRequestHandler.class); + @Override public boolean supports(HttpMethod method) { return HttpMethod.PATCH.equals(method); @@ -55,16 +59,49 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); + // Per RUFH §4.2.1: If upload length was deferred and the client completes the upload + // via Upload-Complete: ?1, derive and set the total length from current offset + + // Content-Length. + long contentLength = servletRequest.getContentLengthLong(); + if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && contentLength >= 0) { + long currentOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; + uploadInfo.setLength(currentOffset + contentLength); + uploadStorageService.update(uploadInfo); + } + InputStream is = servletRequest.getContentInputStream(); + InterruptibleInputStream interruptibleStream = null; if (is != null) { if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); + interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(requestUri, interruptibleStream); is = interruptibleStream; } - UploadInfo appended = uploadStorageService.append(uploadInfo, is); - if (appended != null) { - uploadInfo = appended; + try { + UploadInfo appended = uploadStorageService.append(uploadInfo, is); + if (appended != null) { + uploadInfo = appended; + } + } catch (IOException e) { + // When an append stream is interrupted by the locking service watchdog or a concurrent + // release request, the storage backend commits all bytes received so far and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // append rather than propagating an unhandled error. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Refresh UploadInfo first so the log statement and response reflect the true committed + // byte offset persisted by the storage backend up to the interruption point. + UploadInfo refreshed = uploadStorageService.getUploadInfo(requestUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + log.info( + "RUFH append request for URI {} was interrupted by locking service contention; " + + "saved partial upload up to offset {}", + requestUri, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index 1c47d68c..aa46f7db 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -16,6 +16,9 @@ import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; import me.desair.tus.server.util.Utils; +import org.apache.commons.lang3.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Request handler for upload creation requests via HTTP POST, PUT, or PATCH. @@ -28,6 +31,8 @@ */ public class RufhCreationPostRequestHandler extends AbstractRequestHandler { + private static final Logger log = LoggerFactory.getLogger(RufhCreationPostRequestHandler.class); + public RufhCreationPostRequestHandler() { // Default constructor } @@ -66,17 +71,26 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); + // Per RUFH §4.2.1: If upload length is deferred upon creation and the client completes + // the upload in the creation request via Upload-Complete: ?1, the total length is derived + // from Content-Length. + long contentLength = servletRequest.getContentLengthLong(); + Long announcedLength = uploadLength; + if (announcedLength == null && Boolean.TRUE.equals(uploadComplete) && contentLength >= 0) { + announcedLength = contentLength; + } + UploadInfo uploadInfo; if (preCreatedUploadInfo != null) { uploadInfo = preCreatedUploadInfo; - if (uploadLength != null && uploadLength >= 0) { - uploadInfo.setLength(uploadLength); + if (announcedLength != null && announcedLength >= 0) { + uploadInfo.setLength(announcedLength); } uploadStorageService.update(uploadInfo); } else { uploadInfo = new UploadInfo(); - if (uploadLength != null && uploadLength >= 0) { - uploadInfo.setLength(uploadLength); + if (announcedLength != null && announcedLength >= 0) { + uploadInfo.setLength(announcedLength); } uploadInfo = uploadStorageService.create(uploadInfo, ownerKey); } @@ -84,16 +98,54 @@ public HttpProblemDetails process( String uploadUri = Utils.getUploadUriOnCreation(uploadInfo, servletRequest, uploadStorageService); + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + // An empty creation request carries no body (contentLength <= 0 without chunked encoding). + // In servlet requests without a body, getContentLengthLong() returns -1. We must verify + // that actual payload content is present before invoking storageService.append(); otherwise, + // sending a 0-byte stream to backends with minAppendSize configured (e.g. S3 or Azure Blob) + // would trigger MinAppendSizeNotMetException on an empty creation request. + boolean hasContent = + contentLength > 0 + || (contentLength < 0 + && servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING) != null + && Strings.CI.contains( + servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked")); + InputStream is = servletRequest.getContentInputStream(); - if (is != null && servletRequest.getContentLengthLong() != 0) { + InterruptibleInputStream interruptibleStream = null; + if (is != null && hasContent) { if (uploadLockingService != null) { - InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(is); + interruptibleStream = new InterruptibleInputStream(is); uploadLockingService.registerInputStream(uploadUri, interruptibleStream); is = interruptibleStream; } - UploadInfo appended = uploadStorageService.append(uploadInfo, is); - if (appended != null) { - uploadInfo = appended; + try { + UploadInfo appended = uploadStorageService.append(uploadInfo, is); + if (appended != null) { + uploadInfo = appended; + } + } catch (IOException e) { + // When an upload stream is interrupted by the locking service watchdog or a concurrent + // lock contention release request (e.g. from a concurrent HEAD or DELETE), the storage + // backend (Disk, S3, Azure) commits all bytes received up to the interruption and updates + // the offset in storage. We reload the updated UploadInfo and acknowledge the partial + // upload rather than propagating an unhandled error to the servlet container. + if (interruptibleStream != null && interruptibleStream.isInterrupted()) { + // Refresh UploadInfo first so the log statement and response reflect the true committed + // byte offset persisted by the storage backend up to the interruption point. + UploadInfo refreshed = uploadStorageService.getUploadInfo(uploadUri, ownerKey); + if (refreshed != null) { + uploadInfo = refreshed; + } + log.info( + "RUFH creation request with body for URI {} was interrupted by locking service" + + " contention; saved partial upload up to offset {}", + uploadUri, + uploadInfo != null ? uploadInfo.getOffset() : "unknown"); + } else { + throw e; + } } } diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java index 35b9f2cc..f086036e 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandler.java @@ -46,6 +46,9 @@ public HttpProblemDetails process( } uploadStorageService.terminateUpload(uploadInfo); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(uploadInfo.getId()); + } servletResponse.setStatus(204); return null; } diff --git a/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java b/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java index 4bbb7749..a09dc67e 100644 --- a/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java +++ b/src/main/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtil.java @@ -56,13 +56,22 @@ public static String getRawInterimResponse( String uploadUri; try { - UploadInfo uploadInfo = uploadStorageService.getUploadInfo(existingUploadUri, ownerKey); + UploadInfo uploadInfo = + existingUploadUri != null + ? uploadStorageService.getUploadInfo(existingUploadUri, ownerKey) + : null; if (uploadInfo != null) { long offset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; return getRawInterimResponseForAppend(offset); } + // If there is no existing upload, an interim response with Location is only valid for + // an upload creation request matching this storage service's creation endpoint. + if (!Utils.isCreationEndpoint(servletRequest, uploadStorageService)) { + return null; + } + uploadInfo = new UploadInfo(); uploadInfo = uploadStorageService.create(uploadInfo, ownerKey); uploadUri = Utils.getUploadUriOnCreation(uploadInfo, servletRequest, uploadStorageService); diff --git a/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java b/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java index 7b61176f..25c2a7b0 100644 --- a/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java +++ b/src/main/java/me/desair/tus/server/termination/TerminationDeleteRequestHandler.java @@ -3,8 +3,10 @@ import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.HttpProblemDetails; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.AbstractRequestHandler; import me.desair.tus.server.util.TusServletRequest; @@ -24,12 +26,14 @@ public boolean supports(HttpMethod method) { } @Override - public void process( + public HttpProblemDetails process( HttpMethod method, TusServletRequest servletRequest, TusServletResponse servletResponse, UploadStorageService uploadStorageService, - String ownerKey) + UploadLockingService uploadLockingService, + String ownerKey, + TusException exception) throws IOException, TusException { UploadInfo uploadInfo = @@ -37,8 +41,23 @@ public void process( if (uploadInfo != null) { uploadStorageService.terminateUpload(uploadInfo); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(uploadInfo.getId()); + } } servletResponse.setStatus(HttpServletResponse.SC_NO_CONTENT); + return null; + } + + @Override + public void process( + HttpMethod method, + TusServletRequest servletRequest, + TusServletResponse servletResponse, + UploadStorageService uploadStorageService, + String ownerKey) + throws IOException, TusException { + process(method, servletRequest, servletResponse, uploadStorageService, null, ownerKey, null); } } diff --git a/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java b/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java index 8560af5a..c0b4b022 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractCloseableResourceService.java @@ -2,6 +2,8 @@ import java.io.Closeable; import java.io.IOException; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Abstract base class for services that manage closeable resources and optional JVM shutdown hooks. @@ -9,7 +11,8 @@ public abstract class AbstractCloseableResourceService implements Closeable { private final Thread shutdownHook; - private volatile boolean closed = false; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean closed = false; /** Constructs a closeable resource service without a JVM shutdown hook. */ protected AbstractCloseableResourceService() { @@ -36,7 +39,12 @@ protected AbstractCloseableResourceService(String shutdownHookName) { * @return true if the service is closed, false otherwise */ public boolean isClosed() { - return closed; + thisObjectLock.readLock().lock(); + try { + return closed; + } finally { + thisObjectLock.readLock().unlock(); + } } /** @@ -47,11 +55,17 @@ public boolean isClosed() { */ @Override public void close() throws IOException { - if (!closed) { + thisObjectLock.writeLock().lock(); + try { + if (closed) { + return; + } closed = true; - deregisterShutdownHook(); - cleanupOnClose(); + } finally { + thisObjectLock.writeLock().unlock(); } + deregisterShutdownHook(); + cleanupOnClose(); } /** diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java index 8beec793..ee24adfe 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java @@ -126,25 +126,51 @@ public void close() { * Renew the lock lease by advancing the expiration timestamp and persisting the updated metadata. */ public void renewLease() { + long oldExpiresAt = leaseData.getExpiresAt(); + long candidateExpiresAt = System.currentTimeMillis() + leaseData.getLeaseDurationMs(); try { - leaseData.setExpiresAt(System.currentTimeMillis() + leaseData.getLeaseDurationMs()); + leaseData.setExpiresAt(candidateExpiresAt); doRenewLease(); + } catch (IllegalStateException e) { + leaseData.setExpiresAt(oldExpiresAt); + // Ownership lost (e.g. lease taken over by another node): abort immediately + log.info( + "Aborting active input stream for upload URI {} as lock was taken over: {}", + getRequestUri(), + e.getMessage()); + abortActiveStreamAndHeartbeat(); } catch (Exception e) { - // If lease renewal fails, abort any active input stream immediately - // to prevent the upload thread from continuing to write un-locked bytes. - if (activeInputStreams != null && getRequestUri() != null) { - InputStream stream = activeInputStreams.get(getRequestUri()); - if (stream != null) { - log.info( - "Aborting active input stream for upload URI {} due to lease renewal failure", - getRequestUri()); - Utils.interruptStream(stream); - } + leaseData.setExpiresAt(oldExpiresAt); + // Transient failure (e.g. network timeout). Only abort if the previous lease actually + // expired. + if (System.currentTimeMillis() >= oldExpiresAt) { + log.warn( + "Lease for upload URI {} expired after renewal failure; aborting active stream", + getRequestUri(), + e); + abortActiveStreamAndHeartbeat(); + } else { + log.warn( + "Transient failure renewing lease for upload URI {} (will retry on next tick): {}", + getRequestUri(), + e.getMessage()); } - Utils.shutdownExecutor(heartbeatExecutor); } } + private void abortActiveStreamAndHeartbeat() { + if (activeInputStreams != null && getRequestUri() != null) { + InputStream stream = activeInputStreams.get(getRequestUri()); + if (stream != null) { + log.info( + "Aborting active input stream for upload URI {} due to lease renewal failure", + getRequestUri()); + Utils.interruptStream(stream); + } + } + Utils.shutdownExecutor(heartbeatExecutor); + } + /** * Subclass hook to persist updated lease metadata during heartbeat renewal. * diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java index a1f87707..521c3cc0 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java @@ -37,9 +37,14 @@ public abstract class AbstractLeaseLockingService extends AbstractCloseableResou */ public static final long CLOCK_SKEW_SAFETY_MARGIN_MS = 2000L; + public static final long DEFAULT_JITTER_MIN_MS = 20L; + public static final long DEFAULT_JITTER_MAX_MS = 60L; + protected final long leaseDurationMs; protected final long pollIntervalMs; protected UploadIdFactory idFactory; + protected long jitterMinMs = DEFAULT_JITTER_MIN_MS; + protected long jitterMaxMs = DEFAULT_JITTER_MAX_MS; protected final Map activeInputStreams = new ConcurrentHashMap<>(); protected final ScheduledExecutorService watchdogExecutor; @@ -251,17 +256,76 @@ public void checkStopSignals() { } /** - * Applies randomized jitter backoff to settle concurrent in-flight writes. + * Applies randomized jitter backoff to settle concurrent in-flight writes and arbitrate lock + * acquisition races on storage backends lacking atomic conditional writes. + * + *

If {@code maxMs <= 0}, this method returns immediately without sleeping, allowing callers to + * bypass jitter on backends with guaranteed atomic conditional writes (e.g. AWS S3). * * @param minMs Minimum jitter duration in milliseconds * @param maxMs Maximum jitter duration in milliseconds */ protected void applyJitter(long minMs, long maxMs) { + if (maxMs <= 0) { + return; + } try { - long jitterMs = minMs + (long) (Math.random() * (maxMs - minMs)); - Thread.sleep(jitterMs); + long jitterMs = (minMs >= maxMs) ? minMs : minMs + (long) (Math.random() * (maxMs - minMs)); + if (jitterMs > 0) { + Thread.sleep(jitterMs); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } + + /** + * Applies randomized jitter backoff using the configured {@link #getJitterMinMs()} and {@link + * #getJitterMaxMs()} bounds. + */ + protected void applyJitter() { + applyJitter(jitterMinMs, jitterMaxMs); + } + + /** + * Returns the configured minimum jitter duration in milliseconds. + * + * @return Minimum jitter duration in milliseconds + */ + public long getJitterMinMs() { + return jitterMinMs; + } + + /** + * Returns the configured maximum jitter duration in milliseconds. + * + * @return Maximum jitter duration in milliseconds + */ + public long getJitterMaxMs() { + return jitterMaxMs; + } + + /** + * Configures the jitter bounds used during lock acquisition read-after-write verification. + * + * @param minMs Minimum jitter duration in milliseconds (must be >= 0) + * @param maxMs Maximum jitter duration in milliseconds (must be >= minMs) + * @throws IllegalArgumentException If minMs is negative or maxMs is less than minMs + */ + public void setJitter(long minMs, long maxMs) { + if (minMs < 0) { + throw new IllegalArgumentException("jitterMinMs cannot be negative: " + minMs); + } + if (maxMs < minMs) { + throw new IllegalArgumentException( + "jitterMaxMs (" + maxMs + ") cannot be less than jitterMinMs (" + minMs + ")"); + } + this.jitterMinMs = minMs; + this.jitterMaxMs = maxMs; + } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + cleanupLock(uploadUri != null && idFactory != null ? idFactory.readUploadId(uploadUri) : null); + } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java b/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java index 274dd735..717ffb01 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java +++ b/src/main/java/me/desair/tus/server/upload/UploadIdFactory.java @@ -16,7 +16,7 @@ */ public abstract class UploadIdFactory { - private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); private String uploadUri = "/"; // Read and write operations on uploadUri and uploadUriPattern are guarded by a // ReentrantReadWriteLock. @@ -41,12 +41,12 @@ public void setUploadUri(String uploadUri) { "The upload URI should start with /, http://, or https://"); Validate.isTrue(!Strings.CS.endsWith(uploadUri, "$"), "The upload URI should not end with $"); - lock.writeLock().lock(); + thisObjectLock.writeLock().lock(); try { this.uploadUri = uploadUri; this.uploadUriPattern = null; } finally { - lock.writeLock().unlock(); + thisObjectLock.writeLock().unlock(); } } @@ -57,11 +57,11 @@ public void setUploadUri(String uploadUri) { * @return The URI of the main tus upload endpoint. */ public String getUploadUri() { - lock.readLock().lock(); + thisObjectLock.readLock().lock(); try { return uploadUri; } finally { - lock.readLock().unlock(); + thisObjectLock.readLock().unlock(); } } @@ -107,16 +107,16 @@ public UploadId readUploadId(String url) { * @return A (cached) Pattern to match upload URI's */ protected Pattern getUploadUriPattern() { - lock.readLock().lock(); + thisObjectLock.readLock().lock(); try { if (uploadUriPattern != null) { return uploadUriPattern; } } finally { - lock.readLock().unlock(); + thisObjectLock.readLock().unlock(); } - lock.writeLock().lock(); + thisObjectLock.writeLock().lock(); try { if (uploadUriPattern == null) { // Extract upload IDs by removing upload URI from start of request URI. @@ -127,7 +127,7 @@ protected Pattern getUploadUriPattern() { } return uploadUriPattern; } finally { - lock.writeLock().unlock(); + thisObjectLock.writeLock().unlock(); } } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadInfo.java b/src/main/java/me/desair/tus/server/upload/UploadInfo.java index a843940a..56f6356f 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadInfo.java +++ b/src/main/java/me/desair/tus/server/upload/UploadInfo.java @@ -43,6 +43,7 @@ public class UploadInfo implements Serializable { private String representationDigest; private String requestedRepresentationDigests; private String storageUploadId; + private List uploadPartKeys; /** Default constructor to use if an upload is created without HTTP request. */ public UploadInfo() { @@ -460,6 +461,24 @@ public void setStorageUploadId(String storageUploadId) { this.storageUploadId = storageUploadId; } + /** + * Get the ordered list of storage part keys or block IDs committed for this upload. + * + * @return The list of part keys or block IDs, or null if uninitialized/unsupported + */ + public List getUploadPartKeys() { + return uploadPartKeys; + } + + /** + * Set the ordered list of storage part keys or block IDs committed for this upload. + * + * @param uploadPartKeys The list of part keys or block IDs + */ + public void setUploadPartKeys(List uploadPartKeys) { + this.uploadPartKeys = uploadPartKeys; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -489,6 +508,7 @@ public boolean equals(Object o) { .append(getRepresentationDigest(), that.getRepresentationDigest()) .append(getRequestedRepresentationDigests(), that.getRequestedRepresentationDigests()) .append(getStorageUploadId(), that.getStorageUploadId()) + .append(getUploadPartKeys(), that.getUploadPartKeys()) .isEquals(); } @@ -511,6 +531,7 @@ public int hashCode() { .append(getRepresentationDigest()) .append(getRequestedRepresentationDigests()) .append(getStorageUploadId()) + .append(getUploadPartKeys()) .toHashCode(); } diff --git a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java index 0a3cc427..d97fb701 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java @@ -72,4 +72,30 @@ default void requestLockRelease(String requestUri) { default void close() throws IOException { // No-op by default for backwards compatibility } + + /** + * Cleans up the lock file or resources associated with a specific completed, terminated, or + * expired upload identifier. + * + *

This method is invoked automatically when an upload completes successfully, is terminated + * via DELETE, or is cleaned up during expiration sweeps, freeing underlying lock storage (such as + * Azure Blob Lease lock blobs or on-disk lock files). + * + * @param id The upload identifier of the completed, terminated, or expired upload + * @throws IOException If cleaning up lock resources fails + */ + default void cleanupLock(UploadId id) throws IOException { + // No-op by default for backwards compatibility + } + + /** + * Cleans up the lock file or resources associated with a specific completed, terminated, or + * expired upload URI. + * + * @param uploadUri The request URI of the completed, terminated, or expired upload + * @throws IOException If cleaning up lock resources fails + */ + default void cleanupLock(String uploadUri) throws IOException { + // No-op by default for backwards compatibility + } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index ea5d9f1d..5f168ba4 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; @@ -279,6 +280,43 @@ default boolean isJsonSerializationEnabled() { return false; } + /** + * Set the thread pool size used for asynchronous background chunk uploading to cloud storage. + * + * @param size Number of worker threads + */ + default void setCloudUploadThreadPoolSize(int size) { + // Default no-op for non-cloud implementations + } + + /** + * Get the thread pool size used for asynchronous background chunk uploading to cloud storage. + * + * @return Thread pool size, defaults to 10 + */ + default int getCloudUploadThreadPoolSize() { + return 10; + } + + /** + * Set the timeout for draining in-flight chunks during upload completion or interruption. + * + * @param drainTimeout Drain timeout duration + */ + default void setDrainTimeout(Duration drainTimeout) { + // Default no-op for non-cloud implementations + } + + /** + * Get the timeout for draining in-flight chunks during upload completion or interruption. + * Defaults to 55 seconds (derived from the 60-second default lock wait timeout minus 5 seconds). + * + * @return Drain timeout duration, defaults to 55 seconds + */ + default Duration getDrainTimeout() { + return Duration.ofSeconds(55); + } + /** * Closes any underlying storage resources. * diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java index 19be0000..697f9be1 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java @@ -3,11 +3,14 @@ import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.sas.BlobSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; import com.azure.storage.blob.specialized.BlockBlobClient; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -109,16 +112,20 @@ public void merge(UploadInfo finalUpload) throws IOException, UploadNotFoundExce BlobClient partialBlob = containerClient.getBlobClient(uploadPrefix + partialInfo.getId()); try { - // 1. Attempt zero-copy server-side block copying on Azure Storage cluster - stageBlockFromUrl(finalBlockBlob, blockId, partialBlob.getBlobUrl()); + // 1. Attempt zero-copy server-side block copying on Azure Storage cluster using an + // authorized blob URL (with read SAS token when shared key credentials are present) + String sourceUrl = getAuthorizedBlobUrl(partialBlob); + stageBlockFromUrl(finalBlockBlob, blockId, sourceUrl); } catch (BlobStorageException e) { // 2. In private Azure containers without SAS tokens or in emulators, stageBlockFromUrl - // fails with 403 (ACCESS_DENIED) or 400/501 (API_NOT_IMPLEMENTED). Fall back + // fails with 401/403 (ACCESS_DENIED / CannotVerifyCopySource) or 400/501 + // (API_NOT_IMPLEMENTED). Fall back // gracefully to streaming block staging via getUploadedBytes(). AzureErrorType errorType = AzureUtils.parseErrorResponse(e); if (errorType == AzureErrorType.API_NOT_IMPLEMENTED || errorType == AzureErrorType.ACCESS_DENIED || e.getStatusCode() == 400 + || e.getStatusCode() == 401 || e.getStatusCode() == 403) { try (InputStream partIs = storageService.getUploadedBytes(partialInfo.getId())) { finalBlockBlob.stageBlock(blockId, partIs, partialInfo.getOffset()); @@ -230,4 +237,32 @@ private String sanitizePrefix(String prefix) { void stageBlockFromUrl(BlockBlobClient finalBlockBlob, String blockId, String sourceUrl) { finalBlockBlob.stageBlockFromUrl(blockId, sourceUrl, null); } + + /** + * Resolves an authorized source URL for server-side block copying. If the client possesses shared + * key credentials, generates a short-lived read SAS token so that private Azure containers can be + * accessed directly by the storage service via stageBlockFromUrl. + * + * @param blobClient The BlobClient of the source partial upload + * @return The authorized blob URL (with SAS token if available) or raw blob URL + */ + String getAuthorizedBlobUrl(BlobClient blobClient) { + if (blobClient == null) { + return null; + } + try { + BlobServiceSasSignatureValues sasValues = + new BlobServiceSasSignatureValues( + OffsetDateTime.now().plusMinutes(15), + new BlobSasPermission().setReadPermission(true)); + String sasToken = blobClient.generateSas(sasValues); + if (sasToken != null && !sasToken.isEmpty()) { + return blobClient.getBlobUrl() + "?" + sasToken; + } + } catch (Exception ignored) { + // Client lacks shared key credentials (e.g. TokenCredential without user delegation key) + // or SAS generation is unsupported; fall back to raw blob URL. + } + return blobClient.getBlobUrl(); + } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index c14c5bc7..059d749c 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -3,14 +3,21 @@ import com.azure.core.util.BinaryData; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobItemProperties; import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.DeleteSnapshotsOptionType; +import com.azure.storage.blob.models.LeaseStateType; +import com.azure.storage.blob.models.ListBlobsOptions; import com.azure.storage.blob.specialized.BlobLeaseClient; import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import java.io.IOException; import java.io.InputStream; import java.lang.ref.WeakReference; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -59,11 +66,14 @@ public class AzureBlobLockingService extends AbstractCloseableResourceService public static final String DEFAULT_LOCKS_PREFIX = "locks/"; private static final int LEASE_DURATION_SECONDS = 30; + private static final long DEFAULT_STALE_LOCK_THRESHOLD_SECONDS = 120L; + private static final long STOP_SIGNAL_EXPIRATION_SECONDS = 30L; private final BlobContainerClient containerClient; private final String locksPrefix; final Map> activeStreams = new ConcurrentHashMap<>(); + final Map activeLocks = new ConcurrentHashMap<>(); private UploadIdFactory idFactory = new UuidUploadIdFactory(); @@ -105,6 +115,7 @@ protected void cleanupOnClose() throws IOException { } } activeStreams.clear(); + activeLocks.clear(); } @Override @@ -145,12 +156,17 @@ public UploadLock lockUploadByUri(String requestUri) throws TusException, IOExce // Lock successfully acquired: clear any lingering .stop signal blob deleteStopSignalBlob(idStr); - return new AzureBlobUploadLock(leaseClient, lockBlob, requestUri, idStr, activeStreams); + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + leaseClient, lockBlob, requestUri, idStr, activeStreams, activeLocks); + activeLocks.put(idStr, lock); + return lock; } catch (BlobStorageException e) { AzureErrorType errorType = AzureUtils.parseErrorResponse(e); if (errorType == AzureErrorType.LEASE_ALREADY_PRESENT || errorType == AzureErrorType.CONFLICT) { - log.info("Lock contention for upload URI {}: Azure blob lease is already held", requestUri); + log.debug( + "Lock contention for upload URI {}: Azure blob lease is already held", requestUri); throw new UploadAlreadyLockedException( "Upload with URI " + requestUri + " is currently locked"); } @@ -158,9 +174,149 @@ public UploadLock lockUploadByUri(String requestUri) throws TusException, IOExce } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + String idStr = id.toString(); + try { + // 1. Delete any lingering cross-pod .stop signal blob from prior lock contention + deleteStopSignalBlob(idStr); + + // 2. Delete the target .lock blob in the locks prefix + BlobClient lockBlob = getBlobClient(locksPrefix + idStr + ".lock"); + AzureBlobUploadLock activeLock = activeLocks.get(idStr); + if (activeLock != null && activeLock.getLeaseClient() != null) { + // The lock is actively held by this instance (e.g. during DELETE termination). + // Safely delete the blob specifying the active lease ID condition. + deleteLockBlobWithLease(lockBlob, activeLock.getLeaseClient().getLeaseId()); + activeLock.markBlobDeleted(); + } else { + deleteLockBlobIfUnleased(lockBlob); + } + } catch (Exception e) { + log.debug("Failed to clean up Azure lock blob for upload ID {}: {}", idStr, e.getMessage()); + } + } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + cleanupLock(uploadUri != null && idFactory != null ? idFactory.readUploadId(uploadUri) : null); + } + + /** + * Safely deletes an actively leased lock blob using its active lease ID. + * + *

When an upload is terminated (e.g. via {@code DELETE}) while holding the request lock, + * standard unleased deletion cannot be used because Azure returns HTTP 412 (LeaseIdMissing). + * Providing the active lease ID in {@link BlobRequestConditions} allows Azure to delete the + * actively leased lock blob immediately. + * + * @param lockBlob The blob client for the lock blob + * @param leaseId The active lease ID + * @return true if successfully deleted, false otherwise + */ + boolean deleteLockBlobWithLease(BlobClient lockBlob, String leaseId) { + if (lockBlob == null || leaseId == null) { + return false; + } + try { + lockBlob.deleteWithResponse( + DeleteSnapshotsOptionType.INCLUDE, + new BlobRequestConditions().setLeaseId(leaseId), + null, + null); + log.debug("Cleaned up actively leased Azure lock blob {}", lockBlob.getBlobName()); + return true; + } catch (Exception e) { + log.debug("Failed to delete leased lock blob {}: {}", lockBlob.getBlobName(), e.getMessage()); + return false; + } + } + + /** + * Safely deletes an unleased lock blob. Attempting deleteIfExists() on an actively leased blob + * without the lease ID causes Azure Blob Storage to reject the request with HTTP 412 + * (LeaseIdMissing). This helper verifies lease state before deletion. + */ + boolean deleteLockBlobIfUnleased(BlobClient lockBlob) { + if (lockBlob == null) { + return false; + } + try { + if (Boolean.TRUE.equals(lockBlob.exists())) { + BlobProperties props = lockBlob.getProperties(); + // Check lease state before deletion: if actively held by another thread or node, + // do not attempt deletion to avoid lease conflict errors. + if (props.getLeaseState() != null + && !Strings.CS.equals(props.getLeaseState().toString(), "leased")) { + lockBlob.deleteIfExists(); + log.debug("Cleaned up Azure lock blob {}", lockBlob.getBlobName()); + return true; + } + } + } catch (Exception e) { + log.debug( + "Failed to delete unleased lock blob {}: {}", lockBlob.getBlobName(), e.getMessage()); + } + return false; + } + @Override public void cleanupStaleLocks() throws IOException { - // Azure Blob Leases auto-expire after 30s on holder failure; no manual sweeps needed + try { + ListBlobsOptions options = new ListBlobsOptions().setPrefix(locksPrefix); + OffsetDateTime now = OffsetDateTime.now(); + + for (BlobItem item : listBlobs(options)) { + if (item == null || item.getName() == null) { + continue; + } + String blobName = item.getName(); + BlobItemProperties props = item.getProperties(); + if (props == null) { + continue; + } + // Defensively retrieve properties; uninitialized BlobItem instances in Azure SDK v12 + // throw NullPointerException on property access if internalProperties is null. + OffsetDateTime lastModified = null; + LeaseStateType leaseState = null; + try { + lastModified = props.getLastModified(); + leaseState = props.getLeaseState(); + } catch (NullPointerException ignored) { + continue; + } + + // 1. Clean up stale .stop signal blobs older than 30s + if (blobName.endsWith(".stop")) { + if (lastModified != null + && lastModified.isBefore(now.minusSeconds(STOP_SIGNAL_EXPIRATION_SECONDS))) { + getBlobClient(blobName).deleteIfExists(); + log.info("Cleaned up stale Azure stop signal blob: {}", blobName); + } + } + // 2. Clean up un-leased .lock blobs older than 120s + else if (blobName.endsWith(".lock")) { + // Never delete actively leased blobs to prevent interrupting active upload threads + if (leaseState != null && leaseState == LeaseStateType.LEASED) { + continue; + } + // The 120s grace period ensures we do not delete lock blobs for uploads actively + // between chunk requests, while cleaning up abandoned or pre-existing lock blobs. + if (lastModified != null + && lastModified.isBefore(now.minusSeconds(DEFAULT_STALE_LOCK_THRESHOLD_SECONDS))) { + BlobClient lockBlob = getBlobClient(blobName); + if (deleteLockBlobIfUnleased(lockBlob)) { + log.info("Cleaned up stale unleased Azure lock blob: {}", blobName); + } + } + } + } + } catch (Exception e) { + throw new IOException("Failed to cleanup stale Azure locks in container", e); + } } @Override @@ -210,7 +366,7 @@ private void interruptLocalStream(String idStr) { } /** Creates a .stop signal blob to request remote pods to halt active streaming appends. */ - private void createStopSignalBlob(String idStr) { + void createStopSignalBlob(String idStr) { try { BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); stopBlob.upload(BinaryData.fromString("stop"), true); @@ -220,7 +376,7 @@ private void createStopSignalBlob(String idStr) { } /** Deletes the .stop signal blob after lock acquisition. */ - private void deleteStopSignalBlob(String idStr) { + void deleteStopSignalBlob(String idStr) { try { BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); stopBlob.deleteIfExists(); @@ -251,6 +407,14 @@ BlobLeaseClient createBlobLeaseClient(BlobClient lockBlob) { return new BlobLeaseClientBuilder().blobClient(lockBlob).buildClient(); } + Iterable listBlobs(ListBlobsOptions options) { + return containerClient.listBlobs(options, null); + } + + BlobClient getBlobClient(String blobName) { + return containerClient.getBlobClient(blobName); + } + /** * Periodically polled by {@link #watchdogExecutor} to detect remote .stop signal blobs on Azure * and interrupt local active streams. diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index 69c6e70e..f300ac92 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -4,6 +4,7 @@ import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobProperties; import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.Block; import com.azure.storage.blob.models.BlockList; @@ -21,10 +22,16 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinAppendSizeNotMetException; @@ -37,11 +44,14 @@ import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import me.desair.tus.server.upload.util.AsyncChunkUploader; +import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,6 +96,10 @@ public class AzureBlobStorageService implements UploadStorageService { private long preferredBlockSize = DEFAULT_PREFERRED_BLOCK_SIZE; + private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); + private final ThreadPoolExecutor uploadExecutor; + private Long maxUploadSize; private Long maxAppendSize; private Long minAppendSize; @@ -144,6 +158,21 @@ public AzureBlobStorageService( } Utils.cleanupTempFiles(this.tempBufferDir, "tus-azure-chunk-*.tmp", 24L * 3600_000L); + AtomicInteger threadNum = new AtomicInteger(1); + this.uploadExecutor = + new ThreadPoolExecutor( + cloudUploadThreadPoolSize, + cloudUploadThreadPoolSize, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + r -> { + Thread t = new Thread(r, "tus-azure-upload-" + threadNum.getAndIncrement()); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.CallerRunsPolicy()); + this.concatenationService = new AzureBlobConcatenationService(containerClient, this.uploadPrefix, this); } @@ -199,6 +228,14 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) containerClient.getBlobClient(getAzureBlobName(upload)).getBlockBlobClient(); List blockIds = getCommittedBlockIds(blockBlobClient); + // If metadata records committed uploadPartKeys, ensure the blob's committed blocks do not + // exceed the authoritative metadata offset (e.g. if a prior commitBlockList succeeded + // but updating UploadInfo metadata failed or was interrupted). + if (upload.getUploadPartKeys() != null && blockIds.size() > upload.getUploadPartKeys().size()) { + blockIds = new ArrayList<>(upload.getUploadPartKeys()); + blockBlobClient.commitBlockList(blockIds, true); + } + // 2. Validate block budget: Azure Block Blobs support up to 50,000 blocks validateRemainingBlockBudget(upload, blockIds.size()); @@ -210,67 +247,108 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) int initialBlockCount = blockIds.size(); long totalAppended = 0L; IOException streamException = null; + IOException drainException = null; TusException pendingTusException = null; - // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob - while (true) { - File chunkFile = null; - try { - chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile(); - ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, optimalBlockSize); - long chunkSize = chunkResult.bytesRead; - if (chunkResult.exception != null) { - streamException = chunkResult.exception; - } + List plannedBlockIds = new ArrayList<>(); + List plannedChunkSizes = new ArrayList<>(); - if (chunkSize <= 0) { - break; - } - - if (effectiveMaxAppendSize != null - && (totalAppended + chunkSize) > effectiveMaxAppendSize) { - // If maxAppendSize is exceeded, stop reading from the stream but commit - // previously staged blocks so no uploaded data or offset is corrupted. - pendingTusException = - new MaxAppendSizeExceededException( - "Append payload size " - + (totalAppended + chunkSize) - + " exceeded limit of " - + effectiveMaxAppendSize); - break; - } - - totalAppended += chunkSize; + InterruptibleInputStream interruptibleStream = Utils.toInterruptibleStream(inputStream); + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis(), interruptibleStream)) { + // 4. Read incoming stream in chunks, staging blocks directly to Azure Block Blob + while (true) { + File chunkFile = null; + boolean handedOff = false; + try { + chunkFile = Files.createTempFile(tempBufferDir, "tus-azure-chunk-", ".tmp").toFile(); + ReadChunkResult chunkResult = readChunk(inputStream, chunkFile, optimalBlockSize); + long chunkSize = chunkResult.bytesRead; + if (chunkResult.exception != null) { + streamException = chunkResult.exception; + } - // Validate chunk against remaining block budget - validateRemainingBlockBudget(upload, blockIds.size()); + if (chunkSize <= 0) { + break; + } - // Stage block directly to Azure Block Blob - stageChunkFile(chunkFile, chunkSize, blockBlobClient, blockIds); + if (effectiveMaxAppendSize != null + && (totalAppended + chunkSize) > effectiveMaxAppendSize) { + // If maxAppendSize is exceeded, stop reading from the stream but commit + // previously staged blocks so no uploaded data or offset is corrupted. + pendingTusException = + new MaxAppendSizeExceededException( + "Append payload size " + + (totalAppended + chunkSize) + + " exceeded limit of " + + effectiveMaxAppendSize); + break; + } - if (streamException != null) { + int plannedIndex = blockIds.size() + plannedBlockIds.size(); + // Validate chunk against remaining block budget + validateRemainingBlockBudget(upload, plannedIndex); + + String blockId = generateBlockId(plannedIndex); + plannedBlockIds.add(blockId); + plannedChunkSizes.add(chunkSize); + totalAppended += chunkSize; + + File fileToUpload = chunkFile; + uploader.submitChunk( + chunkFile, + chunkSize, + blockId, + () -> stageBlock(blockBlobClient, blockId, fileToUpload, chunkSize)); + handedOff = true; + + if (streamException != null) { + break; + } + } catch (IOException e) { + streamException = e; break; + } catch (TusException te) { + pendingTusException = te; + break; + } finally { + if (!handedOff) { + deleteFileQuietly(chunkFile); + } } + } + + // Drain remaining staged chunks with configured timeout (defaults to 55s, lock wait - 5s). + // Catch drainException so any chunks confirmed uploaded before timeout or error + // are committed and the metadata offset is preserved without loss. + try { + uploader.drainAndComplete(); } catch (IOException e) { - streamException = e; - break; - } catch (TusException te) { - pendingTusException = te; - break; + drainException = e; } finally { - deleteFileQuietly(chunkFile); + int confirmed = Math.min(uploader.getConfirmedCount(), plannedBlockIds.size()); + for (int i = 0; i < confirmed; i++) { + blockIds.add(plannedBlockIds.get(i)); + } } } + // Calculate actual confirmed bytes appended + long confirmedAppended = 0L; + for (int i = 0; i < blockIds.size() - initialBlockCount; i++) { + confirmedAppended += plannedChunkSizes.get(i); + } + // 5. Commit any newly staged blocks in batch (1 single commit call for entire request) // Batching block commits into a single call at the end eliminates redundant network // round-trips for every chunk, drastically improving performance. In addition, committing // here before throwing any pending stream or limit exception guarantees zero data loss - // if network drops or limits are hit midway. + // if client pauses, network drops, or limits are hit midway. if (blockIds.size() > initialBlockCount) { try { blockBlobClient.commitBlockList(blockIds, true); - upload.setOffset(initialOffset + totalAppended); + upload.setOffset(initialOffset + confirmedAppended); + upload.setUploadPartKeys(new ArrayList<>(blockIds)); if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { upload.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); } @@ -292,6 +370,10 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } } + if (drainException != null) { + throw drainException; + } + if (streamException != null) { throw streamException; } @@ -301,7 +383,16 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } // 6. Validate minimum append size constraints if configured - validateMinAppendSize(totalAppended); + // Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no content, + // or to requests completing the upload by including the Upload-Complete: ?1 header field." + boolean isCompletingOrEmpty = + !upload.isUploadInProgress() + || (upload.getLength() != null + && upload.getOffset() != null + && upload.getOffset() >= upload.getLength()); + if (!isCompletingOrEmpty) { + validateMinAppendSize(confirmedAppended); + } return upload; } @@ -432,16 +523,25 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio containerClient.getBlobClient(checksumKey).deleteIfExists(); } - // 4. Delete lock target and stop signal blobs (handling active lease exceptions gracefully) + // 4. Delete lock target and stop signal blobs try { containerClient.getBlobClient(locksPrefix + id + ".stop").deleteIfExists(); } catch (Exception ignored) { } try { - containerClient.getBlobClient(locksPrefix + id + ".lock").deleteIfExists(); + BlobClient lockBlob = containerClient.getBlobClient(locksPrefix + id + ".lock"); + // Check if lock blob is actively leased by the current request before attempting deletion. + // Attempting deleteIfExists() on a leased blob without the lease ID causes Azure Blob + // Storage to reject the request with HTTP 412 (LeaseIdMissing) and logs an SDK error. + if (Boolean.TRUE.equals(lockBlob.exists())) { + BlobProperties props = lockBlob.getProperties(); + if (props.getLeaseState() != null + && !Strings.CS.equals(props.getLeaseState().toString(), "leased")) { + lockBlob.deleteIfExists(); + } + } } catch (Exception ignored) { - // Lock blob may be actively leased by current request lock (Azure 412 LeaseIdMissing) } log.debug("Terminated upload with ID {}", id); @@ -552,6 +652,9 @@ public void cleanupExpiredUploads(UploadLockingService lockingService) throws IO } catch (UploadNotFoundException ignored) { // Upload was already terminated or deleted concurrently } + if (lockingService != null) { + lockingService.cleanupLock(id); + } } } } @@ -653,6 +756,52 @@ public long getPreferredBlockSize() { return preferredBlockSize; } + @Override + public void setCloudUploadThreadPoolSize(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "The cloud upload thread pool size must be greater than 0"); + } + this.cloudUploadThreadPoolSize = size; + if (size > uploadExecutor.getMaximumPoolSize()) { + uploadExecutor.setMaximumPoolSize(size); + uploadExecutor.setCorePoolSize(size); + } else { + uploadExecutor.setCorePoolSize(size); + uploadExecutor.setMaximumPoolSize(size); + } + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + + @Override + public void close() throws IOException { + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + uploadExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + uploadExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + // --- Helper Methods --- /** Calculates auto-calibrated optimal block size based on total upload length. */ @@ -747,7 +896,7 @@ private List getCommittedBlockIds(BlockBlobClient blockBlobClient) { } /** Validates effective max append size limit. */ - private void validateMaxAppendSize(long totalAppended, Long effectiveMaxAppendSize) + void validateMaxAppendSize(long totalAppended, Long effectiveMaxAppendSize) throws MaxAppendSizeExceededException { if (effectiveMaxAppendSize != null && totalAppended > effectiveMaxAppendSize) { throw new MaxAppendSizeExceededException( @@ -759,23 +908,24 @@ private void validateMaxAppendSize(long totalAppended, Long effectiveMaxAppendSi } /** Validates min append size limit. */ - private void validateMinAppendSize(long totalAppended) throws MinAppendSizeNotMetException { + void validateMinAppendSize(long totalAppended) throws MinAppendSizeNotMetException { if (minAppendSize != null && totalAppended < minAppendSize) { throw new MinAppendSizeNotMetException( "Append size " + totalAppended + " is less than minimum allowed of " + minAppendSize); } } - /** Stages local chunk temp file as a Block Blob block. */ - private void stageChunkFile( - File chunkFile, long chunkSize, BlockBlobClient blockBlobClient, List blockIds) + /** + * Stages local chunk temp file as a Block Blob block without mutating blockIds on the worker + * thread. + */ + private void stageBlock( + BlockBlobClient blockBlobClient, String blockId, File chunkFile, long chunkSize) throws IOException { if (chunkSize > 0) { - String chunkBlockId = generateBlockId(blockIds.size()); try (InputStream chunkIs = new java.io.BufferedInputStream(new FileInputStream(chunkFile))) { - blockBlobClient.stageBlock(chunkBlockId, chunkIs, chunkSize); + blockBlobClient.stageBlock(blockId, chunkIs, chunkSize); } - blockIds.add(chunkBlockId); } } @@ -830,9 +980,18 @@ private void commitEmptyDataBlob(UploadId id) { blockBlobClient.commitBlockList(new ArrayList<>(), true); } - /** Generates Base64 encoded block ID matching Azure Block Blob standards. */ - private String generateBlockId(int index) { - String idString = String.format("block-%06d", index); + /** + * Generates a Base64-encoded block ID matching Azure Block Blob standards. + * + *

Incorporates an 8-character random UUID suffix to guarantee that late-landing blocks from + * previous aborted or timed-out worker threads never collide with or overwrite blocks in + * + * @param index The block index + * @return The Base64-encoded block ID + */ + String generateBlockId(int index) { + String idString = + String.format("blk-%06d-%s", index, UUID.randomUUID().toString().substring(0, 8)); return Base64.getEncoder().encodeToString(idString.getBytes(StandardCharsets.UTF_8)); } @@ -881,7 +1040,7 @@ private String sanitizePrefix(String prefix) { } /** Deletes local file suppressing exceptions. */ - private void deleteFileQuietly(File file) { + void deleteFileQuietly(File file) { if (file != null && file.exists()) { try { Files.delete(file.toPath()); diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java index c0daa6b5..e9898f58 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java @@ -1,6 +1,7 @@ package me.desair.tus.server.upload.azure; import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.specialized.BlobLeaseClient; import java.io.IOException; import java.lang.ref.WeakReference; @@ -8,6 +9,8 @@ import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.Utils; @@ -25,7 +28,6 @@ public class AzureBlobUploadLock implements UploadLock { private static final Logger log = LoggerFactory.getLogger(AzureBlobUploadLock.class); - private static final long RENEWAL_INTERVAL_SECONDS = 10L; private final BlobLeaseClient leaseClient; @@ -33,8 +35,11 @@ public class AzureBlobUploadLock implements UploadLock { private final String uploadUri; private final String uploadId; private final Map> activeStreams; + private final Map activeLocks; private final ScheduledExecutorService renewalExecutor; - private volatile boolean released = false; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean released = false; + private boolean blobDeleted = false; /** * Constructs an {@link AzureBlobUploadLock} wrapping an acquired Azure Blob lease. @@ -50,6 +55,7 @@ public AzureBlobUploadLock(BlobLeaseClient leaseClient, BlobClient lockBlob, Str uploadUri, null, null, + null, null); } @@ -74,7 +80,35 @@ public AzureBlobUploadLock( uploadUri, null, uploadId, - activeStreams); + activeStreams, + null); + } + + /** + * Full constructor for active lock with stream registration, lease tracking, and active lock map. + * + * @param leaseClient The pre-acquired {@link BlobLeaseClient} holding the lease + * @param lockBlob The target lock {@link BlobClient} + * @param uploadUri The upload URI associated with this lock + * @param uploadId The upload ID associated with this lock + * @param activeStreams JVM-wide map of active input streams + * @param activeLocks Map of currently active locks in the locking service + */ + public AzureBlobUploadLock( + BlobLeaseClient leaseClient, + BlobClient lockBlob, + String uploadUri, + String uploadId, + Map> activeStreams, + Map activeLocks) { + this( + Objects.requireNonNull(leaseClient, "leaseClient must not be null"), + Objects.requireNonNull(lockBlob, "lockBlob must not be null"), + uploadUri, + null, + uploadId, + activeStreams, + activeLocks); } AzureBlobUploadLock( @@ -82,7 +116,7 @@ public AzureBlobUploadLock( BlobClient lockBlob, String uploadUri, ScheduledExecutorService renewalExecutor) { - this(leaseClient, lockBlob, uploadUri, renewalExecutor, null, null); + this(leaseClient, lockBlob, uploadUri, renewalExecutor, null, null, null); } AzureBlobUploadLock( @@ -92,11 +126,23 @@ public AzureBlobUploadLock( ScheduledExecutorService renewalExecutor, String uploadId, Map> activeStreams) { + this(leaseClient, lockBlob, uploadUri, renewalExecutor, uploadId, activeStreams, null); + } + + AzureBlobUploadLock( + BlobLeaseClient leaseClient, + BlobClient lockBlob, + String uploadUri, + ScheduledExecutorService renewalExecutor, + String uploadId, + Map> activeStreams, + Map activeLocks) { this.leaseClient = leaseClient; this.lockBlob = lockBlob; this.uploadUri = Objects.requireNonNull(uploadUri, "uploadUri must not be null"); this.uploadId = uploadId; this.activeStreams = activeStreams; + this.activeLocks = activeLocks; if (renewalExecutor != null) { this.renewalExecutor = renewalExecutor; @@ -114,32 +160,107 @@ public AzureBlobUploadLock( } } + private static final long LEASE_DURATION_MS = 60000L; + private long leaseExpiresAt = System.currentTimeMillis() + LEASE_DURATION_MS; + BlobLeaseClient getLeaseClient() { return leaseClient; } + void markBlobDeleted() { + thisObjectLock.writeLock().lock(); + try { + this.blobDeleted = true; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + + boolean isBlobDeleted() { + thisObjectLock.readLock().lock(); + try { + return blobDeleted; + } finally { + thisObjectLock.readLock().unlock(); + } + } + + void setLeaseExpiresAt(long leaseExpiresAt) { + thisObjectLock.writeLock().lock(); + try { + this.leaseExpiresAt = leaseExpiresAt; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + /** Attempts to renew the lease with Azure Blob Storage. */ void renewLease() { - if (released) { - return; + thisObjectLock.readLock().lock(); + try { + if (released) { + return; + } + } finally { + thisObjectLock.readLock().unlock(); } try { executeRenew(); + thisObjectLock.writeLock().lock(); + try { + leaseExpiresAt = System.currentTimeMillis() + LEASE_DURATION_MS; + } finally { + thisObjectLock.writeLock().unlock(); + } log.trace("Successfully renewed Azure blob lease for upload URI {}", uploadUri); } catch (Exception e) { - log.warn("Failed to renew Azure blob lease for upload URI {}: {}", uploadUri, e.getMessage()); - released = true; - shutdownExecutor(); - // If lease renewal fails, abort any active input stream immediately - // so the upload thread does not continue writing un-locked data to Azure. - if (activeStreams != null && uploadId != null) { - WeakReference streamRef = activeStreams.remove(uploadId); - if (streamRef != null) { - log.info( - "Aborting active stream for upload ID {} due to lease renewal failure", uploadId); - Utils.interruptStream(streamRef.get()); + boolean leaseLost = false; + if (e instanceof BlobStorageException) { + AzureErrorType errorType = AzureUtils.parseErrorResponse((BlobStorageException) e); + if (errorType == AzureErrorType.LEASE_ALREADY_PRESENT + || errorType == AzureErrorType.LEASE_NOT_PRESENT + || errorType == AzureErrorType.CONFLICT) { + leaseLost = true; } } + + boolean expired; + thisObjectLock.readLock().lock(); + try { + expired = System.currentTimeMillis() >= leaseExpiresAt; + } finally { + thisObjectLock.readLock().unlock(); + } + + if (leaseLost || expired) { + log.warn( + "Azure blob lease definitively lost/expired for upload URI {}: {}", + uploadUri, + e.getMessage()); + thisObjectLock.writeLock().lock(); + try { + released = true; + } finally { + thisObjectLock.writeLock().unlock(); + } + shutdownExecutor(); + // If lease renewal fails definitively, abort any active input stream immediately + // so the upload thread does not continue writing un-locked data to Azure. + if (activeStreams != null && uploadId != null) { + WeakReference streamRef = activeStreams.remove(uploadId); + if (streamRef != null) { + log.info( + "Aborting active stream for upload ID {} due to lease renewal failure", uploadId); + Utils.interruptStream(streamRef.get()); + } + } + } else { + log.warn( + "Transient failure renewing Azure blob lease for upload URI {} (will retry on next tick):" + + " {}", + uploadUri, + e.getMessage()); + } } } @@ -152,24 +273,40 @@ void executeRenew() { @Override public void release() { - if (!released) { - released = true; - shutdownExecutor(); - // Remove active stream registration from JVM map upon release - // to avoid stale references lingering in heap. - if (activeStreams != null && uploadId != null) { - activeStreams.remove(uploadId); + thisObjectLock.writeLock().lock(); + try { + if (released) { + return; } - if (leaseClient != null) { - try { - leaseClient.releaseLease(); - log.trace("Released Azure blob lease for upload URI {}", uploadUri); - } catch (Exception e) { - log.debug( - "Azure blob lease release failed (may have already expired/broken) for URI {}: {}", - uploadUri, - e.getMessage()); - } + released = true; + } finally { + thisObjectLock.writeLock().unlock(); + } + shutdownExecutor(); + // Remove active stream and lock registrations from service maps upon release + // to avoid stale references lingering in heap. + if (activeStreams != null && uploadId != null) { + activeStreams.remove(uploadId); + } + if (activeLocks != null && uploadId != null) { + activeLocks.remove(uploadId); + } + boolean alreadyDeleted; + thisObjectLock.readLock().lock(); + try { + alreadyDeleted = blobDeleted; + } finally { + thisObjectLock.readLock().unlock(); + } + if (leaseClient != null && !alreadyDeleted) { + try { + leaseClient.releaseLease(); + log.trace("Released Azure blob lease for upload URI {}", uploadUri); + } catch (Exception e) { + log.debug( + "Azure blob lease release failed (may have already expired/broken) for URI {}: {}", + uploadUri, + e.getMessage()); } } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java index b7e1a105..af579f28 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java @@ -58,7 +58,11 @@ public static AzureErrorType parseErrorResponse(BlobStorageException exception) return AzureErrorType.API_NOT_IMPLEMENTED; } - if (statusCode == 403 || errorCodeStr.contains("authorizationfailure")) { + if (statusCode == 401 + || statusCode == 403 + || errorCodeStr.contains("authorizationfailure") + || errorCodeStr.contains("cannotverifycopysource") + || errorCodeStr.contains("noauthenticationinformation")) { return AzureErrorType.ACCESS_DENIED; } diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index 76241689..9e8a1280 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.ref.WeakReference; +import java.time.Duration; import java.util.Objects; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; @@ -233,6 +234,26 @@ public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm alg return storageServiceDelegate.getUploadInfoByChecksum(checksum, algorithm); } + @Override + public void setCloudUploadThreadPoolSize(int size) { + storageServiceDelegate.setCloudUploadThreadPoolSize(size); + } + + @Override + public int getCloudUploadThreadPoolSize() { + return storageServiceDelegate.getCloudUploadThreadPoolSize(); + } + + @Override + public void setDrainTimeout(Duration drainTimeout) { + storageServiceDelegate.setDrainTimeout(drainTimeout); + } + + @Override + public Duration getDrainTimeout() { + return storageServiceDelegate.getDrainTimeout(); + } + @Override public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { UploadLock uploadLock = lockingServiceDelegate.lockUploadByUri(requestUri); @@ -260,6 +281,16 @@ public void requestLockRelease(String requestUri) { lockingServiceDelegate.requestLockRelease(requestUri); } + @Override + public void cleanupLock(UploadId id) throws IOException { + lockingServiceDelegate.cleanupLock(id); + } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + lockingServiceDelegate.cleanupLock(uploadUri); + } + @Override public void close() throws IOException { lockingServiceDelegate.close(); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java index dfb7c696..067fef49 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java @@ -114,6 +114,32 @@ public void cleanupStaleLocks() throws IOException { } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + // Only delete lock file if it exists and is not actively locked by another process/request. + // Notice we check Files.exists first because calling isLocked on a non-existent lock path + // would open a FileChannel with CREATE and instantiate a lock file on disk unnecessarily. + Path lockPath = getLockPath(id); + if (lockPath != null && Files.exists(lockPath)) { + if (!isLocked(id)) { + Files.deleteIfExists(lockPath); + } + } + Path stopPath = getStopPath(id); + if (stopPath != null) { + Files.deleteIfExists(stopPath); + } + } + + @Override + public void cleanupLock(String uploadUri) throws IOException { + UploadId id = idFactory != null ? idFactory.readUploadId(uploadUri) : null; + cleanupLock(id); + } + /** Checks whether the upload is locked by attempting to obtain a short-lived file lock. */ @Override public boolean isLocked(UploadId id) { diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 2269c5e2..234e7dba 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -15,6 +15,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -34,6 +35,7 @@ import me.desair.tus.server.util.Utils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +52,8 @@ public class DiskStorageService extends AbstractDiskBasedService implements Uplo private Long maxAppendSize = null; private Long minAppendSize = null; private Long minSize = null; + private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); private Long uploadExpirationPeriod = null; private UploadIdFactory idFactory; private UploadConcatenationService uploadConcatenationService; @@ -117,6 +121,29 @@ public Long getMinSize() { return minSize; } + @Override + public void setCloudUploadThreadPoolSize(int size) { + Validate.isTrue(size > 0, "The cloud upload thread pool size must be greater than 0"); + this.cloudUploadThreadPoolSize = size; + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + @Override public void setUploadDeduplicationEnabled(boolean enabled) { this.isUploadDeduplicationEnabled = enabled; @@ -338,8 +365,18 @@ public UploadInfo append(UploadInfo info, InputStream inputStream) + " bytes. You can only append to the end of an upload"); } - // write all bytes in the channel up to the configured maximum - transferred = file.transferFrom(uploadedBytes, offset, max - offset); + // write all bytes in the channel up to the configured maximum in a loop to support + // transfers larger than 2GB (OS syscall limits) + long bytesTransferred = 0; + while (offset + bytesTransferred < max) { + long toTransfer = max - (offset + bytesTransferred); + long n = file.transferFrom(uploadedBytes, offset + bytesTransferred, toTransfer); + if (n <= 0) { + break; + } + bytesTransferred += n; + } + transferred = bytesTransferred; file.force(true); newOffset = offset + transferred; @@ -492,8 +529,18 @@ public void copyUploadTo(UploadInfo info, OutputStream outputStream) "The upload bytes for id " + readId + " could not be found."); } try (FileChannel file = FileChannel.open(bytesPath, READ)) { - // Efficiently copy the bytes to the output stream - file.transferTo(0, upload.getLength(), outputChannel); + // Efficiently copy the bytes to the output stream in a loop to handle transfers + // exceeding OS single-syscall limits (e.g. 2GB sendfile limit) + long position = 0; + long remaining = upload.getLength() != null ? upload.getLength() : file.size(); + while (remaining > 0) { + long bytesTransferred = file.transferTo(position, remaining, outputChannel); + if (bytesTransferred <= 0) { + break; + } + position += bytesTransferred; + remaining -= bytesTransferred; + } } } } @@ -508,6 +555,9 @@ public void cleanupExpiredUploads(UploadLockingService uploadLockingService) thr for (Path path : expiredUploadsStream) { FileUtils.deleteDirectory(path.toFile()); + if (uploadLockingService != null) { + uploadLockingService.cleanupLock(new UploadId(path.getFileName().toString())); + } } } } diff --git a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java index f144f25d..635ce45b 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java @@ -160,6 +160,24 @@ public void cleanupStaleLocks() throws IOException { } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + Path lockDirPath = getLockDirPath(id); + if (lockDirPath != null && Files.exists(lockDirPath)) { + // Safe eviction under sibling mutex to prevent deleting an active successor's lease + if (isLockExpired(id)) { + atomicEvictExpiredLock(lockDirPath); + } + } + Path stopFilePath = getStopFilePath(id); + if (stopFilePath != null) { + Utils.deletePathQuietly(stopFilePath); + } + } + /** * Attempts primary lock acquisition or in-place takeover of an expired lease under the protection * of a {@link LeaseFileMutex}. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index 640cf20a..b6207758 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -1,9 +1,7 @@ package me.desair.tus.server.upload.s3; -import io.minio.ComposeObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.SourceObject; import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; @@ -47,75 +45,126 @@ public class S3ConcatenationService implements UploadConcatenationService { private final String objectPrefix; private final long minPartSize; private final Path temporaryDirectory; - private UploadStorageService uploadStorageService; + private final UploadStorageService uploadStorageService; + private S3ServerSideComposeHelper s3ComposeHelper; /** - * Basic constructor using default object prefix ("uploads/") and Java temp directory. + * Convenience constructor for local S3-compatible backends taking backing {@link + * UploadStorageService}. Defaults the region to "local". * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + * @param uploadStorageService Underlying storage service */ - public S3ConcatenationService(MinioClient minioClient, String bucket) { - this(minioClient, bucket, "uploads/", null, null); + public S3ConcatenationService( + String endpoint, + String accessKey, + String secretKey, + String bucket, + UploadStorageService uploadStorageService) { + this(endpoint, "local", accessKey, secretKey, bucket, uploadStorageService); } /** - * Convenient constructor taking MinioClient, bucket, and UploadStorageService. + * Convenient constructor taking connection parameters and backing {@link UploadStorageService}. * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name * @param uploadStorageService Underlying storage service */ public S3ConcatenationService( - MinioClient minioClient, String bucket, UploadStorageService uploadStorageService) { - this(minioClient, bucket, "uploads/", uploadStorageService, null); + String endpoint, + String region, + String accessKey, + String secretKey, + String bucket, + UploadStorageService uploadStorageService) { + this( + endpoint, + region, + accessKey, + secretKey, + bucket, + "uploads/", + uploadStorageService, + null, + DEFAULT_MIN_PART_SIZE); } /** - * Constructs an S3ConcatenationService. + * Full constructor allowing custom object prefix, temporary directory, and minimum part size. * - * @param minioClient The MinIO client - * @param bucket The S3 bucket name + *

Delegates to the internal package-private constructor accepting {@link MinioClient}. + * + * @param endpoint S3 endpoint URL + * @param region S3 region name + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name * @param objectPrefix Key prefix for data objects * @param uploadStorageService Underlying storage service * @param temporaryDirectory Directory for temporary buffer files + * @param minPartSize Minimum part chunk size for server-side composition */ public S3ConcatenationService( - MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, String bucket, String objectPrefix, UploadStorageService uploadStorageService, - Path temporaryDirectory) { + Path temporaryDirectory, + long minPartSize) { this( - minioClient, + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, objectPrefix, uploadStorageService, temporaryDirectory, - DEFAULT_MIN_PART_SIZE); + minPartSize, + new S3ServerSideComposeHelper(endpoint, region, accessKey, secretKey)); } - /** Full constructor allowing custom minimum part size. */ - public S3ConcatenationService( + /** + * Internal package-private constructor accepting {@link MinioClient} where all parameter + * configuration is concentrated. + */ + S3ConcatenationService( MinioClient minioClient, String bucket, String objectPrefix, UploadStorageService uploadStorageService, Path temporaryDirectory, - long minPartSize) { + long minPartSize, + S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); this.objectPrefix = objectPrefix != null ? objectPrefix : ""; - this.uploadStorageService = uploadStorageService; + this.uploadStorageService = + Objects.requireNonNull(uploadStorageService, "UploadStorageService must not be null"); this.temporaryDirectory = temporaryDirectory != null ? temporaryDirectory : java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); this.minPartSize = minPartSize; + this.s3ComposeHelper = + s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); } - public void setUploadStorageService(UploadStorageService uploadStorageService) { - this.uploadStorageService = uploadStorageService; + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); } @Override @@ -126,8 +175,7 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep return; } - Long expirationPeriod = - uploadStorageService != null ? uploadStorageService.getUploadExpirationPeriod() : null; + Long expirationPeriod = uploadStorageService.getUploadExpirationPeriod(); List partialUploads = getPartialUploads(uploadInfo); Long totalLength = calculateTotalLength(partialUploads); @@ -162,7 +210,7 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep if (canUseServerSideCopy) { // Fast path: Compose S3 objects on cluster server-side without downloading data - mergeUsingServerSideCopy(targetObjectKey, partialUploads); + mergeUsingServerSideCopy(targetObjectKey, partialUploads, totalLength); } else { // Fallback path: Sequential stream re-upload for sub-5MB parts mergeUsingStreamingReupload(targetObjectKey, partialUploads, totalLength); @@ -175,12 +223,10 @@ public void merge(UploadInfo uploadInfo) throws IOException, UploadNotFoundExcep } uploadInfo.setStorageUploadId(targetObjectKey); - if (uploadStorageService != null) { - try { - uploadStorageService.update(uploadInfo); - } catch (UploadNotFoundException e) { - log.warn("Failed to update concatenated upload info for " + uploadInfo.getId(), e); - } + try { + uploadStorageService.update(uploadInfo); + } catch (UploadNotFoundException e) { + log.warn("Failed to update concatenated upload info for " + uploadInfo.getId(), e); } } } @@ -192,11 +238,6 @@ public InputStream getConcatenatedBytes(UploadInfo uploadInfo) return null; } - if (uploadStorageService == null) { - throw new IOException( - "UploadStorageService must be configured to retrieve concatenated upload bytes"); - } - if (uploadInfo.isUploadInProgress()) { merge(uploadInfo); } @@ -219,10 +260,7 @@ public List getPartialUploads(UploadInfo info) List output = new ArrayList<>(concatenationParts.size()); for (String childUri : concatenationParts) { - UploadInfo childInfo = - uploadStorageService != null - ? uploadStorageService.getUploadInfo(childUri, info.getOwnerKey()) - : null; + UploadInfo childInfo = uploadStorageService.getUploadInfo(childUri, info.getOwnerKey()); if (childInfo == null) { throw new UploadNotFoundException( "Upload with URI " + childUri + " was not found for owner " + info.getOwnerKey()); @@ -246,23 +284,34 @@ public List getPartialUploads(UploadInfo info) return output; } - private void mergeUsingServerSideCopy(String targetKey, List partialUploads) - throws IOException { + private void mergeUsingServerSideCopy( + String targetKey, List partialUploads, long totalLength) throws IOException { try { - List sources = new ArrayList<>(); + List partKeys = new ArrayList<>(); for (UploadInfo partial : partialUploads) { - String partKey = partial.getStorageUploadId(); - sources.add(SourceObject.builder().bucket(bucket).object(partKey).build()); + partKeys.add(partial.getStorageUploadId()); } - // Execute S3 server-side object composition - minioClient.composeObject( - ComposeObjectArgs.builder().bucket(bucket).object(targetKey).sources(sources).build()); + // Execute S3 server-side object composition via native S3 multipart copy helper. + // S3ServerSideComposeHelper executes an UploadPartCopy sequence without Content-MD5, + // avoiding MinIO SDK's internal EMPTY_BODY issue on Amazon AWS S3. + s3ComposeHelper.compose(bucket, targetKey, partKeys); } catch (Exception e) { - throw new IOException("Failed server-side S3 composeObject merge for key " + targetKey, e); + // If server-side composition fails for any reason (e.g. S3 permissions, regional policy, + // or unhandled multipart copy restriction), fall back to streaming re-upload as a safety net. + // This ensures concatenated uploads always succeed even if server-side copy is denied. + log.warn( + "Server-side S3 composition failed for target key {}; falling back to streaming re-upload: {}", + targetKey, + e.getMessage()); + mergeUsingStreamingReupload(targetKey, partialUploads, totalLength); } } + void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) { + this.s3ComposeHelper = s3ComposeHelper; + } + private void mergeUsingStreamingReupload( String targetKey, List partialUploads, long totalLength) throws IOException { try (InputStream combinedStream = @@ -297,12 +346,10 @@ private boolean checkAllCompleted(Long expirationPeriod, List partia completed = false; } else if (expirationPeriod != null) { childInfo.updateExpiration(expirationPeriod); - if (uploadStorageService != null) { - try { - uploadStorageService.update(childInfo); - } catch (UploadNotFoundException e) { - log.debug("Failed to update child upload expiration for " + childInfo.getId(), e); - } + try { + uploadStorageService.update(childInfo); + } catch (UploadNotFoundException e) { + log.debug("Failed to update child upload expiration for " + childInfo.getId(), e); } } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 02b721aa..eddf2b45 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -14,6 +14,8 @@ import java.io.InputStream; import java.util.Collections; import java.util.Objects; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.upload.AbstractLeaseLockingService; import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadId; @@ -61,58 +63,85 @@ public class S3LockingService extends AbstractLeaseLockingService { private final MinioClient minioClient; private final String bucket; private final String locksPrefix; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean s3ConditionalWritesSupported = true; /** - * Basic constructor using default lock prefix ("locks/"), 30s lease duration, and 2s polling - * interval. + * Convenience constructor for local S3-compatible backends where region is omitted. Defaults the + * region to "local". * - * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket Target S3 bucket name */ - public S3LockingService(MinioClient minioClient, String bucket) { + public S3LockingService(String endpoint, String accessKey, String secretKey, String bucket) { + this(endpoint, "local", accessKey, secretKey, bucket); + } + + /** + * Basic constructor accepting explicit connection parameters without exposing underlying client + * libraries. + * + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "eu-central-1", "us-east-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket Target S3 bucket name + */ + public S3LockingService( + String endpoint, String region, String accessKey, String secretKey, String bucket) { this( - minioClient, + endpoint, + region, + accessKey, + secretKey, bucket, DEFAULT_LOCKS_PREFIX, DEFAULT_LEASE_DURATION_MS, - DEFAULT_POLL_INTERVAL_MS); + DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); } /** - * Full constructor allowing custom configuration for all locking parameters. + * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. + * + *

Delegates to the internal package-private constructor that accepts {@link MinioClient}. * - * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL + * @param region S3 region name + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket Target S3 bucket name * @param locksPrefix Object key prefix for locks and stop signals * @param leaseDurationMs Lock lease duration in milliseconds * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals + * @param idFactory Custom {@link UploadIdFactory} */ public S3LockingService( - MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, String bucket, String locksPrefix, long leaseDurationMs, - long pollIntervalMs) { + long pollIntervalMs, + UploadIdFactory idFactory) { this( - minioClient, + buildMinioClient(endpoint, region, accessKey, secretKey), bucket, locksPrefix, leaseDurationMs, pollIntervalMs, - new UuidUploadIdFactory()); + idFactory); } /** - * Full constructor allowing custom configuration including a custom {@link UploadIdFactory}. - * - * @param minioClient Pre-configured MinIO Client - * @param bucket Target S3 bucket name - * @param locksPrefix Object key prefix for locks and stop signals - * @param leaseDurationMs Lock lease duration in milliseconds - * @param pollIntervalMs Watchdog poll interval for lock contention interrupt signals - * @param idFactory Custom {@link UploadIdFactory} + * Package-private constructor accepting {@link MinioClient} where all parameter configuration is + * concentrated. */ - public S3LockingService( + S3LockingService( MinioClient minioClient, String bucket, String locksPrefix, @@ -125,6 +154,61 @@ public S3LockingService( this.locksPrefix = sanitizePrefix(locksPrefix); } + /** + * Configures custom jitter bounds used during lock acquisition read-after-write verification. + * + *

On S3-compatible backends lacking atomic conditional writes (e.g., Wasabi, SeaweedFS, Ceph, + * Backblaze B2, older MinIO), randomized jitter backoff resolves last-write-wins races. For + * high-latency or cross-region backends, configure higher bounds (e.g. 50–200 ms). For pure AWS + * S3 or Cloudflare R2 deployments with strong conditional write enforcement, jitter can be + * disabled by passing {@code 0, 0} to maximize throughput. + * + * @param minMs Minimum jitter duration in milliseconds (must be >= 0) + * @param maxMs Maximum jitter duration in milliseconds (must be >= minMs) + * @return This service instance for fluent chaining + */ + public S3LockingService withJitter(long minMs, long maxMs) { + setJitter(minMs, maxMs); + return this; + } + + /** + * Configures whether the underlying S3 endpoint supports atomic conditional writes via {@code + * If-None-Match: *}. + * + *

Defaults to {@code true} (optimistic). If the endpoint returns HTTP 501 Not Implemented, + * this is automatically downgraded to {@code false}. For endpoints known to silently ignore + * {@code If-None-Match: *} (such as Wasabi), setting this to {@code false} ensures the service + * pre-checks existing lock status before writing. + * + * @param supported Whether conditional writes are supported + * @return This service instance for fluent chaining + */ + public S3LockingService withS3ConditionalWritesSupported(boolean supported) { + thisObjectLock.writeLock().lock(); + try { + this.s3ConditionalWritesSupported = supported; + } finally { + thisObjectLock.writeLock().unlock(); + } + return this; + } + + /** + * Returns whether the underlying S3 endpoint currently supports atomic conditional writes. + * + * @return true if conditional writes are supported or assumed supported; false if downgraded or + * disabled + */ + public boolean isS3ConditionalWritesSupported() { + thisObjectLock.readLock().lock(); + try { + return s3ConditionalWritesSupported; + } finally { + thisObjectLock.readLock().unlock(); + } + } + @Override public void cleanupStaleLocks() throws IOException { try { @@ -145,6 +229,21 @@ public void cleanupStaleLocks() throws IOException { } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id == null) { + return; + } + String lockKey = buildLockKey(id); + String stopKey = buildStopKey(id); + + // Only delete the lock object if it is expired or already absent + if (isLockExpired(lockKey)) { + deleteObjectQuietly(lockKey); + } + deleteObjectQuietly(stopKey); + } + @Override protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { if (leaseData == null) { @@ -153,9 +252,13 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { String lockKey = buildLockKey(uploadId); String stopKey = buildStopKey(uploadId); - if (!isLockExpired(lockKey)) { - return null; - } + // Optimistic conditional write: + // Skip redundant isLockExpired() pre-check. In >99.9% of requests, no lock exists, + // so an initial isLockExpired() issues an expensive S3 GET that 404s (~320ms penalty). + // By issuing putObject with "If-None-Match: *" directly, happy-path acquisition + // takes only 1 network call. If a lock already exists, S3 atomically returns 412 + // Precondition Failed, causing this method to return null. The caller + // (acquireOrEvictExpiredLock) will then inspect isLockExpired() and evict if expired. try { leaseData.setLockPath(lockKey); @@ -163,21 +266,67 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { byte[] lockContentBytes = LeaseDataJsonSerializer.serializeToBytes(leaseData); - // Layer 1: Conditional PutObject with "If-None-Match: *" - // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object already - // exists - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucket) - .object(lockKey) - .extraHeaders(Collections.singletonMap("If-None-Match", "*")) - .stream( - new ByteArrayInputStream(lockContentBytes), (long) lockContentBytes.length, -1L) - .build()); + boolean conditionalSupported; + thisObjectLock.readLock().lock(); + try { + conditionalSupported = s3ConditionalWritesSupported; + } finally { + thisObjectLock.readLock().unlock(); + } + + if (conditionalSupported) { + // Layer 1: Optimistic Conditional PutObject with "If-None-Match: *" + // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object + // already + // exists + try { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(lockKey) + .extraHeaders(Collections.singletonMap("If-None-Match", "*")) + .stream( + new ByteArrayInputStream(lockContentBytes), + (long) lockContentBytes.length, + -1L) + .build()); + } catch (ErrorResponseException e) { + S3ErrorType errorType = S3Utils.parseErrorResponse(e); + if (errorType == S3ErrorType.API_NOT_IMPLEMENTED) { + // Backend (e.g. Backblaze B2, Ceph RGW) does not support conditional writes. + // Downgrade to non-CAS arbitration mode and proceed with safe pre-check + unconditional + // write. + log.info( + "S3 endpoint does not support conditional writes (If-None-Match: *). " + + "Downgrading to non-CAS lock arbitration for key {}", + lockKey); + thisObjectLock.writeLock().lock(); + try { + s3ConditionalWritesSupported = false; + } finally { + thisObjectLock.writeLock().unlock(); + } + if (!isLockExpired(lockKey)) { + return null; + } + writeUnconditionalLockObject(lockKey, lockContentBytes); + } else { + throw e; + } + } + } else { + // Non-CAS mode (e.g., Wasabi, Ceph RGW, Backblaze B2): + // Verify no active unexpired lock exists before performing unconditional write + if (!isLockExpired(lockKey)) { + return null; + } + writeUnconditionalLockObject(lockKey, lockContentBytes); + } // Layer 2: Jittered Read-After-Write Verification - // For emulators or S3 backends where If-None-Match is not strictly enforced, - // pause for a small randomized jitter (20-60ms) and verify our holderId is still the owner + // For non-CAS backends or backends where If-None-Match is not strictly enforced, + // pause for a randomized jitter duration to allow competing writes to settle, + // then verify our holderId is still the owner applyJitter(); if (!verifyLockOwnership(lockKey, leaseData.getHolderId())) { return null; @@ -187,7 +336,7 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { } catch (ErrorResponseException e) { S3ErrorType errorType = S3Utils.parseErrorResponse(e); if (errorType == S3ErrorType.PRECONDITION_FAILED || errorType == S3ErrorType.CONFLICT) { - log.info("Lock contention for key {}: S3 conditional write precondition failed", lockKey); + log.debug("Lock contention for key {}: S3 conditional write precondition failed", lockKey); return null; } log.warn("Unexpected S3 error response acquiring lock for key {}", lockKey, e); @@ -198,6 +347,14 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { } } + private void writeUnconditionalLockObject(String lockKey, byte[] lockContentBytes) + throws Exception { + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(lockKey).stream( + new ByteArrayInputStream(lockContentBytes), (long) lockContentBytes.length, -1L) + .build()); + } + @Override protected boolean isLockExpired(UploadId uploadId) { if (uploadId == null) { @@ -212,15 +369,24 @@ protected boolean evictExpiredLock(UploadId uploadId) { return false; } String lockKey = buildLockKey(uploadId); - if (!isLockExpired(lockKey)) { - return false; - } - // Delete the expired .lock object from S3 so the subsequent tryAcquireLock() conditional - // write with "If-None-Match: *" can succeed and take over the abandoned lock. - try { + try (InputStream stream = + minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(lockKey).build())) { + LeaseData lock = LeaseDataJsonSerializer.deserialize(stream); + if (!isLeaseExpired(lock, System.currentTimeMillis())) { + return false; + } + // Actual expired lock found in S3: delete it so next tryAcquireLock() can succeed minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(lockKey).build()); log.info("Evicted expired S3 lock for key {}", lockKey); return true; + } catch (ErrorResponseException e) { + if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { + // Lock key was already released normally and removed from S3. + // Return true to indicate the resource is clear without emitting false eviction logs. + return true; + } + log.warn("Failed checking S3 lock expiration for key {}", lockKey, e); + return false; } catch (Exception e) { log.warn("Failed to evict expired S3 lock for key {}", lockKey, e); return false; @@ -307,8 +473,9 @@ private void deleteObjectQuietly(String key) { } } - void applyJitter() { - applyJitter(20L, 60L); + @Override + protected void applyJitter() { + super.applyJitter(); } private String sanitizePrefix(String prefix) { @@ -326,4 +493,14 @@ private String buildLockKey(UploadId uploadId) { private String buildStopKey(UploadId uploadId) { return locksPrefix + uploadId.toString() + ".stop"; } + + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); + } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java new file mode 100644 index 00000000..a805391d --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelper.java @@ -0,0 +1,300 @@ +package me.desair.tus.server.upload.s3; + +import io.minio.AbortMultipartUploadArgs; +import io.minio.Checksum; +import io.minio.CompleteMultipartUploadArgs; +import io.minio.ComposeObjectArgs; +import io.minio.CreateMultipartUploadArgs; +import io.minio.CreateMultipartUploadResponse; +import io.minio.Http; +import io.minio.MinioAsyncClient; +import io.minio.MinioClient; +import io.minio.Signer; +import io.minio.SourceObject; +import io.minio.Time; +import io.minio.Utils; +import io.minio.credentials.Credentials; +import io.minio.credentials.Provider; +import io.minio.credentials.StaticProvider; +import io.minio.messages.Part; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper executing native S3 Multipart Copy operations to combine part objects server-side. + * + *

Why this helper exists: MinIO Java SDK's {@code composeObject()} implementation delegates to + * {@code UploadPartCopy} with an internal {@code EMPTY_BODY} placeholder. In MinIO SDK's {@code + * Http.toRequest()}, that placeholder automatically attaches {@code Content-MD5} and {@code + * Content-Type} headers to the HTTP request. While MinIO server ignores those extra headers, Amazon + * AWS S3 strictly forbids the {@code Content-MD5} header on {@code UploadPartCopy} requests, + * returning HTTP 400 InvalidArgument ("The specified header is not valid in this context"). + * Attempting to remove the header via an OkHttp network interceptor also fails because AWS SigV4 + * signature calculation includes all present headers, resulting in a SignatureDoesNotMatch error. + * + *

Workaround: This helper executes standard S3 Multipart Upload Copy operations directly via + * OkHttp and AWS SigV4 signing without adding {@code Content-MD5}. This allows fast zero-bandwidth + * server-side composition on both Amazon AWS S3 and MinIO/Ceph clusters without needing heavy + * external AWS SDK dependencies. + * + *

Safety Fallback: If native server-side compose fails for any reason (e.g. S3 permissions, + * regional restrictions, or client reflection unavailability), callers (such as {@link + * S3StorageService} and {@link S3ConcatenationService}) catch the exception and fall back to + * sequential streaming concatenation (re-uploading the combined parts). + */ +public class S3ServerSideComposeHelper { + + private static final Logger log = LoggerFactory.getLogger(S3ServerSideComposeHelper.class); + + private final MinioClient minioClient; + private final MinioAsyncClient asyncClient; + private final Http.BaseUrl baseUrl; + private final Provider provider; + private final OkHttpClient httpClient; + private final String explicitRegion; + + /** + * Constructs an instance with explicit connection parameters without requiring a {@link + * MinioClient}. + * + * @param endpoint The S3 endpoint URL (e.g. "https://s3.amazonaws.com" or + * "http://localhost:9000") + * @param region S3 region name (optional, defaults to "local" if null or empty) + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + */ + public S3ServerSideComposeHelper( + String endpoint, String region, String accessKey, String secretKey) { + this(null, endpoint, region, accessKey, secretKey); + } + + /** + * Constructs an instance wrapping the given {@link MinioClient} without reflection. + * + *

Package-private constructor for internal testing. + * + * @param minioClient The MinIO client instance + */ + S3ServerSideComposeHelper(MinioClient minioClient) { + this(minioClient, null, null, null, null); + } + + private S3ServerSideComposeHelper( + MinioClient minioClient, String endpoint, String region, String accessKey, String secretKey) { + this.minioClient = minioClient; + this.explicitRegion = (region != null && !region.isEmpty()) ? region : "local"; + + Http.BaseUrl base = null; + Provider prov = null; + OkHttpClient client = null; + MinioAsyncClient async = null; + + if (endpoint != null && !endpoint.isEmpty()) { + try { + base = new Http.BaseUrl(endpoint); + if (region != null && !region.isEmpty()) { + base.setRegion(region); + } + prov = new StaticProvider(accessKey, secretKey, null); + client = new OkHttpClient(); + + MinioAsyncClient.Builder asyncBuilder = + MinioAsyncClient.builder().endpoint(endpoint).credentials(accessKey, secretKey); + if (region != null && !region.isEmpty()) { + asyncBuilder.region(region); + } + async = asyncBuilder.build(); + } catch (Exception e) { + log.warn( + "Failed to initialize native S3 connection from endpoint {}: {}", + endpoint, + e.getMessage()); + } + } + + this.baseUrl = base; + this.provider = prov; + this.httpClient = client; + this.asyncClient = async; + } + + /** + * Returns whether native S3 multipart copy is available with the extracted client properties. + * + * @return true if all required fields are accessible; false otherwise + */ + public boolean isAvailable() { + return asyncClient != null && baseUrl != null && httpClient != null; + } + + /** + * Composes the given part keys into the target object key entirely server-side on S3. + * + * @param bucket The S3 bucket name + * @param targetKey The destination object key + * @param partKeys The source part object keys to combine in order + * @throws Exception If composition fails + */ + public void compose(String bucket, String targetKey, List partKeys) throws Exception { + if (!isAvailable()) { + // Fallback for mocked MinioClient instances in tests + List sources = new ArrayList<>(); + for (String pk : partKeys) { + sources.add(SourceObject.builder().bucket(bucket).object(pk).build()); + } + minioClient.composeObject( + ComposeObjectArgs.builder().bucket(bucket).object(targetKey).sources(sources).build()); + return; + } + + String region = baseUrl.region(); + if (region == null || region.isEmpty()) { + region = explicitRegion; + } + if (region == null || region.isEmpty()) { + region = "local"; + } + + Credentials credentials = (provider != null) ? provider.fetch() : null; + + // 1. Create Multipart Upload + CreateMultipartUploadArgs createArgs = + CreateMultipartUploadArgs.builder().bucket(bucket).object(targetKey).build(); + CreateMultipartUploadResponse createResponse = + asyncClient.createMultipartUpload(createArgs).join(); + String uploadId = createResponse.result().uploadId(); + log.info( + "Started server-side multipart copy upload (ID: {}) for target {} in bucket {} with {} parts", + uploadId, + targetKey, + bucket, + partKeys.size()); + + List parts = new ArrayList<>(); + try { + // 2. UploadPartCopy for each part chunk without Content-MD5 + for (int i = 0; i < partKeys.size(); i++) { + int partNumber = i + 1; + String partKey = partKeys.get(i); + + Http.QueryParameters queryParams = + new Http.QueryParameters( + "partNumber", Integer.toString(partNumber), "uploadId", uploadId); + HttpUrl url = baseUrl.buildUrl(Http.Method.PUT, bucket, targetKey, region, queryParams); + + // x-amz-copy-source must be URL-encoded "/bucket/key" + String copySource = "/" + bucket + "/" + Utils.encodePath(partKey); + String now = ZonedDateTime.now().format(Time.AMZ_DATE_FORMAT); + + Request.Builder requestBuilder = + new Request.Builder() + .url(url) + .put(RequestBody.create(new byte[0], null)) + .header("Host", Utils.getHostHeader(url)) + .header("x-amz-date", now) + .header("x-amz-copy-source", copySource) + .header("x-amz-content-sha256", Checksum.ZERO_SHA256_HASH); + + if (credentials != null + && credentials.sessionToken() != null + && !credentials.sessionToken().isEmpty()) { + requestBuilder.header("x-amz-security-token", credentials.sessionToken()); + } + + Request httpRequest = requestBuilder.build(); + if (credentials != null) { + httpRequest = + Signer.signV4S3( + httpRequest, + region, + credentials.accessKey(), + credentials.secretKey(), + Checksum.ZERO_SHA256_HASH); + } + + try (Response response = httpClient.newCall(httpRequest).execute()) { + if (!response.isSuccessful()) { + String errorBody = response.body() != null ? response.body().string() : ""; + throw new IOException( + "S3 UploadPartCopy failed for part " + + partNumber + + " (" + + partKey + + "): HTTP " + + response.code() + + " " + + errorBody); + } + String responseBody = response.body() != null ? response.body().string() : ""; + String etag = extractEtagFromXml(responseBody); + if (etag == null || etag.isEmpty()) { + etag = response.header("ETag"); + } + if (etag == null || etag.isEmpty()) { + throw new IOException("Missing ETag in UploadPartCopy response for part " + partNumber); + } + parts.add(new Part(partNumber, etag)); + } + } + + // 3. Complete Multipart Upload + CompleteMultipartUploadArgs completeArgs = + CompleteMultipartUploadArgs.builder() + .bucket(bucket) + .object(targetKey) + .uploadId(uploadId) + .parts(parts.toArray(new Part[0])) + .build(); + asyncClient.completeMultipartUpload(completeArgs).join(); + log.info( + "Successfully completed server-side multipart copy upload (ID: {}) for target {} in bucket {}", + uploadId, + targetKey, + bucket); + + } catch (Exception e) { + try { + asyncClient + .abortMultipartUpload( + AbortMultipartUploadArgs.builder() + .bucket(bucket) + .object(targetKey) + .uploadId(uploadId) + .build()) + .join(); + } catch (Exception abortEx) { + log.warn("Failed to abort multipart upload {} for target {}", uploadId, targetKey, abortEx); + } + throw e; + } + } + + /** + * Extracts the ETag value from an S3 {@code CopyPartResult} XML string. + * + * @param xml The XML response from S3 UploadPartCopy + * @return Extracted ETag string, or null if not found + */ + static String extractEtagFromXml(String xml) { + if (xml == null) { + return null; + } + int start = xml.indexOf(""); + int end = xml.indexOf(""); + if (start != -1 && end != -1 && end > start + 6) { + String etag = xml.substring(start + 6, end).trim(); + etag = etag.replace(""", "\""); + return etag; + } + return null; + } +} diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index d3b9c911..1d0f50e7 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -1,16 +1,17 @@ package me.desair.tus.server.upload.s3; -import io.minio.ComposeObjectArgs; import io.minio.GetObjectArgs; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; import io.minio.Result; -import io.minio.SourceObject; import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteRequest; +import io.minio.messages.DeleteResult; import io.minio.messages.Item; import java.io.ByteArrayInputStream; import java.io.File; @@ -24,12 +25,22 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MaxUploadLengthExceededException; @@ -45,6 +56,8 @@ import me.desair.tus.server.upload.UploadType; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.upload.concatenation.UploadConcatenationService; +import me.desair.tus.server.upload.util.AsyncChunkUploader; +import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.io.FileUtils; @@ -86,12 +99,12 @@ public class S3StorageService implements UploadStorageService { public static final String DEFAULT_LOCKS_PREFIX = "locks/"; // Part Sizing Constraints (per AWS S3 & MinIO specifications) - private static final long DEFAULT_MIN_PART_SIZE = - 5L * 1024 * 1024; // 5 MB (S3 minimum part limit) - private static final long DEFAULT_PREFERRED_PART_SIZE = - 50L * 1024 * 1024; // 50 MB (Optimal chunk size) - private static final long DEFAULT_MAX_PART_SIZE = + public static final long DEFAULT_MIN_PART_SIZE = 5L * 1024 * 1024; // 5 MB (S3 minimum part limit) + public static final long DEFAULT_PREFERRED_PART_SIZE = + 8L * 1024 * 1024; // 8 MB (Optimal chunk size aligned with Azure) + public static final long DEFAULT_MAX_PART_SIZE = 5L * 1024 * 1024 * 1024L; // 5 GB (S3 maximum object/part limit) + public static final int MAX_PARTS_PER_UPLOAD = 10_000; private final MinioClient minioClient; private final String bucket; @@ -104,6 +117,10 @@ public class S3StorageService implements UploadStorageService { private long minPartSize = DEFAULT_MIN_PART_SIZE; private long preferredPartSize = DEFAULT_PREFERRED_PART_SIZE; + private int cloudUploadThreadPoolSize = 10; + private Duration drainTimeout = Duration.ofSeconds(55); + private final ThreadPoolExecutor uploadExecutor; + private Long maxUploadSize; private Long maxAppendSize; private Long minAppendSize; @@ -113,16 +130,41 @@ public class S3StorageService implements UploadStorageService { private UploadIdFactory idFactory = new UuidUploadIdFactory(); private UploadConcatenationService concatenationService; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean s3ComposeObjectSupported = true; + private boolean supportsBatchDelete = true; + private S3ServerSideComposeHelper s3ComposeHelper; + + /** + * Convenience constructor for local S3-compatible backends (e.g., MinIO, RustFS, Ceph) where + * region is omitted. Defaults the region to "local". + * + * @param endpoint S3 endpoint URL (e.g. "http://localhost:9000") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @param bucket S3 bucket name + */ + public S3StorageService(String endpoint, String accessKey, String secretKey, String bucket) { + this(endpoint, "local", accessKey, secretKey, bucket); + } /** - * Basic constructor using default object key prefixes and standard system temp directory. + * Basic constructor accepting explicit connection parameters without exposing underlying client + * libraries. * - * @param minioClient Pre-configured MinIO Client + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket S3 bucket name */ - public S3StorageService(MinioClient minioClient, String bucket) { + public S3StorageService( + String endpoint, String region, String accessKey, String secretKey, String bucket) { this( - minioClient, + endpoint, + region, + accessKey, + secretKey, bucket, DEFAULT_OBJECT_PREFIX, DEFAULT_METADATA_PREFIX, @@ -132,9 +174,14 @@ public S3StorageService(MinioClient minioClient, String bucket) { } /** - * Full constructor allowing full customization of object prefixes and local disk buffer path. + * Full constructor accepting explicit connection parameters and prefix/buffer customization. * - * @param minioClient Pre-configured MinIO Client + *

Delegates to the internal package-private constructor accepting {@link MinioClient}. + * + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (e.g. "us-east-1", "eu-central-1") + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password * @param bucket S3 bucket name * @param objectPrefix Key prefix for final completed file objects * @param metadataPrefix Key prefix for metadata (.info JSON and .part buffer) objects @@ -143,13 +190,40 @@ public S3StorageService(MinioClient minioClient, String bucket) { * @param temporaryDirectory Local directory path for staging chunks before S3 upload */ public S3StorageService( - MinioClient minioClient, + String endpoint, + String region, + String accessKey, + String secretKey, String bucket, String objectPrefix, String metadataPrefix, String checksumsPrefix, String locksPrefix, Path temporaryDirectory) { + this( + buildMinioClient(endpoint, region, accessKey, secretKey), + bucket, + objectPrefix, + metadataPrefix, + checksumsPrefix, + locksPrefix, + temporaryDirectory, + new S3ServerSideComposeHelper(endpoint, region, accessKey, secretKey)); + } + + /** + * Internal package-private constructor accepting {@link MinioClient} where all parameter + * configuration and initialization logic is concentrated. + */ + S3StorageService( + MinioClient minioClient, + String bucket, + String objectPrefix, + String metadataPrefix, + String checksumsPrefix, + String locksPrefix, + Path temporaryDirectory, + S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); this.objectPrefix = sanitizePrefix(objectPrefix); @@ -169,9 +243,42 @@ public S3StorageService( log.debug("Unable to ensure temporary directory exists: {}", e.getMessage()); } + AtomicInteger threadNum = new AtomicInteger(1); + this.uploadExecutor = + new ThreadPoolExecutor( + cloudUploadThreadPoolSize, + cloudUploadThreadPoolSize, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + r -> { + Thread t = new Thread(r, "tus-s3-upload-" + threadNum.getAndIncrement()); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.CallerRunsPolicy()); + + this.s3ComposeHelper = + s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); this.concatenationService = new S3ConcatenationService( - this.minioClient, this.bucket, this.objectPrefix, this, this.temporaryDirectory); + this.minioClient, + this.bucket, + this.objectPrefix, + this, + this.temporaryDirectory, + DEFAULT_MIN_PART_SIZE, + this.s3ComposeHelper); + } + + private static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); } /** @@ -292,6 +399,12 @@ public UploadInfo create(UploadInfo info, String ownerKey) throws IOException { } info.setOwnerKey(ownerKey); info.setStorageUploadId(buildObjectKey(info.getId())); + if (info.getUploadPartKeys() == null) { + info.setUploadPartKeys(new ArrayList<>()); + } + if (info.getOffset() == null) { + info.setOffset(0L); + } // Persist initial UploadInfo metadata object (.info) to S3 try { @@ -307,52 +420,88 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) throws IOException, TusException { // Step 1: Verify upload existence and check configured size limits UploadInfo info = fetchAndValidateUpload(upload.getId()); + if (upload.getLength() != null && info.getLength() == null) { + info.setLength(upload.getLength()); + } String objectKey = getS3ObjectKey(info); - String partObjectKey = buildIncompletePartKey(info.getId()); + InterruptibleInputStream interruptibleStream = Utils.toInterruptibleStream(inputStream); - // Step 2: If a previous sub-5MB .part buffer exists in S3, download & prepend it to incoming - // stream. Also roll back any existing sub-5MB numbered parts (e.g. from prior aborted - // sessions). + // Step 2: If the last part is sub-5MB, download & prepend its bytes to the incoming stream. + // The previous tail part is intentionally kept in S3 as a staleTailKey and is ONLY deleted + // after the new manifest is successfully committed to S3. This ensures zero data loss if + // this request is paused or interrupted immediately. PreparedStream preparedStream = - prepareStreamWithExistingIncompletePart(info.getId(), partObjectKey, inputStream); - - // Step 3: Process payload stream in optimal chunk parts and upload to S3 - boolean successfullyFinished = false; - try { - AppendResult appendResult = - processPayloadChunks(info, preparedStream, info.getId(), partObjectKey); - - // Step 4: Validate minimum append size constraints if configured - // Subtract prependedBytes so minAppendSize accurately measures the payload transferred - // in THIS request rather than earlier buffered bytes. - long requestPayloadAppended = - Math.max(0L, appendResult.totalBytesAppended - preparedStream.prependedBytes); - if (minAppendSize != null && requestPayloadAppended < minAppendSize) { - throw new MinAppendSizeNotMetException( - "Append payload size " - + requestPayloadAppended - + " is below minimum limit " - + minAppendSize); - } - - // Step 5: Recalculate total uploaded byte offset across all uploaded part objects in S3 - long newOffset = calculateCurrentOffset(objectKey, info.getId(), partObjectKey); + prepareStreamWithExistingIncompletePart(info, interruptibleStream); + + // Validate that the upload has not exceeded S3's 10,000 multipart parts ceiling + validateRemainingPartBudget(info, preparedStream.remainingPartKeys.size()); + + // Clean up any unmanifested orphan part objects left in S3 from earlier aborted attempts. + // Pruning unmanifested parts under the upload lock prevents listing delays and ensures + // that only parts explicitly tracked in the manifest are considered valid. + pruneOrphanParts(info.getId(), preparedStream.remainingPartKeys, preparedStream.staleTailKeys); + + // Step 3: Process payload stream in optimal chunk parts and upload to S3. + AppendResult appendResult = processPayloadChunks(info, preparedStream, info.getId()); + + // Step 4: Persist any newly confirmed chunk parts and update the byte offset. + // Drained chunks and trailing parts are committed to the manifest before rethrowing any + // stream interruption exception, ensuring maximum byte persistence. + boolean hasNewParts = + appendResult.allPartKeys.size() > preparedStream.remainingPartKeys.size() + || appendResult.confirmedBytesAppended > 0; + + if (hasNewParts) { + long currentOffsetBeforeStream = + Math.max( + info.getOffset() != null ? info.getOffset() : 0L, + preparedStream.existingPartsTotalSize + preparedStream.prependedBytes); + long baseOffset = currentOffsetBeforeStream - preparedStream.prependedBytes; + long newOffset = baseOffset + appendResult.confirmedBytesAppended; info.setOffset(newOffset); upload.setOffset(newOffset); - - // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object - finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), appendResult, newOffset); + info.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); + upload.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); update(info); - successfullyFinished = true; - return info; - } finally { - if (!successfullyFinished) { - long newOffset = calculateCurrentOffset(objectKey, info.getId(), partObjectKey); - info.setOffset(newOffset); - upload.setOffset(newOffset); - update(info); + + // Safe transactional cleanup: Delete old tail parts only AFTER the new manifest is saved. + // If the upload was interrupted earlier, staleTailKeys were untouched, preserving bytes. + for (String staleKey : preparedStream.staleTailKeys) { + deleteObjectQuietly(staleKey); } } + + // Rethrow any stream reading or size limit exceptions after metadata has been safely saved + if (appendResult.readException != null) { + throw appendResult.readException; + } + if (appendResult.maxAppendSizeException != null) { + throw appendResult.maxAppendSizeException; + } + + // Step 5: Validate minimum append size constraints if configured. + // Subtract prependedBytes so minAppendSize accurately measures payload transferred in THIS + // request. Per RUFH §4.1.4: "This limit does not apply to upload creation requests with no + // content, or to requests completing the upload by including the Upload-Complete: ?1 header + // field." + long currentOffset = info.getOffset() != null ? info.getOffset() : 0L; + boolean isCompletingOrEmpty = + !info.isUploadInProgress() + || (info.getLength() != null && currentOffset >= info.getLength()); + long requestPayloadAppended = + Math.max(0L, appendResult.confirmedBytesAppended - preparedStream.prependedBytes); + if (minAppendSize != null && !isCompletingOrEmpty && requestPayloadAppended < minAppendSize) { + throw new MinAppendSizeNotMetException( + "Append payload size " + + requestPayloadAppended + + " is below minimum limit " + + minAppendSize); + } + + // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object + finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), currentOffset); + update(info); + return info; } @Override @@ -457,6 +606,10 @@ public void cleanupExpiredUploads(UploadLockingService uploadLockingService) thr && info.isExpired() && (uploadLockingService == null || !uploadLockingService.isLocked(id))) { terminateUpload(info); + if (uploadLockingService != null) { + // Clean up any remaining lock files now that the expired upload is terminated + uploadLockingService.cleanupLock(id); + } } } } @@ -475,20 +628,24 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) return; } String objectKey = getS3ObjectKey(uploadInfo); - String partKey = buildIncompletePartKey(uploadInfo.getId()); - long newOffset = Math.max(0L, uploadInfo.getOffset() - byteCount); uploadInfo.setOffset(newOffset); - update(uploadInfo); // If final completed object exists in S3, truncate it if (objectExists(objectKey)) { - truncateFromCompletedObject(objectKey, partKey, newOffset); + truncateCompletedObject(uploadInfo, objectKey, newOffset); + update(uploadInfo); + return; + } + + // If upload has manifest parts in uploadPartKeys, truncate backwards from parts + if (uploadInfo.getUploadPartKeys() != null && !uploadInfo.getUploadPartKeys().isEmpty()) { + truncateManifestParts(uploadInfo, byteCount); + update(uploadInfo); return; } - // Otherwise truncate from incomplete .part object - truncateFromIncompletePart(partKey, byteCount); + update(uploadInfo); } @Override @@ -498,12 +655,10 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio } String objectKey = getS3ObjectKey(uploadInfo); String metadataKey = buildMetadataKey(uploadInfo.getId()); - String partKey = buildIncompletePartKey(uploadInfo.getId()); - // Delete final object, metadata object, and incomplete part object from S3 + // Delete final object and metadata object from S3 deleteObjectQuietly(objectKey); deleteObjectQuietly(metadataKey); - deleteObjectQuietly(partKey); // Delete all temporary part chunk objects (e.g. metadata/.part.00001) deleteAllPartObjectsQuietly(uploadInfo.getId()); @@ -680,6 +835,32 @@ public UploadConcatenationService getUploadConcatenationService() { return concatenationService; } + boolean isS3ComposeObjectSupported() { + thisObjectLock.readLock().lock(); + try { + return s3ComposeObjectSupported; + } finally { + thisObjectLock.readLock().unlock(); + } + } + + void setS3ComposeObjectSupported(boolean s3ComposeObjectSupported) { + thisObjectLock.writeLock().lock(); + try { + this.s3ComposeObjectSupported = s3ComposeObjectSupported; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + + void setS3ServerSideComposeHelper(S3ServerSideComposeHelper s3ComposeHelper) { + this.s3ComposeHelper = s3ComposeHelper; + } + + S3ServerSideComposeHelper getS3ServerSideComposeHelper() { + return s3ComposeHelper; + } + @Override public void setIdFactory(UploadIdFactory idFactory) { if (idFactory != null) { @@ -687,6 +868,96 @@ public void setIdFactory(UploadIdFactory idFactory) { } } + @Override + public void setCloudUploadThreadPoolSize(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "The cloud upload thread pool size must be greater than 0"); + } + this.cloudUploadThreadPoolSize = size; + if (size > uploadExecutor.getMaximumPoolSize()) { + uploadExecutor.setMaximumPoolSize(size); + uploadExecutor.setCorePoolSize(size); + } else { + uploadExecutor.setCorePoolSize(size); + uploadExecutor.setMaximumPoolSize(size); + } + } + + @Override + public int getCloudUploadThreadPoolSize() { + return cloudUploadThreadPoolSize; + } + + @Override + public void setDrainTimeout(Duration drainTimeout) { + if (drainTimeout != null) { + this.drainTimeout = drainTimeout; + } + } + + @Override + public Duration getDrainTimeout() { + return drainTimeout; + } + + /** + * Set the preferred chunk part size in bytes used when buffering and uploading parts to S3. + * + * @param preferredPartSize Part size in bytes (must be between 5 MB and 5 GB) + */ + public void setPreferredPartSize(long preferredPartSize) { + if (preferredPartSize < minPartSize || preferredPartSize > DEFAULT_MAX_PART_SIZE) { + throw new IllegalArgumentException( + "Preferred part size must be between " + + minPartSize + + " and " + + DEFAULT_MAX_PART_SIZE + + " bytes"); + } + this.preferredPartSize = preferredPartSize; + } + + /** + * Return the preferred chunk part size in bytes used when buffering and uploading parts to S3. + * + * @return Preferred part size in bytes + */ + public long getPreferredPartSize() { + return preferredPartSize; + } + + boolean isSupportsBatchDelete() { + thisObjectLock.readLock().lock(); + try { + return supportsBatchDelete; + } finally { + thisObjectLock.readLock().unlock(); + } + } + + void setSupportsBatchDelete(boolean supportsBatchDelete) { + thisObjectLock.writeLock().lock(); + try { + this.supportsBatchDelete = supportsBatchDelete; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + + @Override + public void close() throws IOException { + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + uploadExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + uploadExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + // PRIVATE HELPER METHODS & S3 PROCESSING LOGIC private UploadInfo fetchAndValidateUpload(UploadId uploadId) @@ -713,52 +984,34 @@ private void validateUploadLimits(UploadInfo info) throws TusException { } /** - * Checks if an incomplete sub-5MB {@code .part} buffer object exists in S3 from a previous - * interrupted or paused request. If present, downloads its bytes and prepends them to the - * incoming stream. + * Prepares the incoming input stream by rolling back any prior sub-5MB incomplete tail part into + * the stream buffer so it can continue growing. * - *

Additionally, if prior paused sessions promoted a sub-5MB chunk to a numbered part (e.g. - * {@code .part.00001}), this method dynamically rolls back that sub-5MB chunk from S3 into the - * stream buffer so that subsequent appends can grow the chunk past AWS S3's 5 MB minimum part - * size. + *

The previous tail part is intentionally kept in S3 as a {@code staleTailKey} and is ONLY + * deleted after the new manifest is successfully committed to S3. This ensures zero data loss if + * this request is paused or interrupted immediately. * - * @param id The upload identifier - * @param partObjectKey The S3 object key for the incomplete .part buffer - * @param inputStream The incoming request input stream - * @return {@link PreparedStream} containing the combined stream and prepended byte count + * @param info The upload metadata + * @param interruptibleStream The incoming request interruptible input stream + * @return {@link PreparedStream} containing the combined stream, prepended bytes, remaining + * parts, and stale tail keys * @throws IOException If reading or downloading from S3 fails */ private PreparedStream prepareStreamWithExistingIncompletePart( - UploadId id, String partObjectKey, InputStream inputStream) throws IOException { + UploadInfo info, InterruptibleInputStream interruptibleStream) throws IOException { long prependedBytes = 0L; - InputStream combinedStream = inputStream; + InputStream combinedStream = interruptibleStream; + List staleTailKeys = new ArrayList<>(); - // 1. Check for leftover incomplete sub-5MB .part buffer object - try { - StatObjectResponse partHead = - minioClient.statObject( - StatObjectArgs.builder().bucket(bucket).object(partObjectKey).build()); - if (partHead != null && partHead.size() > 0) { - try (InputStream partStream = - minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(partObjectKey).build())) { - byte[] partBytes = IOUtils.toByteArray(partStream); - prependedBytes += partBytes.length; - combinedStream = - new SequenceInputStream(new ByteArrayInputStream(partBytes), combinedStream); - } - deleteObjectQuietly(partObjectKey); - } - } catch (ErrorResponseException e) { - if (S3Utils.parseErrorResponse(e) != S3ErrorType.NO_SUCH_KEY) { - log.debug("Error checking incomplete part object {}: {}", partObjectKey, e.getMessage()); - } - } catch (Exception e) { - log.debug("Unexpected error reading incomplete part {}: {}", partObjectKey, e.getMessage()); - } + // 1. Fetch current committed part keys from manifest + List existingPartKeys = + info.getUploadPartKeys() != null + ? new ArrayList<>(info.getUploadPartKeys()) + : new ArrayList<>(); - // 2. Roll back any existing sub-5MB numbered parts (e.g. from prior aborted or paused uploads) - List existingPartKeys = fetchExistingPartKeys(id); + long existingPartsTotalSize = calculateTotalPartsSize(existingPartKeys); + + // 2. Roll back any existing sub-5MB numbered parts into the stream buffer so they can grow if (!existingPartKeys.isEmpty()) { String lastPartKey = existingPartKeys.get(existingPartKeys.size() - 1); try { @@ -767,7 +1020,7 @@ private PreparedStream prepareStreamWithExistingIncompletePart( StatObjectArgs.builder().bucket(bucket).object(lastPartKey).build()); if (stat.size() < minPartSize) { log.info( - "Rolling back sub-5MB numbered part {} (size: {}) into stream buffer", + "Rolling back sub-5MB part {} (size: {}) into stream buffer", lastPartKey, stat.size()); try (InputStream lastPartStream = @@ -777,132 +1030,233 @@ private PreparedStream prepareStreamWithExistingIncompletePart( prependedBytes += lastPartBytes.length; combinedStream = new SequenceInputStream(new ByteArrayInputStream(lastPartBytes), combinedStream); + staleTailKeys.add(lastPartKey); + existingPartsTotalSize -= lastPartBytes.length; + existingPartKeys.remove(existingPartKeys.size() - 1); } - deleteObjectQuietly(lastPartKey); - existingPartKeys.remove(existingPartKeys.size() - 1); } } catch (Exception e) { log.debug("Error inspecting last part key {}: {}", lastPartKey, e.getMessage()); } } - return new PreparedStream(combinedStream, prependedBytes, existingPartKeys); + return new PreparedStream( + combinedStream, + interruptibleStream, + prependedBytes, + existingPartsTotalSize, + existingPartKeys, + staleTailKeys); + } + + private long calculateTotalPartsSize(List partKeys) { + long totalSize = 0L; + for (String key : partKeys) { + try { + StatObjectResponse stat = + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(key).build()); + if (stat != null) { + totalSize += stat.size(); + } + } catch (Exception ignored) { + } + } + return totalSize; + } + + /** + * Prunes unmanifested orphaned part objects left in S3 from earlier aborted parallel attempts. + * + * @param id The upload identifier + * @param manifestedParts Parts actively tracked in the manifest + * @param staleTailKeys Stale tail parts pending replacement in this request + */ + private void pruneOrphanParts( + UploadId id, List manifestedParts, List staleTailKeys) { + List s3Parts = fetchExistingPartKeys(id); + for (String s3Part : s3Parts) { + if (!manifestedParts.contains(s3Part) && !staleTailKeys.contains(s3Part)) { + log.debug("Pruning unmanifested orphan S3 part {}", s3Part); + deleteObjectQuietly(s3Part); + } + } } /** * Reads bytes from the incoming stream into temporary local files of optimal part size (default - * 50MB). Parts ≥ 5MB are uploaded immediately to S3 as part chunk objects. Any trailing chunk - * under 5MB is saved as a temporary .part object unless it completes the overall upload. + * 8MB). Parts ≥ 5MB are uploaded asynchronously to S3 via {@link AsyncChunkUploader}, + * overlapping client stream reading with cloud upload. Any trailing chunk under 5MB is uploaded + * directly to S3 and tracked in the manifest. */ private AppendResult processPayloadChunks( - UploadInfo info, PreparedStream preparedStream, UploadId id, String partObjectKey) - throws IOException, MaxAppendSizeExceededException { + UploadInfo info, PreparedStream preparedStream, UploadId id) { List allPartKeys = new ArrayList<>(preparedStream.remainingPartKeys); int nextPartNumber = allPartKeys.size() + 1; - InputStream streamToRead = preparedStream.stream; - long optimalPartSize = calcOptimalPartSize(info.getLength() != null ? info.getLength() : 0); + long optimalPartSize = calcOptimalPartSize(info.getLength()); byte[] buffer = new byte[8192]; long totalBytesAppended = 0; - - // Calibrate base offset: info.getOffset() already includes prependedBytes from prior writes. - // Subtracting prependedBytes ensures currentTotalOffset accurately tracks progress from - // baseOffset. - long baseOffset = info.getOffset() - preparedStream.prependedBytes; + long confirmedBytesAppended = 0; + long currentOffsetBeforeStream = + Math.max( + info.getOffset() != null ? info.getOffset() : 0L, + preparedStream.existingPartsTotalSize + preparedStream.prependedBytes); + long baseOffset = currentOffsetBeforeStream - preparedStream.prependedBytes; boolean streamFinished = false; MaxAppendSizeExceededException maxAppendSizeException = null; + IOException readException = null; - while (!streamFinished) { - File tempChunkFile = - Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); - // Do not call tempChunkFile.deleteOnExit() here. In high-throughput long-running services, - // deleteOnExit() registers entries in a static JVM set that cannot be garbage collected, - // creating an unbounded memory leak. Temp files are deleted in try-finally blocks below. + List plannedPartKeys = new ArrayList<>(); + List plannedChunkSizes = new ArrayList<>(); - long chunkBytesWritten = 0; - IOException readException = null; - try { - try (FileOutputStream fos = new FileOutputStream(tempChunkFile)) { - int bytesRead; - while (chunkBytesWritten < optimalPartSize - && (bytesRead = streamToRead.read(buffer)) != -1) { - long requestBytesSoFar = - (totalBytesAppended + bytesRead) - preparedStream.prependedBytes; - if (maxAppendSize != null && requestBytesSoFar > maxAppendSize) { - // If maxAppendSize is exceeded, do not discard tempChunkFile immediately. - // If preparedStream had prepended bytes from a previous incomplete .part, discarding - // tempChunkFile would permanently lose those bytes. Instead, stop reading and record - // maxAppendSizeException so the bytes currently in tempChunkFile are flushed to S3 - // and UploadInfo offset is accurately preserved. - maxAppendSizeException = - new MaxAppendSizeExceededException( - "Append payload exceeded limit of " + maxAppendSize); + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(uploadExecutor, drainTimeout.toMillis(), preparedStream)) { + while (!streamFinished) { + File tempChunkFile = null; + long chunkBytesWritten = 0; + boolean handedOff = false; + try { + tempChunkFile = + Files.createTempFile(temporaryDirectory, "tus-s3-chunk-", ".tmp").toFile(); + + try (FileOutputStream fos = new FileOutputStream(tempChunkFile)) { + int bytesRead; + while (chunkBytesWritten < optimalPartSize + && (bytesRead = preparedStream.read(buffer)) != -1) { + long requestBytesSoFar = + (totalBytesAppended + bytesRead) - preparedStream.prependedBytes; + if (maxAppendSize != null && requestBytesSoFar > maxAppendSize) { + maxAppendSizeException = + new MaxAppendSizeExceededException( + "Append payload exceeded limit of " + maxAppendSize); + streamFinished = true; + break; + } + fos.write(buffer, 0, bytesRead); + chunkBytesWritten += bytesRead; + totalBytesAppended += bytesRead; + } + + if (chunkBytesWritten < optimalPartSize) { streamFinished = true; - break; } - fos.write(buffer, 0, bytesRead); - chunkBytesWritten += bytesRead; - totalBytesAppended += bytesRead; + } catch (IOException e) { + readException = e; + streamFinished = true; } - if (chunkBytesWritten < optimalPartSize) { - streamFinished = true; + if (chunkBytesWritten == 0) { + break; } - } catch (IOException e) { - readException = e; - streamFinished = true; - } - if (chunkBytesWritten == 0) { - if (readException != null) { - throw readException; + // Validate remaining part capacity before allocating next S3 part number + if (allPartKeys.size() + plannedPartKeys.size() >= MAX_PARTS_PER_UPLOAD) { + maxAppendSizeException = + new MaxAppendSizeExceededException( + "Upload has reached the maximum allowed S3 limit of " + + MAX_PARTS_PER_UPLOAD + + " parts."); + int confirmedCount = uploader.drainAndComplete(); + int confirmed = Math.min(confirmedCount, plannedPartKeys.size()); + for (int i = 0; i < confirmed; i++) { + allPartKeys.add(plannedPartKeys.get(i)); + confirmedBytesAppended += plannedChunkSizes.get(i); + } + plannedPartKeys.clear(); + plannedChunkSizes.clear(); + + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); + allPartKeys.add(chunkKey); + confirmedBytesAppended += chunkBytesWritten; + handedOff = true; + streamFinished = true; + break; } - if (maxAppendSizeException != null) { - throw maxAppendSizeException; + + long currentTotalOffset = baseOffset + totalBytesAppended; + boolean isUploadComplete = + readException == null + && maxAppendSizeException == null + && info.getLength() != null + && currentTotalOffset >= info.getLength(); + + // AWS S3 / MinIO Rule: Parts must be >= 5 MB unless it's the final part completing the + // upload or an incomplete sub-5MB tail chunk + if (chunkBytesWritten >= minPartSize) { + // Full part chunk (>= 5 MB): Submit to AsyncChunkUploader pipeline for background + // upload + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + plannedPartKeys.add(chunkKey); + plannedChunkSizes.add(chunkBytesWritten); + File fileToUpload = tempChunkFile; + long bytesToUpload = chunkBytesWritten; + uploader.submitChunk( + tempChunkFile, + bytesToUpload, + chunkKey, + () -> uploadChunkToS3(chunkKey, fileToUpload, bytesToUpload)); + handedOff = true; + + } else { + // Sub-5MB chunk (e.g. final completing chunk, or partial chunk when + // paused/interrupted): + // First drain all preceding parts in the pipeline + int confirmedCount = uploader.drainAndComplete(); + int confirmed = Math.min(confirmedCount, plannedPartKeys.size()); + for (int i = 0; i < confirmed; i++) { + allPartKeys.add(plannedPartKeys.get(i)); + confirmedBytesAppended += plannedChunkSizes.get(i); + } + plannedPartKeys.clear(); + plannedChunkSizes.clear(); + + String chunkKey = buildChunkPartKey(id, nextPartNumber++); + uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); + allPartKeys.add(chunkKey); + confirmedBytesAppended += chunkBytesWritten; + handedOff = true; } - break; - } - // Base offset plus total bytes appended accurately measures uploaded progress without - // double-counting - long currentTotalOffset = baseOffset + totalBytesAppended; - - // Interruption Guard: If an IOException or limit exception occurred (e.g. client pause or - // connection drop), - // the chunk must NEVER be considered complete, preventing sub-5MB chunks from being - // promoted. - boolean isUploadComplete = - readException == null - && maxAppendSizeException == null - && info.getLength() != null - && currentTotalOffset >= info.getLength(); - - // AWS S3 / MinIO Rule: Parts must be >= 5 MB unless it's the final part completing the - // upload - if (chunkBytesWritten >= minPartSize || (streamFinished && isUploadComplete)) { - String chunkKey = buildChunkPartKey(id, nextPartNumber); - uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); - allPartKeys.add(chunkKey); - nextPartNumber++; - } else { - // Store sub-5MB tail chunk as temporary .part object in S3 for subsequent appends - storeIncompletePartToS3(partObjectKey, tempChunkFile, chunkBytesWritten); + } catch (Exception e) { + if (e instanceof IOException && readException == null) { + readException = (IOException) e; + } + streamFinished = true; + } finally { + if (!handedOff && tempChunkFile != null) { + FileUtils.deleteQuietly(tempChunkFile); + } } + } - if (readException != null) { - throw readException; - } - if (maxAppendSizeException != null) { - throw maxAppendSizeException; + // Drain any remaining in-flight chunks in the pipeline with configured drain timeout + int confirmedCount = 0; + try { + confirmedCount = uploader.drainAndComplete(); + } catch (IOException e) { + if (readException == null) { + readException = e; } } finally { - FileUtils.deleteQuietly(tempChunkFile); + int confirmed = + Math.min( + Math.max(confirmedCount, uploader.getConfirmedCount()), plannedPartKeys.size()); + for (int i = 0; i < confirmed; i++) { + allPartKeys.add(plannedPartKeys.get(i)); + confirmedBytesAppended += plannedChunkSizes.get(i); + } + } + } catch (Exception e) { + if (e instanceof IOException && readException == null) { + readException = (IOException) e; } } - return new AppendResult(totalBytesAppended, allPartKeys); + return new AppendResult( + confirmedBytesAppended, allPartKeys, readException, maxAppendSizeException); } private void uploadChunkToS3(String chunkKey, File tempChunkFile, long chunkLength) @@ -918,19 +1272,6 @@ private void uploadChunkToS3(String chunkKey, File tempChunkFile, long chunkLeng } } - private void storeIncompletePartToS3(String partObjectKey, File tempChunkFile, long chunkLength) - throws IOException { - try (FileInputStream fis = new FileInputStream(tempChunkFile)) { - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(partObjectKey).stream(fis, chunkLength, -1L) - .build()); - } catch (Exception e) { - throw new IOException("Failed to write incomplete part object to S3 key " + partObjectKey, e); - } finally { - FileUtils.deleteQuietly(tempChunkFile); - } - } - /** * When all expected bytes have been received, this method combines all part chunk objects in S3 * into the final destination object key using MinIO's {@code composeObject} API. @@ -940,30 +1281,26 @@ private void storeIncompletePartToS3(String partObjectKey, File tempChunkFile, l * #mergeUsingStreamingReupload(String, List, long)}, guaranteeing successful upload finalization. */ private void finalizeCompletedUploadIfFinished( - UploadInfo info, String objectKey, UploadId id, AppendResult appendResult, long newOffset) - throws IOException { + UploadInfo info, String objectKey, UploadId id, long newOffset) throws IOException { if (info.getLength() != null && newOffset >= info.getLength()) { - List partKeys = fetchExistingPartKeys(id); - - // If leftover sub-5MB .part exists, save it as final part chunk - String leftoverPartKey = buildIncompletePartKey(id); - if (objectExists(leftoverPartKey)) { - int nextPartNum = partKeys.size() + 1; - String finalChunkKey = buildChunkPartKey(id, nextPartNum); - try (InputStream stream = - minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(leftoverPartKey).build())) { - byte[] bytes = IOUtils.toByteArray(stream); - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(finalChunkKey).stream( - new ByteArrayInputStream(bytes), (long) bytes.length, -1L) - .build()); - partKeys.add(finalChunkKey); - } catch (Exception e) { - throw new IOException("Failed to finalize incomplete part for ID " + id, e); - } - deleteObjectQuietly(leftoverPartKey); + List partKeys = + info.getUploadPartKeys() != null + ? new ArrayList<>(info.getUploadPartKeys()) + : new ArrayList<>(); + + long sumPartSizes = calculateTotalPartsSize(partKeys); + + // Guard: Enforce strict byte length equality before final object composition. + // This invariant prevents corrupted uploads where duplicate parts or missed chunk offsets + // would cause the final object to exceed or fall short of the declared length. + if (info.getLength() > 0 && sumPartSizes > 0 && sumPartSizes != info.getLength()) { + throw new IOException( + "Upload finalization aborted: sum of part sizes (" + + sumPartSizes + + ") does not match expected length (" + + info.getLength() + + ")"); } if (!partKeys.isEmpty()) { @@ -987,38 +1324,50 @@ private void finalizeCompletedUploadIfFinished( } } - if (canUseServerSideCompose) { + if (canUseServerSideCompose && isS3ComposeObjectSupported()) { try { - List sources = new ArrayList<>(); - for (String pk : partKeys) { - sources.add(SourceObject.builder().bucket(bucket).object(pk).build()); - } - - // Perform S3 server-side object composition (composeObject) - minioClient.composeObject( - ComposeObjectArgs.builder() - .bucket(bucket) - .object(objectKey) - .sources(sources) - .build()); + // Perform S3 server-side object composition via native S3 multipart copy + s3ComposeHelper.compose(bucket, objectKey, partKeys); } catch (Exception e) { - log.warn( - "S3 composeObject failed for object {}, falling back to streaming concatenation:" - + " {}", + // MinIO Java SDK's composeObject implementation delegates to UploadPartCopy with an + // EMPTY_BODY, which automatically attaches Content-MD5 and Content-Type headers. + // AWS S3 strictly forbids Content-MD5 on UploadPartCopy and rejects it with 400 + // InvalidArgument ("The specified header is not valid in this context"). + // When server-side compose fails on this S3 endpoint, disable it dynamically + // to avoid redundant failing S3 API roundtrips on subsequent uploads, log at INFO, + // and seamlessly merge chunks via streaming part composition. + setS3ComposeObjectSupported(false); + log.info( + "S3 server-side compose failed for object {}, falling back to streaming part" + + " composition: {}", objectKey, e.getMessage()); mergeUsingStreamingReupload(objectKey, partKeys, newOffset); } } else { - log.info( - "Detected sub-5MB non-final parts for ID {}. Using streaming concatenation fallback.", - id); + if (!canUseServerSideCompose) { + log.info( + "Detected sub-5MB non-final parts for ID {}. Using streaming part composition" + + " fallback.", + id); + } mergeUsingStreamingReupload(objectKey, partKeys, newOffset); } - // Clean up temporary part chunk objects in S3 - for (String pk : partKeys) { - deleteObjectQuietly(pk); + // Clean up temporary part chunk objects in S3 using multi-delete with fallback + deleteS3ObjectsQuietly(partKeys); + info.setUploadPartKeys(null); + + } else if (info.getLength() == 0L) { + // Zero-byte upload: create the empty destination object in S3 + try { + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(objectKey).stream( + new ByteArrayInputStream(new byte[0]), 0L, -1L) + .build()); + info.setUploadPartKeys(null); + } catch (Exception e) { + throw new IOException("Failed to create empty completed object for ID " + id, e); } } @@ -1060,23 +1409,23 @@ private InputStream fetchS3ByteStream(UploadId id, UploadInfo info) GetObjectArgs.builder().bucket(bucket).object(objectKey).build()); } catch (ErrorResponseException e) { if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { - // Step 2: Fallback to reading from incomplete .part object if upload is in-progress - String partKey = buildIncompletePartKey(id); - try { - return minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(partKey).build()); - } catch (ErrorResponseException ex) { - // If the incomplete .part object is also missing (NoSuchKey) and the offset is - // zero or null, it means no bytes have been uploaded yet (e.g. immediately after - // creation). In this case, we return an empty stream rather than throwing an exception. - if (S3Utils.parseErrorResponse(ex) == S3ErrorType.NO_SUCH_KEY) { - if (info != null && (info.getOffset() == null || info.getOffset() == 0L)) { - return new ByteArrayInputStream(new byte[0]); - } + // Step 2: Fallback to reading from manifest part objects if upload is in-progress + List partKeys = + info != null && info.getUploadPartKeys() != null + ? info.getUploadPartKeys() + : Collections.emptyList(); + if (!partKeys.isEmpty()) { + try { + Enumeration inputStreams = + new S3PartInputStreamEnumeration(minioClient, bucket, partKeys); + return new SequenceInputStream(inputStreams); + } catch (Exception ex) { + log.debug("Failed to stream manifest parts for upload ID {}", id, ex); } - log.debug("Failed to read incomplete .part object {}", partKey, ex); - } catch (Exception exception) { - log.debug("Failed to read incomplete .part object {}", partKey, exception); + } + + if (info != null && (info.getOffset() == null || info.getOffset() == 0L)) { + return new ByteArrayInputStream(new byte[0]); } } throw new UploadNotFoundException("Uploaded bytes object not found for ID " + id); @@ -1085,99 +1434,125 @@ private InputStream fetchS3ByteStream(UploadId id, UploadInfo info) } } - private void truncateFromCompletedObject(String objectKey, String partKey, long newOffset) + /** + * Truncates bytes backwards across committed manifest parts in reverse chronological order. + * + *

AWS S3 objects are immutable, so truncation cannot truncate in-place without replacing the + * affected object. If a part's total size is less than or equal to the bytes to remove, the + * entire part object is deleted from S3 and removed from the manifest. If a part's size exceeds + * the bytes to remove, {@link #truncateSinglePart(String, long, long)} rewrites the remaining + * prefix bytes to S3. + * + * @param uploadInfo The upload metadata containing the list of part keys + * @param bytesToRemove The total number of bytes to strip from the tail + */ + private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) { + List partKeys = new ArrayList<>(uploadInfo.getUploadPartKeys()); + long bytesToRemoveRemaining = bytesToRemove; + + while (!partKeys.isEmpty() && bytesToRemoveRemaining > 0) { + String lastKey = partKeys.get(partKeys.size() - 1); + try { + StatObjectResponse stat = + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(lastKey).build()); + long partSize = stat.size(); + + if (bytesToRemoveRemaining >= partSize) { + // The entire part is subsumed by the truncation; delete it and adjust remaining budget + deleteObjectQuietly(lastKey); + partKeys.remove(partKeys.size() - 1); + bytesToRemoveRemaining -= partSize; + } else { + // Only a fraction of this part is being removed; rewrite the remaining prefix + truncateSinglePart(lastKey, partSize, bytesToRemoveRemaining); + bytesToRemoveRemaining = 0; + } + } catch (Exception e) { + log.debug("Error truncating manifest part {}", lastKey, e); + break; + } + } + uploadInfo.setUploadPartKeys(partKeys); + } + + /** + * Partially truncates an individual S3 part object by downloading its content and overwriting it + * in S3 with only the remaining byte prefix. + * + * @param partKey S3 object key of the part to partially truncate + * @param partSize Current total size of the part object + * @param bytesToRemove Number of tail bytes to remove from this specific part + * @throws IOException If reading or re-uploading the truncated part fails + */ + private void truncateSinglePart(String partKey, long partSize, long bytesToRemove) + throws IOException { + try (InputStream partStream = + minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(partKey).build())) { + byte[] bytes = IOUtils.toByteArray(partStream); + int newLength = (int) (partSize - bytesToRemove); + byte[] remaining = Arrays.copyOf(bytes, newLength); + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(partKey).stream( + new ByteArrayInputStream(remaining), (long) remaining.length, -1L) + .build()); + } catch (Exception e) { + throw new IOException("Failed to partially truncate S3 part object " + partKey, e); + } + } + + /** + * Truncates a final completed S3 object by converting the remaining byte prefix into a manifest + * part and deleting the completed object. + * + * @param uploadInfo The upload metadata to update with the new manifest part + * @param objectKey S3 object key of the completed object + * @param newOffset The truncated target byte length + * @throws IOException If downloading or rewriting the object fails + */ + private void truncateCompletedObject(UploadInfo uploadInfo, String objectKey, long newOffset) throws IOException { if (newOffset > 0) { try (InputStream objStream = minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(objectKey).build())) { byte[] remainingBytes = new byte[(int) newOffset]; IOUtils.readFully(objStream, remainingBytes); + String chunkKey = buildChunkPartKey(uploadInfo.getId(), 1); minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(partKey).stream( + PutObjectArgs.builder().bucket(bucket).object(chunkKey).stream( new ByteArrayInputStream(remainingBytes), (long) remainingBytes.length, -1L) .build()); + uploadInfo.setUploadPartKeys(new ArrayList<>(Collections.singletonList(chunkKey))); } catch (Exception e) { throw new IOException("Failed to truncate completed object key " + objectKey, e); } + } else { + uploadInfo.setUploadPartKeys(new ArrayList<>()); } deleteObjectQuietly(objectKey); } - private void truncateFromIncompletePart(String partKey, long byteCount) { - try { - StatObjectResponse head = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(partKey).build()); - long partSize = head.size(); - - if (byteCount >= partSize) { - deleteObjectQuietly(partKey); - } else { - try (InputStream partStream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(partKey).build())) { - byte[] bytes = IOUtils.toByteArray(partStream); - int newLength = (int) (bytes.length - byteCount); - byte[] remaining = Arrays.copyOf(bytes, newLength); - - minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(partKey).stream( - new ByteArrayInputStream(remaining), (long) remaining.length, -1L) - .build()); - } - } - } catch (ErrorResponseException ignored) { - } catch (Exception e) { - log.debug("Error truncating incomplete part object {}", partKey, e); - } - } - - private void calculateAndSetOffset(UploadInfo info) { + void calculateAndSetOffset(UploadInfo info) { if (info == null || info.getId() == null) { return; } String objectKey = getS3ObjectKey(info); - String partKey = buildIncompletePartKey(info.getId()); - - long offset = calculateCurrentOffset(objectKey, info.getId(), partKey); - info.setOffset(offset); - } - - private long calculateCurrentOffset(String objectKey, UploadId id, String partKey) { - long offset = 0; - if (objectExists(objectKey)) { try { StatObjectResponse head = minioClient.statObject( StatObjectArgs.builder().bucket(bucket).object(objectKey).build()); - offset += head.size(); - } catch (Exception ignored) { - } - } - - List partKeys = fetchExistingPartKeys(id); - for (String pk : partKeys) { - try { - StatObjectResponse stat = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(pk).build()); - offset += stat.size(); - } catch (Exception ignored) { - } - } - - if (!partKeys.contains(partKey)) { - try { - StatObjectResponse partHead = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(partKey).build()); - if (partHead != null) { - offset += partHead.size(); + if (head != null) { + info.setOffset(head.size()); + return; } - } catch (ErrorResponseException ignored) { - } catch (Exception e) { - log.debug("Error reading head for incomplete part object {}", partKey, e); + } catch (Exception ignored) { } } - return offset; + List partKeys = + info.getUploadPartKeys() != null ? info.getUploadPartKeys() : Collections.emptyList(); + long offset = calculateTotalPartsSize(partKeys); + info.setOffset(offset); } private List fetchExistingPartKeys(UploadId id) { @@ -1191,24 +1566,127 @@ private List fetchExistingPartKeys(UploadId id) { } } catch (Exception ignored) { } + Collections.sort(partKeys); return partKeys; } private void deleteAllPartObjectsQuietly(UploadId id) { List partKeys = fetchExistingPartKeys(id); - for (String pk : partKeys) { - deleteObjectQuietly(pk); + deleteS3ObjectsQuietly(partKeys); + } + + /** + * Deletes multiple S3 objects efficiently using batch Multi-Object Delete (POST /?delete), with + * automatic fallback to individual single-object DELETE requests if batch delete is unsupported + * or rejected by bucket IAM permissions. + * + *

If batch delete fails once, it is permanently disabled for this storage service instance to + * avoid unnecessary failing S3 API round-trips. + * + * @param keys The S3 object keys to delete + */ + void deleteS3ObjectsQuietly(Collection keys) { + if (keys == null || keys.isEmpty()) { + return; + } + + if (!isSupportsBatchDelete()) { + deleteObjectsIndividually(keys); + return; + } + + List objects = toDeleteRequestObjects(keys); + if (objects.isEmpty()) { + return; + } + + boolean batchFailed = false; + try { + Iterable> results = + minioClient.removeObjects( + RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); + if (results != null) { + for (Result res : results) { + if (res != null) { + DeleteResult.Error error = res.get(); + if (error != null) { + log.debug( + "Batch delete reported error for key {}: {}; switching to individual delete", + error.objectName(), + error.message()); + batchFailed = true; + break; + } + } + } + } + } catch (Exception e) { + log.info( + "Batch multi-delete failed; permanently switching to individual delete: {}", + e.getMessage()); + batchFailed = true; + } + + if (batchFailed) { + setSupportsBatchDelete(false); + deleteObjectsIndividually(keys); } } - private long calcOptimalPartSize(long totalSize) { + private List toDeleteRequestObjects(Collection keys) { + List objects = new ArrayList<>(); + for (String key : keys) { + if (key != null) { + objects.add(new DeleteRequest.Object(key)); + } + } + return objects; + } + + private void deleteObjectsIndividually(Collection keys) { + for (String key : keys) { + deleteObjectQuietly(key); + } + } + + /** + * Calculates auto-calibrated optimal chunk part size based on total upload length. + * + *

AWS S3 multipart uploads enforce a strict ceiling of 10,000 parts per object. When an upload + * length exceeds 80 GB (10,000 * 8 MB), the part size dynamically scales up (e.g. ~105 MB for 1 + * TB, ~525 MB for 5 TB) so that the entire upload is guaranteed to fit within 10,000 parts, + * bounded by S3's 5 GB maximum part limit. + * + * @param totalLength The announced total upload length in bytes, or null if unknown/deferred + * @return The calibrated optimal part size in bytes + */ + long calcOptimalPartSize(Long totalLength) { long partSize = preferredPartSize; - if (totalSize > 0 && totalSize / partSize >= 10000) { - partSize = (totalSize / 10000) + 1; + if (totalLength != null && totalLength > 0 && totalLength / partSize >= MAX_PARTS_PER_UPLOAD) { + partSize = (totalLength / MAX_PARTS_PER_UPLOAD) + 1; } return Math.max(minPartSize, Math.min(partSize, DEFAULT_MAX_PART_SIZE)); } + /** + * Validates that the upload has sufficient part budget remaining within S3's 10,000 parts limit. + * + *

Why: AWS S3 enforces a strict maximum ceiling of 10,000 parts per multipart upload. + * If an upload receives too many small chunks, it risks hitting this limit before finishing. This + * check runs in O(1) time at the trust boundary to prevent deadlocked uploads. + * + * @param upload The current upload metadata + * @param currentPartCount Number of already committed parts + * @throws MaxAppendSizeExceededException If remaining part capacity is exhausted + */ + void validateRemainingPartBudget(UploadInfo upload, int currentPartCount) + throws MaxAppendSizeExceededException { + if (currentPartCount >= MAX_PARTS_PER_UPLOAD) { + throw new MaxAppendSizeExceededException( + "Upload has reached the maximum allowed S3 limit of " + MAX_PARTS_PER_UPLOAD + " parts."); + } + } + private void putChecksumIndex(String checksum, ChecksumAlgorithm algorithm, UploadId parentId) { String key = buildChecksumKey(checksum, algorithm); try { @@ -1263,12 +1741,11 @@ private String buildMetadataKey(UploadId id) { return metadataPrefix + Objects.toString(id) + ".info"; } - private String buildIncompletePartKey(UploadId id) { - return metadataPrefix + Objects.toString(id) + ".part"; - } - private String buildChunkPartKey(UploadId id, int partNumber) { - return metadataPrefix + Objects.toString(id) + ".part." + String.format("%05d", partNumber); + return metadataPrefix + + Objects.toString(id) + + ".part." + + String.format("%05d-%s", partNumber, UUID.randomUUID().toString().substring(0, 8)); } private String buildChecksumKey(String checksum, ChecksumAlgorithm algorithm) { @@ -1285,28 +1762,60 @@ private String buildStopKey(UploadId id) { } private static class AppendResult { - final long totalBytesAppended; + final long confirmedBytesAppended; final List allPartKeys; - - AppendResult(long totalBytesAppended, List allPartKeys) { - this.totalBytesAppended = totalBytesAppended; + final IOException readException; + final MaxAppendSizeExceededException maxAppendSizeException; + + AppendResult( + long confirmedBytesAppended, + List allPartKeys, + IOException readException, + MaxAppendSizeExceededException maxAppendSizeException) { + this.confirmedBytesAppended = confirmedBytesAppended; this.allPartKeys = allPartKeys; + this.readException = readException; + this.maxAppendSizeException = maxAppendSizeException; } } /** - * Helper value object encapsulating an input stream prepended with prior incomplete chunk bytes, - * the number of prepended bytes (for base offset calibration), and the remaining part keys. + * Helper stream wrapper encapsulating an input stream prepended with prior incomplete chunk + * bytes, delegating interruption state to the original incoming stream. */ - private static class PreparedStream { - final InputStream stream; + static class PreparedStream extends InterruptibleInputStream { + private final InterruptibleInputStream originalStream; final long prependedBytes; + final long existingPartsTotalSize; final List remainingPartKeys; - - PreparedStream(InputStream stream, long prependedBytes, List remainingPartKeys) { - this.stream = stream; + final List staleTailKeys; + + PreparedStream( + InputStream combinedStream, + InterruptibleInputStream originalStream, + long prependedBytes, + long existingPartsTotalSize, + List remainingPartKeys, + List staleTailKeys) { + super(combinedStream); + this.originalStream = originalStream; this.prependedBytes = prependedBytes; + this.existingPartsTotalSize = existingPartsTotalSize; this.remainingPartKeys = remainingPartKeys; + this.staleTailKeys = staleTailKeys; + } + + @Override + public boolean isInterrupted() { + return super.isInterrupted() || (originalStream != null && originalStream.isInterrupted()); + } + + @Override + public void interrupt() { + super.interrupt(); + if (originalStream != null) { + originalStream.interrupt(); + } } } @@ -1314,7 +1823,7 @@ private static class PreparedStream { * Enumeration that lazily queries and opens S3 input streams for a list of part keys, enabling * streaming re-upload concatenation without holding all parts in memory. */ - private static class S3PartInputStreamEnumeration implements Enumeration { + static class S3PartInputStreamEnumeration implements Enumeration { private final MinioClient minioClient; private final String bucket; private final List partKeys; diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index 5b1c8899..318a7a0b 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -4,11 +4,19 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteRequest; +import io.minio.messages.DeleteResult; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; +import java.util.stream.Stream; import me.desair.tus.server.upload.AbstractLeaseLock; import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadLock; @@ -44,7 +52,7 @@ public class S3UploadLock extends AbstractLeaseLock { * @param stopKey The S3 object key for the interrupt stop signal * @param inputStreamMap Map of active request input streams */ - public S3UploadLock( + S3UploadLock( LeaseData leaseData, MinioClient minioClient, String bucket, @@ -104,24 +112,93 @@ protected void doRenewLease() throws Exception { @Override protected void releaseLockResource() { - // Owner-Safe Lock Release: only delete .lock lease if it is still owned by this holder - deleteS3LockObjectIfOwner(lockKey); - deleteS3ObjectQuietly(stopKey); + // Owner-Safe Lock Release: only delete .lock lease if it is still owned by this holder. + // If ownership was stolen by another node (e.g. heartbeat lost or execution ran too long), + // skip deletion entirely so we do not delete another node's lock or any stop signal + // potentially intended for that new holder. + if (lockKey != null && !doesLockOwnershipMatch(lockKey)) { + log.info( + "Skipping deletion of S3 lock key {}: lock is currently held by another node", lockKey); + return; + } + + // Batch deletion of .lock and .stop keys via S3 Multi-Object Delete. + // AWS S3 and MinIO execute multi-object delete in a single network round trip (POST /?delete), + // eliminating an extra round trip on every lock release. + deleteS3Objects(lockKey, stopKey); } - void deleteS3LockObjectIfOwner(String key) { - if (key == null || minioClient == null || bucket == null) { + void deleteS3Objects(String firstKey, String secondKey) { + if (minioClient == null || bucket == null) { + return; + } + List keysToDelete = + Stream.of(firstKey, secondKey).filter(Objects::nonNull).collect(Collectors.toList()); + if (keysToDelete.isEmpty()) { return; } + + boolean batchFailed = false; + List objects = + keysToDelete.stream().map(DeleteRequest.Object::new).collect(Collectors.toList()); + try { - if (!doesLockOwnershipMatch(key)) { - log.info( - "Skipping deletion of S3 lock key {}: lock is currently held by another node", key); - return; + // First attempt fast S3 Multi-Object Delete (POST /?delete) in a single round trip. + // Supported by MinIO and compliant S3 endpoints where bucket policies permit multi-delete. + Iterable> results = + minioClient.removeObjects( + RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); + if (results != null) { + for (Result result : results) { + try { + if (result != null) { + DeleteResult.Error error = result.get(); + if (error != null) { + // S3 returned an error for an object (e.g. AccessDenied on multi-delete POST + // /?delete) + log.debug( + "Batch delete reported error for S3 object {}: {}; falling back to individual" + + " delete", + error.objectName(), + error.message()); + batchFailed = true; + break; + } + } + } catch (Exception e) { + log.debug( + "Exception iterating batch delete results; falling back to individual delete", e); + batchFailed = true; + break; + } + } } + } catch (Exception e) { + log.debug( + "Batch multi-delete failed for keys {}; falling back to individual single-object DELETE", + keysToDelete, + e); + batchFailed = true; + } + + // Fall back to resilient individual single-object DELETE calls (DELETE /bucket/key). + // Standard AWS IAM policies often grant s3:DeleteObject exclusively on arn:aws:s3:::bucket/*, + // which allows individual object deletions while rejecting bucket-level multi-object delete. + if (batchFailed) { + for (String key : keysToDelete) { + deleteSingleObject(key); + } + } + } + + void deleteSingleObject(String key) { + if (minioClient == null || bucket == null || key == null) { + return; + } + try { minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); } catch (Exception e) { - log.debug("Failed to delete S3 lock object {}", key, e); + log.warn("Failed to delete S3 lock object key {} in bucket {}", key, bucket, e); } } @@ -149,15 +226,4 @@ boolean doesLockOwnershipMatch(String key) { } return true; } - - private void deleteS3ObjectQuietly(String key) { - if (key == null || minioClient == null || bucket == null) { - return; - } - try { - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); - } catch (Exception e) { - log.debug("Failed to delete S3 object {}", key, e); - } - } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java index 618f41c4..2d0c2eaa 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java @@ -43,7 +43,9 @@ public static S3ErrorType parseErrorResponse(ErrorResponseException exception) { return S3ErrorType.ACCESS_DENIED; } - if ("APINotImplemented".equalsIgnoreCase(code) || "NotImplemented".equalsIgnoreCase(code)) { + if ("APINotImplemented".equalsIgnoreCase(code) + || "NotImplemented".equalsIgnoreCase(code) + || (exception.response() != null && exception.response().code() == 501)) { return S3ErrorType.API_NOT_IMPLEMENTED; } diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java new file mode 100644 index 00000000..6f6aa912 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -0,0 +1,374 @@ +package me.desair.tus.server.upload.util; + +import java.io.File; +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import me.desair.tus.server.util.InterruptibleInputStream; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Coordinates asynchronous, pipelined uploading of chunk files to cloud storage backends (S3, + * Azure). + * + *

Uses a bounded 3-slot pipeline (triple-buffering): + * + *

    + *
  • Slot 1 (Receiving): The caller/servlet thread reads incoming request bytes into a + * local temp chunk file. + *
  • Slot 2 (Waiting): At most one completed chunk file held on disk waiting for the + * active cloud upload to complete. + *
  • Slot 3 (Uploading): A discrete task running on the shared thread pool actively + * streaming a chunk to the cloud. + *
+ * + *

This pipeline overlaps network reception from the client with upstream cloud staging, + * eliminating client TCP stall periods while maintaining strict upper bounds on memory, disk space, + * and thread usage. + */ +public class AsyncChunkUploader implements AutoCloseable { + + private static final Logger log = LoggerFactory.getLogger(AsyncChunkUploader.class); + + public static final long DEFAULT_DRAIN_TIMEOUT_MS = 55_000L; + + @FunctionalInterface + public interface ChunkUploadAction { + /** + * Executes the cloud upload action for a single chunk. + * + * @throws Exception If the upload to the cloud provider fails + */ + void upload() throws Exception; + } + + private final ExecutorService executor; + private final long drainTimeoutMs; + + // Slot 3: In-flight upload task and its associated local file and key + private Future inFlightUpload; + private File inFlightFile; + private String inFlightKey; + + // Slot 2: Waiting chunk awaiting execution + private File waitingChunkFile; + private long waitingChunkSize; + private String waitingKey; + private ChunkUploadAction waitingAction; + + // Number of chunks confirmed uploaded to the cloud + private int confirmedCount; + + private final InterruptibleInputStream inputStream; + + // Set to true once drainAndComplete() has successfully finished + private boolean completed; + + /** + * Constructs an uploader using the given shared executor and default 55-second drain timeout. + * + * @param executor Shared thread pool executor for background chunk uploads + */ + public AsyncChunkUploader(ExecutorService executor) { + this(executor, DEFAULT_DRAIN_TIMEOUT_MS, null); + } + + /** + * Constructs an uploader using the given shared executor and explicit drain timeout. + * + * @param executor Shared thread pool executor for background chunk uploads + * @param drainTimeoutMs Maximum duration in milliseconds to drain in-flight chunks + */ + public AsyncChunkUploader(ExecutorService executor, long drainTimeoutMs) { + this(executor, drainTimeoutMs, null); + } + + /** + * Constructs an uploader using the given shared executor, drain timeout, and interruptible input + * stream. + * + * @param executor Shared thread pool executor for background chunk uploads + * @param drainTimeoutMs Maximum duration in milliseconds to drain in-flight chunks + * @param inputStream The interruptible upload stream to monitor for contention signals + */ + public AsyncChunkUploader( + ExecutorService executor, long drainTimeoutMs, InterruptibleInputStream inputStream) { + this.executor = Objects.requireNonNull(executor, "ExecutorService must not be null"); + this.drainTimeoutMs = drainTimeoutMs; + this.inputStream = inputStream; + } + + /** + * Submits a newly completed local chunk file into the upload pipeline. + * + *

If Slot 3 is idle or completed, the chunk begins uploading immediately. If Slot 3 is busy + * and Slot 2 is empty, the chunk is placed into Slot 2. If both Slot 3 and Slot 2 are occupied, + * this call blocks until Slot 3 finishes (applying backpressure to the client stream). + * + * @param tempFile The local chunk file containing the chunk bytes + * @param size The size of the chunk in bytes + * @param preassignedKey The pre-assigned cloud object key or block ID + * @param uploadAction The upload action to execute on the worker thread + * @throws IOException If a previous chunk upload failed or thread was interrupted + */ + public void submitChunk( + File tempFile, long size, String preassignedKey, ChunkUploadAction uploadAction) + throws IOException { + Objects.requireNonNull(tempFile, "tempFile must not be null"); + Objects.requireNonNull(uploadAction, "uploadAction must not be null"); + + // Check if previous in-flight upload has finished, advancing confirmation count + if (inFlightUpload != null && inFlightUpload.isDone()) { + try { + checkAndConfirmInFlight(); + } catch (IOException e) { + FileUtils.deleteQuietly(tempFile); + throw e; + } + } + + try { + if (waitingChunkFile == null) { + if (inFlightUpload == null) { + // Branch 1: Slot 3 is free and Slot 2 is empty. Submit directly to Slot 3. + submitToSlot3(tempFile, size, preassignedKey, uploadAction); + } else { + // Branch 2: Slot 3 is actively uploading, but Slot 2 (waiting) is free. + // Store in Slot 2 and return immediately so the caller can read the next chunk. + this.waitingChunkFile = tempFile; + this.waitingChunkSize = size; + this.waitingKey = preassignedKey; + this.waitingAction = uploadAction; + } + } else { + // Branch 3: Slot 2 already has a waiting chunk. + // To preserve strict sequential chunk ordering, Slot 2's chunk must be uploaded before the + // new chunk. + // If Slot 3 is still active, apply backpressure by waiting for it to finish. + if (inFlightUpload != null) { + waitForInFlight(); + } + + // Slot 3 is now free. Promote Slot 2 into Slot 3. + submitToSlot3(waitingChunkFile, waitingChunkSize, waitingKey, waitingAction); + this.waitingChunkFile = null; + this.waitingAction = null; + + // Place the newly arrived chunk into the now-empty Slot 2. + this.waitingChunkFile = tempFile; + this.waitingChunkSize = size; + this.waitingKey = preassignedKey; + this.waitingAction = uploadAction; + } + } catch (IOException | RuntimeException e) { + FileUtils.deleteQuietly(tempFile); + throw e; + } + } + + /** + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) using the configured drain + * timeout (defaults to {@link #DEFAULT_DRAIN_TIMEOUT_MS}, 55 seconds). + * + * @return The total number of confirmed successfully uploaded chunks + * @throws IOException If any chunk upload fails or times out + */ + public int drainAndComplete() throws IOException { + return drainAndComplete(drainTimeoutMs); + } + + /** + * Returns the configured drain timeout in milliseconds. + * + * @return Drain timeout in milliseconds + */ + public long getDrainTimeoutMs() { + return drainTimeoutMs; + } + + /** + * Drains all remaining chunks in the pipeline (Slot 3 and Slot 2) up to the specified timeout. + * + *

Used upon stream completion or client interruption to ensure all received bytes are + * persisted to cloud storage before releasing the upload lock. + * + * @param timeoutMs Maximum time in milliseconds to wait for in-flight and waiting chunks + * @return The total number of confirmed successfully uploaded chunks + * @throws IOException If any chunk upload fails or times out + */ + public int drainAndComplete(long timeoutMs) throws IOException { + long deadline = System.currentTimeMillis() + timeoutMs; + + // 1. Await Slot 3 if active + if (inFlightUpload != null) { + long remaining = Math.max(1L, deadline - System.currentTimeMillis()); + awaitFuture(inFlightUpload, remaining, inFlightKey); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } + + // 2. If Slot 2 has a waiting chunk, promote it to Slot 3 and await its completion + if (waitingChunkFile != null) { + submitToSlot3(waitingChunkFile, waitingChunkSize, waitingKey, waitingAction); + waitingChunkFile = null; + waitingAction = null; + + long remaining = Math.max(1L, deadline - System.currentTimeMillis()); + awaitFuture(inFlightUpload, remaining, inFlightKey); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } + + completed = true; + return confirmedCount; + } + + /** + * Aborts the pipeline, cancelling any active cloud upload and deleting any remaining temp files. + */ + public void abort() { + if (inFlightUpload != null && !inFlightUpload.isDone()) { + inFlightUpload.cancel(true); + try { + // Wait briefly for worker thread to exit so no background thread outlives the lock + inFlightUpload.get(500, TimeUnit.MILLISECONDS); + } catch (Exception ignored) { + } + } + if (inFlightFile != null) { + FileUtils.deleteQuietly(inFlightFile); + inFlightFile = null; + } + if (waitingChunkFile != null) { + FileUtils.deleteQuietly(waitingChunkFile); + waitingChunkFile = null; + waitingAction = null; + } + } + + /** + * Returns the number of chunks confirmed successfully uploaded. + * + * @return Confirmed chunk count + */ + public int getConfirmedCount() { + return confirmedCount; + } + + @Override + public void close() { + // If drainAndComplete() did not finish successfully, abort and clean up uncommitted files + if (!completed) { + abort(); + } + } + + private void submitToSlot3( + File tempFile, long size, String preassignedKey, ChunkUploadAction uploadAction) { + this.inFlightFile = tempFile; + this.inFlightKey = preassignedKey; + + // Wrap the upload action so temp file cleanup is guaranteed on the worker thread + this.inFlightUpload = + executor.submit( + () -> { + try { + uploadAction.upload(); + } finally { + // The worker owns tempFile once submitted; delete it immediately upon upload + // completion + FileUtils.deleteQuietly(tempFile); + } + return null; + }); + } + + void checkAndConfirmInFlight() throws IOException { + try { + inFlightUpload.get(); + confirmedCount++; + inFlightUpload = null; + inFlightFile = null; + inFlightKey = null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException( + "Interrupted while checking in-flight chunk upload for " + inFlightKey, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, inFlightKey); + } + } + + private void waitForInFlight() throws IOException { + // When waiting under backpressure, poll with contention checks so that if another + // thread signals lock contention (e.g. HEAD or DELETE request), the servlet thread wakes up + // quickly rather than remaining blocked indefinitely on in-flight chunk upload. + while (inFlightUpload != null && !inFlightUpload.isDone()) { + if (isContentionRequested()) { + throw new IOException("Upload was interrupted by locking service contention"); + } + try { + inFlightUpload.get(50, TimeUnit.MILLISECONDS); + break; + } catch (TimeoutException e) { + // Poll tick: loop and re-evaluate contention + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException( + "Interrupted waiting for in-flight chunk upload for " + inFlightKey, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, inFlightKey); + } + } + if (inFlightUpload != null && inFlightUpload.isDone()) { + checkAndConfirmInFlight(); + } + } + + private boolean isContentionRequested() { + return inputStream != null && inputStream.isInterrupted(); + } + + private void awaitFuture(Future future, long timeoutMs, String key) throws IOException { + try { + future.get(timeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + abort(); + throw new IOException("Interrupted waiting for chunk upload for key " + key, e); + } catch (TimeoutException e) { + abort(); + throw new IOException( + "Timed out after " + timeoutMs + "ms waiting for chunk upload for key " + key, e); + } catch (ExecutionException e) { + abort(); + throw translateExecutionException(e, key); + } + } + + private IOException translateExecutionException(ExecutionException e, String key) { + Throwable cause = e.getCause(); + if (cause instanceof IOException) { + return (IOException) cause; + } else if (cause instanceof RuntimeException) { + return new IOException("Upload failed for key " + key + ": " + cause.getMessage(), cause); + } else { + return new IOException( + "Unexpected error during chunk upload for key " + key, cause != null ? cause : e); + } + } +} diff --git a/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java b/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java index 7fe2257a..5a39e302 100644 --- a/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java +++ b/src/main/java/me/desair/tus/server/util/InterruptibleInputStream.java @@ -2,6 +2,8 @@ import java.io.IOException; import java.io.InputStream; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * An InputStream wrapper that can be interrupted by another thread. When interrupted, it throws an @@ -10,7 +12,8 @@ public class InterruptibleInputStream extends InputStream { private final InputStream delegate; - private volatile boolean interrupted = false; + private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); + private boolean interrupted = false; /** * Constructs an interruptible input stream wrapping the given delegate stream. @@ -26,7 +29,7 @@ public InterruptibleInputStream(InputStream delegate) { } private void checkInterrupted() throws IOException { - if (interrupted) { + if (isInterrupted()) { throw new IOException("Stream was interrupted by the upload locking service watchdog"); } } @@ -43,7 +46,12 @@ private void checkInterrupted() throws IOException { * HEAD requests) wait for all in-flight bytes to be committed before reading the upload offset. */ public void interrupt() { - interrupted = true; + thisObjectLock.writeLock().lock(); + try { + interrupted = true; + } finally { + thisObjectLock.writeLock().unlock(); + } try { delegate.close(); } catch (IOException e) { @@ -52,7 +60,12 @@ public void interrupt() { } public boolean isInterrupted() { - return interrupted; + thisObjectLock.readLock().lock(); + try { + return interrupted; + } finally { + thisObjectLock.readLock().unlock(); + } } @Override diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index e884f3c7..906cc477 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -605,6 +605,24 @@ public static void shutdownExecutor(ScheduledExecutorService executor) { } } + /** + * Ensures the given input stream is an {@link InterruptibleInputStream}. If the input stream is + * already an {@code InterruptibleInputStream}, it is cast and returned; otherwise, it is wrapped + * in a new {@code InterruptibleInputStream}. Returns null if the input stream is null. + * + * @param inputStream The input stream to check or wrap + * @return An InterruptibleInputStream wrapping or casting the input stream, or null if null + */ + public static InterruptibleInputStream toInterruptibleStream(InputStream inputStream) { + if (inputStream == null) { + return null; + } + if (inputStream instanceof InterruptibleInputStream) { + return (InterruptibleInputStream) inputStream; + } + return new InterruptibleInputStream(inputStream); + } + /** * Safely interrupts an input stream if it is an instance of {@link InterruptibleInputStream}, or * closes it quietly if it is a standard input stream. Any exceptions encountered during diff --git a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java index e818de35..504dd8b2 100644 --- a/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java +++ b/src/test/java/me/desair/tus/server/AbstractITRufhProtocol.java @@ -801,6 +801,95 @@ public void testUploadCompletionListenerRufhPatchAppend() throws Exception { } } + // =============================================================================================== + // USE CASE 16: Min-Append-Size Exemption for Completing Uploads (§4.1.4) + // =============================================================================================== + + /** + * Section 4.1.4 (Limits): "min-append-size: An Integer indicating the minimum number of bytes + * that MUST be appended with each upload append request. This limit does not apply to upload + * creation requests with no content, or to requests completing the upload by including the + * Upload-Complete: ?1 header field." + * + *

Use Case: Configure min-append-size (1 MB), then verify that an optimistic single-request + * creation upload with Upload-Complete: ?1 and a payload smaller than 1 MB bypasses the + * min-append-size limit and completes successfully. + */ + @Test + public void testCompletingCreationBypassesMinAppendSize() throws Exception { + tusFileUploadService.withMinAppendSize(1024L * 1024L); + + String payload = "Small optimistic payload under min-append-size."; + + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + servletRequest.setContent(payload.getBytes(StandardCharsets.UTF_8)); + + UploadInfo completedInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + + assertResponseStatus(HttpServletResponse.SC_OK); + assertResponseHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + assertNotNull(completedInfo); + assertFalse(completedInfo.isUploadInProgress()); + assertEquals(Long.valueOf(payload.length()), completedInfo.getOffset()); + + try (InputStream stream = tusFileUploadService.getUploadedBytes(completedInfo)) { + assertThat(IOUtils.toString(stream, StandardCharsets.UTF_8), is(payload)); + } + } + + /** + * Section 4.1.4 (Limits): "min-append-size: An Integer indicating the minimum number of bytes + * that MUST be appended with each upload append request. This limit does not apply to upload + * creation requests with no content, or to requests completing the upload by including the + * Upload-Complete: ?1 header field." + * + *

Use Case: Configure min-append-size (1 MB), initiate an upload without content, and append a + * completing chunk smaller than 1 MB with Upload-Complete: ?1. Verify that the completing append + * is exempt from min-append-size and completes the upload cleanly. + */ + @Test + public void testCompletingAppendPatchBypassesMinAppendSize() throws Exception { + tusFileUploadService.withMinAppendSize(1024L * 1024L); + + String uploadContent = "Small final chunk under min-append-size."; + + // Step 1: Initiate upload with POST, Upload-Complete: ?0, declared Upload-Length, and no body + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, String.valueOf(uploadContent.length())); + + UploadInfo createdInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + assertNotNull(createdInfo); + String uploadLocation = servletResponse.getHeader(HttpHeader.LOCATION); + + // Step 2: Append completing chunk with Upload-Complete: ?1 + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(uploadLocation); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + servletRequest.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + servletRequest.setContent(uploadContent.getBytes(StandardCharsets.UTF_8)); + + UploadInfo completedInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_OK); + assertResponseHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + assertNotNull(completedInfo); + assertFalse(completedInfo.isUploadInProgress()); + assertEquals(Long.valueOf(uploadContent.length()), completedInfo.getOffset()); + + try (InputStream stream = tusFileUploadService.getUploadedBytes(completedInfo)) { + assertThat(IOUtils.toString(stream, StandardCharsets.UTF_8), is(uploadContent)); + } + } + // =============================================================================================== // ASSERTION HELPERS // =============================================================================================== diff --git a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java index 13812ae1..40ce876b 100644 --- a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java @@ -1583,7 +1583,7 @@ public void testLockContentionAndHeadRelease() throws Exception { InputStream blockingStream = new InputStream() { - private volatile boolean closed = false; + private boolean closed = false; @Override public int read() throws IOException { @@ -1727,7 +1727,7 @@ public void testInterruptedUploadPersistsBytesBeforeHeadResponds() throws Except InputStream partialBlockingStream = new InputStream() { private final InputStream byteStream = new ByteArrayInputStream(payload); - private volatile boolean closed = false; + private boolean closed = false; @Override public int read() throws IOException { diff --git a/src/test/java/me/desair/tus/server/TestUtils.java b/src/test/java/me/desair/tus/server/TestUtils.java index 32433879..a8358eea 100644 --- a/src/test/java/me/desair/tus/server/TestUtils.java +++ b/src/test/java/me/desair/tus/server/TestUtils.java @@ -7,8 +7,8 @@ import org.testcontainers.containers.GenericContainer; /** - * Helper utility class for S3 integration tests running against Testcontainers MinIO using the - * MinIO Java SDK. Supports both Docker and Podman container engines automatically. + * Helper utility class for integration tests running against Testcontainers RustFS (S3) and Azurite + * (Azure Blob). Supports both Docker and Podman container engines automatically. */ public final class TestUtils { @@ -78,12 +78,13 @@ public static GenericContainer createRustFsContainer() { } /** - * Alias for {@link #createRustFsContainer()} for compatibility. + * Returns the S3 endpoint URL for the active S3 Testcontainer (RustFS). * - * @return A configured GenericContainer instance (not started yet) + * @param s3Container The active S3 Testcontainer (RustFS) + * @return S3 endpoint URL */ - public static GenericContainer createMinioContainer() { - return createRustFsContainer(); + public static String getS3Endpoint(GenericContainer s3Container) { + return "http://" + s3Container.getHost() + ":" + s3Container.getMappedPort(9000); } /** @@ -93,7 +94,7 @@ public static GenericContainer createMinioContainer() { * @return Pre-configured MinioClient */ public static MinioClient createMinioClient(GenericContainer s3Container) { - String s3Url = "http://" + s3Container.getHost() + ":" + s3Container.getMappedPort(9000); + String s3Url = getS3Endpoint(s3Container); return MinioClient.builder().endpoint(s3Url).credentials("rustfsadmin", "rustfsadmin").build(); } diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 90dc8a99..b1b8ec93 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -14,6 +14,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; +import java.util.UUID; import me.desair.tus.server.exception.UploadAlreadyLockedException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; @@ -62,7 +64,7 @@ public void testAcquireUploadLockFallback() throws Exception { TusFileUploadService service = new TusFileUploadService() .withUploadLockingService(mockLockingService) - .withMaxLockRetries(5); + .withLockWaitTimeout(Duration.ofSeconds(6)); UploadLock lock = service.acquireUploadLock(HttpMethod.HEAD, "/files/test"); assertNotNull(lock); @@ -331,6 +333,7 @@ public void testWithUploadStorageServicePreservesConfiguration() { service.withMaxAppendSize(1024L); service.withMinAppendSize(512L); service.withMinSize(2048L); + service.withCloudUploadThreadPoolSize(18); service.withUploadDeduplication(true); UploadStorageService newStorage = mock(UploadStorageService.class); @@ -340,7 +343,9 @@ public void testWithUploadStorageServicePreservesConfiguration() { verify(newStorage).setMaxAppendSize(1024L); verify(newStorage).setMinAppendSize(512L); verify(newStorage).setMinSize(2048L); + verify(newStorage).setCloudUploadThreadPoolSize(18); verify(newStorage).setUploadDeduplicationEnabled(true); + verify(newStorage).setDrainTimeout(Duration.ofSeconds(55)); } @Test @@ -349,11 +354,26 @@ public void testThreadLocalCacheDelegatesAppendAndMinSizes() { service.withMaxAppendSize(1024L); service.withMinAppendSize(512L); service.withMinSize(2048L); + service.withCloudUploadThreadPoolSize(16); service.withThreadLocalCache(true); assertThat(service.getUploadStorageService().getMaxAppendSize(), is(1024L)); assertThat(service.getUploadStorageService().getMinAppendSize(), is(512L)); assertThat(service.getUploadStorageService().getMinSize(), is(2048L)); + assertThat(service.getUploadStorageService().getDrainTimeout(), is(Duration.ofSeconds(55))); + assertThat(service.getUploadStorageService().getCloudUploadThreadPoolSize(), is(16)); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithCloudUploadThreadPoolSizeZeroThrowsException() { + TusFileUploadService service = new TusFileUploadService(); + service.withCloudUploadThreadPoolSize(0); + } + + @Test(expected = IllegalArgumentException.class) + public void testWithCloudUploadThreadPoolSizeNegativeThrowsException() { + TusFileUploadService service = new TusFileUploadService(); + service.withCloudUploadThreadPoolSize(-5); } @Test @@ -613,20 +633,53 @@ public void testDeleteUploadSingleArg() throws Exception { } @Test - public void testWithMaxLockRetries() { + public void testWithLockWaitTimeout() { TusFileUploadService service = new TusFileUploadService(); - assertEquals(40, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(60), service.getLockWaitTimeout()); + assertEquals(300, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(55), service.getUploadStorageService().getDrainTimeout()); + + service.withLockWaitTimeout(Duration.ofSeconds(30)); + assertEquals(Duration.ofSeconds(30), service.getLockWaitTimeout()); + assertEquals(150, service.getMaxLockRetries()); + assertEquals(Duration.ofSeconds(25), service.getUploadStorageService().getDrainTimeout()); + + service.withLockWaitTimeout(Duration.ofMillis(6000)); + assertEquals(Duration.ofMillis(6000), service.getLockWaitTimeout()); + assertEquals(30, service.getMaxLockRetries()); + assertEquals(Duration.ofMillis(1000), service.getUploadStorageService().getDrainTimeout()); + } + + @Test(expected = NullPointerException.class) + public void testWithLockWaitTimeoutNullThrows() { + new TusFileUploadService().withLockWaitTimeout(null); + } - service.withMaxLockRetries(5); - assertEquals(5, service.getMaxLockRetries()); + @Test(expected = IllegalArgumentException.class) + public void testWithLockWaitTimeoutFiveSecondsOrLessThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(5)); + } - service.withMaxLockRetries(0); - assertEquals(0, service.getMaxLockRetries()); + @Test(expected = IllegalArgumentException.class) + public void testWithLockWaitTimeoutZeroThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ZERO); } @Test(expected = IllegalArgumentException.class) - public void testWithMaxLockRetriesNegativeThrows() { - new TusFileUploadService().withMaxLockRetries(-1); + public void testWithLockWaitTimeoutNegativeThrows() { + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(-1)); + } + + @Test + public void testWithUploadStorageServiceCopiesDrainTimeout() { + TusFileUploadService service = + new TusFileUploadService().withLockWaitTimeout(Duration.ofSeconds(20)); + + UploadStorageService newStorageService = mock(UploadStorageService.class); + when(newStorageService.isUploadDeduplicationEnabled()).thenReturn(false); + + service.withUploadStorageService(newStorageService); + verify(newStorageService).setDrainTimeout(Duration.ofSeconds(15)); } @Test @@ -643,7 +696,7 @@ public void testAcquireUploadLockWithConfiguredRetries() throws Exception { TusFileUploadService service = new TusFileUploadService() .withUploadLockingService(mockLockingService) - .withMaxLockRetries(2); + .withLockWaitTimeout(Duration.ofMillis(5200)); UploadLock lock = service.acquireUploadLock(HttpMethod.HEAD, "/files/test"); assertNotNull(lock); @@ -879,4 +932,123 @@ public void testProcessLockedRequestCatchesUnexpectedRuntimeException() throws E assertThat(result, is(nullValue())); verify(mockResp, atLeastOnce()).setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } + + @Test + public void testProcessCleansUpLockOnUploadCompletion() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadLock mockLock = mock(UploadLock.class); + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + + UploadStorageService mockStorage = mock(UploadStorageService.class); + UUID uuid = UUID.randomUUID(); + UploadId testId = new UploadId(uuid); + String uploadUri = "/files/upload/" + uuid; + + UploadInfo inProgressInfo = new UploadInfo(); + inProgressInfo.setId(testId); + inProgressInfo.setOffset(5L); + inProgressInfo.setLength(10L); + + UploadInfo completedInfo = new UploadInfo(); + completedInfo.setId(testId); + completedInfo.setOffset(10L); + completedInfo.setLength(10L); + + when(mockStorage.getUploadInfo(uploadUri, "owner")) + .thenReturn(inProgressInfo) + .thenReturn(inProgressInfo) + .thenReturn(inProgressInfo) + .thenReturn(completedInfo); + when(mockStorage.append(any(), any())).thenReturn(completedInfo); + + jakarta.servlet.http.HttpServletRequest mockReq = + mock(jakarta.servlet.http.HttpServletRequest.class); + jakarta.servlet.http.HttpServletResponse mockResp = + mock(jakarta.servlet.http.HttpServletResponse.class); + when(mockReq.getMethod()).thenReturn("PATCH"); + when(mockReq.getRequestURI()).thenReturn(uploadUri); + when(mockReq.getHeader(anyString())).thenReturn(null); + when(mockReq.getHeader(HttpHeader.TUS_RESUMABLE)).thenReturn("1.0.0"); + when(mockReq.getHeader(HttpHeader.CONTENT_TYPE)).thenReturn("application/offset+octet-stream"); + when(mockReq.getHeader(HttpHeader.UPLOAD_OFFSET)).thenReturn("5"); + when(mockReq.getInputStream()) + .thenReturn( + new me.desair.tus.server.util.MockServletInputStream( + new ByteArrayInputStream(new byte[5]))); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorage); + + service.process(mockReq, mockResp, "owner"); + + verify(mockLockingService).cleanupLock(testId); + } + + @Test + public void testProcessCleansUpLockOnDeleteTermination() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadLock mockLock = mock(UploadLock.class); + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + + UploadStorageService mockStorage = mock(UploadStorageService.class); + UUID uuid = UUID.randomUUID(); + UploadId testId = new UploadId(uuid); + String uploadUri = "/files/upload/" + uuid; + + UploadInfo info = new UploadInfo(); + info.setId(testId); + info.setOffset(5L); + info.setLength(10L); + + when(mockStorage.getUploadInfo(uploadUri, "owner")).thenReturn(info); + + jakarta.servlet.http.HttpServletRequest mockReq = + mock(jakarta.servlet.http.HttpServletRequest.class); + org.springframework.mock.web.MockHttpServletResponse mockResp = + new org.springframework.mock.web.MockHttpServletResponse(); + when(mockReq.getMethod()).thenReturn("DELETE"); + when(mockReq.getRequestURI()).thenReturn(uploadUri); + when(mockReq.getHeader(anyString())).thenReturn(null); + when(mockReq.getHeader(HttpHeader.TUS_RESUMABLE)).thenReturn("1.0.0"); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorage); + + service.process(mockReq, mockResp, "owner"); + + verify(mockLockingService).cleanupLock(testId); + } + + @Test + public void testDeleteUploadCleansUpLock() throws Exception { + UploadLockingService mockLockingService = mock(UploadLockingService.class); + UploadLock mockLock = mock(UploadLock.class); + when(mockLockingService.lockUploadByUri(anyString())).thenReturn(mockLock); + + UploadStorageService mockStorage = mock(UploadStorageService.class); + UUID uuid = UUID.randomUUID(); + UploadId testId = new UploadId(uuid); + String uploadUri = "/files/upload/" + uuid; + + UploadInfo info = new UploadInfo(); + info.setId(testId); + when(mockStorage.getUploadInfo(uploadUri, "owner")).thenReturn(info); + + TusFileUploadService service = + new TusFileUploadService() + .withUploadUri("/files/upload") + .withUploadLockingService(mockLockingService) + .withUploadStorageService(mockStorage); + + service.deleteUpload(uploadUri, "owner"); + + verify(mockStorage).terminateUpload(info); + verify(mockLockingService).cleanupLock(testId); + } } diff --git a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java index 05576a70..1656ae53 100644 --- a/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/core/CorePatchRequestHandlerTest.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.when; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; import java.io.InputStream; import java.util.UUID; import me.desair.tus.server.HttpHeader; @@ -226,4 +227,77 @@ public void testProcess7ParamsBranchCoverage() throws Exception { "owner", null); } + + @Test + public void testProcessInterruptedByLockingServiceContentionReturns204WithUpdatedOffset() + throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + + servletRequest.setRequestURI("/test/upload/123"); + servletRequest.setContent("partial data".getBytes()); + + UploadInfo initialInfo = new UploadInfo(); + initialInfo.setId(new UploadId("123")); + initialInfo.setOffset(0L); + initialInfo.setLength(100L); + + UploadInfo refreshedInfo = new UploadInfo(); + refreshedInfo.setId(new UploadId("123")); + refreshedInfo.setOffset(50L); + refreshedInfo.setLength(100L); + + when(uploadStorageService.getUploadInfo("/test/upload/123", "owner")) + .thenReturn(initialInfo) + .thenReturn(refreshedInfo); + + when(uploadStorageService.append(any(UploadInfo.class), any(InputStream.class))) + .thenAnswer( + invocation -> { + InputStream stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + "owner", + null); + + assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); + assertThat(servletResponse.getHeader(HttpHeader.UPLOAD_OFFSET), is("50")); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + + servletRequest.setRequestURI("/test/upload/123"); + servletRequest.setContent("data".getBytes()); + + UploadInfo initialInfo = new UploadInfo(); + initialInfo.setId(new UploadId("123")); + initialInfo.setOffset(0L); + initialInfo.setLength(100L); + + when(uploadStorageService.getUploadInfo("/test/upload/123", "owner")).thenReturn(initialInfo); + + when(uploadStorageService.append(any(UploadInfo.class), any(InputStream.class))) + .thenThrow(new IOException("Disk failure")); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + "owner", + null); + } } diff --git a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java index 9001c2c0..bf7848a9 100644 --- a/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandlerTest.java @@ -10,6 +10,7 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -95,4 +96,67 @@ public void testProcessWithoutLockingService() throws Exception { verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "5"); } + + @Test + public void testProcessInterruptedByLockingServiceContention() throws Exception { + TusServletRequest request = mock(TusServletRequest.class); + TusServletResponse response = mock(TusServletResponse.class); + UploadStorageService storageService = mock(UploadStorageService.class); + UploadLockingService lockingService = mock(UploadLockingService.class); + + when(request.getHeader(HttpHeader.CONTENT_LENGTH)).thenReturn("100"); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/interrupted-cwu"); + when(request.getContentInputStream()).thenReturn(new ByteArrayInputStream("data".getBytes())); + + UploadInfo uploadInfo = new UploadInfo(); + uploadInfo.setLength(100L); + uploadInfo.setOffset(0L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setLength(100L); + refreshed.setOffset(50L); + + when(storageService.getUploadInfo("/files/interrupted-cwu", "owner")) + .thenReturn(uploadInfo) + .thenReturn(refreshed); + + when(storageService.append(eq(uploadInfo), any(InputStream.class))) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + + verify(response).setHeader(HttpHeader.UPLOAD_OFFSET, "50"); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + TusServletRequest request = mock(TusServletRequest.class); + TusServletResponse response = mock(TusServletResponse.class); + UploadStorageService storageService = mock(UploadStorageService.class); + UploadLockingService lockingService = mock(UploadLockingService.class); + + when(request.getHeader(HttpHeader.CONTENT_LENGTH)).thenReturn("100"); + when(response.getHeader(HttpHeader.LOCATION)).thenReturn("/files/cwu-error"); + when(request.getContentInputStream()).thenReturn(new ByteArrayInputStream("data".getBytes())); + + UploadInfo uploadInfo = new UploadInfo(); + uploadInfo.setLength(100L); + uploadInfo.setOffset(0L); + + when(storageService.getUploadInfo("/files/cwu-error", "owner")).thenReturn(uploadInfo); + when(storageService.append(eq(uploadInfo), any(InputStream.class))) + .thenThrow(new IOException("S3 network error")); + + handler.process( + HttpMethod.POST, request, response, storageService, lockingService, "owner", null); + } } diff --git a/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java b/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java index 7e2652c0..da6944f9 100644 --- a/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/expiration/ExpirationRequestHandlerTest.java @@ -15,7 +15,10 @@ import java.util.TimeZone; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; +import me.desair.tus.server.exception.UploadNotFoundException; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; @@ -209,6 +212,37 @@ public void testNegativeExpiration() throws Exception { assertThat(tusResponse.getHeader(HttpHeader.UPLOAD_EXPIRES), is(nullValue())); } + @Test(expected = UploadNotFoundException.class) + public void testExpiredUploadTerminatesUploadAndCleansUpLock() throws Exception { + UploadInfo info = createUploadInfo(); + info.setId(new UploadId("expired-upload-id")); + info.setOffset(2L); + info.setLength(10L); + // Set expiration in the past relative to mock time (2018-01-20 10:43:11) + info.setExpirationTimestamp(1000L); + + when(uploadStorageService.getUploadInfo(nullable(String.class), nullable(String.class))) + .thenReturn(info); + when(uploadStorageService.getUploadExpirationPeriod()).thenReturn(172800000L); + + UploadLockingService mockLocking = org.mockito.Mockito.mock(UploadLockingService.class); + TusServletResponse tusResponse = new TusServletResponse(this.servletResponse); + + try { + handler.process( + HttpMethod.PATCH, + new TusServletRequest(servletRequest), + tusResponse, + uploadStorageService, + mockLocking, + null, + null); + } finally { + verify(uploadStorageService, times(1)).terminateUpload(info); + verify(mockLocking, times(1)).cleanupLock(info.getId()); + } + } + private UploadInfo createUploadInfo() { return new UploadInfo() { @Override diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index 46f3a3d1..27441b1e 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -10,6 +10,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; @@ -162,6 +164,7 @@ public void testProcessCallFiveParameterMethodAndAppendReturnsNull() throws Exce request.setRequestURI("/files/append-id"); request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.addHeader(HttpHeader.CONTENT_LENGTH, "4"); request.setContent("data".getBytes()); UploadInfo info = new UploadInfo(); @@ -215,4 +218,217 @@ public void testProcessIsFinishedCombinations() throws Exception { assertThat(response.getStatus(), is(200)); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + * + *

§4.2.1: "If the upload length is not known when creating the upload resource, the + * Upload-Length header field is omitted, and the length is deferred... In subsequent requests, + * the upload length can be indicated by including the Upload-Length header field or by completing + * the upload using the Upload-Complete: ?1 header field." + */ + @Test + public void testProcessCompletingAppendOnDeferredLengthUploadSetsLength() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "100"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "final-bytes".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(100L); + info.setLength(null); // Deferred length + + UploadInfo updated = new UploadInfo(); + updated.setId(info.getId()); + updated.setOffset(100L + content.length); + updated.setLength(100L + content.length); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(updated); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).update(info); + assertThat(info.getLength(), is(100L + content.length)); + assertThat(response.getStatus(), is(200)); + assertThat( + response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(100L + content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } + + @Test + public void testProcessCompletingAppendOnDeferredLengthUploadWithNullOffsetSetsLength() + throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "initial-and-final-bytes".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(null); + info.setLength(null); // Deferred length + + UploadInfo updated = new UploadInfo(); + updated.setId(info.getId()); + updated.setOffset((long) content.length); + updated.setLength((long) content.length); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(updated); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).update(info); + assertThat(info.getLength(), is((long) content.length)); + assertThat(response.getStatus(), is(200)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } + + @Test + public void testProcessInterruptedByLockingServiceContention() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + byte[] content = "partial-data".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(0L); + info.setLength(1000L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("append-id")); + refreshed.setOffset(500L); + refreshed.setLength(1000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")) + .thenReturn(info) + .thenReturn(refreshed); + + when(storageService.append(any(UploadInfo.class), any())) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(204)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("500")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } + + @Test(expected = IOException.class) + public void testProcessUninterruptedIoExceptionRethrown() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "0"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + byte[] content = "data".getBytes(); + request.setContent(content); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("append-id")); + info.setOffset(0L); + info.setLength(1000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); + when(storageService.append(any(UploadInfo.class), any())) + .thenThrow(new IOException("Storage failure")); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } + + @Test + public void testProcessInterruptedByLockContention() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.setContent("hello world".getBytes()); + + UploadInfo initial = new UploadInfo(); + initial.setId(new UploadId("append-id")); + initial.setOffset(1000L); + initial.setLength(2000L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("append-id")); + refreshed.setOffset(1005L); + refreshed.setLength(2000L); + + when(storageService.getUploadInfo("/files/append-id", "owner")) + .thenReturn(initial) + .thenReturn(refreshed); + + when(storageService.append(eq(initial), any(InputStream.class))) + .thenAnswer( + inv -> { + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Stream interrupted by locking service contention"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(204)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("1005")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index f6dce0c7..34ad2bf4 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -11,6 +11,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.io.InputStream; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; @@ -473,4 +475,218 @@ public void testProcessCompletedUploadCreationWithAbsoluteUploadUri() throws Exc assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("100")); assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); } + + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + * + *

§4.2.1: "If the upload length is not known when creating the upload resource, the + * Upload-Length header field is omitted, and the length is deferred... In subsequent requests, + * the upload length can be indicated by including the Upload-Length header field or by completing + * the upload using the Upload-Complete: ?1 header field." + */ + @Test + public void testProcessCreationWithUploadCompleteAndContentLengthSetsAnnouncedLength() + throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + byte[] content = "hello world".getBytes(); + request.setContent(content); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("complete-no-length-id")); + createdInfo.setLength((long) content.length); + createdInfo.setOffset((long) content.length); + + when(storageService.create(captor.capture(), nullable(String.class))).thenReturn(createdInfo); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(createdInfo); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(captor.getValue().getLength(), is((long) content.length)); + assertThat(response.getStatus(), is(200)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?1")); + } + + @Test + public void testProcessCreationWithChunkedTransferEncodingCallsAppend() throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.addHeader(HttpHeader.TRANSFER_ENCODING, "chunked"); + byte[] content = "chunked stream data".getBytes(); + // Do not set content via setContent to keep Content-Length at -1, provide via InputStream + request.setContent(content); + + // Custom request to simulate chunked request without Content-Length header (cl < 0) + TusServletRequest tusRequest = + new TusServletRequest(request) { + @Override + public long getContentLengthLong() { + return -1L; + } + + @Override + public InputStream getContentInputStream() { + return new java.io.ByteArrayInputStream(content); + } + }; + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("chunked-id")); + createdInfo.setOffset(0L); + + UploadInfo appendedInfo = new UploadInfo(); + appendedInfo.setId(new UploadId("chunked-id")); + appendedInfo.setOffset((long) content.length); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + when(storageService.append(any(UploadInfo.class), any())).thenReturn(appendedInfo); + + handler.process( + HttpMethod.POST, + tusRequest, + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + verify(storageService).append(eq(createdInfo), any(InputStream.class)); + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is(String.valueOf(content.length))); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } + + @Test + public void testProcessCreationWithBodyInterrupted() throws Exception { + byte[] content = "partial-creation-data".getBytes(); + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.setContent(content); + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("interrupted-create-id")); + createdInfo.setOffset(0L); + createdInfo.setLength(1000L); + + UploadInfo refreshedInfo = new UploadInfo(); + refreshedInfo.setId(new UploadId("interrupted-create-id")); + refreshedInfo.setOffset(500L); + refreshedInfo.setLength(1000L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + when(storageService.getUploadInfo("/files/interrupted-create-id", "owner")) + .thenReturn(refreshedInfo); + + when(storageService.append(any(UploadInfo.class), any())) + .thenAnswer( + invocation -> { + Object stream = invocation.getArgument(1); + if (stream instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) stream).interrupt(); + } + throw new IOException( + "Stream was interrupted by the upload locking service watchdog"); + }); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("500")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + } + + @Test(expected = IOException.class) + public void testProcessCreationWithBodyUninterruptedIoException() throws Exception { + byte[] content = "creation-data".getBytes(); + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.setContent(content); + + UploadInfo createdInfo = new UploadInfo(); + createdInfo.setId(new UploadId("fail-create-id")); + createdInfo.setOffset(0L); + createdInfo.setLength(1000L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))) + .thenReturn(createdInfo); + + when(storageService.append(any(UploadInfo.class), any())) + .thenThrow(new IOException("Disk write failure")); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + } + + @Test + public void testProcessCreationInterruptedByLockContention() throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + request.addHeader(HttpHeader.UPLOAD_LENGTH, "1000"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + request.setContent("creation body".getBytes()); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("creation-id")); + info.setLength(1000L); + info.setOffset(0L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("creation-id")); + refreshed.setLength(1000L); + refreshed.setOffset(13L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))).thenReturn(info); + when(storageService.getUploadInfo("/files/creation-id", "owner")).thenReturn(refreshed); + + when(storageService.append(eq(info), any(InputStream.class))) + .thenAnswer( + inv -> { + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Stream interrupted by locking service contention"); + }); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("13")); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java index 024993b2..e451523d 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhDeleteRequestHandlerTest.java @@ -10,6 +10,7 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; @@ -56,17 +57,20 @@ public void testProcessDeleteRequest() throws Exception { info.setId(new UploadId("delete-id")); when(storageService.getUploadInfo("/files/delete-id", "owner")).thenReturn(info); + UploadLockingService mockLocking = org.mockito.Mockito.mock(UploadLockingService.class); + handler.process( HttpMethod.DELETE, new TusServletRequest(request), new TusServletResponse(response), storageService, - null, + mockLocking, "owner", null); assertThat(response.getStatus(), is(204)); verify(storageService).terminateUpload(info); + verify(mockLocking).cleanupLock(info.getId()); } @Test(expected = me.desair.tus.server.exception.TusException.class) diff --git a/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java b/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java index 27f03d77..406d7b52 100644 --- a/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java +++ b/src/test/java/me/desair/tus/server/rufh/util/RufhInterimResponseUtilTest.java @@ -96,8 +96,8 @@ public void testGetRawInterimResponseWithExistingUploadAndStorageException() thr @Test public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() throws Exception { MockHttpServletRequest request = new MockHttpServletRequest(); - request.setMethod("PATCH"); - request.setRequestURI("/files/not-found-123"); + request.setMethod("POST"); + request.setRequestURI("/files"); request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); // Host header is not set (null host) @@ -107,8 +107,6 @@ public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() thr me.desair.tus.server.upload.UploadInfo created = new me.desair.tus.server.upload.UploadInfo(); created.setId(new me.desair.tus.server.upload.UploadId("created-456")); - org.mockito.Mockito.when(mockStorage.getUploadInfo("/files/not-found-123", "owner")) - .thenReturn(null); org.mockito.Mockito.when( mockStorage.create( org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.eq("owner"))) @@ -119,6 +117,23 @@ public void testGetRawInterimResponseWithExistingUploadNotFoundAndNullHost() thr assertTrue(raw.contains("Location: /files/created-456")); } + @Test + public void testGetRawInterimResponseWithPatchUploadNotFoundReturnsNull() throws Exception { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("PATCH"); + request.setRequestURI("/files/not-found-123"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?0"); + + me.desair.tus.server.upload.UploadStorageService mockStorage = + org.mockito.Mockito.mock(me.desair.tus.server.upload.UploadStorageService.class); + org.mockito.Mockito.when(mockStorage.getUploadUri()).thenReturn("/files"); + org.mockito.Mockito.when(mockStorage.getUploadInfo("/files/not-found-123", "owner")) + .thenReturn(null); + + String raw = RufhInterimResponseUtil.getRawInterimResponse(request, mockStorage, "owner"); + assertNull(raw); + } + @Test public void testGetRawInterimResponseWithAbsoluteUploadUri() throws Exception { MockHttpServletRequest request = new MockHttpServletRequest(); diff --git a/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java b/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java index 54521e97..c4753fd2 100644 --- a/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/termination/TerminationDeleteRequestHandlerTest.java @@ -14,6 +14,7 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; @@ -92,4 +93,31 @@ public void testWithExistingUpload() throws Exception { verify(uploadStorageService, times(1)).terminateUpload(info); assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); } + + @Test + public void testWithExistingUploadAndLockingService() throws Exception { + final UploadId id = new UploadId(UUID.randomUUID()); + + UploadInfo info = new UploadInfo(); + info.setId(id); + info.setOffset(2L); + info.setLength(10L); + when(uploadStorageService.getUploadInfo(nullable(String.class), nullable(String.class))) + .thenReturn(info); + + UploadLockingService mockLocking = org.mockito.Mockito.mock(UploadLockingService.class); + + handler.process( + HttpMethod.DELETE, + new TusServletRequest(servletRequest), + new TusServletResponse(servletResponse), + uploadStorageService, + mockLocking, + null, + null); + + verify(uploadStorageService, times(1)).terminateUpload(info); + verify(mockLocking, times(1)).cleanupLock(id); + assertThat(servletResponse.getStatus(), is(HttpServletResponse.SC_NO_CONTENT)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java index 38098a52..bc7715c3 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java @@ -150,7 +150,7 @@ public void testRenewLeaseFailureInterruptsActiveStreamAndShutsDownExecutor() { LeaseData leaseData = new LeaseData( - "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() + 10000L); + "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() - 1000L); TestLeaseLock lock = new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { @@ -167,4 +167,61 @@ protected void doRenewLease() { assertTrue(stream.isInterrupted()); verify(mockExecutor).shutdownNow(); } + + @Test + public void testRenewLeaseTransientFailureDoesNotInterruptActiveStreamIfLeaseValid() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + Map activeStreams = new ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new ByteArrayInputStream("test".getBytes())); + activeStreams.put("/files/upload-failing", stream); + + LeaseData leaseData = + new LeaseData( + "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() + 10000L); + + TestLeaseLock lock = + new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { + @Override + protected void doRenewLease() { + throw new RuntimeException("Simulated transient network glitch"); + } + }; + + // Trigger renewLease and verify that transient glitch does not abort while lease is still valid + lock.renewLease(); + + org.junit.Assert.assertFalse(stream.isInterrupted()); + org.mockito.Mockito.verify(mockExecutor, org.mockito.Mockito.never()).shutdownNow(); + } + + @Test + public void testRenewLeaseOwnershipLostInterruptsActiveStreamAndShutsDownExecutor() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + Map activeStreams = new ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new ByteArrayInputStream("test".getBytes())); + activeStreams.put("/files/upload-lost", stream); + + LeaseData leaseData = + new LeaseData( + "holder-lost", "/files/upload-lost", 10000L, System.currentTimeMillis() + 10000L); + + TestLeaseLock lock = + new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { + @Override + protected void doRenewLease() { + throw new IllegalStateException("Lease taken over by rival holder"); + } + }; + + lock.renewLease(); + + assertTrue(stream.isInterrupted()); + verify(mockExecutor).shutdownNow(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java index d547ef9c..7b4bbb74 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java @@ -4,6 +4,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -32,11 +33,17 @@ private static class TestLeaseLockingService extends AbstractLeaseLockingService private final List stopSignalsWritten = new ArrayList<>(); private final List stopSignalsChecked = new ArrayList<>(); private final AtomicBoolean cleanedUp = new AtomicBoolean(false); + private final List cleanedUpLocks = new ArrayList<>(); public TestLeaseLockingService() { super(new UuidUploadIdFactory(), 30000L, 1000L, null, "test-lease-locking-service-watchdog"); } + @Override + public void cleanupLock(UploadId id) throws IOException { + cleanedUpLocks.add(id); + } + @Override protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { if (acquireShouldSucceed) { @@ -77,6 +84,10 @@ public void testJitter(long min, long max) { applyJitter(min, max); } + public void testJitterDefault() { + applyJitter(); + } + public void testCheckStopSignals() { checkStopSignals(); } @@ -228,8 +239,60 @@ public void testCheckStopSignalsIteratesActiveStreams() { } @Test - public void testApplyJitter() { - service.testJitter(1L, 5L); + public void testDefaultJitterBounds() { + assertEquals(20L, service.getJitterMinMs()); + assertEquals(60L, service.getJitterMaxMs()); + } + + @Test + public void testSetJitterValidBounds() { + service.setJitter(10L, 50L); + assertEquals(10L, service.getJitterMinMs()); + assertEquals(50L, service.getJitterMaxMs()); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetJitterNegativeMinThrows() { + service.setJitter(-1L, 50L); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetJitterMaxLessThanMinThrows() { + service.setJitter(50L, 10L); + } + + @Test + public void testApplyJitterWithZeroOrNegativeMaxMsReturnsImmediately() { + long start = System.currentTimeMillis(); + service.testJitter(0L, 0L); + service.testJitter(0L, -5L); + long elapsed = System.currentTimeMillis() - start; + // KISS: zero/negative maxMs returns immediately without sleep + assertTrue(elapsed < 100L); + } + + @Test + public void testApplyJitterWhenMinGreaterThanOrEqualMax() { + long start = System.currentTimeMillis(); + service.testJitter(2L, 2L); + service.testJitter(3L, 2L); + long elapsed = System.currentTimeMillis() - start; + assertTrue(elapsed >= 2L); + } + + @Test + public void testApplyJitterDefault() { + service.setJitter(1L, 3L); + service.testJitterDefault(); + assertEquals(1L, service.getJitterMinMs()); + assertEquals(3L, service.getJitterMaxMs()); + } + + @Test + public void testApplyJitterPreservesInterrupt() { + Thread.currentThread().interrupt(); + service.testJitter(10L, 20L); + assertTrue(Thread.interrupted()); } @Test @@ -244,4 +307,19 @@ public void testCleanupOnClose() throws Exception { assertTrue(service.cleanedUp.get()); assertTrue(service.isClosed()); } + + @Test + public void testCleanupLockByUriDelegatesToUploadId() throws Exception { + UUID uuid = UUID.randomUUID(); + UploadId id = new UploadId(uuid); + String uri = "/files/" + uuid; + service.cleanupLock(uri); + + assertEquals(1, service.cleanedUpLocks.size()); + assertEquals(id, service.cleanedUpLocks.get(0)); + + service.cleanupLock((String) null); + assertEquals(2, service.cleanedUpLocks.size()); + assertThat(service.cleanedUpLocks.get(1), is(nullValue())); + } } diff --git a/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java b/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java index a209f3fb..9d7371f5 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadInfoTest.java @@ -282,4 +282,23 @@ public void testGetCreatorIpAddressesWithXforwardedFor() throws Exception { UploadInfo info = new UploadInfo(servletRequest); assertThat(info.getCreatorIpAddresses(), is("24.23.22.21, 192.168.1.1, 10.11.12.13")); } + + @Test + public void testUploadPartKeysEqualsAndHashCode() { + UploadInfo info1 = new UploadInfo(); + UploadInfo info2 = new UploadInfo(); + + assertThat(info1.getUploadPartKeys(), nullValue()); + assertEquals(info1, info2); + assertEquals(info1.hashCode(), info2.hashCode()); + + java.util.List parts = java.util.Arrays.asList("part-1", "part-2"); + info1.setUploadPartKeys(parts); + assertNotEquals(info1, info2); + + info2.setUploadPartKeys(new java.util.ArrayList<>(parts)); + assertEquals(info1, info2); + assertEquals(info1.hashCode(), info2.hashCode()); + assertEquals(parts, info1.getUploadPartKeys()); + } } diff --git a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java index 10b7d436..f5ddb64e 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java @@ -33,6 +33,8 @@ public void setIdFactory(UploadIdFactory idFactory) {} // Test default methods for coverage dummyService.registerInputStream("/test/upload/123", new ByteArrayInputStream(new byte[0])); dummyService.requestLockRelease("/test/upload/123"); + dummyService.cleanupLock(new UploadId("123")); + dummyService.cleanupLock("/test/upload/123"); dummyService.close(); assertFalse(dummyService.isLocked(new UploadId("123"))); diff --git a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java index a3d2307e..151db73e 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadStorageServiceTest.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; @@ -142,6 +143,12 @@ public void testDefaultAppendAndSizeConfigurationMethods() throws Exception { dummyStorageService.setJsonSerializationEnabled(true); assertThat(dummyStorageService.isJsonSerializationEnabled(), is(false)); + dummyStorageService.setCloudUploadThreadPoolSize(15); + assertThat(dummyStorageService.getCloudUploadThreadPoolSize(), is(10)); + + dummyStorageService.setDrainTimeout(Duration.ofSeconds(20)); + assertThat(dummyStorageService.getDrainTimeout(), is(Duration.ofSeconds(55))); + // Default close is a no-op dummyStorageService.close(); } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java index 32360897..875edeaa 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java @@ -250,4 +250,51 @@ public void getPartialUploadsShouldThrowOnOwnerMismatch() throws Exception { // final upload's ownerKey, preventing unauthorized partial stitching. concatenationService.getPartialUploads(finalInfo); } + + @Test + public void getAuthorizedBlobUrlShouldReturnNullOnNullBlobClient() { + org.junit.Assert.assertNull(concatenationService.getAuthorizedBlobUrl(null)); + } + + @Test + public void getAuthorizedBlobUrlShouldAppendSasWhenGenerationSucceeds() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())).thenReturn("sig=mocked-token&sp=r"); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals( + "https://account.blob.core.windows.net/container/blob?sig=mocked-token&sp=r", result); + } + + @Test + public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasGenerationThrows() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())) + .thenThrow(new IllegalStateException("No shared key credentials")); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals("https://account.blob.core.windows.net/container/blob", result); + } + + @Test + public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasIsEmpty() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())).thenReturn(""); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals("https://account.blob.core.windows.net/container/blob", result); + } + + @Test + public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasIsNull() { + com.azure.storage.blob.BlobClient mockBlob = mock(com.azure.storage.blob.BlobClient.class); + when(mockBlob.getBlobUrl()).thenReturn("https://account.blob.core.windows.net/container/blob"); + when(mockBlob.generateSas(any())).thenReturn(null); + + String result = concatenationService.getAuthorizedBlobUrl(mockBlob); + assertEquals("https://account.blob.core.windows.net/container/blob", result); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java index 03897dbb..2da5badb 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java @@ -3,14 +3,25 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.LeaseStateType; +import com.azure.storage.blob.models.ListBlobsOptions; import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.List; import me.desair.tus.server.upload.TimeBasedUploadIdFactory; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.util.InterruptibleInputStream; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; /** * Offline unit tests for {@link AzureBlobLockingService} verifying parameter validation, prefix @@ -87,9 +98,69 @@ public void requestLockReleaseShouldDoNothingOnInvalidUri() { } @Test - public void cleanupStaleLocksShouldNotThrow() throws Exception { - // KISS: verifying method executes cleanly without throwing an exception - lockingService.cleanupStaleLocks(); + public void cleanupLockWithNullOrInvalidShouldDoNothing() throws Exception { + // KISS: verifying null/invalid parameters are handled safely without throwing exceptions + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + lockingService.cleanupLock("invalid-uri-no-id"); + } + + @Test + public void deleteLockBlobIfUnleasedNullReturnsFalse() { + assertFalse(lockingService.deleteLockBlobIfUnleased(null)); + } + + @Test + public void deleteLockBlobWithLeaseNullArguments() { + assertFalse(lockingService.deleteLockBlobWithLease(null, "lease-id")); + BlobClient mockBlob = Mockito.mock(BlobClient.class); + assertFalse(lockingService.deleteLockBlobWithLease(mockBlob, null)); + } + + @Test + public void deleteLockBlobWithLeaseExceptionHandled() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.doThrow(new RuntimeException("Simulated delete lease error")) + .when(mockBlob) + .deleteWithResponse(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + assertFalse(lockingService.deleteLockBlobWithLease(mockBlob, "lease-id")); + } + + @Test + public void cleanupLockWithActiveLockDeletesWithLease() throws Exception { + UploadId testId = new UploadId("test-active-lock-id"); + BlobClient mockBlob = Mockito.mock(BlobClient.class); + com.azure.storage.blob.specialized.BlobLeaseClient realLease = + new com.azure.storage.blob.specialized.BlobLeaseClientBuilder() + .blobClient(mockBlob) + .leaseId("mock-lease-123") + .buildClient(); + + AzureBlobUploadLock mockLock = Mockito.mock(AzureBlobUploadLock.class); + Mockito.when(mockLock.getLeaseClient()).thenReturn(realLease); + + AzureBlobLockingService testService = + new AzureBlobLockingService(containerClient) { + @Override + BlobClient getBlobClient(String blobName) { + return mockBlob; + } + + @Override + void deleteStopSignalBlob(String idStr) { + // KISS: no-op in unit test to avoid network call against dummy endpoint + } + + @Override + boolean deleteLockBlobWithLease(BlobClient lockBlob, String leaseId) { + return true; + } + }; + + testService.activeLocks.put("test-active-lock-id", mockLock); + testService.cleanupLock(testId); + + Mockito.verify(mockLock).markBlobDeleted(); } @Test @@ -105,19 +176,136 @@ public void closeInterruptsActiveWatchdogThread() throws Exception { InterruptibleInputStream stream = new InterruptibleInputStream(bais); lockingService.registerInputStream("/test/upload/88888", stream); - org.junit.Assert.assertFalse(stream.isInterrupted()); + assertFalse(stream.isInterrupted()); lockingService.close(); - org.junit.Assert.assertTrue(stream.isInterrupted()); + assertTrue(stream.isInterrupted()); } @Test public void testPollStopSignalsCleansStaleWeakReferences() { lockingService.activeStreams.put("stale-upload-id", new java.lang.ref.WeakReference<>(null)); - org.junit.Assert.assertTrue(lockingService.activeStreams.containsKey("stale-upload-id")); + assertTrue(lockingService.activeStreams.containsKey("stale-upload-id")); lockingService.pollStopSignals(); - org.junit.Assert.assertFalse(lockingService.activeStreams.containsKey("stale-upload-id")); + assertFalse(lockingService.activeStreams.containsKey("stale-upload-id")); + } + + @Test + public void cleanupLockHandlesExceptionGracefully() throws Exception { + AzureBlobLockingService throwingService = + new AzureBlobLockingService(containerClient) { + @Override + void deleteStopSignalBlob(String idStr) { + throw new RuntimeException("Simulated delete stop failure"); + } + }; + // KISS: verify method catches exception and logs debug without propagating + throwingService.cleanupLock(new UploadId("12345")); + } + + @Test + public void deleteLockBlobIfUnleasedCatchesExceptionAndReturnsFalse() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.when(mockBlob.exists()).thenThrow(new RuntimeException("Simulated exists failure")); + assertFalse(lockingService.deleteLockBlobIfUnleased(mockBlob)); + } + + @Test + public void testCleanupStaleLocksWithVariousBlobItems() throws Exception { + OffsetDateTime oldDate = OffsetDateTime.now().minusSeconds(200); + OffsetDateTime freshDate = OffsetDateTime.now(); + + BlobItem nullPropsItem = createTestBlobItemWithNullProps("locks/test.lock"); + BlobItem staleStopItem = createTestBlobItem("locks/stale.stop", oldDate, null); + BlobItem freshStopItem = createTestBlobItem("locks/fresh.stop", freshDate, null); + BlobItem leasedLockItem = + createTestBlobItem("locks/leased.lock", oldDate, LeaseStateType.LEASED); + BlobItem staleUnleasedLockItem = + createTestBlobItem("locks/stale.lock", oldDate, LeaseStateType.AVAILABLE); + BlobItem otherItem = createTestBlobItem("locks/other.txt", oldDate, null); + + List items = + Arrays.asList( + null, + nullPropsItem, + staleStopItem, + freshStopItem, + leasedLockItem, + staleUnleasedLockItem, + otherItem); + + BlobClient mockStaleBlob = Mockito.mock(BlobClient.class); + + AzureBlobLockingService mockService = + new AzureBlobLockingService(containerClient) { + @Override + Iterable listBlobs(ListBlobsOptions options) { + return items; + } + + @Override + BlobClient getBlobClient(String blobName) { + return mockStaleBlob; + } + + @Override + boolean deleteLockBlobIfUnleased(BlobClient lockBlob) { + return true; + } + }; + + mockService.cleanupStaleLocks(); + Mockito.verify(mockStaleBlob).deleteIfExists(); + } + + @Test(expected = IOException.class) + public void testCleanupStaleLocksThrowsIOExceptionOnListFailure() throws Exception { + AzureBlobLockingService failingService = + new AzureBlobLockingService(containerClient) { + @Override + Iterable listBlobs(ListBlobsOptions options) { + throw new RuntimeException("Simulated list failure"); + } + }; + failingService.cleanupStaleLocks(); + } + + @Test + public void ensureLockBlobExistsCatchesGenericException() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.doThrow(new RuntimeException("Simulated upload error")) + .when(mockBlob) + .upload(Mockito.any(), Mockito.anyLong()); + // KISS: verify method catches generic exception without propagating + lockingService.ensureLockBlobExists(mockBlob); + } + + private BlobItem createTestBlobItem( + String name, OffsetDateTime lastModified, LeaseStateType leaseState) { + com.azure.storage.blob.implementation.models.BlobName blobName = + new com.azure.storage.blob.implementation.models.BlobName() + .setContent(name) + .setEncoded(false); + com.azure.storage.blob.implementation.models.BlobItemPropertiesInternal props = + new com.azure.storage.blob.implementation.models.BlobItemPropertiesInternal() + .setLastModified(lastModified) + .setLeaseState(leaseState); + com.azure.storage.blob.implementation.models.BlobItemInternal itemInternal = + new com.azure.storage.blob.implementation.models.BlobItemInternal() + .setName(blobName) + .setProperties(props); + return com.azure.storage.blob.implementation.util.ModelHelper.populateBlobItem(itemInternal); + } + + private BlobItem createTestBlobItemWithNullProps(String name) { + com.azure.storage.blob.implementation.models.BlobName blobName = + new com.azure.storage.blob.implementation.models.BlobName() + .setContent(name) + .setEncoded(false); + com.azure.storage.blob.implementation.models.BlobItemInternal itemInternal = + new com.azure.storage.blob.implementation.models.BlobItemInternal().setName(blobName); + return com.azure.storage.blob.implementation.util.ModelHelper.populateBlobItem(itemInternal); } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index afffb6ef..27dfdb53 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -1,6 +1,7 @@ package me.desair.tus.server.upload.azure; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -8,9 +9,11 @@ import com.azure.storage.blob.BlobContainerClientBuilder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.upload.TimeBasedUploadIdFactory; @@ -233,4 +236,137 @@ public void calcOptimalBlockSizeCalculations() { long largeLength = 50_000L * 16 * 1024 * 1024L; assertEquals((largeLength / 50_000L) + 1, storageService.calcOptimalBlockSize(largeLength)); } + + @Test + public void testCloudUploadThreadPoolSizeConfiguration() { + assertEquals( + "Default thread pool size is 10", 10, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(30); + assertEquals( + "Updated thread pool size is 30", 30, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(4); + assertEquals("Reduced thread pool size is 4", 4, storageService.getCloudUploadThreadPoolSize()); + + try { + storageService.setCloudUploadThreadPoolSize(0); + org.junit.Assert.fail("Should reject 0 pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + + try { + storageService.setCloudUploadThreadPoolSize(-1); + org.junit.Assert.fail("Should reject negative pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + } + + @Test + public void testDrainTimeoutConfiguration() { + assertEquals( + "Default drain timeout is 55 seconds", + Duration.ofSeconds(55), + storageService.getDrainTimeout()); + + storageService.setDrainTimeout(Duration.ofSeconds(25)); + assertEquals( + "Updated drain timeout is 25 seconds", + Duration.ofSeconds(25), + storageService.getDrainTimeout()); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertEquals( + "Null drain timeout preserves previous value", + Duration.ofSeconds(25), + storageService.getDrainTimeout()); + } + + @Test + public void testCloseGracefulShutdown() throws Exception { + // Verify closing storage service cleanly terminates background upload executor without error + storageService.close(); + + // Verify close handles thread interruption gracefully on an active executor + AzureBlobStorageService interruptedService = new AzureBlobStorageService(containerClient); + Thread.currentThread().interrupt(); + try { + interruptedService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + } + + @Test + public void testGenerateBlockIdEntropyAndFormat() { + String blockId0 = storageService.generateBlockId(0); + String blockId1 = storageService.generateBlockId(0); + + assertNotNull(blockId0); + assertNotNull(blockId1); + // Two block IDs generated for the same index must have different UUID entropy + org.junit.Assert.assertNotEquals(blockId0, blockId1); + + byte[] decoded0 = java.util.Base64.getDecoder().decode(blockId0); + String text0 = new String(decoded0, java.nio.charset.StandardCharsets.UTF_8); + assertTrue(text0.startsWith("blk-000000-")); + assertEquals(19, text0.length()); // "blk-000000-" (11) + 8 hex chars = 19 + } + + @Test(expected = me.desair.tus.server.exception.MaxAppendSizeExceededException.class) + public void testValidateMaxAppendSizeThrowsWhenExceeded() throws Exception { + storageService.validateMaxAppendSize(100L, 50L); + } + + @Test + public void testValidateMaxAppendSizePassesWhenWithinLimit() throws Exception { + storageService.validateMaxAppendSize(50L, 100L); + storageService.validateMaxAppendSize(50L, null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test(expected = me.desair.tus.server.exception.MinAppendSizeNotMetException.class) + public void testValidateMinAppendSizeThrowsWhenNotMet() throws Exception { + storageService.setMinAppendSize(100L); + storageService.validateMinAppendSize(50L); + } + + @Test + public void testValidateMinAppendSizePassesWhenMet() throws Exception { + storageService.setMinAppendSize(100L); + storageService.validateMinAppendSize(150L); + storageService.setMinAppendSize(null); + storageService.validateMinAppendSize(10L); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testDeleteFileQuietly() throws Exception { + // Null and non-existent file are no-ops + storageService.deleteFileQuietly(null); + storageService.deleteFileQuietly(new File("/path/to/non/existent/file/tus.tmp")); + + // Normal file deletion + Path tempFile = Files.createTempFile("tus-delete-quietly", ".tmp"); + assertTrue(Files.exists(tempFile)); + storageService.deleteFileQuietly(tempFile.toFile()); + assertFalse(Files.exists(tempFile)); + + // Non-empty directory causes Files.delete to throw DirectoryNotEmptyException, caught quietly + Path tempDir = Files.createTempDirectory("tus-delete-quietly-dir"); + Path childFile = Files.createTempFile(tempDir, "child", ".tmp"); + try { + storageService.deleteFileQuietly(tempDir.toFile()); + assertTrue( + "Directory should still exist because deletion failed quietly", Files.exists(tempDir)); + } finally { + Files.deleteIfExists(childFile); + Files.deleteIfExists(tempDir); + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java index 7ee5988f..10ef2a7f 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java @@ -5,9 +5,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.specialized.BlobLeaseClient; import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import java.util.concurrent.ScheduledExecutorService; @@ -69,7 +73,8 @@ public void testLockGettersAndReleaseWithExecutor() throws Exception { @Test public void testNullRenewalExecutorAndNullLeaseClient() { AzureBlobUploadLock lock = - new AzureBlobUploadLock(null, null, "/test/upload/null-exec", null, null, null); + new AzureBlobUploadLock( + null, null, "/test/upload/null-exec", (ScheduledExecutorService) null, null, null); org.junit.Assert.assertNull(lock.getLeaseClient()); assertEquals("/test/upload/null-exec", lock.getUploadUri()); lock.executeRenew(); @@ -139,11 +144,198 @@ void executeRenew() { } }; - // Verify that failed renewal immediately aborts the active stream + // When the lease has already expired, renewal failure immediately aborts + lock.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); lock.renewLease(); org.junit.Assert.assertTrue(stream.isInterrupted()); org.junit.Assert.assertFalse(activeStreams.containsKey("upload-failing")); verify(mockExecutor).shutdownNow(); } + + @Test + public void testRenewLeaseTransientFailureTolerated() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new java.io.ByteArrayInputStream("test".getBytes())); + activeStreams.put("upload-transient", new java.lang.ref.WeakReference<>(stream)); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, + null, + "/test/upload/upload-transient", + mockExecutor, + "upload-transient", + activeStreams) { + @Override + void executeRenew() { + throw new RuntimeException("Simulated transient network glitch"); + } + }; + + // Lease valid for another 30 seconds: transient blip must NOT abort + lock.setLeaseExpiresAt(System.currentTimeMillis() + 30000L); + lock.renewLease(); + + org.junit.Assert.assertFalse(stream.isInterrupted()); + org.junit.Assert.assertTrue(activeStreams.containsKey("upload-transient")); + org.mockito.Mockito.verify(mockExecutor, org.mockito.Mockito.never()).shutdownNow(); + } + + @Test + public void testRenewLeaseLostInterruptsActiveStream() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new java.io.ByteArrayInputStream("test".getBytes())); + activeStreams.put("upload-lost", new java.lang.ref.WeakReference<>(stream)); + + HttpResponse response = mock(HttpResponse.class); + org.mockito.Mockito.when(response.getStatusCode()).thenReturn(409); + HttpHeaders headers = new HttpHeaders(); + headers.set("x-ms-error-code", BlobErrorCode.LEASE_NOT_PRESENT_WITH_LEASE_OPERATION.toString()); + org.mockito.Mockito.when(response.getHeaders()).thenReturn(headers); + + BlobStorageException leaseLostException = + new BlobStorageException( + "Lease lost", response, BlobErrorCode.LEASE_NOT_PRESENT_WITH_LEASE_OPERATION); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, null, "/test/upload/upload-lost", mockExecutor, "upload-lost", activeStreams) { + @Override + void executeRenew() { + throw leaseLostException; + } + }; + + // Even if lease was theoretically valid, leaseLost aborts immediately + lock.setLeaseExpiresAt(System.currentTimeMillis() + 30000L); + lock.renewLease(); + + org.junit.Assert.assertTrue(stream.isInterrupted()); + org.junit.Assert.assertFalse(activeStreams.containsKey("upload-lost")); + verify(mockExecutor).shutdownNow(); + } + + @Test + public void testRenewLeaseConflictErrorInterruptsStream() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new java.io.ByteArrayInputStream("test".getBytes())); + activeStreams.put("upload-conflict", new java.lang.ref.WeakReference<>(stream)); + + HttpResponse response = mock(HttpResponse.class); + org.mockito.Mockito.when(response.getStatusCode()).thenReturn(409); + HttpHeaders headers = new HttpHeaders(); + headers.set("x-ms-error-code", "Conflict"); + org.mockito.Mockito.when(response.getHeaders()).thenReturn(headers); + + BlobStorageException conflictException = new BlobStorageException("Conflict", response, null); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, + null, + "/test/upload/upload-conflict", + mockExecutor, + "upload-conflict", + activeStreams) { + @Override + void executeRenew() { + throw conflictException; + } + }; + + lock.setLeaseExpiresAt(System.currentTimeMillis() + 30000L); + lock.renewLease(); + + org.junit.Assert.assertTrue(stream.isInterrupted()); + org.junit.Assert.assertFalse(activeStreams.containsKey("upload-conflict")); + verify(mockExecutor).shutdownNow(); + } + + @Test + public void testRenewLeaseLostWithoutActiveStreamHandlesCleanly() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + + // activeStreams and uploadId both null + AzureBlobUploadLock lockWithoutStreams = + new AzureBlobUploadLock(null, null, "/test/upload/upload-null", mockExecutor, null, null) { + @Override + void executeRenew() { + throw new RuntimeException("Renewal failed"); + } + }; + + lockWithoutStreams.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); + lockWithoutStreams.renewLease(); + verify(mockExecutor).shutdownNow(); + + // activeStreams present but no stream registered for uploadId + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + ScheduledExecutorService mockExecutor2 = mock(ScheduledExecutorService.class); + + AzureBlobUploadLock lockWithEmptyStreams = + new AzureBlobUploadLock( + null, null, "/test/upload/upload-empty", mockExecutor2, "upload-empty", activeStreams) { + @Override + void executeRenew() { + throw new RuntimeException("Renewal failed"); + } + }; + + lockWithEmptyStreams.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); + lockWithEmptyStreams.renewLease(); + verify(mockExecutor2).shutdownNow(); + } + + @Test + public void testReleaseRemovesActiveLockAndSkipsReleaseWhenBlobDeleted() { + BlobLeaseClient realLease = + new BlobLeaseClientBuilder().blobClient(lockBlob).leaseId("mock-lease-123").buildClient(); + java.util.Map activeLocks = + new java.util.concurrent.ConcurrentHashMap<>(); + + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + realLease, + lockBlob, + "/test/upload/lock-deleted-test", + mockExecutor, + "lock-deleted-test", + null, + activeLocks); + + activeLocks.put("lock-deleted-test", lock); + org.junit.Assert.assertFalse(lock.isBlobDeleted()); + + lock.markBlobDeleted(); + org.junit.Assert.assertTrue(lock.isBlobDeleted()); + + // Release skips leaseClient.releaseLease() because blob is marked deleted, + // so no network call is attempted against the unmocked endpoint. + lock.release(); + + // Verify activeLocks is cleared + org.junit.Assert.assertFalse(activeLocks.containsKey("lock-deleted-test")); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java index d11f2a7f..6102383d 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java @@ -69,6 +69,9 @@ public void testParseErrorResponseApiNotImplemented() { public void testParseErrorResponseAccessDenied() { BlobStorageException ex = createException(403, BlobErrorCode.AUTHORIZATION_FAILURE); assertEquals(AzureErrorType.ACCESS_DENIED, AzureUtils.parseErrorResponse(ex)); + + BlobStorageException ex401 = createException(401, null); + assertEquals(AzureErrorType.ACCESS_DENIED, AzureUtils.parseErrorResponse(ex401)); } @Test @@ -90,4 +93,12 @@ public void testParseErrorResponseErrorCodeStringMatching() { BlobStorageException exLeaseMissing = createException(409, BlobErrorCode.LEASE_ID_MISSING); assertEquals(AzureErrorType.LEASE_NOT_PRESENT, AzureUtils.parseErrorResponse(exLeaseMissing)); } + + @Test + public void testParseErrorResponseErrorCodeThrowsExceptionHandled() { + HttpResponse response = mock(HttpResponse.class); + when(response.getHeaders()).thenThrow(new RuntimeException("Simulated error code failure")); + BlobStorageException ex = new BlobStorageException("Test exception", response, null); + assertEquals(AzureErrorType.UNKNOWN, AzureUtils.parseErrorResponse(ex)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java index d85d88d6..fc4c7f36 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java @@ -203,4 +203,126 @@ public void lockUploadByUriShouldThrowIOExceptionOnStorageException() throws Exc service.lockUploadByUri("/test/upload/12345"); } + + @Test + public void testCreateAndDeleteStopSignalBlob() { + lockingService.createStopSignalBlob("test-stop-id"); + com.azure.storage.blob.BlobClient stopBlob = + containerClient.getBlobClient("locks/test-stop-id.stop"); + assertTrue(Boolean.TRUE.equals(stopBlob.exists())); + + lockingService.deleteStopSignalBlob("test-stop-id"); + assertFalse(Boolean.TRUE.equals(stopBlob.exists())); + } + + @Test + public void testStopSignalBlobCatchBlocksHandledGracefully() { + com.azure.storage.blob.BlobServiceClient serviceClient = containerClient.getServiceClient(); + BlobContainerClient nonExistentContainer = + serviceClient.getBlobContainerClient("non-existent-" + System.nanoTime()); + AzureBlobLockingService failingService = new AzureBlobLockingService(nonExistentContainer); + + // Verified against Azurite that non-existent container calls fail fast and are handled quietly + failingService.createStopSignalBlob("fail-stop"); + failingService.deleteStopSignalBlob("fail-stop"); + failingService.ensureLockBlobExists(nonExistentContainer.getBlobClient("fail.lock")); + // KISS: verifying methods execute cleanly without throwing uncaught exceptions + assertTrue(true); + } + + @Test + public void testCleanupLockDeletesLockAndStopBlobs() throws Exception { + UploadId id = new UploadId(111222L); + UploadLock lock = lockingService.lockUploadByUri("/test/upload/111222"); + assertNotNull(lock); + lockingService.createStopSignalBlob("111222"); + lock.release(); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/111222.lock"); + com.azure.storage.blob.BlobClient stopBlob = containerClient.getBlobClient("locks/111222.stop"); + + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + assertTrue(Boolean.TRUE.equals(stopBlob.exists())); + + lockingService.cleanupLock(id); + + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + assertFalse(Boolean.TRUE.equals(stopBlob.exists())); + } + + @Test + public void testCleanupLockByUri() throws Exception { + UploadLock lock = lockingService.lockUploadByUri("/test/upload/333444"); + assertNotNull(lock); + lock.release(); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/333444.lock"); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + lockingService.cleanupLock("/test/upload/333444"); + + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + } + + @Test + public void testCleanupLockActivelyLeasedBlobByExternalClientIsPreserved() throws Exception { + UploadId id = new UploadId(555666L); + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/555666.lock"); + lockBlob.upload(new ByteArrayInputStream(new byte[0]), 0); + + // External lease client simulates another replica / pod holding the lease + com.azure.storage.blob.specialized.BlobLeaseClient externalLease = + new com.azure.storage.blob.specialized.BlobLeaseClientBuilder() + .blobClient(lockBlob) + .buildClient(); + externalLease.acquireLease(30); + + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + // When actively leased by another node, cleanupLock must NOT delete the lock blob + lockingService.cleanupLock(id); + assertTrue( + "Actively leased blob by another client must not be deleted", + Boolean.TRUE.equals(lockBlob.exists())); + + // After the other client releases the lease, cleanupLock should delete it + externalLease.releaseLease(); + lockingService.cleanupLock(id); + assertFalse("Unleased blob should be deleted", Boolean.TRUE.equals(lockBlob.exists())); + } + + @Test + public void testCleanupLockActivelyLeasedBlobByCurrentServiceIsDeleted() throws Exception { + UploadId id = new UploadId(666777L); + UploadLock lock = lockingService.lockUploadByUri("/test/upload/666777"); + assertNotNull(lock); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/666777.lock"); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + // When actively leased by the current service (e.g. during DELETE termination), + // cleanupLock deletes the blob using the active lease ID + lockingService.cleanupLock(id); + assertFalse( + "Actively leased blob by current service should be deleted", + Boolean.TRUE.equals(lockBlob.exists())); + + // Release after cleanup should execute cleanly without error + lock.release(); + } + + @Test + public void testCleanupStaleLocksPreservesFreshBlobs() throws Exception { + UploadLock lock = lockingService.lockUploadByUri("/test/upload/777888"); + assertNotNull(lock); + lock.release(); + + com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/777888.lock"); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + lockingService.cleanupStaleLocks(); + + // Fresh blob (created just now, < 120s old) must NOT be deleted by cleanupStaleLocks + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 6f2bbc67..4a99704e 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -73,6 +73,25 @@ public void minAppendSizeNotMetShouldThrow() throws Exception { storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); } + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + */ + @Test + public void completingUploadBypassesMinAppendSize() throws Exception { + storageService.setMinAppendSize(100L); + + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + UploadInfo completed = + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertNotNull(completed); + assertEquals(Long.valueOf(10L), completed.getOffset()); + assertFalse(completed.isUploadInProgress()); + } + @Test(expected = MaxAppendSizeExceededException.class) public void appendExceedsMaxAppendSizeShouldThrow() throws Exception { storageService.setMaxAppendSize(5L); @@ -239,6 +258,40 @@ public void terminateUploadShouldDeleteBlobs() throws Exception { assertNull(storageService.getUploadInfo(created.getId())); } + @Test + public void terminateUploadShouldHandleLeasedAndUnleasedLockBlobs() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(10L); + UploadInfo created = storageService.create(info, "owner1"); + + storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + assertNotNull(storageService.getUploadInfo(created.getId())); + + // 1. Verify unleased lock blob deletion: create an unleased lock blob and verify it is removed + BlobClient lockBlob = containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + storageService.terminateUpload(created); + assertNull(storageService.getUploadInfo(created.getId())); + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + + // 2. Verify leased lock blob: create upload, hold active lease, and verify terminateUpload + // skips deletion so Azure HTTP 412 (LeaseIdMissing) error is never triggered or logged + UploadInfo info2 = new UploadInfo(); + info2.setLength(10L); + UploadInfo created2 = storageService.create(info2, "owner1"); + + AzureBlobLockingService lockingService = new AzureBlobLockingService(containerClient); + UploadLock lock = lockingService.lockUploadByUri("/test/upload/" + created2.getId()); + try { + storageService.terminateUpload(created2); + assertNull(storageService.getUploadInfo(created2.getId())); + } finally { + lock.release(); + } + } + @Test public void removeLastNumberOfBytesPartBlobOnly() throws Exception { storageService.setPreferredBlockSize(4L * 1024 * 1024); @@ -320,12 +373,17 @@ public void cleanupExpiredUploadsShouldDeleteExpired() throws Exception { info.setLength(20L); // In-progress upload: 10 bytes uploaded out of 20 UploadInfo created = storageService.create(info, "owner1"); - storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); + // Create lock blob for this upload to verify lock is cleaned up too + com.azure.storage.blob.BlobClient lockBlob = + containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(com.azure.core.util.BinaryData.fromString(""), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); Thread.sleep(50L); // Cleanup with expiration period 0 (all uploads expired) storageService.cleanupExpiredUploads(new AzureBlobLockingService(containerClient)); assertNull(storageService.getUploadInfo(created.getId())); + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); } @Test @@ -845,4 +903,51 @@ public void testAppendRecoversPartialBlocksOnMaxAppendSizeExceeded() throws Exce UploadInfo reloaded = storageService.getUploadInfo(created.getId()); assertNotNull(reloaded); } + + @Test + public void testAppendTruncatesCommittedBlocksWhenExceedingAuthoritativeMetadata() + throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(500L); + UploadInfo created = storageService.create(info, "owner1"); + + // Upload first 100 bytes + storageService.append(created, new ByteArrayInputStream(new byte[100])); + // Upload second 100 bytes + storageService.append(created, new ByteArrayInputStream(new byte[100])); + + UploadInfo reloaded = storageService.getUploadInfo(created.getId()); + assertNotNull(reloaded.getUploadPartKeys()); + assertEquals(2, reloaded.getUploadPartKeys().size()); + assertEquals(Long.valueOf(200L), reloaded.getOffset()); + + // Simulate scenario where authoritative metadata records only 1 committed block + reloaded.getUploadPartKeys().remove(1); + reloaded.setOffset(100L); + storageService.update(reloaded); + + // Now call append again: blockIds in Azure blob (2) > uploadPartKeys in metadata (1) + // This exercises reconciliation logic where committed blocks are aligned with authoritative + // metadata + UploadInfo appended = storageService.append(reloaded, new ByteArrayInputStream(new byte[50])); + + assertEquals(Long.valueOf(150L), appended.getOffset()); + assertEquals(2, appended.getUploadPartKeys().size()); + } + + @Test + public void testTerminateUploadWithUnleasedLockBlob() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(100L); + UploadInfo created = storageService.create(info, "owner1"); + + // Create an unleased lock blob to verify safe cleanup of unleased lock blobs + BlobClient lockBlob = containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(BinaryData.fromString("lock-content"), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + storageService.terminateUpload(created); + + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java index 6a9f9d52..7be2e903 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobTusFileUploadService.java @@ -1,12 +1,19 @@ package me.desair.tus.server.upload.azure; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + import com.azure.storage.blob.BlobContainerClient; +import jakarta.servlet.http.HttpServletResponse; import me.desair.tus.server.AbstractITTusFileUploadService; +import me.desair.tus.server.HttpHeader; import me.desair.tus.server.ProtocolVersion; import me.desair.tus.server.TestUtils; import me.desair.tus.server.TusFileUploadService; +import org.apache.commons.lang3.StringUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Test; import org.testcontainers.containers.GenericContainer; /** @@ -65,4 +72,101 @@ protected TusFileUploadService createTusFileUploadService(String uploadUri) { .withDownloadFeature() .withChunkedTransferDecoding(true); } + + /** + * Verifies that when an upload completes via PATCH (offset reaching total length), the lock blob + * in the locks/ directory is automatically cleaned up and deleted. + */ + @Test + public void testUploadCompletionCleansUpLockBlob() throws Exception { + String uploadContent = "Testing Azure lock blob cleanup on upload completion!"; + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, uploadContent.getBytes().length); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_CREATED, servletResponse.getStatus()); + + String location = + UPLOAD_URI + + StringUtils.substringAfter( + servletResponse.getHeader(HttpHeader.LOCATION), UPLOAD_URI); + String uploadIdStr = StringUtils.substringAfterLast(location, "/"); + + // Complete the upload via PATCH + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(location); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, uploadContent.getBytes().length); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, 0); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(uploadContent.getBytes()); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_NO_CONTENT, servletResponse.getStatus()); + + // Verify lock blob is cleaned up on completion + assertFalse( + "Lock blob should be removed upon upload completion", + containerClient.getBlobClient("locks/" + uploadIdStr + ".lock").exists()); + assertFalse( + "Stop signal blob should not exist", + containerClient.getBlobClient("locks/" + uploadIdStr + ".stop").exists()); + } + + /** + * Verifies that when an in-progress upload is terminated via DELETE, the lock blob in the locks/ + * directory is automatically cleaned up and deleted. + */ + @Test + public void testDeleteTerminationCleansUpLockBlob() throws Exception { + String uploadContent = "Incomplete upload to terminate"; + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, uploadContent.getBytes().length * 2); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_CREATED, servletResponse.getStatus()); + + String location = + UPLOAD_URI + + StringUtils.substringAfter( + servletResponse.getHeader(HttpHeader.LOCATION), UPLOAD_URI); + String uploadIdStr = StringUtils.substringAfterLast(location, "/"); + + // Partially upload via PATCH + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(location); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.CONTENT_LENGTH, uploadContent.getBytes().length); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, 0); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(uploadContent.getBytes()); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_NO_CONTENT, servletResponse.getStatus()); + + // Delete the upload + reset(); + servletRequest.setMethod("DELETE"); + servletRequest.setRequestURI(location); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertEquals(HttpServletResponse.SC_NO_CONTENT, servletResponse.getStatus()); + + // Verify lock blob is cleaned up on termination + assertFalse( + "Lock blob should be removed upon DELETE termination", + containerClient.getBlobClient("locks/" + uploadIdStr + ".lock").exists()); + assertFalse( + "Stop signal blob should not exist", + containerClient.getBlobClient("locks/" + uploadIdStr + ".stop").exists()); + } } diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index becc861d..fc5a99d4 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Duration; import java.util.UUID; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.upload.UploadId; @@ -161,6 +162,36 @@ public void testDelegateMethods() throws Exception { assertEquals(0, service.getMaxUploadSize()); verify(mockStorage, times(1)).getMaxUploadSize(); + service.setMaxAppendSize(500L); + verify(mockStorage, times(1)).setMaxAppendSize(500L); + + service.getMaxAppendSize(); + verify(mockStorage, times(1)).getMaxAppendSize(); + + service.setMinAppendSize(200L); + verify(mockStorage, times(1)).setMinAppendSize(200L); + + service.getMinAppendSize(); + verify(mockStorage, times(1)).getMinAppendSize(); + + service.setMinSize(300L); + verify(mockStorage, times(1)).setMinSize(300L); + + service.getMinSize(); + verify(mockStorage, times(1)).getMinSize(); + + service.setCloudUploadThreadPoolSize(12); + verify(mockStorage, times(1)).setCloudUploadThreadPoolSize(12); + + service.getCloudUploadThreadPoolSize(); + verify(mockStorage, times(1)).getCloudUploadThreadPoolSize(); + + service.setDrainTimeout(Duration.ofSeconds(25)); + verify(mockStorage, times(1)).setDrainTimeout(Duration.ofSeconds(25)); + + service.getDrainTimeout(); + verify(mockStorage, times(1)).getDrainTimeout(); + when(mockStorage.create(info, "owner")).thenReturn(info); assertEquals(info, service.create(info, "owner")); verify(mockStorage, times(1)).create(info, "owner"); @@ -221,6 +252,12 @@ public void testDelegateMethods() throws Exception { service.requestLockRelease("/files/1"); verify(mockLocking, times(1)).requestLockRelease("/files/1"); + service.cleanupLock(id); + verify(mockLocking, times(1)).cleanupLock(id); + + service.cleanupLock("/files/1"); + verify(mockLocking, times(1)).cleanupLock("/files/1"); + service.close(); verify(mockLocking, times(1)).close(); } diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java index 39464c3e..b9eee747 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java @@ -685,4 +685,62 @@ public void testCloseInterruptsActiveStreams() throws Exception { lockingService.close(); assertTrue(iis.isInterrupted()); } + + @Test + public void testCleanupLockNullAndInvalid() throws Exception { + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testCleanupLockDeletesLockAndStopFiles() throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + Path lockPath = storagePath.resolve("locks").resolve(id.toString()); + Path stopPath = storagePath.resolve("locks").resolve(id.toString() + ".stop"); + Files.createFile(lockPath); + Files.createFile(stopPath); + + assertTrue(Files.exists(lockPath)); + assertTrue(Files.exists(stopPath)); + + lockingService.cleanupLock(id); + + assertFalse(Files.exists(lockPath)); + assertFalse(Files.exists(stopPath)); + } + + @Test + public void testCleanupLockByUri() throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + when(idFactory.readUploadId(UPLOAD_URL + "/" + id)).thenReturn(id); + + Path lockPath = storagePath.resolve("locks").resolve(id.toString()); + Files.createFile(lockPath); + assertTrue(Files.exists(lockPath)); + + lockingService.cleanupLock(UPLOAD_URL + "/" + id); + + assertFalse(Files.exists(lockPath)); + } + + @Test + public void testCleanupLockWhenActivelyLockedDoesNotDeleteLockFile() throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + when(idFactory.readUploadId(UPLOAD_URL + "/" + id)).thenReturn(id); + + UploadLock lock = lockingService.lockUploadByUri(UPLOAD_URL + "/" + id); + assertThat(lock, is(not(nullValue()))); + + Path lockPath = storagePath.resolve("locks").resolve(id.toString()); + assertTrue(Files.exists(lockPath)); + + // When actively locked, cleanupLock must preserve the lock file + lockingService.cleanupLock(id); + assertTrue(Files.exists(lockPath)); + + lock.release(); + assertFalse(Files.exists(lockPath)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index cd08415d..bbfc4d32 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -537,6 +538,7 @@ public void cleanupExpiredUploads() throws Exception { assertFalse(Files.exists(getUploadInfoPath(info.getId()))); assertFalse(Files.exists(getStoragePath(info.getId()))); + org.mockito.Mockito.verify(uploadLockingService).cleanupLock(info.getId()); } @Test @@ -1106,4 +1108,32 @@ public void testUnsafeChildDataFileDeletionThrowsIllegalArgumentException() thro storageService.update(child); } + + @Test + public void testSetAndGetCloudUploadThreadPoolSize() { + assertThat(storageService.getCloudUploadThreadPoolSize(), is(10)); + storageService.setCloudUploadThreadPoolSize(24); + assertThat(storageService.getCloudUploadThreadPoolSize(), is(24)); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetCloudUploadThreadPoolSizeZeroThrowsIllegalArgumentException() { + storageService.setCloudUploadThreadPoolSize(0); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetCloudUploadThreadPoolSizeNegativeThrowsIllegalArgumentException() { + storageService.setCloudUploadThreadPoolSize(-1); + } + + @Test + public void testDefaultDrainTimeoutOnDiskStorageService() { + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(55))); + storageService.setDrainTimeout(Duration.ofSeconds(20)); + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(20))); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertThat(storageService.getDrainTimeout(), is(Duration.ofSeconds(20))); + } } diff --git a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java index 9f78d6fa..27d0c822 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java @@ -919,4 +919,73 @@ private LeaseData createExpiredLease( lease.setAcquiredAt(expiresAt - 30_000L); return lease; } + + @Test + public void testCleanupLockNullAndInvalid() throws Exception { + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testCleanupLockDeletesExpiredLockDirAndStopFile() throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + UploadId id = new UploadId(uploadIdStr); + Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); + Path stopFile = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); + Files.createDirectories(lockDir); + Files.write(stopFile, new byte[0]); + + long now = System.currentTimeMillis(); + LeaseData expiredLease = + createExpiredLease( + "old-node", UPLOAD_URL + "/" + uploadIdStr, lockDir.toString(), now - 5000L); + LeaseDataJsonSerializer.serializeToPath(expiredLease, lockDir.resolve("lease.json")); + + assertTrue(Files.exists(lockDir)); + assertTrue(Files.exists(stopFile)); + + lockingService.cleanupLock(id); + + assertFalse(Files.exists(lockDir)); + assertFalse(Files.exists(stopFile)); + } + + @Test + public void testCleanupLockByUri() throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + String uri = UPLOAD_URL + "/" + uploadIdStr; + Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); + Files.createDirectories(lockDir); + + long now = System.currentTimeMillis(); + LeaseData expiredLease = createExpiredLease("old-node", uri, lockDir.toString(), now - 5000L); + LeaseDataJsonSerializer.serializeToPath(expiredLease, lockDir.resolve("lease.json")); + assertTrue(Files.exists(lockDir)); + + lockingService.cleanupLock(uri); + + assertFalse(Files.exists(lockDir)); + } + + @Test + public void testCleanupLockPreservesActivelyHeldLease() throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + String uri = UPLOAD_URL + "/" + uploadIdStr; + UploadId id = new UploadId(uploadIdStr); + + UploadLock lock = lockingService.lockUploadByUri(uri); + assertNotNull(lock); + + Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); + assertTrue(Files.exists(lockDir)); + + // Actively held lock must not be deleted by cleanupLock + lockingService.cleanupLock(id); + assertTrue(Files.exists(lockDir)); + + lock.release(); + assertFalse(Files.exists(lockDir)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java index 58248985..02724796 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3LockingService.java @@ -17,7 +17,7 @@ public class ITS3LockingService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-locking-service-bucket"; @@ -26,27 +26,28 @@ public class ITS3LockingService { @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } @Before public void setUp() { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - lockingService = new S3LockingService(minioClient, BUCKET); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + lockingService = new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java index f281926c..97872e3c 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3RufhProtocol.java @@ -14,27 +14,27 @@ */ public class ITS3RufhProtocol extends AbstractITRufhProtocol { - private static org.testcontainers.containers.GenericContainer minio; + private static org.testcontainers.containers.GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-rufh-s3-bucket"; @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } @@ -47,9 +47,13 @@ protected TusFileUploadService createTusFileUploadService() { protected TusFileUploadService createTusFileUploadService(String uploadUri) { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - S3StorageService s3Storage = new S3StorageService(minioClient, BUCKET); - S3LockingService s3Locking = new S3LockingService(minioClient, BUCKET); - S3ConcatenationService s3Concat = new S3ConcatenationService(minioClient, BUCKET, s3Storage); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + S3StorageService s3Storage = + new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3LockingService s3Locking = + new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3ConcatenationService s3Concat = + new S3ConcatenationService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET, s3Storage); s3Storage.setUploadConcatenationService(s3Concat); return new TusFileUploadService() diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java index 544147a0..b230dca9 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java @@ -21,7 +21,7 @@ public class ITS3StorageService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-storage-service-bucket"; @@ -30,27 +30,28 @@ public class ITS3StorageService { @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } @Before public void setUp() { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - storageService = new S3StorageService(minioClient, BUCKET); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + storageService = new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java index af3c0436..6d8da536 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3TusFileUploadService.java @@ -16,27 +16,27 @@ */ public class ITS3TusFileUploadService extends AbstractITTusFileUploadService { - private static GenericContainer minio; + private static GenericContainer rustfsContainer; private static MinioClient minioClient; private static final String BUCKET = "test-service-s3-bucket"; @BeforeClass public static void setUpClass() { org.junit.Assume.assumeTrue( - "Container runtime is not available; skipping Testcontainers MinIO test", + "Container runtime is not available; skipping Testcontainers S3 (RustFS) test", TestUtils.isContainerRuntimeAvailable()); - minio = TestUtils.createMinioContainer(); - minio.start(); + rustfsContainer = TestUtils.createRustFsContainer(); + rustfsContainer.start(); - minioClient = TestUtils.createMinioClient(minio); + minioClient = TestUtils.createMinioClient(rustfsContainer); TestUtils.createBucket(minioClient, BUCKET); } @AfterClass public static void tearDownClass() { - if (minio != null) { - minio.stop(); + if (rustfsContainer != null) { + rustfsContainer.stop(); } } @@ -49,9 +49,13 @@ protected TusFileUploadService createTusFileUploadService() { protected TusFileUploadService createTusFileUploadService(String uploadUri) { org.junit.Assume.assumeTrue(TestUtils.isContainerRuntimeAvailable()); - S3StorageService s3Storage = new S3StorageService(minioClient, BUCKET); - S3LockingService s3Locking = new S3LockingService(minioClient, BUCKET); - S3ConcatenationService s3Concat = new S3ConcatenationService(minioClient, BUCKET, s3Storage); + String endpoint = TestUtils.getS3Endpoint(rustfsContainer); + S3StorageService s3Storage = + new S3StorageService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3LockingService s3Locking = + new S3LockingService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET); + S3ConcatenationService s3Concat = + new S3ConcatenationService(endpoint, "rustfsadmin", "rustfsadmin", BUCKET, s3Storage); s3Storage.setUploadConcatenationService(s3Concat); return new TusFileUploadService() diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index f9970131..4d518f5e 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -39,18 +39,63 @@ public void setUp() { "test-bucket", "uploads/", storageService, - Paths.get(System.getProperty("java.io.tmpdir"))); + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); } @Test - public void testConstructorsAndSetters() { - S3ConcatenationService service1 = new S3ConcatenationService(minioClient, "test-bucket"); - S3ConcatenationService service2 = - new S3ConcatenationService(minioClient, "test-bucket", storageService); - service1.setUploadStorageService(storageService); - - assertNotNull(service1); - assertNotNull(service2); + public void testConstructors() { + S3ConcatenationService service = + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); + assertNotNull(service); + + // Public connection parameter constructors without region (defaults to "local") + S3ConcatenationService serviceParamsNoRegionWithStorage = + new S3ConcatenationService( + "https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket", storageService); + assertNotNull(serviceParamsNoRegionWithStorage); + + // Public connection parameter constructors with region + S3ConcatenationService serviceParamsWithRegion = + new S3ConcatenationService( + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket", + storageService); + assertNotNull(serviceParamsWithRegion); + + S3ConcatenationService serviceParamsFull = + new S3ConcatenationService( + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L); + assertNotNull(serviceParamsFull); + + S3ConcatenationService defaultRegionService = + new S3ConcatenationService( + "https://s3.amazonaws.com", + null, + "accessKey", + "secretKey", + "test-bucket", + storageService); + assertNotNull(defaultRegionService); } @Test @@ -263,8 +308,8 @@ public void testMergeMultiplePartsWithSub5MbNonFinalPartFallsBackToStreaming() t Mockito.verify(minioClient).putObject(Mockito.any(PutObjectArgs.class)); } - @Test(expected = IOException.class) - public void testMergeServerSideCopyFails() throws Exception { + @Test + public void testMergeServerSideCopyFailsFallsBackToStreaming() throws Exception { UploadInfo p1 = new UploadInfo(); p1.setId(new UploadId("part-1")); p1.setOwnerKey("owner-1"); @@ -273,6 +318,8 @@ public void testMergeServerSideCopyFails() throws Exception { p1.setStorageUploadId("uploads/part-1"); Mockito.when(storageService.getUploadInfo("/part-1", "owner-1")).thenReturn(p1); + Mockito.when(storageService.getUploadedBytes(new UploadId("part-1"))) + .thenReturn(new ByteArrayInputStream(new byte[10])); Mockito.when(minioClient.composeObject(Mockito.any(ComposeObjectArgs.class))) .thenThrow(new RuntimeException("Compose error")); @@ -282,6 +329,10 @@ public void testMergeServerSideCopyFails() throws Exception { finalUpload.setConcatenationPartIds(Arrays.asList("/part-1")); concatenationService.merge(finalUpload); + + // Verify fallback to streaming putObject + Mockito.verify(minioClient).putObject(Mockito.any(PutObjectArgs.class)); + assertEquals(Long.valueOf(10L * 1024 * 1024), finalUpload.getLength()); } @Test(expected = IOException.class) @@ -348,14 +399,16 @@ public void testGetConcatenatedBytesTriggersMergeWhenStorageUploadIdIsNull() thr assertNotNull(result); } - @Test(expected = IOException.class) - public void testGetConcatenatedBytesWithoutStorageService() throws Exception { - S3ConcatenationService standalone = new S3ConcatenationService(minioClient, "test-bucket"); - UploadInfo info = new UploadInfo(); - info.setId(new UploadId("concat-1")); - info.setStorageUploadId("uploads/concat-1"); - - standalone.getConcatenatedBytes(info); + @Test(expected = NullPointerException.class) + public void testConstructorNullStorageServiceThrowsException() { + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + null, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); } @Test @@ -490,4 +543,11 @@ public void testMergeBothNullOwnerKeys() throws Exception { assertEquals(Long.valueOf(10L * 1024 * 1024), parent.getLength()); assertEquals(Long.valueOf(10L * 1024 * 1024), parent.getOffset()); } + + @Test + public void testSetS3ServerSideComposeHelper() { + S3ServerSideComposeHelper mockHelper = Mockito.mock(S3ServerSideComposeHelper.class); + concatenationService.setS3ServerSideComposeHelper(mockHelper); + assertNotNull(concatenationService); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index 368d2684..e38b4dcb 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -4,7 +4,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import io.minio.GetObjectArgs; import io.minio.GetObjectResponse; @@ -28,6 +30,7 @@ import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadLock; +import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.InterruptibleInputStream; import org.junit.Before; import org.junit.Test; @@ -42,7 +45,14 @@ public class S3LockingServiceTest { @Before public void setUp() throws Exception { minioClient = Mockito.mock(MinioClient.class); - lockingService = new S3LockingService(minioClient, "test-bucket"); + lockingService = + new S3LockingService( + minioClient, + "test-bucket", + S3LockingService.DEFAULT_LOCKS_PREFIX, + S3LockingService.DEFAULT_LEASE_DURATION_MS, + S3LockingService.DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); s3StorageMap.clear(); Mockito.when(minioClient.putObject(Mockito.any(PutObjectArgs.class))) @@ -90,6 +100,26 @@ public void setUp() throws Exception { }) .when(minioClient) .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); + + Mockito.doAnswer( + invocation -> { + io.minio.RemoveObjectsArgs args = invocation.getArgument(0); + if (args != null && args.objects() != null) { + for (io.minio.messages.DeleteRequest.Object obj : args.objects()) { + try { + java.lang.reflect.Field field = + io.minio.messages.DeleteRequest.Object.class.getDeclaredField("name"); + field.setAccessible(true); + String name = (String) field.get(obj); + s3StorageMap.remove(name); + } catch (Exception ignored) { + } + } + } + return java.util.Collections.emptyList(); + }) + .when(minioClient) + .removeObjects(Mockito.any(io.minio.RemoveObjectsArgs.class)); } @Test @@ -114,6 +144,36 @@ public void testLockUploadByUriInvalidUri() throws Exception { assertNull(lock); } + @Test + public void testExplicitConnectionParametersConstructors() { + S3LockingService serviceWithoutRegion = + new S3LockingService("https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithoutRegion); + + S3LockingService serviceWithParams = + new S3LockingService( + "https://s3.amazonaws.com", "eu-central-1", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithParams); + + S3LockingService fullServiceWithParams = + new S3LockingService( + "https://s3.amazonaws.com", + "eu-central-1", + "accessKey", + "secretKey", + "test-bucket", + "locks/", + 30000L, + 2000L, + new me.desair.tus.server.upload.UuidUploadIdFactory()); + assertNotNull(fullServiceWithParams); + + S3LockingService defaultRegionService = + new S3LockingService( + "https://s3.amazonaws.com", null, "accessKey", "secretKey", "test-bucket"); + assertNotNull(defaultRegionService); + } + @Test public void testIsLocked() throws Exception { assertFalse(lockingService.isLocked((UploadId) null)); @@ -313,11 +373,12 @@ public void testInterruptStreamStandardStreamCloseException() throws Exception { @Test public void testSanitizePrefixNullOrEmpty() throws Exception { S3LockingService serviceWithEmptyPrefix = - new S3LockingService(minioClient, "test-bucket", "", 30000L, 0L); + new S3LockingService(minioClient, "test-bucket", "", 30000L, 0L, new UuidUploadIdFactory()); assertNotNull(serviceWithEmptyPrefix); S3LockingService serviceWithNullPrefix = - new S3LockingService(minioClient, "test-bucket", null, 30000L, 0L); + new S3LockingService( + minioClient, "test-bucket", null, 30000L, 0L, new UuidUploadIdFactory()); assertNotNull(serviceWithNullPrefix); } @@ -326,6 +387,16 @@ public void testClose() throws Exception { lockingService.close(); } + private S3LockingService createLockingService(MinioClient client, String bucket) { + return new S3LockingService( + client, + bucket, + S3LockingService.DEFAULT_LOCKS_PREFIX, + S3LockingService.DEFAULT_LEASE_DURATION_MS, + S3LockingService.DEFAULT_POLL_INTERVAL_MS, + new UuidUploadIdFactory()); + } + @Test public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { lockingService.setIdFactory(new me.desair.tus.server.upload.TimeBasedUploadIdFactory()); @@ -336,7 +407,7 @@ public void testCheckStopSignalForEntryExceptionAndNullId() throws Exception { .when(mockClient) .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); @@ -356,12 +427,12 @@ public void testCheckStopSignalForEntryHappyPath() throws Exception { io.minio.StatObjectResponse mockStat = Mockito.mock(io.minio.StatObjectResponse.class); Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))).thenReturn(mockStat); - S3LockingService service = - new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); - service.setIdFactory(idFactory); + + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L, idFactory); ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); InterruptibleInputStream stream = new InterruptibleInputStream(bais); @@ -391,12 +462,12 @@ public void testCheckStopSignalForEntryNoSuchKey() throws Exception { Mockito.when(mockClient.statObject(Mockito.any(StatObjectArgs.class))) .thenThrow(noSuchKeyException); - S3LockingService service = - new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); - service.setIdFactory(idFactory); + + S3LockingService service = + new S3LockingService(mockClient, "test-bucket", "locks", 30000L, 50L, idFactory); ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); InterruptibleInputStream stream = new InterruptibleInputStream(bais); @@ -470,7 +541,7 @@ public void testCleanupStaleLocksWithExpiredAndNonExpiredLocks() throws Exceptio .thenReturn(expiredStream) .thenReturn(validStream); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); service.cleanupStaleLocks(); // Expired lock object is deleted @@ -489,7 +560,7 @@ public void testWriteStopSignalException() throws Exception { Mockito.when(mockClient.putObject(Mockito.any(PutObjectArgs.class))) .thenThrow(new RuntimeException("S3 Put error")); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); @@ -502,7 +573,7 @@ public void testWriteStopSignalException() throws Exception { @Test public void testCloseInterruptsActiveStreams() throws Exception { MinioClient mockClient = Mockito.mock(MinioClient.class); - S3LockingService service = new S3LockingService(mockClient, "test-bucket"); + S3LockingService service = createLockingService(mockClient, "test-bucket"); ByteArrayInputStream bis = new ByteArrayInputStream(new byte[] {1, 2, 3}); InterruptibleInputStream iis = new InterruptibleInputStream(bis); @@ -558,30 +629,17 @@ public void testLockUploadByUriWithContentionOnPostPutVerificationThrows() throw "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.stop"); String rivalJson = me.desair.tus.server.util.LeaseDataJsonSerializer.serialize(rivalLock); - // Initial check sees expired/missing, but post-put verification sees rival holder - java.util.concurrent.atomic.AtomicInteger getCallCount = - new java.util.concurrent.atomic.AtomicInteger(0); + // PutObject succeeds, but post-put read-after-write verification sees rival holder Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))) - .thenAnswer( - inv -> { - if (getCallCount.incrementAndGet() == 1) { - // First call: check if expired (missing -> not locked) - ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); - Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); - throw new io.minio.errors.ErrorResponseException(errorResponse, null, null); - } - // Second call: read-after-write verification sees rivalLock - return new GetObjectResponse( - null, - "test-bucket", - "eu-central-1", - "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", - new ByteArrayInputStream(rivalJson.getBytes(StandardCharsets.UTF_8))); - }); + .thenReturn( + new GetObjectResponse( + null, + "test-bucket", + "eu-central-1", + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + new ByteArrayInputStream(rivalJson.getBytes(StandardCharsets.UTF_8)))); - UploadLock lock = - lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); - assertNull(lock); + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); } @Test @@ -855,4 +913,226 @@ public void testEvictExpiredLockHandlesRemoveObjectException() throws Exception boolean evicted = lockingService.evictExpiredLock(uploadId); assertFalse(evicted); } + + @Test + public void testEvictExpiredLockHandlesErrorResponseExceptionNonNoSuchKey() throws Exception { + UploadId uploadId = new UploadId("test-access-denied"); + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("AccessDenied"); + ErrorResponseException accessDeniedEx = + new ErrorResponseException(errorResponse, null, "AccessDenied"); + + Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))).thenThrow(accessDeniedEx); + + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertFalse(evicted); + } + + @Test + public void testEvictExpiredLockHandlesNoSuchKeyReturnsTrue() throws Exception { + UploadId uploadId = new UploadId("test-no-such-key"); + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); + ErrorResponseException noSuchKeyEx = + new ErrorResponseException(errorResponse, null, "NoSuchKey"); + + Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))).thenThrow(noSuchKeyEx); + + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertTrue(evicted); + } + + @Test + public void testWithJitterConfiguresBoundsAndReturnsSelf() { + S3LockingService returned = lockingService.withJitter(15L, 75L); + assertSame(lockingService, returned); + assertEquals(15L, lockingService.getJitterMinMs()); + assertEquals(75L, lockingService.getJitterMaxMs()); + } + + @Test + public void testLockAcquisitionWithJitterDisabled() throws Exception { + lockingService.withJitter(0L, 0L); + assertEquals(0L, lockingService.getJitterMinMs()); + assertEquals(0L, lockingService.getJitterMaxMs()); + + UploadLock lock = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock); + lock.close(); + } + + @Test + public void testWithS3ConditionalWritesSupported() { + assertTrue(lockingService.isS3ConditionalWritesSupported()); + S3LockingService returned = lockingService.withS3ConditionalWritesSupported(false); + assertSame(lockingService, returned); + assertFalse(lockingService.isS3ConditionalWritesSupported()); + } + + @Test + public void testConditionalWriteNotImplementedDowngradesToNonCasModeAndSucceeds() + throws Exception { + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NotImplemented"); + ErrorResponseException notImplementedEx = + new ErrorResponseException(errorResponse, null, "NotImplemented"); + + // The first PutObject with If-None-Match: * throws NotImplemented (e.g. Backblaze B2, Ceph RGW) + Mockito.doAnswer( + invocation -> { + PutObjectArgs args = invocation.getArgument(0); + if (args.extraHeaders() != null && args.extraHeaders().containsKey("If-None-Match")) { + throw notImplementedEx; + } + // Unconditional fallback write stores bytes in our in-memory map + java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int read; + java.io.InputStream is = args.stream(); + while ((read = is.read(buf)) != -1) { + baos.write(buf, 0, read); + } + s3StorageMap.put(args.object(), baos.toByteArray()); + return null; + }) + .when(minioClient) + .putObject(Mockito.any(PutObjectArgs.class)); + + assertTrue(lockingService.isS3ConditionalWritesSupported()); + + UploadLock lock = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock); + // Verified that service automatically downgraded to non-CAS mode + assertFalse(lockingService.isS3ConditionalWritesSupported()); + lock.close(); + } + + @Test(expected = UploadAlreadyLockedException.class) + public void testConditionalWriteNotImplementedFailsWhenActiveLockExists() throws Exception { + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("NotImplemented"); + ErrorResponseException notImplementedEx = + new ErrorResponseException(errorResponse, null, "NotImplemented"); + + // PutObject throws NotImplemented when attempting conditional write + Mockito.doThrow(notImplementedEx).when(minioClient).putObject(Mockito.any(PutObjectArgs.class)); + + // Existing lock in S3 is actively held by another contender + LeaseData rivalLock = + new LeaseData( + "rival-holder", + "/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e", + 30000L, + System.currentTimeMillis() + 30000L, + System.currentTimeMillis(), + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.stop"); + s3StorageMap.put( + "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", + me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(rivalLock)); + + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + } + + @Test + public void testExplicitNonCasModeChecksLockExpirationBeforeUnconditionalWrite() + throws Exception { + lockingService.withS3ConditionalWritesSupported(false); + assertFalse(lockingService.isS3ConditionalWritesSupported()); + + // First lock acquisition succeeds + UploadLock lock1 = + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + assertNotNull(lock1); + + // Second lock acquisition while first is active fails because isLockExpired returns false + try { + lockingService.lockUploadByUri("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); + fail("Expected UploadAlreadyLockedException when trying to acquire an actively locked URI"); + } catch (UploadAlreadyLockedException expected) { + assertNotNull(expected.getMessage()); + } + + lock1.close(); + } + + @Test + public void testCleanupLockNullAndInvalid() throws Exception { + lockingService.cleanupLock((UploadId) null); + lockingService.cleanupLock((String) null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testCleanupLockDeletesExpiredLockAndStopObjects() throws Exception { + UploadId id = new UploadId(UUID.randomUUID()); + String lockKey = "locks/" + id + ".lock"; + String stopKey = "locks/" + id + ".stop"; + + LeaseData expiredLock = + new LeaseData( + "old-holder", + "/files/upload/" + id, + 30000L, + System.currentTimeMillis() - 5000L, + System.currentTimeMillis() - 35000L, + lockKey, + stopKey); + s3StorageMap.put( + lockKey, me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(expiredLock)); + s3StorageMap.put(stopKey, new byte[0]); + + assertTrue(s3StorageMap.containsKey(lockKey)); + assertTrue(s3StorageMap.containsKey(stopKey)); + + lockingService.cleanupLock(id); + + assertFalse(s3StorageMap.containsKey(lockKey)); + assertFalse(s3StorageMap.containsKey(stopKey)); + } + + @Test + public void testCleanupLockByUri() throws Exception { + UploadId id = new UploadId(UUID.randomUUID()); + String uri = "/files/upload/" + id; + String lockKey = "locks/" + id + ".lock"; + + LeaseData expiredLock = + new LeaseData( + "old-holder", + uri, + 30000L, + System.currentTimeMillis() - 5000L, + System.currentTimeMillis() - 35000L, + lockKey, + null); + s3StorageMap.put( + lockKey, me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(expiredLock)); + assertTrue(s3StorageMap.containsKey(lockKey)); + + lockingService.cleanupLock(uri); + + assertFalse(s3StorageMap.containsKey(lockKey)); + } + + @Test + public void testCleanupLockPreservesActivelyHeldLock() throws Exception { + UploadId id = new UploadId(UUID.randomUUID()); + String uri = "/files/upload/" + id; + String lockKey = "locks/" + id + ".lock"; + + UploadLock lock = lockingService.lockUploadByUri(uri); + assertNotNull(lock); + assertTrue(s3StorageMap.containsKey(lockKey)); + + // Actively held unexpired lock must not be deleted by cleanupLock + lockingService.cleanupLock(id); + assertTrue(s3StorageMap.containsKey(lockKey)); + + lock.close(); + assertFalse(s3StorageMap.containsKey(lockKey)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java new file mode 100644 index 00000000..552c7b0f --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ServerSideComposeHelperTest.java @@ -0,0 +1,429 @@ +package me.desair.tus.server.upload.s3; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.minio.AbortMultipartUploadArgs; +import io.minio.CompleteMultipartUploadArgs; +import io.minio.ComposeObjectArgs; +import io.minio.CreateMultipartUploadArgs; +import io.minio.CreateMultipartUploadResponse; +import io.minio.Http; +import io.minio.MinioAsyncClient; +import io.minio.MinioClient; +import io.minio.ObjectWriteResponse; +import io.minio.credentials.Credentials; +import io.minio.credentials.Provider; +import io.minio.messages.InitiateMultipartUploadResult; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.Test; + +/** Unit tests for {@link S3ServerSideComposeHelper}. */ +public class S3ServerSideComposeHelperTest { + + @Test + public void testExtractEtagFromXml() { + // Standard XML response + String xml = + "" + + "2026-10-03T12:00:00.000Z" + + "\"1234567890abcdef\"" + + ""; + assertEquals("\"1234567890abcdef\"", S3ServerSideComposeHelper.extractEtagFromXml(xml)); + + // Entity escaped XML quotes + String escapedXml = + ""9876543210fedcba""; + assertEquals("\"9876543210fedcba\"", S3ServerSideComposeHelper.extractEtagFromXml(escapedXml)); + + // Whitespace inside tag + String whitespaceXml = " \"xyz123\" "; + assertEquals("\"xyz123\"", S3ServerSideComposeHelper.extractEtagFromXml(whitespaceXml)); + + // Null or missing tags + assertNull(S3ServerSideComposeHelper.extractEtagFromXml(null)); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + assertNull(S3ServerSideComposeHelper.extractEtagFromXml("")); + } + + @Test + public void testFallbackWhenNotAvailable() throws Exception { + MinioClient mockMinioClient = mock(MinioClient.class); + // Since mock MinioClient does not have initialized asyncClient/baseUrl/httpClient fields, + // isAvailable() will return false, triggering the composeObject() fallback. + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(mockMinioClient); + assertFalse(helper.isAvailable()); + + List partKeys = Arrays.asList("upload/part-1", "upload/part-2"); + helper.compose("my-bucket", "upload/target", partKeys); + + verify(mockMinioClient).composeObject(any(ComposeObjectArgs.class)); + } + + @Test + public void testNullMinioClient() { + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(null); + assertFalse(helper.isAvailable()); + } + + @Test + public void testNativeComposeSuccess() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", "testSessionToken", null); + when(provider.fetch()).thenReturn(creds); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.us-west-2.amazonaws.com")); + + // Prepare createMultipartUpload response + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse( + null, "my-bucket", "us-west-2", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + // Prepare completeMultipartUpload response + ObjectWriteResponse completeResponse = + new ObjectWriteResponse( + null, "my-bucket", "us-west-2", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + // Mock OkHttp execution for uploadPartCopy + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + String partXml = "\"part-etag-1\""; + ResponseBody body = ResponseBody.create(partXml, MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.us-west-2.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + // Assemble helper with reflection fields populated + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + + assertTrue(helper.isAvailable()); + + List partKeys = Collections.singletonList("upload/part-1"); + helper.compose("my-bucket", "upload/target", partKeys); + + verify(asyncClient).createMultipartUpload(any(CreateMultipartUploadArgs.class)); + verify(httpClient).newCall(any(Request.class)); + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + + @Test + public void testNativeComposeHttpErrorAborts() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", null, null); + when(provider.fetch()).thenReturn(creds); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + // Prepare createMultipartUpload response + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + when(asyncClient.abortMultipartUpload(any(AbortMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + + // Mock OkHttp returning HTTP 400 + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + ResponseBody body = + ResponseBody.create( + "Test Error", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(400) + .message("Bad Request") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + + try { + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + fail("Expected IOException to be thrown"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("HTTP 400")); + } + + verify(asyncClient).abortMultipartUpload(any(AbortMultipartUploadArgs.class)); + } + + @Test + public void testNativeComposeMissingEtagHeaderFallback() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + ObjectWriteResponse completeResponse = + new ObjectWriteResponse(null, "my-bucket", "", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + // Response body has no XML ETag tag, but response header has ETag + ResponseBody body = ResponseBody.create("", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .header("ETag", "\"header-etag\"") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + + @Test + public void testExplicitConnectionParametersConstructor() { + MinioClient realClient = + MinioClient.builder() + .endpoint("https://s3.amazonaws.com") + .credentials("testKey", "testSecret") + .region("us-east-1") + .build(); + + S3ServerSideComposeHelper helperWithoutClient = + new S3ServerSideComposeHelper( + "https://s3.amazonaws.com", "us-east-1", "testKey", "testSecret"); + assertTrue(helperWithoutClient.isAvailable()); + + S3ServerSideComposeHelper helperWithNullRegion = + new S3ServerSideComposeHelper("https://s3.amazonaws.com", null, "testKey", "testSecret"); + assertTrue(helperWithNullRegion.isAvailable()); + + S3ServerSideComposeHelper minioOnlyHelper = new S3ServerSideComposeHelper(realClient); + assertFalse(minioOnlyHelper.isAvailable()); + } + + @Test(expected = IOException.class) + public void testNativeComposeMissingEtagThrowsException() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + // Response body has no XML ETag tag, and response header also has no ETag + ResponseBody body = ResponseBody.create("", MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + } + + @Test + public void testNativeComposeAbortFailureHandledQuietly() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("https://s3.amazonaws.com")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + // Abort itself throws an exception to test the log.warn catch block + CompletableFuture failedAbort = + new CompletableFuture<>(); + failedAbort.completeExceptionally(new RuntimeException("Abort S3 error")); + when(asyncClient.abortMultipartUpload(any(AbortMultipartUploadArgs.class))) + .thenReturn(failedAbort); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + Response response = + new Response.Builder() + .request(new Request.Builder().url("https://s3.amazonaws.com").build()) + .protocol(Protocol.HTTP_1_1) + .code(500) + .message("Internal Server Error") + .body(ResponseBody.create("error", MediaType.parse("text/plain"))) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + + try { + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + fail("Expected IOException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("HTTP 500")); + } + } + + @Test + public void testExplicitConnectionParametersInitializationFailureHandledGracefully() { + // When endpoint contains invalid syntax that triggers an exception during MinioAsyncClient + // creation, the constructor catches the exception and logs a warning gracefully without + // throwing. + S3ServerSideComposeHelper helper = + new S3ServerSideComposeHelper( + "http://:::invalid:::", "us-east-1", "accessKey", "secretKey"); + assertFalse(helper.isAvailable()); + } + + @Test + public void testNativeComposeRegionFallbackToLocal() throws Exception { + MinioClient minioClient = mock(MinioClient.class); + MinioAsyncClient asyncClient = mock(MinioAsyncClient.class); + OkHttpClient httpClient = mock(OkHttpClient.class); + Provider provider = mock(Provider.class); + + Credentials creds = new Credentials("testAccessKey", "testSecretKey", null, null); + when(provider.fetch()).thenReturn(creds); + + // Endpoint without AWS region domain (e.g. localhost) so baseUrl.region() is empty + Http.BaseUrl baseUrl = new Http.BaseUrl(HttpUrl.parse("http://localhost:9000")); + + InitiateMultipartUploadResult initResult = mock(InitiateMultipartUploadResult.class); + when(initResult.uploadId()).thenReturn("mock-upload-id"); + CreateMultipartUploadResponse createResponse = + new CreateMultipartUploadResponse(null, "my-bucket", "", "upload/target", initResult); + when(asyncClient.createMultipartUpload(any(CreateMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(createResponse)); + + ObjectWriteResponse completeResponse = + new ObjectWriteResponse(null, "my-bucket", "", "upload/target", "etag-final", "v1"); + when(asyncClient.completeMultipartUpload(any(CompleteMultipartUploadArgs.class))) + .thenReturn(CompletableFuture.completedFuture(completeResponse)); + + Call call = mock(Call.class); + when(httpClient.newCall(any(Request.class))).thenReturn(call); + + String partXml = "\"part-etag-1\""; + ResponseBody body = ResponseBody.create(partXml, MediaType.parse("application/xml")); + Response response = + new Response.Builder() + .request(new Request.Builder().url("http://localhost:9000").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body) + .build(); + when(call.execute()).thenReturn(response); + + S3ServerSideComposeHelper helper = new S3ServerSideComposeHelper(minioClient); + setField(helper, "asyncClient", asyncClient); + setField(helper, "baseUrl", baseUrl); + setField(helper, "httpClient", httpClient); + setField(helper, "provider", provider); + setField(helper, "explicitRegion", null); + + // Verify composition executes with fallback region "local" without errors + helper.compose("my-bucket", "upload/target", Collections.singletonList("upload/part-1")); + verify(asyncClient).completeMultipartUpload(any(CompleteMultipartUploadArgs.class)); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field field = S3ServerSideComposeHelper.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 868a3c1e..b2a5b259 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -22,6 +22,7 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; import io.minio.Result; import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; @@ -32,9 +33,16 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.exception.MinUploadLengthNotReachedException; +import me.desair.tus.server.exception.UploadNotFoundException; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; @@ -50,7 +58,16 @@ public class S3StorageServiceTest { @Before public void setUp() { minioClient = mock(MinioClient.class); - storageService = new S3StorageService(minioClient, "test-bucket"); + storageService = + new S3StorageService( + minioClient, + "test-bucket", + S3StorageService.DEFAULT_OBJECT_PREFIX, + S3StorageService.DEFAULT_METADATA_PREFIX, + S3StorageService.DEFAULT_CHECKSUMS_PREFIX, + S3StorageService.DEFAULT_LOCKS_PREFIX, + Paths.get(System.getProperty("java.io.tmpdir")), + null); } @Test @@ -73,10 +90,53 @@ public void testNullTemporaryDirectoryConstructor() { java.nio.file.Path nullPath = null; S3StorageService serviceWithNullTmp = new S3StorageService( - minioClient, "test-bucket", "uploads/", "uploads/", "checksums/", "locks/", nullPath); + minioClient, + "test-bucket", + "uploads/", + "uploads/", + "checksums/", + "locks/", + nullPath, + null); assertNotNull(serviceWithNullTmp); } + @Test + public void testExplicitConnectionParametersConstructors() { + // 1. Constructor taking connection parameters directly without region (defaults to "local") + S3StorageService serviceWithoutRegion = + new S3StorageService("https://s3.amazonaws.com", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithoutRegion); + + // 2. Constructor taking connection parameters directly with region + S3StorageService serviceWithParams = + new S3StorageService( + "https://s3.amazonaws.com", "us-east-1", "accessKey", "secretKey", "test-bucket"); + assertNotNull(serviceWithParams); + + // 3. Full constructor taking connection parameters with prefix and temporary directory + // configuration + S3StorageService fullServiceWithParams = + new S3StorageService( + "https://s3.amazonaws.com", + "us-east-1", + "accessKey", + "secretKey", + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + java.nio.file.Paths.get(System.getProperty("java.io.tmpdir"))); + assertNotNull(fullServiceWithParams); + + // 4. Null/empty region defaults cleanly + S3StorageService defaultRegionService = + new S3StorageService( + "https://s3.amazonaws.com", null, "accessKey", "secretKey", "test-bucket"); + assertNotNull(defaultRegionService); + } + @Test public void testGetS3ObjectKeyByUri() throws Exception { UploadInfo info = new UploadInfo(); @@ -217,14 +277,15 @@ public void testAppendExceedsMaxAppendSizePreservesIncompletePartAndUpdatesOffse info.setId(uploadId); info.setLength(10000L); info.setOffset(20L); + String partKey1 = "metadata/" + uploadId + ".part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(partKey1))); String json = UploadInfoJsonSerializer.serialize(info); when(minioClient.getObject(any(GetObjectArgs.class))) .thenAnswer( invocation -> { GetObjectArgs args = invocation.getArgument(0); - // Simulate existing .part buffer object of 20 bytes - if (args.object().endsWith(".part")) { + if (args.object().equals(partKey1)) { return mockGetObjectResponse(new byte[20]); } return mockGetObjectResponse(json.getBytes()); @@ -236,7 +297,7 @@ public void testAppendExceedsMaxAppendSizePreservesIncompletePartAndUpdatesOffse .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part")) { + if (args.object().equals(partKey1)) { return partStat; } throw new ErrorResponseException( @@ -254,8 +315,9 @@ public void testAppendExceedsMaxAppendSizePreservesIncompletePartAndUpdatesOffse } // Verify that when maxAppendSize was exceeded, the prepended bytes were NOT lost - // and were flushed back to S3 as an incomplete .part object. - verify(minioClient, atLeastOnce()).putObject(argThat(args -> args.object().endsWith(".part"))); + // and were flushed back to S3 as a part chunk object. + verify(minioClient, atLeastOnce()) + .putObject(argThat(args -> args != null && args.object().contains(".part."))); } @Test(expected = MinUploadLengthNotReachedException.class) @@ -362,6 +424,7 @@ public void testAppendInterruptedMultiPart() throws Exception { info.setId(new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e")); info.setLength(200000000L); info.setOffset(0L); + info.setUploadPartKeys(new ArrayList<>()); String json = UploadInfoJsonSerializer.serialize(info); when(minioClient.getObject(any(GetObjectArgs.class))) @@ -746,7 +809,15 @@ public void testConfigurationSettersAndGetters() { storageService.setIdFactory(new me.desair.tus.server.upload.UuidUploadIdFactory()); - S3ConcatenationService concat = new S3ConcatenationService(minioClient, "test-bucket"); + S3ConcatenationService concat = + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + storageService, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); storageService.setUploadConcatenationService(concat); assertEquals(concat, storageService.getUploadConcatenationService()); @@ -782,6 +853,26 @@ public void testAppendThrowsMinAppendSizeNotMetException() throws Exception { storageService.append(info, new ByteArrayInputStream(new byte[100])); } + /** + * §4.1.4: "This limit does not apply to upload creation requests with no content, or to requests + * completing the upload by including the Upload-Complete: ?1 header field." + */ + @Test + public void testAppendCompletingUploadBypassesMinAppendSize() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e")); + info.setLength(100L); + + String json = UploadInfoJsonSerializer.serialize(info); + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer(invocation -> mockGetObjectResponse(json.getBytes())); + + storageService.setMinAppendSize(500L); + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[100])); + assertEquals(Long.valueOf(100L), result.getOffset()); + assertFalse(result.isUploadInProgress()); + } + @Test(expected = me.desair.tus.server.exception.MaxUploadLengthExceededException.class) public void testAppendThrowsMaxUploadLengthExceededException() throws Exception { UploadInfo info = new UploadInfo(); @@ -844,7 +935,7 @@ public void testCleanupExpiredUploads() throws Exception { String json = UploadInfoJsonSerializer.serialize(expiredInfo); Item item = mock(Item.class); - when(item.objectName()).thenReturn("uploads/expired-123.info"); + when(item.objectName()).thenReturn("metadata/expired-123.info"); Result result = new Result<>(item); when(minioClient.listObjects(any(ListObjectsArgs.class))) .thenReturn(java.util.Collections.singletonList(result)); @@ -856,6 +947,7 @@ public void testCleanupExpiredUploads() throws Exception { when(mockLocking.isLocked(expiredId)).thenReturn(false); storageService.cleanupExpiredUploads(mockLocking); + verify(mockLocking).cleanupLock(expiredId); } @Test(expected = IOException.class) @@ -890,44 +982,6 @@ public void testFinalizeCompletedUploadWithMultipleParts() throws Exception { storageService.append(info, new ByteArrayInputStream(new byte[100])); } - @Test - public void testFinalizeCompletedUploadWithLeftoverIncompletePart() throws Exception { - UploadInfo info = new UploadInfo(); - UploadId id = new UploadId("leftover-part-123"); - info.setId(id); - info.setLength(50L); - info.setOffset(0L); - - String json = UploadInfoJsonSerializer.serialize(info); - - StatObjectResponse leftoverHead = mock(StatObjectResponse.class); - when(leftoverHead.size()).thenReturn(50L); - - when(minioClient.statObject(any(StatObjectArgs.class))) - .thenAnswer( - invocation -> { - StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part")) { - return leftoverHead; - } - ErrorResponse err = mock(ErrorResponse.class); - when(err.code()).thenReturn("NoSuchKey"); - throw new ErrorResponseException(err, null, null); - }); - - when(minioClient.getObject(any(GetObjectArgs.class))) - .thenAnswer( - invocation -> { - GetObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".info")) { - return mockGetObjectResponse(json.getBytes()); - } - return mockGetObjectResponse(new byte[50]); - }); - - storageService.append(info, new ByteArrayInputStream(new byte[50])); - } - @Test public void testFinalizeCompletedUploadZeroLength() throws Exception { UploadInfo info = new UploadInfo(); @@ -1043,6 +1097,8 @@ public void testFetchS3ByteStreamIncompletePartFallback() throws Exception { UploadId id = new UploadId("fallback-123"); info.setId(id); info.setOffset(50L); + String partKey1 = "metadata/fallback-123.part.00001"; + info.setUploadPartKeys(Collections.singletonList(partKey1)); ErrorResponse noSuchKeyErr = mock(ErrorResponse.class); when(noSuchKeyErr.code()).thenReturn("NoSuchKey"); @@ -1055,7 +1111,7 @@ public void testFetchS3ByteStreamIncompletePartFallback() throws Exception { if (args.object().endsWith(".info")) { return mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes()); } - if (args.object().endsWith(".part")) { + if (args.object().equals(partKey1)) { return mockGetObjectResponse("part-data".getBytes()); } throw noSuchKeyEx; @@ -1463,11 +1519,11 @@ public void testDeleteObjectQuietlyNullAndException() throws Exception { public void testSanitizePrefixNullOrEmptyInS3StorageService() throws Exception { java.nio.file.Path tmpDir = java.nio.file.Paths.get(System.getProperty("java.io.tmpdir")); - S3StorageService s1 = new S3StorageService(minioClient, "bucket", "", "", "", "", tmpDir); + S3StorageService s1 = new S3StorageService(minioClient, "bucket", "", "", "", "", tmpDir, null); assertNotNull(s1); S3StorageService s2 = - new S3StorageService(minioClient, "bucket", null, null, null, null, tmpDir); + new S3StorageService(minioClient, "bucket", null, null, null, null, tmpDir, null); assertNotNull(s2); } @@ -1478,21 +1534,12 @@ public void testFinalizeCompletedUploadStreamingReuploadWhenSub5MbIntermediatePa UploadId id = new UploadId("sub5mb-test-123"); info.setId(id); info.setLength(2000L); - info.setOffset(0L); + info.setOffset(2000L); + info.setUploadPartKeys( + Arrays.asList("uploads/sub5mb-test-123.part.00001", "uploads/sub5mb-test-123.part.00002")); String json = UploadInfoJsonSerializer.serialize(info); - Item item1 = mock(Item.class); - when(item1.objectName()).thenReturn("uploads/sub5mb-test-123.part.00001"); - when(item1.size()).thenReturn(1000L); - - Item item2 = mock(Item.class); - when(item2.objectName()).thenReturn("uploads/sub5mb-test-123.part.00002"); - when(item2.size()).thenReturn(1000L); - - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); - StatObjectResponse stat1 = mock(StatObjectResponse.class); when(stat1.size()).thenReturn(1000L); StatObjectResponse stat2 = mock(StatObjectResponse.class); @@ -1502,9 +1549,9 @@ public void testFinalizeCompletedUploadStreamingReuploadWhenSub5MbIntermediatePa .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part.00001")) { + if (args.object().contains(".part.00001")) { return stat1; - } else if (args.object().endsWith(".part.00002")) { + } else if (args.object().contains(".part.00002")) { return stat2; } ErrorResponse err = mock(ErrorResponse.class); @@ -1522,7 +1569,7 @@ public void testFinalizeCompletedUploadStreamingReuploadWhenSub5MbIntermediatePa return mockGetObjectResponse(new byte[1000]); }); - UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[2000])); + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); assertNotNull(result); org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.never()) .composeObject(any(ComposeObjectArgs.class)); @@ -1538,20 +1585,65 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA long fiveMb = 5L * 1024L * 1024L; info.setId(id); info.setLength(fiveMb + 100L); - info.setOffset(0L); + info.setOffset(fiveMb + 100L); + info.setUploadPartKeys( + Arrays.asList( + "uploads/compose-test-123.part.00001", "uploads/compose-test-123.part.00002")); String json = UploadInfoJsonSerializer.serialize(info); - Item item1 = mock(Item.class); - when(item1.objectName()).thenReturn("uploads/compose-test-123.part.00001"); - when(item1.size()).thenReturn(fiveMb); + StatObjectResponse stat1 = mock(StatObjectResponse.class); + when(stat1.size()).thenReturn(fiveMb); + StatObjectResponse stat2 = mock(StatObjectResponse.class); + when(stat2.size()).thenReturn(100L); - Item item2 = mock(Item.class); - when(item2.objectName()).thenReturn("uploads/compose-test-123.part.00002"); - when(item2.size()).thenReturn(100L); + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().contains(".part.00001")) { + return stat1; + } else if (args.object().contains(".part.00002")) { + return stat2; + } + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + }); - when(minioClient.listObjects(any(ListObjectsArgs.class))) - .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); + assertNotNull(result); + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .composeObject(any(ComposeObjectArgs.class)); + } + + @Test + public void testFinalizeCompletedUploadComposeObjectFailsFallsBackToStreamingAndDisablesCompose() + throws Exception { + assertTrue(storageService.isS3ComposeObjectSupported()); + + UploadInfo info = new UploadInfo(); + UploadId id = new UploadId("compose-fail-test-123"); + long fiveMb = 5L * 1024L * 1024L; + info.setId(id); + info.setLength(fiveMb + 100L); + info.setOffset(fiveMb + 100L); + info.setUploadPartKeys( + Arrays.asList( + "uploads/compose-fail-test-123.part.00001", + "uploads/compose-fail-test-123.part.00002")); + + String json = UploadInfoJsonSerializer.serialize(info); StatObjectResponse stat1 = mock(StatObjectResponse.class); when(stat1.size()).thenReturn(fiveMb); @@ -1562,9 +1654,9 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part.00001")) { + if (args.object().contains(".part.00001")) { return stat1; - } else if (args.object().endsWith(".part.00002")) { + } else if (args.object().contains(".part.00002")) { return stat2; } ErrorResponse err = mock(ErrorResponse.class); @@ -1582,11 +1674,53 @@ public void testFinalizeCompletedUploadComposeObjectWhenAllIntermediatePartsAreA return mockGetObjectResponse(new byte[100]); }); - UploadInfo result = - storageService.append(info, new ByteArrayInputStream(new byte[(int) (fiveMb + 100L)])); + when(minioClient.composeObject(any(ComposeObjectArgs.class))) + .thenThrow(new RuntimeException("The specified header is not valid in this context")); + + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); assertNotNull(result); + + // Verify composeObject was attempted and failed + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .composeObject(any(ComposeObjectArgs.class)); + // Verify fallback to streaming putObject was executed + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.atLeastOnce()) + .putObject(any(PutObjectArgs.class)); + // Verify s3ComposeObjectSupported is now disabled + assertFalse(storageService.isS3ComposeObjectSupported()); + + // Subsequent upload should directly use streaming without calling composeObject + UploadInfo info2 = new UploadInfo(); + UploadId id2 = new UploadId("compose-skip-test-456"); + info2.setId(id2); + info2.setLength(fiveMb + 100L); + info2.setOffset(fiveMb + 100L); + info2.setUploadPartKeys( + Arrays.asList( + "uploads/compose-skip-test-456.part.00001", + "uploads/compose-skip-test-456.part.00002")); + + String json2 = UploadInfoJsonSerializer.serialize(info2); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json2.getBytes()); + } + return mockGetObjectResponse(new byte[100]); + }); + + UploadInfo result2 = storageService.append(info2, new ByteArrayInputStream(new byte[0])); + assertNotNull(result2); + + // composeObject count should still be 1 (never called for the second upload) org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) .composeObject(any(ComposeObjectArgs.class)); + + // Reset flag for other tests + storageService.setS3ComposeObjectSupported(true); } @Test @@ -1622,6 +1756,18 @@ public void testPrepareStreamRollsBackSub5MbNumberedPart() throws Exception { return java.util.Collections.emptyList(); }); + java.util.concurrent.atomic.AtomicBoolean partSaved = + new java.util.concurrent.atomic.AtomicBoolean(false); + when(minioClient.putObject(any(PutObjectArgs.class))) + .thenAnswer( + inv -> { + PutObjectArgs args = inv.getArgument(0); + if (args.object().endsWith(".part")) { + partSaved.set(true); + } + return null; + }); + StatObjectResponse stat1 = mock(StatObjectResponse.class); when(stat1.size()).thenReturn(500L); @@ -1632,9 +1778,9 @@ public void testPrepareStreamRollsBackSub5MbNumberedPart() throws Exception { .thenAnswer( invocation -> { StatObjectArgs args = invocation.getArgument(0); - if (args.object().endsWith(".part.00001")) { + if (args.object().contains(".part.00001")) { return stat1; - } else if (args.object().endsWith(".part")) { + } else if (args.object().endsWith(".part") && partSaved.get()) { return statPart; } ErrorResponse err = mock(ErrorResponse.class); @@ -1758,7 +1904,14 @@ public void testCleanupExpiredUploadsPrunesStaleTempFilesAndChecksumIndices() th S3StorageService customService = new S3StorageService( - minioClient, "test-bucket", "uploads/", "metadata/", "checksums/", "locks/", tempDir); + minioClient, + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + tempDir, + null); customService.setUploadDeduplicationEnabled(true); Item checksumItem = mock(Item.class); @@ -1796,6 +1949,997 @@ public void testCleanupExpiredUploadsPrunesStaleTempFilesAndChecksumIndices() th } } + @Test + public void testCloudUploadThreadPoolSizeConfiguration() { + assertEquals( + "Default thread pool size is 10", 10, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(25); + assertEquals( + "Updated thread pool size is 25", 25, storageService.getCloudUploadThreadPoolSize()); + + storageService.setCloudUploadThreadPoolSize(4); + assertEquals("Reduced thread pool size is 4", 4, storageService.getCloudUploadThreadPoolSize()); + + try { + storageService.setCloudUploadThreadPoolSize(0); + fail("Should reject 0 pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + + try { + storageService.setCloudUploadThreadPoolSize(-5); + fail("Should reject negative pool size"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("greater than 0")); + } + } + + @Test + public void testDrainTimeoutConfiguration() { + assertEquals( + "Default drain timeout is 55 seconds", + Duration.ofSeconds(55), + storageService.getDrainTimeout()); + + storageService.setDrainTimeout(Duration.ofSeconds(20)); + assertEquals( + "Updated drain timeout is 20 seconds", + Duration.ofSeconds(20), + storageService.getDrainTimeout()); + + // Null is ignored preserving current value + storageService.setDrainTimeout(null); + assertEquals( + "Null drain timeout preserves previous value", + Duration.ofSeconds(20), + storageService.getDrainTimeout()); + } + + @Test + public void testCloseGracefulShutdown() throws Exception { + // Verify closing storage service cleanly terminates background upload executor without error + storageService.close(); + // KISS: verifying method executes cleanly without throwing an exception + + // Verify close handles thread interruption gracefully + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + } + + @Test + public void testCalculateCurrentOffsetDoesNotDoubleCountOrExceedLength() throws Exception { + UploadId uploadId = new UploadId("calc-offset-test"); + String infoJson = + "{\"id\":\"calc-offset-test\",\"length\":3425070,\"offset\":null,\"uploadPartKeys\":[\"metadata/calc-offset-test.part.00001\"]}"; + + String partKey1 = "metadata/calc-offset-test.part.00001"; + + StatObjectResponse part1Stat = mock(StatObjectResponse.class); + when(part1Stat.size()).thenReturn(3425070L); // 3.42 MB numbered part + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(infoJson.getBytes()); + } + return mockGetObjectResponse(new byte[0]); + }); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals(partKey1)) { + return part1Stat; + } else if (args.object().equals("uploads/calc-offset-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return mock(StatObjectResponse.class); + }); + + UploadInfo fetched = storageService.getUploadInfo(uploadId); + assertNotNull(fetched); + assertEquals(Long.valueOf(3425070L), fetched.getOffset()); + } + + @Test + public void testCalcOptimalPartSizeCalculations() { + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(null)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(0L)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(100L)); + assertEquals(8 * 1024 * 1024L, storageService.calcOptimalPartSize(100L * 1024 * 1024)); + assertEquals( + 8 * 1024 * 1024L, storageService.calcOptimalPartSize(10_000L * 8 * 1024 * 1024L - 1)); + + long largeLength = 10_000L * 16 * 1024 * 1024L; + assertEquals((largeLength / 10_000L) + 1, storageService.calcOptimalPartSize(largeLength)); + + // For 1 TB, part size auto-scales up so upload fits within 10,000 parts + long oneTb = 1024L * 1024 * 1024 * 1024L; + assertEquals((oneTb / 10_000L) + 1, storageService.calcOptimalPartSize(oneTb)); + + // For 5 TB (S3 max limit), part size scales up to ~524.3 MB + long fiveTb = 5L * 1024 * 1024 * 1024 * 1024L; + assertEquals((fiveTb / 10_000L) + 1, storageService.calcOptimalPartSize(fiveTb)); + } + + @Test + public void testSetAndGetPreferredPartSize() { + assertEquals(8 * 1024 * 1024L, storageService.getPreferredPartSize()); + + storageService.setPreferredPartSize(16 * 1024 * 1024L); + assertEquals(16 * 1024 * 1024L, storageService.getPreferredPartSize()); + + try { + storageService.setPreferredPartSize(4 * 1024 * 1024L); // Below 5MB limit + fail("Should reject part size below 5MB"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("Preferred part size must be between")); + } + + try { + storageService.setPreferredPartSize(6L * 1024 * 1024 * 1024L); // Above 5GB limit + fail("Should reject part size above 5GB"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("Preferred part size must be between")); + } + } + + @Test + public void testValidateRemainingPartBudget() throws Exception { + UploadInfo info = new UploadInfo(); + storageService.validateRemainingPartBudget(info, 9999); + // KISS: 9999 parts is within budget, verifies method completes cleanly + + try { + storageService.validateRemainingPartBudget(info, 10000); + fail("Should throw MaxAppendSizeExceededException at 10000 parts"); + } catch (MaxAppendSizeExceededException expected) { + assertTrue(expected.getMessage().contains("maximum allowed S3 limit of 10000 parts")); + } + + try { + storageService.validateRemainingPartBudget(info, 10005); + fail("Should throw MaxAppendSizeExceededException above 10000 parts"); + } catch (MaxAppendSizeExceededException expected) { + assertTrue(expected.getMessage().contains("maximum allowed S3 limit of 10000 parts")); + } + } + + @Test + public void testRemoveLastNumberOfBytesWithManifestParts() throws Exception { + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("trunc-manifest-test")); + info.setLength(300L); + info.setOffset(300L); + info.setUploadPartKeys( + new ArrayList<>(Arrays.asList("metadata/trunc.part.00001", "metadata/trunc.part.00002"))); + + String json = UploadInfoJsonSerializer.serialize(info); + + StatObjectResponse stat1 = mock(StatObjectResponse.class); + when(stat1.size()).thenReturn(150L); + StatObjectResponse stat2 = mock(StatObjectResponse.class); + when(stat2.size()).thenReturn(150L); + + when(minioClient.statObject(any(StatObjectArgs.class))) + .thenAnswer( + invocation -> { + StatObjectArgs args = invocation.getArgument(0); + if (args.object().equals("metadata/trunc.part.00002")) { + return stat2; + } else if (args.object().equals("metadata/trunc.part.00001")) { + return stat1; + } else if (args.object().equals("uploads/trunc-manifest-test")) { + ErrorResponse err = mock(ErrorResponse.class); + when(err.code()).thenReturn("NoSuchKey"); + throw new ErrorResponseException(err, null, null); + } + return mock(StatObjectResponse.class); + }); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer( + invocation -> { + GetObjectArgs args = invocation.getArgument(0); + if (args.object().endsWith(".info")) { + return mockGetObjectResponse(json.getBytes()); + } + return mockGetObjectResponse(new byte[150]); + }); + + // Remove 200 bytes: all 150 bytes of part 2, plus 50 bytes of part 1 + storageService.removeLastNumberOfBytes(info, 200L); + + assertEquals(Long.valueOf(100L), info.getOffset()); + assertEquals(1, info.getUploadPartKeys().size()); + assertEquals("metadata/trunc.part.00001", info.getUploadPartKeys().get(0)); + + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/trunc.part.00002"))); + verify(minioClient) + .putObject( + argThat((PutObjectArgs args) -> args.object().equals("metadata/trunc.part.00001"))); + } + + @Test + public void testDeleteS3ObjectsBatchFailureFallback() throws Exception { + assertTrue(storageService.isSupportsBatchDelete()); + UploadId uploadId = new UploadId("batch-del-fallback"); + List keys = Arrays.asList("metadata/batch.part.00001", "metadata/batch.part.00002"); + + io.minio.messages.DeleteResult.Error delErr = mock(io.minio.messages.DeleteResult.Error.class); + when(delErr.objectName()).thenReturn("metadata/batch.part.00001"); + when(delErr.message()).thenReturn("AccessDenied"); + + @SuppressWarnings("unchecked") + Result errResult = mock(Result.class); + when(errResult.get()).thenReturn(delErr); + + when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenReturn(Collections.singletonList(errResult)); + + storageService.deleteS3ObjectsQuietly(keys); + + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00001"))); + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00002"))); + assertFalse(storageService.isSupportsBatchDelete()); + + // Subsequent batch delete should bypass removeObjects directly + storageService.deleteS3ObjectsQuietly(Collections.singletonList("metadata/batch.part.00003")); + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00003"))); + // removeObjects was only called once (for the first attempt) + org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) + .removeObjects(any(RemoveObjectsArgs.class)); + + // Reset flag for subsequent tests + storageService.setSupportsBatchDelete(true); + } + + @Test + public void testDeleteS3ObjectsThrowsExceptionFallsBackToIndividualDelete() throws Exception { + assertTrue(storageService.isSupportsBatchDelete()); + UploadId uploadId = new UploadId("batch-del-ex"); + List keys = Collections.singletonList("metadata/batch-ex.part.00001"); + + when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenThrow(new RuntimeException("MinIO network failure")); + + storageService.deleteS3ObjectsQuietly(keys); + + verify(minioClient) + .removeObject( + argThat( + (RemoveObjectArgs args) -> args.object().equals("metadata/batch-ex.part.00001"))); + assertFalse(storageService.isSupportsBatchDelete()); + + // Null or empty handling + storageService.deleteS3ObjectsQuietly(null); + storageService.deleteS3ObjectsQuietly(Collections.emptyList()); + storageService.deleteS3ObjectsQuietly(Collections.singletonList(null)); + + // Reset flag + storageService.setSupportsBatchDelete(true); + } + + @Test + public void testGetUploadedBytesFromManifestPartsWhenCompletedObjectMissing() throws Exception { + UploadId uploadId = new UploadId("manifest-stream-test"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(12L); + info.setOffset(12L); + info.setUploadPartKeys( + Arrays.asList( + "metadata/manifest-stream-test.part.00001", + "metadata/manifest-stream-test.part.00002")); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/manifest-stream-test.info".equals(a.object())))) + .thenReturn( + mockGetObjectResponse( + UploadInfoJsonSerializer.serialize(info) + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + ErrorResponse err = new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "uploads/manifest-stream-test".equals(a.object())))) + .thenThrow(new ErrorResponseException(err, null, null)); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/manifest-stream-test.part.00001".equals(a.object())))) + .thenReturn( + mockGetObjectResponse("Hello ".getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/manifest-stream-test.part.00002".equals(a.object())))) + .thenReturn( + mockGetObjectResponse("World!".getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + try (InputStream stream = storageService.getUploadedBytes(uploadId)) { + assertNotNull(stream); + byte[] readBytes = org.apache.commons.io.IOUtils.toByteArray(stream); + assertEquals("Hello World!", new String(readBytes, java.nio.charset.StandardCharsets.UTF_8)); + } + } + + @Test + public void testAppendWithInterruptibleInputStream() throws Exception { + UploadId uploadId = new UploadId("interrupt-stream-append"); + UploadInfo initial = new UploadInfo(); + initial.setId(uploadId); + initial.setLength(10L); + initial.setOffset(null); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/interrupt-stream-append.info".equals(a.object())))) + .thenAnswer( + inv -> + mockGetObjectResponse( + UploadInfoJsonSerializer.serialize(initial) + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + UploadInfo created = storageService.create(initial, "owner"); + assertNotNull(created.getOffset()); + assertEquals(0L, created.getOffset().longValue()); + + byte[] data = "12345".getBytes(java.nio.charset.StandardCharsets.UTF_8); + try (me.desair.tus.server.util.InterruptibleInputStream is = + new me.desair.tus.server.util.InterruptibleInputStream(new ByteArrayInputStream(data))) { + UploadInfo updated = storageService.append(created, is); + assertEquals(5L, updated.getOffset().longValue()); + } + } + + @Test + public void testRemoveLastNumberOfBytesManifestPartPutObjectException() throws Exception { + UploadId uploadId = new UploadId("trunc-put-err"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(10L); + info.setUploadPartKeys( + new ArrayList<>(Collections.singletonList("metadata/trunc-put-err.part.00001"))); + + ErrorResponse noSuchKeyErr = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.statObject( + argThat( + (StatObjectArgs args) -> + args != null && "uploads/trunc-put-err".equals(args.object())))) + .thenThrow(new ErrorResponseException(noSuchKeyErr, null, null)); + + StatObjectResponse stat = mock(StatObjectResponse.class); + when(stat.size()).thenReturn(10L); + when(minioClient.statObject( + argThat( + (StatObjectArgs args) -> + args != null && "metadata/trunc-put-err.part.00001".equals(args.object())))) + .thenReturn(stat); + + byte[] partData = new byte[10]; + when(minioClient.getObject( + argThat( + (GetObjectArgs args) -> + args != null && "metadata/trunc-put-err.part.00001".equals(args.object())))) + .thenReturn(mockGetObjectResponse(partData)); + + doThrow(new RuntimeException("Simulated PutObject failure")) + .when(minioClient) + .putObject( + argThat( + (PutObjectArgs args) -> + args != null && "metadata/trunc-put-err.part.00001".equals(args.object()))); + + storageService.removeLastNumberOfBytes(info, 3L); + assertEquals(7L, info.getOffset().longValue()); + } + + @Test + public void testSupportsBatchDeleteConfiguration() { + assertTrue(storageService.isSupportsBatchDelete()); + storageService.setSupportsBatchDelete(false); + assertFalse(storageService.isSupportsBatchDelete()); + storageService.setSupportsBatchDelete(true); + assertTrue(storageService.isSupportsBatchDelete()); + } + + @Test + public void testCloseInterruptedGracefully() throws Exception { + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testRemoveLastNumberOfBytesTruncatesCompletedObject() throws Exception { + UploadId uploadId = new UploadId("trunc-completed-1"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + + String completedKey = "uploads/trunc-completed-1"; + StatObjectResponse headStat = mock(StatObjectResponse.class); + when(headStat.size()).thenReturn(100L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(headStat); + + byte[] fullBytes = new byte[100]; + for (int i = 0; i < 100; i++) { + fullBytes[i] = (byte) i; + } + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(mockGetObjectResponse(fullBytes)); + + storageService.removeLastNumberOfBytes(info, 20L); + + // 80 bytes should be rewritten as part 1, and completedKey should be deleted + verify(minioClient) + .putObject( + argThat( + (PutObjectArgs a) -> + a != null && a.object().startsWith("metadata/trunc-completed-1.part.00001-"))); + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs a) -> a != null && completedKey.equals(a.object()))); + assertEquals(1, info.getUploadPartKeys().size()); + assertTrue( + info.getUploadPartKeys().get(0).startsWith("metadata/trunc-completed-1.part.00001-")); + } + + @Test + public void testRemoveLastNumberOfBytesTruncatesCompletedObjectToZero() throws Exception { + UploadId uploadId = new UploadId("trunc-completed-zero"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + + String completedKey = "uploads/trunc-completed-zero"; + StatObjectResponse headStat = mock(StatObjectResponse.class); + when(headStat.size()).thenReturn(100L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(headStat); + + storageService.removeLastNumberOfBytes(info, 100L); + + verify(minioClient) + .removeObject( + argThat((RemoveObjectArgs a) -> a != null && completedKey.equals(a.object()))); + assertTrue(info.getUploadPartKeys().isEmpty()); + } + + @Test(expected = IOException.class) + public void testRemoveLastNumberOfBytesTruncatesCompletedObjectThrowsIOExceptionOnReadFailure() + throws Exception { + UploadId uploadId = new UploadId("trunc-completed-err"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + + String completedKey = "uploads/trunc-completed-err"; + StatObjectResponse headStat = mock(StatObjectResponse.class); + when(headStat.size()).thenReturn(100L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenReturn(headStat); + + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenThrow(new RuntimeException("Simulated read failure")); + + storageService.removeLastNumberOfBytes(info, 20L); + } + + @Test + public void testRemoveLastNumberOfBytesTruncatesSinglePartRewrite() throws Exception { + UploadId uploadId = new UploadId("trunc-single-part"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + String part1Key = "uploads/trunc-single-part.part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(part1Key))); + + // completedKey does not exist + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.statObject( + argThat( + (StatObjectArgs a) -> a != null && "uploads/trunc-single-part".equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + StatObjectResponse partStat = mock(StatObjectResponse.class); + when(partStat.size()).thenReturn(100L); + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(partStat); + + byte[] partData = new byte[100]; + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(mockGetObjectResponse(partData)); + + storageService.removeLastNumberOfBytes(info, 30L); + + // Part should be rewritten with 70 bytes + verify(minioClient) + .putObject(argThat((PutObjectArgs a) -> a != null && part1Key.equals(a.object()))); + assertEquals(1, info.getUploadPartKeys().size()); + } + + @Test + public void testRemoveLastNumberOfBytesDeletesEntirePartWhenSubsumed() throws Exception { + UploadId uploadId = new UploadId("trunc-subsumed"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(100L); + String part1Key = "uploads/trunc-subsumed.part.00001"; + String part2Key = "uploads/trunc-subsumed.part.00002"; + info.setUploadPartKeys(new ArrayList<>(Arrays.asList(part1Key, part2Key))); + + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.statObject( + argThat( + (StatObjectArgs a) -> a != null && "uploads/trunc-subsumed".equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + StatObjectResponse statPart1 = mock(StatObjectResponse.class); + when(statPart1.size()).thenReturn(50L); + StatObjectResponse statPart2 = mock(StatObjectResponse.class); + when(statPart2.size()).thenReturn(50L); + + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(statPart1); + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && part2Key.equals(a.object())))) + .thenReturn(statPart2); + + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(mockGetObjectResponse(new byte[50])); + + // Removing 70 bytes should completely remove part2 (50 bytes) and truncate part1 by 20 bytes + storageService.removeLastNumberOfBytes(info, 70L); + + verify(minioClient) + .removeObject(argThat((RemoveObjectArgs a) -> a != null && part2Key.equals(a.object()))); + verify(minioClient) + .putObject(argThat((PutObjectArgs a) -> a != null && part1Key.equals(a.object()))); + assertEquals(1, info.getUploadPartKeys().size()); + assertEquals(part1Key, info.getUploadPartKeys().get(0)); + } + + @Test + public void testFetchS3ByteStreamFallbackToManifestParts() throws Exception { + UploadId uploadId = new UploadId("stream-fallback-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(100L); + info.setOffset(50L); + String part1Key = "uploads/stream-fallback-id.part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(part1Key))); + + String completedKey = "uploads/stream-fallback-id"; + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && completedKey.equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && part1Key.equals(a.object())))) + .thenReturn(mockGetObjectResponse("streamed-bytes".getBytes())); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/stream-fallback-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + InputStream stream = storageService.getUploadedBytes(uploadId); + assertNotNull(stream); + byte[] content = stream.readAllBytes(); + assertEquals("streamed-bytes", new String(content)); + } + + @Test + public void testPruneOrphanPartsDeletesUnmanifestedS3Part() throws Exception { + UploadId uploadId = new UploadId("orphan-test-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000L); + info.setOffset(0L); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/orphan-test-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + Item manifestedItem = mock(Item.class); + when(manifestedItem.objectName()).thenReturn("metadata/orphan-test-id.part.00001"); + Item orphanItem = mock(Item.class); + when(orphanItem.objectName()).thenReturn("metadata/orphan-test-id.part.99999"); + + Result res1 = new Result<>(manifestedItem); + Result res2 = new Result<>(orphanItem); + + when(minioClient.listObjects( + argThat( + (ListObjectsArgs a) -> + a != null && "metadata/orphan-test-id.part.".equals(a.prefix())))) + .thenReturn(Arrays.asList(res1, res2)); + + storageService.append(info, new ByteArrayInputStream(new byte[0])); + + verify(minioClient) + .removeObject( + argThat( + (RemoveObjectArgs a) -> + a != null && "metadata/orphan-test-id.part.99999".equals(a.object()))); + } + + @Test + public void testConstructorUnableToEnsureTempDirLogsDebug() throws Exception { + java.nio.file.Path tempFile = java.nio.file.Files.createTempFile("tus-existing-file", ".tmp"); + try { + S3StorageService service = + new S3StorageService( + minioClient, + "test-bucket", + "uploads/", + "metadata/", + "checksums/", + "locks/", + tempFile, + null); + assertNotNull(service); + } finally { + java.nio.file.Files.deleteIfExists(tempFile); + } + } + + @Test + public void testAppendSetsLengthWhenInfoLengthIsNull() throws Exception { + UploadId uploadId = new UploadId("deferred-len-id"); + UploadInfo infoInS3 = new UploadInfo(); + infoInS3.setId(uploadId); + infoInS3.setLength(null); + infoInS3.setOffset(0L); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/deferred-len-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(infoInS3).getBytes())); + + UploadInfo appendUpload = new UploadInfo(); + appendUpload.setId(uploadId); + appendUpload.setLength(500L); + + UploadInfo result = storageService.append(appendUpload, new ByteArrayInputStream(new byte[0])); + assertNotNull(result); + assertEquals(Long.valueOf(500L), result.getLength()); + } + + @Test + public void testGetUploadInfoByChecksumNoSuchKeyReturnsNull() throws Exception { + storageService.setUploadDeduplicationEnabled(true); + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && a.object().startsWith("checksums/")))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + UploadInfo info = storageService.getUploadInfoByChecksum("hash-123", ChecksumAlgorithm.SHA1); + assertNull(info); + } + + @Test + public void testPruneOrphanedChecksumIndicesListObjectsExceptionHandled() throws Exception { + storageService.setUploadDeduplicationEnabled(true); + when(minioClient.listObjects( + argThat((ListObjectsArgs a) -> a != null && a.prefix().startsWith("checksums/")))) + .thenThrow(new RuntimeException("Simulated S3 listing error")); + + storageService.cleanupExpiredUploads(null); + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testIsJsonSerializationEnabled() { + assertTrue(storageService.isJsonSerializationEnabled()); + } + + @Test + public void testSetAndGetS3ServerSideComposeHelper() { + S3ServerSideComposeHelper mockHelper = mock(S3ServerSideComposeHelper.class); + storageService.setS3ServerSideComposeHelper(mockHelper); + assertEquals(mockHelper, storageService.getS3ServerSideComposeHelper()); + } + + @Test + public void testCloseInterruptedHandledGracefully() throws Exception { + Thread.currentThread().interrupt(); + try { + storageService.close(); + } finally { + Thread.interrupted(); // Clear interrupted status + } + // KISS: verifying method executes cleanly without throwing an exception + assertTrue(true); + } + + @Test + public void testPrepareStreamWithExistingIncompletePartInspectionExceptionHandled() + throws Exception { + UploadId uploadId = new UploadId("part-inspect-err-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000L); + info.setOffset(50L); + String partKey = "metadata/part-inspect-err-id.part.00001"; + info.setUploadPartKeys(new ArrayList<>(Collections.singletonList(partKey))); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/part-inspect-err-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + // StatObject on last part throws Exception to test lines 1033-1034 catch block + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && partKey.equals(a.object())))) + .thenThrow(new RuntimeException("Part inspection failed")); + + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); + assertNotNull(result); + } + + @Test + public void testCalculateAndSetOffsetNullAndStatObjectException() throws Exception { + // Null info / null ID does nothing + storageService.calculateAndSetOffset(null); + UploadInfo nullIdInfo = new UploadInfo(); + storageService.calculateAndSetOffset(nullIdInfo); + + // Object exists but statObject throws Exception + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("stat-fail-id")); + when(minioClient.statObject( + argThat((StatObjectArgs a) -> a != null && "uploads/stat-fail-id".equals(a.object())))) + .thenThrow(new RuntimeException("StatObject failed")); + + storageService.calculateAndSetOffset(info); + assertEquals(Long.valueOf(0L), info.getOffset()); + } + + @Test + public void testDeleteObjectsBatchEmptyListAndErrorReported() throws Exception { + // Empty list returns immediately + storageService.deleteS3ObjectsQuietly(Collections.emptyList()); + + // Batch delete reports error, falling back to individual delete + @SuppressWarnings("unchecked") + Result mockRes = mock(Result.class); + io.minio.messages.DeleteResult.Error mockError = + mock(io.minio.messages.DeleteResult.Error.class); + when(mockError.objectName()).thenReturn("key-1"); + when(mockError.message()).thenReturn("Access Denied"); + when(mockRes.get()).thenReturn(mockError); + + when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenReturn(Collections.singletonList(mockRes)); + + storageService.deleteS3ObjectsQuietly(Arrays.asList("key-1", "key-2")); + verify(minioClient) + .removeObject(argThat((RemoveObjectArgs a) -> a != null && "key-1".equals(a.object()))); + verify(minioClient) + .removeObject(argThat((RemoveObjectArgs a) -> a != null && "key-2".equals(a.object()))); + } + + @Test + public void testPreparedStreamInterruptDelegation() { + me.desair.tus.server.util.InterruptibleInputStream origStream = + new me.desair.tus.server.util.InterruptibleInputStream( + new ByteArrayInputStream(new byte[10])); + S3StorageService.PreparedStream prepStream = + new S3StorageService.PreparedStream( + new ByteArrayInputStream(new byte[10]), + origStream, + 0L, + 0L, + Collections.emptyList(), + Collections.emptyList()); + + assertFalse(prepStream.isInterrupted()); + prepStream.interrupt(); + assertTrue(prepStream.isInterrupted()); + assertTrue(origStream.isInterrupted()); + } + + @Test(expected = java.util.NoSuchElementException.class) + public void testS3PartInputStreamEnumerationThrowsNoSuchElementException() { + S3StorageService.S3PartInputStreamEnumeration enumeration = + new S3StorageService.S3PartInputStreamEnumeration( + minioClient, "test-bucket", Collections.emptyList()); + assertFalse(enumeration.hasMoreElements()); + enumeration.nextElement(); + } + + @Test(expected = UploadNotFoundException.class) + public void testGetUploadedBytesStreamFailureThrowsUploadNotFoundException() throws Exception { + UploadId uploadId = new UploadId("err-bytes-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setOffset(100L); + info.setUploadPartKeys(Collections.singletonList("metadata/err-bytes-id.part.00001")); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> a != null && "metadata/err-bytes-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + ErrorResponse notFound = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject( + argThat((GetObjectArgs a) -> a != null && "uploads/err-bytes-id".equals(a.object())))) + .thenThrow(new ErrorResponseException(notFound, null, null)); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/err-bytes-id.part.00001".equals(a.object())))) + .thenThrow(new RuntimeException("S3 read part error")); + + InputStream is = storageService.getUploadedBytes(uploadId); + if (is != null) { + is.read(); + } + } + + @Test(expected = MaxAppendSizeExceededException.class) + public void testAppendWhenInitialPartsBudgetExceededThrows() throws Exception { + UploadId uploadId = new UploadId("max-parts-init-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000000L); + info.setOffset(50000L); + List tenThousandKeys = new ArrayList<>(Collections.nCopies(10000, "metadata/part.key")); + info.setUploadPartKeys(tenThousandKeys); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> + a != null && "metadata/max-parts-init-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + storageService.append(info, new ByteArrayInputStream(new byte[50])); + } + + @Test + public void testAppendWithMaxPartsReachedFlushesChunkAndThrows() throws Exception { + storageService.setPreferredPartSize(5 * 1024 * 1024L); + UploadId uploadId = new UploadId("max-parts-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(20_000_000L); + info.setOffset(0L); + List existingKeys = + new ArrayList<>(Collections.nCopies(9999, "uploads/max-parts-id.part.0000001")); + info.setUploadPartKeys(existingKeys); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> a != null && "metadata/max-parts-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + StatObjectResponse stat5Mb = mock(StatObjectResponse.class); + when(stat5Mb.size()).thenReturn(5L * 1024 * 1024); + when(minioClient.statObject(any(StatObjectArgs.class))).thenReturn(stat5Mb); + + // Provide 5MB (which fills part 10,000) followed by 100 bytes (which hits the 10,000 max parts + // limit) + InputStream stream = + new InputStream() { + private long remaining = 5L * 1024 * 1024 + 100; + + @Override + public int read() { + return remaining-- > 0 ? 0 : -1; + } + + @Override + public int read(byte[] b, int off, int len) { + if (remaining <= 0) { + return -1; + } + int toRead = (int) Math.min(len, remaining); + Arrays.fill(b, off, off + toRead, (byte) 0); + remaining -= toRead; + return toRead; + } + }; + + try { + storageService.append(info, stream); + fail("Expected MaxAppendSizeExceededException when exceeding 10,000 parts limit"); + } catch (MaxAppendSizeExceededException e) { + assertTrue(e.getMessage().contains("10000 parts")); + // Verify that the trailing chunk was flushed and the offset was updated before throwing + assertEquals(10001, info.getUploadPartKeys().size()); + assertEquals(10000L * 5 * 1024 * 1024L + 100L, info.getOffset().longValue()); + } + } + + @Test + public void testAppendDrainExceptionRecordedAndThrowsIOException() throws Exception { + UploadId uploadId = new UploadId("drain-err-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setLength(1000L); + info.setOffset(0L); + + when(minioClient.getObject( + argThat( + (GetObjectArgs a) -> a != null && "metadata/drain-err-id.info".equals(a.object())))) + .thenReturn(mockGetObjectResponse(UploadInfoJsonSerializer.serialize(info).getBytes())); + + doThrow(new RuntimeException(new IOException("S3 chunk put failed"))) + .when(minioClient) + .putObject(argThat((PutObjectArgs a) -> a != null && a.object().contains(".part."))); + + try { + storageService.append(info, new ByteArrayInputStream(new byte[100])); + fail("Expected IOException from drain failure"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("S3 chunk put failed") || e.getCause() != null); + } + } + private GetObjectResponse mockGetObjectResponse(byte[] bytes) { return new GetObjectResponse( null, "test-bucket", "eu-central-1", "object-key", new ByteArrayInputStream(bytes)); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java index 8b7e66b4..63d5b831 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java @@ -9,8 +9,13 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; +import io.minio.messages.DeleteResult; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import me.desair.tus.server.upload.LeaseData; @@ -87,7 +92,7 @@ public void testRenewLeaseExceptionHandling() throws Exception { public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Exception { Mockito.doThrow(new RuntimeException("Remove failed")) .when(minioClient) - .removeObject(any(RemoveObjectArgs.class)); + .removeObjects(any(RemoveObjectsArgs.class)); LeaseData leaseData = createLeaseData("holder-123", "/files/upload-1"); S3UploadLock lockWithNullKeys = @@ -109,8 +114,28 @@ public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Excep } @Test - public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Exception { - // Simulate remote lock owned by another holder + public void testCloseHeartbeatExecutorShutdownException() throws Exception { + java.util.concurrent.ScheduledExecutorService mockExecutor = + mock(java.util.concurrent.ScheduledExecutorService.class); + Mockito.doThrow(new RuntimeException("Shutdown error")).when(mockExecutor).shutdownNow(); + + LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap, + mockExecutor); + + lock.close(); + assertEquals("holder-123", lock.getHolderId()); + } + + @Test + public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { LeaseData otherLock = new LeaseData( "other-holder", @@ -134,6 +159,10 @@ public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Except Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + InterruptibleInputStream stream = + new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); + inputStreamMap.put("/files/upload-1", stream); + S3UploadLock lock = new S3UploadLock( myLeaseData, @@ -143,18 +172,19 @@ public void testDeleteS3LockObjectIfOwnerSkipsWhenHolderMismatch() throws Except "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); + lock.renewLease(); - // Must NOT remove object because it belongs to other-holder - Mockito.verify(minioClient, Mockito.never()) - .removeObject( - Mockito.argThat( - args -> args != null && "tus-locks/upload-1.lock".equals(args.object()))); + // On lease ownership loss, renewLease must abort the active input stream immediately + // to prevent writing un-locked bytes to S3. + assertTrue(stream.isInterrupted()); + + // Must NOT call putObject because lock is now held by other-holder + Mockito.verify(minioClient, Mockito.never()).putObject(any(PutObjectArgs.class)); + lock.close(); } @Test - public void testDeleteS3LockObjectIfOwnerDeletesWhenHolderMatches() throws Exception { - // Simulate remote lock owned by this holder + public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { LeaseData myLock = new LeaseData( "my-holder", @@ -187,25 +217,15 @@ public void testDeleteS3LockObjectIfOwnerDeletesWhenHolderMatches() throws Excep "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); + lock.renewLease(); - // Must remove object because it matches my-holder - Mockito.verify(minioClient) - .removeObject( - Mockito.argThat( - args -> args != null && "tus-locks/upload-1.lock".equals(args.object()))); + // Must call putObject because holder matches + Mockito.verify(minioClient).putObject(any(PutObjectArgs.class)); + lock.close(); } @Test - public void testDeleteS3LockObjectIfOwnerHandlesNoSuchKey() throws Exception { - io.minio.messages.ErrorResponse errorResponse = - Mockito.mock(io.minio.messages.ErrorResponse.class); - Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); - io.minio.errors.ErrorResponseException ex = - new io.minio.errors.ErrorResponseException(errorResponse, null, null); - - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenThrow(ex); - + public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -216,56 +236,36 @@ public void testDeleteS3LockObjectIfOwnerHandlesNoSuchKey() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); - assertEquals("my-holder", lock.getHolderId()); - } - - @Test - public void testCloseHeartbeatExecutorShutdownException() throws Exception { - java.util.concurrent.ScheduledExecutorService mockExecutor = - mock(java.util.concurrent.ScheduledExecutorService.class); - Mockito.doThrow(new RuntimeException("Shutdown error")).when(mockExecutor).shutdownNow(); + // Null key returns false + org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch(null)); - LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); - S3UploadLock lock = + // Null minioClient returns false + S3UploadLock nullClientLock = new S3UploadLock( - myLeaseData, - minioClient, - "test-bucket", - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop", - inputStreamMap, - mockExecutor); - - lock.close(); - assertEquals("holder-123", lock.getHolderId()); - } + myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + org.junit.Assert.assertFalse(nullClientLock.doesLockOwnershipMatch("key")); - @Test - public void testDeleteS3LockObjectIfOwnerNullChecksAndExceptionHandling() throws Exception { - LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); - S3UploadLock lock = + // Null bucket returns false + S3UploadLock nullBucketLock = new S3UploadLock( - myLeaseData, - minioClient, - "test-bucket", - "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop", - inputStreamMap); + myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + org.junit.Assert.assertFalse(nullBucketLock.doesLockOwnershipMatch("key")); - // Null key check - lock.deleteS3LockObjectIfOwner(null); + // General exception (non-ErrorResponseException) returns true to allow proceed + Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) + .thenThrow(new RuntimeException("Transient S3 error")); + org.junit.Assert.assertTrue(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); - // Exception during removeObject - Mockito.doThrow(new RuntimeException("Remove failed")) - .when(minioClient) - .removeObject(any(RemoveObjectArgs.class)); - lock.deleteS3LockObjectIfOwner("tus-locks/upload-1.lock"); - assertEquals("holder-123", lock.getHolderId()); + // Renew lease with null lockKey or null minioClient + nullClientLock.doRenewLease(); + S3UploadLock nullKeyLock = + new S3UploadLock(myLeaseData, minioClient, "test-bucket", null, null, inputStreamMap); + nullKeyLock.doRenewLease(); } @Test - public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { + public void testReleaseLockResourceWhenOwnershipMismatch() throws Exception { + // Simulate remote lock owned by another holder LeaseData otherLock = new LeaseData( "other-holder", @@ -283,16 +283,11 @@ public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { "test-bucket", "eu-central-1", "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + new ByteArrayInputStream(json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); - InterruptibleInputStream stream = - new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); - inputStreamMap.put("/files/upload-1", stream); - S3UploadLock lock = new S3UploadLock( myLeaseData, @@ -302,41 +297,85 @@ public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.renewLease(); - - // On lease ownership loss, renewLease must abort the active input stream immediately - // to prevent writing un-locked bytes to S3. - assertTrue(stream.isInterrupted()); - - // Must NOT call putObject because lock is now held by other-holder - Mockito.verify(minioClient, Mockito.never()).putObject(any(PutObjectArgs.class)); lock.close(); + + // Must NOT call removeObjects or removeObject because lock was stolen by another node + // and any stop signal might have been set by a third thread for the new holder + Mockito.verify(minioClient, Mockito.never()).removeObjects(any(RemoveObjectsArgs.class)); + Mockito.verify(minioClient, Mockito.never()).removeObject(any(RemoveObjectArgs.class)); } @Test - public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { - LeaseData myLock = - new LeaseData( - "my-holder", - "/files/upload-1", - 60000L, - System.currentTimeMillis() + 60000L, - System.currentTimeMillis(), + public void testDeleteS3ObjectsNullChecksAndEmpty() { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + minioClient, + "test-bucket", "tus-locks/upload-1.lock", - "tus-locks/upload-1.stop"); - String json = LeaseDataJsonSerializer.serialize(myLock); + "tus-locks/upload-1.stop", + inputStreamMap); - io.minio.GetObjectResponse response = - new io.minio.GetObjectResponse( - null, + // Null client + S3UploadLock nullClientLock = + new S3UploadLock( + myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + nullClientLock.deleteS3Objects("key1", "key2"); + + // Null bucket + S3UploadLock nullBucketLock = + new S3UploadLock( + myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + nullBucketLock.deleteS3Objects("key1", "key2"); + + // Both keys null -> empty objects + lock.deleteS3Objects(null, null); + + Mockito.verify(minioClient, Mockito.never()).removeObjects(any(RemoveObjectsArgs.class)); + assertEquals("test-bucket", lock.getBucket()); + assertEquals("tus-locks/upload-1.lock", lock.getLockKey()); + assertEquals("tus-locks/upload-1.stop", lock.getStopKey()); + } + + @Test + public void testDeleteS3ObjectsWithErrorsAndExceptions() throws Exception { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + minioClient, "test-bucket", - "eu-central-1", "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + "tus-locks/upload-1.stop", + inputStreamMap); - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); + DeleteResult.Error deleteError = mock(DeleteResult.Error.class); + Mockito.when(deleteError.objectName()).thenReturn("tus-locks/upload-1.lock"); + Mockito.when(deleteError.message()).thenReturn("Access Denied"); + @SuppressWarnings("unchecked") + Result errorResult = mock(Result.class); + Mockito.when(errorResult.get()).thenReturn(deleteError); + + @SuppressWarnings("unchecked") + Result throwingResult = mock(Result.class); + Mockito.when(throwingResult.get()).thenThrow(new RuntimeException("Result parsing error")); + + List> resultList = Arrays.asList(errorResult, throwingResult, null); + + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))).thenReturn(resultList); + + // Call deleteS3Objects with only firstKey, then with both keys + lock.deleteS3Objects("tus-locks/upload-1.lock", null); + lock.deleteS3Objects(null, "tus-locks/upload-1.stop"); + + Mockito.verify(minioClient, Mockito.times(2)).removeObjects(any(RemoveObjectsArgs.class)); + assertEquals("my-holder", lock.getHolderId()); + } + + @Test + public void testDeleteS3ObjectsWhenMinioClientThrows() { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -347,15 +386,16 @@ public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { "tus-locks/upload-1.stop", inputStreamMap); - lock.renewLease(); + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenThrow(new RuntimeException("S3 connection error")); - // Must call putObject because holder matches - Mockito.verify(minioClient).putObject(any(PutObjectArgs.class)); - lock.close(); + // Should catch exception quietly without propagating + lock.deleteS3Objects("tus-locks/upload-1.lock", "tus-locks/upload-1.stop"); + assertEquals("my-holder", lock.getHolderId()); } @Test - public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception { + public void testDeleteSingleObjectEdgeCases() throws Exception { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( @@ -366,30 +406,52 @@ public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception "tus-locks/upload-1.stop", inputStreamMap); - // Null key returns false - org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch(null)); + lock.deleteSingleObject(null); - // Null minioClient returns false S3UploadLock nullClientLock = - new S3UploadLock( - myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); - org.junit.Assert.assertFalse(nullClientLock.doesLockOwnershipMatch("key")); + new S3UploadLock(myLeaseData, null, "test-bucket", "k1", "k2", inputStreamMap); + nullClientLock.deleteSingleObject("k1"); - // Null bucket returns false S3UploadLock nullBucketLock = + new S3UploadLock(myLeaseData, minioClient, null, "k1", "k2", inputStreamMap); + nullBucketLock.deleteSingleObject("k1"); + + Mockito.doThrow(new RuntimeException("removeObject error")) + .when(minioClient) + .removeObject(any(RemoveObjectArgs.class)); + lock.deleteSingleObject("tus-locks/upload-1.lock"); + assertEquals("my-holder", lock.getHolderId()); + } + + @Test + public void testDeleteS3ObjectsResultIteratorExceptionFallback() throws Exception { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + S3UploadLock lock = new S3UploadLock( - myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); - org.junit.Assert.assertFalse(nullBucketLock.doesLockOwnershipMatch("key")); + myLeaseData, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap); - // General exception (non-ErrorResponseException) returns true to allow proceed - Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) - .thenThrow(new RuntimeException("Transient S3 error")); - org.junit.Assert.assertTrue(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); + @SuppressWarnings("unchecked") + io.minio.Result mockResult = mock(io.minio.Result.class); + Mockito.when(mockResult.get()) + .thenThrow(new RuntimeException("Simulated error while reading delete result")); - // Renew lease with null lockKey or null minioClient - nullClientLock.doRenewLease(); - S3UploadLock nullKeyLock = - new S3UploadLock(myLeaseData, minioClient, "test-bucket", null, null, inputStreamMap); - nullKeyLock.doRenewLease(); + Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) + .thenReturn(java.util.Collections.singletonList(mockResult)); + + lock.deleteS3Objects("tus-locks/upload-1.lock", "tus-locks/upload-1.stop"); + + Mockito.verify(minioClient) + .removeObject( + org.mockito.ArgumentMatchers.argThat( + (RemoveObjectArgs args) -> args.object().equals("tus-locks/upload-1.lock"))); + Mockito.verify(minioClient) + .removeObject( + org.mockito.ArgumentMatchers.argThat( + (RemoveObjectArgs args) -> args.object().equals("tus-locks/upload-1.stop"))); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java index 53ee607d..5dc3b3c9 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java @@ -35,10 +35,28 @@ public void testParseErrorResponseCodes() throws Exception { assertEquals( S3ErrorType.API_NOT_IMPLEMENTED, S3Utils.parseErrorResponse(createExceptionWithCode("APINotImplemented"))); + assertEquals( + S3ErrorType.API_NOT_IMPLEMENTED, + S3Utils.parseErrorResponse(createExceptionWithCode("NotImplemented"))); assertEquals( S3ErrorType.UNKNOWN, S3Utils.parseErrorResponse(createExceptionWithCode("InternalError"))); } + @Test + public void testParseErrorResponseHttp501() { + okhttp3.Response httpResponse = + new okhttp3.Response.Builder() + .request(new okhttp3.Request.Builder().url("https://example.com").build()) + .protocol(okhttp3.Protocol.HTTP_1_1) + .code(501) + .message("Not Implemented") + .build(); + ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); + org.mockito.Mockito.when(errorResponse.code()).thenReturn(""); + ErrorResponseException ex = new ErrorResponseException(errorResponse, httpResponse, null); + assertEquals(S3ErrorType.API_NOT_IMPLEMENTED, S3Utils.parseErrorResponse(ex)); + } + private ErrorResponseException createExceptionWithCode(String code) { ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); org.mockito.Mockito.when(errorResponse.code()).thenReturn(code); diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java new file mode 100644 index 00000000..60f9b921 --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -0,0 +1,820 @@ +package me.desair.tus.server.upload.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import me.desair.tus.server.util.InterruptibleInputStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests verifying bounded 3-slot asynchronous chunk uploading, backpressure, exception + * translation, timeout aborts, and CallerRunsPolicy degradation in {@link AsyncChunkUploader}. + */ +public class AsyncChunkUploaderTest { + + private ExecutorService executor; + private File tempDir; + + @Before + public void setUp() throws Exception { + executor = Executors.newFixedThreadPool(4); + tempDir = Files.createTempDirectory("tus-async-uploader-test").toFile(); + } + + @After + public void tearDown() throws Exception { + if (executor != null) { + executor.shutdownNow(); + } + if (tempDir != null && tempDir.exists()) { + org.apache.commons.io.FileUtils.deleteDirectory(tempDir); + } + } + + @Test + public void testFirstChunkSubmissionDirectToSlot3() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + Files.write(chunk1.toPath(), new byte[] {1, 2, 3}); + + CountDownLatch uploadFinished = new CountDownLatch(1); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 3, + "part-1", + () -> { + uploadFinished.countDown(); + }); + + assertTrue( + "Upload task should execute in background", uploadFinished.await(3, TimeUnit.SECONDS)); + int confirmed = uploader.drainAndComplete(3000); + assertEquals("One chunk confirmed uploaded", 1, confirmed); + } + + // Verify temp file is deleted by the background worker in finally block + assertFalse("Chunk file should be cleaned up after successful upload", chunk1.exists()); + } + + @Test + public void testPipelinedSubmissionThroughSlot2AndSlot3() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + File chunk3 = new File(tempDir, "chunk3.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + Files.write(chunk3.toPath(), new byte[] {3}); + + List uploadedOrder = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch unblockSlot3 = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + // Chunk 1 occupies Slot 3 + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + unblockSlot3.await(3, TimeUnit.SECONDS); + uploadedOrder.add("part-1"); + }); + + // Chunk 2 placed in Slot 2 (waiting) while Chunk 1 is running + uploader.submitChunk( + chunk2, + 1, + "part-2", + () -> { + uploadedOrder.add("part-2"); + }); + + // Unblock Chunk 1 so it finishes + unblockSlot3.countDown(); + + // Chunk 3 submitted: blocks until Chunk 1 finishes, promotes Chunk 2 to Slot 3, stores Chunk + // 3 in Slot 2 + uploader.submitChunk( + chunk3, + 1, + "part-3", + () -> { + uploadedOrder.add("part-3"); + }); + + int confirmed = uploader.drainAndComplete(3000); + assertEquals("All 3 chunks confirmed uploaded", 3, confirmed); + assertEquals( + "Parts uploaded in sequential order", + List.of("part-1", "part-2", "part-3"), + uploadedOrder); + } + + assertFalse("Chunk 1 should be deleted", chunk1.exists()); + assertFalse("Chunk 2 should be deleted", chunk2.exists()); + assertFalse("Chunk 3 should be deleted", chunk3.exists()); + } + + @Test + public void testBackpressureBlocksReaderWhenBothSlotsFull() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + File chunk3 = new File(tempDir, "chunk3.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + Files.write(chunk3.toPath(), new byte[] {3}); + + CountDownLatch chunk1Hold = new CountDownLatch(1); + AtomicBoolean chunk3Submitted = new AtomicBoolean(false); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + chunk1Hold.await(3, TimeUnit.SECONDS); + }); + + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + + // Launch thread trying to submit chunk 3: must block because Slot 3 is busy and Slot 2 is + // occupied + Thread submitThread = + new Thread( + () -> { + try { + uploader.submitChunk(chunk3, 1, "part-3", () -> {}); + chunk3Submitted.set(true); + } catch (IOException ignored) { + } + }); + submitThread.start(); + + // Give thread 150ms to attempt submit; it must be blocked waiting on Slot 3 + Thread.sleep(150); + assertFalse("Submit of chunk 3 must block due to backpressure", chunk3Submitted.get()); + + // Release chunk 1 + chunk1Hold.countDown(); + submitThread.join(3000); + + assertTrue("Submit of chunk 3 unblocked after chunk 1 finished", chunk3Submitted.get()); + int confirmed = uploader.drainAndComplete(3000); + assertEquals("All 3 chunks confirmed", 3, confirmed); + } + } + + @Test + public void testUploadActionFailurePropagatesIOExceptionAndAborts() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + throw new IOException("Simulated S3 network failure"); + }); + + // Submitting next chunk or draining must detect previous failure and throw IOException + try { + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + uploader.drainAndComplete(3000); + fail("Expected IOException from failed chunk upload"); + } catch (IOException e) { + assertTrue( + "Exception message should reflect upload failure", + e.getMessage().contains("Simulated S3 network failure")); + } + } + + assertFalse("Chunk 2 should be cleaned up by abort()", chunk2.exists()); + } + + @Test + public void testDrainTimeoutAbortsInFlightUpload() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + + CountDownLatch hangLatch = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + hangLatch.await(10, TimeUnit.SECONDS); + }); + + try { + // Enforce short 200ms drain timeout + uploader.drainAndComplete(200); + fail("Expected timeout IOException during drain"); + } catch (IOException e) { + assertTrue("Should report timeout error", e.getMessage().contains("Timed out after")); + assertTrue("Should report key", e.getMessage().contains("part-1")); + } + } finally { + hangLatch.countDown(); + } + } + + @Test + public void testCloseAutomaticallyAbortsIfUncompleted() throws Exception { + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + CountDownLatch latch = new CountDownLatch(1); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + latch.await(3, TimeUnit.SECONDS); + }); + uploader.submitChunk(chunk2, 1, "part-2", () -> {}); + // Exiting without calling drainAndComplete() triggers close() which calls abort() + } finally { + latch.countDown(); + } + + assertFalse( + "Waiting chunk file in Slot 2 should be deleted by abort on close", chunk2.exists()); + } + + @Test + public void testSynchronousQueueCallerRunsPolicyDegradation() throws Exception { + // ThreadPool with exactly 1 thread and SynchronousQueue + ThreadPoolExecutor singlePool = + new ThreadPoolExecutor( + 1, + 1, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + File chunk1 = new File(tempDir, "chunk1.tmp"); + File chunk2 = new File(tempDir, "chunk2.tmp"); + Files.write(chunk1.toPath(), new byte[] {1}); + Files.write(chunk2.toPath(), new byte[] {2}); + + CountDownLatch poolThreadBusy = new CountDownLatch(1); + CountDownLatch releaseBusy = new CountDownLatch(1); + + // Occupy the only thread in singlePool + singlePool.submit( + () -> { + poolThreadBusy.countDown(); + try { + releaseBusy.await(3, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue("Single pool thread is now occupied", poolThreadBusy.await(3, TimeUnit.SECONDS)); + + String mainThreadName = Thread.currentThread().getName(); + List executionThreads = Collections.synchronizedList(new ArrayList<>()); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(singlePool)) { + // Since pool is saturated and queue is SynchronousQueue, CallerRunsPolicy executes task + // directly on caller thread + uploader.submitChunk( + chunk1, + 1, + "part-1", + () -> { + executionThreads.add(Thread.currentThread().getName()); + }); + + int confirmed = uploader.drainAndComplete(3000); + assertEquals("One chunk confirmed", 1, confirmed); + assertEquals( + "Executed directly on caller thread without queue stall", + mainThreadName, + executionThreads.get(0)); + } finally { + releaseBusy.countDown(); + singlePool.shutdownNow(); + } + } + + @Test + public void testHighConcurrencyMultiUploaderSimulation() throws Exception { + int concurrentUploaders = 15; + ExecutorService sharedPool = + new ThreadPoolExecutor( + 4, + 4, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + CountDownLatch allDone = new CountDownLatch(concurrentUploaders); + AtomicInteger totalUploadedChunks = new AtomicInteger(0); + + for (int i = 0; i < concurrentUploaders; i++) { + final int uploaderId = i; + new Thread( + () -> { + try { + File f1 = new File(tempDir, "c-" + uploaderId + "-1.tmp"); + File f2 = new File(tempDir, "c-" + uploaderId + "-2.tmp"); + Files.write(f1.toPath(), new byte[] {1, 2}); + Files.write(f2.toPath(), new byte[] {3, 4}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(sharedPool)) { + uploader.submitChunk( + f1, + 2, + "u" + uploaderId + "-p1", + () -> { + Thread.sleep(20); + }); + uploader.submitChunk( + f2, + 2, + "u" + uploaderId + "-p2", + () -> { + Thread.sleep(20); + }); + int confirmed = uploader.drainAndComplete(5000); + totalUploadedChunks.addAndGet(confirmed); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + allDone.countDown(); + } + }) + .start(); + } + + assertTrue( + "All concurrent uploaders must complete within 10s", allDone.await(10, TimeUnit.SECONDS)); + assertEquals( + "All 30 chunks (15 uploaders x 2 chunks) must complete", 30, totalUploadedChunks.get()); + + sharedPool.shutdownNow(); + } + + @Test + public void testGetConfirmedCount() throws Exception { + File f1 = new File(tempDir, "count1.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + assertEquals(0, uploader.getConfirmedCount()); + uploader.submitChunk(f1, 1, "k1", () -> {}); + uploader.drainAndComplete(3000); + assertEquals(1, uploader.getConfirmedCount()); + } + } + + @Test + public void testSubmitChunkDiscoversPreviousDoneUploadFailureAndCleansUp() throws Exception { + File f1 = new File(tempDir, "err1.tmp"); + File f2 = new File(tempDir, "err2.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + + CountDownLatch chunk1Done = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k1", + () -> { + chunk1Done.countDown(); + throw new IOException("Simulated disk read error on cloud worker"); + }); + + assertTrue(chunk1Done.await(3, TimeUnit.SECONDS)); + // Give worker thread a brief moment to update Future state to done + Thread.sleep(50); + + try { + uploader.submitChunk(f2, 1, "k2", () -> {}); + fail("Should have thrown IOException when discovering chunk 1 failure"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Simulated disk read error on cloud worker")); + } + } + + assertFalse("Chunk 2 must be cleaned up on failure", f2.exists()); + } + + @Test + public void testUploadActionThrowsRuntimeExceptionTranslatedToIOException() throws Exception { + File f1 = new File(tempDir, "runtime_err.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "runtime-key", + () -> { + throw new IllegalStateException("Boom!"); + }); + + try { + uploader.drainAndComplete(3000); + fail("Should have thrown IOException wrapping RuntimeException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Upload failed for key runtime-key: Boom!")); + assertTrue(e.getCause() instanceof IllegalStateException); + } + } + } + + @Test + public void testUploadActionThrowsErrorTranslatedToIOException() throws Exception { + File f1 = new File(tempDir, "error_key.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "error-key", + () -> { + throw new AssertionError("Simulated fatal assertion error"); + }); + + try { + uploader.drainAndComplete(3000); + fail("Should have thrown IOException wrapping AssertionError"); + } catch (IOException e) { + assertTrue( + e.getMessage().contains("Unexpected error during chunk upload for key error-key")); + } + } + } + + @Test + public void testDrainInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "interrupted.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + + AtomicBoolean gotInterruptedIOException = new AtomicBoolean(false); + + Thread drainThread = + new Thread( + () -> { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k-interrupt", + () -> { + started.countDown(); + unblock.await(5, TimeUnit.SECONDS); + }); + uploader.drainAndComplete(5000); + } catch (IOException e) { + if (e.getMessage().contains("Interrupted waiting for chunk upload")) { + gotInterruptedIOException.set(true); + } + } catch (Exception ignored) { + } + }); + + drainThread.start(); + assertTrue(started.await(3, TimeUnit.SECONDS)); + drainThread.interrupt(); + drainThread.join(3000); + unblock.countDown(); + + assertTrue( + "Interrupted thread should throw IOException mentioning Interrupted", + gotInterruptedIOException.get()); + } + + @Test + public void testSubmitChunkBackpressureInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "bp-int1.tmp"); + File f2 = new File(tempDir, "bp-int2.tmp"); + File f3 = new File(tempDir, "bp-int3.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + Files.write(f3.toPath(), new byte[] {3}); + + CountDownLatch f1Started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + AtomicBoolean gotBackpressureInterruptedException = new AtomicBoolean(false); + + Thread submitThread = + new Thread( + () -> { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k1", + () -> { + f1Started.countDown(); + unblock.await(5, TimeUnit.SECONDS); + }); + uploader.submitChunk(f2, 1, "k2", () -> {}); + // Third chunk triggers waitForInFlight() backpressure + uploader.submitChunk(f3, 1, "k3", () -> {}); + } catch (IOException e) { + if (e.getMessage().contains("Interrupted") + && e.getMessage().contains("in-flight chunk upload")) { + gotBackpressureInterruptedException.set(true); + } + } catch (Exception ignored) { + } + }); + + submitThread.start(); + assertTrue(f1Started.await(3, TimeUnit.SECONDS)); + // Brief sleep to let submitThread block in waitForInFlight() + try { + Thread.sleep(50); + } catch (InterruptedException ignored) { + } + submitThread.interrupt(); + submitThread.join(3000); + unblock.countDown(); + + assertTrue( + "Interrupted backpressure wait should throw IOException", + gotBackpressureInterruptedException.get()); + } + + @Test + public void testDrainAndCompleteDefaultTimeout() throws Exception { + File chunk = new File(tempDir, "chunk-default-drain.tmp"); + Files.write(chunk.toPath(), "test-data".getBytes()); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk(chunk, 9, "part-default", () -> {}); + int confirmed = uploader.drainAndComplete(); + assertEquals("One chunk confirmed uploaded using default timeout", 1, confirmed); + } + + assertFalse("Chunk file should be cleaned up", chunk.exists()); + } + + @Test + public void testCustomDrainTimeoutConstructorAndGetter() { + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor, 12_345L)) { + assertEquals(12_345L, uploader.getDrainTimeoutMs()); + } + } + + @Test + public void testDefaultDrainTimeoutConstant() { + assertEquals(55_000L, AsyncChunkUploader.DEFAULT_DRAIN_TIMEOUT_MS); + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + assertEquals(55_000L, uploader.getDrainTimeoutMs()); + } + } + + @Test + public void testSubmitChunkBackpressureContentionSignalThrowsIOException() throws Exception { + File f1 = new File(tempDir, "contention-c1.tmp"); + File f2 = new File(tempDir, "contention-c2.tmp"); + File f3 = new File(tempDir, "contention-c3.tmp"); + Files.write(f1.toPath(), "data-1".getBytes()); + Files.write(f2.toPath(), "data-2".getBytes()); + Files.write(f3.toPath(), "data-3".getBytes()); + + CountDownLatch f1Started = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + InterruptibleInputStream interruptibleStream = new InterruptibleInputStream(bais); + + try (AsyncChunkUploader uploader = + new AsyncChunkUploader(executor, 5000L, interruptibleStream)) { + uploader.submitChunk( + f1, + 6, + "k1", + () -> { + f1Started.countDown(); + try { + unblock.await(3, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + uploader.submitChunk(f2, 6, "k2", () -> {}); + assertTrue(f1Started.await(3, TimeUnit.SECONDS)); + + // Signal contention before 3rd submitChunk which requires waiting + interruptibleStream.interrupt(); + + try { + uploader.submitChunk(f3, 6, "k3", () -> {}); + fail("Expected IOException due to lock contention"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("contention")); + } + } finally { + unblock.countDown(); + } + } + + @Test + public void testAbortWithActiveInFlightUpload() throws Exception { + File chunk1 = new File(tempDir, "abort-active-c1.tmp"); + Files.write(chunk1.toPath(), "data-abort".getBytes()); + + CountDownLatch uploadStarted = new CountDownLatch(1); + CountDownLatch unblock = new CountDownLatch(1); + + AsyncChunkUploader uploader = new AsyncChunkUploader(executor); + uploader.submitChunk( + chunk1, + 10, + "k-abort", + () -> { + uploadStarted.countDown(); + try { + unblock.await(5, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue(uploadStarted.await(3, TimeUnit.SECONDS)); + // Calling abort while worker is running cancels inFlightUpload and waits up to 500ms + uploader.abort(); + unblock.countDown(); + uploader.close(); + } + + @Test + public void testWaitForInFlightExecutionExceptionPropagated() throws Exception { + File f1 = new File(tempDir, "exec-err-1.tmp"); + File f2 = new File(tempDir, "exec-err-2.tmp"); + File f3 = new File(tempDir, "exec-err-3.tmp"); + Files.write(f1.toPath(), "err1".getBytes()); + Files.write(f2.toPath(), "err2".getBytes()); + Files.write(f3.toPath(), "err3".getBytes()); + + CountDownLatch f1Started = new CountDownLatch(1); + CountDownLatch triggerFailure = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 4, + "k-fail-1", + () -> { + f1Started.countDown(); + try { + triggerFailure.await(5, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + throw new RuntimeException("S3 simulated upload failure"); + }); + + assertTrue(f1Started.await(3, TimeUnit.SECONDS)); + // Submit slot 2 while slot 1 is blocked in latch + uploader.submitChunk(f2, 4, "k-fail-2", () -> {}); + + // In a background thread release triggerFailure after 100ms so failure happens while + // waitForInFlight polls + Executors.newSingleThreadScheduledExecutor() + .schedule(triggerFailure::countDown, 100, TimeUnit.MILLISECONDS); + + // Submitting 3rd chunk triggers waitForInFlight() for slot 1 which throws ExecutionException + try { + uploader.submitChunk(f3, 4, "k-fail-3", () -> {}); + fail("Expected IOException when in-flight upload threw ExecutionException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("S3 simulated upload failure")); + } + } + } + + @Test + public void testAbortWithActiveInFlightUploadWaitsAndCancels() throws Exception { + File f1 = new File(tempDir, "abort-f1.tmp"); + Files.write(f1.toPath(), new byte[] {1, 2, 3}); + + CountDownLatch taskStarted = new CountDownLatch(1); + CountDownLatch hangLatch = new CountDownLatch(1); + + AsyncChunkUploader uploader = new AsyncChunkUploader(executor); + uploader.submitChunk( + f1, + 3, + "k-abort", + () -> { + taskStarted.countDown(); + try { + hangLatch.await(10, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue(taskStarted.await(3, TimeUnit.SECONDS)); + uploader.abort(); + + hangLatch.countDown(); + assertFalse("File should be cleaned up by abort", f1.exists()); + } + + @Test + public void testWaitForInFlightUploadTimeoutPolling() throws Exception { + File f1 = new File(tempDir, "poll-f1.tmp"); + File f2 = new File(tempDir, "poll-f2.tmp"); + File f3 = new File(tempDir, "poll-f3.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + Files.write(f2.toPath(), new byte[] {2}); + Files.write(f3.toPath(), new byte[] {3}); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k-poll-1", + () -> { + try { + Thread.sleep(120); + } catch (InterruptedException ignored) { + } + }); + + uploader.submitChunk(f2, 1, "k-poll-2", () -> {}); + + // Submitting 3rd chunk triggers waitForInFlightUpload(), which will encounter + // TimeoutException + // on the 50ms tick before succeeding on the 3rd tick. + uploader.submitChunk(f3, 1, "k-poll-3", () -> {}); + uploader.drainAndComplete(3000); + } + } + + @Test + public void testCheckAndConfirmInFlightInterruptedThrowsIOException() throws Exception { + File f1 = new File(tempDir, "interrupt-f1.tmp"); + Files.write(f1.toPath(), new byte[] {1}); + + CountDownLatch uploadStarted = new CountDownLatch(1); + CountDownLatch holdUpload = new CountDownLatch(1); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(executor)) { + uploader.submitChunk( + f1, + 1, + "k-int-1", + () -> { + uploadStarted.countDown(); + try { + holdUpload.await(10, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + } + }); + + assertTrue(uploadStarted.await(3, TimeUnit.SECONDS)); + + Thread.currentThread().interrupt(); + try { + uploader.checkAndConfirmInFlight(); + fail("Expected IOException"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Interrupted while checking in-flight chunk upload")); + } finally { + Thread.interrupted(); // clear + holdUpload.countDown(); + } + } + } +} diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index 1443dab7..a8d36f72 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -4,10 +4,12 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import jakarta.servlet.http.HttpServletRequest; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.Serializable; import java.nio.channels.FileChannel; @@ -971,6 +973,19 @@ public void testExtractMediaType() { is("application/offset+octet-stream")); } + @Test + public void testToInterruptibleStream() throws Exception { + assertThat(Utils.toInterruptibleStream(null), is(nullValue())); + + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[] {1, 2, 3}); + InterruptibleInputStream wrapped = Utils.toInterruptibleStream(bais); + assertThat(wrapped, is(notNullValue())); + assertThat(wrapped.read(), is(1)); + + InterruptibleInputStream same = Utils.toInterruptibleStream(wrapped); + assertThat(same, is(sameInstance(wrapped))); + } + /** Simple serializable class for testing. */ public static class TestSerializable implements Serializable { private static final long serialVersionUID = 1L;