From 760be89de7d8965168a159c120b85288efc0432c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 19:59:50 +0000 Subject: [PATCH 01/17] Rewrite FileSecure in Kotlin, merging MediaConverter features FileSecure 13 processes files through pipelines of ordered stages: name and date filters, folder/rename transformations from FileSecure, and a Convert stage bringing MediaConverter's ffmpeg/ImageMagick conversions, probe filters and H2 scan cache. Files are then copied or moved, with a single conflict policy (rename or skip), and folder actions run afterwards. Also brings MediaConverter's parallel conversions, progress bars, p/r/q console commands and Windows CPU affinity. The configuration format is new, the README documents it and how to migrate from both tools. Fixes carried over bugs: transformers ran in an unspecified order, extensionless name collisions aborted a mapping, folders beyond maxDepth were moved as files, DeleteIfEmpty could delete the input folder, a converted input could be stranded in the temp folder, and the affinity mask was applied to a pid instead of a process handle. The Docker image now uses the ffmpeg base image, like MediaConverter. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019ozsCgvJdvHoNqTQzgMJMU --- .editorconfig | 3 + .github/workflows/gradle_ci.yml | 4 + README.md | 218 +++++++++++--- build.gradle.kts | 65 ++-- gradle.properties | 6 +- gradle/libs.versions.toml | 39 ++- lombok.config | 2 - .../fr/rakambda/filesecure/CLIParameters.java | 24 -- .../java/fr/rakambda/filesecure/Main.java | 55 ---- .../filesecure/config/Configuration.java | 57 ---- .../rakambda/filesecure/config/Mapping.java | 25 -- .../rakambda/filesecure/config/Operation.java | 34 --- .../fr/rakambda/filesecure/config/Rule.java | 52 ---- .../config/options/FileTransformer.java | 43 --- .../config/options/FolderTransformer.java | 24 -- .../config/options/Transformer.java | 17 -- .../file/FolderCanonicalTransformer.java | 71 ----- .../options/file/FolderPerDayTransformer.java | 37 --- .../file/FolderPerMonthTransformer.java | 36 --- .../file/FolderPerWeekTransformer.java | 38 --- .../file/FolderPerYearTransformer.java | 35 --- .../file/FolderUniquePerDayTransformer.java | 35 --- .../options/file/IfOlderThanTransformer.java | 44 --- .../options/file/LowercaseTransformer.java | 33 -- .../options/file/NoRenameTransformer.java | 31 -- .../RenameWithDatePatternTransformer.java | 83 ------ .../file/RenameWithDateTransformer.java | 71 ----- .../folder/DeleteChildrenTransformer.java | 44 --- .../folder/DeleteIfEmptyTransformer.java | 52 ---- .../folder/DeleteIfOlderThanTransformer.java | 74 ----- .../exceptions/AbandonBackupException.java | 9 - .../exceptions/FlagsProcessingException.java | 7 - .../metadata/MetadataExtractor.java | 281 ------------------ .../metadata/media/MediaDateExtractor.java | 29 -- .../media/SimpleMediaDateExtractor.java | 28 -- .../metadata/media/XmpMediaDateExtractor.java | 39 --- .../name/DatePatternNameExtractor.java | 27 -- .../metadata/name/NameDateExtractor.java | 16 - .../name/Pattern1NameDateExtractorImpl.java | 23 -- .../name/Pattern2NameDateExtractorImpl.java | 23 -- .../name/Pattern3NameDateExtractorImpl.java | 23 -- .../name/Pattern4NameDateExtractorImpl.java | 23 -- .../name/Pattern5NameDateExtractorImpl.java | 28 -- .../name/Pattern6NameDateExtractorImpl.java | 28 -- .../processor/DifferenceVisitor.java | 62 ---- .../filesecure/processor/FileMetadata.java | 27 -- .../filesecure/processor/Processor.java | 144 --------- .../filesecure/utils/FileOperations.java | 61 ---- .../filesecure/utils/ThrowingTriConsumer.java | 7 - .../utils/json/DateTimeDeserializer.java | 17 -- .../utils/json/GeonamesTimeZone.java | 43 --- .../utils/json/JacksonObjectMapper.java | 65 ---- .../utils/json/PatternDeserializer.java | 17 -- .../utils/json/ZoneIdDeserializer.java | 14 - src/main/java/module-info.java.old | 16 - .../kotlin/fr/rakambda/filesecure/Main.kt | 139 +++++++++ .../filesecure/config/Configuration.kt | 55 ++++ .../fr/rakambda/filesecure/config/Pipeline.kt | 53 ++++ .../filesecure/console/ConsoleHandler.kt | 43 +++ .../rakambda/filesecure/console/PauseGate.kt | 29 ++ .../filesecure/conversion/ConversionTools.kt | 57 ++++ .../filesecure/conversion/Converter.kt | 130 ++++++++ .../filesecure/conversion/CustomFFmpeg.kt | 28 ++ .../filesecure/conversion/MediaProcessor.kt | 187 ++++++++++++ .../filesecure/conversion/ProbeFilter.kt | 31 ++ .../filesecure/conversion/WindowsAffinity.kt | 44 +++ .../filesecure/folder/FolderAction.kt | 113 +++++++ .../filesecure/metadata/FileMetadata.kt | 15 + .../filesecure/metadata/MediaDateExtractor.kt | 44 +++ .../filesecure/metadata/MetadataExtractor.kt | 149 ++++++++++ .../filesecure/metadata/NameDateExtractor.kt | 44 +++ .../filesecure/metadata/TimeZoneResolver.kt | 57 ++++ .../pipeline/DestinationReserver.kt | 49 +++ .../rakambda/filesecure/pipeline/FileItem.kt | 31 ++ .../filesecure/pipeline/FileOperations.kt | 152 ++++++++++ .../filesecure/pipeline/FileScanner.kt | 60 ++++ .../filesecure/pipeline/PipelineContext.kt | 35 +++ .../filesecure/pipeline/PipelineRunner.kt | 248 ++++++++++++++++ .../filesecure/progress/ProgressReporter.kt | 133 +++++++++ .../fr/rakambda/filesecure/stage/Convert.kt | 133 +++++++++ .../rakambda/filesecure/stage/FilterStages.kt | 40 +++ .../rakambda/filesecure/stage/FolderStages.kt | 108 +++++++ .../rakambda/filesecure/stage/RenameStages.kt | 77 +++++ .../fr/rakambda/filesecure/stage/Stage.kt | 82 +++++ .../filesecure/storage/UselessFileStorage.kt | 121 ++++++++ src/main/resources/log4j2.xml | 3 + .../file/FolderCanonicalTransformerTest.java | 41 --- .../file/FolderPerDayTransformerTest.java | 36 --- .../file/FolderPerMonthTransformerTest.java | 36 --- .../file/FolderPerWeekTransformerTest.java | 36 --- .../file/FolderPerYearTransformerTest.java | 36 --- .../FolderUniquePerDayTransformerTest.java | 36 --- .../file/LowercaseTransformerTest.java | 32 -- .../options/file/NoRenameTransformerTest.java | 32 -- .../file/RenameWithDateTransformerTest.java | 39 --- .../folder/DeleteIfEmptyTransformerTest.java | 84 ------ .../filesecure/config/ConfigurationTest.kt | 94 ++++++ .../filesecure/folder/FolderActionTest.kt | 83 ++++++ .../pipeline/ConversionPipelineTest.kt | 222 ++++++++++++++ .../pipeline/DestinationReserverTest.kt | 57 ++++ .../filesecure/pipeline/PipelineRunnerTest.kt | 223 ++++++++++++++ .../filesecure/stage/FilterStagesTest.kt | 43 +++ .../filesecure/stage/PathStagesTest.kt | 119 ++++++++ .../fr/rakambda/filesecure/stage/TestItems.kt | 17 ++ .../filesecure/storage/H2StorageTest.kt | 26 ++ 105 files changed, 3613 insertions(+), 2578 deletions(-) delete mode 100644 lombok.config delete mode 100644 src/main/java/fr/rakambda/filesecure/CLIParameters.java delete mode 100644 src/main/java/fr/rakambda/filesecure/Main.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/Configuration.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/Mapping.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/Operation.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/Rule.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/FileTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/FolderTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/Transformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/IfOlderThanTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDatePatternTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteChildrenTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfOlderThanTransformer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/exceptions/AbandonBackupException.java delete mode 100644 src/main/java/fr/rakambda/filesecure/exceptions/FlagsProcessingException.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/MetadataExtractor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/media/MediaDateExtractor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/media/SimpleMediaDateExtractor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/media/XmpMediaDateExtractor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/DatePatternNameExtractor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/NameDateExtractor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/Pattern1NameDateExtractorImpl.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/Pattern2NameDateExtractorImpl.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/Pattern3NameDateExtractorImpl.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/Pattern4NameDateExtractorImpl.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/Pattern5NameDateExtractorImpl.java delete mode 100644 src/main/java/fr/rakambda/filesecure/metadata/name/Pattern6NameDateExtractorImpl.java delete mode 100644 src/main/java/fr/rakambda/filesecure/processor/DifferenceVisitor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/processor/FileMetadata.java delete mode 100644 src/main/java/fr/rakambda/filesecure/processor/Processor.java delete mode 100644 src/main/java/fr/rakambda/filesecure/utils/FileOperations.java delete mode 100644 src/main/java/fr/rakambda/filesecure/utils/ThrowingTriConsumer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/utils/json/DateTimeDeserializer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/utils/json/GeonamesTimeZone.java delete mode 100644 src/main/java/fr/rakambda/filesecure/utils/json/JacksonObjectMapper.java delete mode 100644 src/main/java/fr/rakambda/filesecure/utils/json/PatternDeserializer.java delete mode 100644 src/main/java/fr/rakambda/filesecure/utils/json/ZoneIdDeserializer.java delete mode 100644 src/main/java/module-info.java.old create mode 100644 src/main/kotlin/fr/rakambda/filesecure/Main.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/config/Configuration.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/config/Pipeline.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/console/ConsoleHandler.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/console/PauseGate.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/conversion/ConversionTools.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/conversion/Converter.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/conversion/CustomFFmpeg.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/conversion/MediaProcessor.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/conversion/ProbeFilter.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/conversion/WindowsAffinity.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/folder/FolderAction.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/metadata/FileMetadata.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/metadata/MediaDateExtractor.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/metadata/MetadataExtractor.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/metadata/NameDateExtractor.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/metadata/TimeZoneResolver.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserver.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/pipeline/FileItem.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/pipeline/FileOperations.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/pipeline/FileScanner.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineContext.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunner.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/progress/ProgressReporter.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/stage/Convert.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/stage/FilterStages.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/stage/FolderStages.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/stage/RenameStages.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/stage/Stage.kt create mode 100644 src/main/kotlin/fr/rakambda/filesecure/storage/UselessFileStorage.kt delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformerTest.java delete mode 100644 src/test/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformerTest.java create mode 100644 src/test/kotlin/fr/rakambda/filesecure/config/ConfigurationTest.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/folder/FolderActionTest.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/pipeline/ConversionPipelineTest.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserverTest.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunnerTest.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/stage/FilterStagesTest.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/stage/PathStagesTest.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/stage/TestItems.kt create mode 100644 src/test/kotlin/fr/rakambda/filesecure/storage/H2StorageTest.kt diff --git a/.editorconfig b/.editorconfig index 7f167a8..eac265f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -260,6 +260,9 @@ ij_java_wrap_comments = false ij_java_wrap_first_method_in_call_chain = false ij_java_wrap_long_lines = false +[*.kt] +indent_style = tab + [{*.ad,*.adoc,*.asciidoc,.asciidoctorconfig}] ij_asciidoc_blank_lines_after_header = 1 ij_asciidoc_blank_lines_keep_after_header = 1 diff --git a/.github/workflows/gradle_ci.yml b/.github/workflows/gradle_ci.yml index 8a53c30..48b982f 100644 --- a/.github/workflows/gradle_ci.yml +++ b/.github/workflows/gradle_ci.yml @@ -42,3 +42,7 @@ jobs: docker-repo: ${{ secrets.DOCKER_REPO }} docker-username: ${{ secrets.DOCKER_USERNAME }} docker-token: ${{ secrets.DOCKER_TOKEN }} + jib-extra-params: | + -Djib.from.image=${{ secrets.DOCKER_REPO }}/rakambda/eclipse-temurin-ffmpeg:9.0-27-jre \ + -Djib.from.auth.username=${{ secrets.DOCKER_USERNAME }} \ + -Djib.from.auth.password=${{ secrets.DOCKER_TOKEN }} diff --git a/README.md b/README.md index ee26f45..3dca386 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,201 @@ # FileSecure -Backup files from a base directory to a target directory. +Moves, backs up and converts files through pipelines of stages. -To work this program needs a configuration file that will be passed as a parameter. The config file file must be in a json format. Here's an example: +Every file found in the input folder of a pipeline goes through its stages in order. A stage can filter the file out, change its destination (folders, name) +or plan its conversion into another format. Once all stages accepted the file, the conversion (if any) runs and the file is copied or moved to its +destination. -```json +Version 13 is a rewrite in Kotlin merging FileSecure and [MediaConverter](https://github.com/RakambdaOrg/MediaConverter). Its configuration format is +new, see [migrating](#migrating-from-previous-versions). + +## Running + +```shell +java -jar FileSecure.jar -c config.json +``` + +| Flag | Description | +|----------------------------|---------------------------------------------------------------------------------------------------------------| +| `-c`, `--config` | **Mandatory**. Path to the configuration file. | +| `--dry` | Dry run: everything is logged, no file is copied, moved, converted or deleted (the `Convert` cache is still updated). | +| `-t`, `--threads` | How many conversions run in parallel. Default 1. | +| `-w`, `--workers` | How many files are processed in parallel (scanning, probing, copying). Default 8. | +| `-ft`, `--ffmpeg-threads` | Value of ffmpeg's `-threads` option for each conversion. | +| `--ffmpeg` | Folder containing the `ffmpeg` executable. Uses the `PATH` if not set. | +| `--ffprobe` | Folder containing the `ffprobe` executable. Uses the `PATH` if not set. | +| `--magick` | ImageMagick executable used for photo conversions. Default `magick`. | +| `--affinity` | Windows only. Mask of the CPU cores conversions run on, e.g. `7` for cores 1, 2 and 3. | +| `--progress/--no-progress` | Show progress bars. Enabled by default when running in a terminal. | +| `--geonames-username` | [geonames.org](https://www.geonames.org) account used to find the time zone of geotagged photos and videos. | + +While running, these commands can be typed in the console: + +* `p`: pause, conversions already running finish but nothing new starts. +* `r`: resume. +* `q`: quit, running conversions are stopped and their temporary files removed. Stopping the process (Ctrl+C, `SIGTERM`) does the same. + +Conversions need [ffmpeg](https://ffmpeg.org) (video and audio) and [ImageMagick](https://imagemagick.org) (photos) to be installed. + +## Configuration + +The configuration is a JSON file (comments and trailing commas are allowed). Unknown keys are rejected to catch typos. + +```json5 { - "rules": [ + "pipelines": [ { + // Back up photos from the phone, sorted by month and renamed with their date + "name": "Phone photos", "mappings": [ - { - "input": "/Input", - "output": "/Output" - } - ] + {"input": "/phone/DCIM", "output": "/backup/Photos"} + ], + "scan": {"skipHidden": true, "excludedFolders": ["\\.thumbnails"]}, + "stages": [ + {"type": "NameFilter", "includes": [".*\\.jpg", ".*\\.mp4"]}, + {"type": "IfOlderThan", "dayOffset": 2}, + {"type": "FolderPerMonth"}, + {"type": "RenameWithDate"} + ], + "operation": "MOVE", + "setFileCreationDate": true, + "folderActions": [{"type": "DeleteIfEmpty"}] }, { - "operation": "move", - "depth": 2, - "filters": [ - ".*\\.png", - ".*\\.jpg" + // Convert videos to HEVC in place, files that can't be converted are left untouched + "name": "Videos", + "mappings": [ + {"input": "/media/Videos", "output": "/media/Videos"} ], - "excludes": [ - ".*\\.ini" + "stages": [ + {"type": "Convert", "processors": ["VIDEO_TO_HEVC_MKV"], "cache": "/data/videos.db"} ], - "mappings": [ - { - "input": "/Input2", - "output": "/Output2" - } - ] + "operation": "MOVE", + "temp": "/data/temp" } ] } ``` -Each backup to do is an object in the array called "mappings". Two fields are mandatory: +### Pipeline + +| Key | Default | Description | +|-----------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `name` | `#index` | Name used in logs. | +| `enabled` | `true` | Disabled pipelines are skipped. | +| `mappings` | | **Mandatory**. List of `{"input": ..., "output": ...}` folders. A file at `input/a/b.txt` has `output/a/b.txt` as initial destination. | +| `scan.maxDepth` | no limit | How many levels of sub-folders are visited, `1` only processes the files directly in `input`. | +| `scan.skipHidden` | `false` | Ignore hidden files and folders. | +| `scan.excludedFolders`| none | Regexes matched against the folder path relative to `input`, using `/` as separator (e.g. `"a/b"`). Matching folders are not visited. | +| `stages` | none | The [stages](#stages), applied in the given order. | +| `operation` | `NONE` | What is done with accepted files: `COPY`, `MOVE`, `MOVE_WITH_COPY` (copy then delete, for moves across file systems) or `NONE` (only logs what would be done). | +| `conflict` | `RENAME` | When the destination already exists: `RENAME` uses the first free name among `name (1).ext`, `name (2).ext`...; `SKIP` leaves the file untouched. | +| `setFileCreationDate` | `false` | Set the creation and modification dates of the destination to the date found in the file name or in its metadata. | +| `temp` | system | Folder where conversions write before their result is moved to its destination. | +| `folderActions` | none | [Actions](#folder-actions) applied to every visited input folder once all files are processed, deepest folders first. | + +Enum values are case-insensitive. A file whose destination is itself (input and output being the same folder) is left untouched, unless it is converted. + +### Stages + +Several stages use **the date of the file**: the date found in its name (e.g. `2023-05-12 10.15.12.jpg`, `IMG_20230512_101512.jpg`), otherwise the +date it was taken from its metadata (EXIF, QuickTime, MP4, XMP), otherwise its last modification date. + +| Type | Parameters | Description | +|-------------------------|--------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `NameFilter` | `includes`, `excludes`: regex lists | Keeps files whose name fully matches one of the `includes` (all if empty) and none of the `excludes`. | +| `IfOlderThan` | `dayOffset`, `minuteOffset` | Keeps files whose date is older than the offset. | +| `FolderPerYear` | | Prefixes the destination with `yyyy/` from the file date. | +| `FolderPerMonth` | | Prefixes the destination with `yyyy/MM/`. | +| `FolderPerWeek` | | Prefixes the destination with `yyyy/MM/w/`, `w` being the week of the month. | +| `FolderPerDay` | | Prefixes the destination with `yyyy/MM/dd/`. | +| `FolderUniquePerDay` | | Prefixes the destination with `yyyy-MM-dd/`. | +| `FolderCanonical` | `elementCount` (mandatory) | Prefixes the destination with one folder per character of its first path element: `going/file` with `3` becomes `g/o/i/going/file`. | +| `Lowercase` | | Lowercases the destination folders (not the file name). | +| `NoRename` | | Restores the original file name. | +| `RenameWithDate` | `format` (`yyyy-MM-dd HH.mm.ss`), `locale` (`en`), `zone`, `filters` | Renames the file with its date, keeping its extension. When `filters` (regexes) is set, only files whose original name matches one of them are renamed. | +| `RenameWithDatePattern` | `regex`, `dateFormat`, `timeFormat` (mandatory), `outputFormat`, `locale`, `zone` | Renames the file with a date read from its current name. `regex` must define the groups `date` and `time`, parsed with `dateFormat` and `timeFormat`. Files not matching are skipped. | +| `Convert` | see below | Plans the conversion of media files. | + +### Convert + +The file is probed with ffprobe and converted by the first processor able to handle it. Its destination extension changes accordingly, so stages placed +after `Convert` see the new extension. The conversion only runs once all stages accepted the file and its destination is free, so with +`"conflict": "SKIP"` already converted files are not converted again. + +With `MOVE` or `MOVE_WITH_COPY` the source is sent to the trash (deleted where there is none) once the converted file is in place. When the destination +is the source itself, the source is replaced. A pipeline can contain only one `Convert` stage. + +| Key | Default | Description | +|--------------|----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `processors` | `VIDEO_TO_HEVC_MKV`, `MP3_TO_AAC`, `TIFF_TO_JPG` | Processors tried in order. | +| `extensions` | `mp4`, `mov`, `mkv`, `avi`, `tiff`, `mp3`, `m4a`, `m4v`, `ts`, `m2ts`, `mts`, `heic`, `jpg`, `raf` | Only files with these extensions are probed. | +| `filters` | none | Probe filters, all must pass: `{"type": "LengthMoreThan", "value": seconds}`, `{"type": "LengthLessThan", "value": seconds}`. | +| `unhandled` | `SKIP` | What happens to files that won't be converted: `SKIP` leaves them untouched, `PASS` lets them continue through the pipeline unchanged (e.g. to back them up too). | +| `cache` | none | H2 database file remembering files that can't be converted, so they aren't probed again on the next runs (until they are modified). | + +| Processor | Converts | Into | +|---------------------|-------------------------------------------------------------|---------------------------------------------------------------------------------| +| `VIDEO_TO_HEVC_MKV` | `H264`, `VP9`, `WMV3`, `MPEG2` videos, HEVC with another tag or container | `HEVC` + `AAC` in `MKV` (HEVC sources are only remuxed) | +| `VIDEO_TO_HEVC_MP4` | same | `HEVC` + `AAC` in `MP4` | +| `VIDEO_TO_AV1_MKV` | `H264`, `HEVC`, `VP9`, `WMV3`, `MPEG2` videos | `AV1` + `OPUS` in `MKV` | +| `MP3_TO_AAC` | `MP3` audio files | `AAC` (`.m4a`) | +| `AUDIO_TO_OPUS` | `MP3`, `AAC` audio files | `OPUS` | +| `TIFF_TO_JPG` | `TIFF` images | `JPEG` | +| `PHOTO_TO_AVIF` | `heic`, `tiff`, `jpg`, `raf` files | `AVIF` | +| `PHOTO_TO_JXL` | `heic`, `tiff`, `jpg`, `raf` files | `JXL` | + +### Folder actions + +| Type | Parameters | Description | +|---------------------|----------------------------------------------|----------------------------------------------------------------------------------------------| +| `DeleteIfEmpty` | `depth` (`1`) | Deletes empty folders at least `depth` levels below the input folder (the input is level 0). | +| `DeleteIfOlderThan` | `dayOffset`, `minuteOffset`, `depth` (`1`) | Same, for empty folders last modified before the offset. | +| `DeleteChildren` | `names` | Deletes the given files in every folder, e.g. `["Thumbs.db", "desktop.ini"]`. | + +## Migrating from previous versions + +### FileSecure 12 + +Each `rule` becomes a pipeline, with the same `mappings` and `operation`. + +| FileSecure 12 | FileSecure 13 | +|---------------------------------------|------------------------------------------------------------------------------------| +| `filters`, `excludes` | A `NameFilter` stage with `includes`, `excludes` | +| `folderExcludes` | `scan.excludedFolders` | +| `maxDepth` | `scan.maxDepth` | +| `skipIfAlreadyExists: true` | `"conflict": "SKIP"` | +| `fileTransformers` | `stages`, without the `Transformer` suffix (`FolderPerYearTransformer` → `FolderPerYear`), `name` becomes `type` | +| `folderTransformers` | `folderActions`, same renaming | +| `setFileCreationDate` | unchanged | + +Transformers used to run in an unspecified order (their priority was never applied). Stages run in the order they are written, list them in the order +you expect: filters first, then folders, then renaming. + +### MediaConverter -* input -* output +Each `conversion` becomes a pipeline with a `Convert` stage. -The input folder is backed up recursively up to a certain depth. +| MediaConverter | FileSecure 13 | +|--------------------------------|-----------------------------------------------------------------------------------| +| `input`, `output` | A mapping | +| `deleteInput: true` (default) | `"operation": "MOVE"` | +| `deleteInput: false` | `"operation": "COPY", "conflict": "SKIP"` | +| `database` | `cache` of the `Convert` stage, existing database files can be reused | +| `temp` | `temp` | +| `excluded` | `scan.excludedFolders` (regexes, so escape special characters such as `.`) | +| `processors`, `extensions` | Same keys in the `Convert` stage | +| `filters` (`name`, `value`) | `filters` of the `Convert` stage, `name` becomes `type` | +| hidden files were ignored | `"scan": {"skipHidden": true}` | +| `-ft`, `--ffmpegThreads` | `-ft`, `--ffmpeg-threads` | -Other fields are optionals: +### Behaviour changes -* operation: define the strategy to copy files. Can be "copy", "move" or "none" (just displays in standard output). Default is none. -* depth: The number of sub-folders to visit. -* filters: Regexes to apply to determine which files to keep. By default, all files are kept. -* excludes: Regexes to apply to determine which files to not process. By default, none are excluded. -* folderExcludes: Regexes to apply to determine which folders to not process. This is a relative path from the `input` folder, separated by `/`. By default, none are excluded. -* skipIfAlreadyExists: If destination file already exists, skip it instead of generating a unique name for it. -* fileTransformers: Transformations to apply when moving the file. - * TODO describe options -* folderTransformers: Transformations to apply to input folders after operation has been applied. - * TODO describe options +* Only regular files are processed. Previously, folders beyond `maxDepth` were copied or moved as if they were files. +* `DeleteIfEmpty` never deletes the input folder itself with its default depth (it used to, unlike `DeleteIfOlderThan`). +* Files without extension no longer abort the whole mapping when their destination already exists. +* With MediaConverter, an input file could be left in the temporary folder if its output already existed. The source is now only removed once the + converted file is in place. +* The Windows CPU affinity was never applied by MediaConverter (a process id was used as a process handle), it now is. +* An error on one file or folder is logged and the others are still processed. +* Pipelines run one after another (MediaConverter ran its conversions in parallel), `--threads` conversions run in parallel within a pipeline. diff --git a/build.gradle.kts b/build.gradle.kts index bf292f5..827f03a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,10 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer +import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { idea - java application + alias(libs.plugins.kotlin) alias(libs.plugins.shadow) alias(libs.plugins.jib) } @@ -11,58 +12,51 @@ plugins { group = "fr.rakambda" description = "FileSecure" +val javaVersion = providers.gradleProperty("javaVersion").get() + dependencies { implementation(platform(libs.jacksonBom)) implementation(platform(libs.log4j2Bom)) + implementation(libs.coroutines) + implementation(libs.slf4j) implementation(libs.bundles.log4j2) + implementation(libs.kotlinLogging) - implementation(libs.unirest) - implementation(libs.pointLocation) - implementation(libs.metadataExtractor) - - implementation(libs.picocli) + implementation(libs.clikt) implementation(libs.bundles.jackson) - implementation(libs.jSpecify) - compileOnly(libs.lombok) + implementation(libs.pointLocation) + implementation(libs.metadataExtractor) - annotationProcessor(libs.lombok) + implementation(libs.jaffree) + implementation(libs.progressbar) + implementation(libs.h2) + implementation(libs.bundles.jna) - implementation(platform(libs.junitBom)) + testImplementation(platform(libs.junitBom)) testImplementation(libs.bundles.junit) testRuntimeOnly(libs.junitPlatformLauncher) - testImplementation(libs.bundles.assertj) - testImplementation(libs.bundles.mockito) - - testCompileOnly(libs.lombok) - - testAnnotationProcessor(libs.lombok) } repositories { mavenCentral() } -tasks { - processResources { - expand(project.properties) - } - - compileJava { - val moduleName: String by project - inputs.property("moduleName", moduleName) - - options.encoding = "UTF-8" - options.isDeprecation = true - } +java { + sourceCompatibility = JavaVersion.toVersion(javaVersion) + targetCompatibility = JavaVersion.toVersion(javaVersion) +} - compileTestJava { - options.encoding = "UTF-8" +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget(javaVersion)) } +} +tasks { test { useJUnitPlatform() } @@ -84,16 +78,7 @@ tasks { } application { - val moduleName: String by project - val className: String by project - - mainModule.set(moduleName) - mainClass.set(className) -} - -java { - sourceCompatibility = JavaVersion.VERSION_26 - targetCompatibility = JavaVersion.VERSION_26 + mainClass.set(providers.gradleProperty("className")) } jib { diff --git a/gradle.properties b/gradle.properties index d39f502..6d88781 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ -version=12.1 -moduleName=fr.rakambda.filesecure -className=fr.rakambda.filesecure.Main +version=13.0 +javaVersion=26 +className=fr.rakambda.filesecure.MainKt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5b365aa..49f61f0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,52 +1,59 @@ [versions] +kotlin-version = "2.4.21" +coroutines-version = "1.11.0" slf4j-version = "2.0.20" log4j2-version = "2.26.1" -picocli-version = "4.7.7" +kotlin-logging-version = "8.0.4" +clikt-version = "5.1.0" jackson-version = "2.22.3" -jspecify-version = "1.0.1" metadata-extractor-version = "2.21.0" pointlocation6709-version = "5.0.1" -unirest-version = "4.0.0-RC2" -lombok-version = "1.18.48" +jaffree-version = "2024.08.29" +progressbar-version = "0.10.2" +h2-version = "2.5.252" +jna-version = "5.19.1" junitVersion = "6.1.3" assertj-core-version = "3.27.7" -mockito-version = "5.24.0" shadow-version = "9.6.1" jib-version = "3.5.4" [libraries] +coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines-version" } slf4j = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j-version" } -log4j2Bom = { group = "org.apache.logging.log4j", name = "log4j-bom", version.ref = "log4j2-version"} +log4j2Bom = { group = "org.apache.logging.log4j", name = "log4j-bom", version.ref = "log4j2-version" } log4j2-core = { group = "org.apache.logging.log4j", name = "log4j-core" } log4j2-slf4j = { group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl" } log4j2-json = { group = "org.apache.logging.log4j", name = "log4j-layout-template-json" } -picocli = { group = "info.picocli", name = "picocli", version.ref = "picocli-version" } -jacksonBom = { group = "com.fasterxml.jackson", name = "jackson-bom", version.ref = "jackson-version"} +kotlinLogging = { group = "io.github.oshai", name = "kotlin-logging-jvm", version.ref = "kotlin-logging-version" } +clikt = { group = "com.github.ajalt.clikt", name = "clikt", version.ref = "clikt-version" } +jacksonBom = { group = "com.fasterxml.jackson", name = "jackson-bom", version.ref = "jackson-version" } jackson-core = { group = "com.fasterxml.jackson.core", name = "jackson-core" } jackson-annotations = { group = "com.fasterxml.jackson.core", name = "jackson-annotations" } jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind" } -jSpecify = { group = "org.jspecify", name = "jspecify", version.ref = "jspecify-version" } +jackson-kotlin = { group = "com.fasterxml.jackson.module", name = "jackson-module-kotlin" } metadataExtractor = { group = "com.drewnoakes", name = "metadata-extractor", version.ref = "metadata-extractor-version" } pointLocation = { group = "us.fatehi", name = "pointlocation6709", version.ref = "pointlocation6709-version" } -unirest = { group = "com.konghq", name = "unirest-java", version.ref = "unirest-version" } -lombok = { group = "org.projectlombok", name = "lombok", version.ref = "lombok-version" } -junitBom = { group = "org.junit", name = "junit-bom", version.ref = "junitVersion"} +jaffree = { group = "com.github.kokorin.jaffree", name = "jaffree", version.ref = "jaffree-version" } +progressbar = { group = "me.tongfei", name = "progressbar", version.ref = "progressbar-version" } +h2 = { group = "com.h2database", name = "h2", version.ref = "h2-version" } +jna = { group = "net.java.dev.jna", name = "jna", version.ref = "jna-version" } +jna-platform = { group = "net.java.dev.jna", name = "jna-platform", version.ref = "jna-version" } +junitBom = { group = "org.junit", name = "junit-bom", version.ref = "junitVersion" } junit-api = { group = "org.junit.jupiter", name = "junit-jupiter-api" } junit-params = { group = "org.junit.jupiter", name = "junit-jupiter-params" } junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine" } junitPlatformLauncher = { group = "org.junit.platform", name = "junit-platform-launcher" } assertj-core = { group = "org.assertj", name = "assertj-core", version.ref = "assertj-core-version" } -mockito-core = { group = "org.mockito", name = "mockito-core", version.ref = "mockito-version" } -mockito-junit = { group = "org.mockito", name = "mockito-junit-jupiter", version.ref = "mockito-version" } [bundles] -jackson = ["jackson-core", "jackson-annotations", "jackson-databind"] +jackson = ["jackson-core", "jackson-annotations", "jackson-databind", "jackson-kotlin"] log4j2 = ["log4j2-core", "log4j2-slf4j", "log4j2-json"] +jna = ["jna", "jna-platform"] junit = ["junit-api", "junit-params", "junit-engine"] -mockito = ["mockito-core", "mockito-junit"] assertj = ["assertj-core"] [plugins] +kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin-version" } shadow = { id = "com.gradleup.shadow", version.ref = "shadow-version" } jib = { id = "com.google.cloud.tools.jib", version.ref = "jib-version" } diff --git a/lombok.config b/lombok.config deleted file mode 100644 index 6aa51d7..0000000 --- a/lombok.config +++ /dev/null @@ -1,2 +0,0 @@ -# This file is generated by the 'io.freefair.lombok' Gradle plugin -config.stopBubbling = true diff --git a/src/main/java/fr/rakambda/filesecure/CLIParameters.java b/src/main/java/fr/rakambda/filesecure/CLIParameters.java deleted file mode 100644 index 3c79b96..0000000 --- a/src/main/java/fr/rakambda/filesecure/CLIParameters.java +++ /dev/null @@ -1,24 +0,0 @@ -package fr.rakambda.filesecure; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import picocli.CommandLine; - -@NoArgsConstructor -@Getter -@CommandLine.Command(name = "filesecure", mixinStandardHelpOptions = true) -public class CLIParameters{ - @CommandLine.Option(names = { - "-c", - "--config" - }, - description = "The path to the configuration file", - required = true) - public String configurationFile; - @CommandLine.Option(names = { - "--dry" - }, - description = "Performs a dry run") - public boolean dryRun; -} - diff --git a/src/main/java/fr/rakambda/filesecure/Main.java b/src/main/java/fr/rakambda/filesecure/Main.java deleted file mode 100644 index 30cedb1..0000000 --- a/src/main/java/fr/rakambda/filesecure/Main.java +++ /dev/null @@ -1,55 +0,0 @@ -package fr.rakambda.filesecure; - -import fr.rakambda.filesecure.config.Configuration; -import fr.rakambda.filesecure.metadata.MetadataExtractor; -import fr.rakambda.filesecure.processor.Processor; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.extern.slf4j.Slf4j; -import picocli.CommandLine; -import java.nio.file.Path; -import java.nio.file.Paths; - -@Slf4j -public class Main{ - public static void main(String[] args){ - var cli = parseArgs(args); - - var configurationOptional = Configuration.loadConfiguration(Path.of(cli.getConfigurationFile())); - if(configurationOptional.isEmpty()){ - log.error("Failed to load configuration from {}", cli.getConfigurationFile()); - System.exit(1); - return; - } - - var configuration = configurationOptional.get(); - var metadataExtractor = new MetadataExtractor(); - var fileOperations = new FileOperations(cli.isDryRun()); - - for(var rule : configuration.getRules()){ - for(var mapping : rule.getMappings()){ - try{ - new Processor(rule, mapping, metadataExtractor, fileOperations).process(); - } - catch(Exception e){ - log.error("Failed to run processor", e); - } - } - } - } - - private static CLIParameters parseArgs(String[] args){ - var parameters = new CLIParameters(); - var cli = new CommandLine(parameters); - cli.registerConverter(Path.class, Paths::get); - cli.setUnmatchedArgumentsAllowed(true); - try{ - cli.parseArgs(args); - } - catch(CommandLine.ParameterException e){ - log.error("Failed to parse arguments", e); - cli.usage(System.out); - System.exit(1); - } - return parameters; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/Configuration.java b/src/main/java/fr/rakambda/filesecure/config/Configuration.java deleted file mode 100644 index c8c61b1..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Configuration.java +++ /dev/null @@ -1,57 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - -@Slf4j -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -@AllArgsConstructor -public class Configuration{ - private static final ObjectReader objectReader; - - @JsonProperty - private List rules; - - @NonNull - public static Optional loadConfiguration(@NonNull Path path){ - if(Files.isRegularFile(path)){ - try(var fis = Files.newBufferedReader(path)){ - return Optional.ofNullable(objectReader.readValue(fis)); - } - catch(IOException e){ - log.error("Failed to read settings in {}", path, e); - } - } - return Optional.empty(); - } - - static{ - var mapper = new ObjectMapper(); - mapper.setVisibility(mapper.getSerializationConfig() - .getDefaultVisibilityChecker() - .withFieldVisibility(JsonAutoDetect.Visibility.ANY) - .withGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); - mapper.enable(SerializationFeature.INDENT_OUTPUT); - objectReader = mapper.readerFor(Configuration.class); - } -} \ No newline at end of file diff --git a/src/main/java/fr/rakambda/filesecure/config/Mapping.java b/src/main/java/fr/rakambda/filesecure/config/Mapping.java deleted file mode 100644 index 1eb3221..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Mapping.java +++ /dev/null @@ -1,25 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.ext.NioPathDeserializer; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import java.nio.file.Path; - -@Slf4j -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -public class Mapping{ - @JsonProperty - @JsonDeserialize(using = NioPathDeserializer.class) - private Path input; - @JsonProperty - @JsonDeserialize(using = NioPathDeserializer.class) - private Path output; -} diff --git a/src/main/java/fr/rakambda/filesecure/config/Operation.java b/src/main/java/fr/rakambda/filesecure/config/Operation.java deleted file mode 100644 index cc8b0bd..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Operation.java +++ /dev/null @@ -1,34 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import fr.rakambda.filesecure.utils.FileOperations; -import fr.rakambda.filesecure.utils.ThrowingTriConsumer; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; - -@Getter -@RequiredArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public enum Operation{ - COPY(FileOperations::copy), - MOVE(FileOperations::move), - MOVE_WITH_COPY(FileOperations::moveWithCopy), - NONE((op, in, out) -> {}); - - private final ThrowingTriConsumer processor; - - @JsonCreator - public static Operation getByName(@NonNull String name){ - return valueOf(name); - } - - @SuppressWarnings("SameReturnValue") - public static Operation getDefault(){ - return NONE; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/Rule.java b/src/main/java/fr/rakambda/filesecure/config/Rule.java deleted file mode 100644 index 2a6f503..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Rule.java +++ /dev/null @@ -1,52 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.json.PatternDeserializer; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.regex.Pattern; - -@Slf4j -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -public class Rule{ - @JsonProperty - private Operation operation = Operation.getDefault(); - @JsonProperty - private List mappings = new ArrayList<>(); - - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private Set filters = new HashSet<>(); - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private Set excludes = new HashSet<>(); - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private Set folderExcludes = new HashSet<>(); - - @JsonProperty - private Set fileTransformers = new HashSet<>(); - @JsonProperty - private Set folderTransformers = new HashSet<>(); - - @JsonProperty - private boolean skipIfAlreadyExists = false; - @JsonProperty - private boolean setFileCreationDate = false; - - @JsonProperty - private int maxDepth = Integer.MAX_VALUE; -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/FileTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/FileTransformer.java deleted file mode 100644 index c62a827..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/FileTransformer.java +++ /dev/null @@ -1,43 +0,0 @@ -package fr.rakambda.filesecure.config.options; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import fr.rakambda.filesecure.config.options.file.FolderCanonicalTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerDayTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerMonthTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerWeekTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerYearTransformer; -import fr.rakambda.filesecure.config.options.file.FolderUniquePerDayTransformer; -import fr.rakambda.filesecure.config.options.file.IfOlderThanTransformer; -import fr.rakambda.filesecure.config.options.file.LowercaseTransformer; -import fr.rakambda.filesecure.config.options.file.NoRenameTransformer; -import fr.rakambda.filesecure.config.options.file.RenameWithDatePatternTransformer; -import fr.rakambda.filesecure.config.options.file.RenameWithDateTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -@JsonSubTypes(value = { - @JsonSubTypes.Type(value = FolderCanonicalTransformer.class, name = "FolderCanonicalTransformer"), - @JsonSubTypes.Type(value = FolderPerDayTransformer.class, name = "FolderPerDayTransformer"), - @JsonSubTypes.Type(value = FolderPerWeekTransformer.class, name = "FolderPerWeekTransformer"), - @JsonSubTypes.Type(value = FolderPerMonthTransformer.class, name = "FolderPerMonthTransformer"), - @JsonSubTypes.Type(value = FolderPerYearTransformer.class, name = "FolderPerYearTransformer"), - @JsonSubTypes.Type(value = FolderUniquePerDayTransformer.class, name = "FolderUniquePerDayTransformer"), - @JsonSubTypes.Type(value = IfOlderThanTransformer.class, name = "IfOlderThanTransformer"), - @JsonSubTypes.Type(value = LowercaseTransformer.class, name = "LowercaseTransformer"), - @JsonSubTypes.Type(value = NoRenameTransformer.class, name = "NoRenameTransformer"), - @JsonSubTypes.Type(value = RenameWithDateTransformer.class, name = "RenameWithDateTransformer"), - @JsonSubTypes.Type(value = RenameWithDatePatternTransformer.class, name = "RenameWithDatePatternTransformer"), -}) -public interface FileTransformer extends Transformer{ - @NonNull - Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException; -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/FolderTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/FolderTransformer.java deleted file mode 100644 index 35c4c05..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/FolderTransformer.java +++ /dev/null @@ -1,24 +0,0 @@ -package fr.rakambda.filesecure.config.options; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import fr.rakambda.filesecure.config.options.folder.DeleteChildrenTransformer; -import fr.rakambda.filesecure.config.options.folder.DeleteIfEmptyTransformer; -import fr.rakambda.filesecure.config.options.folder.DeleteIfOlderThanTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -@JsonSubTypes(value = { - @JsonSubTypes.Type(value = DeleteIfOlderThanTransformer.class, name = "DeleteIfOlderThanTransformer"), - @JsonSubTypes.Type(value = DeleteIfEmptyTransformer.class, name = "DeleteIfEmptyTransformer"), - @JsonSubTypes.Type(value = DeleteChildrenTransformer.class, name = "DeleteChildrenTransformer"), -}) -public interface FolderTransformer extends Transformer{ - void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder); -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/Transformer.java b/src/main/java/fr/rakambda/filesecure/config/options/Transformer.java deleted file mode 100644 index 711af0b..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/Transformer.java +++ /dev/null @@ -1,17 +0,0 @@ -package fr.rakambda.filesecure.config.options; - -import org.jspecify.annotations.NonNull; - -public interface Transformer extends Comparable{ - @Override - default int compareTo(@NonNull Transformer o){ - return Integer.compare(getPriority(), o.getPriority()); - } - - /** - * Get the priority of this option. - * - * @return The priority, lower will be executed first. - */ - int getPriority(); -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformer.java deleted file mode 100644 index d636128..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformer.java +++ /dev/null @@ -1,71 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderCanonicalTransformer") -@NoArgsConstructor -@AllArgsConstructor -public class FolderCanonicalTransformer implements FileTransformer{ - private static final String UNKNOWN_PART = "_"; - private static final Set FORBIDDEN_PART = Set.of(".", "/", "\\"); - - @JsonProperty(required = true) - private int elementCount = Integer.MAX_VALUE; - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var relative = baseOutput.relativize(currentOutput); - var base = getBase(relative); - var canonical = getCanonicalPath(base.getFileName().toString()); - return Optional.of(baseOutput.resolve(canonical).resolve(relative)); - } - - @NonNull - private Path getBase(@NonNull Path path){ - var parent = path.getParent(); - if(Objects.isNull(parent) || Objects.equals(parent, path)){ - return path; - } - return getBase(parent); - } - - private Path getCanonicalPath(String name){ - var path = Paths.get(""); - var max = Math.min(elementCount, name.length()); - for(var i = 0; i < max; i++){ - var part = String.valueOf(name.charAt(i)).toLowerCase(); - if(FORBIDDEN_PART.contains(part)){ - part = UNKNOWN_PART; - } - path = path.resolve(part); - } - while(path.getNameCount() < elementCount){ - path = path.resolve(UNKNOWN_PART); - } - return path; - } - - @Override - public int getPriority(){ - return 12; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformer.java deleted file mode 100644 index e335b57..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformer.java +++ /dev/null @@ -1,37 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerDayTransformer") -@NoArgsConstructor -public class FolderPerDayTransformer implements FileTransformer{ - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - var month = "%02d".formatted(date.getMonthValue()); - var day = "%02d".formatted(date.getDayOfMonth()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(month).resolve(day).resolve(relative)); - } - - @Override - public int getPriority(){ - return 11; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformer.java deleted file mode 100644 index 95e02ed..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformer.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerMonthTransformer") -@NoArgsConstructor -public class FolderPerMonthTransformer implements FileTransformer{ - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - var month = "%02d".formatted(date.getMonthValue()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(month).resolve(relative)); - } - - @Override - public int getPriority(){ - return 13; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformer.java deleted file mode 100644 index 3af6ede..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformer.java +++ /dev/null @@ -1,38 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.temporal.ChronoField; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerWeekTransformer") -@NoArgsConstructor -public class FolderPerWeekTransformer implements FileTransformer{ - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - var month = "%02d".formatted(date.getMonthValue()); - var week = "%1d".formatted(date.get(ChronoField.ALIGNED_WEEK_OF_MONTH)); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(month).resolve(week).resolve(relative)); - } - - @Override - public int getPriority(){ - return 12; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformer.java deleted file mode 100644 index 3a364f0..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformer.java +++ /dev/null @@ -1,35 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerYearTransformer") -@NoArgsConstructor -public class FolderPerYearTransformer implements FileTransformer { - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(relative)); - } - - @Override - public int getPriority(){ - return 14; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformer.java deleted file mode 100644 index df017d4..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformer.java +++ /dev/null @@ -1,35 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderUniquePerDayTransformer") -@NoArgsConstructor -public class FolderUniquePerDayTransformer implements FileTransformer { - @Override - public int getPriority(){ - return 10; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var dateFormat = "%4d-%02d-%02d".formatted(date.getYear(), date.getMonthValue(), date.getDayOfMonth()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(dateFormat).resolve(relative)); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/IfOlderThanTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/IfOlderThanTransformer.java deleted file mode 100644 index 35cb549..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/IfOlderThanTransformer.java +++ /dev/null @@ -1,44 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.ZonedDateTime; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("IfOlderThanTransformer") -@NoArgsConstructor -@AllArgsConstructor -public class IfOlderThanTransformer implements FileTransformer { - @JsonProperty - private int dayOffset = 0; - @JsonProperty - private int minuteOffset = 0; - - @Override - public int getPriority(){ - return 0; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - if(date.isAfter(ZonedDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset))){ - throw new AbandonBackupException(sourceFile); - } - return Optional.empty(); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformer.java deleted file mode 100644 index 7207e13..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformer.java +++ /dev/null @@ -1,33 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Locale; -import java.util.Optional; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("LowercaseTransformer") -@NoArgsConstructor -public class LowercaseTransformer implements FileTransformer { - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var relative = baseOutput.relativize(currentOutput.getParent()); - return Optional.of(baseOutput.resolve(relative.toString().toLowerCase(Locale.ROOT)).resolve(currentOutput.getFileName())); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformer.java deleted file mode 100644 index 0cd7091..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformer.java +++ /dev/null @@ -1,31 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("NoRenameTransformer") -@NoArgsConstructor -public class NoRenameTransformer implements FileTransformer { - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - return Optional.of(currentOutput.getParent().resolve(sourceFile.getFileName())); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDatePatternTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDatePatternTransformer.java deleted file mode 100644 index a1eb995..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDatePatternTransformer.java +++ /dev/null @@ -1,83 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import fr.rakambda.filesecure.utils.json.PatternDeserializer; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.Optional; -import java.util.regex.Pattern; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("RenameWithDatePatternTransformer") -@Log4j2 -@NoArgsConstructor -public class RenameWithDatePatternTransformer implements FileTransformer{ - @JsonProperty - private String outputFormat = "yyyy-MM-dd HH.mm.ss"; - @JsonProperty(required = true) - @JsonDeserialize(using = PatternDeserializer.class) - private Pattern regex; - @JsonProperty(required = true) - private String dateFormat; - @JsonProperty(required = true) - private String timeFormat; - @JsonProperty - private String locale = "en"; - @JsonProperty - private String zone; - - @Override - public int getPriority(){ - return 0; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = extractDate(currentOutput.getFileName().toString()) - .orElseThrow(() -> new AbandonBackupException(sourceFile)); - - var zoneId = Optional.ofNullable(zone).map(ZoneId::of).orElse(ZoneId.systemDefault()); - var formatter = DateTimeFormatter.ofPattern(outputFormat, new Locale.Builder().setLanguage(locale).build()).withZone(zoneId); - - var fileName = currentOutput.getFileName().toString(); - var dotIndex = fileName.lastIndexOf('.'); - var extension = dotIndex < 0 ? "" : fileName.substring(dotIndex); - - return Optional.of(currentOutput.getParent().resolve(formatter.format(date) + extension)); - } - - @NonNull - private Optional extractDate(@NonNull String name){ - try{ - var matcher = regex.matcher(name); - if(!matcher.matches()){ - return Optional.empty(); - } - var dateString = "%s %s".formatted(matcher.group("date"), matcher.group("time")); - - var zoneId = Optional.ofNullable(zone).map(ZoneId::of).orElse(ZoneId.systemDefault()); - var formatter = DateTimeFormatter.ofPattern("%s %s".formatted(dateFormat, timeFormat), new Locale.Builder().setLanguage(locale).build()).withZone(zoneId); - - return Optional.of(ZonedDateTime.parse(dateString, formatter)); - } - catch(Exception e){ - log.error("Failed to decode date from value `{}` and format `{} {}`", name, dateFormat, timeFormat, e); - return Optional.empty(); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformer.java deleted file mode 100644 index 5a00db1..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformer.java +++ /dev/null @@ -1,71 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import fr.rakambda.filesecure.utils.json.PatternDeserializer; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Objects; -import java.util.Optional; -import java.util.regex.Pattern; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("RenameWithDateTransformer") -@Log4j2 -@NoArgsConstructor -public class RenameWithDateTransformer implements FileTransformer{ - @JsonProperty - private String format = "yyyy-MM-dd HH.mm.ss"; - @JsonProperty - private String locale = "en"; - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private List filters = new LinkedList<>(); - @JsonProperty - @Setter(AccessLevel.PACKAGE) - private String zone; - - @Override - public int getPriority(){ - return 0; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - if(Objects.nonNull(filters) && !filters.isEmpty()){ - var matched = this.filters.stream() - .map(Pattern::asMatchPredicate) - .anyMatch(pred -> pred.test(sourceFile.getFileName().toString())); - if(!matched){ - log.debug("Not applying transformer {} because file name {} didn't match any filters", this, sourceFile); - return Optional.empty(); - } - } - var zoneId = Optional.ofNullable(zone).map(ZoneId::of).orElse(ZoneId.systemDefault()); - var formatter = DateTimeFormatter.ofPattern(format, new Locale.Builder().setLanguage(locale).build()).withZone(zoneId); - - var date = metadata.getDate(); - var fileName = currentOutput.getFileName().toString(); - var dotIndex = fileName.lastIndexOf('.'); - var extension = dotIndex < 0 ? "" : fileName.substring(dotIndex); - - return Optional.of(currentOutput.getParent().resolve(formatter.format(date) + extension)); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteChildrenTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteChildrenTransformer.java deleted file mode 100644 index d8cb575..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteChildrenTransformer.java +++ /dev/null @@ -1,44 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("DeleteChildrenTransformer") -@NoArgsConstructor -public class DeleteChildrenTransformer implements FolderTransformer{ - @JsonProperty - private List names = new ArrayList<>(); - - @Override - public int getPriority(){ - return 10; - } - - @Override - public void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder){ - try{ - for(var child : names){ - var childPath = folder.resolve(child); - if(fileOperations.deleteIfExists(childPath)){ - log.info("Deleted child file {}", childPath); - } - } - } - catch(Exception e){ - log.error("Failed to delete children in {}", folder, e); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformer.java deleted file mode 100644 index c7a9acf..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Path; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("DeleteIfEmptyTransformer") -@NoArgsConstructor -@AllArgsConstructor -public class DeleteIfEmptyTransformer implements FolderTransformer{ - @JsonProperty - private int depth = 1; - - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @Override - public void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder){ - try{ - var relativeDepth = baseFolder.relativize(folder).getNameCount(); - if(relativeDepth < depth){ - return; - } - - if(Files.list(folder).toList().isEmpty()){ - log.info("Deleting folder {} because it is empty", folder); - fileOperations.delete(folder); - } - } - catch(DirectoryNotEmptyException e){ - log.error("Failed to delete folder {}, not empty", folder); - } - catch(Exception e){ - log.error("Failed to determine if {} should be deleted, it will not be by default", folder, e); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfOlderThanTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfOlderThanTransformer.java deleted file mode 100644 index 40365b6..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfOlderThanTransformer.java +++ /dev/null @@ -1,74 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.ZoneId; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("DeleteIfOlderThanTransformer") -@NoArgsConstructor -public class DeleteIfOlderThanTransformer implements FolderTransformer{ - @JsonProperty - private int dayOffset = 0; - @JsonProperty - private int minuteOffset = 0; - @JsonProperty - private int depth = 1; - - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @Override - public void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder){ - try{ - var relativize = baseFolder.relativize(folder); - var relativeDepth = relativize.getNameCount(); - - if(baseFolder.equals(folder)){ - relativeDepth = 0; - } - - if(relativeDepth < depth){ - return; - } - - var folderTime = Files.getLastModifiedTime(folder); - var date = LocalDateTime.ofInstant(folderTime.toInstant(), ZoneId.systemDefault()); - if(date.isBefore(LocalDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset))){ - delete(fileOperations, folder); - } - } - catch(DirectoryNotEmptyException e){ - log.error("Failed to delete folder {}, not empty", folder); - } - catch(Exception e){ - log.error("Failed to determine if {} should be deleted, it will not be by default", folder, e); - } - } - - private void delete(@NonNull FileOperations fileOperations, @NonNull Path folder) throws IOException{ - if(Files.list(folder).toList().isEmpty()){ - log.info("Deleting folder {} because it is more than {} days old and empty", folder, dayOffset); - fileOperations.delete(folder); - } - else{ - log.debug("Tried deleting folder {} because it is more than {} days old but wasn't empty", folder, dayOffset); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/exceptions/AbandonBackupException.java b/src/main/java/fr/rakambda/filesecure/exceptions/AbandonBackupException.java deleted file mode 100644 index bc75f82..0000000 --- a/src/main/java/fr/rakambda/filesecure/exceptions/AbandonBackupException.java +++ /dev/null @@ -1,9 +0,0 @@ -package fr.rakambda.filesecure.exceptions; - -import java.nio.file.Path; - -public class AbandonBackupException extends Exception{ - public AbandonBackupException(Path originFile){ - super("Abandoned the backup of " + originFile); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/exceptions/FlagsProcessingException.java b/src/main/java/fr/rakambda/filesecure/exceptions/FlagsProcessingException.java deleted file mode 100644 index 25e748c..0000000 --- a/src/main/java/fr/rakambda/filesecure/exceptions/FlagsProcessingException.java +++ /dev/null @@ -1,7 +0,0 @@ -package fr.rakambda.filesecure.exceptions; - -public class FlagsProcessingException extends RuntimeException{ - public FlagsProcessingException(String message){ - super(message); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/MetadataExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/MetadataExtractor.java deleted file mode 100644 index c77c8e5..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/MetadataExtractor.java +++ /dev/null @@ -1,281 +0,0 @@ -package fr.rakambda.filesecure.metadata; - -import com.drew.imaging.ImageMetadataReader; -import com.drew.metadata.Directory; -import com.drew.metadata.Metadata; -import com.drew.metadata.exif.ExifSubIFDDirectory; -import com.drew.metadata.exif.GpsDirectory; -import com.drew.metadata.mov.QuickTimeDirectory; -import com.drew.metadata.mov.metadata.QuickTimeMetadataDirectory; -import com.drew.metadata.mp4.Mp4Directory; -import com.drew.metadata.xmp.XmpDirectory; -import fr.rakambda.filesecure.metadata.media.MediaDateExtractor; -import fr.rakambda.filesecure.metadata.media.SimpleMediaDateExtractor; -import fr.rakambda.filesecure.metadata.media.XmpMediaDateExtractor; -import fr.rakambda.filesecure.metadata.name.NameDateExtractor; -import fr.rakambda.filesecure.metadata.name.Pattern1NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern2NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern3NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern4NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern5NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern6NameDateExtractorImpl; -import fr.rakambda.filesecure.processor.FileMetadata; -import fr.rakambda.filesecure.utils.json.GeonamesTimeZone; -import kong.unirest.GenericType; -import kong.unirest.Unirest; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; -import us.fatehi.pointlocation6709.Angle; -import us.fatehi.pointlocation6709.Latitude; -import us.fatehi.pointlocation6709.Longitude; -import us.fatehi.pointlocation6709.PointLocation; -import us.fatehi.pointlocation6709.parse.PointLocationParser; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.ParseException; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.TimeZone; -import java.util.regex.Pattern; - -@Slf4j -public class MetadataExtractor{ - private final List dateFormats; - private final List> mediaDateExtractors; - - public MetadataExtractor(){ - dateFormats = new ArrayList<>(); - dateFormats.add(new Pattern1NameDateExtractorImpl()); - dateFormats.add(new Pattern2NameDateExtractorImpl()); - dateFormats.add(new Pattern3NameDateExtractorImpl()); - dateFormats.add(new Pattern4NameDateExtractorImpl()); - dateFormats.add(new Pattern5NameDateExtractorImpl()); - dateFormats.add(new Pattern6NameDateExtractorImpl()); - - mediaDateExtractors = new ArrayList<>(); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(QuickTimeMetadataDirectory.class, QuickTimeMetadataDirectory.TAG_CREATION_DATE)); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(QuickTimeDirectory.class, QuickTimeDirectory.TAG_CREATION_TIME)); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(Mp4Directory.class, Mp4Directory.TAG_CREATION_TIME)); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(ExifSubIFDDirectory.class, ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL)); - mediaDateExtractors.add(new XmpMediaDateExtractor()); - } - - @NonNull - public FileMetadata getMetadata(@NonNull Path path) throws IOException{ - return new FileMetadata(getDateFromMetadata(path), getCreationDate(path), getDateFromFileName(path)); - } - - @NonNull - private ZonedDateTime getCreationDate(@NonNull Path path) throws IOException{ - var attr = Files.readAttributes(path, BasicFileAttributes.class); - var date = Instant.ofEpochMilli(attr.lastModifiedTime().toMillis()).atZone(ZoneId.systemDefault()); - - if(date.getYear() <= 1970){ - date = date.withYear(LocalDateTime.now().getYear()); - } - - return date; - } - - @Nullable - private ZonedDateTime getDateFromMetadata(@NonNull Path path){ - try{ - var metadata = ImageMetadataReader.readMetadata(path.toFile()); - if(Objects.isNull(metadata)){ - return null; - } - - var zoneID = getZoneIdFromMetadata(metadata).orElse(ZoneId.systemDefault()); - var timeZone = TimeZone.getTimeZone(zoneID); - - for(var dataExtractor : mediaDateExtractors){ - for(var directory : metadata.getDirectoriesOfType(dataExtractor.getKlass())){ - if(Objects.nonNull(directory)){ - try{ - var result = runDirectoryExtractor(dataExtractor, directory, timeZone); - if(Objects.nonNull(result)){ - return result; - } - } - catch(ParseException e){ - log.warn("Invalid year with metadata directory {} for file {}", dataExtractor.getKlass().getName(), path); - } - catch(Exception e){ - log.error("Error processing metadata directory {} for {} => {}", dataExtractor.getKlass().getName(), path, e.getMessage()); - } - } - } - } - } - catch(Exception e){ - log.error("Error getting date from metadata for {} => {}", path, e.getMessage()); - } - return null; - } - - @Nullable - private ZonedDateTime runDirectoryExtractor(@NonNull MediaDateExtractor dataExtractor, @NonNull Directory directory, @NonNull TimeZone timeZone) throws ParseException{ - log.debug("Trying {}", dataExtractor.getKlass().getName()); - - var takenDateOptional = dataExtractor.parse(directory, timeZone); - if(takenDateOptional.isEmpty()){ - return null; - } - - var takenDate = takenDateOptional.get(); - log.debug("Matched metadata directory {}", directory); - if(takenDate.getYear() < 1970){ - throw new ParseException("Invalid year", 0); - } - return takenDate; - } - - @Nullable - private ZonedDateTime getDateFromFileName(@NonNull Path path){ - var fileName = path.getFileName().toString(); - - for(var nameDateExtractor : dateFormats){ - try{ - log.debug("Trying name extractor `{}`", nameDateExtractor); - var dateOptional = nameDateExtractor.parse(fileName); - if(dateOptional.isEmpty()){ - continue; - } - - var date = dateOptional.get(); - if(date.getYear() < 1970){ - throw new ParseException("Invalid year", 0); - } - - log.debug("Matched date format for {}", path); - return date; - } - catch(ParseException e){ - log.warn("Invalid year with used format for file {}", path); - } - catch(DateTimeParseException ignored){ - } - catch(Exception e){ - log.error("Error using format {} => {}", nameDateExtractor, e.getMessage()); - } - } - - return null; - } - - @NonNull - private Optional getZoneIdFromMetadata(@NonNull Metadata metadata){ - try{ - for(var gpsDirectory : metadata.getDirectoriesOfType(GpsDirectory.class)){ - var location = gpsDirectory.getGeoLocation(); - if(Objects.isNull(location)){ - continue; - } - var zoneId = getZoneID(location.getLatitude(), location.getLongitude()); - if(zoneId.isPresent()){ - return zoneId; - } - } - for(var quickTimeMetadataDirectory : metadata.getDirectoriesOfType(QuickTimeMetadataDirectory.class)){ - var repr = quickTimeMetadataDirectory.getString(0x050D); - if(Objects.nonNull(repr) && !repr.isBlank()){ - var location = PointLocationParser.parsePointLocation(repr); - var zoneId = getZoneID(location.getLatitude().getDegrees(), location.getLongitude().getDegrees()); - if(zoneId.isPresent()){ - return zoneId; - } - } - } - for(var xmpDirectory : metadata.getDirectoriesOfType(XmpDirectory.class)){ - var xmpValues = xmpDirectory.getXmpProperties(); - if(xmpValues.containsKey("exif:GPSLatitude") && xmpValues.containsKey("exif:GPSLongitude")){ - var zoneId = getAngle(xmpValues.get("exif:GPSLatitude")) - .flatMap(lat -> getAngle(xmpValues.get("exif:GPSLongitude")) - .flatMap(lon -> { - var location = new PointLocation(new Latitude(lat), new Longitude(lon)); - return getZoneID(location.getLatitude().getDegrees(), location.getLongitude().getDegrees()); - })); - if(zoneId.isPresent()){ - return zoneId; - } - } - } - } - catch(Exception e){ - log.warn("Error getting GPS infos", e); - } - return Optional.empty(); - } - - /** - * Get the zoneID of a geolocation. - * - * @param latitude The latitude. - * @param longitude The longitude. - * - * @return The zoneID corresponding to this location. - */ - @NonNull - private static Optional getZoneID(double latitude, double longitude){ - try{ - var request = Unirest.get("http://api.geonames.org/timezoneJSON") - .queryString("lat", latitude) - .queryString("lng", longitude) - .queryString("username", "mrcraftcod") - .asObject(new GenericType(){}); - if(request.isSuccess()){ - var geonamesTimeZone = request.getBody(); - return Optional.ofNullable(geonamesTimeZone.getTimezoneId()); - } - } - catch(Exception e){ - log.error("Error getting zoneID for coordinates {};{}", latitude, longitude, e); - } - return Optional.empty(); - } - - /** - * Convert a N/E/S/W coordinate to an angle one. - * - * @param s The coordinate to convert. - * - * @return The angle. - */ - @NonNull - private static Optional getAngle(@NonNull String s){ - var pattern = Pattern.compile("(\\d{1,3}),(\\d{1,2})\\.(\\d+)([NESW])"); - var matcher = pattern.matcher(s); - if(matcher.matches()){ - var angle = Integer.parseInt(matcher.group(1)) - + (Integer.parseInt(matcher.group(2)) / 60.0) - + (Double.parseDouble("0." + matcher.group(3)) / 60.0); - angle *= getMultiplicand(matcher.group(4)); - return Optional.of(Angle.fromDegrees(angle)); - } - return Optional.empty(); - } - - /** - * Get the sign of the angle depending of it's position (N/E/S/W). - * - * @param group The group (N/E/S/W). - * - * @return The sign of the angle. - */ - private static double getMultiplicand(@NonNull String group){ - return switch(group){ - case "W", "S" -> -1; - default -> 1; - }; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/media/MediaDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/media/MediaDateExtractor.java deleted file mode 100644 index cb42e32..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/media/MediaDateExtractor.java +++ /dev/null @@ -1,29 +0,0 @@ -package fr.rakambda.filesecure.metadata.media; - -import com.drew.metadata.Directory; -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.util.Optional; -import java.util.TimeZone; - -/** - * @param The directory type. - */ -public interface MediaDateExtractor{ - /** - * Get the date from the given {@link Directory}. - * - * @param directory The directory to get from. - * @param tz The timezone of the date. - * - * @return The date or null if not found. - */ - @NonNull Optional parse(@NonNull Directory directory, @NonNull TimeZone tz); - - /** - * Get the class of the directory. - * - * @return The directory's class. - */ - @NonNull Class getKlass(); -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/media/SimpleMediaDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/media/SimpleMediaDateExtractor.java deleted file mode 100644 index 9f2bd61..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/media/SimpleMediaDateExtractor.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.rakambda.filesecure.metadata.media; - -import com.drew.metadata.Directory; -import lombok.Getter; -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.util.Date; -import java.util.Optional; -import java.util.TimeZone; - -public class SimpleMediaDateExtractor implements MediaDateExtractor{ - private final int tag; - @Getter - private final Class klass; - - public SimpleMediaDateExtractor(@NonNull Class klass, int tag){ - this.klass = klass; - this.tag = tag; - } - - @NonNull - @Override - public Optional parse(@NonNull Directory directory, @NonNull TimeZone tz){ - return Optional.ofNullable(directory.getDate(tag, tz)) - .map(Date::toInstant) - .map(date -> date.atZone(tz.toZoneId())); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/media/XmpMediaDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/media/XmpMediaDateExtractor.java deleted file mode 100644 index 911ffcb..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/media/XmpMediaDateExtractor.java +++ /dev/null @@ -1,39 +0,0 @@ -package fr.rakambda.filesecure.metadata.media; - -import com.drew.metadata.Directory; -import com.drew.metadata.xmp.XmpDirectory; -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Optional; -import java.util.TimeZone; - -public class XmpMediaDateExtractor implements MediaDateExtractor{ - private final List keys = List.of("xmp:CreateDate", "photoshop:DateCreated"); - private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); - - @NonNull - @Override - public Optional parse(@NonNull Directory directory, @NonNull TimeZone tz){ - var xmpDirectory = (XmpDirectory) directory; - var values = xmpDirectory.getXmpProperties(); - for(var key : keys){ - if(values.containsKey(key)){ - try{ - return Optional.ofNullable(values.get(key)) - .map(date -> ZonedDateTime.parse(date, dateTimeFormatter.withZone(tz.toZoneId()))); - } - catch(Exception ignored){ - } - } - } - return Optional.empty(); - } - - @NonNull - @Override - public Class getKlass(){ - return XmpDirectory.class; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/DatePatternNameExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/name/DatePatternNameExtractor.java deleted file mode 100644 index ca0fb78..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/DatePatternNameExtractor.java +++ /dev/null @@ -1,27 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Optional; -import java.util.regex.Pattern; - -public abstract class DatePatternNameExtractor implements NameDateExtractor{ - @Override - @NonNull - public Optional parse(@NonNull String name){ - var matcher = getPattern().matcher(name); - if(matcher.find()){ - return Optional.of(ZonedDateTime.parse(matcher.group(getCaptureGroupIndex()), getFormatter())); - } - return Optional.empty(); - } - - protected int getCaptureGroupIndex(){ - return 0; - } - - protected abstract DateTimeFormatter getFormatter(); - - protected abstract Pattern getPattern(); -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/NameDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/name/NameDateExtractor.java deleted file mode 100644 index e97889b..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/NameDateExtractor.java +++ /dev/null @@ -1,16 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.util.Optional; - -public interface NameDateExtractor{ - /** - * Get the date from the filename - * - * @param name The filename - * - * @return The date if found. - */ - @NonNull Optional parse(@NonNull String name); -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern1NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern1NameDateExtractorImpl.java deleted file mode 100644 index 7a73849..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern1NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern1NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2} \\d{2}\\.\\d{2}\\.\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH.mm.ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern2NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern2NameDateExtractorImpl.java deleted file mode 100644 index 6a3ac05..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern2NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern2NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2} \\d{2}-\\d{2}-\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern3NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern3NameDateExtractorImpl.java deleted file mode 100644 index 0b6fcd1..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern3NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern3NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern4NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern4NameDateExtractorImpl.java deleted file mode 100644 index d9a8094..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern4NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern4NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern5NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern5NameDateExtractorImpl.java deleted file mode 100644 index b4bd7ed..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern5NameDateExtractorImpl.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern5NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("IMG_(\\d{8}_\\d{6})"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } - - @Override - protected int getCaptureGroupIndex(){ - return 1; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern6NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern6NameDateExtractorImpl.java deleted file mode 100644 index 5e2afb2..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern6NameDateExtractorImpl.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern6NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("Video-(\\d{8}-\\d{6})"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } - - @Override - protected int getCaptureGroupIndex(){ - return 1; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/processor/DifferenceVisitor.java b/src/main/java/fr/rakambda/filesecure/processor/DifferenceVisitor.java deleted file mode 100644 index 209b28f..0000000 --- a/src/main/java/fr/rakambda/filesecure/processor/DifferenceVisitor.java +++ /dev/null @@ -1,62 +0,0 @@ -package fr.rakambda.filesecure.processor; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Collection; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.regex.Pattern; - -@RequiredArgsConstructor -public class DifferenceVisitor implements FileVisitor{ - private final Path rootFolder; - private final Collection includePatterns; - private final Collection excludePatterns; - private final Collection excludeFolderPatterns; - - @Getter - private final Collection paths = new ConcurrentLinkedDeque<>(); - @Getter - private final Collection folders = new ConcurrentLinkedDeque<>(); - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs){ - var relative = rootFolder.relativize(dir).toString().replace("\\", "/"); - if(excludeFolderPatterns.stream().anyMatch(f -> f.matcher(relative).matches())){ - return FileVisitResult.SKIP_SUBTREE; - } - - folders.add(dir); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs){ - var fileName = file.getFileName().toString(); - - if(excludePatterns.stream().anyMatch(f -> f.matcher(fileName).matches())){ - return FileVisitResult.CONTINUE; - } - if(!includePatterns.isEmpty() && includePatterns.stream().noneMatch(f -> f.matcher(fileName).matches())){ - return FileVisitResult.CONTINUE; - } - - paths.add(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException{ - throw exc; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc){ - return FileVisitResult.CONTINUE; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/processor/FileMetadata.java b/src/main/java/fr/rakambda/filesecure/processor/FileMetadata.java deleted file mode 100644 index 682f84c..0000000 --- a/src/main/java/fr/rakambda/filesecure/processor/FileMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -package fr.rakambda.filesecure.processor; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; -import java.time.ZonedDateTime; -import java.util.Optional; - -@Getter -@RequiredArgsConstructor -public class FileMetadata{ - @Nullable - private final ZonedDateTime takenDate; - @Nullable - private final ZonedDateTime createdDate; - @Nullable - private final ZonedDateTime nameDate; - - @NonNull - public ZonedDateTime getDate(){ - return Optional.ofNullable(nameDate) - .or(() -> Optional.ofNullable(takenDate)) - .or(() -> Optional.ofNullable(createdDate)) - .orElseGet(ZonedDateTime::now); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/processor/Processor.java b/src/main/java/fr/rakambda/filesecure/processor/Processor.java deleted file mode 100644 index 7a64a6d..0000000 --- a/src/main/java/fr/rakambda/filesecure/processor/Processor.java +++ /dev/null @@ -1,144 +0,0 @@ -package fr.rakambda.filesecure.processor; - -import fr.rakambda.filesecure.config.Mapping; -import fr.rakambda.filesecure.config.Rule; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.exceptions.FlagsProcessingException; -import fr.rakambda.filesecure.metadata.MetadataExtractor; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.util.Comparator; -import java.util.Optional; -import java.util.Set; - -@RequiredArgsConstructor -@Slf4j -public class Processor{ - @NonNull - private final Rule rule; - private final Mapping mapping; - private final MetadataExtractor metadataExtractor; - private final FileOperations fileOperations; - - public void process() throws IOException{ - if(!Files.exists(mapping.getInput())){ - log.warn("Input folder {} doesn't exist, skipping mapping", mapping.getInput()); - return; - } - - log.info("Processing mapping ({}) {} ==> {}", rule.getOperation().name(), mapping.getInput(), mapping.getOutput()); - log.debug("Building differences..."); - var differenceVisitor = new DifferenceVisitor(mapping.getInput(), rule.getFilters(), rule.getExcludes(), rule.getFolderExcludes()); - Files.walkFileTree(mapping.getInput(), Set.of(), rule.getMaxDepth(), differenceVisitor); - - log.debug("Applying strategy"); - differenceVisitor.getPaths().forEach(this::applyStrategy); - - log.debug("Cleaning up"); - differenceVisitor.getFolders().stream() - .sorted(Comparator.comparingInt(Path::getNameCount).reversed()) - .forEach(this::cleanup); - } - - private void cleanup(@NonNull Path path){ - for(var option : rule.getFolderTransformers()){ - option.apply(fileOperations, path, mapping.getInput()); - } - } - - @SneakyThrows(IOException.class) - private void applyStrategy(@NonNull Path path) throws FlagsProcessingException{ - var relativeInput = mapping.getInput().relativize(path); - var exactDestination = mapping.getOutput().resolve(relativeInput); - var destination = exactDestination; - var metadata = metadataExtractor.getMetadata(path); - - for(var option : rule.getFileTransformers()){ - try{ - var newDestination = option.apply(path, exactDestination, mapping.getOutput(), destination, metadata); - if(newDestination.isPresent()){ - log.debug("File destination {} changed to {} by {}", destination, newDestination.get(), option.getClass().getSimpleName()); - destination = newDestination.get(); - } - } - catch(AbandonBackupException e){ - log.debug("Transformer {} abandoned backup for file {}", option.getClass().getSimpleName(), path); - return; - } - catch(Exception e){ - log.error("Error applying transformer {} to file {}", option.getClass(), path, e); - throw new FlagsProcessingException("Error applying transformer to file " + path); - } - } - - if(rule.isSkipIfAlreadyExists() && Files.exists(destination)){ - return; - } - var nonExistingDestination = generateUniqueName(destination); - - log.info("{} file {} to {}", rule.getOperation().name(), path, nonExistingDestination); - try{ - rule.getOperation().getProcessor().accept(fileOperations, path, nonExistingDestination); - if(rule.isSetFileCreationDate()){ - Optional.ofNullable(metadata.getNameDate()) - .or(() -> Optional.ofNullable(metadata.getTakenDate())) - .ifPresent(date -> { - try{ - var instant = date.toInstant(); - var fileTime = FileTime.from(instant); - log.info("Setting creation date for {} to {}", exactDestination, instant); - - var attributes = Files.getFileAttributeView(nonExistingDestination, BasicFileAttributeView.class); - BasicFileAttributes attributesRead = attributes.readAttributes(); - attributes.setTimes(fileTime, attributesRead.lastAccessTime(), fileTime); - } - catch(IOException e){ - log.warn("Failed to set file creation date to obtained metadata", e); - } - }); - } - } - catch(Exception e){ - log.warn("Error applying operation {} on file {}", rule.getOperation(), path, e); - } - } - - @NonNull - private Path generateUniqueName(@NonNull Path sourcePath){ - try{ - if(!Files.exists(sourcePath)){ - return sourcePath; - } - - var fileName = sourcePath.getFileName().toString(); - - var extIndex = fileName.lastIndexOf("."); - var prefix = fileName.substring(0, extIndex); - var ext = fileName.substring(extIndex); - - var i = 0; - var finalName = fileName; - do{ - var newName = "%s (%d)%s".formatted(prefix, ++i, ext); - log.debug("File '{}' already exists in target, trying with suffix {}", sourcePath, i); - finalName = newName; - } - while(Files.exists(sourcePath.getParent().resolve(finalName))); - - return sourcePath.getParent().resolve(finalName); - } - catch(Exception e){ - log.error("Failed to generate unique name", e); - throw new RuntimeException("Failed to generate unique name", e); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/FileOperations.java b/src/main/java/fr/rakambda/filesecure/utils/FileOperations.java deleted file mode 100644 index c995e32..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/FileOperations.java +++ /dev/null @@ -1,61 +0,0 @@ -package fr.rakambda.filesecure.utils; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; - -@Slf4j -@RequiredArgsConstructor -public class FileOperations{ - private final boolean dryRun; - - public void copy(@NonNull Path in, @NonNull Path out) throws IOException{ - if(dryRun){ - return; - } - createDirectories(out.getParent()); - Files.copy(in, out, StandardCopyOption.COPY_ATTRIBUTES); - } - - public void move(@NonNull Path in, @NonNull Path out) throws IOException{ - if(dryRun){ - return; - } - createDirectories(out.getParent()); - Files.move(in, out); - } - - public void moveWithCopy(@NonNull Path in, @NonNull Path out) throws IOException{ - if(dryRun){ - return; - } - - copy(in, out); - Files.delete(in); - } - - public void delete(Path path) throws IOException{ - if(dryRun){ - return; - } - Files.delete(path); - } - - public boolean deleteIfExists(Path path) throws IOException{ - if(dryRun){ - return false; - } - return Files.deleteIfExists(path); - } - - private void createDirectories(Path path) throws IOException{ - Files.createDirectories(path); - if(!Files.isDirectory(path)){ - throw new IOException("Destination folder %s already exists as a file".formatted(path)); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/ThrowingTriConsumer.java b/src/main/java/fr/rakambda/filesecure/utils/ThrowingTriConsumer.java deleted file mode 100644 index 56aab7f..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/ThrowingTriConsumer.java +++ /dev/null @@ -1,7 +0,0 @@ -package fr.rakambda.filesecure.utils; - -import java.io.IOException; - -public interface ThrowingTriConsumer{ - void accept(T t, U u, V v) throws IOException; -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/DateTimeDeserializer.java b/src/main/java/fr/rakambda/filesecure/utils/json/DateTimeDeserializer.java deleted file mode 100644 index 9f267e5..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/DateTimeDeserializer.java +++ /dev/null @@ -1,17 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -public class DateTimeDeserializer extends JsonDeserializer{ - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); - - @Override - public LocalDateTime deserialize(JsonParser jsonParser, DeserializationContext context) throws IOException{ - return LocalDateTime.parse(jsonParser.getValueAsString(), FORMATTER); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/GeonamesTimeZone.java b/src/main/java/fr/rakambda/filesecure/utils/json/GeonamesTimeZone.java deleted file mode 100644 index 0889cfe..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/GeonamesTimeZone.java +++ /dev/null @@ -1,43 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import lombok.Getter; -import lombok.NoArgsConstructor; -import java.time.LocalDateTime; -import java.time.ZoneId; - -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -public class GeonamesTimeZone{ - @JsonProperty("lng") - private double longitude; - @JsonProperty("lat") - private double latitude; - @JsonProperty("sunrise") - @JsonDeserialize(using = DateTimeDeserializer.class) - private LocalDateTime sunrise; - @JsonProperty("sunset") - @JsonDeserialize(using = DateTimeDeserializer.class) - private LocalDateTime sunset; - @JsonProperty("time") - @JsonDeserialize(using = DateTimeDeserializer.class) - private LocalDateTime time; - @JsonProperty("gmtOffset") - private int gmtOffset; - @JsonProperty("rawOffset") - private int rawOffset; - @JsonProperty("dstOffset") - private int dstOffset; - @JsonProperty("timezoneId") - @JsonDeserialize(using = ZoneIdDeserializer.class) - private ZoneId timezoneId; - @JsonProperty("countryCode") - private String countryCode; - @JsonProperty("countryName") - private String countryName; -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/JacksonObjectMapper.java b/src/main/java/fr/rakambda/filesecure/utils/json/JacksonObjectMapper.java deleted file mode 100644 index ebe92c8..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/JacksonObjectMapper.java +++ /dev/null @@ -1,65 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.json.JsonMapper; -import kong.unirest.GenericType; -import kong.unirest.ObjectMapper; -import java.io.IOException; -import java.lang.reflect.Type; -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.ANY; -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; -import static com.fasterxml.jackson.annotation.PropertyAccessor.CREATOR; -import static com.fasterxml.jackson.annotation.PropertyAccessor.FIELD; -import static com.fasterxml.jackson.annotation.PropertyAccessor.GETTER; -import static com.fasterxml.jackson.annotation.PropertyAccessor.SETTER; -import static com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS; -import static com.fasterxml.jackson.core.json.JsonReadFeature.ALLOW_TRAILING_COMMA; -import static com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS; - -public class JacksonObjectMapper implements ObjectMapper{ - private final com.fasterxml.jackson.databind.ObjectMapper mapper; - - public JacksonObjectMapper(){ - mapper = JsonMapper.builder() - .enable(ALLOW_TRAILING_COMMA) - .enable(ALLOW_COMMENTS) - .visibility(FIELD, ANY) - .visibility(GETTER, NONE) - .visibility(SETTER, NONE) - .visibility(CREATOR, NONE) - .enable(ACCEPT_CASE_INSENSITIVE_ENUMS) - .build(); - } - - public T readValue(String value, Class valueType){ - try{ - return mapper.readValue(value, valueType); - } - catch(IOException var4){ - throw new RuntimeException(var4); - } - } - - public T readValue(String value, GenericType genericType){ - try{ - return mapper.readValue(value, new TypeReference(){ - public Type getType(){ - return genericType.getType(); - } - }); - } - catch(IOException var4){ - throw new RuntimeException(var4); - } - } - - public String writeValue(Object value){ - try{ - return mapper.writeValueAsString(value); - } - catch(JsonProcessingException var3){ - throw new RuntimeException(var3); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/PatternDeserializer.java b/src/main/java/fr/rakambda/filesecure/utils/json/PatternDeserializer.java deleted file mode 100644 index 06ea6fe..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/PatternDeserializer.java +++ /dev/null @@ -1,17 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.util.regex.Pattern; - -public class PatternDeserializer extends JsonDeserializer{ - @Override - public Pattern deserialize(@NonNull JsonParser jsonParser, @NonNull DeserializationContext deserializationContext) throws IOException{ - return Pattern.compile(jsonParser.getValueAsString()); - } -} - - diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/ZoneIdDeserializer.java b/src/main/java/fr/rakambda/filesecure/utils/json/ZoneIdDeserializer.java deleted file mode 100644 index b24ef82..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/ZoneIdDeserializer.java +++ /dev/null @@ -1,14 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import java.time.ZoneId; - -public class ZoneIdDeserializer extends JsonDeserializer{ - @Override - public ZoneId deserialize(JsonParser jsonParser, DeserializationContext context) throws IOException{ - return ZoneId.of(jsonParser.getValueAsString()); - } -} diff --git a/src/main/java/module-info.java.old b/src/main/java/module-info.java.old deleted file mode 100644 index b3b7454..0000000 --- a/src/main/java/module-info.java.old +++ /dev/null @@ -1,16 +0,0 @@ -open module fr.raksrinana.filesecure { - requires unirest.java; - requires metadata.extractor; - requires pointlocation6709; - requires com.fasterxml.jackson.core; - requires com.fasterxml.jackson.annotation; - requires com.fasterxml.jackson.databind; - requires org.slf4j; - requires org.apache.logging.log4j; - requires org.apache.logging.log4j.core; - requires org.apache.logging.log4j.slf4j; - requires java.scripting; - requires info.picocli; - requires static lombok; - requires static org.jetbrains.annotations; -} \ No newline at end of file diff --git a/src/main/kotlin/fr/rakambda/filesecure/Main.kt b/src/main/kotlin/fr/rakambda/filesecure/Main.kt new file mode 100644 index 0000000..4b1285c --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/Main.kt @@ -0,0 +1,139 @@ +package fr.rakambda.filesecure + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.Context +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.core.main +import com.github.ajalt.clikt.parameters.options.check +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.types.int +import com.github.ajalt.clikt.parameters.types.long +import com.github.ajalt.clikt.parameters.types.path +import fr.rakambda.filesecure.config.Configuration +import fr.rakambda.filesecure.console.ConsoleHandler +import fr.rakambda.filesecure.console.PauseGate +import fr.rakambda.filesecure.conversion.ConversionTools +import fr.rakambda.filesecure.conversion.WindowsAffinity +import fr.rakambda.filesecure.metadata.GeonamesTimeZoneResolver +import fr.rakambda.filesecure.metadata.MetadataExtractor +import fr.rakambda.filesecure.pipeline.FileOperations +import fr.rakambda.filesecure.pipeline.PipelineRunner +import fr.rakambda.filesecure.pipeline.RunEnvironment +import fr.rakambda.filesecure.progress.ConsoleProgressReporter +import fr.rakambda.filesecure.progress.NoOpProgressReporter +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.time.Duration.Companion.seconds + +// Lazy so the startup message setting in main is applied before kotlin-logging initializes +private val log by lazy { KotlinLogging.logger {} } + +class FileSecureCommand : CliktCommand(name = "filesecure"){ + private val configurationFile by option("-c", "--config", help = "The path to the configuration file") + .path(mustExist = true, canBeDir = false, mustBeReadable = true) + .required() + private val dryRun by option("--dry", help = "Performs a dry run, nothing is written") + .flag() + private val threads by option("-t", "--threads", help = "How many conversions are performed in parallel") + .int() + .default(1) + .check("must be >= 1") { it >= 1 } + private val workers by option("-w", "--workers", help = "How many files are processed in parallel (scan, probe, copy)") + .int() + .default(8) + .check("must be >= 1") { it >= 1 } + private val ffmpegThreads by option("-ft", "--ffmpeg-threads", help = "Value of ffmpeg's -threads option for each conversion") + .int() + .check("must be >= 0") { it >= 0 } + private val ffmpegDirectory by option("--ffmpeg", help = "Folder containing the ffmpeg executable, uses the PATH if not set") + .path(mustExist = true, canBeFile = false) + private val ffprobeDirectory by option("--ffprobe", help = "Folder containing the ffprobe executable, uses the PATH if not set") + .path(mustExist = true, canBeFile = false) + private val magick by option("--magick", help = "ImageMagick executable used for photo conversions") + .default("magick") + private val affinityMask by option("--affinity", help = "Windows only, mask of the CPU cores conversions can run on, e.g. 7 for cores 1, 2 and 3") + .long() + .check("must be > 0") { it > 0 } + private val progress by option("--progress", help = "Show progress bars, enabled by default when running in a terminal") + .flag("--no-progress", default = System.console()?.isTerminal == true) + private val geonamesUsername by option("--geonames-username", help = "geonames.org account used to find the time zone of geotagged media") + .default("mrcraftcod") + + override fun help(context: Context): String = "Moves, backs up and converts files through configurable pipelines." + + override fun run(){ + val configuration = try{ + Configuration.load(configurationFile) + } + catch(e: Exception){ + log.error(e) { "Failed to load configuration from $configurationFile" } + throw ProgramResult(1) + } + + val pauseGate = PauseGate() + val affinity = affinityMask?.let(::WindowsAffinity) + val progressReporter = if(progress) ConsoleProgressReporter() else NoOpProgressReporter + progressReporter.use { + val environment = RunEnvironment( + fileOperations = FileOperations(dryRun), + metadataExtractor = MetadataExtractor(GeonamesTimeZoneResolver(geonamesUsername)), + tools = ConversionTools(ffmpegDirectory, ffprobeDirectory, magick, ffmpegThreads, affinity?.let { it::apply }), + conversionThreads = threads, + workers = workers, + pauseGate = pauseGate, + progress = progressReporter, + ) + runPipelines(configuration, environment, pauseGate) + } + } + + private fun runPipelines(configuration: Configuration, environment: RunEnvironment, pauseGate: PauseGate) = runBlocking { + val runner = PipelineRunner(environment) + val job = launch(Dispatchers.Default) { + configuration.pipelines.forEachIndexed { index, pipeline -> + val name = pipeline.displayName(index) + if(!pipeline.enabled){ + log.info { "Pipeline $name is disabled" } + return@forEachIndexed + } + try{ + runner.run(pipeline, name) + } + catch(e: CancellationException){ + throw e + } + catch(e: Exception){ + log.error(e) { "Failed to run pipeline $name" } + } + } + } + + ConsoleHandler(System.`in`, pauseGate) { job.cancel() }.start() + val shutdownHook = Thread { + job.cancel() + // Give running conversions the time to be stopped and their temporary files removed + runBlocking { withTimeoutOrNull(30.seconds) { job.join() } } + } + Runtime.getRuntime().addShutdownHook(shutdownHook) + + job.join() + try{ + Runtime.getRuntime().removeShutdownHook(shutdownHook) + } + catch(_: IllegalStateException){ + // Already shutting down + } + } +} + +fun main(args: Array){ + System.setProperty("kotlin-logging.logStartupMessage", "false") + FileSecureCommand().main(args) +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/config/Configuration.kt b/src/main/kotlin/fr/rakambda/filesecure/config/Configuration.kt new file mode 100644 index 0000000..41589b3 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/config/Configuration.kt @@ -0,0 +1,55 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.core.json.JsonReadFeature +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.MapperFeature +import com.fasterxml.jackson.databind.json.JsonMapper +import com.fasterxml.jackson.module.kotlin.kotlinModule +import com.fasterxml.jackson.module.kotlin.readValue +import fr.rakambda.filesecure.stage.Convert +import java.io.Reader +import java.nio.file.Files +import java.nio.file.Path + +data class Configuration( + val pipelines: List = emptyList(), +){ + /** + * Checks constraints the JSON schema alone can't express. + * + * @throws InvalidConfigurationException If the configuration can't be run. + */ + fun validate(){ + pipelines.forEachIndexed { index, pipeline -> + val name = pipeline.displayName(index) + if(pipeline.mappings.isEmpty()){ + throw InvalidConfigurationException("Pipeline '$name' has no mappings") + } + if(pipeline.stages.count { it is Convert } > 1){ + throw InvalidConfigurationException("Pipeline '$name' has more than one Convert stage") + } + if(pipeline.scan.maxDepth < 0){ + throw InvalidConfigurationException("Pipeline '$name' has a negative maxDepth") + } + } + } + + companion object{ + private val mapper = JsonMapper.builder() + .addModule(kotlinModule()) + .enable(JsonReadFeature.ALLOW_JAVA_COMMENTS) + .enable(JsonReadFeature.ALLOW_TRAILING_COMMA) + .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) + .enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) + .build() + + fun load(path: Path): Configuration = Files.newBufferedReader(path).use { parse(it) } + + fun parse(reader: Reader): Configuration = mapper.readValue(reader).also { it.validate() } + + fun parse(json: String): Configuration = parse(json.reader()) + } +} + +class InvalidConfigurationException(message: String) : RuntimeException(message) diff --git a/src/main/kotlin/fr/rakambda/filesecure/config/Pipeline.kt b/src/main/kotlin/fr/rakambda/filesecure/config/Pipeline.kt new file mode 100644 index 0000000..f766227 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/config/Pipeline.kt @@ -0,0 +1,53 @@ +package fr.rakambda.filesecure.config + +import fr.rakambda.filesecure.folder.FolderAction +import fr.rakambda.filesecure.stage.Stage +import java.nio.file.Path +import java.util.regex.Pattern + +/** + * A sequence of stages applied to every file found in the inputs of its mappings. + */ +data class Pipeline( + val name: String? = null, + val enabled: Boolean = true, + val mappings: List = emptyList(), + val scan: ScanOptions = ScanOptions(), + val stages: List = emptyList(), + val operation: Operation = Operation.NONE, + val conflict: ConflictStrategy = ConflictStrategy.RENAME, + val setFileCreationDate: Boolean = false, + val temp: Path? = null, + val folderActions: List = emptyList(), +){ + fun displayName(index: Int): String = name ?: "#${index + 1}" +} + +data class Mapping( + val input: Path, + val output: Path, +) + +data class ScanOptions( + val maxDepth: Int = Int.MAX_VALUE, + val skipHidden: Boolean = false, + /** + * Regexes matched against the folder path relative to the input, using `/` as separator. + */ + val excludedFolders: List = emptyList(), +) + +enum class Operation(val removesSource: Boolean){ + COPY(false), + MOVE(true), + MOVE_WITH_COPY(true), + NONE(false), +} + +enum class ConflictStrategy{ + /** + * Use the first free name among `name (1).ext`, `name (2).ext`... + */ + RENAME, + SKIP, +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/console/ConsoleHandler.kt b/src/main/kotlin/fr/rakambda/filesecure/console/ConsoleHandler.kt new file mode 100644 index 0000000..abd97b6 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/console/ConsoleHandler.kt @@ -0,0 +1,43 @@ +package fr.rakambda.filesecure.console + +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.InputStream + +private val log = KotlinLogging.logger {} + +/** + * Reads commands from the standard input: `p` pauses, `r` resumes and `q` quits. + */ +class ConsoleHandler( + private val input: InputStream, + private val pauseGate: PauseGate, + private val onQuit: () -> Unit, +) : Thread("Console watcher"){ + init{ + isDaemon = true + } + + override fun run(){ + try{ + input.bufferedReader().lineSequence().forEach { line -> + when(line.trim()){ + "q" -> { + log.info { "Exiting" } + onQuit() + } + "p" -> { + log.info { "Pausing, running conversions will finish" } + pauseGate.pause() + } + "r" -> { + log.info { "Resuming" } + pauseGate.resume() + } + } + } + } + catch(e: Exception){ + log.warn(e) { "Error reading console commands" } + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/console/PauseGate.kt b/src/main/kotlin/fr/rakambda/filesecure/console/PauseGate.kt new file mode 100644 index 0000000..491c2ae --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/console/PauseGate.kt @@ -0,0 +1,29 @@ +package fr.rakambda.filesecure.console + +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first + +/** + * Lets work be paused and resumed, work already started is not interrupted. + */ +class PauseGate{ + private val paused = MutableStateFlow(false) + + val isPaused: Boolean + get() = paused.value + + fun pause(){ + paused.value = true + } + + fun resume(){ + paused.value = false + } + + /** + * Suspends while paused. + */ + suspend fun awaitResumed(){ + paused.first { !it } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/ConversionTools.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/ConversionTools.kt new file mode 100644 index 0000000..b658c41 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/ConversionTools.kt @@ -0,0 +1,57 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.ffprobe.FFprobe +import com.github.kokorin.jaffree.ffprobe.FFprobeResult +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import java.nio.file.Path + +private val log = KotlinLogging.logger {} + +/** + * External programs used to probe and convert files. + * + * @param ffmpegDirectory Folder containing the ffmpeg executable, null to look it up in the PATH. + * @param ffprobeDirectory Folder containing the ffprobe executable, null to look it up in the PATH. + * @param magick ImageMagick executable. + * @param ffmpegThreads Value of ffmpeg's `-threads` option, null to let ffmpeg decide. + * @param processListener Called with every external process started for a conversion. + */ +class ConversionTools( + private val ffmpegDirectory: Path? = null, + private val ffprobeDirectory: Path? = null, + val magick: String = "magick", + val ffmpegThreads: Int? = null, + val processListener: ((Process) -> Unit)? = null, +){ + private val probeLock = Semaphore(MAX_CONCURRENT_PROBES) + + fun ffmpeg(): CustomFFmpeg = CustomFFmpeg(ffmpegDirectory?.resolve("ffmpeg") ?: Path.of("ffmpeg"), processListener) + + /** + * @return The probe result, or null if the file couldn't be probed (not a media file for example). + */ + suspend fun probe(file: Path): FFprobeResult? = probeLock.withPermit { + runInterruptible(Dispatchers.IO) { + try{ + log.debug { "Probing file $file" } + val ffprobe = ffprobeDirectory?.let { FFprobe.atPath(it) } ?: FFprobe.atPath() + ffprobe.setShowStreams(true) + .setShowFormat(true) + .setInput(file.toString()) + .execute() + } + catch(e: Exception){ + log.warn { "Failed to probe file $file" } + null + } + } + } + + private companion object{ + const val MAX_CONCURRENT_PROBES = 10 + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/Converter.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/Converter.kt new file mode 100644 index 0000000..1f65b89 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/Converter.kt @@ -0,0 +1,130 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.ffmpeg.FFmpegProgress +import com.github.kokorin.jaffree.ffmpeg.ProgressListener +import com.github.kokorin.jaffree.ffmpeg.UrlInput +import com.github.kokorin.jaffree.ffmpeg.UrlOutput +import com.github.kokorin.jaffree.ffprobe.FFprobeResult +import fr.rakambda.filesecure.progress.TaskProgress +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.future.await +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext +import java.nio.file.Path +import java.time.Duration + +private val log = KotlinLogging.logger {} + +/** + * Converts one file into another. + */ +interface Converter{ + /** + * Human-readable command, used for dry runs. + */ + val description: String + + /** + * Runs the conversion, the external process is killed if the coroutine is cancelled. + * + * @throws ConversionException If the conversion failed. + */ + suspend fun convert(progress: TaskProgress) +} + +class ConversionException(message: String, cause: Throwable? = null) : Exception(message, cause) + +class FfmpegConverter( + private val tools: ConversionTools, + private val probeResult: FFprobeResult?, + private val input: Path, + private val output: Path, + private val configureOutput: (UrlOutput) -> UrlOutput, +) : Converter{ + override val description: String + get() = build(null).commandLine + + override suspend fun convert(progress: TaskProgress){ + val duration = probeResult?.format?.duration?.let { Duration.ofSeconds(it.toLong()) } ?: Duration.ZERO + log.debug { "Converting $input ($duration) to $output" } + progress.setMax(duration.toMillis()) + + val future = build { ffmpegProgress -> ffmpegProgress.timeMillis?.takeIf { it >= 0 }?.let(progress::stepTo) } + .executeAsync() + try{ + future.toCompletableFuture().await() + } + catch(e: CancellationException){ + future.forceStop() + throw e + } + catch(e: Exception){ + throw ConversionException("ffmpeg failed to convert $input", e) + } + } + + private fun build(progressListener: ProgressListener?): CustomFFmpeg{ + val output = configureOutput(UrlOutput.toPath(output)) + tools.ffmpegThreads?.takeIf { it >= 0 }?.let { output.addArguments("-threads", it.toString()) } + + val ffmpeg = tools.ffmpeg() + ffmpeg.addInput(UrlInput.fromPath(input)) + .addOutput(output) + .setOverwriteOutput(false) + progressListener?.let(ffmpeg::setProgressListener) + return ffmpeg + } +} + +/** + * Runs an external command expected to write the output file. + */ +class CommandConverter( + private val tools: ConversionTools, + private val command: List, +) : Converter{ + override val description: String + get() = command.joinToString(" ") { "\"$it\"" } + + override suspend fun convert(progress: TaskProgress){ + progress.setMax(1) + val process = runInterruptible(Dispatchers.IO) { + ProcessBuilder(command) + .redirectErrorStream(true) + .start() + } + try{ + tools.processListener?.invoke(process) + val exitCode = coroutineScope { + val output = async(Dispatchers.IO) { process.inputStream.bufferedReader().use { it.readText() } } + val exitCode = try{ + process.onExit().await().exitValue() + } + catch(e: CancellationException){ + // Killing the process ends its output stream, letting the reader above complete + process.destroyForcibly() + throw e + } + if(exitCode != 0){ + val text = output.await() + log.warn { "Command `$description` exited with code $exitCode: $text" } + } + exitCode + } + if(exitCode != 0){ + throw ConversionException("Command `$description` exited with code $exitCode") + } + progress.stepTo(1) + } + finally{ + if(process.isAlive){ + withContext(NonCancellable + Dispatchers.IO) { process.destroyForcibly().waitFor() } + } + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/CustomFFmpeg.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/CustomFFmpeg.kt new file mode 100644 index 0000000..9bcc670 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/CustomFFmpeg.kt @@ -0,0 +1,28 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.ffmpeg.FFmpeg +import com.github.kokorin.jaffree.process.Stopper +import java.nio.file.Path + +/** + * FFmpeg notifying when its process starts and exposing its command line. + */ +class CustomFFmpeg( + private val executable: Path, + private val processListener: ((Process) -> Unit)?, +) : FFmpeg(executable){ + override fun createStopper(): Stopper{ + val delegate = super.createStopper() + return object : Stopper by delegate{ + override fun setProcess(process: Process){ + delegate.setProcess(process) + processListener?.invoke(process) + } + } + } + + public override fun buildArguments(): List = super.buildArguments() + + val commandLine: String + get() = (listOf(executable.toString()) + buildArguments()).joinToString(" ") { "\"$it\"" } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/MediaProcessor.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/MediaProcessor.kt new file mode 100644 index 0000000..bed6442 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/MediaProcessor.kt @@ -0,0 +1,187 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.StreamType +import com.github.kokorin.jaffree.ffmpeg.UrlOutput +import com.github.kokorin.jaffree.ffprobe.FFprobeResult +import com.github.kokorin.jaffree.ffprobe.Stream +import java.nio.file.Path +import java.util.Locale + +/** + * Decides whether a file should be converted and how. + */ +sealed interface MediaProcessor{ + /** + * Extension of the converted file, without the dot. + */ + val extension: String + + fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean + + fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter +} + +/** + * A conversion decided by a Convert stage, executed once all stages accepted the file. + */ +data class PlannedConversion( + val processor: MediaProcessor, + val probeResult: FFprobeResult?, +) + +enum class ProcessorType(val processor: MediaProcessor){ + VIDEO_TO_HEVC_MP4(VideoToHevcProcessor("mp4", "mov,mp4,m4a,3gp,3g2,mj2")), + VIDEO_TO_HEVC_MKV(VideoToHevcProcessor("mkv", "matroska,webm")), + VIDEO_TO_AV1_MKV(VideoToAv1Processor), + MP3_TO_AAC(SingleAudioStreamProcessor("m4a", setOf("mp3")) { it.setCodec(StreamType.AUDIO, "aac").addArguments("-b:a", "192k") }), + AUDIO_TO_OPUS(SingleAudioStreamProcessor("opus", setOf("mp3", "aac")) { it.setCodec(StreamType.AUDIO, "libopus").addArguments("-b:a", "128000") }), + TIFF_TO_JPG(TiffToJpgProcessor), + PHOTO_TO_AVIF(PhotoProcessor("avif")), + PHOTO_TO_JXL(PhotoProcessor("jxl")), +} + +class VideoToHevcProcessor( + override val extension: String, + /** + * ffprobe's format name of the target container. + */ + private val formatName: String, +) : MediaProcessor{ + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean{ + if(probeResult == null || isPicture(probeResult)){ + return false + } + return isOtherVideoCodec(probeResult) || isHevcWithOtherTag(probeResult) || isHevcWithOtherContainer(probeResult) + } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter{ + // Already HEVC: only the container or codec tag needs to change, no re-encoding + val remuxOnly = probeResult != null && isHevcWithOtherContainer(probeResult) + return FfmpegConverter(tools, probeResult, input, output) { + if(remuxOnly){ + it.copyAllCodecs() + .addArguments("-tag:v", "hvc1") + } + else{ + it.setCodec(StreamType.SUBTITLE, "copy") + .setCodec(StreamType.AUDIO, "aac") + .addArguments("-b:a", "128000") + .setCodec(StreamType.VIDEO, "libx265") + .addArguments("-preset", "medium") + .addArguments("-crf", "21") + .addArguments("-vf", "format=yuv420p10le") + .addArguments("-movflags", "+faststart+use_metadata_tags") + .addArguments("-map_metadata", "0") + .addArguments("-map", "0") + .addArguments("-map", "-0:d") + .addArguments("-max_muxing_queue_size", "512") + .addArguments("-tag:v", "hvc1") + } + } + } + + private fun isPicture(probeResult: FFprobeResult) = probeResult.streams.any { it.profile == "Main Still Picture" } + + private fun isOtherVideoCodec(probeResult: FFprobeResult) = probeResult.streams.any { it.codecName in CODECS } + + private fun isHevcWithOtherTag(probeResult: FFprobeResult) = probeResult.streams.any { isHevc(it) && !isTargetCodecTag(it) } + + private fun isHevcWithOtherContainer(probeResult: FFprobeResult): Boolean{ + val isTargetContainer = probeResult.format?.formatName == formatName + val isTargetTag = probeResult.streams.any(::isTargetCodecTag) + val isHevc = probeResult.streams.any(::isHevc) + return isHevc && (!isTargetContainer || !isTargetTag) + } + + private fun isHevc(stream: Stream) = stream.codecName == "hevc" + + private fun isTargetCodecTag(stream: Stream) = stream.codecTagString == "hvc1" || stream.codecTagString == "[0][0][0][0]" + + private companion object{ + val CODECS = setOf("h264", "vp9", "wmv3", "mpeg2video") + } +} + +object VideoToAv1Processor : MediaProcessor{ + private val CODECS = setOf("h264", "vp9", "wmv3", "mpeg2video", "hevc", "h265") + + override val extension = "mkv" + + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean = probeResult?.streams.orEmpty().any { it.codecName in CODECS } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + FfmpegConverter(tools, probeResult, input, output) { + it.setCodec(StreamType.SUBTITLE, "copy") + .setCodec(StreamType.AUDIO, "libopus") + .addArguments("-b:a", "128000") + .setCodec(StreamType.VIDEO, "libsvtav1") + .addArguments("-preset", "4") + .addArguments("-crf", "32") + .addArguments("-g", "240") + .addArguments("-svtav1-params", "tune=0:enable-overlays=1:scd=1:scm=2") + .addArguments("-pix_fmt", "yuv420p10le") + .addArguments("-movflags", "+faststart+use_metadata_tags") + .addArguments("-map_metadata", "0") + .addArguments("-map", "0") + .addArguments("-map", "-0:d") + .addArguments("-max_muxing_queue_size", "512") + } +} + +/** + * Converts files made of a single audio stream. + */ +class SingleAudioStreamProcessor( + override val extension: String, + private val codecs: Set, + private val configureOutput: (UrlOutput) -> UrlOutput, +) : MediaProcessor{ + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean{ + val streams = probeResult?.streams ?: return false + return streams.size == 1 && streams.all { it.codecName in codecs } + } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + FfmpegConverter(tools, probeResult, input, output, configureOutput) +} + +object TiffToJpgProcessor : MediaProcessor{ + override val extension = "jpg" + + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean = probeResult?.streams.orEmpty().any { it.codecName == "tiff" } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + CommandConverter(tools, listOf( + tools.magick, + "-quality", "85%", + "-sampling-factor", "4:2:0", + "-interlace", "JPEG", + "-colorspace", "sRGB", + input.toAbsolutePath().toString(), + output.toAbsolutePath().toString(), + )) +} + +/** + * Converts photos with ImageMagick, the output format being deduced from the extension. + */ +class PhotoProcessor(override val extension: String) : MediaProcessor{ + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean{ + val name = file.fileName.toString().lowercase(Locale.ROOT) + if(name.endsWith(".$extension")){ + return false + } + return PHOTO_EXTENSIONS.any { name.endsWith(".$it") } + } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + CommandConverter(tools, listOf( + tools.magick, + input.toAbsolutePath().toString(), + output.toAbsolutePath().toString(), + )) + + private companion object{ + val PHOTO_EXTENSIONS = setOf("heic", "tiff", "jpg", "raf") + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/ProbeFilter.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/ProbeFilter.kt new file mode 100644 index 0000000..217267e --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/ProbeFilter.kt @@ -0,0 +1,31 @@ +package fr.rakambda.filesecure.conversion + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.github.kokorin.jaffree.ffprobe.FFprobeResult + +/** + * Filters files on their probe result, files that couldn't be probed never pass. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes( + JsonSubTypes.Type(value = LengthMoreThan::class, name = "LengthMoreThan"), + JsonSubTypes.Type(value = LengthLessThan::class, name = "LengthLessThan"), +) +sealed interface ProbeFilter{ + fun test(probeResult: FFprobeResult?): Boolean +} + +/** + * Keeps media lasting at least [value] seconds. + */ +data class LengthMoreThan(val value: Long) : ProbeFilter{ + override fun test(probeResult: FFprobeResult?): Boolean = probeResult?.format?.duration?.let { it >= value } ?: false +} + +/** + * Keeps media lasting at most [value] seconds. + */ +data class LengthLessThan(val value: Long) : ProbeFilter{ + override fun test(probeResult: FFprobeResult?): Boolean = probeResult?.format?.duration?.let { it <= value } ?: false +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/WindowsAffinity.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/WindowsAffinity.kt new file mode 100644 index 0000000..0d54be4 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/WindowsAffinity.kt @@ -0,0 +1,44 @@ +package fr.rakambda.filesecure.conversion + +import com.sun.jna.Platform +import com.sun.jna.platform.win32.BaseTSD +import com.sun.jna.platform.win32.Kernel32 +import com.sun.jna.platform.win32.WinNT +import io.github.oshai.kotlinlogging.KotlinLogging + +private val log = KotlinLogging.logger {} + +/** + * Restricts processes to some CPU cores, only supported on Windows. + * + * @param mask Bit mask of the allowed cores, e.g. 7 allows cores 1, 2 and 3. + */ +class WindowsAffinity(private val mask: Long){ + init{ + require(mask > 0) { "Affinity mask must be positive" } + if(!Platform.isWindows()){ + log.warn { "CPU affinity is only supported on Windows, it will be ignored" } + } + } + + fun apply(process: Process){ + if(!Platform.isWindows()){ + return + } + log.info { "Setting affinity $mask to pid ${process.pid()}" } + val kernel = Kernel32.INSTANCE + val handle = kernel.OpenProcess(WinNT.PROCESS_SET_INFORMATION or WinNT.PROCESS_QUERY_INFORMATION, false, process.pid().toInt()) + if(handle == null){ + log.warn { "Failed to open process ${process.pid()} to set its affinity, error ${kernel.GetLastError()}" } + return + } + try{ + if(!kernel.SetProcessAffinityMask(handle, BaseTSD.ULONG_PTR(mask))){ + log.warn { "Failed to set affinity of process ${process.pid()}, error ${kernel.GetLastError()}" } + } + } + finally{ + kernel.CloseHandle(handle) + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/folder/FolderAction.kt b/src/main/kotlin/fr/rakambda/filesecure/folder/FolderAction.kt new file mode 100644 index 0000000..e79ae8b --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/folder/FolderAction.kt @@ -0,0 +1,113 @@ +package fr.rakambda.filesecure.folder + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import fr.rakambda.filesecure.pipeline.FileOperations +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.Files +import java.nio.file.Path +import java.time.LocalDateTime +import java.time.ZoneId + +private val log = KotlinLogging.logger {} + +/** + * An action applied to every scanned input folder once all files have been processed, deepest folders first. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes( + JsonSubTypes.Type(value = DeleteIfEmpty::class, name = "DeleteIfEmpty"), + JsonSubTypes.Type(value = DeleteIfOlderThan::class, name = "DeleteIfOlderThan"), + JsonSubTypes.Type(value = DeleteChildren::class, name = "DeleteChildren"), +) +sealed interface FolderAction{ + /** + * @param folder The folder to apply the action to. + * @param baseFolder The input folder of the mapping. + */ + fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path) +} + +/** + * Depth of the folder below the input folder, the input folder itself being at depth 0. + */ +internal fun depthOf(folder: Path, baseFolder: Path): Int{ + if(folder == baseFolder){ + return 0 + } + return baseFolder.relativize(folder).nameCount +} + +internal fun isEmptyFolder(folder: Path): Boolean = Files.list(folder).use { it.findAny().isEmpty } + +/** + * Deletes empty folders at least [depth] levels below the input folder. + */ +data class DeleteIfEmpty( + val depth: Int = 1, +) : FolderAction{ + override fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path){ + try{ + if(depthOf(folder, baseFolder) < depth || !isEmptyFolder(folder)){ + return + } + log.info { "Deleting folder $folder because it is empty" } + fileOperations.delete(folder) + } + catch(e: Exception){ + log.error(e) { "Failed to determine if $folder should be deleted, it will not be by default" } + } + } +} + +/** + * Deletes empty folders at least [depth] levels below the input folder, whose last modification is older than the given offset. + */ +data class DeleteIfOlderThan( + val dayOffset: Long = 0, + val minuteOffset: Long = 0, + val depth: Int = 1, +) : FolderAction{ + override fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path){ + try{ + if(depthOf(folder, baseFolder) < depth){ + return + } + val modified = LocalDateTime.ofInstant(Files.getLastModifiedTime(folder).toInstant(), ZoneId.systemDefault()) + if(!modified.isBefore(LocalDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset))){ + return + } + if(isEmptyFolder(folder)){ + log.info { "Deleting folder $folder because it is more than $dayOffset days old and empty" } + fileOperations.delete(folder) + } + else{ + log.debug { "Tried deleting folder $folder because it is more than $dayOffset days old but wasn't empty" } + } + } + catch(e: Exception){ + log.error(e) { "Failed to determine if $folder should be deleted, it will not be by default" } + } + } +} + +/** + * Deletes the given children of every folder, if they exist. + */ +data class DeleteChildren( + val names: List = emptyList(), +) : FolderAction{ + override fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path){ + for(name in names){ + val child = folder.resolve(name) + try{ + if(fileOperations.deleteIfExists(child)){ + log.info { "Deleted child file $child" } + } + } + catch(e: Exception){ + log.error(e) { "Failed to delete child $child" } + } + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/FileMetadata.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/FileMetadata.kt new file mode 100644 index 0000000..e5832b3 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/FileMetadata.kt @@ -0,0 +1,15 @@ +package fr.rakambda.filesecure.metadata + +import java.time.ZonedDateTime + +data class FileMetadata( + val takenDate: ZonedDateTime? = null, + val createdDate: ZonedDateTime? = null, + val nameDate: ZonedDateTime? = null, +){ + /** + * The best known date of the file: from its name, then its media metadata, then its last modification. + */ + val date: ZonedDateTime + get() = nameDate ?: takenDate ?: createdDate ?: ZonedDateTime.now() +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/MediaDateExtractor.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/MediaDateExtractor.kt new file mode 100644 index 0000000..5520cde --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/MediaDateExtractor.kt @@ -0,0 +1,44 @@ +package fr.rakambda.filesecure.metadata + +import com.drew.metadata.Directory +import com.drew.metadata.xmp.XmpDirectory +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.TimeZone + +/** + * Reads a date from one kind of metadata directory. + */ +sealed interface MediaDateExtractor{ + val directoryClass: Class + + fun parse(directory: Directory, timeZone: TimeZone): ZonedDateTime? +} + +class TagMediaDateExtractor( + override val directoryClass: Class, + private val tag: Int, +) : MediaDateExtractor{ + override fun parse(directory: Directory, timeZone: TimeZone): ZonedDateTime? = + directory.getDate(tag, timeZone)?.toInstant()?.atZone(timeZone.toZoneId()) +} + +object XmpMediaDateExtractor : MediaDateExtractor{ + private val KEYS = listOf("xmp:CreateDate", "photoshop:DateCreated") + private val FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss") + + override val directoryClass = XmpDirectory::class.java + + override fun parse(directory: Directory, timeZone: TimeZone): ZonedDateTime?{ + val values = (directory as XmpDirectory).xmpProperties + for(key in KEYS){ + val value = values[key] ?: continue + try{ + return ZonedDateTime.parse(value, FORMATTER.withZone(timeZone.toZoneId())) + } + catch(_: Exception){ + } + } + return null + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/MetadataExtractor.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/MetadataExtractor.kt new file mode 100644 index 0000000..5ab3236 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/MetadataExtractor.kt @@ -0,0 +1,149 @@ +package fr.rakambda.filesecure.metadata + +import com.drew.imaging.ImageMetadataReader +import com.drew.metadata.Metadata +import com.drew.metadata.exif.ExifSubIFDDirectory +import com.drew.metadata.exif.GpsDirectory +import com.drew.metadata.mov.QuickTimeDirectory +import com.drew.metadata.mov.metadata.QuickTimeMetadataDirectory +import com.drew.metadata.mp4.Mp4Directory +import com.drew.metadata.xmp.XmpDirectory +import io.github.oshai.kotlinlogging.KotlinLogging +import us.fatehi.pointlocation6709.parse.PointLocationParser +import java.nio.file.Files +import java.nio.file.Path +import java.time.LocalDateTime +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeParseException +import java.util.TimeZone +import java.util.regex.Pattern + +private val log = KotlinLogging.logger {} + +class MetadataExtractor( + private val timeZoneResolver: TimeZoneResolver, + private val nameDateExtractors: List = NameDateExtractor.DEFAULTS, +){ + private val mediaDateExtractors: List = listOf( + TagMediaDateExtractor(QuickTimeMetadataDirectory::class.java, QuickTimeMetadataDirectory.TAG_CREATION_DATE), + TagMediaDateExtractor(QuickTimeDirectory::class.java, QuickTimeDirectory.TAG_CREATION_TIME), + TagMediaDateExtractor(Mp4Directory::class.java, Mp4Directory.TAG_CREATION_TIME), + TagMediaDateExtractor(ExifSubIFDDirectory::class.java, ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL), + XmpMediaDateExtractor, + ) + + fun getMetadata(path: Path): FileMetadata = FileMetadata( + takenDate = getDateFromMetadata(path), + createdDate = getModificationDate(path), + nameDate = getDateFromFileName(path), + ) + + private fun getModificationDate(path: Path): ZonedDateTime{ + val date = Files.getLastModifiedTime(path).toInstant().atZone(ZoneId.systemDefault()) + if(date.year <= 1970){ + return date.withYear(LocalDateTime.now().year) + } + return date + } + + private fun getDateFromMetadata(path: Path): ZonedDateTime?{ + try{ + val metadata = ImageMetadataReader.readMetadata(path.toFile()) ?: return null + val timeZone = TimeZone.getTimeZone(getZoneIdFromMetadata(metadata) ?: ZoneId.systemDefault()) + + for(extractor in mediaDateExtractors){ + for(directory in metadata.getDirectoriesOfType(extractor.directoryClass)){ + try{ + log.debug { "Trying ${extractor.directoryClass.name}" } + val date = extractor.parse(directory, timeZone) ?: continue + if(date.year < 1970){ + log.warn { "Invalid year with metadata directory ${extractor.directoryClass.name} for file $path" } + continue + } + log.debug { "Matched metadata directory $directory" } + return date + } + catch(e: Exception){ + log.error { "Error processing metadata directory ${extractor.directoryClass.name} for $path => ${e.message}" } + } + } + } + } + catch(e: Exception){ + log.error { "Error getting date from metadata for $path => ${e.message}" } + } + return null + } + + private fun getDateFromFileName(path: Path): ZonedDateTime?{ + val fileName = path.fileName.toString() + for(extractor in nameDateExtractors){ + try{ + log.debug { "Trying name extractor `$extractor`" } + val date = extractor.parse(fileName) ?: continue + if(date.year < 1970){ + log.warn { "Invalid year with used format for file $path" } + continue + } + log.debug { "Matched date format for $path" } + return date + } + catch(_: DateTimeParseException){ + } + catch(e: Exception){ + log.error { "Error using format $extractor => ${e.message}" } + } + } + return null + } + + private fun getZoneIdFromMetadata(metadata: Metadata): ZoneId?{ + try{ + for(directory in metadata.getDirectoriesOfType(GpsDirectory::class.java)){ + val location = directory.geoLocation ?: continue + timeZoneResolver.resolve(location.latitude, location.longitude)?.let { return it } + } + for(directory in metadata.getDirectoriesOfType(QuickTimeMetadataDirectory::class.java)){ + val representation = directory.getString(QUICKTIME_LOCATION_TAG) + if(representation.isNullOrBlank()){ + continue + } + val location = PointLocationParser.parsePointLocation(representation) + timeZoneResolver.resolve(location.latitude.degrees, location.longitude.degrees)?.let { return it } + } + for(directory in metadata.getDirectoriesOfType(XmpDirectory::class.java)){ + val values = directory.xmpProperties + val latitude = values["exif:GPSLatitude"]?.let(::parseAngle) ?: continue + val longitude = values["exif:GPSLongitude"]?.let(::parseAngle) ?: continue + timeZoneResolver.resolve(latitude, longitude)?.let { return it } + } + } + catch(e: Exception){ + log.warn(e) { "Error getting GPS infos" } + } + return null + } + + companion object{ + private const val QUICKTIME_LOCATION_TAG = 0x050D + private val ANGLE_PATTERN = Pattern.compile("""(\d{1,3}),(\d{1,2})\.(\d+)([NESW])""") + + /** + * Converts an XMP coordinate such as `48,51.2345N` to signed degrees. + */ + internal fun parseAngle(value: String): Double?{ + val matcher = ANGLE_PATTERN.matcher(value) + if(!matcher.matches()){ + return null + } + val angle = matcher.group(1).toInt() + + matcher.group(2).toInt() / 60.0 + + "0.${matcher.group(3)}".toDouble() / 60.0 + return when(matcher.group(4)){ + "W", "S" -> -angle + else -> angle + } + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/NameDateExtractor.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/NameDateExtractor.kt new file mode 100644 index 0000000..d58e296 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/NameDateExtractor.kt @@ -0,0 +1,44 @@ +package fr.rakambda.filesecure.metadata + +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.Locale +import java.util.regex.Pattern + +/** + * Finds a date in a file name with a regex, the matched group being parsed with a date format. + */ +class NameDateExtractor( + private val pattern: Pattern, + format: String, + private val group: Int = 0, +){ + private val formatter = DateTimeFormatter.ofPattern(format) + .withLocale(Locale.ENGLISH) + .withZone(ZoneId.systemDefault()) + + /** + * @throws java.time.format.DateTimeParseException If the matched text is not a valid date. + */ + fun parse(name: String): ZonedDateTime?{ + val matcher = pattern.matcher(name) + if(!matcher.find()){ + return null + } + return ZonedDateTime.parse(matcher.group(group), formatter) + } + + override fun toString(): String = "NameDateExtractor(${pattern.pattern()})" + + companion object{ + val DEFAULTS = listOf( + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2}"""), "yyyy-MM-dd HH.mm.ss"), + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2} \d{2}-\d{2}-\d{2}"""), "yyyy-MM-dd HH-mm-ss"), + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}"""), "yyyy-MM-dd_HH-mm-ss"), + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}"""), "yyyy-MM-dd-HH-mm-ss"), + NameDateExtractor(Pattern.compile("""IMG_(\d{8}_\d{6})"""), "yyyyMMdd_HHmmss", 1), + NameDateExtractor(Pattern.compile("""Video-(\d{8}-\d{6})"""), "yyyyMMdd-HHmmss", 1), + ) + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/TimeZoneResolver.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/TimeZoneResolver.kt new file mode 100644 index 0000000..8c1b841 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/TimeZoneResolver.kt @@ -0,0 +1,57 @@ +package fr.rakambda.filesecure.metadata + +import com.fasterxml.jackson.databind.json.JsonMapper +import io.github.oshai.kotlinlogging.KotlinLogging +import java.net.URI +import java.net.URLEncoder +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.charset.StandardCharsets +import java.time.Duration +import java.time.ZoneId +import java.util.concurrent.ConcurrentHashMap + +private val log = KotlinLogging.logger {} + +fun interface TimeZoneResolver{ + fun resolve(latitude: Double, longitude: Double): ZoneId? +} + +/** + * Resolves the time zone of coordinates with the geonames.org web service. + */ +class GeonamesTimeZoneResolver(private val username: String) : TimeZoneResolver{ + private val client = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(30)) + .build() + private val mapper = JsonMapper() + private val cache = ConcurrentHashMap, ZoneId>() + + override fun resolve(latitude: Double, longitude: Double): ZoneId?{ + cache[latitude to longitude]?.let { return it } + try{ + val query = "lat=$latitude&lng=$longitude&username=${URLEncoder.encode(username, StandardCharsets.UTF_8)}" + val request = HttpRequest.newBuilder(URI.create("http://api.geonames.org/timezoneJSON?$query")) + .timeout(Duration.ofSeconds(30)) + .GET() + .build() + val response = client.send(request, HttpResponse.BodyHandlers.ofString()) + if(response.statusCode() !in 200..299){ + log.warn { "Geonames answered ${response.statusCode()} for coordinates $latitude;$longitude" } + return null + } + val zoneId = mapper.readTree(response.body()).path("timezoneId").textValue()?.let(ZoneId::of) + zoneId?.let { cache[latitude to longitude] = it } + return zoneId + } + catch(e: InterruptedException){ + Thread.currentThread().interrupt() + return null + } + catch(e: Exception){ + log.error(e) { "Error getting zoneID for coordinates $latitude;$longitude" } + return null + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserver.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserver.kt new file mode 100644 index 0000000..eb075e9 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserver.kt @@ -0,0 +1,49 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.ConflictStrategy +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.Path + +/** + * Picks destinations that are neither existing nor already given to another file being processed concurrently. + */ +class DestinationReserver{ + private val mutex = Mutex() + private val reserved = HashSet() + + /** + * @return The destination to use, or null if the file should be skipped. + */ + suspend fun reserve(destination: Path, strategy: ConflictStrategy): Path? = mutex.withLock { + withContext(Dispatchers.IO) { + val candidate = when{ + isFree(destination) -> destination + strategy == ConflictStrategy.SKIP -> null + else -> generateSequence(1) { it + 1 } + .map { destination.resolveSibling(numberedName(destination.fileName.toString(), it)) } + .first(::isFree) + } + candidate?.also { reserved.add(it) } + } + } + + private fun isFree(path: Path) = path !in reserved && !Files.exists(path, LinkOption.NOFOLLOW_LINKS) + + companion object{ + /** + * `name.ext` becomes `name (index).ext`, names without extension or starting with a dot get the suffix at their end. + */ + internal fun numberedName(name: String, index: Int): String{ + val dotIndex = name.lastIndexOf('.') + if(dotIndex <= 0){ + return "$name ($index)" + } + return "${name.substring(0, dotIndex)} ($index)${name.substring(dotIndex)}" + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileItem.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileItem.kt new file mode 100644 index 0000000..4d6f686 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileItem.kt @@ -0,0 +1,31 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.conversion.PlannedConversion +import fr.rakambda.filesecure.metadata.FileMetadata +import java.nio.file.Path + +/** + * A file going through the stages of a pipeline. + * + * Stages only plan what will happen to the file (destination, conversion), the plan is executed once all stages accepted the file. + */ +class FileItem( + val source: Path, + val inputBase: Path, + val outputBase: Path, + metadataProvider: () -> FileMetadata, +){ + /** + * Same relative path in the output as in the input. + */ + val originalDestination: Path = outputBase.resolve(inputBase.relativize(source)) + var destination: Path = originalDestination + var conversion: PlannedConversion? = null + + /** + * Computed on first access as reading media metadata can be costly. + */ + val metadata: FileMetadata by lazy(metadataProvider) + + override fun toString(): String = source.toString() +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileOperations.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileOperations.kt new file mode 100644 index 0000000..a9e71b9 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileOperations.kt @@ -0,0 +1,152 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.Operation +import io.github.oshai.kotlinlogging.KotlinLogging +import java.awt.Desktop +import java.awt.GraphicsEnvironment +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.nio.file.attribute.BasicFileAttributeView +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime + +private val log = KotlinLogging.logger {} + +/** + * Every write to the file system goes through here, doing nothing during dry runs. + */ +class FileOperations(val dryRun: Boolean){ + fun transfer(operation: Operation, input: Path, output: Path){ + when(operation){ + Operation.COPY -> copy(input, output) + Operation.MOVE -> move(input, output) + Operation.MOVE_WITH_COPY -> moveWithCopy(input, output) + Operation.NONE -> {} + } + } + + fun copy(input: Path, output: Path){ + if(dryRun){ + return + } + createDirectories(output.parent) + Files.copy(input, output, StandardCopyOption.COPY_ATTRIBUTES) + } + + fun move(input: Path, output: Path){ + if(dryRun){ + return + } + createDirectories(output.parent) + Files.move(input, output) + } + + fun moveWithCopy(input: Path, output: Path){ + if(dryRun){ + return + } + copy(input, output) + Files.delete(input) + } + + /** + * Puts a converted file at its destination, removing the source if the operation requires it. + * + * @param replaceSource true when the destination is the source itself, the source then being replaced by the converted file. + */ + fun commitConverted(operation: Operation, source: Path, converted: Path, destination: Path, replaceSource: Boolean){ + if(dryRun || operation == Operation.NONE){ + return + } + createDirectories(destination.parent) + val sourceAttributes = Files.readAttributes(source, BasicFileAttributes::class.java) + + if(replaceSource){ + // Keep the original next to it until the converted file is in place, so it can be restored on failure + val backup = source.resolveSibling("${source.fileName}.filesecure-original") + Files.move(source, backup) + try{ + Files.move(converted, destination) + } + catch(e: Exception){ + Files.move(backup, source) + throw e + } + copyTimes(sourceAttributes, destination) + trash(backup) + return + } + + Files.move(converted, destination) + copyTimes(sourceAttributes, destination) + if(operation.removesSource){ + trash(source) + } + } + + fun delete(path: Path){ + if(dryRun){ + return + } + Files.delete(path) + } + + fun deleteIfExists(path: Path): Boolean{ + if(dryRun){ + return false + } + return Files.deleteIfExists(path) + } + + /** + * Moves a file to the trash when the platform has one, deletes it otherwise. + */ + fun trash(path: Path){ + if(dryRun){ + return + } + if(moveToTrash(path)){ + log.debug { "Moved file $path to trash" } + return + } + if(Files.deleteIfExists(path)){ + log.debug { "Deleted file $path" } + } + } + + fun setTimes(path: Path, lastModified: FileTime, creation: FileTime){ + if(dryRun){ + return + } + val view = Files.getFileAttributeView(path, BasicFileAttributeView::class.java) + view.setTimes(lastModified, view.readAttributes().lastAccessTime(), creation) + } + + private fun copyTimes(attributes: BasicFileAttributes, path: Path){ + Files.getFileAttributeView(path, BasicFileAttributeView::class.java) + .setTimes(attributes.lastModifiedTime(), attributes.lastAccessTime(), attributes.creationTime()) + } + + private fun moveToTrash(path: Path): Boolean{ + try{ + if(GraphicsEnvironment.isHeadless() || !Desktop.isDesktopSupported()){ + return false + } + val desktop = Desktop.getDesktop() + return desktop.isSupported(Desktop.Action.MOVE_TO_TRASH) && desktop.moveToTrash(path.toFile()) + } + catch(e: Throwable){ + log.debug(e) { "Trash is not available" } + return false + } + } + + private fun createDirectories(path: Path){ + Files.createDirectories(path) + if(!Files.isDirectory(path)){ + throw IOException("Destination folder $path already exists as a file") + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileScanner.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileScanner.kt new file mode 100644 index 0000000..50493bf --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileScanner.kt @@ -0,0 +1,60 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.ScanOptions +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.IOException +import java.nio.file.FileVisitResult +import java.nio.file.FileVisitor +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributes + +private val log = KotlinLogging.logger {} + +/** + * Walks an input folder, reporting the regular files and the folders that aren't excluded. + */ +class FileScanner( + private val root: Path, + private val options: ScanOptions, + private val onFolder: (Path) -> Unit, + private val onFile: (Path) -> Boolean, +) : FileVisitor{ + fun scan(){ + Files.walkFileTree(root, emptySet(), options.maxDepth, this) + } + + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult{ + if(options.skipHidden && dir != root && Files.isHidden(dir)){ + return FileVisitResult.SKIP_SUBTREE + } + val relative = root.relativize(dir).toString().replace("\\", "/") + if(options.excludedFolders.any { it.matcher(relative).matches() }){ + return FileVisitResult.SKIP_SUBTREE + } + log.debug { "Entering folder $dir" } + onFolder(dir) + return FileVisitResult.CONTINUE + } + + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult{ + // Folders beyond the max depth are reported here, they must not be processed as files + if(!attrs.isRegularFile){ + return FileVisitResult.CONTINUE + } + if(options.skipHidden && Files.isHidden(file)){ + return FileVisitResult.CONTINUE + } + return if(onFile(file)) FileVisitResult.CONTINUE else FileVisitResult.TERMINATE + } + + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult{ + log.warn { "Failed to visit $file: ${exc.message}" } + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult{ + exc?.let { log.warn { "Error while listing $dir: ${it.message}" } } + return FileVisitResult.CONTINUE + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineContext.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineContext.kt new file mode 100644 index 0000000..896ff0d --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineContext.kt @@ -0,0 +1,35 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.Pipeline +import fr.rakambda.filesecure.console.PauseGate +import fr.rakambda.filesecure.conversion.ConversionTools +import fr.rakambda.filesecure.metadata.MetadataExtractor +import fr.rakambda.filesecure.progress.NoOpProgressReporter +import fr.rakambda.filesecure.progress.ProgressReporter +import kotlinx.coroutines.sync.Semaphore + +/** + * Everything shared by all pipelines of a run. + * + * @param conversionThreads Maximum number of conversions running at the same time. + * @param workers Maximum number of files processed at the same time. + */ +class RunEnvironment( + val fileOperations: FileOperations, + val metadataExtractor: MetadataExtractor, + val tools: ConversionTools, + conversionThreads: Int = 1, + val workers: Int = 8, + val pauseGate: PauseGate = PauseGate(), + val progress: ProgressReporter = NoOpProgressReporter, +){ + val conversionLock = Semaphore(conversionThreads) + + val dryRun: Boolean + get() = fileOperations.dryRun +} + +class PipelineContext( + val environment: RunEnvironment, + val pipeline: Pipeline, +) diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunner.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunner.kt new file mode 100644 index 0000000..56fccdd --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunner.kt @@ -0,0 +1,248 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.Mapping +import fr.rakambda.filesecure.config.Operation +import fr.rakambda.filesecure.config.Pipeline +import fr.rakambda.filesecure.conversion.ConversionException +import fr.rakambda.filesecure.conversion.PlannedConversion +import fr.rakambda.filesecure.stage.Convert +import fr.rakambda.filesecure.stage.StageExecutor +import fr.rakambda.filesecure.stage.StageResult +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.trySendBlocking +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.FileTime +import java.util.concurrent.ConcurrentLinkedQueue + +private val log = KotlinLogging.logger {} + +class PipelineRunner(private val environment: RunEnvironment){ + suspend fun run(pipeline: Pipeline, name: String){ + log.info { "Running pipeline $name" } + val context = PipelineContext(environment, pipeline) + val executors = mutableListOf() + try{ + withContext(Dispatchers.IO) { + pipeline.stages.forEach { executors += it.open(context) } + } + val tempDirectory = if(pipeline.stages.any { it is Convert } && !environment.dryRun) createTempDirectory(pipeline.temp) else null + try{ + for(mapping in pipeline.mappings){ + MappingRun(pipeline, mapping, executors, tempDirectory).run() + } + } + finally{ + tempDirectory?.let { withContext(NonCancellable) { deleteTempDirectory(it) } } + } + } + finally{ + executors.forEach { executor -> + try{ + executor.close() + } + catch(e: Exception){ + log.error(e) { "Failed to close stage $executor" } + } + } + } + } + + private suspend fun createTempDirectory(parent: Path?): Path = withContext(Dispatchers.IO) { + if(parent == null){ + Files.createTempDirectory(TEMP_PREFIX) + } + else{ + Files.createDirectories(parent) + Files.createTempDirectory(parent, TEMP_PREFIX) + } + } + + private suspend fun deleteTempDirectory(directory: Path) = withContext(Dispatchers.IO) { + try{ + Files.delete(directory) + } + catch(e: Exception){ + log.warn { "Failed to delete temporary directory $directory, it may still contain files: ${e.message}" } + } + } + + private inner class MappingRun( + private val pipeline: Pipeline, + private val mapping: Mapping, + private val executors: List, + private val tempDirectory: Path?, + ){ + private val reserver = DestinationReserver() + private val fileOperations = environment.fileOperations + private val progress = environment.progress + + suspend fun run(){ + if(!withContext(Dispatchers.IO) { Files.isDirectory(mapping.input) }){ + log.warn { "Input folder ${mapping.input} doesn't exist, skipping mapping" } + return + } + log.info { "Processing mapping (${pipeline.operation}) ${mapping.input} ==> ${mapping.output}" } + + val folders = ConcurrentLinkedQueue() + coroutineScope { + val files = Channel(FILE_BUFFER_SIZE) + launch { + try{ + runInterruptible(Dispatchers.IO) { + FileScanner(mapping.input, pipeline.scan, folders::add) { file -> + progress.fileFound() + files.trySendBlocking(file).isSuccess + }.scan() + } + } + finally{ + files.close() + } + } + repeat(environment.workers) { + launch(Dispatchers.IO) { + for(file in files){ + processFile(file) + } + } + } + } + + if(pipeline.folderActions.isNotEmpty()){ + log.debug { "Applying folder actions" } + withContext(Dispatchers.IO) { + folders.sortedByDescending { it.nameCount }.forEach { folder -> + pipeline.folderActions.forEach { it.apply(fileOperations, folder, mapping.input) } + } + } + } + } + + private suspend fun processFile(file: Path){ + val item = FileItem(file, mapping.input, mapping.output) { environment.metadataExtractor.getMetadata(file) } + try{ + for(executor in executors){ + environment.pauseGate.awaitResumed() + val result = executor.process(item) + if(result is StageResult.Skip){ + log.debug { "Skipping $file: ${result.reason}" } + return + } + } + commit(item) + } + catch(e: CancellationException){ + throw e + } + catch(e: Exception){ + log.error(e) { "Failed to process $file" } + } + finally{ + progress.fileProcessed() + } + } + + private suspend fun commit(item: FileItem){ + val source = item.source + val conversion = item.conversion + val operation = pipeline.operation + + val isSource = item.destination.toAbsolutePath().normalize() == source.toAbsolutePath().normalize() + if(isSource && conversion == null){ + log.debug { "$source is already at its destination" } + return + } + // The source is replaced by its converted version, there's no conflict as the source will be gone + val replaceSource = isSource && operation.removesSource + val destination = if(replaceSource) item.destination else reserver.reserve(item.destination, pipeline.conflict) + if(destination == null){ + log.info { "Skipping $source, destination ${item.destination} already exists" } + return + } + + val conversionDescription = conversion?.let { " converted to ${it.processor.extension}" } ?: "" + log.info { "$operation file $source to $destination$conversionDescription" } + if(operation == Operation.NONE){ + return + } + + // Read before the source may disappear + val fileDate = if(pipeline.setFileCreationDate) item.metadata.nameDate ?: item.metadata.takenDate else null + + if(conversion == null){ + withContext(Dispatchers.IO) { fileOperations.transfer(operation, source, destination) } + } + else{ + val converted = convert(item, conversion, destination) ?: return + try{ + withContext(Dispatchers.IO) { fileOperations.commitConverted(operation, source, converted, destination, replaceSource) } + } + finally{ + withContext(NonCancellable + Dispatchers.IO) { Files.deleteIfExists(converted) } + } + } + + fileDate?.let { + log.info { "Setting creation date for $destination to ${it.toInstant()}" } + try{ + val time = FileTime.from(it.toInstant()) + withContext(Dispatchers.IO) { fileOperations.setTimes(destination, time, time) } + } + catch(e: Exception){ + log.warn(e) { "Failed to set file creation date to obtained metadata" } + } + } + } + + /** + * @return The converted file in the temporary directory, null during dry runs. + */ + private suspend fun convert(item: FileItem, conversion: PlannedConversion, destination: Path): Path?{ + // No temporary directory is created during dry runs, the path is only used in logs + val directory = tempDirectory ?: pipeline.temp ?: Path.of(System.getProperty("java.io.tmpdir")) + val converted = directory.resolve("${System.nanoTime()}-${item.source.hashCode()}-${destination.fileName}") + val converter = conversion.processor.createConverter(environment.tools, conversion.probeResult, item.source, converted) + if(environment.dryRun){ + log.info { "Dry run: would have executed `${converter.description}`" } + return null + } + + progress.conversionQueued() + try{ + environment.conversionLock.withPermit { + environment.pauseGate.awaitResumed() + log.info { "Converting ${item.source} to $converted" } + progress.task(destination.fileName.toString()).use { converter.convert(it) } + } + val size = withContext(Dispatchers.IO) { if(Files.exists(converted)) Files.size(converted) else -1 } + if(size <= 0){ + throw ConversionException("Converted file $converted is missing or empty") + } + log.debug { "Converted ${item.source} to $converted" } + return converted + } + catch(e: Throwable){ + withContext(NonCancellable + Dispatchers.IO) { Files.deleteIfExists(converted) } + throw e + } + finally{ + progress.conversionDone() + } + } + } + + private companion object{ + const val TEMP_PREFIX = "FileSecure" + const val FILE_BUFFER_SIZE = 1000 + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/progress/ProgressReporter.kt b/src/main/kotlin/fr/rakambda/filesecure/progress/ProgressReporter.kt new file mode 100644 index 0000000..ff00410 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/progress/ProgressReporter.kt @@ -0,0 +1,133 @@ +package fr.rakambda.filesecure.progress + +import me.tongfei.progressbar.InteractiveConsoleProgressBarConsumer +import me.tongfei.progressbar.ProgressBar +import me.tongfei.progressbar.ProgressBarBuilder +import java.io.FileDescriptor +import java.io.FileOutputStream +import java.io.PrintStream +import java.time.temporal.ChronoUnit +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +interface ProgressReporter : AutoCloseable{ + fun fileFound() + + fun fileProcessed() + + fun conversionQueued() + + fun conversionDone() + + /** + * Starts tracking a single task, closing it releases its display. + */ + fun task(name: String): TaskProgress + + override fun close(){ + } +} + +interface TaskProgress : AutoCloseable{ + fun setMax(max: Long) + + fun stepTo(value: Long) + + override fun close(){ + } +} + +object NoOpProgressReporter : ProgressReporter{ + private object NoOpTask : TaskProgress{ + override fun setMax(max: Long){ + } + + override fun stepTo(value: Long){ + } + } + + override fun fileFound(){ + } + + override fun fileProcessed(){ + } + + override fun conversionQueued(){ + } + + override fun conversionDone(){ + } + + override fun task(name: String): TaskProgress = NoOpTask +} + +/** + * Console progress bars: one for files, one for conversions, and one per running conversion, reused once their conversion ends. + */ +class ConsoleProgressReporter : ProgressReporter{ + private val filesFound = AtomicLong() + private val conversionsQueued = AtomicLong() + private val filesBar = ProgressBarBuilder() + .setTaskName("Files") + .setUnit(" files", 1) + .setInitialMax(0) + .hideEta() + .showSpeed() + .build() + private val conversionsBar = ProgressBarBuilder() + .setTaskName("Conversions") + .setInitialMax(0) + .build() + private val taskCounter = AtomicInteger() + private val idleTaskBars = ConcurrentLinkedQueue() + + override fun fileFound(){ + filesBar.maxHint(filesFound.incrementAndGet()) + } + + override fun fileProcessed(){ + filesBar.step() + } + + override fun conversionQueued(){ + conversionsBar.maxHint(conversionsQueued.incrementAndGet()) + } + + override fun conversionDone(){ + conversionsBar.step() + } + + override fun task(name: String): TaskProgress{ + val bar = (idleTaskBars.poll() ?: createTaskBar()).resume() + bar.extraMessage = name + return object : TaskProgress{ + override fun setMax(max: Long){ + bar.maxHint(max) + } + + override fun stepTo(value: Long){ + bar.stepTo(value) + } + + override fun close(){ + idleTaskBars.offer(bar.setExtraMessage("").maxHint(-1).reset().pause()) + } + } + } + + private fun createTaskBar(): ProgressBar = ProgressBarBuilder() + .setTaskName("Converter ${taskCounter.incrementAndGet()}") + .setUnit("s", 1000) + .setSpeedUnit(ChronoUnit.SECONDS) + .setConsumer(InteractiveConsoleProgressBarConsumer(PrintStream(FileOutputStream(FileDescriptor.err)), -1)) + .clearDisplayOnFinish() + .build() + + override fun close(){ + idleTaskBars.forEach(ProgressBar::close) + idleTaskBars.clear() + conversionsBar.close() + filesBar.close() + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/Convert.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/Convert.kt new file mode 100644 index 0000000..ed9a22d --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/Convert.kt @@ -0,0 +1,133 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.conversion.PlannedConversion +import fr.rakambda.filesecure.conversion.ProbeFilter +import fr.rakambda.filesecure.conversion.ProcessorType +import fr.rakambda.filesecure.pipeline.FileItem +import fr.rakambda.filesecure.pipeline.PipelineContext +import fr.rakambda.filesecure.storage.H2Storage +import fr.rakambda.filesecure.storage.NoOpStorage +import fr.rakambda.filesecure.storage.UselessFileStorage +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Timestamp +import java.util.Locale + +private val log = KotlinLogging.logger {} + +/** + * Plans the conversion of media files: the first processor able to handle the file is used, and the destination extension changes accordingly. + * The conversion itself runs after all stages accepted the file, and only if its destination is free. + */ +data class Convert( + /** + * Processors tried in order, empty to use the defaults. + */ + val processors: List = emptyList(), + /** + * Extensions of the files to probe, empty to use the defaults. + */ + val extensions: Set = emptySet(), + val filters: List = emptyList(), + /** + * What happens to files that won't be converted. + */ + val unhandled: UnhandledStrategy = UnhandledStrategy.SKIP, + /** + * Database remembering files that can't be converted, to avoid probing them on every run. + */ + val cache: Path? = null, +) : Stage{ + override fun open(context: PipelineContext): StageExecutor{ + val storage = cache?.let(::H2Storage) ?: NoOpStorage + return ConvertExecutor( + context, + processors.ifEmpty { DEFAULT_PROCESSORS }, + extensions.map { it.lowercase(Locale.ROOT) }.toSet().ifEmpty { DEFAULT_EXTENSIONS }, + filters, + unhandled, + storage, + ) + } + + companion object{ + val DEFAULT_PROCESSORS = listOf(ProcessorType.VIDEO_TO_HEVC_MKV, ProcessorType.MP3_TO_AAC, ProcessorType.TIFF_TO_JPG) + val DEFAULT_EXTENSIONS = setOf("mp4", "mov", "mkv", "avi", "tiff", "mp3", "m4a", "m4v", "ts", "m2ts", "mts", "heic", "jpg", "raf") + } +} + +enum class UnhandledStrategy{ + /** + * The file continues through the pipeline unchanged. + */ + PASS, + + /** + * The file is left where it is. + */ + SKIP, +} + +private class ConvertExecutor( + private val context: PipelineContext, + private val processors: List, + private val extensions: Set, + private val filters: List, + private val unhandled: UnhandledStrategy, + private val storage: UselessFileStorage, +) : StageExecutor{ + override suspend fun process(item: FileItem): StageResult{ + val file = item.source + val lastModified = withContext(Dispatchers.IO) { Timestamp(Files.getLastModifiedTime(file).toMillis()) } + if(storage.isUseless(file, lastModified)){ + return unhandled("known as not convertible") + } + if(file.extension !in extensions){ + storage.setUseless(file, lastModified) + return unhandled("extension is not handled") + } + + val probeResult = context.environment.tools.probe(file) + val processor = processors.map { it.processor }.firstOrNull { + try{ + it.canHandle(probeResult, file) + } + catch(e: Exception){ + log.error(e) { "Failed to check if ${it.javaClass.simpleName} can handle $file" } + false + } + } + if(processor == null){ + storage.setUseless(file, lastModified) + return unhandled("no processor can handle it") + } + if(!filters.all { it.test(probeResult) }){ + return unhandled("probe filters did not pass") + } + + log.debug { "Processor ${processor.javaClass.simpleName} matched $file" } + item.conversion = PlannedConversion(processor, probeResult) + item.destination = item.destination.withFileName(item.destination.nameWithoutExtension + "." + processor.extension) + return StageResult.Continue + } + + private fun unhandled(reason: String): StageResult = when(unhandled){ + UnhandledStrategy.PASS -> StageResult.Continue + UnhandledStrategy.SKIP -> StageResult.Skip("not converted, $reason") + } + + override fun close() = storage.close() + + private val Path.extension: String + get() = fileName.toString().substringAfterLast('.', "").lowercase(Locale.ROOT) + + private val Path.nameWithoutExtension: String + get(){ + val name = fileName.toString() + val dotIndex = name.lastIndexOf('.') + return if(dotIndex <= 0) name else name.substring(0, dotIndex) + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/FilterStages.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/FilterStages.kt new file mode 100644 index 0000000..0d3665e --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/FilterStages.kt @@ -0,0 +1,40 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.pipeline.FileItem +import java.time.ZonedDateTime +import java.util.regex.Pattern + +/** + * Keeps files whose name fully matches one of the includes (all files if empty) and none of the excludes. + */ +data class NameFilter( + val includes: List = emptyList(), + val excludes: List = emptyList(), +) : SimpleStage{ + override fun process(item: FileItem): StageResult{ + val name = item.source.fileName.toString() + if(excludes.any { it.matcher(name).matches() }){ + return StageResult.Skip("name is excluded") + } + if(includes.isNotEmpty() && includes.none { it.matcher(name).matches() }){ + return StageResult.Skip("name is not included") + } + return StageResult.Continue + } +} + +/** + * Keeps files whose date is older than the given offset. + */ +data class IfOlderThan( + val dayOffset: Long = 0, + val minuteOffset: Long = 0, +) : SimpleStage{ + override fun process(item: FileItem): StageResult{ + val limit = ZonedDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset) + if(item.metadata.date.isAfter(limit)){ + return StageResult.Skip("file is too recent") + } + return StageResult.Continue + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/FolderStages.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/FolderStages.kt new file mode 100644 index 0000000..270b867 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/FolderStages.kt @@ -0,0 +1,108 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.pipeline.FileItem +import java.nio.file.Path +import java.time.temporal.ChronoField +import java.util.Locale + +/** + * Prefixes the destination with folders built from the first characters of its first path element, e.g. `going/file` with 3 elements becomes `g/o/i/going/file`. + */ +data class FolderCanonical( + val elementCount: Int, +) : PathStage{ + override fun transform(item: FileItem): Path{ + val relative = item.relativeDestination + val firstElement = relative.getName(0).toString() + + var canonical = item.outputBase + for(index in 0 until elementCount){ + val part = firstElement.getOrNull(index)?.toString()?.lowercase() + canonical = canonical.resolve(if(part == null || part in FORBIDDEN_PARTS) UNKNOWN_PART else part) + } + return canonical.resolve(relative) + } + + private companion object{ + const val UNKNOWN_PART = "_" + val FORBIDDEN_PARTS = setOf(".", "/", "\\") + } +} + +/** + * Prefixes the destination with `yyyy/MM/dd` folders from the file date. + */ +class FolderPerDay : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve("%02d".format(date.monthValue)) + .resolve("%02d".format(date.dayOfMonth)) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with `yyyy/MM/w` folders from the file date, `w` being the aligned week of the month. + */ +class FolderPerWeek : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve("%02d".format(date.monthValue)) + .resolve("%1d".format(date.get(ChronoField.ALIGNED_WEEK_OF_MONTH))) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with `yyyy/MM` folders from the file date. + */ +class FolderPerMonth : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve("%02d".format(date.monthValue)) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with a `yyyy` folder from the file date. + */ +class FolderPerYear : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with a `yyyy-MM-dd` folder from the file date. + */ +class FolderUniquePerDay : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d-%02d-%02d".format(date.year, date.monthValue, date.dayOfMonth)) + .resolve(item.relativeDestination) + } +} + +/** + * Lowercases the folders of the destination below the output folder, the file name is kept as is. + */ +class Lowercase : PathStage{ + override fun transform(item: FileItem): Path{ + val folder = item.destination.parent + val relativeFolder = item.outputBase.relativize(folder).toString().lowercase(Locale.ROOT) + return item.outputBase.resolve(relativeFolder).resolve(item.destination.fileName) + } +} + +/** + * Restores the original file name, keeping the current destination folder. + */ +class NoRename : PathStage{ + override fun transform(item: FileItem): Path = item.destination.withFileName(item.source.fileName.toString()) +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/RenameStages.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/RenameStages.kt new file mode 100644 index 0000000..b8fe420 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/RenameStages.kt @@ -0,0 +1,77 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.pipeline.FileItem +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.Path +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.Locale +import java.util.regex.Pattern + +private val log = KotlinLogging.logger {} + +/** + * Renames the file with its date, keeping its extension. + */ +data class RenameWithDate( + val format: String = "yyyy-MM-dd HH.mm.ss", + val locale: String = "en", + /** + * When not empty, only files whose original name fully matches one of these are renamed. + */ + val filters: List = emptyList(), + val zone: String? = null, +) : PathStage{ + private val formatter: DateTimeFormatter = DateTimeFormatter.ofPattern(format, Locale.Builder().setLanguage(locale).build()) + .withZone(zone?.let(ZoneId::of) ?: ZoneId.systemDefault()) + + override fun transform(item: FileItem): Path?{ + val sourceName = item.source.fileName.toString() + if(filters.isNotEmpty() && filters.none { it.matcher(sourceName).matches() }){ + log.debug { "Not renaming $sourceName as it doesn't match any filter" } + return null + } + return item.destination.withFileName(formatter.format(item.metadata.date) + item.destination.extensionWithDot) + } +} + +/** + * Renames the file with a date read from its current name, keeping its extension. Files whose name doesn't contain a date are skipped. + * + * The regex must define the named groups `date` and `time`, parsed with [dateFormat] and [timeFormat]. + */ +data class RenameWithDatePattern( + val regex: Pattern, + val dateFormat: String, + val timeFormat: String, + val outputFormat: String = "yyyy-MM-dd HH.mm.ss", + val locale: String = "en", + val zone: String? = null, +) : SimpleStage{ + private val zoneId: ZoneId = zone?.let(ZoneId::of) ?: ZoneId.systemDefault() + private val inputFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern("$dateFormat $timeFormat", Locale.Builder().setLanguage(locale).build()) + .withZone(zoneId) + private val outputFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern(outputFormat, Locale.Builder().setLanguage(locale).build()) + .withZone(zoneId) + + override fun process(item: FileItem): StageResult{ + val date = extractDate(item.destination.fileName.toString()) ?: return StageResult.Skip("no date found in name") + item.destination = item.destination.withFileName(outputFormatter.format(date) + item.destination.extensionWithDot) + return StageResult.Continue + } + + private fun extractDate(name: String): ZonedDateTime?{ + try{ + val matcher = regex.matcher(name) + if(!matcher.matches()){ + return null + } + return ZonedDateTime.parse("${matcher.group("date")} ${matcher.group("time")}", inputFormatter) + } + catch(e: Exception){ + log.error(e) { "Failed to decode date from value `$name` and format `$dateFormat $timeFormat`" } + return null + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/Stage.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/Stage.kt new file mode 100644 index 0000000..19a6ab9 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/Stage.kt @@ -0,0 +1,82 @@ +package fr.rakambda.filesecure.stage + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import fr.rakambda.filesecure.pipeline.FileItem +import fr.rakambda.filesecure.pipeline.PipelineContext +import java.nio.file.Path + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes( + JsonSubTypes.Type(value = NameFilter::class, name = "NameFilter"), + JsonSubTypes.Type(value = IfOlderThan::class, name = "IfOlderThan"), + JsonSubTypes.Type(value = FolderCanonical::class, name = "FolderCanonical"), + JsonSubTypes.Type(value = FolderPerDay::class, name = "FolderPerDay"), + JsonSubTypes.Type(value = FolderPerWeek::class, name = "FolderPerWeek"), + JsonSubTypes.Type(value = FolderPerMonth::class, name = "FolderPerMonth"), + JsonSubTypes.Type(value = FolderPerYear::class, name = "FolderPerYear"), + JsonSubTypes.Type(value = FolderUniquePerDay::class, name = "FolderUniquePerDay"), + JsonSubTypes.Type(value = Lowercase::class, name = "Lowercase"), + JsonSubTypes.Type(value = NoRename::class, name = "NoRename"), + JsonSubTypes.Type(value = RenameWithDate::class, name = "RenameWithDate"), + JsonSubTypes.Type(value = RenameWithDatePattern::class, name = "RenameWithDatePattern"), + JsonSubTypes.Type(value = Convert::class, name = "Convert"), +) +sealed interface Stage{ + /** + * Creates what runs this stage during one execution of a pipeline. + */ + fun open(context: PipelineContext): StageExecutor +} + +fun interface StageExecutor : AutoCloseable{ + suspend fun process(item: FileItem): StageResult + + override fun close(){ + } +} + +sealed interface StageResult{ + data object Continue : StageResult + + data class Skip(val reason: String) : StageResult +} + +/** + * A stage without any state or resource. + */ +sealed interface SimpleStage : Stage{ + fun process(item: FileItem): StageResult + + override fun open(context: PipelineContext): StageExecutor = StageExecutor { process(it) } +} + +/** + * A stage changing the destination of the file. + */ +sealed interface PathStage : SimpleStage{ + /** + * @return The new destination, or null to keep the current one. + */ + fun transform(item: FileItem): Path? + + override fun process(item: FileItem): StageResult{ + transform(item)?.let { item.destination = it } + return StageResult.Continue + } +} + +/** + * The destination path relative to the output folder. + */ +internal val FileItem.relativeDestination: Path + get() = outputBase.relativize(destination) + +internal fun Path.withFileName(name: String): Path = parent?.resolve(name) ?: fileSystem.getPath(name) + +internal val Path.extensionWithDot: String + get(){ + val name = fileName.toString() + val dotIndex = name.lastIndexOf('.') + return if(dotIndex <= 0) "" else name.substring(dotIndex) + } diff --git a/src/main/kotlin/fr/rakambda/filesecure/storage/UselessFileStorage.kt b/src/main/kotlin/fr/rakambda/filesecure/storage/UselessFileStorage.kt new file mode 100644 index 0000000..c86098e --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/storage/UselessFileStorage.kt @@ -0,0 +1,121 @@ +package fr.rakambda.filesecure.storage + +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection +import java.sql.DriverManager +import java.sql.Timestamp +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +private val log = KotlinLogging.logger {} + +/** + * Remembers files that don't need to be converted, to avoid probing them again on the next runs. + */ +interface UselessFileStorage : AutoCloseable{ + /** + * @return true if the file was marked as useless and hasn't been modified since. + */ + fun isUseless(path: Path, lastModified: Timestamp): Boolean + + fun setUseless(path: Path, lastModified: Timestamp) + + /** + * Persists the files marked since the last save. + */ + fun save() + + override fun close() = save() +} + +object NoOpStorage : UselessFileStorage{ + override fun isUseless(path: Path, lastModified: Timestamp) = false + + override fun setUseless(path: Path, lastModified: Timestamp){ + } + + override fun save(){ + } +} + +/** + * H2 database storage, compatible with MediaConverter's database files. + */ +class H2Storage(private val databaseFile: Path) : UselessFileStorage{ + private val useless = ConcurrentHashMap() + private val newUseless = ConcurrentLinkedQueue>() + private val saveLock = ReentrantLock() + + init{ + log.info { "Loading useless files from $databaseFile" } + databaseFile.toAbsolutePath().parent?.let(Files::createDirectories) + connect().use { connection -> + connection.createStatement().use { statement -> + statement.executeUpdate(""" + CREATE TABLE IF NOT EXISTS Useless( + Filee VARCHAR(512) NOT NULL, + LastModified TIMESTAMP, + PRIMARY KEY(Filee) + )""".trimIndent()) + statement.executeUpdate("ALTER TABLE Useless ADD COLUMN IF NOT EXISTS LastModified TIMESTAMP") + statement.executeQuery("SELECT Filee, LastModified FROM Useless").use { result -> + while(result.next()){ + useless[result.getString("Filee")] = result.getTimestamp("LastModified") + } + } + } + } + log.info { "Loaded ${useless.size} useless files" } + } + + override fun isUseless(path: Path, lastModified: Timestamp): Boolean{ + val key = key(path) + if(!useless.containsKey(key)){ + return false + } + val entryLastModified = useless[key] ?: return true + return entryLastModified >= lastModified + } + + override fun setUseless(path: Path, lastModified: Timestamp){ + log.debug { "Marking $path as useless" } + val key = key(path) + useless[key] = lastModified + newUseless.add(key to lastModified) + if(newUseless.size >= SAVE_THRESHOLD){ + save() + } + } + + override fun save(): Unit = saveLock.withLock { + if(newUseless.isEmpty()){ + return + } + connect().use { connection -> + connection.prepareStatement("MERGE INTO Useless(Filee, LastModified) VALUES(?, ?)").use { statement -> + var count = 0 + while(true){ + val (path, lastModified) = newUseless.poll() ?: break + statement.setString(1, path) + statement.setTimestamp(2, lastModified) + statement.addBatch() + count++ + } + statement.executeBatch() + log.info { "Saved $count new useless files" } + } + } + } + + private fun connect(): Connection = DriverManager.getConnection("jdbc:h2:${databaseFile.toAbsolutePath()}") + + private fun key(path: Path) = path.toString().replace("\\", "/") + + private companion object{ + const val SAVE_THRESHOLD = 100 + } +} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 5ef4908..bb4cbaa 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -6,6 +6,9 @@ + + + diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformerTest.java deleted file mode 100644 index 4a04c1e..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformerTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(MockitoExtension.class) -class FolderCanonicalTransformerTest{ - private final FolderCanonicalTransformer tested = new FolderCanonicalTransformer(3); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - var base = Paths.get("/path"); - var out = Paths.get("/path/going/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/g/o/i/going/to/my/file.ext")); - } - - @Test - void itShouldWorkWhenPartIsSmaller() throws AbandonBackupException{ - var base = Paths.get("/path"); - var out = Paths.get("/path/go/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/g/o/_/go/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformerTest.java deleted file mode 100644 index 89dd3e3..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerDayTransformerTest{ - private final FolderPerDayTransformer tested = new FolderPerDayTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/05/12/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformerTest.java deleted file mode 100644 index 6f5a81d..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerMonthTransformerTest{ - private final FolderPerMonthTransformer tested = new FolderPerMonthTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/05/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformerTest.java deleted file mode 100644 index a9a98c6..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerWeekTransformerTest{ - private final FolderPerWeekTransformer tested = new FolderPerWeekTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/05/2/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformerTest.java deleted file mode 100644 index 28d467d..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerYearTransformerTest{ - private final FolderPerYearTransformer tested = new FolderPerYearTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformerTest.java deleted file mode 100644 index 97f72b1..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderUniquePerDayTransformerTest{ - private final FolderUniquePerDayTransformer tested = new FolderUniquePerDayTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023-05-12/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformerTest.java deleted file mode 100644 index c9adf79..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformerTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(MockitoExtension.class) -class LowercaseTransformerTest{ - private final LowercaseTransformer tested = new LowercaseTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - var base = Paths.get("/pAth/tO"); - var out = Paths.get("/pAth/tO/mY/sUper/fIle.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/pAth/tO/my/super/fIle.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformerTest.java deleted file mode 100644 index a3e1423..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformerTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(MockitoExtension.class) -class NoRenameTransformerTest{ - private final NoRenameTransformer tested = new NoRenameTransformer(); - - @Mock - private Path originalOut; - @Mock - private Path baseOutput; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - var source = Paths.get("/path/to/source.other"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, baseOutput, out, metadata)) - .contains(Paths.get("/path/to/my/source.other")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformerTest.java deleted file mode 100644 index edb1846..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformerTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class RenameWithDateTransformerTest{ - private final RenameWithDateTransformer tested = new RenameWithDateTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private Path base; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException, IllegalAccessException, NoSuchFieldException{ - tested.setZone("UTC+2"); - - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/to/my/2023-05-12 12.15.12.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformerTest.java deleted file mode 100644 index 46b55a9..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformerTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import fr.rakambda.filesecure.utils.FileOperations; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -@ExtendWith(MockitoExtension.class) -class DeleteIfEmptyTransformerTest{ - private final DeleteIfEmptyTransformer tested = new DeleteIfEmptyTransformer(2); - - @Mock - private FileOperations fileOperations; - - private static Stream validPaths(){ - return Stream.of( - Arguments.of(Paths.get("/path/to/the")), - Arguments.of(Paths.get("/path/to/the/folder")), - Arguments.of(Paths.get("/path/to/the/folder/yes")) - ); - } - - private static Stream depthInvalidPaths(){ - return Stream.of( - Arguments.of(Paths.get("/path")), - Arguments.of(Paths.get("/path/to")) - ); - } - - @ParameterizedTest - @MethodSource("validPaths") - void itShouldWorkOnEmptyFolder(Path folder) throws IOException{ - var base = Paths.get("/path"); - - try(var filesMock = mockStatic(Files.class)){ - filesMock.when(() -> Files.list(folder)).thenReturn(Stream.of()); - - tested.apply(fileOperations, folder, base); - - verify(fileOperations).delete(folder); - } - } - - @Test - void itShouldSkipNotEmpty() throws IOException{ - var folder = Paths.get("/path/to/the/folder"); - var base = Paths.get("/path"); - - try(var filesMock = mockStatic(Files.class)){ - filesMock.when(() -> Files.list(folder)).thenReturn(Stream.of(folder.resolve("fake"))); - - tested.apply(fileOperations, folder, base); - - verify(fileOperations, never()).delete(any()); - } - } - - @ParameterizedTest - @MethodSource("depthInvalidPaths") - void itShouldSkipIfDepthIsNotEnough(Path folder) throws IOException{ - var base = Paths.get("/path"); - - try(var filesMock = mockStatic(Files.class)){ - filesMock.when(() -> Files.list(folder)).thenReturn(Stream.empty()); - - tested.apply(fileOperations, folder, base); - - verify(fileOperations, never()).delete(any()); - } - } -} \ No newline at end of file diff --git a/src/test/kotlin/fr/rakambda/filesecure/config/ConfigurationTest.kt b/src/test/kotlin/fr/rakambda/filesecure/config/ConfigurationTest.kt new file mode 100644 index 0000000..54745e2 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/config/ConfigurationTest.kt @@ -0,0 +1,94 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException +import fr.rakambda.filesecure.conversion.LengthMoreThan +import fr.rakambda.filesecure.conversion.ProcessorType +import fr.rakambda.filesecure.folder.DeleteIfEmpty +import fr.rakambda.filesecure.stage.Convert +import fr.rakambda.filesecure.stage.FolderPerMonth +import fr.rakambda.filesecure.stage.NameFilter +import fr.rakambda.filesecure.stage.RenameWithDate +import fr.rakambda.filesecure.stage.UnhandledStrategy +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import java.nio.file.Path + +class ConfigurationTest{ + @Test + fun parsesFullPipeline(){ + val configuration = Configuration.parse(""" + { + // Comments and trailing commas are allowed + "pipelines": [ + { + "name": "Phone", + "mappings": [{"input": "/in", "output": "/out"}], + "scan": {"maxDepth": 3, "skipHidden": true, "excludedFolders": ["\\.thumbnails"]}, + "stages": [ + {"type": "NameFilter", "includes": [".*\\.mp4"]}, + {"type": "Convert", "processors": ["video_to_hevc_mkv"], "filters": [{"type": "LengthMoreThan", "value": 50}], "unhandled": "PASS", "cache": "/db/cache"}, + {"type": "FolderPerMonth"}, + {"type": "RenameWithDate", "format": "yyyy-MM-dd"}, + ], + "operation": "move", + "conflict": "skip", + "setFileCreationDate": true, + "folderActions": [{"type": "DeleteIfEmpty", "depth": 2}], + }, + ], + } + """.trimIndent()) + + val pipeline = configuration.pipelines.single() + assertThat(pipeline.name).isEqualTo("Phone") + assertThat(pipeline.mappings).containsExactly(Mapping(Path.of("/in"), Path.of("/out"))) + assertThat(pipeline.scan.maxDepth).isEqualTo(3) + assertThat(pipeline.scan.skipHidden).isTrue() + assertThat(pipeline.scan.excludedFolders.single().pattern()).isEqualTo("""\.thumbnails""") + assertThat(pipeline.stages).hasSize(4) + assertThat(pipeline.stages[0]).isInstanceOf(NameFilter::class.java) + assertThat(pipeline.stages[1]).isEqualTo(Convert( + processors = listOf(ProcessorType.VIDEO_TO_HEVC_MKV), + filters = listOf(LengthMoreThan(50)), + unhandled = UnhandledStrategy.PASS, + cache = Path.of("/db/cache"), + )) + assertThat(pipeline.stages[2]).isInstanceOf(FolderPerMonth::class.java) + assertThat((pipeline.stages[3] as RenameWithDate).format).isEqualTo("yyyy-MM-dd") + assertThat(pipeline.operation).isEqualTo(Operation.MOVE) + assertThat(pipeline.conflict).isEqualTo(ConflictStrategy.SKIP) + assertThat(pipeline.setFileCreationDate).isTrue() + assertThat(pipeline.folderActions).containsExactly(DeleteIfEmpty(2)) + } + + @Test + fun appliesDefaults(){ + val pipeline = Configuration.parse("""{"pipelines": [{"mappings": [{"input": "/in", "output": "/out"}]}]}""").pipelines.single() + + assertThat(pipeline.enabled).isTrue() + assertThat(pipeline.operation).isEqualTo(Operation.NONE) + assertThat(pipeline.conflict).isEqualTo(ConflictStrategy.RENAME) + assertThat(pipeline.scan.maxDepth).isEqualTo(Int.MAX_VALUE) + assertThat(pipeline.stages).isEmpty() + } + + @Test + fun rejectsUnknownProperties(){ + assertThatThrownBy { Configuration.parse("""{"pipelines": [{"mappings": [], "depth": 2}]}""") } + .isInstanceOf(UnrecognizedPropertyException::class.java) + } + + @Test + fun rejectsSeveralConversions(){ + assertThatThrownBy { + Configuration.parse("""{"pipelines": [{"mappings": [{"input": "/in", "output": "/out"}], "stages": [{"type": "Convert"}, {"type": "Convert"}]}]}""") + }.isInstanceOf(InvalidConfigurationException::class.java) + } + + @Test + fun rejectsPipelineWithoutMapping(){ + assertThatThrownBy { Configuration.parse("""{"pipelines": [{}]}""") } + .isInstanceOf(InvalidConfigurationException::class.java) + } +} diff --git a/src/test/kotlin/fr/rakambda/filesecure/folder/FolderActionTest.kt b/src/test/kotlin/fr/rakambda/filesecure/folder/FolderActionTest.kt new file mode 100644 index 0000000..1431401 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/folder/FolderActionTest.kt @@ -0,0 +1,83 @@ +package fr.rakambda.filesecure.folder + +import fr.rakambda.filesecure.pipeline.FileOperations +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.FileTime +import java.time.Instant +import java.time.temporal.ChronoUnit + +class FolderActionTest{ + @TempDir + lateinit var root: Path + + private val fileOperations = FileOperations(false) + + @Test + fun deleteIfEmptyRespectsDepth(){ + val level1 = Files.createDirectories(root.resolve("a")) + val level2 = Files.createDirectories(level1.resolve("b")) + val action = DeleteIfEmpty(depth = 2) + + action.apply(fileOperations, level2, root) + action.apply(fileOperations, level1, root) + + assertThat(level2).doesNotExist() + assertThat(level1).exists() + } + + @Test + fun deleteIfEmptyNeverDeletesInputWithDefaultDepth(){ + DeleteIfEmpty().apply(fileOperations, root, root) + + assertThat(root).exists() + } + + @Test + fun deleteIfEmptyKeepsNonEmptyFolders(){ + val folder = Files.createDirectories(root.resolve("a")) + Files.createFile(folder.resolve("file")) + + DeleteIfEmpty().apply(fileOperations, folder, root) + + assertThat(folder).exists() + } + + @Test + fun deleteIfEmptyDoesNothingInDryRun(){ + val folder = Files.createDirectories(root.resolve("a")) + + DeleteIfEmpty().apply(FileOperations(true), folder, root) + + assertThat(folder).exists() + } + + @Test + fun deleteIfOlderThan(){ + val old = Files.createDirectories(root.resolve("old")) + val recent = Files.createDirectories(root.resolve("recent")) + Files.setLastModifiedTime(old, FileTime.from(Instant.now().minus(3, ChronoUnit.DAYS))) + val action = DeleteIfOlderThan(dayOffset = 2) + + action.apply(fileOperations, old, root) + action.apply(fileOperations, recent, root) + + assertThat(old).doesNotExist() + assertThat(recent).exists() + } + + @Test + fun deleteChildren(){ + val folder = Files.createDirectories(root.resolve("a")) + Files.createFile(folder.resolve("Thumbs.db")) + Files.createFile(folder.resolve("keep.jpg")) + + DeleteChildren(listOf("Thumbs.db", "desktop.ini")).apply(fileOperations, folder, root) + + assertThat(folder.resolve("Thumbs.db")).doesNotExist() + assertThat(folder.resolve("keep.jpg")).exists() + } +} diff --git a/src/test/kotlin/fr/rakambda/filesecure/pipeline/ConversionPipelineTest.kt b/src/test/kotlin/fr/rakambda/filesecure/pipeline/ConversionPipelineTest.kt new file mode 100644 index 0000000..708a672 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/pipeline/ConversionPipelineTest.kt @@ -0,0 +1,222 @@ +package fr.rakambda.filesecure.pipeline + +import com.github.kokorin.jaffree.ffprobe.FFprobe +import fr.rakambda.filesecure.config.ConflictStrategy +import fr.rakambda.filesecure.config.Mapping +import fr.rakambda.filesecure.config.Operation +import fr.rakambda.filesecure.config.Pipeline +import fr.rakambda.filesecure.conversion.ConversionTools +import fr.rakambda.filesecure.conversion.LengthMoreThan +import fr.rakambda.filesecure.conversion.ProcessorType +import fr.rakambda.filesecure.metadata.MetadataExtractor +import fr.rakambda.filesecure.stage.Convert +import fr.rakambda.filesecure.stage.UnhandledStrategy +import fr.rakambda.filesecure.storage.H2Storage +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Timestamp +import kotlin.io.path.createDirectories +import kotlin.io.path.getLastModifiedTime +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.writeText +import kotlin.time.Duration.Companion.seconds + +/** + * Runs real conversions, skipped when ffmpeg isn't installed. + */ +class ConversionPipelineTest{ + @TempDir + lateinit var root: Path + + private val input by lazy { root.resolve("in").createDirectories() } + private val output by lazy { root.resolve("out") } + private val temp by lazy { root.resolve("temp") } + + @BeforeEach + fun requireFfmpeg(){ + assumeTrue(commandSucceeds("ffmpeg", "-version"), "ffmpeg is not installed") + } + + @Test + fun convertsAndMovesAudio(){ + val mp3 = generateMp3(input.resolve("music/song.mp3")) + val modified = mp3.getLastModifiedTime() + val text = input.resolve("music/notes.txt").also { it.writeText("notes") } + + run(convertPipeline(Operation.MOVE, Convert(processors = listOf(ProcessorType.MP3_TO_AAC)))) + + val converted = output.resolve("music/song.m4a") + assertThat(converted).exists() + assertThat(codecs(converted)).containsExactly("aac") + assertThat(converted.getLastModifiedTime()).describedAs("times copied from source").isEqualTo(modified) + assertThat(mp3).doesNotExist() + assertThat(text).describedAs("not convertible files are skipped by default").exists() + assertThat(output.resolve("music/notes.txt")).doesNotExist() + assertThat(temp.listDirectoryEntries()).describedAs("temporary files cleaned").isEmpty() + } + + @Test + fun passesNotConvertibleFiles(){ + generateMp3(input.resolve("song.mp3")) + input.resolve("notes.txt").writeText("notes") + + run(convertPipeline(Operation.COPY, Convert(processors = listOf(ProcessorType.MP3_TO_AAC), unhandled = UnhandledStrategy.PASS))) + + assertThat(output.resolve("song.m4a")).exists() + assertThat(output.resolve("notes.txt")).hasContent("notes") + assertThat(input.resolve("song.mp3")).exists() + } + + @Test + fun skipsConversionWhenDestinationExists(){ + generateMp3(input.resolve("song.mp3")) + val existing = output.createDirectories().resolve("song.m4a").also { it.writeText("existing") } + + run(convertPipeline(Operation.MOVE, Convert(processors = listOf(ProcessorType.MP3_TO_AAC))).copy(conflict = ConflictStrategy.SKIP)) + + assertThat(existing).hasContent("existing") + assertThat(input.resolve("song.mp3")).exists() + } + + @Test + fun probeFiltersSkipShortMedia(){ + generateMp3(input.resolve("song.mp3")) + + run(convertPipeline(Operation.COPY, Convert(processors = listOf(ProcessorType.MP3_TO_AAC), filters = listOf(LengthMoreThan(60))))) + + assertThat(output).doesNotExist() + } + + @Test + fun dryRunDoesNotConvert(){ + val mp3 = generateMp3(input.resolve("song.mp3")) + + run(convertPipeline(Operation.MOVE, Convert(processors = listOf(ProcessorType.MP3_TO_AAC))), dryRun = true) + + assertThat(mp3).exists() + assertThat(output).doesNotExist() + } + + @Test + fun replacesSourceInPlace(){ + val video = input.resolve("video.mkv") + ffmpeg("-f", "lavfi", "-i", "testsrc=duration=1:size=64x64:rate=10", "-c:v", "libx264", video.toString()) + + run(Pipeline( + mappings = listOf(Mapping(input, input)), + stages = listOf(Convert(processors = listOf(ProcessorType.VIDEO_TO_HEVC_MKV))), + operation = Operation.MOVE, + temp = temp, + )) + + assertThat(codecs(video)).containsExactly("hevc") + assertThat(input.listDirectoryEntries()).containsExactly(video) + } + + @Test + fun cancellationStopsRunningConversion(): Unit = runBlocking { + val video = input.resolve("long.mp4") + ffmpeg("-f", "lavfi", "-i", "testsrc=duration=60:size=640x360:rate=30", "-c:v", "libx264", "-preset", "ultrafast", video.toString()) + val environment = RunEnvironment(FileOperations(false), MetadataExtractor({ _, _ -> null }), ConversionTools()) + + val job = launch(Dispatchers.Default) { + PipelineRunner(environment).run(convertPipeline(Operation.MOVE, Convert(processors = listOf(ProcessorType.VIDEO_TO_HEVC_MKV))), "test") + } + withTimeout(30.seconds) { + while(childFfmpegCount() == 0L){ + delay(100) + } + } + job.cancelAndJoin() + + withTimeout(5.seconds) { + while(childFfmpegCount() > 0){ + delay(100) + } + } + assertThat(video).exists() + assertThat(output).doesNotExist() + assertThat(temp.listDirectoryEntries()).isEmpty() + } + + private fun childFfmpegCount() = ProcessHandle.current().descendants() + .filter { it.info().command().orElse("").endsWith("ffmpeg") } + .count() + + @Test + fun convertsTiffWithImageMagick(){ + assumeTrue(commandSucceeds("convert", "-version"), "ImageMagick is not installed") + val tiff = input.resolve("image.tiff") + assertThat(commandSucceeds("convert", "-size", "8x8", "xc:red", tiff.toString())).isTrue() + + run(convertPipeline(Operation.COPY, Convert(processors = listOf(ProcessorType.TIFF_TO_JPG))), magick = "convert") + + assertThat(codecs(output.resolve("image.jpg"))).containsExactly("mjpeg") + } + + @Test + fun cachesNotConvertibleFiles(){ + val text = input.resolve("notes.txt").also { it.writeText("notes") } + val video = input.resolve("video.mkv").also { it.writeText("not really a video") } + val database = root.resolve("cache/db") + + run(convertPipeline(Operation.COPY, Convert(cache = database))) + + H2Storage(database).use { + assertThat(it.isUseless(text, Timestamp(Files.getLastModifiedTime(text).toMillis()))).isTrue() + assertThat(it.isUseless(video, Timestamp(Files.getLastModifiedTime(video).toMillis()))).isTrue() + } + } + + private fun convertPipeline(operation: Operation, convert: Convert) = Pipeline( + mappings = listOf(Mapping(input, output)), + stages = listOf(convert), + operation = operation, + temp = temp, + ) + + private fun run(pipeline: Pipeline, dryRun: Boolean = false, magick: String = "magick") = runBlocking { + val environment = RunEnvironment( + fileOperations = FileOperations(dryRun), + metadataExtractor = MetadataExtractor({ _, _ -> null }), + tools = ConversionTools(magick = magick), + conversionThreads = 2, + ) + PipelineRunner(environment).run(pipeline, "test") + } + + private fun generateMp3(path: Path): Path{ + path.parent.createDirectories() + ffmpeg("-f", "lavfi", "-i", "sine=frequency=440:duration=1", "-c:a", "libmp3lame", path.toString()) + return path + } + + private fun ffmpeg(vararg args: String){ + assertThat(commandSucceeds("ffmpeg", "-hide_banner", "-loglevel", "error", *args)).isTrue() + } + + private fun codecs(path: Path): List = FFprobe.atPath() + .setShowStreams(true) + .setInput(path.toString()) + .execute() + .streams + .map { it.codecName } + + private fun commandSucceeds(vararg command: String): Boolean = try{ + ProcessBuilder(*command).redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start().waitFor() == 0 + } + catch(_: Exception){ + false + } +} diff --git a/src/test/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserverTest.kt b/src/test/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserverTest.kt new file mode 100644 index 0000000..e638657 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserverTest.kt @@ -0,0 +1,57 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.ConflictStrategy +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +class DestinationReserverTest{ + @TempDir + lateinit var root: Path + + @Test + fun numberedName(){ + assertThat(DestinationReserver.numberedName("file.ext", 2)).isEqualTo("file (2).ext") + assertThat(DestinationReserver.numberedName("archive.tar.gz", 1)).isEqualTo("archive.tar (1).gz") + assertThat(DestinationReserver.numberedName("Makefile", 1)).isEqualTo("Makefile (1)") + assertThat(DestinationReserver.numberedName(".bashrc", 1)).isEqualTo(".bashrc (1)") + } + + @Test + fun keepsFreeDestination(): Unit = runBlocking { + val destination = root.resolve("file.txt") + + assertThat(DestinationReserver().reserve(destination, ConflictStrategy.SKIP)).isEqualTo(destination) + } + + @Test + fun renamesExistingDestination(): Unit = runBlocking { + val destination = Files.createFile(root.resolve("file.txt")) + Files.createFile(root.resolve("file (1).txt")) + + assertThat(DestinationReserver().reserve(destination, ConflictStrategy.RENAME)).isEqualTo(root.resolve("file (2).txt")) + } + + @Test + fun skipsExistingDestination(): Unit = runBlocking { + val destination = Files.createFile(root.resolve("file.txt")) + + assertThat(DestinationReserver().reserve(destination, ConflictStrategy.SKIP)).isNull() + } + + @Test + fun concurrentReservationsGetDistinctNames(): Unit = runBlocking { + val reserver = DestinationReserver() + val destination = root.resolve("file.txt") + + val reserved = (1..50).map { async(Dispatchers.Default) { reserver.reserve(destination, ConflictStrategy.RENAME) } }.awaitAll() + + assertThat(reserved).doesNotHaveDuplicates().hasSize(50) + } +} diff --git a/src/test/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunnerTest.kt b/src/test/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunnerTest.kt new file mode 100644 index 0000000..1d0a1c4 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunnerTest.kt @@ -0,0 +1,223 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.ConflictStrategy +import fr.rakambda.filesecure.config.Mapping +import fr.rakambda.filesecure.config.Operation +import fr.rakambda.filesecure.config.Pipeline +import fr.rakambda.filesecure.config.ScanOptions +import fr.rakambda.filesecure.conversion.ConversionTools +import fr.rakambda.filesecure.folder.DeleteIfEmpty +import fr.rakambda.filesecure.metadata.MetadataExtractor +import fr.rakambda.filesecure.stage.FolderPerYear +import fr.rakambda.filesecure.stage.NameFilter +import fr.rakambda.filesecure.stage.RenameWithDate +import fr.rakambda.filesecure.stage.Stage +import kotlinx.coroutines.runBlocking +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.FileTime +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneId +import java.util.regex.Pattern +import kotlin.io.path.createDirectories +import kotlin.io.path.getLastModifiedTime +import kotlin.io.path.readText +import kotlin.io.path.setLastModifiedTime +import kotlin.io.path.writeText + +class PipelineRunnerTest{ + @TempDir + lateinit var root: Path + + private val input by lazy { root.resolve("in").createDirectories() } + private val output by lazy { root.resolve("out") } + + private fun file(relative: String, content: String = relative): Path = input.resolve(relative).also { + it.parent.createDirectories() + it.writeText(content) + } + + private fun run(pipeline: Pipeline, dryRun: Boolean = false) = runBlocking { + val environment = RunEnvironment( + fileOperations = FileOperations(dryRun), + metadataExtractor = MetadataExtractor({ _, _ -> null }), + tools = ConversionTools(), + workers = 4, + ) + PipelineRunner(environment).run(pipeline, "test") + } + + private fun pipeline(operation: Operation, vararg stages: Stage) = Pipeline( + mappings = listOf(Mapping(input, output)), + operation = operation, + stages = stages.toList(), + ) + + @Test + fun copyKeepsStructure(){ + val a = file("a/b.txt") + val c = file("c.txt") + + run(pipeline(Operation.COPY)) + + assertThat(output.resolve("a/b.txt")).hasContent("a/b.txt") + assertThat(output.resolve("c.txt")).hasContent("c.txt") + assertThat(a).exists() + assertThat(c).exists() + } + + @Test + fun moveRemovesSourcesAndCleansFolders(){ + val a = file("a/b.txt") + + run(pipeline(Operation.MOVE).copy(folderActions = listOf(DeleteIfEmpty()))) + + assertThat(output.resolve("a/b.txt")).hasContent("a/b.txt") + assertThat(a).doesNotExist() + assertThat(a.parent).doesNotExist() + assertThat(input).describedAs("input folder itself is kept").exists() + } + + @Test + fun noneOperationWritesNothing(){ + val a = file("a.txt") + + run(pipeline(Operation.NONE)) + + assertThat(a).exists() + assertThat(output).doesNotExist() + } + + @Test + fun dryRunWritesNothing(){ + val a = file("a/b.txt") + + run(pipeline(Operation.MOVE).copy(folderActions = listOf(DeleteIfEmpty())), dryRun = true) + + assertThat(a).exists() + assertThat(output).doesNotExist() + } + + @Test + fun conflictRename(){ + file("a.txt", "new") + output.createDirectories().resolve("a.txt").writeText("old") + + run(pipeline(Operation.COPY)) + + assertThat(output.resolve("a.txt")).hasContent("old") + assertThat(output.resolve("a (1).txt")).hasContent("new") + } + + @Test + fun conflictRenameWithoutExtension(){ + file("Makefile", "new") + output.createDirectories().resolve("Makefile").writeText("old") + + run(pipeline(Operation.COPY)) + + assertThat(output.resolve("Makefile (1)")).hasContent("new") + } + + @Test + fun conflictSkip(){ + val a = file("a.txt", "new") + output.createDirectories().resolve("a.txt").writeText("old") + + run(pipeline(Operation.MOVE).copy(conflict = ConflictStrategy.SKIP)) + + assertThat(output.resolve("a.txt")).hasContent("old") + assertThat(a).exists() + assertThat(output.resolve("a (1).txt")).doesNotExist() + } + + @Test + fun sameDestinationForSeveralFilesIsRenamed(){ + val date = FileTime.from(Instant.parse("2020-06-01T12:00:00Z")) + file("x.txt", "x").setLastModifiedTime(date) + file("y.txt", "y").setLastModifiedTime(date) + + run(pipeline(Operation.COPY, RenameWithDate(format = "yyyy", zone = "UTC"))) + + assertThat(listOf(output.resolve("2020.txt").readText(), output.resolve("2020 (1).txt").readText())).containsExactlyInAnyOrder("x", "y") + } + + @Test + fun inPlaceMoveDoesNothing(){ + val a = file("a.txt") + + run(Pipeline(mappings = listOf(Mapping(input, input)), operation = Operation.MOVE)) + + assertThat(a).hasContent("a.txt") + assertThat(input.resolve("a (1).txt")).doesNotExist() + } + + @Test + fun scanOptions(){ + file("keep.txt") + file("deep/deeper.txt") + file("excluded/file.txt") + file(".hidden") + file(".hiddenFolder/file.txt") + + run(pipeline(Operation.COPY).copy(scan = ScanOptions(maxDepth = 2, skipHidden = true, excludedFolders = listOf(Pattern.compile("excl.*"))))) + + assertThat(output.resolve("keep.txt")).exists() + assertThat(output.resolve("deep/deeper.txt")).exists() + assertThat(output.resolve("excluded")).doesNotExist() + assertThat(output.resolve(".hidden")).doesNotExist() + assertThat(output.resolve(".hiddenFolder")).doesNotExist() + } + + @Test + fun maxDepthDoesNotProcessFoldersAsFiles(){ + val deep = file("deep/deeper.txt") + file("keep.txt") + + run(pipeline(Operation.MOVE).copy(scan = ScanOptions(maxDepth = 1))) + + assertThat(output.resolve("keep.txt")).exists() + assertThat(output.resolve("deep")).doesNotExist() + assertThat(deep).exists() + } + + @Test + fun stagesRunInOrder(){ + file("2023-05-12 10.15.12.jpg") + file("2022-01-02 03.04.05.txt") + + run(pipeline( + Operation.COPY, + NameFilter(includes = listOf(Pattern.compile(""".*\.jpg"""))), + FolderPerYear(), + RenameWithDate(format = "yyyyMMdd_HHmmss"), + )) + + assertThat(output.resolve("2023/20230512_101512.jpg")).exists() + assertThat(output.resolve("2022")).doesNotExist() + } + + @Test + fun setFileCreationDate(){ + file("2023-05-12 10.15.12.txt") + + run(pipeline(Operation.MOVE).copy(setFileCreationDate = true)) + + val expected = LocalDateTime.of(2023, 5, 12, 10, 15, 12).atZone(ZoneId.systemDefault()).toInstant() + assertThat(output.resolve("2023-05-12 10.15.12.txt").getLastModifiedTime().toInstant()).isEqualTo(expected) + } + + @Test + fun missingInputIsSkipped(){ + Files.delete(input) + + run(pipeline(Operation.COPY)) + + assertThat(output).doesNotExist() + } +} + diff --git a/src/test/kotlin/fr/rakambda/filesecure/stage/FilterStagesTest.kt b/src/test/kotlin/fr/rakambda/filesecure/stage/FilterStagesTest.kt new file mode 100644 index 0000000..3412264 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/stage/FilterStagesTest.kt @@ -0,0 +1,43 @@ +package fr.rakambda.filesecure.stage + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import java.nio.file.Path +import java.time.ZonedDateTime +import java.util.regex.Pattern + +class FilterStagesTest{ + private val base = Path.of("/out") + + private fun item(name: String, date: ZonedDateTime? = null) = fileItem(base, base.resolve(name), source = Path.of("/input", name), date = date) + + @Test + fun nameFilterKeepsEverythingByDefault(){ + assertThat(NameFilter().process(item("a.txt"))).isEqualTo(StageResult.Continue) + } + + @Test + fun nameFilterIncludes(){ + val filter = NameFilter(includes = listOf(Pattern.compile(""".*\.png"""), Pattern.compile(""".*\.jpg"""))) + + assertThat(filter.process(item("a.png"))).isEqualTo(StageResult.Continue) + assertThat(filter.process(item("a.jpg"))).isEqualTo(StageResult.Continue) + assertThat(filter.process(item("a.png.txt"))).isInstanceOf(StageResult.Skip::class.java) + } + + @Test + fun nameFilterExcludesWin(){ + val filter = NameFilter(includes = listOf(Pattern.compile(".*")), excludes = listOf(Pattern.compile(""".*\.ini"""))) + + assertThat(filter.process(item("desktop.ini"))).isInstanceOf(StageResult.Skip::class.java) + assertThat(filter.process(item("a.png"))).isEqualTo(StageResult.Continue) + } + + @Test + fun ifOlderThan(){ + val stage = IfOlderThan(dayOffset = 2) + + assertThat(stage.process(item("a", ZonedDateTime.now().minusDays(3)))).isEqualTo(StageResult.Continue) + assertThat(stage.process(item("a", ZonedDateTime.now().minusDays(1)))).isInstanceOf(StageResult.Skip::class.java) + } +} diff --git a/src/test/kotlin/fr/rakambda/filesecure/stage/PathStagesTest.kt b/src/test/kotlin/fr/rakambda/filesecure/stage/PathStagesTest.kt new file mode 100644 index 0000000..8fd9929 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/stage/PathStagesTest.kt @@ -0,0 +1,119 @@ +package fr.rakambda.filesecure.stage + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import java.nio.file.Path +import java.time.ZoneOffset.UTC +import java.time.ZonedDateTime +import java.util.regex.Pattern + +class PathStagesTest{ + private val date = ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC) + private val base = Path.of("/path") + private val out = Path.of("/path/to/my/file.ext") + + @Test + fun folderPerDay(){ + assertThat(FolderPerDay().transform(fileItem(base, out, date = date))).isEqualTo(Path.of("/path/2023/05/12/to/my/file.ext")) + } + + @Test + fun folderPerWeek(){ + assertThat(FolderPerWeek().transform(fileItem(base, out, date = date))).isEqualTo(Path.of("/path/2023/05/2/to/my/file.ext")) + } + + @Test + fun folderPerMonth(){ + assertThat(FolderPerMonth().transform(fileItem(base, out, date = date))).isEqualTo(Path.of("/path/2023/05/to/my/file.ext")) + } + + @Test + fun folderPerYear(){ + assertThat(FolderPerYear().transform(fileItem(base, out, date = date))).isEqualTo(Path.of("/path/2023/to/my/file.ext")) + } + + @Test + fun folderUniquePerDay(){ + assertThat(FolderUniquePerDay().transform(fileItem(base, out, date = date))).isEqualTo(Path.of("/path/2023-05-12/to/my/file.ext")) + } + + @Test + fun folderCanonical(){ + val item = fileItem(base, Path.of("/path/going/to/my/file.ext")) + assertThat(FolderCanonical(3).transform(item)).isEqualTo(Path.of("/path/g/o/i/going/to/my/file.ext")) + } + + @Test + fun folderCanonicalWhenPartIsSmaller(){ + val item = fileItem(base, Path.of("/path/go/to/my/file.ext")) + assertThat(FolderCanonical(3).transform(item)).isEqualTo(Path.of("/path/g/o/_/go/to/my/file.ext")) + } + + @Test + fun folderCanonicalReplacesForbiddenCharacters(){ + val item = fileItem(base, Path.of("/path/.git/file.ext")) + assertThat(FolderCanonical(2).transform(item)).isEqualTo(Path.of("/path/_/g/.git/file.ext")) + } + + @Test + fun lowercaseOnlyChangesFolders(){ + val item = fileItem(Path.of("/pAth/tO"), Path.of("/pAth/tO/mY/sUper/fIle.ext")) + assertThat(Lowercase().transform(item)).isEqualTo(Path.of("/pAth/tO/my/super/fIle.ext")) + } + + @Test + fun lowercaseFileAtRoot(){ + val item = fileItem(Path.of("/pAth"), Path.of("/pAth/fIle.ext")) + assertThat(Lowercase().transform(item)).isEqualTo(Path.of("/pAth/fIle.ext")) + } + + @Test + fun noRename(){ + val item = fileItem(Path.of("/base"), out, source = Path.of("/path/to/source.other")) + assertThat(NoRename().transform(item)).isEqualTo(Path.of("/path/to/my/source.other")) + } + + @Test + fun renameWithDate(){ + val item = fileItem(base, out, date = date) + assertThat(RenameWithDate(zone = "UTC+2").transform(item)).isEqualTo(Path.of("/path/to/my/2023-05-12 12.15.12.ext")) + } + + @Test + fun renameWithDateWithoutExtension(){ + val item = fileItem(base, Path.of("/path/to/file"), date = date) + assertThat(RenameWithDate(zone = "UTC").transform(item)).isEqualTo(Path.of("/path/to/2023-05-12 10.15.12")) + } + + @Test + fun renameWithDateIgnoresFilesNotMatchingFilters(){ + val item = fileItem(base, out, date = date) + assertThat(RenameWithDate(filters = listOf(Pattern.compile("IMG_.*"))).transform(item)).isNull() + } + + @Test + fun renameWithDatePattern(){ + val stage = RenameWithDatePattern( + regex = Pattern.compile("""Screenshot (?\d{8}) (?