diff --git a/.editorconfig b/.editorconfig index 7f167a8..eac265f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -260,6 +260,9 @@ ij_java_wrap_comments = false ij_java_wrap_first_method_in_call_chain = false ij_java_wrap_long_lines = false +[*.kt] +indent_style = tab + [{*.ad,*.adoc,*.asciidoc,.asciidoctorconfig}] ij_asciidoc_blank_lines_after_header = 1 ij_asciidoc_blank_lines_keep_after_header = 1 diff --git a/.github/workflows/gradle_ci.yml b/.github/workflows/gradle_ci.yml index 8a53c30..48b982f 100644 --- a/.github/workflows/gradle_ci.yml +++ b/.github/workflows/gradle_ci.yml @@ -42,3 +42,7 @@ jobs: docker-repo: ${{ secrets.DOCKER_REPO }} docker-username: ${{ secrets.DOCKER_USERNAME }} docker-token: ${{ secrets.DOCKER_TOKEN }} + jib-extra-params: | + -Djib.from.image=${{ secrets.DOCKER_REPO }}/rakambda/eclipse-temurin-ffmpeg:9.0-27-jre \ + -Djib.from.auth.username=${{ secrets.DOCKER_USERNAME }} \ + -Djib.from.auth.password=${{ secrets.DOCKER_TOKEN }} diff --git a/.gitignore b/.gitignore index c79f519..e9ffcd9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ target/ *.iml config.json FileSecure.json +config.yaml +config.yml diff --git a/README.md b/README.md index ee26f45..e29c401 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,433 @@ # FileSecure -Backup files from a base directory to a target directory. +Moves, backs up and converts files through pipelines of stages. -To work this program needs a configuration file that will be passed as a parameter. The config file file must be in a json format. Here's an example: +Every file found in the input folder of a pipeline goes through its stages in order. A stage can filter the file out, change its destination (folders, name) +or plan its conversion into another format. Once all stages accepted the file, the conversion (if any) runs and the file is copied or moved to its +destination. + +Version 13 is a rewrite in Kotlin merging FileSecure and [MediaConverter](https://github.com/RakambdaOrg/MediaConverter). Its configuration format is +new, see [migrating](#migrating-from-previous-versions). + +## Running + +```shell +java -jar FileSecure.jar -c config.yaml +``` + +| Flag | Description | +|----------------------------|---------------------------------------------------------------------------------------------------------------| +| `-c`, `--config` | **Mandatory**. Path to the configuration file. | +| `--dry` | Dry run: everything is logged, no file is copied, moved, converted or deleted (the `Convert` cache is still updated). | +| `-t`, `--threads` | How many conversions run in parallel. Default 1. | +| `-w`, `--workers` | How many files are processed in parallel (scanning, probing, copying). Default 8. | +| `-ft`, `--ffmpeg-threads` | Value of ffmpeg's `-threads` option for each conversion. | +| `--ffmpeg` | Folder containing the `ffmpeg` executable. Uses the `PATH` if not set. | +| `--ffprobe` | Folder containing the `ffprobe` executable. Uses the `PATH` if not set. | +| `--magick` | ImageMagick executable used for photo conversions. Default `magick`. | +| `--affinity` | Windows only. Mask of the CPU cores conversions run on, e.g. `7` for cores 1, 2 and 3. | +| `--progress/--no-progress` | Show progress bars. Enabled by default when running in a terminal. | + +While running, these commands can be typed in the console: + +* `p`: pause, conversions already running finish but nothing new starts. +* `r`: resume. +* `q`: quit, running conversions are stopped and their temporary files removed. Stopping the process (Ctrl+C, `SIGTERM`) does the same. + +At the end of the run (also when stopped with `q`), a summary is logged: one line per enabled pipeline, then a total. Counters that are zero are +omitted, except `found` and `errors`. The lines are logged at `ERROR` level when there are errors, `INFO` otherwise. + +```text +Pipeline 'Photos': 123 found, 100 moved, 3 converted, 10 skipped by stages, 5 destination exists, 2 errors +Pipeline 'Backup': 40 found, 38 copied, 2 already in place, 0 errors +Total: 163 found, 100 moved, 38 copied, 3 converted, 10 skipped by stages, 5 destination exists, 2 already in place, 2 errors +``` + +* `moved`, `copied`, `converted`: files transferred (`MOVE_WITH_COPY` counts as moved, a converted file counts as converted whatever the operation). + During a dry run, they read `would move`, `would copy` and `would convert`, and the lines are marked `(dry run)`. With the `NONE` operation, accepted + files are counted as `would process (operation NONE)`. +* `skipped by stages`: filtered out by a stage. +* `destination exists`: skipped because of the `SKIP` conflict strategy. +* `already in place`: the file is already at its destination. +* `errors`: files that failed (stage, conversion, transfer, setting dates), failed folder actions, and pipelines that failed as a whole (e.g. a `Convert` + cache that can't be opened). A missing input folder is only a warning. + +Each counted error is logged once at `ERROR` level when it happens. Problems the run works around (missing input folder, file that can't be probed, +unreadable metadata, time zone lookup failure) are logged at `WARN` level and aren't counted. + +Exit codes: + +| Code | Meaning | +|------|----------------------------------------------------------------------------| +| 0 | Success. | +| 1 | Invalid command line or configuration. | +| 2 | At least one error happened during the run. | +| 3 | Stopped with the `q` command (takes precedence over 2). | + +When the process is stopped by Ctrl+C or `SIGTERM`, running conversions are stopped and the summary is still logged, but the exit code is the one set by +the JVM for the signal (usually 130 for Ctrl+C and 143 for `SIGTERM`). + +Conversions need [ffmpeg](https://ffmpeg.org) (video and audio) and [ImageMagick](https://imagemagick.org) (photos) to be installed. + +## Configuration + +The configuration is a YAML file. Unknown keys are rejected to catch typos. Files whose name ends with `.json` are read as JSON instead +(comments and trailing commas allowed), with the same keys. + +```yaml +# Settings shared by all pipelines, see below +defaults: + operation: MOVE + +pipelines: + # Back up photos from the phone, sorted by month and renamed with their date + - name: Phone photos + mappings: + - input: /phone/DCIM + output: /backup/Photos + scan: + skipHidden: true + excludedFolders: ['\.thumbnails'] + stages: + - type: NameFilter + includes: ['.*\.jpg', '.*\.mp4'] + - type: IfOlderThan + dayOffset: 2 + - type: FolderPerMonth + - type: RenameWithDate + setFileCreationDate: true + folderActions: + - type: DeleteIfEmpty + + # Convert videos to HEVC in place, files that can't be converted are left untouched + - name: Videos + mappings: + - input: /media/Videos + output: /media/Videos + stages: + - type: Convert + processors: [VIDEO_TO_HEVC_MKV] + cache: /data/videos.db + temp: /data/temp +``` + +Regexes are best written between single quotes: backslashes are kept as is, and values starting with characters such as `[`, `*` or `{` would +otherwise be read as YAML syntax. A single quote inside is written `''`. + +### Editor support + +[`config.schema.json`](config.schema.json) is a JSON schema of the configuration, giving autocompletion, documentation on hover and validation in +editors. With VS Code's YAML extension (and other editors using the YAML language server), reference it in the first line of the configuration: + +```text +# yaml-language-server: $schema=https://raw.githubusercontent.com/RakambdaOrg/FileSecure/main/config.schema.json +``` + +A local path to the file works too (`$schema=../FileSecure/config.schema.json`, relative to the configuration). In IntelliJ IDEA, the schema can be +mapped in *Settings → Languages & Frameworks → Schemas and DTDs → JSON Schema Mappings*: add a mapping to `config.schema.json` (schema version 7) and +add the configuration file to it. + +* Enum values are proposed in uppercase. FileSecure also accepts them in lowercase, but the editor flags them. +* The schema describes the file as written, before `defaults` are merged and `mappingTable` is expanded. Some rules are only checked when FileSecure + loads the file (a pipeline without mappings, several `Convert` stages, invalid regexes...). +* `<<` merge keys are a YAML 1.1 feature: editors not resolving them while validating may flag `<<` or report keys coming from the merged template + as missing. FileSecure resolves them. + +### Pipeline + +| Key | Default | Description | +|-----------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `name` | `#index` | Name used in logs. | +| `enabled` | `true` | Disabled pipelines are skipped. | +| `mappings` | | **Mandatory** unless `mappingTable` or `mappingsFile` is set. List of `{input: ..., output: ...}` folders. A file at `input/a/b.txt` has `output/a/b.txt` as initial destination. | +| `mappingTable` | none | Many mappings sharing base folders, see [mapping table](#mapping-table). | +| `mappingsFile` | none | File holding more mappings, rewritten by another tool, see [mappings file](#mappings-file). | +| `scan.maxDepth` | no limit | How many levels of sub-folders are visited, `1` only processes the files directly in `input`. | +| `scan.skipHidden` | `false` | Ignore hidden files and folders. | +| `scan.excludedFolders`| none | Regexes matched against the folder path relative to `input`, using `/` as separator (e.g. `"a/b"`). Matching folders are not visited. | +| `stages` | none | The [stages](#stages), applied in the given order. | +| `operation` | `NONE` | What is done with accepted files: `COPY`, `MOVE`, `MOVE_WITH_COPY` (copy then delete, for moves across file systems) or `NONE` (only logs what would be done). | +| `conflict` | `RENAME` | When the destination already exists: `RENAME` uses the first free name among `name (1).ext`, `name (2).ext`...; `SKIP` leaves the file untouched. | +| `verify` | `SIZE` | How files copied to their destination are checked: `SIZE` compares the sizes, `CHECKSUM` also compares their SHA-256, see below. | +| `setFileCreationDate` | `false` | Set the creation and modification dates of the destination to the date found in the file name or in its metadata. | +| `temp` | system | Folder where conversions write before their result is moved to its destination. | +| `folderActions` | none | [Actions](#folder-actions) applied to every visited input folder once all files are processed, deepest folders first. | +| `useDefaults` | `true` | Merge the [defaults](#defaults) into this pipeline. | + +Enum values are case-insensitive. A file whose destination is itself (input and output being the same folder) is left untouched, unless it is converted. + +Moves and copies never leave an incomplete file under its final name, even if the run is interrupted (network drop, crash): + +* `MOVE` first tries an atomic rename, which succeeds when the source and the destination are on the same disk. +* Otherwise (different disks, e.g. a local disk and a NAS), and for `COPY` and `MOVE_WITH_COPY`, the file is copied to `..filesecure-partial` in + the destination folder, checked against the source (see `verify`), then renamed to its final name. The source is only deleted after that, and + is left untouched if anything fails. +* With `verify: SIZE` (default), the size of the copy is compared with the one of the source. With `verify: CHECKSUM`, the SHA-256 of the source + is computed while it is read for the copy, the copy is forced to storage then read back, and both SHA-256 must match (sizes are compared too). It + catches corruptions that keep the size, at the cost of reading the copy again. The copy keeps the dates of the source, but not its other + attributes (permissions, Windows file attributes) that `SIZE` copies where the file systems allow it. Reading back right after writing may be + served from the cache of the operating system or of the SMB client: it proves the data went through the copy correctly up to the destination + file system, not what the NAS disks physically hold. +* Converted files are put in place the same way from the `temp` folder. +* An existing destination is never overwritten. `.filesecure-partial` files left by an interrupted run are ignored by the scan (a warning is logged) + and can safely be deleted. + +### Mapping table + +`mappingTable` writes many mappings between two base folders as a table, one line per mapping. Each entry `key: value` becomes the mapping +`{input: /, output: /}`, in the order of the entries, after the pipeline's own `mappings` (which can then be omitted). + +| Key | Description | +|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `input` | **Mandatory**. Base folder of the entry keys. | +| `output` | **Mandatory**. Base folder of the entry values. | +| `entries` | **Mandatory**. Map of relative input path to relative output path. Paths can't be empty, absolute (`/a`, `D:/a`) or contain `..`, and each key can only appear once. | +| `paths` | Rewrites applied separately to every key and every value: the first rule whose `regex` fully matches it is applied, `replacement` can use its groups (`$1`, `${name}`). Paths matching no rule are kept. | + +This moves `D:/base/in/LINE/alice_42` to `D:/base/out/Whatsapp/3/3/6/0/0/0/0/33600000001` and `D:/base/in/Whatsapp/3/3/6/0/0/0/0/33600000002` to +`D:/base/out/Instagram/someone`: the rewrite applies to keys as well as values, splitting Whatsapp folders by digit: + +```yaml +mappingTable: + input: D:/base/in + output: D:/base/out + paths: + - regex: 'Whatsapp/(\d)(\d)(\d)(\d)(\d)(\d)(\d)(\d*)' + replacement: 'Whatsapp/$1/$2/$3/$4/$5/$6/$7/$1$2$3$4$5$6$7$8' + entries: + LINE/alice_42: Whatsapp/33600000001 + Whatsapp/33600000002: Instagram/someone +``` + +Keys are always read as text. Values read as numbers or booleans (`123`, `1e3`, `true`) are rejected and must be quoted. + +### Mappings file + +`mappingsFile` reads more mappings from a separate YAML or JSON file holding a list of `{input, output}`, added after the pipeline's own `mappings` +and before those of its `mappingTable`. It lets another tool rewrite that file without touching the main configuration, which keeps its comments +and anchors. A relative path is resolved against the working directory, an empty file holds no mapping. For example +`mappingsFile: D:/config/aliases.json` with this file: ```json -{ - "rules": [ - { - "mappings": [ - { - "input": "/Input", - "output": "/Output" - } - ] - }, - { - "operation": "move", - "depth": 2, - "filters": [ - ".*\\.png", - ".*\\.jpg" - ], - "excludes": [ - ".*\\.ini" - ], - "mappings": [ - { - "input": "/Input2", - "output": "/Output2" - } - ] - } - ] -} -``` - -Each backup to do is an object in the array called "mappings". Two fields are mandatory: - -* input -* output - -The input folder is backed up recursively up to a certain depth. - -Other fields are optionals: - -* operation: define the strategy to copy files. Can be "copy", "move" or "none" (just displays in standard output). Default is none. -* depth: The number of sub-folders to visit. -* filters: Regexes to apply to determine which files to keep. By default, all files are kept. -* excludes: Regexes to apply to determine which files to not process. By default, none are excluded. -* folderExcludes: Regexes to apply to determine which folders to not process. This is a relative path from the `input` folder, separated by `/`. By default, none are excluded. -* skipIfAlreadyExists: If destination file already exists, skip it instead of generating a unique name for it. -* fileTransformers: Transformations to apply when moving the file. - * TODO describe options -* folderTransformers: Transformations to apply to input folders after operation has been applied. - * TODO describe options +[ + {"input": "D:/base/LINE/alice_42", "output": "D:/base/Whatsapp/3/3/6/0/0/0/0/33600000001"} +] +``` + +### Defaults + +`defaults` holds settings written once for all pipelines. It accepts the same keys as a pipeline, except `name`, `enabled`, `mappings` and `mappingTable`, and is +merged into every pipeline: + +* `stages` and `folderActions`: the default entries are placed before the pipeline's own. +* `scan`: merged key by key, the pipeline's keys win. +* Other keys (`operation`, `conflict`, `verify`, `setFileCreationDate`, `temp`): the default applies when the pipeline doesn't set the key. + +A pipeline with `useDefaults: false` ignores `defaults`. + +```yaml +defaults: + scan: + skipHidden: true + stages: + - type: NameFilter + excludes: ['.*\.db', '.*\.ini'] + operation: MOVE + +pipelines: + # Excludes .db and .ini files, then sorts by year, moved + - mappings: + - input: /in/Photos + output: /out/Photos + stages: + - type: FolderPerYear + # Excludes .db, .ini and .txt files (several NameFilter stages exclude every pattern of each), copied + - mappings: + - input: /in/Notes + output: /out/Notes + stages: + - type: NameFilter + excludes: ['.*\.txt'] + operation: COPY + # Not affected by defaults: copies hidden files too, nothing excluded, operation NONE + - mappings: + - input: /in/Other + output: /out/Other + useDefaults: false +``` + +### Anchors and templates + +YAML anchors (`&name`) and aliases (`*name`) repeat a value written once, and the merge key `<<` copies the keys of a mapping, the keys written +next to it taking precedence. Anchors are best defined under the top-level `templates` key, which FileSecure ignores and whose content is free. + +```yaml +templates: + cleanup: &cleanup + - type: DeleteChildren + names: [Thumbs.db] + - type: DeleteIfEmpty + archive: &archive + stages: + - type: FolderPerYear + conflict: SKIP + folderActions: *cleanup + +pipelines: + - <<: *archive + mappings: + - input: /in/Scans + output: /archive/Scans + # Same as above, with conflict: RENAME + - <<: *archive + mappings: + - input: /in/Letters + output: /archive/Letters + conflict: RENAME + - mappings: + - input: /in/Photos + output: /out/Photos + folderActions: *cleanup +``` + +`<<` replaces whole keys: a pipeline merging `*archive` and writing its own `stages` doesn't get the template's stages. Unlike `defaults`, anchors +are a YAML feature and don't exist in JSON configurations. + +### Stages + +Several stages use **the date of the file**: the date found in its name (e.g. `2023-05-12 10.15.12.jpg`, `IMG_20230512_101512.jpg`), otherwise the +date it was taken from its metadata (EXIF, QuickTime, MP4, XMP), otherwise its last modification date. Time zones of geotagged media are found +offline from their GPS coordinates: metadata dates without a time zone are read in the time zone of the place the file was taken, or in the +system time zone when the file isn't geotagged or no time zone is found there. + +| Type | Parameters | Description | +|-------------------------|--------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `NameFilter` | `includes`, `excludes`: regex lists | Keeps files whose name fully matches one of the `includes` (all if empty) and none of the `excludes`. | +| `IfOlderThan` | `dayOffset`, `minuteOffset` | Keeps files whose date is older than the offset. | +| `FolderPerYear` | | Prefixes the destination with `yyyy/` from the file date. | +| `FolderPerMonth` | | Prefixes the destination with `yyyy/MM/`. | +| `FolderPerWeek` | | Prefixes the destination with `yyyy/MM/w/`, `w` being the week of the month. | +| `FolderPerDay` | | Prefixes the destination with `yyyy/MM/dd/`. | +| `FolderUniquePerDay` | | Prefixes the destination with `yyyy-MM-dd/`. | +| `FolderCanonical` | `elementCount` (mandatory) | Prefixes the destination with one folder per character of its first path element: `going/file` with `3` becomes `g/o/i/going/file`. | +| `Lowercase` | | Lowercases the destination folders (not the file name). | +| `NoRename` | | Restores the original file name. | +| `RenameWithDate` | `format` (`yyyy-MM-dd HH.mm.ss`), `locale` (`en`), `zone`, `filters` | Renames the file with its date, keeping its extension. When `filters` (regexes) is set, only files whose original name matches one of them are renamed. | +| `RenameWithDatePattern` | `regex`, `dateFormat`, `timeFormat` (mandatory), `outputFormat`, `locale`, `zone` | Renames the file with a date read from its current name. `regex` must define the groups `date` and `time`, parsed with `dateFormat` and `timeFormat`. Files not matching are skipped. | +| `RegexPath` | `rules` (mandatory), `unmatched` (`SKIP`) | Rewrites the destination relative to the output folder (with `/` separators) with the first rule whose `regex` fully matches it, `replacement` can use its groups (`$1`, `${name}`). Files matching no rule are skipped, or kept unchanged with `unmatched: PASS`. See the example below. | +| `Convert` | see below | Plans the conversion of media files. | + +`RegexPath` can replace many similar mappings. This moves `DropOff/Bob - 123/a.jpg` to `Whatsapp/123/a.jpg` and `DropOff/bob_tw/a.jpg` to +`Twitter/bob_tw/a.jpg`, other files are left in place: + +```yaml +mappings: + - input: /data/DropOff + output: /data +stages: + - type: RegexPath + rules: + - regex: '(?:[^/]* - )?(\d+)/(.*)' + replacement: Whatsapp/$1/$2 + - regex: '(bob_tw|carol_tw)/(.*)' + replacement: Twitter/$1/$2 +operation: MOVE +``` + +### Convert + +The file is probed with ffprobe and converted by the first processor able to handle it. Its destination extension changes accordingly, so stages placed +after `Convert` see the new extension. The conversion only runs once all stages accepted the file and its destination is free, so with +`conflict: SKIP` already converted files are not converted again. + +With `MOVE` or `MOVE_WITH_COPY` the source is sent to the trash (deleted where there is none) once the converted file is in place. When the destination +is the source itself, the source is replaced. A pipeline can contain only one `Convert` stage. + +| Key | Default | Description | +|--------------|----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `processors` | `VIDEO_TO_HEVC_MKV`, `MP3_TO_AAC`, `TIFF_TO_JPG` | Processors tried in order. | +| `extensions` | `mp4`, `mov`, `mkv`, `avi`, `tiff`, `mp3`, `m4a`, `m4v`, `ts`, `m2ts`, `mts`, `heic`, `jpg`, `raf` | Only files with these extensions are probed. | +| `filters` | none | Probe filters, all must pass: `{type: LengthMoreThan, value: seconds}`, `{type: LengthLessThan, value: seconds}`. | +| `unhandled` | `SKIP` | What happens to files that won't be converted: `SKIP` leaves them untouched, `PASS` lets them continue through the pipeline unchanged (e.g. to back them up too). | +| `cache` | none | H2 database file remembering files that can't be converted, so they aren't probed again on the next runs (until they are modified). | + +| Processor | Converts | Into | +|---------------------|-------------------------------------------------------------|---------------------------------------------------------------------------------| +| `VIDEO_TO_HEVC_MKV` | `H264`, `VP9`, `WMV3`, `MPEG2` videos, HEVC with another tag or container | `HEVC` + `AAC` in `MKV` (HEVC sources are only remuxed) | +| `VIDEO_TO_HEVC_MP4` | same | `HEVC` + `AAC` in `MP4` | +| `VIDEO_TO_AV1_MKV` | `H264`, `HEVC`, `VP9`, `WMV3`, `MPEG2` videos | `AV1` + `OPUS` in `MKV` | +| `MP3_TO_AAC` | `MP3` audio files | `AAC` (`.m4a`) | +| `AUDIO_TO_OPUS` | `MP3`, `AAC` audio files | `OPUS` | +| `TIFF_TO_JPG` | `TIFF` images | `JPEG` | +| `PHOTO_TO_AVIF` | `heic`, `tiff`, `jpg`, `raf` files | `AVIF` | +| `PHOTO_TO_JXL` | `heic`, `tiff`, `jpg`, `raf` files | `JXL` | + +### Folder actions + +| Type | Parameters | Description | +|---------------------|----------------------------------------------|----------------------------------------------------------------------------------------------| +| `DeleteIfEmpty` | `depth` (`1`) | Deletes empty folders at least `depth` levels below the input folder (the input is level 0). | +| `DeleteIfOlderThan` | `dayOffset`, `minuteOffset`, `depth` (`1`) | Same, for empty folders last modified before the offset. | +| `DeleteChildren` | `names` | Deletes the given files in every folder, e.g. `["Thumbs.db", "desktop.ini"]`. | + +## Migrating from previous versions + +A converter turns FileSecure 12 and MediaConverter configuration files into the new YAML format. Several files can be given, they are merged into one +configuration, one pipeline per old rule or conversion: + +```shell +java -cp FileSecure.jar fr.rakambda.filesecure.migration.MigrateConfigCommandKt FileSecure.json MediaConverter.json -o config.yaml +# or from the sources +./gradlew migrateConfig --args="/absolute/path/FileSecure.json -o /absolute/path/config.yaml" +``` + +Without `-o` the result is printed. An existing output file is only replaced with `--force`. The result is checked against the new format before +being written, and warnings list what couldn't be converted exactly (unknown keys that were already ignored, transformer order, behaviour +changes): review them before using the new configuration, ideally with a `--dry` run first. + +The tables below detail the conversion. + +### FileSecure 12 + +Each `rule` becomes a pipeline, with the same `mappings` and `operation`. + +| FileSecure 12 | FileSecure 13 | +|---------------------------------------|------------------------------------------------------------------------------------| +| `filters`, `excludes` | A `NameFilter` stage with `includes`, `excludes` | +| `folderExcludes` | `scan.excludedFolders` | +| `maxDepth` | `scan.maxDepth` | +| `skipIfAlreadyExists: true` | `conflict: SKIP` | +| `fileTransformers` | `stages`, without the `Transformer` suffix (`FolderPerYearTransformer` → `FolderPerYear`), `name` becomes `type` | +| `folderTransformers` | `folderActions`, same renaming | +| `setFileCreationDate` | unchanged | + +Transformers used to run in an unspecified order (their priority was never applied). Stages run in the order they are written, list them in the order +you expect: filters first, then folders, then renaming. + +### MediaConverter + +Each `conversion` becomes a pipeline with a `Convert` stage. + +| MediaConverter | FileSecure 13 | +|--------------------------------|-----------------------------------------------------------------------------------| +| `input`, `output` | A mapping | +| `deleteInput: true` (default) | `operation: MOVE` | +| `deleteInput: false` | `operation: COPY` and `conflict: SKIP` | +| `database` | `cache` of the `Convert` stage, existing database files can be reused | +| `temp` | `temp` | +| `excluded` | `scan.excludedFolders` (regexes, so escape special characters such as `.`) | +| `processors`, `extensions` | Same keys in the `Convert` stage | +| `filters` (`name`, `value`) | `filters` of the `Convert` stage, `name` becomes `type` | +| hidden files were ignored | `scan: {skipHidden: true}` | +| `-ft`, `--ffmpegThreads` | `-ft`, `--ffmpeg-threads` | + +### Behaviour changes + +* Only regular files are processed. Previously, folders beyond `maxDepth` were copied or moved as if they were files. +* `DeleteIfEmpty` never deletes the input folder itself with its default depth (it used to, unlike `DeleteIfOlderThan`). +* Files without extension no longer abort the whole mapping when their destination already exists. +* With MediaConverter, an input file could be left in the temporary folder if its output already existed. The source is now only removed once the + converted file is in place. +* The Windows CPU affinity was never applied by MediaConverter (a process id was used as a process handle), it now is. +* An error on one file or folder is logged and the others are still processed. +* Pipelines run one after another (MediaConverter ran its conversions in parallel), `--threads` conversions run in parallel within a pipeline. diff --git a/build.gradle.kts b/build.gradle.kts index bf292f5..80d4936 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,10 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer +import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { idea - java application + alias(libs.plugins.kotlin) alias(libs.plugins.shadow) alias(libs.plugins.jib) } @@ -11,68 +12,80 @@ plugins { group = "fr.rakambda" description = "FileSecure" +val javaVersion = providers.gradleProperty("javaVersion").get() + dependencies { implementation(platform(libs.jacksonBom)) implementation(platform(libs.log4j2Bom)) + implementation(libs.coroutines) + implementation(libs.slf4j) implementation(libs.bundles.log4j2) + implementation(libs.kotlinLogging) - implementation(libs.unirest) - implementation(libs.pointLocation) - implementation(libs.metadataExtractor) - - implementation(libs.picocli) + implementation(libs.clikt) implementation(libs.bundles.jackson) - implementation(libs.jSpecify) - compileOnly(libs.lombok) + implementation(libs.pointLocation) + implementation(libs.timeshape) + constraints { + implementation(libs.protobuf) { + because("Versions before 4.36 call sun.misc.Unsafe memory methods, printing JVM warnings since JDK 24") + } + } + implementation(libs.metadataExtractor) - annotationProcessor(libs.lombok) + implementation(libs.jaffree) + implementation(libs.progressbar) + implementation(libs.h2) + implementation(libs.bundles.jna) - implementation(platform(libs.junitBom)) + testImplementation(platform(libs.junitBom)) testImplementation(libs.bundles.junit) testRuntimeOnly(libs.junitPlatformLauncher) - testImplementation(libs.bundles.assertj) - testImplementation(libs.bundles.mockito) - - testCompileOnly(libs.lombok) - - testAnnotationProcessor(libs.lombok) + testImplementation(libs.jsonSchemaValidator) } repositories { mavenCentral() } -tasks { - processResources { - expand(project.properties) - } - - compileJava { - val moduleName: String by project - inputs.property("moduleName", moduleName) - - options.encoding = "UTF-8" - options.isDeprecation = true - } +java { + sourceCompatibility = JavaVersion.toVersion(javaVersion) + targetCompatibility = JavaVersion.toVersion(javaVersion) +} - compileTestJava { - options.encoding = "UTF-8" +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget(javaVersion)) } +} +tasks { test { useJUnitPlatform() + jvmArgs("--enable-native-access=ALL-UNNAMED") + inputs.files("config.schema.json", "README.md").withPathSensitivity(PathSensitivity.RELATIVE) } jar { manifest { attributes["Multi-Release"] = "true" + // The zstd library used by timeshape loads a native library + attributes["Enable-Native-Access"] = "ALL-UNNAMED" } } + register("migrateConfig") { + group = "application" + description = "Converts FileSecure 12 and MediaConverter configurations, pass the files with --args" + classpath = sourceSets["main"].runtimeClasspath + mainClass.set("fr.rakambda.filesecure.migration.MigrateConfigCommandKt") + jvmArgs("--enable-native-access=ALL-UNNAMED") + } + shadowJar { archiveBaseName.set(project.name) archiveClassifier.set("shaded") @@ -84,16 +97,7 @@ tasks { } application { - val moduleName: String by project - val className: String by project - - mainModule.set(moduleName) - mainClass.set(className) -} - -java { - sourceCompatibility = JavaVersion.VERSION_26 - targetCompatibility = JavaVersion.VERSION_26 + mainClass.set(providers.gradleProperty("className")) } jib { @@ -111,6 +115,7 @@ jib { } } container { + jvmFlags = listOf("--enable-native-access=ALL-UNNAMED") creationTime.set("USE_CURRENT_TIMESTAMP") user = "1027:100" } diff --git a/config.schema.json b/config.schema.json new file mode 100644 index 0000000..86fa954 --- /dev/null +++ b/config.schema.json @@ -0,0 +1,835 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "FileSecure configuration", + "description": "Configuration of FileSecure 13: pipelines of stages moving, copying and converting files. See the README for details.", + "type": "object", + "properties": { + "pipelines": { + "description": "The pipelines, run one after another.", + "type": "array", + "items": { + "$ref": "#/definitions/pipeline" + } + }, + "defaults": { + "$ref": "#/definitions/defaults" + }, + "templates": { + "description": "Ignored by FileSecure, free content. The place to define YAML anchors (&name) reused elsewhere with aliases (*name) or merge keys (<<: *name)." + } + }, + "additionalProperties": false, + "definitions": { + "pipeline": { + "description": "A sequence of stages applied to every file found in the inputs of its mappings.", + "type": "object", + "properties": { + "name": { + "description": "Name used in logs. Default: #index.", + "type": "string" + }, + "enabled": { + "description": "Disabled pipelines are skipped.", + "type": "boolean", + "default": true + }, + "useDefaults": { + "description": "Merge the top-level defaults into this pipeline.", + "type": "boolean", + "default": true + }, + "mappings": { + "description": "Mandatory unless mappingTable or mappingsFile is set. Folders to process: a file at input/a/b.txt has output/a/b.txt as initial destination.", + "type": "array", + "items": { + "$ref": "#/definitions/mapping" + } + }, + "mappingTable": { + "$ref": "#/definitions/mappingTable" + }, + "mappingsFile": { + "description": "YAML or JSON file holding a list of mappings, added after the pipeline's own mappings. Meant to be rewritten by another tool without touching this configuration. A relative path is resolved against the working directory.", + "type": "string", + "minLength": 1 + }, + "scan": { + "$ref": "#/definitions/scan" + }, + "stages": { + "description": "The stages, applied in the given order. Stages from defaults are placed before them.", + "type": "array", + "items": { + "$ref": "#/definitions/stage" + } + }, + "operation": { + "$ref": "#/definitions/operation" + }, + "conflict": { + "$ref": "#/definitions/conflict" + }, + "verify": { + "$ref": "#/definitions/verify" + }, + "setFileCreationDate": { + "description": "Set the creation and modification dates of the destination to the date found in the file name or in its metadata.", + "type": "boolean", + "default": false + }, + "temp": { + "description": "Folder where conversions write before their result is moved to its destination. Default: the system temporary folder.", + "type": [ + "string", + "null" + ] + }, + "folderActions": { + "description": "Actions applied to every visited input folder once all files are processed, deepest folders first. Actions from defaults are placed before them.", + "type": "array", + "items": { + "$ref": "#/definitions/folderAction" + } + } + }, + "additionalProperties": false + }, + "defaults": { + "description": "Settings merged into every pipeline not having useDefaults: false. Their stages and folderActions are placed before the pipeline's own, scan is merged key by key (the pipeline's keys win), other keys apply when the pipeline doesn't set them.", + "type": [ + "object", + "null" + ], + "properties": { + "scan": { + "$ref": "#/definitions/pipeline/properties/scan" + }, + "stages": { + "$ref": "#/definitions/pipeline/properties/stages" + }, + "operation": { + "$ref": "#/definitions/pipeline/properties/operation" + }, + "conflict": { + "$ref": "#/definitions/pipeline/properties/conflict" + }, + "verify": { + "$ref": "#/definitions/pipeline/properties/verify" + }, + "setFileCreationDate": { + "$ref": "#/definitions/pipeline/properties/setFileCreationDate" + }, + "temp": { + "$ref": "#/definitions/pipeline/properties/temp" + }, + "folderActions": { + "$ref": "#/definitions/pipeline/properties/folderActions" + } + }, + "additionalProperties": false + }, + "mapping": { + "description": "An input folder and the output folder its files are sent to.", + "type": "object", + "properties": { + "input": { + "description": "Folder scanned for files.", + "type": "string" + }, + "output": { + "description": "Folder the files are sent to, keeping their path relative to input.", + "type": "string" + } + }, + "required": [ + "input", + "output" + ], + "additionalProperties": false + }, + "mappingTable": { + "description": "Many mappings sharing base folders: each entry key: value becomes the mapping {input: /, output: /}, added after the pipeline's own mappings.", + "type": "object", + "properties": { + "input": { + "description": "Base folder of the entry keys.", + "type": "string", + "minLength": 1 + }, + "output": { + "description": "Base folder of the entry values.", + "type": "string", + "minLength": 1 + }, + "paths": { + "description": "Rewrites applied separately to every key and every value: the first rule whose regex fully matches it is applied. Paths matching no rule are kept.", + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/regexRule" + } + }, + "entries": { + "description": "Map of relative input path to relative output path. Paths can't be empty, absolute or contain '..'. Values read as numbers or booleans must be quoted.", + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "input", + "output", + "entries" + ], + "additionalProperties": false + }, + "regexRule": { + "description": "Rewrites a path fully matching regex with replacement.", + "type": "object", + "properties": { + "regex": { + "description": "Java regex that must match the whole path, '/' being the separator. Best written between single quotes.", + "type": "string" + }, + "replacement": { + "description": "Replacement where $1 or ${name} refer to the groups of the regex.", + "type": "string" + } + }, + "required": [ + "regex", + "replacement" + ], + "additionalProperties": false + }, + "scan": { + "description": "How the input folders are visited.", + "type": "object", + "properties": { + "maxDepth": { + "description": "How many levels of sub-folders are visited, 1 only processes the files directly in input. Default: no limit.", + "type": "integer", + "minimum": 0 + }, + "skipHidden": { + "description": "Ignore hidden files and folders.", + "type": "boolean", + "default": false + }, + "excludedFolders": { + "description": "Java regexes matched against the folder path relative to input, using '/' as separator (e.g. 'a/b'). Matching folders are not visited.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "operation": { + "description": "What is done with accepted files: COPY, MOVE, MOVE_WITH_COPY (copy then delete, for moves across file systems) or NONE (only logs what would be done). Case-insensitive for FileSecure.", + "type": "string", + "enum": [ + "COPY", + "MOVE", + "MOVE_WITH_COPY", + "NONE" + ], + "default": "NONE" + }, + "conflict": { + "description": "When the destination already exists: RENAME uses the first free name among 'name (1).ext', 'name (2).ext'...; SKIP leaves the file untouched. Case-insensitive for FileSecure.", + "type": "string", + "enum": [ + "RENAME", + "SKIP" + ], + "default": "RENAME" + }, + "verify": { + "description": "How files copied to their destination (COPY, MOVE_WITH_COPY, MOVE across file systems, converted files) are checked before being renamed to their final name: SIZE compares the sizes; CHECKSUM also compares the SHA-256 of the source, computed while copying, with the one of the copy read back (slower: the copy is forced to storage then read again). Atomic renames on the same file system aren't checked, nothing being copied. Case-insensitive for FileSecure.", + "type": "string", + "enum": [ + "SIZE", + "CHECKSUM" + ], + "default": "SIZE" + }, + "processor": { + "description": "VIDEO_TO_HEVC_MKV, VIDEO_TO_HEVC_MP4: H264, VP9, WMV3, MPEG2 videos into HEVC + AAC. VIDEO_TO_AV1_MKV: into AV1 + OPUS. MP3_TO_AAC: MP3 into AAC (.m4a). AUDIO_TO_OPUS: MP3, AAC into OPUS. TIFF_TO_JPG: TIFF into JPEG. PHOTO_TO_AVIF, PHOTO_TO_JXL: heic, tiff, jpg, raf into AVIF, JXL. Case-insensitive for FileSecure.", + "type": "string", + "enum": [ + "VIDEO_TO_HEVC_MP4", + "VIDEO_TO_HEVC_MKV", + "VIDEO_TO_AV1_MKV", + "MP3_TO_AAC", + "AUDIO_TO_OPUS", + "TIFF_TO_JPG", + "PHOTO_TO_AVIF", + "PHOTO_TO_JXL" + ] + }, + "pattern": { + "description": "Java regex. Best written between single quotes.", + "type": "string" + }, + "zone": { + "description": "Time zone ID (e.g. Europe/Paris) of the date. Default: the system time zone.", + "type": [ + "string", + "null" + ] + }, + "locale": { + "description": "Language of the date format (month and day names).", + "type": "string", + "default": "en" + }, + "stage": { + "description": "A stage, chosen by its type.", + "type": "object", + "properties": { + "type": { + "description": "Kind of stage.", + "type": "string", + "enum": [ + "NameFilter", + "IfOlderThan", + "FolderCanonical", + "FolderPerDay", + "FolderPerWeek", + "FolderPerMonth", + "FolderPerYear", + "FolderUniquePerDay", + "Lowercase", + "NoRename", + "RenameWithDate", + "RenameWithDatePattern", + "RegexPath", + "Convert" + ] + } + }, + "required": [ + "type" + ], + "oneOf": [ + { + "$ref": "#/definitions/NameFilter" + }, + { + "$ref": "#/definitions/IfOlderThan" + }, + { + "$ref": "#/definitions/FolderCanonical" + }, + { + "$ref": "#/definitions/FolderPerDay" + }, + { + "$ref": "#/definitions/FolderPerWeek" + }, + { + "$ref": "#/definitions/FolderPerMonth" + }, + { + "$ref": "#/definitions/FolderPerYear" + }, + { + "$ref": "#/definitions/FolderUniquePerDay" + }, + { + "$ref": "#/definitions/Lowercase" + }, + { + "$ref": "#/definitions/NoRename" + }, + { + "$ref": "#/definitions/RenameWithDate" + }, + { + "$ref": "#/definitions/RenameWithDatePattern" + }, + { + "$ref": "#/definitions/RegexPath" + }, + { + "$ref": "#/definitions/Convert" + } + ] + }, + "NameFilter": { + "description": "Keeps files whose name fully matches one of the includes (all files if empty) and none of the excludes.", + "type": "object", + "properties": { + "type": { + "const": "NameFilter" + }, + "includes": { + "description": "Java regexes, the file name must fully match one of them. All files are included when empty.", + "type": "array", + "items": { + "$ref": "#/definitions/pattern" + } + }, + "excludes": { + "description": "Java regexes, files whose name fully matches one of them are skipped.", + "type": "array", + "items": { + "$ref": "#/definitions/pattern" + } + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "IfOlderThan": { + "description": "Keeps files whose date (from their name, metadata or last modification) is older than the offset.", + "type": "object", + "properties": { + "type": { + "const": "IfOlderThan" + }, + "dayOffset": { + "description": "Days of the offset.", + "type": "integer", + "default": 0 + }, + "minuteOffset": { + "description": "Minutes added to the offset.", + "type": "integer", + "default": 0 + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "FolderCanonical": { + "description": "Prefixes the destination with one folder per character of its first path element: 'going/file' with 3 becomes 'g/o/i/going/file'.", + "type": "object", + "properties": { + "type": { + "const": "FolderCanonical" + }, + "elementCount": { + "description": "How many characters, and so folders, are used.", + "type": "integer" + } + }, + "required": [ + "type", + "elementCount" + ], + "additionalProperties": false + }, + "FolderPerDay": { + "description": "Prefixes the destination with yyyy/MM/dd/ from the file date.", + "type": "object", + "properties": { + "type": { + "const": "FolderPerDay" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "FolderPerWeek": { + "description": "Prefixes the destination with yyyy/MM/w/ from the file date, w being the week of the month.", + "type": "object", + "properties": { + "type": { + "const": "FolderPerWeek" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "FolderPerMonth": { + "description": "Prefixes the destination with yyyy/MM/ from the file date.", + "type": "object", + "properties": { + "type": { + "const": "FolderPerMonth" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "FolderPerYear": { + "description": "Prefixes the destination with yyyy/ from the file date.", + "type": "object", + "properties": { + "type": { + "const": "FolderPerYear" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "FolderUniquePerDay": { + "description": "Prefixes the destination with yyyy-MM-dd/ from the file date.", + "type": "object", + "properties": { + "type": { + "const": "FolderUniquePerDay" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "Lowercase": { + "description": "Lowercases the destination folders (not the file name).", + "type": "object", + "properties": { + "type": { + "const": "Lowercase" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "NoRename": { + "description": "Restores the original file name.", + "type": "object", + "properties": { + "type": { + "const": "NoRename" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "RenameWithDate": { + "description": "Renames the file with its date (from its name, metadata or last modification), keeping its extension.", + "type": "object", + "properties": { + "type": { + "const": "RenameWithDate" + }, + "format": { + "description": "Java DateTimeFormatter pattern of the new name.", + "type": "string", + "default": "yyyy-MM-dd HH.mm.ss" + }, + "locale": { + "$ref": "#/definitions/locale" + }, + "filters": { + "description": "When set, only files whose original name fully matches one of these Java regexes are renamed.", + "type": "array", + "items": { + "$ref": "#/definitions/pattern" + } + }, + "zone": { + "$ref": "#/definitions/zone" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "RenameWithDatePattern": { + "description": "Renames the file with a date read from its current name, keeping its extension. Files not matching regex are skipped.", + "type": "object", + "properties": { + "type": { + "const": "RenameWithDatePattern" + }, + "regex": { + "description": "Java regex fully matching the name, defining the named groups date and time.", + "type": "string" + }, + "dateFormat": { + "description": "Java DateTimeFormatter pattern of the date group.", + "type": "string" + }, + "timeFormat": { + "description": "Java DateTimeFormatter pattern of the time group.", + "type": "string" + }, + "outputFormat": { + "description": "Java DateTimeFormatter pattern of the new name.", + "type": "string", + "default": "yyyy-MM-dd HH.mm.ss" + }, + "locale": { + "$ref": "#/definitions/locale" + }, + "zone": { + "$ref": "#/definitions/zone" + } + }, + "required": [ + "type", + "regex", + "dateFormat", + "timeFormat" + ], + "additionalProperties": false + }, + "RegexPath": { + "description": "Rewrites the destination relative to the output folder (with '/' separators) with the first rule whose regex fully matches it.", + "type": "object", + "properties": { + "type": { + "const": "RegexPath" + }, + "rules": { + "description": "Rules tried in order.", + "type": "array", + "items": { + "$ref": "#/definitions/regexRule" + } + }, + "unmatched": { + "description": "What happens to files matching no rule: SKIP leaves them untouched, PASS keeps them unchanged in the pipeline. Case-insensitive for FileSecure.", + "type": "string", + "enum": [ + "PASS", + "SKIP" + ], + "default": "SKIP" + } + }, + "required": [ + "type", + "rules" + ], + "additionalProperties": false + }, + "Convert": { + "description": "Plans the conversion of media files with the first processor able to handle them, the destination extension changes accordingly. Only one per pipeline.", + "type": "object", + "properties": { + "type": { + "const": "Convert" + }, + "processors": { + "description": "Processors tried in order. Default: VIDEO_TO_HEVC_MKV, MP3_TO_AAC, TIFF_TO_JPG.", + "type": "array", + "items": { + "$ref": "#/definitions/processor" + } + }, + "extensions": { + "description": "Only files with these extensions are probed. Default: mp4, mov, mkv, avi, tiff, mp3, m4a, m4v, ts, m2ts, mts, heic, jpg, raf.", + "type": "array", + "items": { + "type": "string" + } + }, + "filters": { + "description": "Probe filters, all must pass.", + "type": "array", + "items": { + "$ref": "#/definitions/probeFilter" + } + }, + "unhandled": { + "description": "What happens to files that won't be converted: SKIP leaves them untouched, PASS lets them continue through the pipeline unchanged. Case-insensitive for FileSecure.", + "type": "string", + "enum": [ + "PASS", + "SKIP" + ], + "default": "SKIP" + }, + "cache": { + "description": "H2 database file remembering files that can't be converted, so they aren't probed again on the next runs.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "probeFilter": { + "description": "A filter on the media probed by ffprobe, chosen by its type. Files that couldn't be probed never pass.", + "type": "object", + "properties": { + "type": { + "description": "Kind of filter.", + "type": "string", + "enum": [ + "LengthMoreThan", + "LengthLessThan" + ] + } + }, + "required": [ + "type" + ], + "oneOf": [ + { + "$ref": "#/definitions/LengthMoreThan" + }, + { + "$ref": "#/definitions/LengthLessThan" + } + ] + }, + "LengthMoreThan": { + "description": "Keeps media lasting at least value seconds.", + "type": "object", + "properties": { + "type": { + "const": "LengthMoreThan" + }, + "value": { + "description": "Duration in seconds.", + "type": "integer" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + "LengthLessThan": { + "description": "Keeps media lasting at most value seconds.", + "type": "object", + "properties": { + "type": { + "const": "LengthLessThan" + }, + "value": { + "description": "Duration in seconds.", + "type": "integer" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + "folderAction": { + "description": "An action applied to every visited input folder, chosen by its type.", + "type": "object", + "properties": { + "type": { + "description": "Kind of folder action.", + "type": "string", + "enum": [ + "DeleteIfEmpty", + "DeleteIfOlderThan", + "DeleteChildren" + ] + } + }, + "required": [ + "type" + ], + "oneOf": [ + { + "$ref": "#/definitions/DeleteIfEmpty" + }, + { + "$ref": "#/definitions/DeleteIfOlderThan" + }, + { + "$ref": "#/definitions/DeleteChildren" + } + ] + }, + "DeleteIfEmpty": { + "description": "Deletes empty folders at least depth levels below the input folder.", + "type": "object", + "properties": { + "type": { + "const": "DeleteIfEmpty" + }, + "depth": { + "$ref": "#/definitions/depth" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "DeleteIfOlderThan": { + "description": "Deletes empty folders at least depth levels below the input folder, last modified before the offset.", + "type": "object", + "properties": { + "type": { + "const": "DeleteIfOlderThan" + }, + "dayOffset": { + "description": "Days of the offset.", + "type": "integer", + "default": 0 + }, + "minuteOffset": { + "description": "Minutes added to the offset.", + "type": "integer", + "default": 0 + }, + "depth": { + "$ref": "#/definitions/depth" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "DeleteChildren": { + "description": "Deletes the given files in every folder.", + "type": "object", + "properties": { + "type": { + "const": "DeleteChildren" + }, + "names": { + "description": "Names of the files to delete, e.g. [Thumbs.db, desktop.ini].", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "depth": { + "description": "Minimum depth of the deleted folders below the input folder, the input itself being at depth 0.", + "type": "integer", + "default": 1 + } + } +} diff --git a/gradle.properties b/gradle.properties index d39f502..6d88781 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ -version=12.1 -moduleName=fr.rakambda.filesecure -className=fr.rakambda.filesecure.Main +version=13.0 +javaVersion=26 +className=fr.rakambda.filesecure.MainKt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 816ffbc..db6f9b3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,52 +1,66 @@ [versions] +kotlin-version = "2.4.21" +coroutines-version = "1.11.0" slf4j-version = "2.0.20" log4j2-version = "2.26.1" -picocli-version = "4.7.7" +kotlin-logging-version = "8.0.4" +clikt-version = "5.1.0" jackson-version = "2.22.4" -jspecify-version = "1.0.1" metadata-extractor-version = "2.22.0" pointlocation6709-version = "5.0.1" -unirest-version = "4.0.0-RC2" -lombok-version = "1.18.48" +timeshape-version = "2026b.29" +protobuf-version = "4.36.2" +jaffree-version = "2024.08.29" +progressbar-version = "0.10.2" +h2-version = "2.5.252" +jna-version = "5.19.1" junitVersion = "6.1.3" assertj-core-version = "3.27.7" -mockito-version = "5.24.0" +json-schema-validator-version = "3.0.8" shadow-version = "9.6.1" jib-version = "3.5.4" [libraries] +coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines-version" } slf4j = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j-version" } -log4j2Bom = { group = "org.apache.logging.log4j", name = "log4j-bom", version.ref = "log4j2-version"} +log4j2Bom = { group = "org.apache.logging.log4j", name = "log4j-bom", version.ref = "log4j2-version" } log4j2-core = { group = "org.apache.logging.log4j", name = "log4j-core" } log4j2-slf4j = { group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl" } log4j2-json = { group = "org.apache.logging.log4j", name = "log4j-layout-template-json" } -picocli = { group = "info.picocli", name = "picocli", version.ref = "picocli-version" } -jacksonBom = { group = "com.fasterxml.jackson", name = "jackson-bom", version.ref = "jackson-version"} +kotlinLogging = { group = "io.github.oshai", name = "kotlin-logging-jvm", version.ref = "kotlin-logging-version" } +clikt = { group = "com.github.ajalt.clikt", name = "clikt", version.ref = "clikt-version" } +jacksonBom = { group = "com.fasterxml.jackson", name = "jackson-bom", version.ref = "jackson-version" } jackson-core = { group = "com.fasterxml.jackson.core", name = "jackson-core" } jackson-annotations = { group = "com.fasterxml.jackson.core", name = "jackson-annotations" } jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind" } -jSpecify = { group = "org.jspecify", name = "jspecify", version.ref = "jspecify-version" } +jackson-kotlin = { group = "com.fasterxml.jackson.module", name = "jackson-module-kotlin" } +jackson-yaml = { group = "com.fasterxml.jackson.dataformat", name = "jackson-dataformat-yaml" } metadataExtractor = { group = "com.drewnoakes", name = "metadata-extractor", version.ref = "metadata-extractor-version" } pointLocation = { group = "us.fatehi", name = "pointlocation6709", version.ref = "pointlocation6709-version" } -unirest = { group = "com.konghq", name = "unirest-java", version.ref = "unirest-version" } -lombok = { group = "org.projectlombok", name = "lombok", version.ref = "lombok-version" } -junitBom = { group = "org.junit", name = "junit-bom", version.ref = "junitVersion"} +timeshape = { group = "net.iakovlev", name = "timeshape", version.ref = "timeshape-version" } +protobuf = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protobuf-version" } +jaffree = { group = "com.github.kokorin.jaffree", name = "jaffree", version.ref = "jaffree-version" } +progressbar = { group = "me.tongfei", name = "progressbar", version.ref = "progressbar-version" } +h2 = { group = "com.h2database", name = "h2", version.ref = "h2-version" } +jna = { group = "net.java.dev.jna", name = "jna", version.ref = "jna-version" } +jna-platform = { group = "net.java.dev.jna", name = "jna-platform", version.ref = "jna-version" } +junitBom = { group = "org.junit", name = "junit-bom", version.ref = "junitVersion" } junit-api = { group = "org.junit.jupiter", name = "junit-jupiter-api" } junit-params = { group = "org.junit.jupiter", name = "junit-jupiter-params" } junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine" } junitPlatformLauncher = { group = "org.junit.platform", name = "junit-platform-launcher" } assertj-core = { group = "org.assertj", name = "assertj-core", version.ref = "assertj-core-version" } -mockito-core = { group = "org.mockito", name = "mockito-core", version.ref = "mockito-version" } -mockito-junit = { group = "org.mockito", name = "mockito-junit-jupiter", version.ref = "mockito-version" } +jsonSchemaValidator = { group = "com.networknt", name = "json-schema-validator", version.ref = "json-schema-validator-version" } [bundles] -jackson = ["jackson-core", "jackson-annotations", "jackson-databind"] +jackson = ["jackson-core", "jackson-annotations", "jackson-databind", "jackson-kotlin", "jackson-yaml"] log4j2 = ["log4j2-core", "log4j2-slf4j", "log4j2-json"] +jna = ["jna", "jna-platform"] junit = ["junit-api", "junit-params", "junit-engine"] -mockito = ["mockito-core", "mockito-junit"] assertj = ["assertj-core"] [plugins] +kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin-version" } shadow = { id = "com.gradleup.shadow", version.ref = "shadow-version" } jib = { id = "com.google.cloud.tools.jib", version.ref = "jib-version" } diff --git a/lombok.config b/lombok.config deleted file mode 100644 index 6aa51d7..0000000 --- a/lombok.config +++ /dev/null @@ -1,2 +0,0 @@ -# This file is generated by the 'io.freefair.lombok' Gradle plugin -config.stopBubbling = true diff --git a/src/main/java/fr/rakambda/filesecure/CLIParameters.java b/src/main/java/fr/rakambda/filesecure/CLIParameters.java deleted file mode 100644 index 3c79b96..0000000 --- a/src/main/java/fr/rakambda/filesecure/CLIParameters.java +++ /dev/null @@ -1,24 +0,0 @@ -package fr.rakambda.filesecure; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import picocli.CommandLine; - -@NoArgsConstructor -@Getter -@CommandLine.Command(name = "filesecure", mixinStandardHelpOptions = true) -public class CLIParameters{ - @CommandLine.Option(names = { - "-c", - "--config" - }, - description = "The path to the configuration file", - required = true) - public String configurationFile; - @CommandLine.Option(names = { - "--dry" - }, - description = "Performs a dry run") - public boolean dryRun; -} - diff --git a/src/main/java/fr/rakambda/filesecure/Main.java b/src/main/java/fr/rakambda/filesecure/Main.java deleted file mode 100644 index 30cedb1..0000000 --- a/src/main/java/fr/rakambda/filesecure/Main.java +++ /dev/null @@ -1,55 +0,0 @@ -package fr.rakambda.filesecure; - -import fr.rakambda.filesecure.config.Configuration; -import fr.rakambda.filesecure.metadata.MetadataExtractor; -import fr.rakambda.filesecure.processor.Processor; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.extern.slf4j.Slf4j; -import picocli.CommandLine; -import java.nio.file.Path; -import java.nio.file.Paths; - -@Slf4j -public class Main{ - public static void main(String[] args){ - var cli = parseArgs(args); - - var configurationOptional = Configuration.loadConfiguration(Path.of(cli.getConfigurationFile())); - if(configurationOptional.isEmpty()){ - log.error("Failed to load configuration from {}", cli.getConfigurationFile()); - System.exit(1); - return; - } - - var configuration = configurationOptional.get(); - var metadataExtractor = new MetadataExtractor(); - var fileOperations = new FileOperations(cli.isDryRun()); - - for(var rule : configuration.getRules()){ - for(var mapping : rule.getMappings()){ - try{ - new Processor(rule, mapping, metadataExtractor, fileOperations).process(); - } - catch(Exception e){ - log.error("Failed to run processor", e); - } - } - } - } - - private static CLIParameters parseArgs(String[] args){ - var parameters = new CLIParameters(); - var cli = new CommandLine(parameters); - cli.registerConverter(Path.class, Paths::get); - cli.setUnmatchedArgumentsAllowed(true); - try{ - cli.parseArgs(args); - } - catch(CommandLine.ParameterException e){ - log.error("Failed to parse arguments", e); - cli.usage(System.out); - System.exit(1); - } - return parameters; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/Configuration.java b/src/main/java/fr/rakambda/filesecure/config/Configuration.java deleted file mode 100644 index c8c61b1..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Configuration.java +++ /dev/null @@ -1,57 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - -@Slf4j -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -@AllArgsConstructor -public class Configuration{ - private static final ObjectReader objectReader; - - @JsonProperty - private List rules; - - @NonNull - public static Optional loadConfiguration(@NonNull Path path){ - if(Files.isRegularFile(path)){ - try(var fis = Files.newBufferedReader(path)){ - return Optional.ofNullable(objectReader.readValue(fis)); - } - catch(IOException e){ - log.error("Failed to read settings in {}", path, e); - } - } - return Optional.empty(); - } - - static{ - var mapper = new ObjectMapper(); - mapper.setVisibility(mapper.getSerializationConfig() - .getDefaultVisibilityChecker() - .withFieldVisibility(JsonAutoDetect.Visibility.ANY) - .withGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); - mapper.enable(SerializationFeature.INDENT_OUTPUT); - objectReader = mapper.readerFor(Configuration.class); - } -} \ No newline at end of file diff --git a/src/main/java/fr/rakambda/filesecure/config/Mapping.java b/src/main/java/fr/rakambda/filesecure/config/Mapping.java deleted file mode 100644 index 1eb3221..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Mapping.java +++ /dev/null @@ -1,25 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.ext.NioPathDeserializer; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import java.nio.file.Path; - -@Slf4j -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -public class Mapping{ - @JsonProperty - @JsonDeserialize(using = NioPathDeserializer.class) - private Path input; - @JsonProperty - @JsonDeserialize(using = NioPathDeserializer.class) - private Path output; -} diff --git a/src/main/java/fr/rakambda/filesecure/config/Operation.java b/src/main/java/fr/rakambda/filesecure/config/Operation.java deleted file mode 100644 index cc8b0bd..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Operation.java +++ /dev/null @@ -1,34 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import fr.rakambda.filesecure.utils.FileOperations; -import fr.rakambda.filesecure.utils.ThrowingTriConsumer; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; - -@Getter -@RequiredArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public enum Operation{ - COPY(FileOperations::copy), - MOVE(FileOperations::move), - MOVE_WITH_COPY(FileOperations::moveWithCopy), - NONE((op, in, out) -> {}); - - private final ThrowingTriConsumer processor; - - @JsonCreator - public static Operation getByName(@NonNull String name){ - return valueOf(name); - } - - @SuppressWarnings("SameReturnValue") - public static Operation getDefault(){ - return NONE; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/Rule.java b/src/main/java/fr/rakambda/filesecure/config/Rule.java deleted file mode 100644 index 2a6f503..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/Rule.java +++ /dev/null @@ -1,52 +0,0 @@ -package fr.rakambda.filesecure.config; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.json.PatternDeserializer; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.regex.Pattern; - -@Slf4j -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -public class Rule{ - @JsonProperty - private Operation operation = Operation.getDefault(); - @JsonProperty - private List mappings = new ArrayList<>(); - - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private Set filters = new HashSet<>(); - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private Set excludes = new HashSet<>(); - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private Set folderExcludes = new HashSet<>(); - - @JsonProperty - private Set fileTransformers = new HashSet<>(); - @JsonProperty - private Set folderTransformers = new HashSet<>(); - - @JsonProperty - private boolean skipIfAlreadyExists = false; - @JsonProperty - private boolean setFileCreationDate = false; - - @JsonProperty - private int maxDepth = Integer.MAX_VALUE; -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/FileTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/FileTransformer.java deleted file mode 100644 index c62a827..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/FileTransformer.java +++ /dev/null @@ -1,43 +0,0 @@ -package fr.rakambda.filesecure.config.options; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import fr.rakambda.filesecure.config.options.file.FolderCanonicalTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerDayTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerMonthTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerWeekTransformer; -import fr.rakambda.filesecure.config.options.file.FolderPerYearTransformer; -import fr.rakambda.filesecure.config.options.file.FolderUniquePerDayTransformer; -import fr.rakambda.filesecure.config.options.file.IfOlderThanTransformer; -import fr.rakambda.filesecure.config.options.file.LowercaseTransformer; -import fr.rakambda.filesecure.config.options.file.NoRenameTransformer; -import fr.rakambda.filesecure.config.options.file.RenameWithDatePatternTransformer; -import fr.rakambda.filesecure.config.options.file.RenameWithDateTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -@JsonSubTypes(value = { - @JsonSubTypes.Type(value = FolderCanonicalTransformer.class, name = "FolderCanonicalTransformer"), - @JsonSubTypes.Type(value = FolderPerDayTransformer.class, name = "FolderPerDayTransformer"), - @JsonSubTypes.Type(value = FolderPerWeekTransformer.class, name = "FolderPerWeekTransformer"), - @JsonSubTypes.Type(value = FolderPerMonthTransformer.class, name = "FolderPerMonthTransformer"), - @JsonSubTypes.Type(value = FolderPerYearTransformer.class, name = "FolderPerYearTransformer"), - @JsonSubTypes.Type(value = FolderUniquePerDayTransformer.class, name = "FolderUniquePerDayTransformer"), - @JsonSubTypes.Type(value = IfOlderThanTransformer.class, name = "IfOlderThanTransformer"), - @JsonSubTypes.Type(value = LowercaseTransformer.class, name = "LowercaseTransformer"), - @JsonSubTypes.Type(value = NoRenameTransformer.class, name = "NoRenameTransformer"), - @JsonSubTypes.Type(value = RenameWithDateTransformer.class, name = "RenameWithDateTransformer"), - @JsonSubTypes.Type(value = RenameWithDatePatternTransformer.class, name = "RenameWithDatePatternTransformer"), -}) -public interface FileTransformer extends Transformer{ - @NonNull - Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException; -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/FolderTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/FolderTransformer.java deleted file mode 100644 index 35c4c05..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/FolderTransformer.java +++ /dev/null @@ -1,24 +0,0 @@ -package fr.rakambda.filesecure.config.options; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import fr.rakambda.filesecure.config.options.folder.DeleteChildrenTransformer; -import fr.rakambda.filesecure.config.options.folder.DeleteIfEmptyTransformer; -import fr.rakambda.filesecure.config.options.folder.DeleteIfOlderThanTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -@JsonSubTypes(value = { - @JsonSubTypes.Type(value = DeleteIfOlderThanTransformer.class, name = "DeleteIfOlderThanTransformer"), - @JsonSubTypes.Type(value = DeleteIfEmptyTransformer.class, name = "DeleteIfEmptyTransformer"), - @JsonSubTypes.Type(value = DeleteChildrenTransformer.class, name = "DeleteChildrenTransformer"), -}) -public interface FolderTransformer extends Transformer{ - void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder); -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/Transformer.java b/src/main/java/fr/rakambda/filesecure/config/options/Transformer.java deleted file mode 100644 index 711af0b..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/Transformer.java +++ /dev/null @@ -1,17 +0,0 @@ -package fr.rakambda.filesecure.config.options; - -import org.jspecify.annotations.NonNull; - -public interface Transformer extends Comparable{ - @Override - default int compareTo(@NonNull Transformer o){ - return Integer.compare(getPriority(), o.getPriority()); - } - - /** - * Get the priority of this option. - * - * @return The priority, lower will be executed first. - */ - int getPriority(); -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformer.java deleted file mode 100644 index d636128..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformer.java +++ /dev/null @@ -1,71 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderCanonicalTransformer") -@NoArgsConstructor -@AllArgsConstructor -public class FolderCanonicalTransformer implements FileTransformer{ - private static final String UNKNOWN_PART = "_"; - private static final Set FORBIDDEN_PART = Set.of(".", "/", "\\"); - - @JsonProperty(required = true) - private int elementCount = Integer.MAX_VALUE; - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var relative = baseOutput.relativize(currentOutput); - var base = getBase(relative); - var canonical = getCanonicalPath(base.getFileName().toString()); - return Optional.of(baseOutput.resolve(canonical).resolve(relative)); - } - - @NonNull - private Path getBase(@NonNull Path path){ - var parent = path.getParent(); - if(Objects.isNull(parent) || Objects.equals(parent, path)){ - return path; - } - return getBase(parent); - } - - private Path getCanonicalPath(String name){ - var path = Paths.get(""); - var max = Math.min(elementCount, name.length()); - for(var i = 0; i < max; i++){ - var part = String.valueOf(name.charAt(i)).toLowerCase(); - if(FORBIDDEN_PART.contains(part)){ - part = UNKNOWN_PART; - } - path = path.resolve(part); - } - while(path.getNameCount() < elementCount){ - path = path.resolve(UNKNOWN_PART); - } - return path; - } - - @Override - public int getPriority(){ - return 12; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformer.java deleted file mode 100644 index e335b57..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformer.java +++ /dev/null @@ -1,37 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerDayTransformer") -@NoArgsConstructor -public class FolderPerDayTransformer implements FileTransformer{ - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - var month = "%02d".formatted(date.getMonthValue()); - var day = "%02d".formatted(date.getDayOfMonth()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(month).resolve(day).resolve(relative)); - } - - @Override - public int getPriority(){ - return 11; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformer.java deleted file mode 100644 index 95e02ed..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformer.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerMonthTransformer") -@NoArgsConstructor -public class FolderPerMonthTransformer implements FileTransformer{ - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - var month = "%02d".formatted(date.getMonthValue()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(month).resolve(relative)); - } - - @Override - public int getPriority(){ - return 13; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformer.java deleted file mode 100644 index 3af6ede..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformer.java +++ /dev/null @@ -1,38 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.temporal.ChronoField; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerWeekTransformer") -@NoArgsConstructor -public class FolderPerWeekTransformer implements FileTransformer{ - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - var month = "%02d".formatted(date.getMonthValue()); - var week = "%1d".formatted(date.get(ChronoField.ALIGNED_WEEK_OF_MONTH)); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(month).resolve(week).resolve(relative)); - } - - @Override - public int getPriority(){ - return 12; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformer.java deleted file mode 100644 index 3a364f0..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformer.java +++ /dev/null @@ -1,35 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderPerYearTransformer") -@NoArgsConstructor -public class FolderPerYearTransformer implements FileTransformer { - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var year = "%4d".formatted(date.getYear()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(year).resolve(relative)); - } - - @Override - public int getPriority(){ - return 14; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformer.java deleted file mode 100644 index df017d4..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformer.java +++ /dev/null @@ -1,35 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("FolderUniquePerDayTransformer") -@NoArgsConstructor -public class FolderUniquePerDayTransformer implements FileTransformer { - @Override - public int getPriority(){ - return 10; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - var dateFormat = "%4d-%02d-%02d".formatted(date.getYear(), date.getMonthValue(), date.getDayOfMonth()); - - var relative = baseOutput.relativize(currentOutput); - return Optional.of(baseOutput.resolve(dateFormat).resolve(relative)); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/IfOlderThanTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/IfOlderThanTransformer.java deleted file mode 100644 index 35cb549..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/IfOlderThanTransformer.java +++ /dev/null @@ -1,44 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.ZonedDateTime; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("IfOlderThanTransformer") -@NoArgsConstructor -@AllArgsConstructor -public class IfOlderThanTransformer implements FileTransformer { - @JsonProperty - private int dayOffset = 0; - @JsonProperty - private int minuteOffset = 0; - - @Override - public int getPriority(){ - return 0; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = metadata.getDate(); - if(date.isAfter(ZonedDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset))){ - throw new AbandonBackupException(sourceFile); - } - return Optional.empty(); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformer.java deleted file mode 100644 index 7207e13..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformer.java +++ /dev/null @@ -1,33 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Locale; -import java.util.Optional; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("LowercaseTransformer") -@NoArgsConstructor -public class LowercaseTransformer implements FileTransformer { - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var relative = baseOutput.relativize(currentOutput.getParent()); - return Optional.of(baseOutput.resolve(relative.toString().toLowerCase(Locale.ROOT)).resolve(currentOutput.getFileName())); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformer.java deleted file mode 100644 index 0cd7091..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformer.java +++ /dev/null @@ -1,31 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.Optional; - -@Log4j2 -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("NoRenameTransformer") -@NoArgsConstructor -public class NoRenameTransformer implements FileTransformer { - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - return Optional.of(currentOutput.getParent().resolve(sourceFile.getFileName())); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDatePatternTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDatePatternTransformer.java deleted file mode 100644 index a1eb995..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDatePatternTransformer.java +++ /dev/null @@ -1,83 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import fr.rakambda.filesecure.utils.json.PatternDeserializer; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.Optional; -import java.util.regex.Pattern; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("RenameWithDatePatternTransformer") -@Log4j2 -@NoArgsConstructor -public class RenameWithDatePatternTransformer implements FileTransformer{ - @JsonProperty - private String outputFormat = "yyyy-MM-dd HH.mm.ss"; - @JsonProperty(required = true) - @JsonDeserialize(using = PatternDeserializer.class) - private Pattern regex; - @JsonProperty(required = true) - private String dateFormat; - @JsonProperty(required = true) - private String timeFormat; - @JsonProperty - private String locale = "en"; - @JsonProperty - private String zone; - - @Override - public int getPriority(){ - return 0; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - var date = extractDate(currentOutput.getFileName().toString()) - .orElseThrow(() -> new AbandonBackupException(sourceFile)); - - var zoneId = Optional.ofNullable(zone).map(ZoneId::of).orElse(ZoneId.systemDefault()); - var formatter = DateTimeFormatter.ofPattern(outputFormat, new Locale.Builder().setLanguage(locale).build()).withZone(zoneId); - - var fileName = currentOutput.getFileName().toString(); - var dotIndex = fileName.lastIndexOf('.'); - var extension = dotIndex < 0 ? "" : fileName.substring(dotIndex); - - return Optional.of(currentOutput.getParent().resolve(formatter.format(date) + extension)); - } - - @NonNull - private Optional extractDate(@NonNull String name){ - try{ - var matcher = regex.matcher(name); - if(!matcher.matches()){ - return Optional.empty(); - } - var dateString = "%s %s".formatted(matcher.group("date"), matcher.group("time")); - - var zoneId = Optional.ofNullable(zone).map(ZoneId::of).orElse(ZoneId.systemDefault()); - var formatter = DateTimeFormatter.ofPattern("%s %s".formatted(dateFormat, timeFormat), new Locale.Builder().setLanguage(locale).build()).withZone(zoneId); - - return Optional.of(ZonedDateTime.parse(dateString, formatter)); - } - catch(Exception e){ - log.error("Failed to decode date from value `{}` and format `{} {}`", name, dateFormat, timeFormat, e); - return Optional.empty(); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformer.java deleted file mode 100644 index 5a00db1..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformer.java +++ /dev/null @@ -1,71 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import fr.rakambda.filesecure.config.options.FileTransformer; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import fr.rakambda.filesecure.utils.json.PatternDeserializer; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.extern.log4j.Log4j2; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Objects; -import java.util.Optional; -import java.util.regex.Pattern; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("RenameWithDateTransformer") -@Log4j2 -@NoArgsConstructor -public class RenameWithDateTransformer implements FileTransformer{ - @JsonProperty - private String format = "yyyy-MM-dd HH.mm.ss"; - @JsonProperty - private String locale = "en"; - @JsonProperty - @JsonDeserialize(contentUsing = PatternDeserializer.class) - private List filters = new LinkedList<>(); - @JsonProperty - @Setter(AccessLevel.PACKAGE) - private String zone; - - @Override - public int getPriority(){ - return 0; - } - - @NonNull - @Override - public Optional apply(@NonNull Path sourceFile, @NonNull Path originalOutput, @NonNull Path baseOutput, @NonNull Path currentOutput, @NonNull FileMetadata metadata) throws AbandonBackupException{ - if(Objects.nonNull(filters) && !filters.isEmpty()){ - var matched = this.filters.stream() - .map(Pattern::asMatchPredicate) - .anyMatch(pred -> pred.test(sourceFile.getFileName().toString())); - if(!matched){ - log.debug("Not applying transformer {} because file name {} didn't match any filters", this, sourceFile); - return Optional.empty(); - } - } - var zoneId = Optional.ofNullable(zone).map(ZoneId::of).orElse(ZoneId.systemDefault()); - var formatter = DateTimeFormatter.ofPattern(format, new Locale.Builder().setLanguage(locale).build()).withZone(zoneId); - - var date = metadata.getDate(); - var fileName = currentOutput.getFileName().toString(); - var dotIndex = fileName.lastIndexOf('.'); - var extension = dotIndex < 0 ? "" : fileName.substring(dotIndex); - - return Optional.of(currentOutput.getParent().resolve(formatter.format(date) + extension)); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteChildrenTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteChildrenTransformer.java deleted file mode 100644 index d8cb575..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteChildrenTransformer.java +++ /dev/null @@ -1,44 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("DeleteChildrenTransformer") -@NoArgsConstructor -public class DeleteChildrenTransformer implements FolderTransformer{ - @JsonProperty - private List names = new ArrayList<>(); - - @Override - public int getPriority(){ - return 10; - } - - @Override - public void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder){ - try{ - for(var child : names){ - var childPath = folder.resolve(child); - if(fileOperations.deleteIfExists(childPath)){ - log.info("Deleted child file {}", childPath); - } - } - } - catch(Exception e){ - log.error("Failed to delete children in {}", folder, e); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformer.java deleted file mode 100644 index c7a9acf..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Path; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("DeleteIfEmptyTransformer") -@NoArgsConstructor -@AllArgsConstructor -public class DeleteIfEmptyTransformer implements FolderTransformer{ - @JsonProperty - private int depth = 1; - - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @Override - public void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder){ - try{ - var relativeDepth = baseFolder.relativize(folder).getNameCount(); - if(relativeDepth < depth){ - return; - } - - if(Files.list(folder).toList().isEmpty()){ - log.info("Deleting folder {} because it is empty", folder); - fileOperations.delete(folder); - } - } - catch(DirectoryNotEmptyException e){ - log.error("Failed to delete folder {}, not empty", folder); - } - catch(Exception e){ - log.error("Failed to determine if {} should be deleted, it will not be by default", folder, e); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfOlderThanTransformer.java b/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfOlderThanTransformer.java deleted file mode 100644 index 40365b6..0000000 --- a/src/main/java/fr/rakambda/filesecure/config/options/folder/DeleteIfOlderThanTransformer.java +++ /dev/null @@ -1,74 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import fr.rakambda.filesecure.config.options.FolderTransformer; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.ZoneId; - -@Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeName("DeleteIfOlderThanTransformer") -@NoArgsConstructor -public class DeleteIfOlderThanTransformer implements FolderTransformer{ - @JsonProperty - private int dayOffset = 0; - @JsonProperty - private int minuteOffset = 0; - @JsonProperty - private int depth = 1; - - @Override - public int getPriority(){ - return Integer.MAX_VALUE; - } - - @Override - public void apply(@NonNull FileOperations fileOperations, @NonNull Path folder, @NonNull Path baseFolder){ - try{ - var relativize = baseFolder.relativize(folder); - var relativeDepth = relativize.getNameCount(); - - if(baseFolder.equals(folder)){ - relativeDepth = 0; - } - - if(relativeDepth < depth){ - return; - } - - var folderTime = Files.getLastModifiedTime(folder); - var date = LocalDateTime.ofInstant(folderTime.toInstant(), ZoneId.systemDefault()); - if(date.isBefore(LocalDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset))){ - delete(fileOperations, folder); - } - } - catch(DirectoryNotEmptyException e){ - log.error("Failed to delete folder {}, not empty", folder); - } - catch(Exception e){ - log.error("Failed to determine if {} should be deleted, it will not be by default", folder, e); - } - } - - private void delete(@NonNull FileOperations fileOperations, @NonNull Path folder) throws IOException{ - if(Files.list(folder).toList().isEmpty()){ - log.info("Deleting folder {} because it is more than {} days old and empty", folder, dayOffset); - fileOperations.delete(folder); - } - else{ - log.debug("Tried deleting folder {} because it is more than {} days old but wasn't empty", folder, dayOffset); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/exceptions/AbandonBackupException.java b/src/main/java/fr/rakambda/filesecure/exceptions/AbandonBackupException.java deleted file mode 100644 index bc75f82..0000000 --- a/src/main/java/fr/rakambda/filesecure/exceptions/AbandonBackupException.java +++ /dev/null @@ -1,9 +0,0 @@ -package fr.rakambda.filesecure.exceptions; - -import java.nio.file.Path; - -public class AbandonBackupException extends Exception{ - public AbandonBackupException(Path originFile){ - super("Abandoned the backup of " + originFile); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/exceptions/FlagsProcessingException.java b/src/main/java/fr/rakambda/filesecure/exceptions/FlagsProcessingException.java deleted file mode 100644 index 25e748c..0000000 --- a/src/main/java/fr/rakambda/filesecure/exceptions/FlagsProcessingException.java +++ /dev/null @@ -1,7 +0,0 @@ -package fr.rakambda.filesecure.exceptions; - -public class FlagsProcessingException extends RuntimeException{ - public FlagsProcessingException(String message){ - super(message); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/MetadataExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/MetadataExtractor.java deleted file mode 100644 index c77c8e5..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/MetadataExtractor.java +++ /dev/null @@ -1,281 +0,0 @@ -package fr.rakambda.filesecure.metadata; - -import com.drew.imaging.ImageMetadataReader; -import com.drew.metadata.Directory; -import com.drew.metadata.Metadata; -import com.drew.metadata.exif.ExifSubIFDDirectory; -import com.drew.metadata.exif.GpsDirectory; -import com.drew.metadata.mov.QuickTimeDirectory; -import com.drew.metadata.mov.metadata.QuickTimeMetadataDirectory; -import com.drew.metadata.mp4.Mp4Directory; -import com.drew.metadata.xmp.XmpDirectory; -import fr.rakambda.filesecure.metadata.media.MediaDateExtractor; -import fr.rakambda.filesecure.metadata.media.SimpleMediaDateExtractor; -import fr.rakambda.filesecure.metadata.media.XmpMediaDateExtractor; -import fr.rakambda.filesecure.metadata.name.NameDateExtractor; -import fr.rakambda.filesecure.metadata.name.Pattern1NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern2NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern3NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern4NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern5NameDateExtractorImpl; -import fr.rakambda.filesecure.metadata.name.Pattern6NameDateExtractorImpl; -import fr.rakambda.filesecure.processor.FileMetadata; -import fr.rakambda.filesecure.utils.json.GeonamesTimeZone; -import kong.unirest.GenericType; -import kong.unirest.Unirest; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; -import us.fatehi.pointlocation6709.Angle; -import us.fatehi.pointlocation6709.Latitude; -import us.fatehi.pointlocation6709.Longitude; -import us.fatehi.pointlocation6709.PointLocation; -import us.fatehi.pointlocation6709.parse.PointLocationParser; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.ParseException; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.TimeZone; -import java.util.regex.Pattern; - -@Slf4j -public class MetadataExtractor{ - private final List dateFormats; - private final List> mediaDateExtractors; - - public MetadataExtractor(){ - dateFormats = new ArrayList<>(); - dateFormats.add(new Pattern1NameDateExtractorImpl()); - dateFormats.add(new Pattern2NameDateExtractorImpl()); - dateFormats.add(new Pattern3NameDateExtractorImpl()); - dateFormats.add(new Pattern4NameDateExtractorImpl()); - dateFormats.add(new Pattern5NameDateExtractorImpl()); - dateFormats.add(new Pattern6NameDateExtractorImpl()); - - mediaDateExtractors = new ArrayList<>(); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(QuickTimeMetadataDirectory.class, QuickTimeMetadataDirectory.TAG_CREATION_DATE)); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(QuickTimeDirectory.class, QuickTimeDirectory.TAG_CREATION_TIME)); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(Mp4Directory.class, Mp4Directory.TAG_CREATION_TIME)); - mediaDateExtractors.add(new SimpleMediaDateExtractor<>(ExifSubIFDDirectory.class, ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL)); - mediaDateExtractors.add(new XmpMediaDateExtractor()); - } - - @NonNull - public FileMetadata getMetadata(@NonNull Path path) throws IOException{ - return new FileMetadata(getDateFromMetadata(path), getCreationDate(path), getDateFromFileName(path)); - } - - @NonNull - private ZonedDateTime getCreationDate(@NonNull Path path) throws IOException{ - var attr = Files.readAttributes(path, BasicFileAttributes.class); - var date = Instant.ofEpochMilli(attr.lastModifiedTime().toMillis()).atZone(ZoneId.systemDefault()); - - if(date.getYear() <= 1970){ - date = date.withYear(LocalDateTime.now().getYear()); - } - - return date; - } - - @Nullable - private ZonedDateTime getDateFromMetadata(@NonNull Path path){ - try{ - var metadata = ImageMetadataReader.readMetadata(path.toFile()); - if(Objects.isNull(metadata)){ - return null; - } - - var zoneID = getZoneIdFromMetadata(metadata).orElse(ZoneId.systemDefault()); - var timeZone = TimeZone.getTimeZone(zoneID); - - for(var dataExtractor : mediaDateExtractors){ - for(var directory : metadata.getDirectoriesOfType(dataExtractor.getKlass())){ - if(Objects.nonNull(directory)){ - try{ - var result = runDirectoryExtractor(dataExtractor, directory, timeZone); - if(Objects.nonNull(result)){ - return result; - } - } - catch(ParseException e){ - log.warn("Invalid year with metadata directory {} for file {}", dataExtractor.getKlass().getName(), path); - } - catch(Exception e){ - log.error("Error processing metadata directory {} for {} => {}", dataExtractor.getKlass().getName(), path, e.getMessage()); - } - } - } - } - } - catch(Exception e){ - log.error("Error getting date from metadata for {} => {}", path, e.getMessage()); - } - return null; - } - - @Nullable - private ZonedDateTime runDirectoryExtractor(@NonNull MediaDateExtractor dataExtractor, @NonNull Directory directory, @NonNull TimeZone timeZone) throws ParseException{ - log.debug("Trying {}", dataExtractor.getKlass().getName()); - - var takenDateOptional = dataExtractor.parse(directory, timeZone); - if(takenDateOptional.isEmpty()){ - return null; - } - - var takenDate = takenDateOptional.get(); - log.debug("Matched metadata directory {}", directory); - if(takenDate.getYear() < 1970){ - throw new ParseException("Invalid year", 0); - } - return takenDate; - } - - @Nullable - private ZonedDateTime getDateFromFileName(@NonNull Path path){ - var fileName = path.getFileName().toString(); - - for(var nameDateExtractor : dateFormats){ - try{ - log.debug("Trying name extractor `{}`", nameDateExtractor); - var dateOptional = nameDateExtractor.parse(fileName); - if(dateOptional.isEmpty()){ - continue; - } - - var date = dateOptional.get(); - if(date.getYear() < 1970){ - throw new ParseException("Invalid year", 0); - } - - log.debug("Matched date format for {}", path); - return date; - } - catch(ParseException e){ - log.warn("Invalid year with used format for file {}", path); - } - catch(DateTimeParseException ignored){ - } - catch(Exception e){ - log.error("Error using format {} => {}", nameDateExtractor, e.getMessage()); - } - } - - return null; - } - - @NonNull - private Optional getZoneIdFromMetadata(@NonNull Metadata metadata){ - try{ - for(var gpsDirectory : metadata.getDirectoriesOfType(GpsDirectory.class)){ - var location = gpsDirectory.getGeoLocation(); - if(Objects.isNull(location)){ - continue; - } - var zoneId = getZoneID(location.getLatitude(), location.getLongitude()); - if(zoneId.isPresent()){ - return zoneId; - } - } - for(var quickTimeMetadataDirectory : metadata.getDirectoriesOfType(QuickTimeMetadataDirectory.class)){ - var repr = quickTimeMetadataDirectory.getString(0x050D); - if(Objects.nonNull(repr) && !repr.isBlank()){ - var location = PointLocationParser.parsePointLocation(repr); - var zoneId = getZoneID(location.getLatitude().getDegrees(), location.getLongitude().getDegrees()); - if(zoneId.isPresent()){ - return zoneId; - } - } - } - for(var xmpDirectory : metadata.getDirectoriesOfType(XmpDirectory.class)){ - var xmpValues = xmpDirectory.getXmpProperties(); - if(xmpValues.containsKey("exif:GPSLatitude") && xmpValues.containsKey("exif:GPSLongitude")){ - var zoneId = getAngle(xmpValues.get("exif:GPSLatitude")) - .flatMap(lat -> getAngle(xmpValues.get("exif:GPSLongitude")) - .flatMap(lon -> { - var location = new PointLocation(new Latitude(lat), new Longitude(lon)); - return getZoneID(location.getLatitude().getDegrees(), location.getLongitude().getDegrees()); - })); - if(zoneId.isPresent()){ - return zoneId; - } - } - } - } - catch(Exception e){ - log.warn("Error getting GPS infos", e); - } - return Optional.empty(); - } - - /** - * Get the zoneID of a geolocation. - * - * @param latitude The latitude. - * @param longitude The longitude. - * - * @return The zoneID corresponding to this location. - */ - @NonNull - private static Optional getZoneID(double latitude, double longitude){ - try{ - var request = Unirest.get("http://api.geonames.org/timezoneJSON") - .queryString("lat", latitude) - .queryString("lng", longitude) - .queryString("username", "mrcraftcod") - .asObject(new GenericType(){}); - if(request.isSuccess()){ - var geonamesTimeZone = request.getBody(); - return Optional.ofNullable(geonamesTimeZone.getTimezoneId()); - } - } - catch(Exception e){ - log.error("Error getting zoneID for coordinates {};{}", latitude, longitude, e); - } - return Optional.empty(); - } - - /** - * Convert a N/E/S/W coordinate to an angle one. - * - * @param s The coordinate to convert. - * - * @return The angle. - */ - @NonNull - private static Optional getAngle(@NonNull String s){ - var pattern = Pattern.compile("(\\d{1,3}),(\\d{1,2})\\.(\\d+)([NESW])"); - var matcher = pattern.matcher(s); - if(matcher.matches()){ - var angle = Integer.parseInt(matcher.group(1)) - + (Integer.parseInt(matcher.group(2)) / 60.0) - + (Double.parseDouble("0." + matcher.group(3)) / 60.0); - angle *= getMultiplicand(matcher.group(4)); - return Optional.of(Angle.fromDegrees(angle)); - } - return Optional.empty(); - } - - /** - * Get the sign of the angle depending of it's position (N/E/S/W). - * - * @param group The group (N/E/S/W). - * - * @return The sign of the angle. - */ - private static double getMultiplicand(@NonNull String group){ - return switch(group){ - case "W", "S" -> -1; - default -> 1; - }; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/media/MediaDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/media/MediaDateExtractor.java deleted file mode 100644 index cb42e32..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/media/MediaDateExtractor.java +++ /dev/null @@ -1,29 +0,0 @@ -package fr.rakambda.filesecure.metadata.media; - -import com.drew.metadata.Directory; -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.util.Optional; -import java.util.TimeZone; - -/** - * @param The directory type. - */ -public interface MediaDateExtractor{ - /** - * Get the date from the given {@link Directory}. - * - * @param directory The directory to get from. - * @param tz The timezone of the date. - * - * @return The date or null if not found. - */ - @NonNull Optional parse(@NonNull Directory directory, @NonNull TimeZone tz); - - /** - * Get the class of the directory. - * - * @return The directory's class. - */ - @NonNull Class getKlass(); -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/media/SimpleMediaDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/media/SimpleMediaDateExtractor.java deleted file mode 100644 index 9f2bd61..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/media/SimpleMediaDateExtractor.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.rakambda.filesecure.metadata.media; - -import com.drew.metadata.Directory; -import lombok.Getter; -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.util.Date; -import java.util.Optional; -import java.util.TimeZone; - -public class SimpleMediaDateExtractor implements MediaDateExtractor{ - private final int tag; - @Getter - private final Class klass; - - public SimpleMediaDateExtractor(@NonNull Class klass, int tag){ - this.klass = klass; - this.tag = tag; - } - - @NonNull - @Override - public Optional parse(@NonNull Directory directory, @NonNull TimeZone tz){ - return Optional.ofNullable(directory.getDate(tag, tz)) - .map(Date::toInstant) - .map(date -> date.atZone(tz.toZoneId())); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/media/XmpMediaDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/media/XmpMediaDateExtractor.java deleted file mode 100644 index 911ffcb..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/media/XmpMediaDateExtractor.java +++ /dev/null @@ -1,39 +0,0 @@ -package fr.rakambda.filesecure.metadata.media; - -import com.drew.metadata.Directory; -import com.drew.metadata.xmp.XmpDirectory; -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Optional; -import java.util.TimeZone; - -public class XmpMediaDateExtractor implements MediaDateExtractor{ - private final List keys = List.of("xmp:CreateDate", "photoshop:DateCreated"); - private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); - - @NonNull - @Override - public Optional parse(@NonNull Directory directory, @NonNull TimeZone tz){ - var xmpDirectory = (XmpDirectory) directory; - var values = xmpDirectory.getXmpProperties(); - for(var key : keys){ - if(values.containsKey(key)){ - try{ - return Optional.ofNullable(values.get(key)) - .map(date -> ZonedDateTime.parse(date, dateTimeFormatter.withZone(tz.toZoneId()))); - } - catch(Exception ignored){ - } - } - } - return Optional.empty(); - } - - @NonNull - @Override - public Class getKlass(){ - return XmpDirectory.class; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/DatePatternNameExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/name/DatePatternNameExtractor.java deleted file mode 100644 index ca0fb78..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/DatePatternNameExtractor.java +++ /dev/null @@ -1,27 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Optional; -import java.util.regex.Pattern; - -public abstract class DatePatternNameExtractor implements NameDateExtractor{ - @Override - @NonNull - public Optional parse(@NonNull String name){ - var matcher = getPattern().matcher(name); - if(matcher.find()){ - return Optional.of(ZonedDateTime.parse(matcher.group(getCaptureGroupIndex()), getFormatter())); - } - return Optional.empty(); - } - - protected int getCaptureGroupIndex(){ - return 0; - } - - protected abstract DateTimeFormatter getFormatter(); - - protected abstract Pattern getPattern(); -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/NameDateExtractor.java b/src/main/java/fr/rakambda/filesecure/metadata/name/NameDateExtractor.java deleted file mode 100644 index e97889b..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/NameDateExtractor.java +++ /dev/null @@ -1,16 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import org.jspecify.annotations.NonNull; -import java.time.ZonedDateTime; -import java.util.Optional; - -public interface NameDateExtractor{ - /** - * Get the date from the filename - * - * @param name The filename - * - * @return The date if found. - */ - @NonNull Optional parse(@NonNull String name); -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern1NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern1NameDateExtractorImpl.java deleted file mode 100644 index 7a73849..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern1NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern1NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2} \\d{2}\\.\\d{2}\\.\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH.mm.ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern2NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern2NameDateExtractorImpl.java deleted file mode 100644 index 6a3ac05..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern2NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern2NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2} \\d{2}-\\d{2}-\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern3NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern3NameDateExtractorImpl.java deleted file mode 100644 index 0b6fcd1..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern3NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern3NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern4NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern4NameDateExtractorImpl.java deleted file mode 100644 index d9a8094..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern4NameDateExtractorImpl.java +++ /dev/null @@ -1,23 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern4NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern5NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern5NameDateExtractorImpl.java deleted file mode 100644 index b4bd7ed..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern5NameDateExtractorImpl.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern5NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("IMG_(\\d{8}_\\d{6})"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } - - @Override - protected int getCaptureGroupIndex(){ - return 1; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern6NameDateExtractorImpl.java b/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern6NameDateExtractorImpl.java deleted file mode 100644 index 5e2afb2..0000000 --- a/src/main/java/fr/rakambda/filesecure/metadata/name/Pattern6NameDateExtractorImpl.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.rakambda.filesecure.metadata.name; - -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Locale; -import java.util.regex.Pattern; - -public class Pattern6NameDateExtractorImpl extends DatePatternNameExtractor{ - private static final Pattern NAME_PATTERN = Pattern.compile("Video-(\\d{8}-\\d{6})"); - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss") - .withLocale(Locale.ENGLISH) - .withZone(ZoneId.systemDefault()); - - @Override - protected DateTimeFormatter getFormatter(){ - return DATE_TIME_FORMATTER; - } - - @Override - protected Pattern getPattern(){ - return NAME_PATTERN; - } - - @Override - protected int getCaptureGroupIndex(){ - return 1; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/processor/DifferenceVisitor.java b/src/main/java/fr/rakambda/filesecure/processor/DifferenceVisitor.java deleted file mode 100644 index 209b28f..0000000 --- a/src/main/java/fr/rakambda/filesecure/processor/DifferenceVisitor.java +++ /dev/null @@ -1,62 +0,0 @@ -package fr.rakambda.filesecure.processor; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Collection; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.regex.Pattern; - -@RequiredArgsConstructor -public class DifferenceVisitor implements FileVisitor{ - private final Path rootFolder; - private final Collection includePatterns; - private final Collection excludePatterns; - private final Collection excludeFolderPatterns; - - @Getter - private final Collection paths = new ConcurrentLinkedDeque<>(); - @Getter - private final Collection folders = new ConcurrentLinkedDeque<>(); - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs){ - var relative = rootFolder.relativize(dir).toString().replace("\\", "/"); - if(excludeFolderPatterns.stream().anyMatch(f -> f.matcher(relative).matches())){ - return FileVisitResult.SKIP_SUBTREE; - } - - folders.add(dir); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs){ - var fileName = file.getFileName().toString(); - - if(excludePatterns.stream().anyMatch(f -> f.matcher(fileName).matches())){ - return FileVisitResult.CONTINUE; - } - if(!includePatterns.isEmpty() && includePatterns.stream().noneMatch(f -> f.matcher(fileName).matches())){ - return FileVisitResult.CONTINUE; - } - - paths.add(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException{ - throw exc; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc){ - return FileVisitResult.CONTINUE; - } -} diff --git a/src/main/java/fr/rakambda/filesecure/processor/FileMetadata.java b/src/main/java/fr/rakambda/filesecure/processor/FileMetadata.java deleted file mode 100644 index 682f84c..0000000 --- a/src/main/java/fr/rakambda/filesecure/processor/FileMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -package fr.rakambda.filesecure.processor; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; -import java.time.ZonedDateTime; -import java.util.Optional; - -@Getter -@RequiredArgsConstructor -public class FileMetadata{ - @Nullable - private final ZonedDateTime takenDate; - @Nullable - private final ZonedDateTime createdDate; - @Nullable - private final ZonedDateTime nameDate; - - @NonNull - public ZonedDateTime getDate(){ - return Optional.ofNullable(nameDate) - .or(() -> Optional.ofNullable(takenDate)) - .or(() -> Optional.ofNullable(createdDate)) - .orElseGet(ZonedDateTime::now); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/processor/Processor.java b/src/main/java/fr/rakambda/filesecure/processor/Processor.java deleted file mode 100644 index 7a64a6d..0000000 --- a/src/main/java/fr/rakambda/filesecure/processor/Processor.java +++ /dev/null @@ -1,144 +0,0 @@ -package fr.rakambda.filesecure.processor; - -import fr.rakambda.filesecure.config.Mapping; -import fr.rakambda.filesecure.config.Rule; -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.exceptions.FlagsProcessingException; -import fr.rakambda.filesecure.metadata.MetadataExtractor; -import fr.rakambda.filesecure.utils.FileOperations; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.util.Comparator; -import java.util.Optional; -import java.util.Set; - -@RequiredArgsConstructor -@Slf4j -public class Processor{ - @NonNull - private final Rule rule; - private final Mapping mapping; - private final MetadataExtractor metadataExtractor; - private final FileOperations fileOperations; - - public void process() throws IOException{ - if(!Files.exists(mapping.getInput())){ - log.warn("Input folder {} doesn't exist, skipping mapping", mapping.getInput()); - return; - } - - log.info("Processing mapping ({}) {} ==> {}", rule.getOperation().name(), mapping.getInput(), mapping.getOutput()); - log.debug("Building differences..."); - var differenceVisitor = new DifferenceVisitor(mapping.getInput(), rule.getFilters(), rule.getExcludes(), rule.getFolderExcludes()); - Files.walkFileTree(mapping.getInput(), Set.of(), rule.getMaxDepth(), differenceVisitor); - - log.debug("Applying strategy"); - differenceVisitor.getPaths().forEach(this::applyStrategy); - - log.debug("Cleaning up"); - differenceVisitor.getFolders().stream() - .sorted(Comparator.comparingInt(Path::getNameCount).reversed()) - .forEach(this::cleanup); - } - - private void cleanup(@NonNull Path path){ - for(var option : rule.getFolderTransformers()){ - option.apply(fileOperations, path, mapping.getInput()); - } - } - - @SneakyThrows(IOException.class) - private void applyStrategy(@NonNull Path path) throws FlagsProcessingException{ - var relativeInput = mapping.getInput().relativize(path); - var exactDestination = mapping.getOutput().resolve(relativeInput); - var destination = exactDestination; - var metadata = metadataExtractor.getMetadata(path); - - for(var option : rule.getFileTransformers()){ - try{ - var newDestination = option.apply(path, exactDestination, mapping.getOutput(), destination, metadata); - if(newDestination.isPresent()){ - log.debug("File destination {} changed to {} by {}", destination, newDestination.get(), option.getClass().getSimpleName()); - destination = newDestination.get(); - } - } - catch(AbandonBackupException e){ - log.debug("Transformer {} abandoned backup for file {}", option.getClass().getSimpleName(), path); - return; - } - catch(Exception e){ - log.error("Error applying transformer {} to file {}", option.getClass(), path, e); - throw new FlagsProcessingException("Error applying transformer to file " + path); - } - } - - if(rule.isSkipIfAlreadyExists() && Files.exists(destination)){ - return; - } - var nonExistingDestination = generateUniqueName(destination); - - log.info("{} file {} to {}", rule.getOperation().name(), path, nonExistingDestination); - try{ - rule.getOperation().getProcessor().accept(fileOperations, path, nonExistingDestination); - if(rule.isSetFileCreationDate()){ - Optional.ofNullable(metadata.getNameDate()) - .or(() -> Optional.ofNullable(metadata.getTakenDate())) - .ifPresent(date -> { - try{ - var instant = date.toInstant(); - var fileTime = FileTime.from(instant); - log.info("Setting creation date for {} to {}", exactDestination, instant); - - var attributes = Files.getFileAttributeView(nonExistingDestination, BasicFileAttributeView.class); - BasicFileAttributes attributesRead = attributes.readAttributes(); - attributes.setTimes(fileTime, attributesRead.lastAccessTime(), fileTime); - } - catch(IOException e){ - log.warn("Failed to set file creation date to obtained metadata", e); - } - }); - } - } - catch(Exception e){ - log.warn("Error applying operation {} on file {}", rule.getOperation(), path, e); - } - } - - @NonNull - private Path generateUniqueName(@NonNull Path sourcePath){ - try{ - if(!Files.exists(sourcePath)){ - return sourcePath; - } - - var fileName = sourcePath.getFileName().toString(); - - var extIndex = fileName.lastIndexOf("."); - var prefix = fileName.substring(0, extIndex); - var ext = fileName.substring(extIndex); - - var i = 0; - var finalName = fileName; - do{ - var newName = "%s (%d)%s".formatted(prefix, ++i, ext); - log.debug("File '{}' already exists in target, trying with suffix {}", sourcePath, i); - finalName = newName; - } - while(Files.exists(sourcePath.getParent().resolve(finalName))); - - return sourcePath.getParent().resolve(finalName); - } - catch(Exception e){ - log.error("Failed to generate unique name", e); - throw new RuntimeException("Failed to generate unique name", e); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/FileOperations.java b/src/main/java/fr/rakambda/filesecure/utils/FileOperations.java deleted file mode 100644 index c995e32..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/FileOperations.java +++ /dev/null @@ -1,61 +0,0 @@ -package fr.rakambda.filesecure.utils; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; - -@Slf4j -@RequiredArgsConstructor -public class FileOperations{ - private final boolean dryRun; - - public void copy(@NonNull Path in, @NonNull Path out) throws IOException{ - if(dryRun){ - return; - } - createDirectories(out.getParent()); - Files.copy(in, out, StandardCopyOption.COPY_ATTRIBUTES); - } - - public void move(@NonNull Path in, @NonNull Path out) throws IOException{ - if(dryRun){ - return; - } - createDirectories(out.getParent()); - Files.move(in, out); - } - - public void moveWithCopy(@NonNull Path in, @NonNull Path out) throws IOException{ - if(dryRun){ - return; - } - - copy(in, out); - Files.delete(in); - } - - public void delete(Path path) throws IOException{ - if(dryRun){ - return; - } - Files.delete(path); - } - - public boolean deleteIfExists(Path path) throws IOException{ - if(dryRun){ - return false; - } - return Files.deleteIfExists(path); - } - - private void createDirectories(Path path) throws IOException{ - Files.createDirectories(path); - if(!Files.isDirectory(path)){ - throw new IOException("Destination folder %s already exists as a file".formatted(path)); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/ThrowingTriConsumer.java b/src/main/java/fr/rakambda/filesecure/utils/ThrowingTriConsumer.java deleted file mode 100644 index 56aab7f..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/ThrowingTriConsumer.java +++ /dev/null @@ -1,7 +0,0 @@ -package fr.rakambda.filesecure.utils; - -import java.io.IOException; - -public interface ThrowingTriConsumer{ - void accept(T t, U u, V v) throws IOException; -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/DateTimeDeserializer.java b/src/main/java/fr/rakambda/filesecure/utils/json/DateTimeDeserializer.java deleted file mode 100644 index 9f267e5..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/DateTimeDeserializer.java +++ /dev/null @@ -1,17 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -public class DateTimeDeserializer extends JsonDeserializer{ - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); - - @Override - public LocalDateTime deserialize(JsonParser jsonParser, DeserializationContext context) throws IOException{ - return LocalDateTime.parse(jsonParser.getValueAsString(), FORMATTER); - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/GeonamesTimeZone.java b/src/main/java/fr/rakambda/filesecure/utils/json/GeonamesTimeZone.java deleted file mode 100644 index 0889cfe..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/GeonamesTimeZone.java +++ /dev/null @@ -1,43 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import lombok.Getter; -import lombok.NoArgsConstructor; -import java.time.LocalDateTime; -import java.time.ZoneId; - -@Getter -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -public class GeonamesTimeZone{ - @JsonProperty("lng") - private double longitude; - @JsonProperty("lat") - private double latitude; - @JsonProperty("sunrise") - @JsonDeserialize(using = DateTimeDeserializer.class) - private LocalDateTime sunrise; - @JsonProperty("sunset") - @JsonDeserialize(using = DateTimeDeserializer.class) - private LocalDateTime sunset; - @JsonProperty("time") - @JsonDeserialize(using = DateTimeDeserializer.class) - private LocalDateTime time; - @JsonProperty("gmtOffset") - private int gmtOffset; - @JsonProperty("rawOffset") - private int rawOffset; - @JsonProperty("dstOffset") - private int dstOffset; - @JsonProperty("timezoneId") - @JsonDeserialize(using = ZoneIdDeserializer.class) - private ZoneId timezoneId; - @JsonProperty("countryCode") - private String countryCode; - @JsonProperty("countryName") - private String countryName; -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/JacksonObjectMapper.java b/src/main/java/fr/rakambda/filesecure/utils/json/JacksonObjectMapper.java deleted file mode 100644 index ebe92c8..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/JacksonObjectMapper.java +++ /dev/null @@ -1,65 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.json.JsonMapper; -import kong.unirest.GenericType; -import kong.unirest.ObjectMapper; -import java.io.IOException; -import java.lang.reflect.Type; -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.ANY; -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; -import static com.fasterxml.jackson.annotation.PropertyAccessor.CREATOR; -import static com.fasterxml.jackson.annotation.PropertyAccessor.FIELD; -import static com.fasterxml.jackson.annotation.PropertyAccessor.GETTER; -import static com.fasterxml.jackson.annotation.PropertyAccessor.SETTER; -import static com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS; -import static com.fasterxml.jackson.core.json.JsonReadFeature.ALLOW_TRAILING_COMMA; -import static com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS; - -public class JacksonObjectMapper implements ObjectMapper{ - private final com.fasterxml.jackson.databind.ObjectMapper mapper; - - public JacksonObjectMapper(){ - mapper = JsonMapper.builder() - .enable(ALLOW_TRAILING_COMMA) - .enable(ALLOW_COMMENTS) - .visibility(FIELD, ANY) - .visibility(GETTER, NONE) - .visibility(SETTER, NONE) - .visibility(CREATOR, NONE) - .enable(ACCEPT_CASE_INSENSITIVE_ENUMS) - .build(); - } - - public T readValue(String value, Class valueType){ - try{ - return mapper.readValue(value, valueType); - } - catch(IOException var4){ - throw new RuntimeException(var4); - } - } - - public T readValue(String value, GenericType genericType){ - try{ - return mapper.readValue(value, new TypeReference(){ - public Type getType(){ - return genericType.getType(); - } - }); - } - catch(IOException var4){ - throw new RuntimeException(var4); - } - } - - public String writeValue(Object value){ - try{ - return mapper.writeValueAsString(value); - } - catch(JsonProcessingException var3){ - throw new RuntimeException(var3); - } - } -} diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/PatternDeserializer.java b/src/main/java/fr/rakambda/filesecure/utils/json/PatternDeserializer.java deleted file mode 100644 index 06ea6fe..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/PatternDeserializer.java +++ /dev/null @@ -1,17 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import org.jspecify.annotations.NonNull; -import java.io.IOException; -import java.util.regex.Pattern; - -public class PatternDeserializer extends JsonDeserializer{ - @Override - public Pattern deserialize(@NonNull JsonParser jsonParser, @NonNull DeserializationContext deserializationContext) throws IOException{ - return Pattern.compile(jsonParser.getValueAsString()); - } -} - - diff --git a/src/main/java/fr/rakambda/filesecure/utils/json/ZoneIdDeserializer.java b/src/main/java/fr/rakambda/filesecure/utils/json/ZoneIdDeserializer.java deleted file mode 100644 index b24ef82..0000000 --- a/src/main/java/fr/rakambda/filesecure/utils/json/ZoneIdDeserializer.java +++ /dev/null @@ -1,14 +0,0 @@ -package fr.rakambda.filesecure.utils.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import java.time.ZoneId; - -public class ZoneIdDeserializer extends JsonDeserializer{ - @Override - public ZoneId deserialize(JsonParser jsonParser, DeserializationContext context) throws IOException{ - return ZoneId.of(jsonParser.getValueAsString()); - } -} diff --git a/src/main/java/module-info.java.old b/src/main/java/module-info.java.old deleted file mode 100644 index b3b7454..0000000 --- a/src/main/java/module-info.java.old +++ /dev/null @@ -1,16 +0,0 @@ -open module fr.raksrinana.filesecure { - requires unirest.java; - requires metadata.extractor; - requires pointlocation6709; - requires com.fasterxml.jackson.core; - requires com.fasterxml.jackson.annotation; - requires com.fasterxml.jackson.databind; - requires org.slf4j; - requires org.apache.logging.log4j; - requires org.apache.logging.log4j.core; - requires org.apache.logging.log4j.slf4j; - requires java.scripting; - requires info.picocli; - requires static lombok; - requires static org.jetbrains.annotations; -} \ No newline at end of file diff --git a/src/main/kotlin/fr/rakambda/filesecure/Main.kt b/src/main/kotlin/fr/rakambda/filesecure/Main.kt new file mode 100644 index 0000000..03add10 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/Main.kt @@ -0,0 +1,214 @@ +package fr.rakambda.filesecure + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.Context +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.core.main +import com.github.ajalt.clikt.parameters.options.check +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.types.int +import com.github.ajalt.clikt.parameters.types.long +import com.github.ajalt.clikt.parameters.types.path +import fr.rakambda.filesecure.config.Configuration +import fr.rakambda.filesecure.console.ConsoleHandler +import fr.rakambda.filesecure.console.PauseGate +import fr.rakambda.filesecure.conversion.ConversionTools +import fr.rakambda.filesecure.conversion.WindowsAffinity +import fr.rakambda.filesecure.metadata.MetadataExtractor +import fr.rakambda.filesecure.metadata.OfflineTimeZoneResolver +import fr.rakambda.filesecure.pipeline.FileOperations +import fr.rakambda.filesecure.pipeline.PipelineRunner +import fr.rakambda.filesecure.pipeline.PipelineStats +import fr.rakambda.filesecure.pipeline.RunEnvironment +import fr.rakambda.filesecure.progress.ConsoleProgressReporter +import fr.rakambda.filesecure.progress.NoOpProgressReporter +import io.github.oshai.kotlinlogging.KotlinLogging +import io.github.oshai.kotlinlogging.Level +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.time.Duration.Companion.seconds + +// Lazy so the startup message setting in main is applied before kotlin-logging initializes +private val log by lazy { KotlinLogging.logger {} } + +class FileSecureCommand : CliktCommand(name = "filesecure"){ + private val configurationFile by option("-c", "--config", help = "The path to the configuration file") + .path(mustExist = true, canBeDir = false, mustBeReadable = true) + .required() + private val dryRun by option("--dry", help = "Performs a dry run, nothing is written") + .flag() + private val threads by option("-t", "--threads", help = "How many conversions are performed in parallel") + .int() + .default(1) + .check("must be >= 1") { it >= 1 } + private val workers by option("-w", "--workers", help = "How many files are processed in parallel (scan, probe, copy)") + .int() + .default(8) + .check("must be >= 1") { it >= 1 } + private val ffmpegThreads by option("-ft", "--ffmpeg-threads", help = "Value of ffmpeg's -threads option for each conversion") + .int() + .check("must be >= 0") { it >= 0 } + private val ffmpegDirectory by option("--ffmpeg", help = "Folder containing the ffmpeg executable, uses the PATH if not set") + .path(mustExist = true, canBeFile = false) + private val ffprobeDirectory by option("--ffprobe", help = "Folder containing the ffprobe executable, uses the PATH if not set") + .path(mustExist = true, canBeFile = false) + private val magick by option("--magick", help = "ImageMagick executable used for photo conversions") + .default("magick") + private val affinityMask by option("--affinity", help = "Windows only, mask of the CPU cores conversions can run on, e.g. 7 for cores 1, 2 and 3") + .long() + .check("must be > 0") { it > 0 } + private val progress by option("--progress", help = "Show progress bars, enabled by default when running in a terminal") + .flag("--no-progress", default = System.console()?.isTerminal == true) + + override fun help(context: Context): String = "Moves, backs up and converts files through configurable pipelines." + + override fun run(){ + val configuration = try{ + Configuration.load(configurationFile) + } + catch(e: Exception){ + log.error(e) { "Failed to load configuration from $configurationFile" } + throw ProgramResult(1) + } + + val pauseGate = PauseGate() + val affinity = affinityMask?.let(::WindowsAffinity) + val progressReporter = if(progress) ConsoleProgressReporter() else NoOpProgressReporter + val report = RunReport(configuration, dryRun) + val completed = progressReporter.use { + val environment = RunEnvironment( + fileOperations = FileOperations(dryRun), + metadataExtractor = MetadataExtractor(OfflineTimeZoneResolver()), + tools = ConversionTools(ffmpegDirectory, ffprobeDirectory, magick, ffmpegThreads, affinity?.let { it::apply }), + conversionThreads = threads, + workers = workers, + pauseGate = pauseGate, + progress = progressReporter, + ) + runPipelines(configuration, environment, pauseGate, report) + } + if(!completed){ + // The JVM is shutting down, the shutdown hook logs the summary + return + } + report.log() + val code = exitCode(report.errors, report.stopped) + if(code != 0){ + throw ProgramResult(code) + } + } + + /** + * @return false if the JVM is shutting down. + */ + private fun runPipelines(configuration: Configuration, environment: RunEnvironment, pauseGate: PauseGate, report: RunReport): Boolean = runBlocking { + val runner = PipelineRunner(environment) + val job = launch(Dispatchers.Default) { + configuration.pipelines.forEachIndexed { index, pipeline -> + val name = pipeline.displayName(index) + if(!pipeline.enabled){ + log.info { "Pipeline $name is disabled" } + return@forEachIndexed + } + val stats = report.start(index) + try{ + runner.run(pipeline, name, stats) + } + catch(e: CancellationException){ + throw e + } + catch(e: Exception){ + log.error(e) { "Failed to run pipeline $name" } + stats.error() + } + } + } + + ConsoleHandler(System.`in`, pauseGate) { + if(job.isActive){ + report.stopped = true + } + job.cancel() + }.start() + val shutdownHook = Thread { + job.cancel() + // Give running conversions the time to be stopped and their temporary files removed + runBlocking { withTimeoutOrNull(30.seconds) { job.join() } } + report.stopped = true + report.log() + } + Runtime.getRuntime().addShutdownHook(shutdownHook) + + job.join() + try{ + Runtime.getRuntime().removeShutdownHook(shutdownHook) + true + } + catch(_: IllegalStateException){ + // Already shutting down + false + } + } +} + +/** + * Counts of every enabled pipeline of a run. + */ +private class RunReport(private val configuration: Configuration, private val dryRun: Boolean){ + private val stats = arrayOfNulls(configuration.pipelines.size) + + /** + * Set when the run is stopped with the `q` command or by the JVM shutting down. + */ + @Volatile + var stopped = false + + val errors: Long + get() = stats.sumOf { it?.errorCount ?: 0 } + + @Synchronized + fun start(index: Int): PipelineStats = PipelineStats().also { stats[index] = it } + + @Synchronized + fun log(){ + val mode = if(dryRun) " (dry run)" else "" + val total = PipelineStats() + configuration.pipelines.forEachIndexed { index, pipeline -> + if(!pipeline.enabled){ + return@forEachIndexed + } + val name = pipeline.displayName(index) + val pipelineStats = stats[index] + if(pipelineStats == null){ + log.info { "Pipeline '$name'$mode: not run" } + return@forEachIndexed + } + total += pipelineStats + val level = if(pipelineStats.errorCount > 0) Level.ERROR else Level.INFO + log.at(level) { message = "Pipeline '$name'$mode: ${pipelineStats.describe(dryRun)}" } + } + val level = if(total.errorCount > 0) Level.ERROR else Level.INFO + val suffix = if(stopped) ", stopped before the end" else "" + log.at(level) { message = "Total$mode: ${total.describe(dryRun)}$suffix" } + } +} + +/** + * @return 3 if the run was stopped with the `q` command, 2 if any error happened, 0 otherwise. + */ +internal fun exitCode(errors: Long, stopped: Boolean): Int = when{ + stopped -> 3 + errors > 0 -> 2 + else -> 0 +} + +fun main(args: Array){ + System.setProperty("kotlin-logging.logStartupMessage", "false") + FileSecureCommand().main(args) +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/config/Configuration.kt b/src/main/kotlin/fr/rakambda/filesecure/config/Configuration.kt new file mode 100644 index 0000000..fe08faa --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/config/Configuration.kt @@ -0,0 +1,93 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.json.JsonReadFeature +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.MapperFeature +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.cfg.MapperBuilder +import com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer +import com.fasterxml.jackson.databind.json.JsonMapper +import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper +import com.fasterxml.jackson.module.kotlin.kotlinModule +import fr.rakambda.filesecure.stage.Convert +import java.io.Reader +import java.nio.file.Files +import java.nio.file.Path + +data class Configuration( + val pipelines: List = emptyList(), +){ + /** + * Checks constraints the JSON schema alone can't express. + * + * @throws InvalidConfigurationException If the configuration can't be run. + */ + fun validate(){ + pipelines.forEachIndexed { index, pipeline -> + val name = pipeline.displayName(index) + if(pipeline.mappings.isEmpty()){ + throw InvalidConfigurationException("Pipeline '$name' has no mappings") + } + if(pipeline.stages.count { it is Convert } > 1){ + throw InvalidConfigurationException("Pipeline '$name' has more than one Convert stage") + } + if(pipeline.scan.maxDepth < 0){ + throw InvalidConfigurationException("Pipeline '$name' has a negative maxDepth") + } + } + } + + companion object{ + private const val TEMPLATES = "templates" + private val yamlMapper = configure(YAMLMapper.builder()).build() + private val jsonMapper = configure(JsonMapper.builder()) + .enable(JsonReadFeature.ALLOW_JAVA_COMMENTS) + .enable(JsonReadFeature.ALLOW_TRAILING_COMMA) + .build() + + private fun > configure(builder: B): B = builder + .addModule(kotlinModule()) + .addModule(SimpleModule().addDeserializer(Path::class.java, PathDeserializer)) + .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) + .enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) + + /** + * Reads a YAML configuration, or a JSON one if the file name ends with `.json`. + */ + fun load(path: Path): Configuration = Files.newBufferedReader(path).use { + parse(it, json = path.fileName.toString().endsWith(".json", ignoreCase = true)) + } + + /** + * Reads a configuration: YAML anchors and aliases are resolved, `templates` is dropped, `defaults` is merged into the pipelines, + * `mappingsFile`s are read and `mappingTable`s are expanded into mappings before binding. + */ + fun parse(reader: Reader, json: Boolean = false): Configuration{ + val mapper = if(json) jsonMapper else yamlMapper + val tree = if(json) ConfigurationTree(mapper.readTree(reader)) else ConfigurationTree.readYaml(reader) + (tree.root as? ObjectNode)?.remove(TEMPLATES) + applyDefaults(tree, mapper) + includeMappingsFiles(tree) + expandMappingTables(tree) + return tree.bind(mapper, tree.root, Configuration::class.java).also { it.validate() } + } + + fun parse(content: String, json: Boolean = false): Configuration = parse(content.reader(), json) + } +} + +/** + * Reads paths as plain file paths. Jackson's default reads `D:/folder` as a URI with the scheme `D` when not running on Windows. + */ +private object PathDeserializer : StdScalarDeserializer(Path::class.java){ + private fun readResolve(): Any = PathDeserializer + + override fun deserialize(parser: JsonParser, context: DeserializationContext): Path = Path.of(parser.valueAsString) +} + +class InvalidConfigurationException(message: String) : RuntimeException(message) diff --git a/src/main/kotlin/fr/rakambda/filesecure/config/ConfigurationTree.kt b/src/main/kotlin/fr/rakambda/filesecure/config/ConfigurationTree.kt new file mode 100644 index 0000000..a290de6 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/config/ConfigurationTree.kt @@ -0,0 +1,133 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.core.JsonLocation +import com.fasterxml.jackson.core.io.ContentReference +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.DoubleNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.LongNode +import com.fasterxml.jackson.databind.node.MissingNode +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.TextNode +import com.fasterxml.jackson.databind.node.TreeTraversingParser +import org.yaml.snakeyaml.LoaderOptions +import org.yaml.snakeyaml.Yaml +import org.yaml.snakeyaml.constructor.SafeConstructor +import org.yaml.snakeyaml.error.Mark +import org.yaml.snakeyaml.nodes.MappingNode +import org.yaml.snakeyaml.nodes.Node +import org.yaml.snakeyaml.nodes.ScalarNode +import org.yaml.snakeyaml.nodes.SequenceNode +import org.yaml.snakeyaml.nodes.Tag +import java.io.Reader +import java.math.BigInteger +import java.util.Collections +import java.util.IdentityHashMap + +/** + * A configuration tree remembering where its nodes were read, so binding errors can tell the line of the faulty value. + */ +internal class ConfigurationTree( + val root: JsonNode, + private val locations: Map = emptyMap(), +){ + fun bind(mapper: ObjectMapper, node: JsonNode, type: Class): T = mapper.readValue(LocatingParser(node, mapper), type) + + private inner class LocatingParser(node: JsonNode, mapper: ObjectMapper) : TreeTraversingParser(node, mapper){ + override fun currentTokenLocation(): JsonLocation = currentNode()?.let { locations[it] } ?: JsonLocation.NA + + override fun currentLocation(): JsonLocation = currentTokenLocation() + + @Deprecated("Deprecated in Jackson") + override fun getTokenLocation(): JsonLocation = currentTokenLocation() + + @Deprecated("Deprecated in Jackson") + override fun getCurrentLocation(): JsonLocation = currentTokenLocation() + } + + companion object{ + private val INTEGER = Regex("[-+]?(0|[1-9][0-9]*)") + private val DECIMAL = Regex("[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?") + private val BOOLEAN = Regex("true|True|TRUE|false|False|FALSE") + + /** + * Reads a YAML document, resolving anchors, aliases and `<<` merge keys. Scalars are typed like Jackson's YAML parser does: only `true` and + * `false` are booleans, other YAML 1.1 forms (`yes`, `on`, dates, octal numbers...) stay text. + */ + fun readYaml(reader: Reader): ConfigurationTree{ + val options = LoaderOptions().apply { + maxAliasesForCollections = 1000 + } + val document = Yaml(SafeConstructor(options)).compose(reader) ?: return ConfigurationTree(MissingNode.getInstance()) + val converter = YamlConverter() + return ConfigurationTree(converter.convert(document), converter.locations) + } + + private class YamlConverter{ + val locations = IdentityHashMap() + private val nodes = JsonNodeFactory.instance + private val visiting = Collections.newSetFromMap(IdentityHashMap()) + + fun convert(node: Node): JsonNode{ + if(!visiting.add(node)){ + throw InvalidConfigurationException("Recursive alias at ${position(node.startMark)}") + } + try{ + val result = when(node){ + is ScalarNode -> scalar(node) + is SequenceNode -> nodes.arrayNode().addAll(node.value.map(::convert)) + is MappingNode -> mapping(node) + else -> throw InvalidConfigurationException("Unsupported YAML node at ${position(node.startMark)}") + } + locations[result] = node.startMark.let { JsonLocation(ContentReference.unknown(), -1L, -1L, it.line + 1, it.column + 1) } + return result + } + finally{ + visiting.remove(node) + } + } + + private fun scalar(node: ScalarNode): JsonNode{ + val value = node.value + if(!node.isPlain){ + return TextNode(value) + } + return when{ + node.tag == Tag.NULL -> nodes.nullNode() + node.tag == Tag.BOOL && BOOLEAN.matches(value) -> nodes.booleanNode(value.equals("true", ignoreCase = true)) + node.tag == Tag.INT && INTEGER.matches(value) -> value.toLongOrNull()?.let(::LongNode) ?: nodes.numberNode(BigInteger(value)) + node.tag == Tag.FLOAT && DECIMAL.matches(value) -> DoubleNode(value.toDouble()) + else -> TextNode(value) + } + } + + private fun mapping(node: MappingNode): ObjectNode{ + val result = nodes.objectNode() + val merged = mutableListOf() + for(tuple in node.value){ + val key = tuple.keyNode + if(key.tag == Tag.MERGE){ + merged += tuple.valueNode + continue + } + if(key !is ScalarNode){ + throw InvalidConfigurationException("Keys must be plain values, at ${position(key.startMark)}") + } + if(result.has(key.value)){ + throw InvalidConfigurationException("Duplicate key '${key.value}' at ${position(key.startMark)}") + } + result.set(key.value, convert(tuple.valueNode)) + } + merged.flatMap { if(it is SequenceNode) it.value else listOf(it) }.forEach { source -> + val values = convert(source) as? ObjectNode ?: throw InvalidConfigurationException("'<<' must be given mappings, at ${position(source.startMark)}") + values.properties().filter { !result.has(it.key) }.forEach { result.set(it.key, it.value) } + } + return result + } + + private fun position(mark: Mark): String = "line ${mark.line + 1}, column ${mark.column + 1}" + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/config/Defaults.kt b/src/main/kotlin/fr/rakambda/filesecure/config/Defaults.kt new file mode 100644 index 0000000..7818893 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/config/Defaults.kt @@ -0,0 +1,70 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode + +private const val DEFAULTS = "defaults" +internal const val USE_DEFAULTS = "useDefaults" +internal val FORBIDDEN_KEYS = listOf("name", "mappings", MAPPING_TABLE, MAPPINGS_FILE, "enabled", USE_DEFAULTS) +private val LIST_KEYS = setOf("stages", "folderActions") +private const val SCAN = "scan" + +/** + * Removes the `defaults` section of the configuration and merges it into every pipeline not opting out with `useDefaults: false`: + * its `stages` and `folderActions` are placed before the pipeline's own, `scan` is merged key by key and other keys only apply when the pipeline + * doesn't set them. + * + * @throws InvalidConfigurationException If `defaults` or `useDefaults` is misused. + * @throws JsonMappingException If `defaults` isn't a valid pipeline. + */ +internal fun applyDefaults(tree: ConfigurationTree, mapper: ObjectMapper){ + val root = tree.root as? ObjectNode ?: return + val defaults = root.remove(DEFAULTS)?.takeUnless { it.isNull } + if(defaults != null){ + validateDefaults(defaults, tree, mapper) + } + (root.get("pipelines") as? ArrayNode)?.forEachIndexed { index, pipeline -> + if(pipeline !is ObjectNode){ + return@forEachIndexed + } + val useDefaults = pipeline.remove(USE_DEFAULTS) + if(useDefaults != null && !useDefaults.isBoolean){ + throw InvalidConfigurationException("'$USE_DEFAULTS' of pipeline #${index + 1} must be true or false") + } + if(defaults != null && useDefaults?.booleanValue() != false){ + merge(defaults as ObjectNode, pipeline) + } + } +} + +private fun validateDefaults(defaults: JsonNode, tree: ConfigurationTree, mapper: ObjectMapper){ + if(defaults !is ObjectNode){ + throw InvalidConfigurationException("'$DEFAULTS' must be a map of pipeline keys") + } + FORBIDDEN_KEYS.firstOrNull { defaults.has(it) }?.let { + throw InvalidConfigurationException("'$it' can't be set in '$DEFAULTS'") + } + try{ + tree.bind(mapper, defaults, Pipeline::class.java) + } + catch(e: JsonMappingException){ + throw JsonMappingException.wrapWithPath(e, Configuration::class.java, DEFAULTS) + } +} + +/** + * Default nodes are shared between pipelines rather than copied, so they keep their location. They are never modified afterwards. + */ +private fun merge(defaults: ObjectNode, pipeline: ObjectNode){ + defaults.properties().forEach { (key, value) -> + val own = pipeline.get(key) + when{ + own == null -> pipeline.set(key, value) + key in LIST_KEYS && own is ArrayNode && value is ArrayNode -> pipeline.set(key, pipeline.arrayNode().addAll(value).addAll(own)) + key == SCAN && own is ObjectNode && value is ObjectNode -> pipeline.set(key, pipeline.objectNode().setAll(value).setAll(own)) + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/config/MappingTable.kt b/src/main/kotlin/fr/rakambda/filesecure/config/MappingTable.kt new file mode 100644 index 0000000..00ed0aa --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/config/MappingTable.kt @@ -0,0 +1,163 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.MissingNode +import com.fasterxml.jackson.databind.node.NullNode +import com.fasterxml.jackson.databind.node.ObjectNode +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import java.util.regex.Pattern +import java.util.regex.PatternSyntaxException + +internal const val MAPPING_TABLE = "mappingTable" +internal const val MAPPINGS_FILE = "mappingsFile" +private const val MAPPINGS = "mappings" +internal val TABLE_KEYS = setOf("input", "output", "paths", "entries") +internal val RULE_KEYS = setOf("regex", "replacement") +private val DRIVE = Regex("[A-Za-z]:.*") + +/** + * Replaces the `mappingTable` of every pipeline with regular mappings, appended after the pipeline's own: each entry `key: value` becomes + * `{input: /, output: /}`, keys and values being first rewritten by the first `paths` rule fully matching them. + * + * @throws InvalidConfigurationException If a `mappingTable` is invalid. + */ +internal fun expandMappingTables(tree: ConfigurationTree) = forEachPipeline(tree) { pipeline, name -> + val table = pipeline.remove(MAPPING_TABLE) ?: return@forEachPipeline + val generated = MappingTable.read(table, name).expand() + val mappings = ownMappings(pipeline, name, MAPPING_TABLE) + generated.forEach { (input, output) -> mappings.addObject().put("input", input).put("output", output) } +} + +/** + * Appends to the mappings of every pipeline the list of mappings read from its `mappingsFile` (YAML or JSON, an empty file meaning no mapping), so a + * tool can rewrite that file without touching the main configuration. A relative path is resolved against the working directory. + * + * @throws InvalidConfigurationException If a `mappingsFile` is invalid or can't be read. + */ +internal fun includeMappingsFiles(tree: ConfigurationTree) = forEachPipeline(tree) { pipeline, name -> + val node = pipeline.remove(MAPPINGS_FILE) ?: return@forEachPipeline + fun error(message: String) = InvalidConfigurationException("'$MAPPINGS_FILE' of pipeline '$name' $message") + val path = node.takeIf { it.isTextual }?.asText()?.takeIf { it.isNotEmpty() } ?: throw error("must be a non-empty text") + val included = try{ + Files.newBufferedReader(Path.of(path)).use { ConfigurationTree.readYaml(it).root } + } + catch(e: IOException){ + throw error("can't be read: $e") + } + when(included){ + is MissingNode, is NullNode -> {} + is ArrayNode -> ownMappings(pipeline, name, MAPPINGS_FILE).addAll(included) + else -> throw error("must contain a list of mappings: $path") + } +} + +private fun forEachPipeline(tree: ConfigurationTree, action: (ObjectNode, String) -> Unit){ + val root = tree.root as? ObjectNode ?: return + (root.get("pipelines") as? ArrayNode)?.forEachIndexed { index, pipeline -> + if(pipeline is ObjectNode){ + action(pipeline, pipeline.get("name")?.takeIf { it.isTextual }?.asText() ?: "#${index + 1}") + } + } +} + +private fun ownMappings(pipeline: ObjectNode, name: String, combinedWith: String): ArrayNode = when(val own = pipeline.get(MAPPINGS)){ + null, is NullNode -> pipeline.putArray(MAPPINGS) + is ArrayNode -> own + else -> throw InvalidConfigurationException("'$MAPPINGS' of pipeline '$name' must be a list to be combined with '$combinedWith'") +} + +private class MappingTable( + private val pipeline: String, + private val input: String, + private val output: String, + private val rules: List>, + private val entries: List>, +){ + fun expand(): List> = entries.map { (key, value) -> + join(input, rewrite(key)) to join(output, rewrite(value)) + } + + private fun rewrite(path: String): String{ + val result = rules.firstNotNullOfOrNull { (regex, replacement) -> + val matcher = regex.matcher(path) + if(!matcher.matches()){ + return@firstNotNullOfOrNull null + } + try{ + matcher.replaceFirst(replacement) + } + catch(e: RuntimeException){ + throw tableError(pipeline, "has an invalid replacement '$replacement' for '$path': ${e.message}") + } + } ?: return path + checkRelative(result, "rewritten path '$result' (from '$path')") + return result + } + + private fun join(base: String, path: String): String = base.trimEnd('/', '\\') + "/" + path + + private fun checkRelative(path: String, label: String){ + when{ + path.isEmpty() -> throw tableError(pipeline, "has an empty entry") + path.startsWith('/') || path.startsWith('\\') || DRIVE.matches(path) -> throw tableError(pipeline, "must have relative entries: $label is absolute") + path.split('/', '\\').any { it == ".." } -> throw tableError(pipeline, "must have entries without '..': $label") + } + } + + companion object{ + fun read(node: JsonNode, pipeline: String): MappingTable{ + fun error(message: String) = tableError(pipeline, message) + fun checkKeys(node: ObjectNode, allowed: Set, label: String){ + node.properties().map { it.key }.firstOrNull { it !in allowed }?.let { + throw error("has an unknown key '$it'$label, expected one of ${allowed.joinToString()}") + } + } + fun text(node: JsonNode?, label: String): String = node?.takeIf { it.isTextual }?.asText()?.takeIf { it.isNotEmpty() } + ?: throw error(if(node == null || node.isNull) "has no '$label'" else "must have a non-empty text as '$label'") + + if(node !is ObjectNode){ + throw error("must be a map with keys ${TABLE_KEYS.joinToString()}") + } + checkKeys(node, TABLE_KEYS, "") + val input = text(node.get("input"), "input") + val output = text(node.get("output"), "output") + val rules = when(val paths = node.get("paths")){ + null, is NullNode -> emptyList() + is ArrayNode -> paths.mapIndexed { index, rule -> + val label = " in rule #${index + 1} of 'paths'" + if(rule !is ObjectNode){ + throw error("must have maps with keys regex, replacement in 'paths'") + } + checkKeys(rule, RULE_KEYS, label) + val regex = text(rule.get("regex"), "regex") + val pattern = try{ + Pattern.compile(regex) + } + catch(e: PatternSyntaxException){ + throw error("has an invalid regex '$regex'$label: ${e.description}") + } + val replacement = rule.get("replacement")?.takeIf { it.isTextual }?.asText() ?: throw error("has no text 'replacement'$label") + pattern to replacement + } + else -> throw error("must have a list as 'paths'") + } + val entries = node.get("entries") as? ObjectNode ?: throw error(if(node.get("entries").let { it == null || it.isNull }) "has no 'entries'" else "must have a map as 'entries'") + val table = MappingTable(pipeline, input, output, rules, entries.properties().map { (key, value) -> + if(!value.isTextual){ + throw error("must have a text value for entry '$key', quote it") + } + key to value.asText() + }) + table.entries.forEach { (key, value) -> + table.checkRelative(key, "key '$key'") + table.checkRelative(value, "value '$value' of '$key'") + } + return table + } + } +} + +private fun tableError(pipeline: String, message: String) = InvalidConfigurationException("'$MAPPING_TABLE' of pipeline '$pipeline' $message") diff --git a/src/main/kotlin/fr/rakambda/filesecure/config/Pipeline.kt b/src/main/kotlin/fr/rakambda/filesecure/config/Pipeline.kt new file mode 100644 index 0000000..1a930a6 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/config/Pipeline.kt @@ -0,0 +1,65 @@ +package fr.rakambda.filesecure.config + +import fr.rakambda.filesecure.folder.FolderAction +import fr.rakambda.filesecure.stage.Stage +import java.nio.file.Path +import java.util.regex.Pattern + +/** + * A sequence of stages applied to every file found in the inputs of its mappings. + */ +data class Pipeline( + val name: String? = null, + val enabled: Boolean = true, + val mappings: List = emptyList(), + val scan: ScanOptions = ScanOptions(), + val stages: List = emptyList(), + val operation: Operation = Operation.NONE, + val conflict: ConflictStrategy = ConflictStrategy.RENAME, + val verify: Verification = Verification.SIZE, + val setFileCreationDate: Boolean = false, + val temp: Path? = null, + val folderActions: List = emptyList(), +){ + fun displayName(index: Int): String = name ?: "#${index + 1}" +} + +data class Mapping( + val input: Path, + val output: Path, +) + +data class ScanOptions( + val maxDepth: Int = Int.MAX_VALUE, + val skipHidden: Boolean = false, + /** + * Regexes matched against the folder path relative to the input, using `/` as separator. + */ + val excludedFolders: List = emptyList(), +) + +enum class Operation(val removesSource: Boolean){ + COPY(false), + MOVE(true), + MOVE_WITH_COPY(true), + NONE(false), +} + +enum class ConflictStrategy{ + /** + * Use the first free name among `name (1).ext`, `name (2).ext`... + */ + RENAME, + SKIP, +} + +/** + * How a file copied to its destination is checked before being renamed to its final name. + */ +enum class Verification{ + SIZE, + /** + * The SHA-256 of the source, computed while copying, is compared with the one of the copy read back. + */ + CHECKSUM, +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/console/ConsoleHandler.kt b/src/main/kotlin/fr/rakambda/filesecure/console/ConsoleHandler.kt new file mode 100644 index 0000000..abd97b6 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/console/ConsoleHandler.kt @@ -0,0 +1,43 @@ +package fr.rakambda.filesecure.console + +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.InputStream + +private val log = KotlinLogging.logger {} + +/** + * Reads commands from the standard input: `p` pauses, `r` resumes and `q` quits. + */ +class ConsoleHandler( + private val input: InputStream, + private val pauseGate: PauseGate, + private val onQuit: () -> Unit, +) : Thread("Console watcher"){ + init{ + isDaemon = true + } + + override fun run(){ + try{ + input.bufferedReader().lineSequence().forEach { line -> + when(line.trim()){ + "q" -> { + log.info { "Exiting" } + onQuit() + } + "p" -> { + log.info { "Pausing, running conversions will finish" } + pauseGate.pause() + } + "r" -> { + log.info { "Resuming" } + pauseGate.resume() + } + } + } + } + catch(e: Exception){ + log.warn(e) { "Error reading console commands" } + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/console/PauseGate.kt b/src/main/kotlin/fr/rakambda/filesecure/console/PauseGate.kt new file mode 100644 index 0000000..491c2ae --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/console/PauseGate.kt @@ -0,0 +1,29 @@ +package fr.rakambda.filesecure.console + +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first + +/** + * Lets work be paused and resumed, work already started is not interrupted. + */ +class PauseGate{ + private val paused = MutableStateFlow(false) + + val isPaused: Boolean + get() = paused.value + + fun pause(){ + paused.value = true + } + + fun resume(){ + paused.value = false + } + + /** + * Suspends while paused. + */ + suspend fun awaitResumed(){ + paused.first { !it } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/ConversionTools.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/ConversionTools.kt new file mode 100644 index 0000000..01113e5 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/ConversionTools.kt @@ -0,0 +1,75 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.ffprobe.FFprobe +import com.github.kokorin.jaffree.ffprobe.FFprobeResult +import com.github.kokorin.jaffree.process.JaffreeAbnormalExitException +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import java.nio.file.Path + +private val log = KotlinLogging.logger {} + +/** + * External programs used to probe and convert files. + * + * @param ffmpegDirectory Folder containing the ffmpeg executable, null to look it up in the PATH. + * @param ffprobeDirectory Folder containing the ffprobe executable, null to look it up in the PATH. + * @param magick ImageMagick executable. + * @param ffmpegThreads Value of ffmpeg's `-threads` option, null to let ffmpeg decide. + * @param processListener Called with every external process started for a conversion. + */ +class ConversionTools( + private val ffmpegDirectory: Path? = null, + private val ffprobeDirectory: Path? = null, + val magick: String = "magick", + val ffmpegThreads: Int? = null, + val processListener: ((Process) -> Unit)? = null, +){ + private val probeLock = Semaphore(MAX_CONCURRENT_PROBES) + + fun ffmpeg(): CustomFFmpeg = CustomFFmpeg(ffmpegDirectory?.resolve("ffmpeg") ?: Path.of("ffmpeg"), processListener) + + /** + * @return The probe result, or null if the file couldn't be probed (not a media file for example). + */ + suspend fun probe(file: Path): FFprobeResult? = probeLock.withPermit { + runInterruptible(Dispatchers.IO) { + try{ + log.debug { "Probing file $file" } + val ffprobe = ffprobeDirectory?.let { FFprobe.atPath(it) } ?: FFprobe.atPath() + ffprobe.setShowStreams(true) + .setShowFormat(true) + .setInput(file.toString()) + .execute() + } + catch(e: Exception){ + log.warn { "Failed to probe file $file, it is considered not convertible: ${describeFailure(e)}" } + null + } + } + } + + private companion object{ + const val MAX_CONCURRENT_PROBES = 10 + } +} + +/** + * Message of the exception, followed by the error lines the process printed if it is a Jaffree process failure (they aren't logged by Jaffree). + */ +internal fun describeFailure(e: Throwable): String{ + val processErrors = generateSequence(e) { it.cause } + .filterIsInstance() + .firstOrNull() + ?.processErrorLogMessages + .orEmpty() + .joinToString(" | ") { it.message.trim() } + if(processErrors.isEmpty()){ + return "${e.message}" + } + // Jaffree's message refers to its logs, where these lines aren't written + return "${e.message.orEmpty().removeSuffix(" Check logs for detailed error message.").removeSuffix(".")}: $processErrors" +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/Converter.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/Converter.kt new file mode 100644 index 0000000..0fc6f9f --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/Converter.kt @@ -0,0 +1,125 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.ffmpeg.FFmpegProgress +import com.github.kokorin.jaffree.ffmpeg.ProgressListener +import com.github.kokorin.jaffree.ffmpeg.UrlInput +import com.github.kokorin.jaffree.ffmpeg.UrlOutput +import com.github.kokorin.jaffree.ffprobe.FFprobeResult +import fr.rakambda.filesecure.progress.TaskProgress +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.future.await +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext +import java.nio.file.Path +import java.time.Duration + +private val log = KotlinLogging.logger {} + +/** + * Converts one file into another. + */ +interface Converter{ + /** + * Human-readable command, used for dry runs. + */ + val description: String + + /** + * Runs the conversion, the external process is killed if the coroutine is cancelled. + * + * @throws ConversionException If the conversion failed. + */ + suspend fun convert(progress: TaskProgress) +} + +class ConversionException(message: String, cause: Throwable? = null) : Exception(message, cause) + +class FfmpegConverter( + private val tools: ConversionTools, + private val probeResult: FFprobeResult?, + private val input: Path, + private val output: Path, + private val configureOutput: (UrlOutput) -> UrlOutput, +) : Converter{ + override val description: String + get() = build(null).commandLine + + override suspend fun convert(progress: TaskProgress){ + val duration = probeResult?.format?.duration?.let { Duration.ofSeconds(it.toLong()) } ?: Duration.ZERO + log.debug { "Converting $input ($duration) to $output" } + progress.setMax(duration.toMillis()) + + val future = build { ffmpegProgress -> ffmpegProgress.timeMillis?.takeIf { it >= 0 }?.let(progress::stepTo) } + .executeAsync() + try{ + future.toCompletableFuture().await() + } + catch(e: CancellationException){ + future.forceStop() + throw e + } + catch(e: Exception){ + throw ConversionException("ffmpeg failed to convert $input: ${describeFailure(e)}", e) + } + } + + private fun build(progressListener: ProgressListener?): CustomFFmpeg{ + val output = configureOutput(UrlOutput.toPath(output)) + tools.ffmpegThreads?.takeIf { it >= 0 }?.let { output.addArguments("-threads", it.toString()) } + + val ffmpeg = tools.ffmpeg() + ffmpeg.addInput(UrlInput.fromPath(input)) + .addOutput(output) + .setOverwriteOutput(false) + progressListener?.let(ffmpeg::setProgressListener) + return ffmpeg + } +} + +/** + * Runs an external command expected to write the output file. + */ +class CommandConverter( + private val tools: ConversionTools, + private val command: List, +) : Converter{ + override val description: String + get() = command.joinToString(" ") { "\"$it\"" } + + override suspend fun convert(progress: TaskProgress){ + progress.setMax(1) + val process = runInterruptible(Dispatchers.IO) { + ProcessBuilder(command) + .redirectErrorStream(true) + .start() + } + try{ + tools.processListener?.invoke(process) + coroutineScope { + val output = async(Dispatchers.IO) { process.inputStream.bufferedReader().use { it.readText() } } + val exitCode = try{ + process.onExit().await().exitValue() + } + catch(e: CancellationException){ + // Killing the process ends its output stream, letting the reader above complete + process.destroyForcibly() + throw e + } + if(exitCode != 0){ + throw ConversionException("Command `$description` exited with code $exitCode: ${output.await().trim()}") + } + } + progress.stepTo(1) + } + finally{ + if(process.isAlive){ + withContext(NonCancellable + Dispatchers.IO) { process.destroyForcibly().waitFor() } + } + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/CustomFFmpeg.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/CustomFFmpeg.kt new file mode 100644 index 0000000..9bcc670 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/CustomFFmpeg.kt @@ -0,0 +1,28 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.ffmpeg.FFmpeg +import com.github.kokorin.jaffree.process.Stopper +import java.nio.file.Path + +/** + * FFmpeg notifying when its process starts and exposing its command line. + */ +class CustomFFmpeg( + private val executable: Path, + private val processListener: ((Process) -> Unit)?, +) : FFmpeg(executable){ + override fun createStopper(): Stopper{ + val delegate = super.createStopper() + return object : Stopper by delegate{ + override fun setProcess(process: Process){ + delegate.setProcess(process) + processListener?.invoke(process) + } + } + } + + public override fun buildArguments(): List = super.buildArguments() + + val commandLine: String + get() = (listOf(executable.toString()) + buildArguments()).joinToString(" ") { "\"$it\"" } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/MediaProcessor.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/MediaProcessor.kt new file mode 100644 index 0000000..bed6442 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/MediaProcessor.kt @@ -0,0 +1,187 @@ +package fr.rakambda.filesecure.conversion + +import com.github.kokorin.jaffree.StreamType +import com.github.kokorin.jaffree.ffmpeg.UrlOutput +import com.github.kokorin.jaffree.ffprobe.FFprobeResult +import com.github.kokorin.jaffree.ffprobe.Stream +import java.nio.file.Path +import java.util.Locale + +/** + * Decides whether a file should be converted and how. + */ +sealed interface MediaProcessor{ + /** + * Extension of the converted file, without the dot. + */ + val extension: String + + fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean + + fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter +} + +/** + * A conversion decided by a Convert stage, executed once all stages accepted the file. + */ +data class PlannedConversion( + val processor: MediaProcessor, + val probeResult: FFprobeResult?, +) + +enum class ProcessorType(val processor: MediaProcessor){ + VIDEO_TO_HEVC_MP4(VideoToHevcProcessor("mp4", "mov,mp4,m4a,3gp,3g2,mj2")), + VIDEO_TO_HEVC_MKV(VideoToHevcProcessor("mkv", "matroska,webm")), + VIDEO_TO_AV1_MKV(VideoToAv1Processor), + MP3_TO_AAC(SingleAudioStreamProcessor("m4a", setOf("mp3")) { it.setCodec(StreamType.AUDIO, "aac").addArguments("-b:a", "192k") }), + AUDIO_TO_OPUS(SingleAudioStreamProcessor("opus", setOf("mp3", "aac")) { it.setCodec(StreamType.AUDIO, "libopus").addArguments("-b:a", "128000") }), + TIFF_TO_JPG(TiffToJpgProcessor), + PHOTO_TO_AVIF(PhotoProcessor("avif")), + PHOTO_TO_JXL(PhotoProcessor("jxl")), +} + +class VideoToHevcProcessor( + override val extension: String, + /** + * ffprobe's format name of the target container. + */ + private val formatName: String, +) : MediaProcessor{ + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean{ + if(probeResult == null || isPicture(probeResult)){ + return false + } + return isOtherVideoCodec(probeResult) || isHevcWithOtherTag(probeResult) || isHevcWithOtherContainer(probeResult) + } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter{ + // Already HEVC: only the container or codec tag needs to change, no re-encoding + val remuxOnly = probeResult != null && isHevcWithOtherContainer(probeResult) + return FfmpegConverter(tools, probeResult, input, output) { + if(remuxOnly){ + it.copyAllCodecs() + .addArguments("-tag:v", "hvc1") + } + else{ + it.setCodec(StreamType.SUBTITLE, "copy") + .setCodec(StreamType.AUDIO, "aac") + .addArguments("-b:a", "128000") + .setCodec(StreamType.VIDEO, "libx265") + .addArguments("-preset", "medium") + .addArguments("-crf", "21") + .addArguments("-vf", "format=yuv420p10le") + .addArguments("-movflags", "+faststart+use_metadata_tags") + .addArguments("-map_metadata", "0") + .addArguments("-map", "0") + .addArguments("-map", "-0:d") + .addArguments("-max_muxing_queue_size", "512") + .addArguments("-tag:v", "hvc1") + } + } + } + + private fun isPicture(probeResult: FFprobeResult) = probeResult.streams.any { it.profile == "Main Still Picture" } + + private fun isOtherVideoCodec(probeResult: FFprobeResult) = probeResult.streams.any { it.codecName in CODECS } + + private fun isHevcWithOtherTag(probeResult: FFprobeResult) = probeResult.streams.any { isHevc(it) && !isTargetCodecTag(it) } + + private fun isHevcWithOtherContainer(probeResult: FFprobeResult): Boolean{ + val isTargetContainer = probeResult.format?.formatName == formatName + val isTargetTag = probeResult.streams.any(::isTargetCodecTag) + val isHevc = probeResult.streams.any(::isHevc) + return isHevc && (!isTargetContainer || !isTargetTag) + } + + private fun isHevc(stream: Stream) = stream.codecName == "hevc" + + private fun isTargetCodecTag(stream: Stream) = stream.codecTagString == "hvc1" || stream.codecTagString == "[0][0][0][0]" + + private companion object{ + val CODECS = setOf("h264", "vp9", "wmv3", "mpeg2video") + } +} + +object VideoToAv1Processor : MediaProcessor{ + private val CODECS = setOf("h264", "vp9", "wmv3", "mpeg2video", "hevc", "h265") + + override val extension = "mkv" + + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean = probeResult?.streams.orEmpty().any { it.codecName in CODECS } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + FfmpegConverter(tools, probeResult, input, output) { + it.setCodec(StreamType.SUBTITLE, "copy") + .setCodec(StreamType.AUDIO, "libopus") + .addArguments("-b:a", "128000") + .setCodec(StreamType.VIDEO, "libsvtav1") + .addArguments("-preset", "4") + .addArguments("-crf", "32") + .addArguments("-g", "240") + .addArguments("-svtav1-params", "tune=0:enable-overlays=1:scd=1:scm=2") + .addArguments("-pix_fmt", "yuv420p10le") + .addArguments("-movflags", "+faststart+use_metadata_tags") + .addArguments("-map_metadata", "0") + .addArguments("-map", "0") + .addArguments("-map", "-0:d") + .addArguments("-max_muxing_queue_size", "512") + } +} + +/** + * Converts files made of a single audio stream. + */ +class SingleAudioStreamProcessor( + override val extension: String, + private val codecs: Set, + private val configureOutput: (UrlOutput) -> UrlOutput, +) : MediaProcessor{ + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean{ + val streams = probeResult?.streams ?: return false + return streams.size == 1 && streams.all { it.codecName in codecs } + } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + FfmpegConverter(tools, probeResult, input, output, configureOutput) +} + +object TiffToJpgProcessor : MediaProcessor{ + override val extension = "jpg" + + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean = probeResult?.streams.orEmpty().any { it.codecName == "tiff" } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + CommandConverter(tools, listOf( + tools.magick, + "-quality", "85%", + "-sampling-factor", "4:2:0", + "-interlace", "JPEG", + "-colorspace", "sRGB", + input.toAbsolutePath().toString(), + output.toAbsolutePath().toString(), + )) +} + +/** + * Converts photos with ImageMagick, the output format being deduced from the extension. + */ +class PhotoProcessor(override val extension: String) : MediaProcessor{ + override fun canHandle(probeResult: FFprobeResult?, file: Path): Boolean{ + val name = file.fileName.toString().lowercase(Locale.ROOT) + if(name.endsWith(".$extension")){ + return false + } + return PHOTO_EXTENSIONS.any { name.endsWith(".$it") } + } + + override fun createConverter(tools: ConversionTools, probeResult: FFprobeResult?, input: Path, output: Path): Converter = + CommandConverter(tools, listOf( + tools.magick, + input.toAbsolutePath().toString(), + output.toAbsolutePath().toString(), + )) + + private companion object{ + val PHOTO_EXTENSIONS = setOf("heic", "tiff", "jpg", "raf") + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/ProbeFilter.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/ProbeFilter.kt new file mode 100644 index 0000000..217267e --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/ProbeFilter.kt @@ -0,0 +1,31 @@ +package fr.rakambda.filesecure.conversion + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.github.kokorin.jaffree.ffprobe.FFprobeResult + +/** + * Filters files on their probe result, files that couldn't be probed never pass. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes( + JsonSubTypes.Type(value = LengthMoreThan::class, name = "LengthMoreThan"), + JsonSubTypes.Type(value = LengthLessThan::class, name = "LengthLessThan"), +) +sealed interface ProbeFilter{ + fun test(probeResult: FFprobeResult?): Boolean +} + +/** + * Keeps media lasting at least [value] seconds. + */ +data class LengthMoreThan(val value: Long) : ProbeFilter{ + override fun test(probeResult: FFprobeResult?): Boolean = probeResult?.format?.duration?.let { it >= value } ?: false +} + +/** + * Keeps media lasting at most [value] seconds. + */ +data class LengthLessThan(val value: Long) : ProbeFilter{ + override fun test(probeResult: FFprobeResult?): Boolean = probeResult?.format?.duration?.let { it <= value } ?: false +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/conversion/WindowsAffinity.kt b/src/main/kotlin/fr/rakambda/filesecure/conversion/WindowsAffinity.kt new file mode 100644 index 0000000..00108e6 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/conversion/WindowsAffinity.kt @@ -0,0 +1,44 @@ +package fr.rakambda.filesecure.conversion + +import com.sun.jna.Platform +import com.sun.jna.platform.win32.BaseTSD +import com.sun.jna.platform.win32.Kernel32 +import com.sun.jna.platform.win32.WinNT +import io.github.oshai.kotlinlogging.KotlinLogging + +private val log = KotlinLogging.logger {} + +/** + * Restricts processes to some CPU cores, only supported on Windows. + * + * @param mask Bit mask of the allowed cores, e.g. 7 allows cores 1, 2 and 3. + */ +class WindowsAffinity(private val mask: Long){ + init{ + require(mask > 0) { "Affinity mask must be positive" } + if(!Platform.isWindows()){ + log.warn { "CPU affinity is only supported on Windows, it will be ignored" } + } + } + + fun apply(process: Process){ + if(!Platform.isWindows()){ + return + } + log.debug { "Setting affinity $mask to pid ${process.pid()}" } + val kernel = Kernel32.INSTANCE + val handle = kernel.OpenProcess(WinNT.PROCESS_SET_INFORMATION or WinNT.PROCESS_QUERY_INFORMATION, false, process.pid().toInt()) + if(handle == null){ + log.warn { "Failed to open process ${process.pid()} to set its affinity, error ${kernel.GetLastError()}" } + return + } + try{ + if(!kernel.SetProcessAffinityMask(handle, BaseTSD.ULONG_PTR(mask))){ + log.warn { "Failed to set affinity of process ${process.pid()}, error ${kernel.GetLastError()}" } + } + } + finally{ + kernel.CloseHandle(handle) + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/folder/FolderAction.kt b/src/main/kotlin/fr/rakambda/filesecure/folder/FolderAction.kt new file mode 100644 index 0000000..1d56dd4 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/folder/FolderAction.kt @@ -0,0 +1,135 @@ +package fr.rakambda.filesecure.folder + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import fr.rakambda.filesecure.pipeline.FileOperations +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.DirectoryNotEmptyException +import java.nio.file.Files +import java.nio.file.Path +import java.time.LocalDateTime +import java.time.ZoneId + +private val log = KotlinLogging.logger {} + +/** + * An action applied to every scanned input folder once all files have been processed, deepest folders first. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes( + JsonSubTypes.Type(value = DeleteIfEmpty::class, name = "DeleteIfEmpty"), + JsonSubTypes.Type(value = DeleteIfOlderThan::class, name = "DeleteIfOlderThan"), + JsonSubTypes.Type(value = DeleteChildren::class, name = "DeleteChildren"), +) +sealed interface FolderAction{ + /** + * @param folder The folder to apply the action to. + * @param baseFolder The input folder of the mapping. + * @throws FolderActionException If the action failed. + */ + fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path) +} + +/** + * Depth of the folder below the input folder, the input folder itself being at depth 0. + */ +internal fun depthOf(folder: Path, baseFolder: Path): Int{ + if(folder == baseFolder){ + return 0 + } + return baseFolder.relativize(folder).nameCount +} + +class FolderActionException(message: String, cause: Throwable) : Exception(message, cause) + +/** + * Deletes the folder, unless a file appeared in it since it was found empty. + */ +private fun deleteEmptyFolder(fileOperations: FileOperations, folder: Path){ + try{ + fileOperations.delete(folder) + } + catch(_: DirectoryNotEmptyException){ + log.info { "Not deleting folder $folder, it is not empty anymore" } + } +} + +private inline fun deletionCheck(folder: Path, block: () -> Unit){ + try{ + block() + } + catch(e: Exception){ + throw FolderActionException("Failed to determine if $folder should be deleted, it will not be by default", e) + } +} + +internal fun isEmptyFolder(folder: Path): Boolean = Files.list(folder).use { it.findAny().isEmpty } + +/** + * Deletes empty folders at least [depth] levels below the input folder. + */ +data class DeleteIfEmpty( + val depth: Int = 1, +) : FolderAction{ + override fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path){ + deletionCheck(folder) { + if(depthOf(folder, baseFolder) < depth || !isEmptyFolder(folder)){ + return + } + log.info { "Deleting folder $folder because it is empty" } + deleteEmptyFolder(fileOperations, folder) + } + } +} + +/** + * Deletes empty folders at least [depth] levels below the input folder, whose last modification is older than the given offset. + */ +data class DeleteIfOlderThan( + val dayOffset: Long = 0, + val minuteOffset: Long = 0, + val depth: Int = 1, +) : FolderAction{ + override fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path){ + deletionCheck(folder) { + if(depthOf(folder, baseFolder) < depth){ + return + } + val modified = LocalDateTime.ofInstant(Files.getLastModifiedTime(folder).toInstant(), ZoneId.systemDefault()) + if(!modified.isBefore(LocalDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset))){ + return + } + if(isEmptyFolder(folder)){ + log.info { "Deleting folder $folder because it is more than $dayOffset days old and empty" } + deleteEmptyFolder(fileOperations, folder) + } + else{ + log.debug { "Tried deleting folder $folder because it is more than $dayOffset days old but wasn't empty" } + } + } + } +} + +/** + * Deletes the given children of every folder, if they exist. All children are tried even if one fails. + */ +data class DeleteChildren( + val names: List = emptyList(), +) : FolderAction{ + override fun apply(fileOperations: FileOperations, folder: Path, baseFolder: Path){ + var failure: FolderActionException? = null + for(name in names){ + val child = folder.resolve(name) + try{ + if(fileOperations.deleteIfExists(child)){ + log.info { "Deleted child file $child" } + } + } + catch(e: Exception){ + val error = FolderActionException("Failed to delete child $child", e) + failure?.addSuppressed(error) ?: run { failure = error } + } + } + failure?.let { throw it } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/FileMetadata.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/FileMetadata.kt new file mode 100644 index 0000000..e5832b3 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/FileMetadata.kt @@ -0,0 +1,15 @@ +package fr.rakambda.filesecure.metadata + +import java.time.ZonedDateTime + +data class FileMetadata( + val takenDate: ZonedDateTime? = null, + val createdDate: ZonedDateTime? = null, + val nameDate: ZonedDateTime? = null, +){ + /** + * The best known date of the file: from its name, then its media metadata, then its last modification. + */ + val date: ZonedDateTime + get() = nameDate ?: takenDate ?: createdDate ?: ZonedDateTime.now() +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/MediaDateExtractor.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/MediaDateExtractor.kt new file mode 100644 index 0000000..5520cde --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/MediaDateExtractor.kt @@ -0,0 +1,44 @@ +package fr.rakambda.filesecure.metadata + +import com.drew.metadata.Directory +import com.drew.metadata.xmp.XmpDirectory +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.TimeZone + +/** + * Reads a date from one kind of metadata directory. + */ +sealed interface MediaDateExtractor{ + val directoryClass: Class + + fun parse(directory: Directory, timeZone: TimeZone): ZonedDateTime? +} + +class TagMediaDateExtractor( + override val directoryClass: Class, + private val tag: Int, +) : MediaDateExtractor{ + override fun parse(directory: Directory, timeZone: TimeZone): ZonedDateTime? = + directory.getDate(tag, timeZone)?.toInstant()?.atZone(timeZone.toZoneId()) +} + +object XmpMediaDateExtractor : MediaDateExtractor{ + private val KEYS = listOf("xmp:CreateDate", "photoshop:DateCreated") + private val FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss") + + override val directoryClass = XmpDirectory::class.java + + override fun parse(directory: Directory, timeZone: TimeZone): ZonedDateTime?{ + val values = (directory as XmpDirectory).xmpProperties + for(key in KEYS){ + val value = values[key] ?: continue + try{ + return ZonedDateTime.parse(value, FORMATTER.withZone(timeZone.toZoneId())) + } + catch(_: Exception){ + } + } + return null + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/MetadataExtractor.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/MetadataExtractor.kt new file mode 100644 index 0000000..65edf22 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/MetadataExtractor.kt @@ -0,0 +1,165 @@ +package fr.rakambda.filesecure.metadata + +import com.drew.imaging.ImageMetadataReader +import com.drew.imaging.ImageProcessingException +import com.drew.metadata.Metadata +import com.drew.metadata.exif.ExifSubIFDDirectory +import com.drew.metadata.exif.GpsDirectory +import com.drew.metadata.mov.QuickTimeDirectory +import com.drew.metadata.mov.metadata.QuickTimeMetadataDirectory +import com.drew.metadata.mp4.Mp4Directory +import com.drew.metadata.xmp.XmpDirectory +import io.github.oshai.kotlinlogging.KotlinLogging +import us.fatehi.pointlocation6709.parse.PointLocationParser +import java.nio.file.Files +import java.nio.file.Path +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeParseException +import java.util.TimeZone +import java.util.regex.Pattern + +private val log = KotlinLogging.logger {} + +class MetadataExtractor( + private val timeZoneResolver: TimeZoneResolver, + private val nameDateExtractors: List = NameDateExtractor.DEFAULTS, +){ + private val mediaDateExtractors: List = listOf( + TagMediaDateExtractor(QuickTimeMetadataDirectory::class.java, QuickTimeMetadataDirectory.TAG_CREATION_DATE), + TagMediaDateExtractor(QuickTimeDirectory::class.java, QuickTimeDirectory.TAG_CREATION_TIME), + TagMediaDateExtractor(Mp4Directory::class.java, Mp4Directory.TAG_CREATION_TIME), + TagMediaDateExtractor(ExifSubIFDDirectory::class.java, ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL), + XmpMediaDateExtractor, + ) + + fun getMetadata(path: Path): FileMetadata = FileMetadata( + takenDate = getDateFromMetadata(path), + createdDate = getModificationDate(path), + nameDate = getDateFromFileName(path), + ) + + private fun getModificationDate(path: Path): ZonedDateTime{ + val date = Files.getLastModifiedTime(path).toInstant().atZone(ZoneId.systemDefault()) + if(date.year <= 1970){ + return date.withYear(LocalDateTime.now().year) + } + return date + } + + private fun getDateFromMetadata(path: Path): ZonedDateTime?{ + try{ + val metadata = ImageMetadataReader.readMetadata(path.toFile()) ?: return null + val timeZone = TimeZone.getTimeZone(getZoneIdFromMetadata(path, metadata) ?: ZoneId.systemDefault()) + + for(extractor in mediaDateExtractors){ + for(directory in metadata.getDirectoriesOfType(extractor.directoryClass)){ + try{ + log.debug { "Trying ${extractor.directoryClass.name}" } + val date = extractor.parse(directory, timeZone) ?: continue + if(date.toInstant() == MP4_EPOCH){ + log.debug { "No date in metadata directory ${extractor.directoryClass.simpleName} of $path, its creation time is unset" } + continue + } + if(date.year < 1970){ + log.warn { "Ignoring date $date of $path from metadata directory ${extractor.directoryClass.simpleName}, its year is before 1970" } + continue + } + log.debug { "Matched metadata directory $directory" } + return date + } + catch(e: Exception){ + log.warn { "Failed to read a date from metadata directory ${extractor.directoryClass.simpleName} of $path: ${e.describe()}" } + } + } + } + } + catch(e: Exception){ + // ImageProcessingException itself (not a format specific subclass) is only thrown when the file isn't a supported image or video + if(e.javaClass == ImageProcessingException::class.java){ + log.debug { "No date metadata for $path, not a supported image or video: ${e.message}" } + } + else{ + log.warn { "Failed to read the metadata of $path, its date is taken from elsewhere: ${e.describe()}" } + } + } + return null + } + + private fun getDateFromFileName(path: Path): ZonedDateTime?{ + val fileName = path.fileName.toString() + for(extractor in nameDateExtractors){ + try{ + log.debug { "Trying name extractor `$extractor`" } + val date = extractor.parse(fileName) ?: continue + if(date.year < 1970){ + log.warn { "Ignoring date $date found in the name of $path, its year is before 1970" } + continue + } + log.debug { "Matched date format for $path" } + return date + } + catch(_: DateTimeParseException){ + } + catch(e: Exception){ + log.warn { "Failed to read a date from the name of $path with $extractor: ${e.describe()}" } + } + } + return null + } + + private fun getZoneIdFromMetadata(path: Path, metadata: Metadata): ZoneId?{ + try{ + for(directory in metadata.getDirectoriesOfType(GpsDirectory::class.java)){ + val location = directory.geoLocation ?: continue + timeZoneResolver.resolve(location.latitude, location.longitude)?.let { return it } + } + for(directory in metadata.getDirectoriesOfType(QuickTimeMetadataDirectory::class.java)){ + val representation = directory.getString(QUICKTIME_LOCATION_TAG) + if(representation.isNullOrBlank()){ + continue + } + val location = PointLocationParser.parsePointLocation(representation) + timeZoneResolver.resolve(location.latitude.degrees, location.longitude.degrees)?.let { return it } + } + for(directory in metadata.getDirectoriesOfType(XmpDirectory::class.java)){ + val values = directory.xmpProperties + val latitude = values["exif:GPSLatitude"]?.let(::parseAngle) ?: continue + val longitude = values["exif:GPSLongitude"]?.let(::parseAngle) ?: continue + timeZoneResolver.resolve(latitude, longitude)?.let { return it } + } + } + catch(e: Exception){ + log.warn { "Failed to read the location of $path, the system time zone is used: ${e.describe()}" } + } + return null + } + + private fun Exception.describe() = "${javaClass.simpleName}: $message" + + companion object{ + private const val QUICKTIME_LOCATION_TAG = 0x050D + // Value of an unset MP4 / QuickTime creation time + private val MP4_EPOCH = Instant.parse("1904-01-01T00:00:00Z") + private val ANGLE_PATTERN = Pattern.compile("""(\d{1,3}),(\d{1,2})\.(\d+)([NESW])""") + + /** + * Converts an XMP coordinate such as `48,51.2345N` to signed degrees. + */ + internal fun parseAngle(value: String): Double?{ + val matcher = ANGLE_PATTERN.matcher(value) + if(!matcher.matches()){ + return null + } + val angle = matcher.group(1).toInt() + + matcher.group(2).toInt() / 60.0 + + "0.${matcher.group(3)}".toDouble() / 60.0 + return when(matcher.group(4)){ + "W", "S" -> -angle + else -> angle + } + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/NameDateExtractor.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/NameDateExtractor.kt new file mode 100644 index 0000000..d58e296 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/NameDateExtractor.kt @@ -0,0 +1,44 @@ +package fr.rakambda.filesecure.metadata + +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.Locale +import java.util.regex.Pattern + +/** + * Finds a date in a file name with a regex, the matched group being parsed with a date format. + */ +class NameDateExtractor( + private val pattern: Pattern, + format: String, + private val group: Int = 0, +){ + private val formatter = DateTimeFormatter.ofPattern(format) + .withLocale(Locale.ENGLISH) + .withZone(ZoneId.systemDefault()) + + /** + * @throws java.time.format.DateTimeParseException If the matched text is not a valid date. + */ + fun parse(name: String): ZonedDateTime?{ + val matcher = pattern.matcher(name) + if(!matcher.find()){ + return null + } + return ZonedDateTime.parse(matcher.group(group), formatter) + } + + override fun toString(): String = "NameDateExtractor(${pattern.pattern()})" + + companion object{ + val DEFAULTS = listOf( + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2}"""), "yyyy-MM-dd HH.mm.ss"), + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2} \d{2}-\d{2}-\d{2}"""), "yyyy-MM-dd HH-mm-ss"), + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}"""), "yyyy-MM-dd_HH-mm-ss"), + NameDateExtractor(Pattern.compile("""\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}"""), "yyyy-MM-dd-HH-mm-ss"), + NameDateExtractor(Pattern.compile("""IMG_(\d{8}_\d{6})"""), "yyyyMMdd_HHmmss", 1), + NameDateExtractor(Pattern.compile("""Video-(\d{8}-\d{6})"""), "yyyyMMdd-HHmmss", 1), + ) + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/metadata/TimeZoneResolver.kt b/src/main/kotlin/fr/rakambda/filesecure/metadata/TimeZoneResolver.kt new file mode 100644 index 0000000..e509a98 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/metadata/TimeZoneResolver.kt @@ -0,0 +1,58 @@ +package fr.rakambda.filesecure.metadata + +import io.github.oshai.kotlinlogging.KotlinLogging +import net.iakovlev.timeshape.TimeZoneEngine +import java.time.ZoneId +import java.util.Optional +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.jvm.optionals.getOrNull + +private val log = KotlinLogging.logger {} + +fun interface TimeZoneResolver{ + fun resolve(latitude: Double, longitude: Double): ZoneId? +} + +/** + * Resolves the time zone of coordinates offline, with the time zone boundaries bundled by timeshape. + * + * The data is loaded on the first call only, as it takes time and memory. If it can't be loaded, every call returns null and loading isn't retried. + */ +class OfflineTimeZoneResolver(engineFactory: () -> TimeZoneEngine = TimeZoneEngine::initialize) : TimeZoneResolver{ + private val engine: Result by lazy { + try{ + Result.success(engineFactory()) + } + catch(e: Throwable){ + if(e !is Exception && e !is LinkageError && e !is OutOfMemoryError){ + throw e + } + Result.failure(e) + } + } + private val cache = ConcurrentHashMap, Optional>() + private val initFailureLogged = AtomicBoolean() + private val notFoundLogged = AtomicBoolean() + + override fun resolve(latitude: Double, longitude: Double): ZoneId?{ + val engine = engine.getOrElse { e -> + if(initFailureLogged.compareAndSet(false, true)){ + log.warn { "Failed to load the time zone data for coordinates $latitude;$longitude, the system time zone is used for every geotagged file: ${e.javaClass.simpleName}: ${e.message}" } + } + return null + } + return cache.computeIfAbsent(latitude to longitude) { + engine.query(latitude, longitude).also { zone -> + if(zone.isEmpty){ + if(notFoundLogged.compareAndSet(false, true)){ + log.warn { "No time zone found for coordinates $latitude;$longitude, the system time zone is used (next ones are logged at debug level)" } + } + else{ + log.debug { "No time zone found for coordinates $latitude;$longitude, the system time zone is used" } + } + } + } + }.getOrNull() + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/migration/ConfigMigrator.kt b/src/main/kotlin/fr/rakambda/filesecure/migration/ConfigMigrator.kt new file mode 100644 index 0000000..13ee794 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/migration/ConfigMigrator.kt @@ -0,0 +1,237 @@ +package fr.rakambda.filesecure.migration + +import com.fasterxml.jackson.core.json.JsonReadFeature +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.json.JsonMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper +import fr.rakambda.filesecure.config.Configuration +import java.util.Locale +import java.util.regex.Pattern + +/** + * Converts FileSecure 12 and MediaConverter configurations into the FileSecure 13 format. + * + * Warnings collect everything that couldn't be converted exactly, they should be reviewed before using the result. + */ +class ConfigMigrator{ + private val nodes = JsonNodeFactory.instance + private val _warnings = mutableListOf() + val warnings: List + get() = _warnings + + /** + * @param sources Old configurations, each being a FileSecure 12 (`rules`) or MediaConverter (`conversions`) configuration. + * @return The new configuration in YAML, already validated by the FileSecure 13 parser. + */ + fun migrate(sources: List): String{ + val pipelines = nodes.arrayNode() + for(source in sources){ + val root = source.content + val known = setOf("rules", "conversions") + if(root.fieldNames().asSequence().none { it in known }){ + throw IllegalArgumentException("${source.name} is neither a FileSecure (rules) nor a MediaConverter (conversions) configuration") + } + warnUnknown(root, known, source.name) + root.path("rules").forEachIndexed { index, rule -> pipelines.add(migrateRule(rule, "${source.name} rule #${index + 1}")) } + root.path("conversions").forEachIndexed { index, conversion -> pipelines.add(migrateConversion(conversion, "${source.name} conversion #${index + 1}")) } + } + val yaml = YAML_WRITER.writeValueAsString(nodes.objectNode().set("pipelines", pipelines)) + Configuration.parse(yaml) + return yaml + } + + private fun migrateRule(rule: JsonNode, name: String): ObjectNode{ + warnUnknown(rule, RULE_KEYS, name) + val pipeline = nodes.objectNode() + pipeline.put("name", name) + pipeline.set("mappings", migrateMappings(rule.path("mappings"), name)) + + val scan = nodes.objectNode() + rule.get("maxDepth")?.let { scan.set("maxDepth", it) } + rule.get("folderExcludes")?.takeIf { it.size() > 0 }?.let { scan.set("excludedFolders", it) } + if(!scan.isEmpty){ + pipeline.set("scan", scan) + } + + val stages = nodes.arrayNode() + val includes = rule.path("filters") + val excludes = rule.path("excludes") + if(includes.size() > 0 || excludes.size() > 0){ + val filter = stages.addObject().put("type", "NameFilter") + if(includes.size() > 0) filter.set("includes", includes) + if(excludes.size() > 0) filter.set("excludes", excludes) + } + stages.addAll(migrateTransformers(withoutUselessNoRename(rule.path("fileTransformers"), name), FILE_TRANSFORMERS, name, "file transformer")) + if(stages.size() > 0){ + pipeline.set("stages", stages) + } + + val operation = rule.path("operation").asText("NONE").uppercase(Locale.ROOT) + if(operation !in OPERATIONS){ + throw IllegalArgumentException("$name: unknown operation '$operation'") + } + pipeline.put("operation", operation) + if(operation == "NONE"){ + _warnings += "$name: operation is NONE (the default), this pipeline only logs what it would do" + } + if(rule.path("skipIfAlreadyExists").asBoolean(false)){ + pipeline.put("conflict", "SKIP") + } + if(rule.path("setFileCreationDate").asBoolean(false)){ + pipeline.put("setFileCreationDate", true) + } + + val folderActions = migrateTransformers(rule.path("folderTransformers"), FOLDER_TRANSFORMERS, name, "folder transformer") + if(folderActions.size() > 0){ + pipeline.set("folderActions", folderActions) + } + if(folderActions.any { it.path("type").asText() == "DeleteIfEmpty" && it.path("depth").asInt(1) <= 1 }){ + _warnings += "$name: DeleteIfEmpty used to delete the input folder itself once empty, it now keeps it" + } + return pipeline + } + + /** + * NoRename comes last (highest priority), it only does something when a stage before it renamed the file. + */ + private fun withoutUselessNoRename(transformers: JsonNode, name: String): JsonNode{ + val types = transformers.map { it.path("name").asText() } + if("NoRenameTransformer" !in types || types.any { it in RENAME_TRANSFORMERS }){ + return transformers + } + _warnings += "$name: NoRename dropped, nothing renames files so it did nothing" + return nodes.arrayNode().addAll(transformers.filter { it.path("name").asText() != "NoRenameTransformer" }) + } + + /** + * Converts transformers into stages or folder actions, sorted by their old priority. + * The old version stored them in an unordered set, the priority was meant to be their order but was never applied. + */ + private fun migrateTransformers(transformers: JsonNode, definitions: Map, name: String, kind: String): ArrayNode{ + val migrated = transformers.mapNotNull { transformer -> + val oldType = transformer.path("name").asText() + val definition = definitions[oldType] + if(definition == null){ + _warnings += "$name: unknown $kind '$oldType' dropped" + return@mapNotNull null + } + warnUnknown(transformer, definition.parameters + "name", "$name $oldType") + val node = nodes.objectNode().put("type", definition.newType) + definition.parameters.forEach { parameter -> transformer.get(parameter)?.let { node.set(parameter, it) } } + definition.priority to node + } + if(migrated.size > 1){ + _warnings += "$name: ${kind}s were applied in an unspecified order, they are now ordered by their intended priority: " + + migrated.sortedBy { it.first }.joinToString(", ") { it.second.path("type").asText() } + ". Check this order is the one you expect" + } + return nodes.arrayNode().addAll(migrated.sortedBy { it.first }.map { it.second }) + } + + private fun migrateConversion(conversion: JsonNode, name: String): ObjectNode{ + warnUnknown(conversion, CONVERSION_KEYS, name) + val pipeline = nodes.objectNode() + pipeline.put("name", name) + conversion.get("enabled")?.let { pipeline.set("enabled", it) } + pipeline.set("mappings", nodes.arrayNode().add(migrateMapping(conversion, name))) + + val scan = pipeline.putObject("scan").put("skipHidden", true) + val excluded = conversion.path("excluded") + if(excluded.size() > 0){ + val patterns = scan.putArray("excludedFolders") + excluded.forEach { patterns.add(Pattern.quote(it.asText().replace("\\", "/"))) } + } + + val convert = pipeline.putArray("stages").addObject().put("type", "Convert") + conversion.path("processors").takeIf { it.size() > 0 }?.let { processors -> + val array = convert.putArray("processors") + processors.forEach { array.add(it.asText().uppercase(Locale.ROOT)) } + } + conversion.path("extensions").takeIf { it.size() > 0 }?.let { convert.set("extensions", it) } + conversion.path("filters").takeIf { it.size() > 0 }?.let { filters -> + val array = convert.putArray("filters") + filters.forEach { filter -> + warnUnknown(filter, setOf("name", "value"), "$name filter") + array.addObject() + .put("type", filter.path("name").asText()) + .set("value", filter.path("value")) + } + } + convert.put("unhandled", "SKIP") + conversion.get("database")?.let { convert.set("cache", it) } + + // The output was never overwritten: kept as is when copying, and the conversion failed when moving + val deleteInput = conversion.path("deleteInput").asBoolean(true) + pipeline.put("operation", if(deleteInput) "MOVE" else "COPY") + pipeline.put("conflict", "SKIP") + conversion.get("temp")?.let { pipeline.set("temp", it) } + return pipeline + } + + private fun migrateMappings(mappings: JsonNode, name: String): ArrayNode{ + if(mappings.size() == 0){ + throw IllegalArgumentException("$name has no mappings") + } + return nodes.arrayNode().addAll(mappings.map { migrateMapping(it, name) }) + } + + private fun migrateMapping(mapping: JsonNode, name: String): ObjectNode{ + val input = mapping.get("input")?.asText() ?: throw IllegalArgumentException("$name has no input") + val output = mapping.get("output")?.asText() ?: throw IllegalArgumentException("$name has no output") + return nodes.objectNode().put("input", input).put("output", output) + } + + private fun warnUnknown(node: JsonNode, known: Set, name: String){ + node.fieldNames().asSequence() + .filter { it !in known } + .forEach { _warnings += "$name: unknown key '$it' was ignored by the previous version, it is dropped" } + } + + data class Source(val name: String, val content: JsonNode) + + private data class TransformerDefinition(val newType: String, val priority: Int, val parameters: Set) + + companion object{ + private val YAML_WRITER = YAMLMapper.builder() + .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .build() + .writer() + + /** + * Reads the old JSON configurations. + */ + val MAPPER: JsonMapper = JsonMapper.builder() + .enable(JsonReadFeature.ALLOW_JAVA_COMMENTS) + .enable(JsonReadFeature.ALLOW_TRAILING_COMMA) + .build() + + private val OPERATIONS = setOf("COPY", "MOVE", "MOVE_WITH_COPY", "NONE") + private val RULE_KEYS = setOf("operation", "mappings", "filters", "excludes", "folderExcludes", "fileTransformers", "folderTransformers", "skipIfAlreadyExists", "setFileCreationDate", "maxDepth") + private val CONVERSION_KEYS = setOf("input", "output", "database", "temp", "deleteInput", "enabled", "excluded", "processors", "extensions", "filters") + + // Priorities of the old transformers: lower ran first. IfOlderThan shared 0 with renames, being a filter it is put first + private val FILE_TRANSFORMERS = mapOf( + "IfOlderThanTransformer" to TransformerDefinition("IfOlderThan", -1, setOf("dayOffset", "minuteOffset")), + "RenameWithDateTransformer" to TransformerDefinition("RenameWithDate", 0, setOf("format", "locale", "filters", "zone")), + "RenameWithDatePatternTransformer" to TransformerDefinition("RenameWithDatePattern", 0, setOf("outputFormat", "regex", "dateFormat", "timeFormat", "locale", "zone")), + "FolderUniquePerDayTransformer" to TransformerDefinition("FolderUniquePerDay", 10, emptySet()), + "FolderPerDayTransformer" to TransformerDefinition("FolderPerDay", 11, emptySet()), + "FolderPerWeekTransformer" to TransformerDefinition("FolderPerWeek", 12, emptySet()), + "FolderCanonicalTransformer" to TransformerDefinition("FolderCanonical", 12, setOf("elementCount")), + "FolderPerMonthTransformer" to TransformerDefinition("FolderPerMonth", 13, emptySet()), + "FolderPerYearTransformer" to TransformerDefinition("FolderPerYear", 14, emptySet()), + "LowercaseTransformer" to TransformerDefinition("Lowercase", Int.MAX_VALUE, emptySet()), + "NoRenameTransformer" to TransformerDefinition("NoRename", Int.MAX_VALUE, emptySet()), + ) + private val RENAME_TRANSFORMERS = setOf("RenameWithDateTransformer", "RenameWithDatePatternTransformer") + private val FOLDER_TRANSFORMERS = mapOf( + "DeleteChildrenTransformer" to TransformerDefinition("DeleteChildren", 10, setOf("names")), + "DeleteIfEmptyTransformer" to TransformerDefinition("DeleteIfEmpty", Int.MAX_VALUE, setOf("depth")), + "DeleteIfOlderThanTransformer" to TransformerDefinition("DeleteIfOlderThan", Int.MAX_VALUE, setOf("dayOffset", "minuteOffset", "depth")), + ) + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/migration/MigrateConfigCommand.kt b/src/main/kotlin/fr/rakambda/filesecure/migration/MigrateConfigCommand.kt new file mode 100644 index 0000000..8e4508c --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/migration/MigrateConfigCommand.kt @@ -0,0 +1,57 @@ +package fr.rakambda.filesecure.migration + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.Context +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.core.main +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.multiple +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import java.nio.file.Files + +/** + * Standalone tool converting old configurations, run with + * `java -cp FileSecure.jar fr.rakambda.filesecure.migration.MigrateConfigCommandKt old.json [other.json...] -o config.yaml`. + */ +class MigrateConfigCommand : CliktCommand(name = "migrate-config"){ + private val inputs by argument(help = "FileSecure 12 or MediaConverter configuration files, all merged into one new configuration") + .path(mustExist = true, canBeDir = false, mustBeReadable = true) + .multiple(required = true) + private val output by option("-o", "--output", help = "YAML file to write the new configuration to, printed if not set") + .path(canBeDir = false) + private val force by option("-f", "--force", help = "Overwrite the output file if it exists") + .flag() + + override fun help(context: Context): String = "Converts FileSecure 12 and MediaConverter configurations to the FileSecure 13 format." + + override fun run(){ + val output = output + if(output != null && Files.exists(output) && !force){ + echo("$output already exists, use --force to overwrite it", err = true) + throw ProgramResult(1) + } + + val migrator = ConfigMigrator() + val result = try{ + migrator.migrate(inputs.map { ConfigMigrator.Source(it.fileName.toString(), Files.newBufferedReader(it).use(ConfigMigrator.MAPPER::readTree)) }) + } + catch(e: Exception){ + echo("Failed to migrate: ${e.message}", err = true) + throw ProgramResult(1) + } + + if(output == null){ + echo(result, trailingNewline = false) + } + else{ + output.toAbsolutePath().parent?.let(Files::createDirectories) + Files.writeString(output, result) + echo("New configuration written to $output", err = true) + } + migrator.warnings.forEach { echo("WARNING: $it", err = true) } + } +} + +fun main(args: Array) = MigrateConfigCommand().main(args) diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserver.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserver.kt new file mode 100644 index 0000000..eb075e9 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/DestinationReserver.kt @@ -0,0 +1,49 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.ConflictStrategy +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.Path + +/** + * Picks destinations that are neither existing nor already given to another file being processed concurrently. + */ +class DestinationReserver{ + private val mutex = Mutex() + private val reserved = HashSet() + + /** + * @return The destination to use, or null if the file should be skipped. + */ + suspend fun reserve(destination: Path, strategy: ConflictStrategy): Path? = mutex.withLock { + withContext(Dispatchers.IO) { + val candidate = when{ + isFree(destination) -> destination + strategy == ConflictStrategy.SKIP -> null + else -> generateSequence(1) { it + 1 } + .map { destination.resolveSibling(numberedName(destination.fileName.toString(), it)) } + .first(::isFree) + } + candidate?.also { reserved.add(it) } + } + } + + private fun isFree(path: Path) = path !in reserved && !Files.exists(path, LinkOption.NOFOLLOW_LINKS) + + companion object{ + /** + * `name.ext` becomes `name (index).ext`, names without extension or starting with a dot get the suffix at their end. + */ + internal fun numberedName(name: String, index: Int): String{ + val dotIndex = name.lastIndexOf('.') + if(dotIndex <= 0){ + return "$name ($index)" + } + return "${name.substring(0, dotIndex)} ($index)${name.substring(dotIndex)}" + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileItem.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileItem.kt new file mode 100644 index 0000000..4d6f686 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileItem.kt @@ -0,0 +1,31 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.conversion.PlannedConversion +import fr.rakambda.filesecure.metadata.FileMetadata +import java.nio.file.Path + +/** + * A file going through the stages of a pipeline. + * + * Stages only plan what will happen to the file (destination, conversion), the plan is executed once all stages accepted the file. + */ +class FileItem( + val source: Path, + val inputBase: Path, + val outputBase: Path, + metadataProvider: () -> FileMetadata, +){ + /** + * Same relative path in the output as in the input. + */ + val originalDestination: Path = outputBase.resolve(inputBase.relativize(source)) + var destination: Path = originalDestination + var conversion: PlannedConversion? = null + + /** + * Computed on first access as reading media metadata can be costly. + */ + val metadata: FileMetadata by lazy(metadataProvider) + + override fun toString(): String = source.toString() +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileOperations.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileOperations.kt new file mode 100644 index 0000000..c2df624 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileOperations.kt @@ -0,0 +1,317 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.Operation +import fr.rakambda.filesecure.config.Verification +import io.github.oshai.kotlinlogging.KotlinLogging +import java.awt.Desktop +import java.awt.GraphicsEnvironment +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.channels.FileChannel +import java.nio.file.AtomicMoveNotSupportedException +import java.nio.file.FileAlreadyExistsException +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.nio.file.StandardOpenOption +import java.nio.file.attribute.BasicFileAttributeView +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime +import java.security.MessageDigest +import java.util.HexFormat + +private val log = KotlinLogging.logger {} + +/** + * Every write to the file system goes through here, doing nothing during dry runs. + */ +class FileOperations internal constructor( + val dryRun: Boolean, + private val atomicMove: (Path, Path) -> Unit, + private val copyToPartial: (Path, Path) -> Unit, + /** + * Called with the source and the partial file once the partial file is written, before it is checked. + */ + private val afterCopy: (Path, Path) -> Unit = { _, _ -> }, +){ + constructor(dryRun: Boolean) : this(dryRun, ::defaultAtomicMove, ::defaultCopyToPartial) + + fun transfer(operation: Operation, input: Path, output: Path, verification: Verification = Verification.SIZE){ + when(operation){ + Operation.COPY -> copy(input, output, verification) + Operation.MOVE -> move(input, output, verification) + Operation.MOVE_WITH_COPY -> moveWithCopy(input, output, verification) + Operation.NONE -> {} + } + } + + fun copy(input: Path, output: Path, verification: Verification = Verification.SIZE){ + if(dryRun){ + return + } + createDirectories(output.parent) + safeCopy(input, output, verification) + } + + fun move(input: Path, output: Path, verification: Verification = Verification.SIZE){ + if(dryRun){ + return + } + createDirectories(output.parent) + moveFile(input, output, verification) + } + + fun moveWithCopy(input: Path, output: Path, verification: Verification = Verification.SIZE){ + if(dryRun){ + return + } + createDirectories(output.parent) + safeCopy(input, output, verification) + Files.delete(input) + } + + /** + * Puts a converted file at its destination, removing the source if the operation requires it. + * + * @param replaceSource true when the destination is the source itself, the source then being replaced by the converted file. + */ + fun commitConverted(operation: Operation, source: Path, converted: Path, destination: Path, replaceSource: Boolean, verification: Verification = Verification.SIZE){ + if(dryRun || operation == Operation.NONE){ + return + } + createDirectories(destination.parent) + val sourceAttributes = Files.readAttributes(source, BasicFileAttributes::class.java) + + if(replaceSource){ + // Keep the original next to it until the converted file is in place, so it can be restored on failure + val backup = source.resolveSibling("${source.fileName}.filesecure-original") + Files.move(source, backup) + try{ + moveFile(converted, destination, verification) + } + catch(e: Exception){ + try{ + Files.move(backup, source) + } + catch(restoreError: Exception){ + e.addSuppressed(restoreError) + } + throw e + } + copyTimes(sourceAttributes, destination) + trash(backup) + return + } + + moveFile(converted, destination, verification) + copyTimes(sourceAttributes, destination) + if(operation.removesSource){ + trash(source) + } + } + + /** + * Renames the file when source and destination are on the same file system, otherwise copies it safely then deletes the source. + */ + private fun moveFile(source: Path, destination: Path, verification: Verification){ + ensureFree(destination) + try{ + atomicMove(source, destination) + return + } + catch(_: AtomicMoveNotSupportedException){ + log.debug { "$source and $destination are on different file systems, copying" } + } + safeCopy(source, destination, verification) + Files.delete(source) + } + + /** + * Copies to a partial file next to the destination, checks it then renames it, so the destination never holds an incomplete file. + * The source is never modified. + */ + private fun safeCopy(source: Path, destination: Path, verification: Verification){ + ensureFree(destination) + val partial = partialFile(destination) + try{ + val sourceAttributes = Files.readAttributes(source, BasicFileAttributes::class.java) + val expectedSize = sourceAttributes.size() + val sourceHash = when(verification){ + Verification.SIZE -> { + copyToPartial(source, partial) + null + } + Verification.CHECKSUM -> copyWithChecksum(source, partial) + } + afterCopy(source, partial) + val size = Files.size(partial) + if(size != expectedSize){ + throw IOException("Copy of $source to $partial is incomplete: $size bytes written out of $expectedSize") + } + if(sourceHash != null){ + val copyHash = checksum(partial) + if(!sourceHash.contentEquals(copyHash)){ + throw IOException("Copy of $source to $partial is corrupted: SHA-256 ${shortHex(sourceHash)}... read from the source, ${shortHex(copyHash)}... read back from the copy") + } + log.debug { "Checksum of $partial matches $source" } + copyTimesBestEffort(sourceAttributes, partial) + } + ensureFree(destination) + try{ + Files.move(partial, destination, StandardCopyOption.ATOMIC_MOVE) + } + catch(_: AtomicMoveNotSupportedException){ + Files.move(partial, destination) + } + } + catch(e: Throwable){ + try{ + Files.deleteIfExists(partial) + } + catch(deleteError: Exception){ + e.addSuppressed(deleteError) + } + throw e + } + } + + /** + * Copies the source to a new partial file, forced to storage. + * + * @return The SHA-256 of the bytes read from the source. + */ + private fun copyWithChecksum(source: Path, partial: Path): ByteArray{ + val digest = MessageDigest.getInstance(CHECKSUM_ALGORITHM) + Files.deleteIfExists(partial) + FileChannel.open(source, StandardOpenOption.READ).use { input -> + FileChannel.open(partial, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW).use { output -> + val buffer = ByteBuffer.allocate(BUFFER_SIZE) + while(input.read(buffer) >= 0){ + buffer.flip() + digest.update(buffer.array(), 0, buffer.limit()) + while(buffer.hasRemaining()){ + output.write(buffer) + } + buffer.clear() + } + output.force(true) + } + } + return digest.digest() + } + + private fun checksum(path: Path): ByteArray{ + val digest = MessageDigest.getInstance(CHECKSUM_ALGORITHM) + FileChannel.open(path, StandardOpenOption.READ).use { input -> + val buffer = ByteBuffer.allocate(BUFFER_SIZE) + while(input.read(buffer) >= 0){ + buffer.flip() + digest.update(buffer.array(), 0, buffer.limit()) + buffer.clear() + } + } + return digest.digest() + } + + /** + * Same as [copyTimes], ignoring the creation time where the file system can't set it. + */ + private fun copyTimesBestEffort(attributes: BasicFileAttributes, path: Path){ + try{ + copyTimes(attributes, path) + } + catch(e: IOException){ + log.debug { "Failed to set the creation time of $path, setting the other times only: ${e.message}" } + Files.getFileAttributeView(path, BasicFileAttributeView::class.java).setTimes(attributes.lastModifiedTime(), attributes.lastAccessTime(), null) + } + } + + private fun ensureFree(destination: Path){ + if(Files.exists(destination, LinkOption.NOFOLLOW_LINKS)){ + throw FileAlreadyExistsException(destination.toString()) + } + } + + fun delete(path: Path){ + if(dryRun){ + return + } + Files.delete(path) + } + + fun deleteIfExists(path: Path): Boolean{ + if(dryRun){ + return false + } + return Files.deleteIfExists(path) + } + + /** + * Moves a file to the trash when the platform has one, deletes it otherwise. + */ + fun trash(path: Path){ + if(dryRun){ + return + } + if(moveToTrash(path)){ + log.debug { "Moved file $path to trash" } + return + } + if(Files.deleteIfExists(path)){ + log.debug { "Deleted file $path" } + } + } + + fun setTimes(path: Path, lastModified: FileTime, creation: FileTime){ + if(dryRun){ + return + } + val view = Files.getFileAttributeView(path, BasicFileAttributeView::class.java) + view.setTimes(lastModified, view.readAttributes().lastAccessTime(), creation) + } + + private fun copyTimes(attributes: BasicFileAttributes, path: Path){ + Files.getFileAttributeView(path, BasicFileAttributeView::class.java) + .setTimes(attributes.lastModifiedTime(), attributes.lastAccessTime(), attributes.creationTime()) + } + + private fun moveToTrash(path: Path): Boolean{ + try{ + if(GraphicsEnvironment.isHeadless() || !Desktop.isDesktopSupported()){ + return false + } + val desktop = Desktop.getDesktop() + return desktop.isSupported(Desktop.Action.MOVE_TO_TRASH) && desktop.moveToTrash(path.toFile()) + } + catch(e: Throwable){ + log.debug(e) { "Trash is not available" } + return false + } + } + + private fun createDirectories(path: Path){ + Files.createDirectories(path) + if(!Files.isDirectory(path)){ + throw IOException("Destination folder $path already exists as a file") + } + } + + companion object{ + const val PARTIAL_SUFFIX = ".filesecure-partial" + private const val CHECKSUM_ALGORITHM = "SHA-256" + private const val BUFFER_SIZE = 1024 * 1024 + + fun partialFile(destination: Path): Path = destination.resolveSibling(".${destination.fileName}$PARTIAL_SUFFIX") + + private fun defaultAtomicMove(source: Path, destination: Path){ + Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE) + } + + private fun defaultCopyToPartial(source: Path, partial: Path){ + Files.copy(source, partial, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING) + } + + private fun shortHex(hash: ByteArray): String = HexFormat.of().formatHex(hash, 0, 8) + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileScanner.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileScanner.kt new file mode 100644 index 0000000..f927fd7 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/FileScanner.kt @@ -0,0 +1,64 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.ScanOptions +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.IOException +import java.nio.file.FileVisitResult +import java.nio.file.FileVisitor +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributes + +private val log = KotlinLogging.logger {} + +/** + * Walks an input folder, reporting the regular files and the folders that aren't excluded. + */ +class FileScanner( + private val root: Path, + private val options: ScanOptions, + private val onFolder: (Path) -> Unit, + private val onFile: (Path) -> Boolean, +) : FileVisitor{ + fun scan(){ + Files.walkFileTree(root, emptySet(), options.maxDepth, this) + } + + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult{ + if(options.skipHidden && dir != root && Files.isHidden(dir)){ + return FileVisitResult.SKIP_SUBTREE + } + val relative = root.relativize(dir).toString().replace("\\", "/") + if(options.excludedFolders.any { it.matcher(relative).matches() }){ + return FileVisitResult.SKIP_SUBTREE + } + log.debug { "Entering folder $dir" } + onFolder(dir) + return FileVisitResult.CONTINUE + } + + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult{ + // Folders beyond the max depth are reported here, they must not be processed as files + if(!attrs.isRegularFile){ + return FileVisitResult.CONTINUE + } + if(options.skipHidden && Files.isHidden(file)){ + return FileVisitResult.CONTINUE + } + if(file.fileName.toString().endsWith(FileOperations.PARTIAL_SUFFIX)){ + log.warn { "Ignoring $file, left by an interrupted copy, it can be deleted" } + return FileVisitResult.CONTINUE + } + return if(onFile(file)) FileVisitResult.CONTINUE else FileVisitResult.TERMINATE + } + + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult{ + log.warn { "Failed to visit $file: ${exc.message}" } + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult{ + exc?.let { log.warn { "Error while listing $dir: ${it.message}" } } + return FileVisitResult.CONTINUE + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineContext.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineContext.kt new file mode 100644 index 0000000..896ff0d --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineContext.kt @@ -0,0 +1,35 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.Pipeline +import fr.rakambda.filesecure.console.PauseGate +import fr.rakambda.filesecure.conversion.ConversionTools +import fr.rakambda.filesecure.metadata.MetadataExtractor +import fr.rakambda.filesecure.progress.NoOpProgressReporter +import fr.rakambda.filesecure.progress.ProgressReporter +import kotlinx.coroutines.sync.Semaphore + +/** + * Everything shared by all pipelines of a run. + * + * @param conversionThreads Maximum number of conversions running at the same time. + * @param workers Maximum number of files processed at the same time. + */ +class RunEnvironment( + val fileOperations: FileOperations, + val metadataExtractor: MetadataExtractor, + val tools: ConversionTools, + conversionThreads: Int = 1, + val workers: Int = 8, + val pauseGate: PauseGate = PauseGate(), + val progress: ProgressReporter = NoOpProgressReporter, +){ + val conversionLock = Semaphore(conversionThreads) + + val dryRun: Boolean + get() = fileOperations.dryRun +} + +class PipelineContext( + val environment: RunEnvironment, + val pipeline: Pipeline, +) diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunner.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunner.kt new file mode 100644 index 0000000..d0e281e --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineRunner.kt @@ -0,0 +1,289 @@ +package fr.rakambda.filesecure.pipeline + +import fr.rakambda.filesecure.config.Mapping +import fr.rakambda.filesecure.config.Operation +import fr.rakambda.filesecure.config.Pipeline +import fr.rakambda.filesecure.conversion.ConversionException +import fr.rakambda.filesecure.conversion.PlannedConversion +import fr.rakambda.filesecure.folder.FolderAction +import fr.rakambda.filesecure.stage.Convert +import fr.rakambda.filesecure.stage.StageExecutor +import fr.rakambda.filesecure.stage.StageResult +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.trySendBlocking +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.FileTime +import java.util.concurrent.ConcurrentLinkedQueue + +private val log = KotlinLogging.logger {} + +class PipelineRunner(private val environment: RunEnvironment){ + /** + * @param stats Updated while the pipeline runs, so counts are kept if it fails or is cancelled. + */ + suspend fun run(pipeline: Pipeline, name: String, stats: PipelineStats = PipelineStats()): PipelineStats{ + log.info { "Running pipeline $name" } + val context = PipelineContext(environment, pipeline) + val executors = mutableListOf() + try{ + withContext(Dispatchers.IO) { + pipeline.stages.forEach { executors += it.open(context) } + } + val tempDirectory = if(pipeline.stages.any { it is Convert } && !environment.dryRun) createTempDirectory(pipeline.temp) else null + try{ + for(mapping in pipeline.mappings){ + MappingRun(pipeline, mapping, executors, tempDirectory, stats).run() + } + } + finally{ + tempDirectory?.let { withContext(NonCancellable) { deleteTempDirectory(it) } } + } + } + finally{ + executors.forEach { executor -> + try{ + executor.close() + } + catch(e: Exception){ + log.error(e) { "Failed to close stage $executor" } + stats.error() + } + } + } + return stats + } + + private suspend fun createTempDirectory(parent: Path?): Path = withContext(Dispatchers.IO) { + if(parent == null){ + Files.createTempDirectory(TEMP_PREFIX) + } + else{ + Files.createDirectories(parent) + Files.createTempDirectory(parent, TEMP_PREFIX) + } + } + + private suspend fun deleteTempDirectory(directory: Path) = withContext(Dispatchers.IO) { + try{ + Files.delete(directory) + } + catch(e: Exception){ + log.warn { "Failed to delete temporary directory $directory, it may still contain files: ${e.message}" } + } + } + + private inner class MappingRun( + private val pipeline: Pipeline, + private val mapping: Mapping, + private val executors: List, + private val tempDirectory: Path?, + private val stats: PipelineStats, + ){ + private val reserver = DestinationReserver() + private val fileOperations = environment.fileOperations + private val progress = environment.progress + + suspend fun run(){ + if(!withContext(Dispatchers.IO) { Files.isDirectory(mapping.input) }){ + log.warn { "Input folder ${mapping.input} doesn't exist, skipping mapping" } + return + } + log.info { "Processing mapping (${pipeline.operation}) ${mapping.input} ==> ${mapping.output}" } + + val folders = ConcurrentLinkedQueue() + coroutineScope { + val files = Channel(FILE_BUFFER_SIZE) + launch { + try{ + runInterruptible(Dispatchers.IO) { + FileScanner(mapping.input, pipeline.scan, folders::add) { file -> + progress.fileFound() + stats.fileFound() + files.trySendBlocking(file).isSuccess + }.scan() + } + } + finally{ + files.close() + } + } + repeat(environment.workers) { + launch(Dispatchers.IO) { + for(file in files){ + processFile(file) + } + } + } + } + + if(pipeline.folderActions.isNotEmpty()){ + log.debug { "Applying folder actions" } + withContext(Dispatchers.IO) { + folders.sortedByDescending { it.nameCount }.forEach { folder -> + pipeline.folderActions.forEach { action -> applyFolderAction(action, folder) } + } + } + } + } + + private fun applyFolderAction(action: FolderAction, folder: Path){ + try{ + action.apply(fileOperations, folder, mapping.input) + } + catch(e: Exception){ + log.error(e) { e.message ?: "Failed to apply $action to $folder" } + stats.error() + } + } + + private suspend fun processFile(file: Path){ + val item = FileItem(file, mapping.input, mapping.output) { environment.metadataExtractor.getMetadata(file) } + try{ + for(executor in executors){ + environment.pauseGate.awaitResumed() + val result = executor.process(item) + if(result is StageResult.Skip){ + log.debug { "Skipping $file: ${result.reason}" } + stats.record(FileOutcome.SKIPPED_BY_STAGE) + return + } + } + commit(item) + } + catch(e: CancellationException){ + throw e + } + catch(e: Exception){ + // Failures caused by a cancellation aren't errors + if(currentCoroutineContext().isActive){ + log.error(e) { "Failed to process $file" } + stats.error() + } + else{ + log.warn { "Processing of $file interrupted by the stop: ${e.message}" } + } + } + finally{ + progress.fileProcessed() + } + } + + private suspend fun commit(item: FileItem){ + val source = item.source + val conversion = item.conversion + val operation = pipeline.operation + + val isSource = item.destination.toAbsolutePath().normalize() == source.toAbsolutePath().normalize() + if(isSource && conversion == null){ + log.debug { "$source is already at its destination" } + stats.record(FileOutcome.ALREADY_IN_PLACE) + return + } + // The source is replaced by its converted version, there's no conflict as the source will be gone + val replaceSource = isSource && operation.removesSource + val destination = if(replaceSource) item.destination else reserver.reserve(item.destination, pipeline.conflict) + if(destination == null){ + log.info { "Skipping $source, destination ${item.destination} already exists" } + stats.record(FileOutcome.DESTINATION_EXISTS) + return + } + + val conversionDescription = conversion?.let { " converted to ${it.processor.extension}" } ?: "" + log.info { "$operation file $source to $destination$conversionDescription" } + if(operation == Operation.NONE){ + stats.record(FileOutcome.PROCESSED) + return + } + + // Read before the source may disappear + val fileDate = if(pipeline.setFileCreationDate) item.metadata.nameDate ?: item.metadata.takenDate else null + + if(conversion == null){ + withContext(Dispatchers.IO) { fileOperations.transfer(operation, source, destination, pipeline.verify) } + stats.record(if(operation == Operation.COPY) FileOutcome.COPIED else FileOutcome.MOVED) + } + else{ + val converted = convert(item, conversion, destination) + if(converted == null){ + stats.record(FileOutcome.CONVERTED) + return + } + try{ + withContext(Dispatchers.IO) { fileOperations.commitConverted(operation, source, converted, destination, replaceSource, pipeline.verify) } + } + finally{ + withContext(NonCancellable + Dispatchers.IO) { Files.deleteIfExists(converted) } + } + stats.record(FileOutcome.CONVERTED) + } + + fileDate?.let { + log.debug { "Setting creation date for $destination to ${it.toInstant()}" } + try{ + val time = FileTime.from(it.toInstant()) + withContext(Dispatchers.IO) { fileOperations.setTimes(destination, time, time) } + } + catch(e: CancellationException){ + throw e + } + catch(e: Exception){ + log.error(e) { "Failed to set the creation date of $destination to ${it.toInstant()}" } + stats.error() + } + } + } + + /** + * @return The converted file in the temporary directory, null during dry runs. + */ + private suspend fun convert(item: FileItem, conversion: PlannedConversion, destination: Path): Path?{ + // No temporary directory is created during dry runs, the path is only used in logs + val directory = tempDirectory ?: pipeline.temp ?: Path.of(System.getProperty("java.io.tmpdir")) + val converted = directory.resolve("${System.nanoTime()}-${item.source.hashCode()}-${destination.fileName}") + val converter = conversion.processor.createConverter(environment.tools, conversion.probeResult, item.source, converted) + if(environment.dryRun){ + log.info { "Dry run: would have executed `${converter.description}`" } + return null + } + + progress.conversionQueued() + try{ + environment.conversionLock.withPermit { + environment.pauseGate.awaitResumed() + log.info { "Converting ${item.source} to $converted" } + progress.task(destination.fileName.toString()).use { converter.convert(it) } + } + val size = withContext(Dispatchers.IO) { if(Files.exists(converted)) Files.size(converted) else -1 } + if(size <= 0){ + throw ConversionException("Converted file $converted is missing or empty") + } + log.debug { "Converted ${item.source} to $converted" } + return converted + } + catch(e: Throwable){ + withContext(NonCancellable + Dispatchers.IO) { Files.deleteIfExists(converted) } + throw e + } + finally{ + progress.conversionDone() + } + } + } + + private companion object{ + const val TEMP_PREFIX = "FileSecure" + const val FILE_BUFFER_SIZE = 1000 + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineStats.kt b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineStats.kt new file mode 100644 index 0000000..8a33ac4 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/pipeline/PipelineStats.kt @@ -0,0 +1,71 @@ +package fr.rakambda.filesecure.pipeline + +import java.util.EnumMap +import java.util.concurrent.atomic.AtomicLong + +/** + * What happened to a file that went through a pipeline. + */ +enum class FileOutcome(private val label: String, private val dryRunLabel: String){ + MOVED("moved", "would move"), + COPIED("copied", "would copy"), + CONVERTED("converted", "would convert"), + /** + * Accepted by every stage in a pipeline whose operation is NONE. + */ + PROCESSED("would process (operation NONE)", "would process (operation NONE)"), + SKIPPED_BY_STAGE("skipped by stages", "skipped by stages"), + DESTINATION_EXISTS("destination exists", "destination exists"), + ALREADY_IN_PLACE("already in place", "already in place"); + + fun label(dryRun: Boolean): String = if(dryRun) dryRunLabel else label +} + +/** + * Counters of a pipeline run, updated concurrently by the workers. + */ +class PipelineStats{ + private val found = AtomicLong() + private val errors = AtomicLong() + private val outcomes = EnumMap(FileOutcome::class.java).apply { + FileOutcome.entries.forEach { put(it, AtomicLong()) } + } + + val foundCount: Long + get() = found.get() + + val errorCount: Long + get() = errors.get() + + fun fileFound(){ + found.incrementAndGet() + } + + fun record(outcome: FileOutcome){ + outcomes.getValue(outcome).incrementAndGet() + } + + fun error(){ + errors.incrementAndGet() + } + + fun count(outcome: FileOutcome): Long = outcomes.getValue(outcome).get() + + operator fun plusAssign(other: PipelineStats){ + found.addAndGet(other.foundCount) + errors.addAndGet(other.errorCount) + FileOutcome.entries.forEach { outcomes.getValue(it).addAndGet(other.count(it)) } + } + + /** + * Counters separated by commas, zero outcomes being omitted, e.g. `12 found, 10 moved, 2 skipped by stages, 0 errors`. + */ + fun describe(dryRun: Boolean): String{ + val parts = mutableListOf("$foundCount found") + FileOutcome.entries.forEach { outcome -> + count(outcome).takeIf { it > 0 }?.let { parts += "$it ${outcome.label(dryRun)}" } + } + parts += "$errorCount errors" + return parts.joinToString(", ") + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/progress/ProgressReporter.kt b/src/main/kotlin/fr/rakambda/filesecure/progress/ProgressReporter.kt new file mode 100644 index 0000000..ff00410 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/progress/ProgressReporter.kt @@ -0,0 +1,133 @@ +package fr.rakambda.filesecure.progress + +import me.tongfei.progressbar.InteractiveConsoleProgressBarConsumer +import me.tongfei.progressbar.ProgressBar +import me.tongfei.progressbar.ProgressBarBuilder +import java.io.FileDescriptor +import java.io.FileOutputStream +import java.io.PrintStream +import java.time.temporal.ChronoUnit +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +interface ProgressReporter : AutoCloseable{ + fun fileFound() + + fun fileProcessed() + + fun conversionQueued() + + fun conversionDone() + + /** + * Starts tracking a single task, closing it releases its display. + */ + fun task(name: String): TaskProgress + + override fun close(){ + } +} + +interface TaskProgress : AutoCloseable{ + fun setMax(max: Long) + + fun stepTo(value: Long) + + override fun close(){ + } +} + +object NoOpProgressReporter : ProgressReporter{ + private object NoOpTask : TaskProgress{ + override fun setMax(max: Long){ + } + + override fun stepTo(value: Long){ + } + } + + override fun fileFound(){ + } + + override fun fileProcessed(){ + } + + override fun conversionQueued(){ + } + + override fun conversionDone(){ + } + + override fun task(name: String): TaskProgress = NoOpTask +} + +/** + * Console progress bars: one for files, one for conversions, and one per running conversion, reused once their conversion ends. + */ +class ConsoleProgressReporter : ProgressReporter{ + private val filesFound = AtomicLong() + private val conversionsQueued = AtomicLong() + private val filesBar = ProgressBarBuilder() + .setTaskName("Files") + .setUnit(" files", 1) + .setInitialMax(0) + .hideEta() + .showSpeed() + .build() + private val conversionsBar = ProgressBarBuilder() + .setTaskName("Conversions") + .setInitialMax(0) + .build() + private val taskCounter = AtomicInteger() + private val idleTaskBars = ConcurrentLinkedQueue() + + override fun fileFound(){ + filesBar.maxHint(filesFound.incrementAndGet()) + } + + override fun fileProcessed(){ + filesBar.step() + } + + override fun conversionQueued(){ + conversionsBar.maxHint(conversionsQueued.incrementAndGet()) + } + + override fun conversionDone(){ + conversionsBar.step() + } + + override fun task(name: String): TaskProgress{ + val bar = (idleTaskBars.poll() ?: createTaskBar()).resume() + bar.extraMessage = name + return object : TaskProgress{ + override fun setMax(max: Long){ + bar.maxHint(max) + } + + override fun stepTo(value: Long){ + bar.stepTo(value) + } + + override fun close(){ + idleTaskBars.offer(bar.setExtraMessage("").maxHint(-1).reset().pause()) + } + } + } + + private fun createTaskBar(): ProgressBar = ProgressBarBuilder() + .setTaskName("Converter ${taskCounter.incrementAndGet()}") + .setUnit("s", 1000) + .setSpeedUnit(ChronoUnit.SECONDS) + .setConsumer(InteractiveConsoleProgressBarConsumer(PrintStream(FileOutputStream(FileDescriptor.err)), -1)) + .clearDisplayOnFinish() + .build() + + override fun close(){ + idleTaskBars.forEach(ProgressBar::close) + idleTaskBars.clear() + conversionsBar.close() + filesBar.close() + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/Convert.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/Convert.kt new file mode 100644 index 0000000..f2945c0 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/Convert.kt @@ -0,0 +1,133 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.conversion.PlannedConversion +import fr.rakambda.filesecure.conversion.ProbeFilter +import fr.rakambda.filesecure.conversion.ProcessorType +import fr.rakambda.filesecure.pipeline.FileItem +import fr.rakambda.filesecure.pipeline.PipelineContext +import fr.rakambda.filesecure.storage.H2Storage +import fr.rakambda.filesecure.storage.NoOpStorage +import fr.rakambda.filesecure.storage.UselessFileStorage +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Timestamp +import java.util.Locale + +private val log = KotlinLogging.logger {} + +/** + * Plans the conversion of media files: the first processor able to handle the file is used, and the destination extension changes accordingly. + * The conversion itself runs after all stages accepted the file, and only if its destination is free. + */ +data class Convert( + /** + * Processors tried in order, empty to use the defaults. + */ + val processors: List = emptyList(), + /** + * Extensions of the files to probe, empty to use the defaults. + */ + val extensions: Set = emptySet(), + val filters: List = emptyList(), + /** + * What happens to files that won't be converted. + */ + val unhandled: UnhandledStrategy = UnhandledStrategy.SKIP, + /** + * Database remembering files that can't be converted, to avoid probing them on every run. + */ + val cache: Path? = null, +) : Stage{ + override fun open(context: PipelineContext): StageExecutor{ + val storage = cache?.let(::H2Storage) ?: NoOpStorage + return ConvertExecutor( + context, + processors.ifEmpty { DEFAULT_PROCESSORS }, + extensions.map { it.lowercase(Locale.ROOT) }.toSet().ifEmpty { DEFAULT_EXTENSIONS }, + filters, + unhandled, + storage, + ) + } + + companion object{ + val DEFAULT_PROCESSORS = listOf(ProcessorType.VIDEO_TO_HEVC_MKV, ProcessorType.MP3_TO_AAC, ProcessorType.TIFF_TO_JPG) + val DEFAULT_EXTENSIONS = setOf("mp4", "mov", "mkv", "avi", "tiff", "mp3", "m4a", "m4v", "ts", "m2ts", "mts", "heic", "jpg", "raf") + } +} + +enum class UnhandledStrategy{ + /** + * The file continues through the pipeline unchanged. + */ + PASS, + + /** + * The file is left where it is. + */ + SKIP, +} + +private class ConvertExecutor( + private val context: PipelineContext, + private val processors: List, + private val extensions: Set, + private val filters: List, + private val unhandled: UnhandledStrategy, + private val storage: UselessFileStorage, +) : StageExecutor{ + override suspend fun process(item: FileItem): StageResult{ + val file = item.source + val lastModified = withContext(Dispatchers.IO) { Timestamp(Files.getLastModifiedTime(file).toMillis()) } + if(storage.isUseless(file, lastModified)){ + return unhandled("known as not convertible") + } + if(file.extension !in extensions){ + storage.setUseless(file, lastModified) + return unhandled("extension is not handled") + } + + val probeResult = context.environment.tools.probe(file) + val processor = processors.map { it.processor }.firstOrNull { + try{ + it.canHandle(probeResult, file) + } + catch(e: Exception){ + log.warn(e) { "Failed to check if ${it.javaClass.simpleName} can handle $file, it is considered not handled" } + false + } + } + if(processor == null){ + storage.setUseless(file, lastModified) + return unhandled("no processor can handle it") + } + if(!filters.all { it.test(probeResult) }){ + return unhandled("probe filters did not pass") + } + + log.debug { "Processor ${processor.javaClass.simpleName} matched $file" } + item.conversion = PlannedConversion(processor, probeResult) + item.destination = item.destination.withFileName(item.destination.nameWithoutExtension + "." + processor.extension) + return StageResult.Continue + } + + private fun unhandled(reason: String): StageResult = when(unhandled){ + UnhandledStrategy.PASS -> StageResult.Continue + UnhandledStrategy.SKIP -> StageResult.Skip("not converted, $reason") + } + + override fun close() = storage.close() + + private val Path.extension: String + get() = fileName.toString().substringAfterLast('.', "").lowercase(Locale.ROOT) + + private val Path.nameWithoutExtension: String + get(){ + val name = fileName.toString() + val dotIndex = name.lastIndexOf('.') + return if(dotIndex <= 0) name else name.substring(0, dotIndex) + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/FilterStages.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/FilterStages.kt new file mode 100644 index 0000000..0d3665e --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/FilterStages.kt @@ -0,0 +1,40 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.pipeline.FileItem +import java.time.ZonedDateTime +import java.util.regex.Pattern + +/** + * Keeps files whose name fully matches one of the includes (all files if empty) and none of the excludes. + */ +data class NameFilter( + val includes: List = emptyList(), + val excludes: List = emptyList(), +) : SimpleStage{ + override fun process(item: FileItem): StageResult{ + val name = item.source.fileName.toString() + if(excludes.any { it.matcher(name).matches() }){ + return StageResult.Skip("name is excluded") + } + if(includes.isNotEmpty() && includes.none { it.matcher(name).matches() }){ + return StageResult.Skip("name is not included") + } + return StageResult.Continue + } +} + +/** + * Keeps files whose date is older than the given offset. + */ +data class IfOlderThan( + val dayOffset: Long = 0, + val minuteOffset: Long = 0, +) : SimpleStage{ + override fun process(item: FileItem): StageResult{ + val limit = ZonedDateTime.now().minusDays(dayOffset).minusMinutes(minuteOffset) + if(item.metadata.date.isAfter(limit)){ + return StageResult.Skip("file is too recent") + } + return StageResult.Continue + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/FolderStages.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/FolderStages.kt new file mode 100644 index 0000000..270b867 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/FolderStages.kt @@ -0,0 +1,108 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.pipeline.FileItem +import java.nio.file.Path +import java.time.temporal.ChronoField +import java.util.Locale + +/** + * Prefixes the destination with folders built from the first characters of its first path element, e.g. `going/file` with 3 elements becomes `g/o/i/going/file`. + */ +data class FolderCanonical( + val elementCount: Int, +) : PathStage{ + override fun transform(item: FileItem): Path{ + val relative = item.relativeDestination + val firstElement = relative.getName(0).toString() + + var canonical = item.outputBase + for(index in 0 until elementCount){ + val part = firstElement.getOrNull(index)?.toString()?.lowercase() + canonical = canonical.resolve(if(part == null || part in FORBIDDEN_PARTS) UNKNOWN_PART else part) + } + return canonical.resolve(relative) + } + + private companion object{ + const val UNKNOWN_PART = "_" + val FORBIDDEN_PARTS = setOf(".", "/", "\\") + } +} + +/** + * Prefixes the destination with `yyyy/MM/dd` folders from the file date. + */ +class FolderPerDay : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve("%02d".format(date.monthValue)) + .resolve("%02d".format(date.dayOfMonth)) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with `yyyy/MM/w` folders from the file date, `w` being the aligned week of the month. + */ +class FolderPerWeek : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve("%02d".format(date.monthValue)) + .resolve("%1d".format(date.get(ChronoField.ALIGNED_WEEK_OF_MONTH))) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with `yyyy/MM` folders from the file date. + */ +class FolderPerMonth : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve("%02d".format(date.monthValue)) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with a `yyyy` folder from the file date. + */ +class FolderPerYear : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d".format(date.year)) + .resolve(item.relativeDestination) + } +} + +/** + * Prefixes the destination with a `yyyy-MM-dd` folder from the file date. + */ +class FolderUniquePerDay : PathStage{ + override fun transform(item: FileItem): Path{ + val date = item.metadata.date + return item.outputBase.resolve("%4d-%02d-%02d".format(date.year, date.monthValue, date.dayOfMonth)) + .resolve(item.relativeDestination) + } +} + +/** + * Lowercases the folders of the destination below the output folder, the file name is kept as is. + */ +class Lowercase : PathStage{ + override fun transform(item: FileItem): Path{ + val folder = item.destination.parent + val relativeFolder = item.outputBase.relativize(folder).toString().lowercase(Locale.ROOT) + return item.outputBase.resolve(relativeFolder).resolve(item.destination.fileName) + } +} + +/** + * Restores the original file name, keeping the current destination folder. + */ +class NoRename : PathStage{ + override fun transform(item: FileItem): Path = item.destination.withFileName(item.source.fileName.toString()) +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/RegexPath.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/RegexPath.kt new file mode 100644 index 0000000..0d590db --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/RegexPath.kt @@ -0,0 +1,45 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.pipeline.FileItem +import java.util.regex.Pattern + +/** + * Rewrites the destination relative to the output folder (using `/` as separator) with the first rule whose regex fully matches it. + * + * For example the rule `(?:[^/]* - )?(\d+)/(.*)` → `Whatsapp/$1/$2` moves `Bob - 123/a.jpg` to `Whatsapp/123/a.jpg`. + */ +data class RegexPath( + val rules: List, + /** + * What happens to files matching no rule. + */ + val unmatched: UnhandledStrategy = UnhandledStrategy.SKIP, +) : SimpleStage{ + override fun process(item: FileItem): StageResult{ + val relative = item.relativeDestination.toString().replace("\\", "/") + for(rule in rules){ + val matcher = rule.regex.matcher(relative) + if(!matcher.matches()){ + continue + } + val replaced = item.outputBase.resolve(matcher.replaceFirst(rule.replacement)).normalize() + if(!replaced.startsWith(item.outputBase.normalize())){ + return StageResult.Skip("rewritten destination $replaced is outside the output folder") + } + item.destination = replaced + return StageResult.Continue + } + return when(unmatched){ + UnhandledStrategy.PASS -> StageResult.Continue + UnhandledStrategy.SKIP -> StageResult.Skip("path matches no rule") + } + } + + data class Rule( + val regex: Pattern, + /** + * Replacement where `$1` or `${name}` refer to the groups of the regex. + */ + val replacement: String, + ) +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/RenameStages.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/RenameStages.kt new file mode 100644 index 0000000..d5de5fc --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/RenameStages.kt @@ -0,0 +1,77 @@ +package fr.rakambda.filesecure.stage + +import fr.rakambda.filesecure.pipeline.FileItem +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.Path +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.Locale +import java.util.regex.Pattern + +private val log = KotlinLogging.logger {} + +/** + * Renames the file with its date, keeping its extension. + */ +data class RenameWithDate( + val format: String = "yyyy-MM-dd HH.mm.ss", + val locale: String = "en", + /** + * When not empty, only files whose original name fully matches one of these are renamed. + */ + val filters: List = emptyList(), + val zone: String? = null, +) : PathStage{ + private val formatter: DateTimeFormatter = DateTimeFormatter.ofPattern(format, Locale.Builder().setLanguage(locale).build()) + .withZone(zone?.let(ZoneId::of) ?: ZoneId.systemDefault()) + + override fun transform(item: FileItem): Path?{ + val sourceName = item.source.fileName.toString() + if(filters.isNotEmpty() && filters.none { it.matcher(sourceName).matches() }){ + log.debug { "Not renaming $sourceName as it doesn't match any filter" } + return null + } + return item.destination.withFileName(formatter.format(item.metadata.date) + item.destination.extensionWithDot) + } +} + +/** + * Renames the file with a date read from its current name, keeping its extension. Files whose name doesn't contain a date are skipped. + * + * The regex must define the named groups `date` and `time`, parsed with [dateFormat] and [timeFormat]. + */ +data class RenameWithDatePattern( + val regex: Pattern, + val dateFormat: String, + val timeFormat: String, + val outputFormat: String = "yyyy-MM-dd HH.mm.ss", + val locale: String = "en", + val zone: String? = null, +) : SimpleStage{ + private val zoneId: ZoneId = zone?.let(ZoneId::of) ?: ZoneId.systemDefault() + private val inputFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern("$dateFormat $timeFormat", Locale.Builder().setLanguage(locale).build()) + .withZone(zoneId) + private val outputFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern(outputFormat, Locale.Builder().setLanguage(locale).build()) + .withZone(zoneId) + + override fun process(item: FileItem): StageResult{ + val date = extractDate(item.destination.fileName.toString()) ?: return StageResult.Skip("no date found in name") + item.destination = item.destination.withFileName(outputFormatter.format(date) + item.destination.extensionWithDot) + return StageResult.Continue + } + + private fun extractDate(name: String): ZonedDateTime?{ + try{ + val matcher = regex.matcher(name) + if(!matcher.matches()){ + return null + } + return ZonedDateTime.parse("${matcher.group("date")} ${matcher.group("time")}", inputFormatter) + } + catch(e: Exception){ + log.warn { "Failed to read a date from `$name` with format `$dateFormat $timeFormat`, the file is skipped: ${e.message}" } + return null + } + } +} diff --git a/src/main/kotlin/fr/rakambda/filesecure/stage/Stage.kt b/src/main/kotlin/fr/rakambda/filesecure/stage/Stage.kt new file mode 100644 index 0000000..e57a3d6 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/stage/Stage.kt @@ -0,0 +1,83 @@ +package fr.rakambda.filesecure.stage + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import fr.rakambda.filesecure.pipeline.FileItem +import fr.rakambda.filesecure.pipeline.PipelineContext +import java.nio.file.Path + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes( + JsonSubTypes.Type(value = NameFilter::class, name = "NameFilter"), + JsonSubTypes.Type(value = IfOlderThan::class, name = "IfOlderThan"), + JsonSubTypes.Type(value = FolderCanonical::class, name = "FolderCanonical"), + JsonSubTypes.Type(value = FolderPerDay::class, name = "FolderPerDay"), + JsonSubTypes.Type(value = FolderPerWeek::class, name = "FolderPerWeek"), + JsonSubTypes.Type(value = FolderPerMonth::class, name = "FolderPerMonth"), + JsonSubTypes.Type(value = FolderPerYear::class, name = "FolderPerYear"), + JsonSubTypes.Type(value = FolderUniquePerDay::class, name = "FolderUniquePerDay"), + JsonSubTypes.Type(value = Lowercase::class, name = "Lowercase"), + JsonSubTypes.Type(value = NoRename::class, name = "NoRename"), + JsonSubTypes.Type(value = RenameWithDate::class, name = "RenameWithDate"), + JsonSubTypes.Type(value = RenameWithDatePattern::class, name = "RenameWithDatePattern"), + JsonSubTypes.Type(value = RegexPath::class, name = "RegexPath"), + JsonSubTypes.Type(value = Convert::class, name = "Convert"), +) +sealed interface Stage{ + /** + * Creates what runs this stage during one execution of a pipeline. + */ + fun open(context: PipelineContext): StageExecutor +} + +fun interface StageExecutor : AutoCloseable{ + suspend fun process(item: FileItem): StageResult + + override fun close(){ + } +} + +sealed interface StageResult{ + data object Continue : StageResult + + data class Skip(val reason: String) : StageResult +} + +/** + * A stage without any state or resource. + */ +sealed interface SimpleStage : Stage{ + fun process(item: FileItem): StageResult + + override fun open(context: PipelineContext): StageExecutor = StageExecutor { process(it) } +} + +/** + * A stage changing the destination of the file. + */ +sealed interface PathStage : SimpleStage{ + /** + * @return The new destination, or null to keep the current one. + */ + fun transform(item: FileItem): Path? + + override fun process(item: FileItem): StageResult{ + transform(item)?.let { item.destination = it } + return StageResult.Continue + } +} + +/** + * The destination path relative to the output folder. + */ +internal val FileItem.relativeDestination: Path + get() = outputBase.relativize(destination) + +internal fun Path.withFileName(name: String): Path = parent?.resolve(name) ?: fileSystem.getPath(name) + +internal val Path.extensionWithDot: String + get(){ + val name = fileName.toString() + val dotIndex = name.lastIndexOf('.') + return if(dotIndex <= 0) "" else name.substring(dotIndex) + } diff --git a/src/main/kotlin/fr/rakambda/filesecure/storage/UselessFileStorage.kt b/src/main/kotlin/fr/rakambda/filesecure/storage/UselessFileStorage.kt new file mode 100644 index 0000000..5624b15 --- /dev/null +++ b/src/main/kotlin/fr/rakambda/filesecure/storage/UselessFileStorage.kt @@ -0,0 +1,121 @@ +package fr.rakambda.filesecure.storage + +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection +import java.sql.DriverManager +import java.sql.Timestamp +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +private val log = KotlinLogging.logger {} + +/** + * Remembers files that don't need to be converted, to avoid probing them again on the next runs. + */ +interface UselessFileStorage : AutoCloseable{ + /** + * @return true if the file was marked as useless and hasn't been modified since. + */ + fun isUseless(path: Path, lastModified: Timestamp): Boolean + + fun setUseless(path: Path, lastModified: Timestamp) + + /** + * Persists the files marked since the last save. + */ + fun save() + + override fun close() = save() +} + +object NoOpStorage : UselessFileStorage{ + override fun isUseless(path: Path, lastModified: Timestamp) = false + + override fun setUseless(path: Path, lastModified: Timestamp){ + } + + override fun save(){ + } +} + +/** + * H2 database storage, compatible with MediaConverter's database files. + */ +class H2Storage(private val databaseFile: Path) : UselessFileStorage{ + private val useless = ConcurrentHashMap() + private val newUseless = ConcurrentLinkedQueue>() + private val saveLock = ReentrantLock() + + init{ + log.debug { "Loading useless files from $databaseFile" } + databaseFile.toAbsolutePath().parent?.let(Files::createDirectories) + connect().use { connection -> + connection.createStatement().use { statement -> + statement.executeUpdate(""" + CREATE TABLE IF NOT EXISTS Useless( + Filee VARCHAR(512) NOT NULL, + LastModified TIMESTAMP, + PRIMARY KEY(Filee) + )""".trimIndent()) + statement.executeUpdate("ALTER TABLE Useless ADD COLUMN IF NOT EXISTS LastModified TIMESTAMP") + statement.executeQuery("SELECT Filee, LastModified FROM Useless").use { result -> + while(result.next()){ + useless[result.getString("Filee")] = result.getTimestamp("LastModified") + } + } + } + } + log.info { "Loaded ${useless.size} useless files from $databaseFile" } + } + + override fun isUseless(path: Path, lastModified: Timestamp): Boolean{ + val key = key(path) + if(!useless.containsKey(key)){ + return false + } + val entryLastModified = useless[key] ?: return true + return entryLastModified >= lastModified + } + + override fun setUseless(path: Path, lastModified: Timestamp){ + log.debug { "Marking $path as useless" } + val key = key(path) + useless[key] = lastModified + newUseless.add(key to lastModified) + if(newUseless.size >= SAVE_THRESHOLD){ + save() + } + } + + override fun save(): Unit = saveLock.withLock { + if(newUseless.isEmpty()){ + return + } + connect().use { connection -> + connection.prepareStatement("MERGE INTO Useless(Filee, LastModified) VALUES(?, ?)").use { statement -> + var count = 0 + while(true){ + val (path, lastModified) = newUseless.poll() ?: break + statement.setString(1, path) + statement.setTimestamp(2, lastModified) + statement.addBatch() + count++ + } + statement.executeBatch() + log.debug { "Saved $count new useless files" } + } + } + } + + private fun connect(): Connection = DriverManager.getConnection("jdbc:h2:${databaseFile.toAbsolutePath()}") + + private fun key(path: Path) = path.toString().replace("\\", "/") + + private companion object{ + const val SAVE_THRESHOLD = 100 + } +} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 5ef4908..5bdef3a 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -1,11 +1,20 @@ - + + + + + + + + + + diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformerTest.java deleted file mode 100644 index 4a04c1e..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderCanonicalTransformerTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(MockitoExtension.class) -class FolderCanonicalTransformerTest{ - private final FolderCanonicalTransformer tested = new FolderCanonicalTransformer(3); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - var base = Paths.get("/path"); - var out = Paths.get("/path/going/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/g/o/i/going/to/my/file.ext")); - } - - @Test - void itShouldWorkWhenPartIsSmaller() throws AbandonBackupException{ - var base = Paths.get("/path"); - var out = Paths.get("/path/go/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/g/o/_/go/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformerTest.java deleted file mode 100644 index 89dd3e3..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerDayTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerDayTransformerTest{ - private final FolderPerDayTransformer tested = new FolderPerDayTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/05/12/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformerTest.java deleted file mode 100644 index 6f5a81d..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerMonthTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerMonthTransformerTest{ - private final FolderPerMonthTransformer tested = new FolderPerMonthTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/05/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformerTest.java deleted file mode 100644 index a9a98c6..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerWeekTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerWeekTransformerTest{ - private final FolderPerWeekTransformer tested = new FolderPerWeekTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/05/2/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformerTest.java deleted file mode 100644 index 28d467d..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderPerYearTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderPerYearTransformerTest{ - private final FolderPerYearTransformer tested = new FolderPerYearTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformerTest.java deleted file mode 100644 index 97f72b1..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/FolderUniquePerDayTransformerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class FolderUniquePerDayTransformerTest{ - private final FolderUniquePerDayTransformer tested = new FolderUniquePerDayTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var base = Paths.get("/path"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/2023-05-12/to/my/file.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformerTest.java deleted file mode 100644 index c9adf79..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/LowercaseTransformerTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(MockitoExtension.class) -class LowercaseTransformerTest{ - private final LowercaseTransformer tested = new LowercaseTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - var base = Paths.get("/pAth/tO"); - var out = Paths.get("/pAth/tO/mY/sUper/fIle.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/pAth/tO/my/super/fIle.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformerTest.java deleted file mode 100644 index a3e1423..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/NoRenameTransformerTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(MockitoExtension.class) -class NoRenameTransformerTest{ - private final NoRenameTransformer tested = new NoRenameTransformer(); - - @Mock - private Path originalOut; - @Mock - private Path baseOutput; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException{ - var source = Paths.get("/path/to/source.other"); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, baseOutput, out, metadata)) - .contains(Paths.get("/path/to/my/source.other")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformerTest.java deleted file mode 100644 index edb1846..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/file/RenameWithDateTransformerTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package fr.rakambda.filesecure.config.options.file; - -import fr.rakambda.filesecure.exceptions.AbandonBackupException; -import fr.rakambda.filesecure.processor.FileMetadata; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class RenameWithDateTransformerTest{ - private final RenameWithDateTransformer tested = new RenameWithDateTransformer(); - - @Mock - private Path source; - @Mock - private Path originalOut; - @Mock - private Path base; - @Mock - private FileMetadata metadata; - - @Test - void itShouldWork() throws AbandonBackupException, IllegalAccessException, NoSuchFieldException{ - tested.setZone("UTC+2"); - - when(metadata.getDate()).thenReturn(ZonedDateTime.of(2023, 5, 12, 10, 15, 12, 0, UTC)); - var out = Paths.get("/path/to/my/file.ext"); - - assertThat(tested.apply(source, originalOut, base, out, metadata)) - .contains(Paths.get("/path/to/my/2023-05-12 12.15.12.ext")); - } -} \ No newline at end of file diff --git a/src/test/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformerTest.java b/src/test/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformerTest.java deleted file mode 100644 index 46b55a9..0000000 --- a/src/test/java/fr/rakambda/filesecure/config/options/folder/DeleteIfEmptyTransformerTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package fr.rakambda.filesecure.config.options.folder; - -import fr.rakambda.filesecure.utils.FileOperations; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -@ExtendWith(MockitoExtension.class) -class DeleteIfEmptyTransformerTest{ - private final DeleteIfEmptyTransformer tested = new DeleteIfEmptyTransformer(2); - - @Mock - private FileOperations fileOperations; - - private static Stream validPaths(){ - return Stream.of( - Arguments.of(Paths.get("/path/to/the")), - Arguments.of(Paths.get("/path/to/the/folder")), - Arguments.of(Paths.get("/path/to/the/folder/yes")) - ); - } - - private static Stream depthInvalidPaths(){ - return Stream.of( - Arguments.of(Paths.get("/path")), - Arguments.of(Paths.get("/path/to")) - ); - } - - @ParameterizedTest - @MethodSource("validPaths") - void itShouldWorkOnEmptyFolder(Path folder) throws IOException{ - var base = Paths.get("/path"); - - try(var filesMock = mockStatic(Files.class)){ - filesMock.when(() -> Files.list(folder)).thenReturn(Stream.of()); - - tested.apply(fileOperations, folder, base); - - verify(fileOperations).delete(folder); - } - } - - @Test - void itShouldSkipNotEmpty() throws IOException{ - var folder = Paths.get("/path/to/the/folder"); - var base = Paths.get("/path"); - - try(var filesMock = mockStatic(Files.class)){ - filesMock.when(() -> Files.list(folder)).thenReturn(Stream.of(folder.resolve("fake"))); - - tested.apply(fileOperations, folder, base); - - verify(fileOperations, never()).delete(any()); - } - } - - @ParameterizedTest - @MethodSource("depthInvalidPaths") - void itShouldSkipIfDepthIsNotEnough(Path folder) throws IOException{ - var base = Paths.get("/path"); - - try(var filesMock = mockStatic(Files.class)){ - filesMock.when(() -> Files.list(folder)).thenReturn(Stream.empty()); - - tested.apply(fileOperations, folder, base); - - verify(fileOperations, never()).delete(any()); - } - } -} \ No newline at end of file diff --git a/src/test/kotlin/fr/rakambda/filesecure/ExitCodeTest.kt b/src/test/kotlin/fr/rakambda/filesecure/ExitCodeTest.kt new file mode 100644 index 0000000..dbe515a --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/ExitCodeTest.kt @@ -0,0 +1,23 @@ +package fr.rakambda.filesecure + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ExitCodeTest{ + @Test + fun success(){ + assertThat(exitCode(0, false)).isEqualTo(0) + } + + @Test + fun errors(){ + assertThat(exitCode(1, false)).isEqualTo(2) + assertThat(exitCode(42, false)).isEqualTo(2) + } + + @Test + fun stoppedTakesPrecedenceOverErrors(){ + assertThat(exitCode(0, true)).isEqualTo(3) + assertThat(exitCode(5, true)).isEqualTo(3) + } +} diff --git a/src/test/kotlin/fr/rakambda/filesecure/config/ConfigSchemaTest.kt b/src/test/kotlin/fr/rakambda/filesecure/config/ConfigSchemaTest.kt new file mode 100644 index 0000000..e81d2d7 --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/config/ConfigSchemaTest.kt @@ -0,0 +1,140 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import fr.rakambda.filesecure.conversion.ProbeFilter +import fr.rakambda.filesecure.conversion.ProcessorType +import fr.rakambda.filesecure.folder.FolderAction +import fr.rakambda.filesecure.stage.RegexPath +import fr.rakambda.filesecure.stage.Stage +import fr.rakambda.filesecure.stage.UnhandledStrategy +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import java.nio.file.Path +import kotlin.reflect.KClass +import kotlin.reflect.full.findAnnotation +import kotlin.reflect.full.primaryConstructor + +/** + * Checks that `config.schema.json` describes exactly what the configuration classes accept. + */ +class ConfigSchemaTest{ + private val schema = ObjectMapper().readTree(SCHEMA_PATH.toFile()) + + @Test + fun usesDraft07(){ + assertThat(schema["\$schema"].asText()).isEqualTo("http://json-schema.org/draft-07/schema#") + } + + @Test + fun describesTopLevelKeys(){ + assertStrictObject(schema, parameters(Configuration::class) + listOf("defaults", "templates")) + } + + @Test + fun describesPipelineKeys(){ + assertStrictObject(definition("pipeline"), parameters(Pipeline::class) + listOf(USE_DEFAULTS, MAPPING_TABLE, MAPPINGS_FILE)) + assertThat(required(definition("pipeline"))).isEmpty() + } + + @Test + fun describesDefaultsKeys(){ + val defaults = definition("defaults") + assertStrictObject(defaults, parameters(Pipeline::class) + listOf(USE_DEFAULTS, MAPPING_TABLE, MAPPINGS_FILE) - FORBIDDEN_KEYS.toSet()) + properties(defaults).forEach { (key, value) -> + assertThat(value["\$ref"].asText()).describedAs(key).isEqualTo("#/definitions/pipeline/properties/$key") + } + } + + @Test + fun describesNestedObjects(){ + assertClass(definition("mapping"), Mapping::class) + assertClass(definition("scan"), ScanOptions::class) + assertClass(definition("regexRule"), RegexPath.Rule::class) + assertStrictObject(definition("mappingTable"), TABLE_KEYS.toList()) + assertThat(required(definition("mappingTable"))).containsExactlyInAnyOrder("input", "output", "entries") + assertThat(RULE_KEYS).isEqualTo(parameters(RegexPath.Rule::class).toSet()) + assertThat(resolve(definition("mappingTable")["properties"]["paths"]["items"])).isSameAs(definition("regexRule")) + } + + @ParameterizedTest + @MethodSource("polymorphicTypes") + fun describesEveryType(base: KClass<*>, name: String){ + val subTypes = base.findAnnotation()!!.value.associate { it.name to it.value } + val definition = definition(name) + assertThat(enumValues(definition["properties"]["type"])).containsExactlyInAnyOrderElementsOf(subTypes.keys) + assertThat(required(definition)).containsExactly("type") + + val variants = definition["oneOf"].map(::resolve).associateBy { it["properties"]["type"]["const"].asText() } + assertThat(variants.keys).containsExactlyInAnyOrderElementsOf(subTypes.keys) + subTypes.forEach { (type, kClass) -> + val variant = variants.getValue(type) + assertClass(variant, kClass, "type") + assertThat(definition(type)).describedAs("definition named after the type").isSameAs(variant) + } + } + + @Test + fun listsEnumValues(){ + assertThat(enumValues(definition("operation"))).containsExactlyElementsOf(Operation.entries.map { it.name }) + assertThat(enumValues(definition("conflict"))).containsExactlyElementsOf(ConflictStrategy.entries.map { it.name }) + assertThat(enumValues(definition("verify"))).containsExactlyElementsOf(Verification.entries.map { it.name }) + assertThat(definition("verify")["default"].asText()).isEqualTo(Pipeline().verify.name) + assertThat(enumValues(definition("processor"))).containsExactlyElementsOf(ProcessorType.entries.map { it.name }) + assertThat(resolve(definition("Convert")["properties"]["processors"]["items"])).isSameAs(definition("processor")) + listOf(definition("Convert")["properties"]["unhandled"], definition("RegexPath")["properties"]["unmatched"]).forEach { + assertThat(enumValues(it)).containsExactlyElementsOf(UnhandledStrategy.entries.map { value -> value.name }) + } + } + + @Test + fun resolvesEveryReference(){ + fun visit(node: JsonNode){ + node["\$ref"]?.let { assertThat(resolve(node)).describedAs(it.asText()).isNotNull() } + node.forEach(::visit) + } + visit(schema) + } + + private fun definition(name: String): JsonNode = schema["definitions"][name] + + private fun resolve(node: JsonNode): JsonNode = schema.at(node["\$ref"].asText().removePrefix("#")) + + private fun properties(node: JsonNode): Map = node["properties"].properties().associate { it.key to it.value } + + private fun required(node: JsonNode): List = node["required"]?.map { it.asText() } ?: emptyList() + + private fun enumValues(node: JsonNode): List = node["enum"].map { it.asText() } + + private fun assertStrictObject(node: JsonNode, keys: List){ + assertThat(properties(node).keys).containsExactlyInAnyOrderElementsOf(keys) + assertThat(node["additionalProperties"].booleanValue()).describedAs("additionalProperties").isFalse() + } + + /** + * The object must have exactly the parameters of the class (plus [extra]), and require the mandatory ones. + */ + private fun assertClass(node: JsonNode, kClass: KClass<*>, vararg extra: String){ + val parameters = kClass.primaryConstructor!!.parameters + assertStrictObject(node, parameters.map { it.name!! } + extra) + assertThat(required(node)).describedAs("required of ${kClass.simpleName}") + .containsExactlyInAnyOrderElementsOf(parameters.filter { !it.isOptional && !it.type.isMarkedNullable }.map { it.name!! } + extra) + } + + private fun parameters(kClass: KClass<*>): List = kClass.primaryConstructor!!.parameters.map { it.name!! } + + companion object{ + val SCHEMA_PATH: Path = Path.of("config.schema.json") + + @JvmStatic + fun polymorphicTypes(): List = listOf( + Arguments.of(Stage::class, "stage"), + Arguments.of(FolderAction::class, "folderAction"), + Arguments.of(ProbeFilter::class, "probeFilter"), + ) + } +} diff --git a/src/test/kotlin/fr/rakambda/filesecure/config/ConfigSchemaValidationTest.kt b/src/test/kotlin/fr/rakambda/filesecure/config/ConfigSchemaValidationTest.kt new file mode 100644 index 0000000..586d3cf --- /dev/null +++ b/src/test/kotlin/fr/rakambda/filesecure/config/ConfigSchemaValidationTest.kt @@ -0,0 +1,152 @@ +package fr.rakambda.filesecure.config + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.TextNode +import com.networknt.schema.InputFormat +import com.networknt.schema.SchemaRegistry +import com.networknt.schema.SpecificationVersion +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import java.nio.file.Files +import java.nio.file.Path +import java.util.Locale +import kotlin.io.path.extension +import kotlin.io.path.readText + +/** + * Validates configurations against `config.schema.json` as an editor would: anchors and merges resolved, before `defaults` and `mappingTable` are + * applied. + */ +class ConfigSchemaValidationTest{ + @Test + fun acceptsReadmeExamples(){ + val blocks = README_BLOCK.findAll(Path.of("README.md").readText()).map { it.groupValues[1] }.toList() + assertThat(blocks).hasSizeGreaterThanOrEqualTo(5) + blocks.forEach { block -> + val tree = readTree(block).let { if(it.properties().all { (key, _) -> key in TOP_LEVEL_KEYS }) it else wrapInPipelines(it) } + assertThat(errors(tree)).describedAs(block).isEmpty() + assertThatCode { Configuration.parse(tree.toString(), json = true) }.describedAs(block).doesNotThrowAnyException() + } + } + + @Test + fun acceptsConfigurationsOfTests(){ + val samples = Files.walk(Path.of("src/test/kotlin")).use { files -> + files.filter { it.extension == "kt" && it.fileName.toString() != "ConfigSchemaValidationTest.kt" }.toList() + }.flatMap { file -> PARSED_SAMPLE.findAll(file.readText()).map { it.groupValues[1].replace("\${'$'}", "$").trimIndent() } } + .filter { runCatching { Configuration.parse(it) }.isSuccess } + assertThat(samples).hasSizeGreaterThanOrEqualTo(15) + samples.forEach { sample -> + assertThat(errors(canonicalEnums(readTree(sample)))).describedAs(sample).isEmpty() + } + } + + @Test + fun flagsLowercaseEnumValuesAcceptedByFileSecure(){ + val content = "pipelines:\n - mappings: [{input: /a, output: /b}]\n operation: move" + + assertThat(Configuration.parse(content).pipelines.single().operation).isEqualTo(Operation.MOVE) + assertThat(errors(readTree(content))).isNotEmpty() + } + + @Test + fun acceptsVerify(){ + listOf( + "pipelines:\n - mappings: [{input: /a, output: /b}]\n verify: CHECKSUM", + "pipelines:\n - mappings: [{input: /a, output: /b}]\n verify: SIZE", + "defaults: {verify: CHECKSUM}\npipelines:\n - mappings: [{input: /a, output: /b}]", + ).forEach { content -> + assertThat(errors(readTree(content))).describedAs(content).isEmpty() + assertThatCode { Configuration.parse(content) }.describedAs(content).doesNotThrowAnyException() + } + } + + @Test + fun rejectsInvalidConfigurations(){ + val pipeline = "pipelines:\n - mappings: [{input: /a, output: /b}]\n" + listOf( + "pipeline: []", + "useDefaults: false\n$pipeline", + "${pipeline} depth: 2", + "${pipeline} templates: {a: b}", + "${pipeline} scan: {useDefaults: false}", + "${pipeline} scan: {maxDepth: -1}", + "${pipeline} operation: DELETE", + "${pipeline} conflict: OVERWRITE", + "${pipeline} verify: MD5", + "${pipeline} verify: true", + "defaults: {verify: SHA256}\npipelines: []", + "${pipeline} enabled: maybe", + "pipelines:\n - mappings: [{input: /a}]", + "pipelines:\n - mappings: [{input: /a, output: /b, depth: 1}]", + "${pipeline} stages: [{type: NameFilter, exclude: [a]}]", + "${pipeline} stages: [{type: FolderPerCentury}]", + "${pipeline} stages: [{exclude: [a]}]", + "${pipeline} stages: [{type: FolderPerYear, depth: 1}]", + "${pipeline} stages: [{type: FolderCanonical}]", + "${pipeline} stages: [{type: IfOlderThan, dayOffset: two}]", + "${pipeline} stages: [{type: RegexPath}]", + "${pipeline} stages: [{type: RegexPath, rules: [{regex: a}]}]", + "${pipeline} stages: [{type: RegexPath, rules: [{regex: a, replacement: b}], unmatched: IGNORE}]", + "${pipeline} stages: [{type: RenameWithDatePattern, regex: '(?.*) (?