diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2082904a2b28..49c7f161190e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,3 +12,4 @@ /java-spanner/ @googleapis/spanner-team @googleapis/cloud-sdk-java-team /java-spanner-jdbc/ @googleapis/spanner-team @googleapis/cloud-sdk-java-team /google-auth-library-java/ @googleapis/cloud-sdk-auth-team @googleapis/cloud-sdk-java-team +/java-storage-nio/ @googleapis/gcs-team @googleapis/cloud-sdk-java-team diff --git a/.github/workflows/java-storage-nio-ci.yaml b/.github/workflows/java-storage-nio-ci.yaml new file mode 100644 index 000000000000..56d39506143c --- /dev/null +++ b/.github/workflows/java-storage-nio-ci.yaml @@ -0,0 +1,139 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Github action job to test core java library features on +# downstream client libraries before they are released. +on: + push: + branches: + - main + pull_request: +name: java-storage-nio ci +env: + BUILD_SUBDIR: java-storage-nio +jobs: + filter: + runs-on: ubuntu-latest + outputs: + library: ${{ steps.filter.outputs.library }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + library: + - 'java-storage-nio/**' + units: + needs: filter + if: ${{ needs.filter.outputs.library == 'true' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + java: [11, 17, 21, 25] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{matrix.java}} + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: test + units-java8: + needs: filter + if: ${{ needs.filter.outputs.library == 'true' }} + # Building using Java 17 and run the tests with Java 8 runtime + name: "units (8)" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: temurin + - name: "Set jvm system property environment variable for surefire plugin (unit tests)" + # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests. + # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm + run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java -P !java17" >> $GITHUB_ENV + shell: bash + - uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: temurin + - run: .kokoro/build.sh + env: + JOB_TYPE: test + windows: + needs: filter + if: ${{ needs.filter.outputs.library == 'true' }} + runs-on: windows-latest + steps: + - name: Support longpaths + run: git config --system core.longpaths true + - name: Support longpaths + run: git config --system core.longpaths true + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: test + dependencies: + needs: filter + if: ${{ needs.filter.outputs.library == 'true' }} + runs-on: ubuntu-latest + strategy: + matrix: + java: [17] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{matrix.java}} + - run: java -version + - run: .kokoro/dependencies.sh + javadoc: + needs: filter + if: ${{ needs.filter.outputs.library == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: javadoc + lint: + needs: filter + if: ${{ needs.filter.outputs.library == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: lint diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 3b3c8745febc..62085b6ddb97 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -32,6 +32,7 @@ excluded_modules=( 'java-spanner' 'java-spanner-jdbc' 'google-auth-library-java' + 'java-storage-nio' ) function retry_with_backoff { diff --git a/.kokoro/presubmit/storage-nio-graalvm-native-presubmit.cfg b/.kokoro/presubmit/storage-nio-graalvm-native-presubmit.cfg new file mode 100644 index 000000000000..a729b538d1d1 --- /dev/null +++ b/.kokoro/presubmit/storage-nio-graalvm-native-presubmit.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.58.0" # {x-version-update:google-cloud-shared-dependencies:current} +} + +env_vars: { + key: "JOB_TYPE" + value: "graalvm-single" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "BUILD_SUBDIR" + value: "java-storage-nio" +} diff --git a/.kokoro/presubmit/storage-nio-integration.cfg b/.kokoro/presubmit/storage-nio-integration.cfg new file mode 100644 index 000000000000..5f988c285c12 --- /dev/null +++ b/.kokoro/presubmit/storage-nio-integration.cfg @@ -0,0 +1,39 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration-single" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + + +env_vars: { + key: "BUILD_SUBDIR" + value: "java-storage-nio" +} diff --git a/generation/check_non_release_please_versions.sh b/generation/check_non_release_please_versions.sh index 633f4fd0ab9a..1841f4084fef 100755 --- a/generation/check_non_release_please_versions.sh +++ b/generation/check_non_release_please_versions.sh @@ -16,6 +16,7 @@ for pomFile in $(find . -mindepth 2 -name pom.xml | sort ); do [[ "${pomFile}" =~ .*java-spanner.* ]] || \ [[ "${pomFile}" =~ .*java-spanner-jdbc.* ]] || \ [[ "${pomFile}" =~ .*google-auth-library-java.* ]] || \ + [[ "${pomFile}" =~ .*java-storage-nio.* ]] || \ [[ "${pomFile}" =~ .*.github*. ]]; then continue fi diff --git a/java-storage-nio/.gemini/config.yaml b/java-storage-nio/.gemini/config.yaml new file mode 100644 index 000000000000..8afb84853471 --- /dev/null +++ b/java-storage-nio/.gemini/config.yaml @@ -0,0 +1,10 @@ +# https://developers.google.com/gemini-code-assist/docs/customize-gemini-behavior-github#custom-configuration +have_fun: false +code_review: + disable: false + comment_severity_threshold: HIGH + max_review_comments: -1 + pull_request_opened: + help: false + summary: false + code_review: false diff --git a/java-storage-nio/.repo-metadata.json b/java-storage-nio/.repo-metadata.json new file mode 100644 index 000000000000..fcc3b4d2a559 --- /dev/null +++ b/java-storage-nio/.repo-metadata.json @@ -0,0 +1,15 @@ +{ + "api_shortname": "storage_nio", + "name_pretty": "NIO Filesystem Provider for Google Cloud Storage", + "api_description": "provides a Google Cloud Storage extension for Java's NIO Filesystem.", + "product_documentation": "https://cloud.google.com/storage/docs", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-nio/latest/history", + "release_level": "preview", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "google-cloud-java", + "distribution_name": "com.google.cloud:google-cloud-nio", + "api_id": "storage.googleapis.com", + "library_type": "OTHER", + "codeowner_team": "@googleapis/gcs-team" +} diff --git a/java-storage-nio/CHANGELOG.md b/java-storage-nio/CHANGELOG.md new file mode 100644 index 000000000000..1760c9384a6a --- /dev/null +++ b/java-storage-nio/CHANGELOG.md @@ -0,0 +1,1423 @@ +# Changelog + +## [0.128.14](https://github.com/googleapis/java-storage-nio/compare/v0.128.13...v0.128.14) (2026-03-26) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20260204-2.0.0 ([#1750](https://github.com/googleapis/java-storage-nio/issues/1750)) ([70eef35](https://github.com/googleapis/java-storage-nio/commit/70eef3562551f9e964269284a004775cd325c2a3)) +* Update dependency com.google.cloud:google-cloud-storage to v2.64.1 ([#1752](https://github.com/googleapis/java-storage-nio/issues/1752)) ([0851342](https://github.com/googleapis/java-storage-nio/commit/08513421255f6b9dbb7ef7904f2b10f04dc87075)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.58.0 ([#1751](https://github.com/googleapis/java-storage-nio/issues/1751)) ([2cd5f71](https://github.com/googleapis/java-storage-nio/commit/2cd5f716c3c67e7a71313abf984ff17537abeab8)) + +## [0.128.13](https://github.com/googleapis/java-storage-nio/compare/v0.128.12...v0.128.13) (2026-02-27) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.64.0 ([#1745](https://github.com/googleapis/java-storage-nio/issues/1745)) ([8c3d9b2](https://github.com/googleapis/java-storage-nio/commit/8c3d9b284db15442449ef9bfe2bec137831ab8f5)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.57.0 ([#1744](https://github.com/googleapis/java-storage-nio/issues/1744)) ([9370e91](https://github.com/googleapis/java-storage-nio/commit/9370e9115196f210db2726c4a04e0c23a8f3119c)) + +## [0.128.12](https://github.com/googleapis/java-storage-nio/compare/v0.128.11...v0.128.12) (2026-02-13) + + +### Dependencies + +* Update actions/setup-node action to v6 ([#1688](https://github.com/googleapis/java-storage-nio/issues/1688)) ([f69b18e](https://github.com/googleapis/java-storage-nio/commit/f69b18edad7f9b5c661588f822322202d86aa172)) +* Update dependency com.google.cloud:google-cloud-storage to v2.63.0 ([#1739](https://github.com/googleapis/java-storage-nio/issues/1739)) ([ab6f616](https://github.com/googleapis/java-storage-nio/commit/ab6f616a9e81a56a55825bad3c633c48c4bb8edd)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.56.1 ([#1736](https://github.com/googleapis/java-storage-nio/issues/1736)) ([3dd4070](https://github.com/googleapis/java-storage-nio/commit/3dd40703954eab70c3e9f4bacfaf626b0a03f8de)) +* Update dependency node to v24 ([#1695](https://github.com/googleapis/java-storage-nio/issues/1695)) ([f2df429](https://github.com/googleapis/java-storage-nio/commit/f2df4297f292244f554a148b6aeace7380b63047)) + +## [0.128.11](https://github.com/googleapis/java-storage-nio/compare/v0.128.10...v0.128.11) (2026-01-28) + + +### Dependencies + +* Update dependency com.google.cloud:sdk-platform-java-config to v3.56.0 ([#1729](https://github.com/googleapis/java-storage-nio/issues/1729)) ([bc62ff5](https://github.com/googleapis/java-storage-nio/commit/bc62ff565e19dd97b4c25e327b94887c7bb0e8f4)) + +## [0.128.10](https://github.com/googleapis/java-storage-nio/compare/v0.128.9...v0.128.10) (2026-01-16) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.62.0 ([#1721](https://github.com/googleapis/java-storage-nio/issues/1721)) ([aea8635](https://github.com/googleapis/java-storage-nio/commit/aea86353f90e29b5da5e9409e45e8e3a27231698)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.55.1 ([#1725](https://github.com/googleapis/java-storage-nio/issues/1725)) ([4701cca](https://github.com/googleapis/java-storage-nio/commit/4701ccaa028482880f087690d48649be71e5604e)) +* Update dependency org.ow2.asm:asm to v9.9.1 ([#1717](https://github.com/googleapis/java-storage-nio/issues/1717)) ([edd4ec4](https://github.com/googleapis/java-storage-nio/commit/edd4ec46b9ad0cd365c1c06d7859611ba59d3dd9)) +* Update dependency org.ow2.asm:asm-commons to v9.9.1 ([#1718](https://github.com/googleapis/java-storage-nio/issues/1718)) ([ecf4ebe](https://github.com/googleapis/java-storage-nio/commit/ecf4ebe09d0a75178457847e5695ba239e171dc9)) + +## [0.128.9](https://github.com/googleapis/java-storage-nio/compare/v0.128.8...v0.128.9) (2025-12-15) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20251118-2.0.0 ([#1710](https://github.com/googleapis/java-storage-nio/issues/1710)) ([55c6f31](https://github.com/googleapis/java-storage-nio/commit/55c6f3147a838eefd79df9012dd6e7f789e3fbab)) +* Update dependency com.google.cloud:google-cloud-storage to v2.61.0 ([#1712](https://github.com/googleapis/java-storage-nio/issues/1712)) ([aaf92d0](https://github.com/googleapis/java-storage-nio/commit/aaf92d044542ddd6dec565207a60c18b30604aba)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.54.2 ([#1709](https://github.com/googleapis/java-storage-nio/issues/1709)) ([abf9691](https://github.com/googleapis/java-storage-nio/commit/abf9691683ed0df4605c7ed16de163806c47eec4)) + +## [0.128.8](https://github.com/googleapis/java-storage-nio/compare/v0.128.7...v0.128.8) (2025-11-10) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.60.0 ([#1698](https://github.com/googleapis/java-storage-nio/issues/1698)) ([fcd73eb](https://github.com/googleapis/java-storage-nio/commit/fcd73eb7382d259315b07f880659a4d7d6666f0e)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.54.1 ([#1696](https://github.com/googleapis/java-storage-nio/issues/1696)) ([bc5c5f4](https://github.com/googleapis/java-storage-nio/commit/bc5c5f45f65df810ac813cb2e59b7d6d5b90052c)) + +## [0.128.7](https://github.com/googleapis/java-storage-nio/compare/v0.128.6...v0.128.7) (2025-10-21) + + +### Bug Fixes + +* Remove explicit guava version from examples module ([#1685](https://github.com/googleapis/java-storage-nio/issues/1685)) ([76291ce](https://github.com/googleapis/java-storage-nio/commit/76291cefe3aba7a72d1fc1396c6a80c645a813e0)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.59.0 ([6c3e5ee](https://github.com/googleapis/java-storage-nio/commit/6c3e5eeb4b815158a86d83f5f60736ec99e9aaee)), closes [#1690](https://github.com/googleapis/java-storage-nio/issues/1690) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.53.0 ([6c3e5ee](https://github.com/googleapis/java-storage-nio/commit/6c3e5eeb4b815158a86d83f5f60736ec99e9aaee)), closes [#1689](https://github.com/googleapis/java-storage-nio/issues/1689) + +## [0.128.6](https://github.com/googleapis/java-storage-nio/compare/v0.128.5...v0.128.6) (2025-10-07) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250925-2.0.0 ([#1668](https://github.com/googleapis/java-storage-nio/issues/1668)) ([998c565](https://github.com/googleapis/java-storage-nio/commit/998c56528b521e7c3604bc13c4c04f047fa0cb42)) +* Update dependency com.google.cloud:google-cloud-storage to v2.58.1 ([#1678](https://github.com/googleapis/java-storage-nio/issues/1678)) ([021de43](https://github.com/googleapis/java-storage-nio/commit/021de436e63f774f00049f2383adffbb4e1a2fff)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.52.3 ([#1671](https://github.com/googleapis/java-storage-nio/issues/1671)) ([30984fd](https://github.com/googleapis/java-storage-nio/commit/30984fdd919a7ea173ce3875b42e1812fb589ebb)) +* Update dependency org.ow2.asm:asm to v9.9 ([#1673](https://github.com/googleapis/java-storage-nio/issues/1673)) ([30e5c66](https://github.com/googleapis/java-storage-nio/commit/30e5c66560805fa67c06e36b938eafcd543602ed)) +* Update dependency org.ow2.asm:asm-commons to v9.9 ([#1674](https://github.com/googleapis/java-storage-nio/issues/1674)) ([fc8bcb2](https://github.com/googleapis/java-storage-nio/commit/fc8bcb2bb0c70cb59a54455d0cd1f934cc6e15f0)) + +## [0.128.5](https://github.com/googleapis/java-storage-nio/compare/v0.128.4...v0.128.5) (2025-09-23) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.58.0 ([#1661](https://github.com/googleapis/java-storage-nio/issues/1661)) ([5eafd92](https://github.com/googleapis/java-storage-nio/commit/5eafd928dd57246ec462735d3cf308de6fa3e685)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.52.2 ([#1658](https://github.com/googleapis/java-storage-nio/issues/1658)) ([2a6ee3f](https://github.com/googleapis/java-storage-nio/commit/2a6ee3fddf3f53dc815a482214301560264f0e0a)) + +## [0.128.4](https://github.com/googleapis/java-storage-nio/compare/v0.128.3...v0.128.4) (2025-09-10) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.57.0 ([#1650](https://github.com/googleapis/java-storage-nio/issues/1650)) ([88e834b](https://github.com/googleapis/java-storage-nio/commit/88e834b56619f57f507090e91ad9d8dca9737872)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.52.1 ([#1647](https://github.com/googleapis/java-storage-nio/issues/1647)) ([0462132](https://github.com/googleapis/java-storage-nio/commit/0462132f9b635a134d20676d504a65d8b7ac5ceb)) + +## [0.128.3](https://github.com/googleapis/java-storage-nio/compare/v0.128.2...v0.128.3) (2025-08-25) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250815-2.0.0 ([#1637](https://github.com/googleapis/java-storage-nio/issues/1637)) ([ab95792](https://github.com/googleapis/java-storage-nio/commit/ab95792e6d6722f6e5ed2edce235099af0eda56f)) +* Update dependency com.google.cloud:google-cloud-storage to v2.56.0 ([#1641](https://github.com/googleapis/java-storage-nio/issues/1641)) ([f20e476](https://github.com/googleapis/java-storage-nio/commit/f20e476b3205e548998cf46f1ac7233e3efb3d7b)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.52.0 ([#1638](https://github.com/googleapis/java-storage-nio/issues/1638)) ([fe3f802](https://github.com/googleapis/java-storage-nio/commit/fe3f80270168251686926f2ff15949cac8a3c73b)) + +## [0.128.2](https://github.com/googleapis/java-storage-nio/compare/v0.128.1...v0.128.2) (2025-08-06) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.55.0 ([#1632](https://github.com/googleapis/java-storage-nio/issues/1632)) ([2b581e2](https://github.com/googleapis/java-storage-nio/commit/2b581e20e36a134443e8f653dc9a12d40cd7f406)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.51.0 ([#1630](https://github.com/googleapis/java-storage-nio/issues/1630)) ([811a28d](https://github.com/googleapis/java-storage-nio/commit/811a28d13c888f8b27a93100e91fea65e8f86832)) + +## [0.128.1](https://github.com/googleapis/java-storage-nio/compare/v0.128.0...v0.128.1) (2025-07-29) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250718-2.0.0 ([#1624](https://github.com/googleapis/java-storage-nio/issues/1624)) ([7a84881](https://github.com/googleapis/java-storage-nio/commit/7a84881ec533e9f9afcac1d342e794784d0d113b)) +* Update dependency com.google.cloud:google-cloud-storage to v2.54.0 ([#1626](https://github.com/googleapis/java-storage-nio/issues/1626)) ([b3a90b6](https://github.com/googleapis/java-storage-nio/commit/b3a90b687fe8f91af3f610252e83da2986b9ed52)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.50.2 ([#1623](https://github.com/googleapis/java-storage-nio/issues/1623)) ([9861f8b](https://github.com/googleapis/java-storage-nio/commit/9861f8b3c5549941744c2bfe65afa5089633a3bf)) + +## [0.128.0](https://github.com/googleapis/java-storage-nio/compare/v0.127.38...v0.128.0) (2025-07-11) + + +### Features + +* Implement CloudStorageFileSystemProvider.move method using the moveBlob API ([#1610](https://github.com/googleapis/java-storage-nio/issues/1610)) ([bdd785e](https://github.com/googleapis/java-storage-nio/commit/bdd785ee34ba3fe5a63ff7edba9ab1b63ed9e383)) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250629-2.0.0 ([#1616](https://github.com/googleapis/java-storage-nio/issues/1616)) ([bdc7489](https://github.com/googleapis/java-storage-nio/commit/bdc7489fc2c97651bd04a7dd115dfd4bc6083518)) +* Update dependency com.google.cloud:google-cloud-storage to v2.53.3 ([#1619](https://github.com/googleapis/java-storage-nio/issues/1619)) ([f4b1105](https://github.com/googleapis/java-storage-nio/commit/f4b11054cf96a6f561d554c1536381462abff379)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.50.1 ([#1617](https://github.com/googleapis/java-storage-nio/issues/1617)) ([2d4aec1](https://github.com/googleapis/java-storage-nio/commit/2d4aec1e948ceede870925ce80603f72a950b2f7)) + +## [0.127.38](https://github.com/googleapis/java-storage-nio/compare/v0.127.37...v0.127.38) (2025-06-26) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250605-2.0.0 ([#1604](https://github.com/googleapis/java-storage-nio/issues/1604)) ([36fb6a9](https://github.com/googleapis/java-storage-nio/commit/36fb6a9a4bd56055d647d7caee330dbf824d95e5)) +* Update dependency com.google.cloud:google-cloud-storage to v2.53.2 ([#1608](https://github.com/googleapis/java-storage-nio/issues/1608)) ([b365a55](https://github.com/googleapis/java-storage-nio/commit/b365a55d4dcbf7f5b9fe52a8578952fa66baeb90)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.49.2 ([#1607](https://github.com/googleapis/java-storage-nio/issues/1607)) ([1c371e7](https://github.com/googleapis/java-storage-nio/commit/1c371e721053980795cc8655be1241c1d0e9792a)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.50.0 ([#1611](https://github.com/googleapis/java-storage-nio/issues/1611)) ([18959ca](https://github.com/googleapis/java-storage-nio/commit/18959ca3f50cd838a7d4e4a1a174d4fc0b472bc6)) + +## [0.127.37](https://github.com/googleapis/java-storage-nio/compare/v0.127.36...v0.127.37) (2025-06-04) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250521-2.0.0 ([#1596](https://github.com/googleapis/java-storage-nio/issues/1596)) ([4a08c52](https://github.com/googleapis/java-storage-nio/commit/4a08c5290aaf8430ed86093b9eb7fac38b5fb681)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250524-2.0.0 ([#1598](https://github.com/googleapis/java-storage-nio/issues/1598)) ([c3af63f](https://github.com/googleapis/java-storage-nio/commit/c3af63f95e0ff65a33ce15ced5ce027e91f12025)) +* Update dependency com.google.cloud:google-cloud-storage to v2.53.0 ([#1600](https://github.com/googleapis/java-storage-nio/issues/1600)) ([d6fea9e](https://github.com/googleapis/java-storage-nio/commit/d6fea9e88e5cbe72aa900bb6994303c0adf5604c)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.49.0 ([#1599](https://github.com/googleapis/java-storage-nio/issues/1599)) ([87fb221](https://github.com/googleapis/java-storage-nio/commit/87fb2210465de19b5626fc69431eceefa4738c5e)) + +## [0.127.36](https://github.com/googleapis/java-storage-nio/compare/v0.127.35...v0.127.36) (2025-05-19) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250509-2.0.0 ([#1590](https://github.com/googleapis/java-storage-nio/issues/1590)) ([c339845](https://github.com/googleapis/java-storage-nio/commit/c3398452201f6d12f1b74cb4071482980888e7d8)) +* Update dependency com.google.cloud:google-cloud-storage to v2.52.3 ([#1592](https://github.com/googleapis/java-storage-nio/issues/1592)) ([7273afe](https://github.com/googleapis/java-storage-nio/commit/7273afe803cbdbba6a3ee3126e8393f746be73ea)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.48.0 ([#1589](https://github.com/googleapis/java-storage-nio/issues/1589)) ([b15e10f](https://github.com/googleapis/java-storage-nio/commit/b15e10fcad77628f3f1240b6198dc8de5791d998)) + +## [0.127.35](https://github.com/googleapis/java-storage-nio/compare/v0.127.34...v0.127.35) (2025-05-06) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250424-2.0.0 ([#1580](https://github.com/googleapis/java-storage-nio/issues/1580)) ([e01241a](https://github.com/googleapis/java-storage-nio/commit/e01241a18c703c60fa194f1ca61711aa56c40e7f)) +* Update dependency com.google.cloud:google-cloud-storage to v2.52.2 ([#1584](https://github.com/googleapis/java-storage-nio/issues/1584)) ([5f62a77](https://github.com/googleapis/java-storage-nio/commit/5f62a779e0079567e31db0d2fcc566fd110a4b55)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.47.0 ([#1583](https://github.com/googleapis/java-storage-nio/issues/1583)) ([7dc52c4](https://github.com/googleapis/java-storage-nio/commit/7dc52c47a7960de1fa8a23078a38d15164c553f4)) + +## [0.127.34](https://github.com/googleapis/java-storage-nio/compare/v0.127.33...v0.127.34) (2025-04-24) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250312-2.0.0 ([#1561](https://github.com/googleapis/java-storage-nio/issues/1561)) ([ff7d9df](https://github.com/googleapis/java-storage-nio/commit/ff7d9dfc168fa12798434675a59b28d992eddad3)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250416-2.0.0 ([#1575](https://github.com/googleapis/java-storage-nio/issues/1575)) ([b900a09](https://github.com/googleapis/java-storage-nio/commit/b900a096380e06a5b3f5ca9ca41f4e88a99d7b18)) +* Update dependency com.google.cloud:google-cloud-storage to v2.51.0 ([#1576](https://github.com/googleapis/java-storage-nio/issues/1576)) ([2ff82dd](https://github.com/googleapis/java-storage-nio/commit/2ff82dd285d923dcc3530c376f6d7e8a602aace0)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.46.0 ([#1572](https://github.com/googleapis/java-storage-nio/issues/1572)) ([01ae240](https://github.com/googleapis/java-storage-nio/commit/01ae24029abaeaeab7b3d625bf64e076c4fde9f3)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.46.1 ([#1573](https://github.com/googleapis/java-storage-nio/issues/1573)) ([9d87790](https://github.com/googleapis/java-storage-nio/commit/9d87790a48fc240d51895b8d437ba012b7e6e8c6)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.46.2 ([#1574](https://github.com/googleapis/java-storage-nio/issues/1574)) ([4c4e91d](https://github.com/googleapis/java-storage-nio/commit/4c4e91d6923723e961421a9767630a31ab0a5912)) +* Update dependency com.google.guava:guava to v33.4.5-android ([#1563](https://github.com/googleapis/java-storage-nio/issues/1563)) ([211e5e8](https://github.com/googleapis/java-storage-nio/commit/211e5e88e00293b72e355ed54d2e4917708bc7d3)) +* Update dependency com.google.guava:guava to v33.4.6-android ([#1566](https://github.com/googleapis/java-storage-nio/issues/1566)) ([38ef153](https://github.com/googleapis/java-storage-nio/commit/38ef15378468fd979cd49eeb4a2c65f1e6f3f07e)) +* Update dependency com.google.guava:guava to v33.4.8-android ([#1570](https://github.com/googleapis/java-storage-nio/issues/1570)) ([47b9ad4](https://github.com/googleapis/java-storage-nio/commit/47b9ad4f3413bb0a5a6d95e659497ae5f542cdc3)) + +## [0.127.33](https://github.com/googleapis/java-storage-nio/compare/v0.127.32...v0.127.33) (2025-03-14) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20250224-2.0.0 ([#1549](https://github.com/googleapis/java-storage-nio/issues/1549)) ([7eb41df](https://github.com/googleapis/java-storage-nio/commit/7eb41df58095ac33bac0b648dc80733e3e2143ca)) +* Update dependency com.google.cloud:google-cloud-storage to v2.50.0 ([#1556](https://github.com/googleapis/java-storage-nio/issues/1556)) ([80f3bd9](https://github.com/googleapis/java-storage-nio/commit/80f3bd9bb5b25c1ae599963b3a0192110cebe2b8)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.45.0 ([#1554](https://github.com/googleapis/java-storage-nio/issues/1554)) ([cdd06f9](https://github.com/googleapis/java-storage-nio/commit/cdd06f9689ebd4b7bf44b54a4319983884f7ad03)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.45.1 ([#1555](https://github.com/googleapis/java-storage-nio/issues/1555)) ([97c3624](https://github.com/googleapis/java-storage-nio/commit/97c3624ac7e375621d78c7eea10b00359d7971fd)) +* Update dependency node to v22 ([#1553](https://github.com/googleapis/java-storage-nio/issues/1553)) ([f4c4920](https://github.com/googleapis/java-storage-nio/commit/f4c4920c6ccdd32969a4ea1cfe004b2537ba2021)) + +## [0.127.32](https://github.com/googleapis/java-storage-nio/compare/v0.127.31...v0.127.32) (2025-02-26) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.49.0 ([#1545](https://github.com/googleapis/java-storage-nio/issues/1545)) ([09af40c](https://github.com/googleapis/java-storage-nio/commit/09af40cc2b1d359341e1dab137eabc404b35a821)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.44.0 ([#1542](https://github.com/googleapis/java-storage-nio/issues/1542)) ([a659c23](https://github.com/googleapis/java-storage-nio/commit/a659c23225d3c4326f3de916a9a63c31ae86b59b)) + +## [0.127.31](https://github.com/googleapis/java-storage-nio/compare/v0.127.30...v0.127.31) (2025-02-13) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.48.1 ([#1533](https://github.com/googleapis/java-storage-nio/issues/1533)) ([f6b9472](https://github.com/googleapis/java-storage-nio/commit/f6b9472fb1970a0ad5865f4d95b9f97c9102fd6d)) +* Update dependency com.google.cloud:google-cloud-storage to v2.48.2 ([#1537](https://github.com/googleapis/java-storage-nio/issues/1537)) ([74b2803](https://github.com/googleapis/java-storage-nio/commit/74b2803e3270ada80c004cdd86fe42c47b9b8d3d)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.43.0 ([#1536](https://github.com/googleapis/java-storage-nio/issues/1536)) ([d024538](https://github.com/googleapis/java-storage-nio/commit/d0245387989f1c371f7545807ef030df1c21c67b)) + +## [0.127.30](https://github.com/googleapis/java-storage-nio/compare/v0.127.29...v0.127.30) (2025-01-28) + + +### Bug Fixes + +* Update FakeStorageRpc to be able to handle incremental resumable upload puts ([#1527](https://github.com/googleapis/java-storage-nio/issues/1527)) ([5ca338e](https://github.com/googleapis/java-storage-nio/commit/5ca338e24fe9fb180b34948c14e216b32bc9a4c9)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.48.0 ([#1528](https://github.com/googleapis/java-storage-nio/issues/1528)) ([b1ee43a](https://github.com/googleapis/java-storage-nio/commit/b1ee43aa29604bdea88ca491af5b5e30e2d4a2d8)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.42.0 ([#1529](https://github.com/googleapis/java-storage-nio/issues/1529)) ([006259b](https://github.com/googleapis/java-storage-nio/commit/006259b4ef962d5683d8b1f092aab8cb9d513273)) + +## [0.127.29](https://github.com/googleapis/java-storage-nio/compare/v0.127.28...v0.127.29) (2025-01-08) + + +### Bug Fixes + +* Minimize initialize at buildtime ([#1465](https://github.com/googleapis/java-storage-nio/issues/1465)) ([fbc842d](https://github.com/googleapis/java-storage-nio/commit/fbc842d42ab3d3f0bf5168e9273edff50bf5a8e0)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.47.0 ([#1522](https://github.com/googleapis/java-storage-nio/issues/1522)) ([133ab80](https://github.com/googleapis/java-storage-nio/commit/133ab803e720359fd0024464517f53483946546e)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.41.1 ([#1521](https://github.com/googleapis/java-storage-nio/issues/1521)) ([7ce2a91](https://github.com/googleapis/java-storage-nio/commit/7ce2a91ecc460df5c84e194ad3f49c42f54b05ff)) +* Update dependency com.google.guava:guava to v33.4.0-android ([#1519](https://github.com/googleapis/java-storage-nio/issues/1519)) ([c496bc6](https://github.com/googleapis/java-storage-nio/commit/c496bc657f0fc1ef93754d0df7df1518cafdc3d6)) + +## [0.127.28](https://github.com/googleapis/java-storage-nio/compare/v0.127.27...v0.127.28) (2024-12-13) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20241206-2.0.0 ([#1512](https://github.com/googleapis/java-storage-nio/issues/1512)) ([ef94ecb](https://github.com/googleapis/java-storage-nio/commit/ef94ecb85fcd326da3449202a84a0ba9129045b2)) +* Update dependency com.google.cloud:google-cloud-storage to v2.46.0 ([#1514](https://github.com/googleapis/java-storage-nio/issues/1514)) ([63f1a16](https://github.com/googleapis/java-storage-nio/commit/63f1a16e0277bf78f5ebd4de6ab0c4c586ac192d)) + +## [0.127.27](https://github.com/googleapis/java-storage-nio/compare/v0.127.26...v0.127.27) (2024-11-19) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20241113-2.0.0 ([#1506](https://github.com/googleapis/java-storage-nio/issues/1506)) ([8400c67](https://github.com/googleapis/java-storage-nio/commit/8400c674938adddf4d5156f136a3ee00771ba151)) +* Update dependency com.google.cloud:google-cloud-storage to v2.45.0 ([#1505](https://github.com/googleapis/java-storage-nio/issues/1505)) ([623c960](https://github.com/googleapis/java-storage-nio/commit/623c9600c1b027e9e39c592860d26a8466fe0eba)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.40.0 ([#1504](https://github.com/googleapis/java-storage-nio/issues/1504)) ([461583f](https://github.com/googleapis/java-storage-nio/commit/461583fab0f7aa8792a8162e5fb026b1ce5d3fd3)) + +## [0.127.26](https://github.com/googleapis/java-storage-nio/compare/v0.127.25...v0.127.26) (2024-10-29) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20241008-2.0.0 ([#1491](https://github.com/googleapis/java-storage-nio/issues/1491)) ([110796b](https://github.com/googleapis/java-storage-nio/commit/110796b7485daefcb5509388954a1efa74fb0c76)) +* Update dependency com.google.cloud:google-cloud-storage to v2.44.0 ([#1495](https://github.com/googleapis/java-storage-nio/issues/1495)) ([5042da0](https://github.com/googleapis/java-storage-nio/commit/5042da072103a4718ca6e091017950d76fdde41d)) +* Update dependency com.google.cloud:google-cloud-storage to v2.44.1 ([#1498](https://github.com/googleapis/java-storage-nio/issues/1498)) ([7354b85](https://github.com/googleapis/java-storage-nio/commit/7354b850fe881f6906a38f466146363c5374b8a7)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.38.0 ([#1493](https://github.com/googleapis/java-storage-nio/issues/1493)) ([67fed77](https://github.com/googleapis/java-storage-nio/commit/67fed7703b4cd24cb44b94287458f6f0377b29d6)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.39.0 ([#1497](https://github.com/googleapis/java-storage-nio/issues/1497)) ([f207c0e](https://github.com/googleapis/java-storage-nio/commit/f207c0e78ddfd50e31aaf1fa5ef58033138762cc)) + +## [0.127.25](https://github.com/googleapis/java-storage-nio/compare/v0.127.24...v0.127.25) (2024-10-09) + + +### Dependencies + +* Update dependency com.google.cloud:sdk-platform-java-config to v3.37.0 ([#1484](https://github.com/googleapis/java-storage-nio/issues/1484)) ([5281b99](https://github.com/googleapis/java-storage-nio/commit/5281b99bbec8d72c5cc7657aece2cefd786bb973)) +* Update dependency org.ow2.asm:asm to v9.7.1 ([#1485](https://github.com/googleapis/java-storage-nio/issues/1485)) ([dd8d9f6](https://github.com/googleapis/java-storage-nio/commit/dd8d9f6cbca5c374d5abd7a37de669db6e69008a)) +* Update dependency org.ow2.asm:asm-commons to v9.7.1 ([#1486](https://github.com/googleapis/java-storage-nio/issues/1486)) ([0001634](https://github.com/googleapis/java-storage-nio/commit/00016341bbd3277d0271e08165fdb5fbb87b7cc1)) + +## [0.127.24](https://github.com/googleapis/java-storage-nio/compare/v0.127.23...v0.127.24) (2024-09-30) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240916-2.0.0 ([#1473](https://github.com/googleapis/java-storage-nio/issues/1473)) ([e66ad2d](https://github.com/googleapis/java-storage-nio/commit/e66ad2d94f4d282ef48fdef1312dc1a0444b91b9)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240924-2.0.0 ([#1478](https://github.com/googleapis/java-storage-nio/issues/1478)) ([a382594](https://github.com/googleapis/java-storage-nio/commit/a38259400cc46ab3154a2d5c4db9fc0aa98ce4fd)) +* Update dependency com.google.cloud:google-cloud-storage to v2.43.1 ([#1476](https://github.com/googleapis/java-storage-nio/issues/1476)) ([21b832c](https://github.com/googleapis/java-storage-nio/commit/21b832ce178190b0ef71aec67dd7af4157651ded)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.36.1 ([#1474](https://github.com/googleapis/java-storage-nio/issues/1474)) ([e7f8d41](https://github.com/googleapis/java-storage-nio/commit/e7f8d41ba3f3820a4012407cc7983deda8f5d6da)) +* Update dependency com.google.guava:guava to v33.3.1-android ([#1475](https://github.com/googleapis/java-storage-nio/issues/1475)) ([b3453fe](https://github.com/googleapis/java-storage-nio/commit/b3453fe43d3c6d14d986527f20c68f86f22034cb)) +* Update dependency ubuntu to v24 ([#1477](https://github.com/googleapis/java-storage-nio/issues/1477)) ([2b5b871](https://github.com/googleapis/java-storage-nio/commit/2b5b871bda84eab953b0fe0c6f3b5e398c9fa111)) + +## [0.127.23](https://github.com/googleapis/java-storage-nio/compare/v0.127.22...v0.127.23) (2024-09-17) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240819-2.0.0 ([#1460](https://github.com/googleapis/java-storage-nio/issues/1460)) ([bf6cfdc](https://github.com/googleapis/java-storage-nio/commit/bf6cfdcc59faf668ceb82b6cb81f62267f2fe2c9)) +* Update dependency com.google.cloud:google-cloud-storage to v2.43.0 ([#1468](https://github.com/googleapis/java-storage-nio/issues/1468)) ([4e2963c](https://github.com/googleapis/java-storage-nio/commit/4e2963cd3ad0b11d145658f61db9edf261efc2f3)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.35.0 ([#1466](https://github.com/googleapis/java-storage-nio/issues/1466)) ([6fe3708](https://github.com/googleapis/java-storage-nio/commit/6fe3708536986b4beb157d6f41fcb3cc6897edb7)) + +## [0.127.22](https://github.com/googleapis/java-storage-nio/compare/v0.127.21...v0.127.22) (2024-08-19) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240809-2.0.0 ([#1451](https://github.com/googleapis/java-storage-nio/issues/1451)) ([117bee9](https://github.com/googleapis/java-storage-nio/commit/117bee90559d97e84df7933227bdbccc47cb2dc4)) +* Update dependency com.google.cloud:google-cloud-storage to v2.42.0 ([#1457](https://github.com/googleapis/java-storage-nio/issues/1457)) ([416fc23](https://github.com/googleapis/java-storage-nio/commit/416fc234e06ef6c42f0ecfdd51004537d9fb29ed)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.34.0 ([#1454](https://github.com/googleapis/java-storage-nio/issues/1454)) ([01f4a81](https://github.com/googleapis/java-storage-nio/commit/01f4a810dda11b27be35cc8b03bad7f427a1abf2)) +* Update dependency com.google.guava:guava to v33.3.0-android ([#1455](https://github.com/googleapis/java-storage-nio/issues/1455)) ([8dce385](https://github.com/googleapis/java-storage-nio/commit/8dce385ffd4577c0424f5d54d91f5f980cc83972)) + +## [0.127.21](https://github.com/googleapis/java-storage-nio/compare/v0.127.20...v0.127.21) (2024-08-05) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240625-2.0.0 ([#1433](https://github.com/googleapis/java-storage-nio/issues/1433)) ([e7dba8b](https://github.com/googleapis/java-storage-nio/commit/e7dba8ba31fb0616afd366da7976291cda5897e0)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240706-2.0.0 ([#1438](https://github.com/googleapis/java-storage-nio/issues/1438)) ([ac99294](https://github.com/googleapis/java-storage-nio/commit/ac99294b518047e35e12ad83850d0d37ca9fc937)) +* Update dependency com.google.cloud:google-cloud-storage to v2.41.0 ([#1441](https://github.com/googleapis/java-storage-nio/issues/1441)) ([11dca80](https://github.com/googleapis/java-storage-nio/commit/11dca80b94d1ea9d8a1c008a4c0dd66bfe83f3ef)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.33.0 ([#1440](https://github.com/googleapis/java-storage-nio/issues/1440)) ([0055ee0](https://github.com/googleapis/java-storage-nio/commit/0055ee08973b6b1330b2ca2c79e031f895035fe7)) + +## [0.127.20](https://github.com/googleapis/java-storage-nio/compare/v0.127.19...v0.127.20) (2024-06-27) + + +### Bug Fixes + +* Update CloudStorageFileSystemProvider#getFileAttributeView to return null rather than throw UnsupportedOperationException ([#1427](https://github.com/googleapis/java-storage-nio/issues/1427)) ([b9e0362](https://github.com/googleapis/java-storage-nio/commit/b9e03621af8c9a24a45ec288cc974719b3bf63d9)) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240524-2.0.0 ([#1410](https://github.com/googleapis/java-storage-nio/issues/1410)) ([011a78f](https://github.com/googleapis/java-storage-nio/commit/011a78f6c0b2dfb51169d1e690a33c4ddde57615)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240621-2.0.0 ([#1425](https://github.com/googleapis/java-storage-nio/issues/1425)) ([2cc3efc](https://github.com/googleapis/java-storage-nio/commit/2cc3efc8a7f17caf4916c7a6ca6803627c26164d)) +* Update dependency com.google.cloud:google-cloud-storage to v2.40.1 ([#1429](https://github.com/googleapis/java-storage-nio/issues/1429)) ([cbeadb8](https://github.com/googleapis/java-storage-nio/commit/cbeadb87b2b24d5a1e6eff1613845fa94c39efa0)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.32.0 ([#1426](https://github.com/googleapis/java-storage-nio/issues/1426)) ([3a4bb4f](https://github.com/googleapis/java-storage-nio/commit/3a4bb4f100f46625e5ca9c43902d6220b0a6617b)) + +## [0.127.19](https://github.com/googleapis/java-storage-nio/compare/v0.127.18...v0.127.19) (2024-06-11) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.40.0 ([#1417](https://github.com/googleapis/java-storage-nio/issues/1417)) ([aba4f07](https://github.com/googleapis/java-storage-nio/commit/aba4f072b66a06fa4c36da048a28d51d32a806bd)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.31.0 ([#1414](https://github.com/googleapis/java-storage-nio/issues/1414)) ([53b46f8](https://github.com/googleapis/java-storage-nio/commit/53b46f8c98c2f40dab7ee4f6d7a2d95a480400ff)) +* Update dependency com.google.guava:guava to v33.2.1-android ([#1413](https://github.com/googleapis/java-storage-nio/issues/1413)) ([3fba0cf](https://github.com/googleapis/java-storage-nio/commit/3fba0cff64a9c74e3ffbec9792255898efe1954d)) + +## [0.127.18](https://github.com/googleapis/java-storage-nio/compare/v0.127.17...v0.127.18) (2024-05-23) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.39.0 ([#1406](https://github.com/googleapis/java-storage-nio/issues/1406)) ([c240857](https://github.com/googleapis/java-storage-nio/commit/c24085763cc46f340c716d5a520ada9ab1f90f00)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.30.1 ([#1403](https://github.com/googleapis/java-storage-nio/issues/1403)) ([3723265](https://github.com/googleapis/java-storage-nio/commit/37232653b5cb597f478d78a4b5d89defc4ba4ae0)) + +## [0.127.17](https://github.com/googleapis/java-storage-nio/compare/v0.127.16...v0.127.17) (2024-05-09) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.38.0 ([#1397](https://github.com/googleapis/java-storage-nio/issues/1397)) ([85ee286](https://github.com/googleapis/java-storage-nio/commit/85ee2868ff885fa8c5d1a2f2bd27cec800f83b1e)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.30.0 ([#1396](https://github.com/googleapis/java-storage-nio/issues/1396)) ([19df4c5](https://github.com/googleapis/java-storage-nio/commit/19df4c56045b71cec71b4eb6bc7b008e1bb4a79d)) +* Update dependency com.google.guava:guava to v33.2.0-android ([#1395](https://github.com/googleapis/java-storage-nio/issues/1395)) ([4cae062](https://github.com/googleapis/java-storage-nio/commit/4cae062b288534be2b9a24044235543f9446e742)) + +## [0.127.16](https://github.com/googleapis/java-storage-nio/compare/v0.127.15...v0.127.16) (2024-04-19) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240319-2.0.0 ([#1381](https://github.com/googleapis/java-storage-nio/issues/1381)) ([203aca7](https://github.com/googleapis/java-storage-nio/commit/203aca783bf3cf54c64bda5ccffbbcb94de51325)) +* Update dependency com.google.cloud:google-cloud-storage to v2.37.0 ([#1387](https://github.com/googleapis/java-storage-nio/issues/1387)) ([ed6f938](https://github.com/googleapis/java-storage-nio/commit/ed6f938601ccb818e1fe9a2bcfb354961f09081b)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.29.0 ([#1386](https://github.com/googleapis/java-storage-nio/issues/1386)) ([ee05692](https://github.com/googleapis/java-storage-nio/commit/ee05692d0a3dbca9445678180290b72c336cdbb9)) +* Update dependency com.google.guava:guava to v33.1.0-android ([#1375](https://github.com/googleapis/java-storage-nio/issues/1375)) ([a5c86a9](https://github.com/googleapis/java-storage-nio/commit/a5c86a9f13941f87492ed8a9b273268adca2f84f)) + +## [0.127.15](https://github.com/googleapis/java-storage-nio/compare/v0.127.14...v0.127.15) (2024-03-28) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240311-2.0.0 ([#1374](https://github.com/googleapis/java-storage-nio/issues/1374)) ([528fc3a](https://github.com/googleapis/java-storage-nio/commit/528fc3a91d51e250db027bc84d91b5468ec115cb)) +* Update dependency com.google.cloud:google-cloud-storage to v2.36.1 ([#1377](https://github.com/googleapis/java-storage-nio/issues/1377)) ([1e16a57](https://github.com/googleapis/java-storage-nio/commit/1e16a57dfe1d911605af2fbb1ff4b628a7924171)) +* Update dependency org.ow2.asm:asm to v9.7 ([#1378](https://github.com/googleapis/java-storage-nio/issues/1378)) ([9b6221c](https://github.com/googleapis/java-storage-nio/commit/9b6221cdaaa5725b17f0ad17e4aca9a7d6087ad8)) +* Update dependency org.ow2.asm:asm-commons to v9.7 ([#1379](https://github.com/googleapis/java-storage-nio/issues/1379)) ([81f4517](https://github.com/googleapis/java-storage-nio/commit/81f45173c5ea7943b99945756a94d20b20061a1e)) + +## [0.127.14](https://github.com/googleapis/java-storage-nio/compare/v0.127.13...v0.127.14) (2024-03-05) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.35.0 ([#1369](https://github.com/googleapis/java-storage-nio/issues/1369)) ([86bc7d1](https://github.com/googleapis/java-storage-nio/commit/86bc7d1c3c2aab455e9c30f8edc36dac09cd1940)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.27.0 ([#1367](https://github.com/googleapis/java-storage-nio/issues/1367)) ([7c25d11](https://github.com/googleapis/java-storage-nio/commit/7c25d1179452f62c63c9f16a31abdcf88e4fd27b)) +* Update dependency com.google.guava:guava to v33 ([#1322](https://github.com/googleapis/java-storage-nio/issues/1322)) ([40b66ce](https://github.com/googleapis/java-storage-nio/commit/40b66ceaa3e1973fbb5a2206da0a0fa242956afe)) + +## [0.127.13](https://github.com/googleapis/java-storage-nio/compare/v0.127.12...v0.127.13) (2024-02-14) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240209-2.0.0 ([#1354](https://github.com/googleapis/java-storage-nio/issues/1354)) ([b194d35](https://github.com/googleapis/java-storage-nio/commit/b194d357cb4b23fe010741c07e5671bb545ffae8)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.25.0 ([#1356](https://github.com/googleapis/java-storage-nio/issues/1356)) ([2dc2886](https://github.com/googleapis/java-storage-nio/commit/2dc2886e8266184ea0be29083c3f586d813a6d40)) +* Update dependency com.google.cloud:google-cloud-storage to v2.34.0 ([#1359](https://github.com/googleapis/java-storage-nio/issues/1359)) ([41d9dfa](https://github.com/googleapis/java-storage-nio/commit/41d9dfaca7b5909f3c6a82f7e02e4e7573d8d266)) + +## [0.127.12](https://github.com/googleapis/java-storage-nio/compare/v0.127.11...v0.127.12) (2024-02-08) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240202-2.0.0 ([#1347](https://github.com/googleapis/java-storage-nio/issues/1347)) ([9b15436](https://github.com/googleapis/java-storage-nio/commit/9b154360529732f60bee717b87868173b482c8b8)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240205-2.0.0 ([#1350](https://github.com/googleapis/java-storage-nio/issues/1350)) ([cb6546b](https://github.com/googleapis/java-storage-nio/commit/cb6546b32c7a464d3d2c8849f7774a7a818dc5cd)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.24.0 ([#1345](https://github.com/googleapis/java-storage-nio/issues/1345)) ([9f80180](https://github.com/googleapis/java-storage-nio/commit/9f8018020bf3cebe1e1a507c19123ebbf74bca7d)) +* Update dependency com.google.cloud:google-cloud-storage to v2.33.0 ([#1349](https://github.com/googleapis/java-storage-nio/issues/1349)) ([6b96ad2](https://github.com/googleapis/java-storage-nio/commit/6b96ad2bae91a57b9655e2f4ca40c776103e4769)) + +## [0.127.11](https://github.com/googleapis/java-storage-nio/compare/v0.127.10...v0.127.11) (2024-01-25) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.23.0 ([#1339](https://github.com/googleapis/java-storage-nio/issues/1339)) ([46685dd](https://github.com/googleapis/java-storage-nio/commit/46685dd704dca859331e4856dc7ba9433a4cc794)) +* Update dependency com.google.cloud:google-cloud-storage to v2.32.1 ([#1341](https://github.com/googleapis/java-storage-nio/issues/1341)) ([05347f7](https://github.com/googleapis/java-storage-nio/commit/05347f759f2f130d37c4c1c9f6c41331e25ef761)) + +## [0.127.10](https://github.com/googleapis/java-storage-nio/compare/v0.127.9...v0.127.10) (2024-01-22) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.22.0 ([#1331](https://github.com/googleapis/java-storage-nio/issues/1331)) ([af36d28](https://github.com/googleapis/java-storage-nio/commit/af36d284bb8c3c36476e065460db6928d4482187)) +* Update dependency com.google.cloud:google-cloud-storage to v2.32.0 ([#1335](https://github.com/googleapis/java-storage-nio/issues/1335)) ([54b07be](https://github.com/googleapis/java-storage-nio/commit/54b07beee25a333fdd3a70fb573b701bc2ed19e1)) + +## [0.127.9](https://github.com/googleapis/java-storage-nio/compare/v0.127.8...v0.127.9) (2024-01-11) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240105-2.0.0 ([#1325](https://github.com/googleapis/java-storage-nio/issues/1325)) ([f6a4925](https://github.com/googleapis/java-storage-nio/commit/f6a49253cb4eaba4e076eb2557a64b12543486f2)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.21.0 ([#1324](https://github.com/googleapis/java-storage-nio/issues/1324)) ([5c07d98](https://github.com/googleapis/java-storage-nio/commit/5c07d988c799ec80538fc08122da8f7f6b74e3cf)) +* Update dependency com.google.cloud:google-cloud-storage to v2.31.0 ([#1326](https://github.com/googleapis/java-storage-nio/issues/1326)) ([ae3e4a1](https://github.com/googleapis/java-storage-nio/commit/ae3e4a100cca6ab834dc3240e4a2ecc86b2d9642)) + +## [0.127.8](https://github.com/googleapis/java-storage-nio/compare/v0.127.7...v0.127.8) (2023-12-06) + + +### Bug Fixes + +* Update FakeStorageRpc to better handle resumable sessions that overwrite existing objects ([#1296](https://github.com/googleapis/java-storage-nio/issues/1296)) ([8051cae](https://github.com/googleapis/java-storage-nio/commit/8051cae055815c882d46167b2c878157c1aa776b)) + + +### Dependencies + +* Update actions/github-script action to v7 ([#1299](https://github.com/googleapis/java-storage-nio/issues/1299)) ([670cc7a](https://github.com/googleapis/java-storage-nio/commit/670cc7a0677cb4f2e5614f536ad211a981579a0e)) +* Update actions/github-script action to v7 ([#1311](https://github.com/googleapis/java-storage-nio/issues/1311)) ([8cba471](https://github.com/googleapis/java-storage-nio/commit/8cba471ecbb1f18a0cc1a1c21ec98acde7ea68cb)) +* Update actions/setup-java action to v4 ([#1301](https://github.com/googleapis/java-storage-nio/issues/1301)) ([042c51c](https://github.com/googleapis/java-storage-nio/commit/042c51c692fbb42eee6dea24a950ace7aee59f7e)) +* Update actions/setup-java action to v4 ([#1312](https://github.com/googleapis/java-storage-nio/issues/1312)) ([f286d00](https://github.com/googleapis/java-storage-nio/commit/f286d00d5e22c15dff361524aef3337479ea1000)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20231117-2.0.0 ([#1303](https://github.com/googleapis/java-storage-nio/issues/1303)) ([20d0e9c](https://github.com/googleapis/java-storage-nio/commit/20d0e9cb4ab25ce8fe78cb94b5dc380a102f1bd1)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.20.0 ([#1302](https://github.com/googleapis/java-storage-nio/issues/1302)) ([5f3d552](https://github.com/googleapis/java-storage-nio/commit/5f3d552111b332a706189d61e2c17763047c3a03)) +* Update dependency com.google.cloud:google-cloud-storage to v2.30.1 ([#1306](https://github.com/googleapis/java-storage-nio/issues/1306)) ([7a29cdb](https://github.com/googleapis/java-storage-nio/commit/7a29cdb129337c7e39829d45e35ef4dd20b015bd)) +* Update dependency com.google.guava:guava to v32.1.3-android ([#1298](https://github.com/googleapis/java-storage-nio/issues/1298)) ([0f24b83](https://github.com/googleapis/java-storage-nio/commit/0f24b83c7cc55527e8db077866646b50fade8fd5)) +* Update the dependency com.google.apis:google-api-services-stora… ([#1314](https://github.com/googleapis/java-storage-nio/issues/1314)) ([b3fb38d](https://github.com/googleapis/java-storage-nio/commit/b3fb38d1b80c471b8c7c6498150942bd638541cd)) + +## [0.127.7](https://github.com/googleapis/java-storage-nio/compare/v0.127.6...v0.127.7) (2023-11-03) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20231028-2.0.0 ([#1288](https://github.com/googleapis/java-storage-nio/issues/1288)) ([97e9fcb](https://github.com/googleapis/java-storage-nio/commit/97e9fcbfbe6b6010d5eb2738b80949aa28c88e24)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.19.0 ([#1292](https://github.com/googleapis/java-storage-nio/issues/1292)) ([b4c5543](https://github.com/googleapis/java-storage-nio/commit/b4c554367ae7dacb1bb5c5a65615247fcfd63295)) +* Update dependency com.google.cloud:google-cloud-storage to v2.29.1 ([#1293](https://github.com/googleapis/java-storage-nio/issues/1293)) ([a1f9322](https://github.com/googleapis/java-storage-nio/commit/a1f93224326633ca9a23280bd820e9062c2874da)) + +## [0.127.6](https://github.com/googleapis/java-storage-nio/compare/v0.127.5...v0.127.6) (2023-10-23) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20231012-2.0.0 ([#1279](https://github.com/googleapis/java-storage-nio/issues/1279)) ([de065f1](https://github.com/googleapis/java-storage-nio/commit/de065f1174dc67bb873f074ec82f30ca4ff30811)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.18.0 ([#1283](https://github.com/googleapis/java-storage-nio/issues/1283)) ([a33eb8e](https://github.com/googleapis/java-storage-nio/commit/a33eb8ebd4e5c0e358c866a000c5dcdb53aa665b)) +* Update dependency com.google.cloud:google-cloud-storage to v2.29.0 ([#1284](https://github.com/googleapis/java-storage-nio/issues/1284)) ([72db89d](https://github.com/googleapis/java-storage-nio/commit/72db89d92e2a18b3b67dfd6dc254c03524780b1f)) + +## [0.127.5](https://github.com/googleapis/java-storage-nio/compare/v0.127.4...v0.127.5) (2023-10-12) + + +### Bug Fixes + +* Add protection against a possible null dereference issue ([#1258](https://github.com/googleapis/java-storage-nio/issues/1258)) ([ade173b](https://github.com/googleapis/java-storage-nio/commit/ade173bdc762d0fe8ac97a5e189f6802156231a7)) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230926-2.0.0 ([#1266](https://github.com/googleapis/java-storage-nio/issues/1266)) ([1cf28cb](https://github.com/googleapis/java-storage-nio/commit/1cf28cb0b86914a1a8ad9e46f31d9372796de038)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.17.0 ([#1272](https://github.com/googleapis/java-storage-nio/issues/1272)) ([876c4b5](https://github.com/googleapis/java-storage-nio/commit/876c4b531f86f0b5af01cb850a9e7116c7b1e181)) +* Update dependency com.google.cloud:google-cloud-storage to v2.28.0 ([#1275](https://github.com/googleapis/java-storage-nio/issues/1275)) ([cc93df1](https://github.com/googleapis/java-storage-nio/commit/cc93df1cda79fb3faf5dc786df01bb1b8fc2897a)) +* Update dependency org.ow2.asm:asm to v9.6 ([#1268](https://github.com/googleapis/java-storage-nio/issues/1268)) ([b216dc9](https://github.com/googleapis/java-storage-nio/commit/b216dc9606297ff430f234d922a44161175daf41)) +* Update dependency org.ow2.asm:asm-commons to v9.6 ([#1267](https://github.com/googleapis/java-storage-nio/issues/1267)) ([bfa0f30](https://github.com/googleapis/java-storage-nio/commit/bfa0f3022065c2e3c071b98c4be37c63548e97fc)) + +## [0.127.4](https://github.com/googleapis/java-storage-nio/compare/v0.127.3...v0.127.4) (2023-09-26) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230914-2.0.0 ([#1254](https://github.com/googleapis/java-storage-nio/issues/1254)) ([efe45f0](https://github.com/googleapis/java-storage-nio/commit/efe45f029dcbf318f36f5681ad10935bfcdc2808)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230922-2.0.0 ([#1259](https://github.com/googleapis/java-storage-nio/issues/1259)) ([80a7dbb](https://github.com/googleapis/java-storage-nio/commit/80a7dbbbaf523d5771d161e9df43415cee990b6d)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.16.0 ([#1257](https://github.com/googleapis/java-storage-nio/issues/1257)) ([7f6d165](https://github.com/googleapis/java-storage-nio/commit/7f6d165e04c3e3bde0416b05d06076493806c1ac)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.16.1 ([#1261](https://github.com/googleapis/java-storage-nio/issues/1261)) ([69f15c0](https://github.com/googleapis/java-storage-nio/commit/69f15c004c96fef4337d9dae30258a38fa29cad3)) +* Update dependency com.google.cloud:google-cloud-storage to v2.27.1 ([#1263](https://github.com/googleapis/java-storage-nio/issues/1263)) ([b559148](https://github.com/googleapis/java-storage-nio/commit/b559148c1e084446c31a10731bfe6810ac8b5245)) + +## [0.127.3](https://github.com/googleapis/java-storage-nio/compare/v0.127.2...v0.127.3) (2023-09-13) + + +### Dependencies + +* Update actions/checkout action to v4 ([#1245](https://github.com/googleapis/java-storage-nio/issues/1245)) ([9d0a175](https://github.com/googleapis/java-storage-nio/commit/9d0a17581020058a35c592dce895a7e601faf584)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230710-2.0.0 ([#1238](https://github.com/googleapis/java-storage-nio/issues/1238)) ([51b12aa](https://github.com/googleapis/java-storage-nio/commit/51b12aa6044407220f8461e92c673607406fd719)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230907-2.0.0 ([#1249](https://github.com/googleapis/java-storage-nio/issues/1249)) ([d362b23](https://github.com/googleapis/java-storage-nio/commit/d362b230aae976ab2ff1eb92da67f34da38fe183)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.15.0 ([#1248](https://github.com/googleapis/java-storage-nio/issues/1248)) ([173db6b](https://github.com/googleapis/java-storage-nio/commit/173db6b6f5dd83a0c25e43f28fe8bf6a28c73c9a)) +* Update dependency com.google.cloud:google-cloud-storage to v2.26.1 ([#1239](https://github.com/googleapis/java-storage-nio/issues/1239)) ([2a0866a](https://github.com/googleapis/java-storage-nio/commit/2a0866adc21f0df5370d20f942a47c5f7cd57496)) +* Update dependency com.google.cloud:google-cloud-storage to v2.27.0 ([#1250](https://github.com/googleapis/java-storage-nio/issues/1250)) ([cd51778](https://github.com/googleapis/java-storage-nio/commit/cd51778387c2635df29fabb445be19505492c9d2)) + +## [0.127.2](https://github.com/googleapis/java-storage-nio/compare/v0.127.1...v0.127.2) (2023-08-17) + + +### Bug Fixes + +* Add precondition to delete operations ([#1240](https://github.com/googleapis/java-storage-nio/issues/1240)) ([57c65c6](https://github.com/googleapis/java-storage-nio/commit/57c65c6a90106e1de384ac38d8b16cce1730894a)) + +## [0.127.1](https://github.com/googleapis/java-storage-nio/compare/v0.127.0...v0.127.1) (2023-08-08) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.14.0 ([#1231](https://github.com/googleapis/java-storage-nio/issues/1231)) ([021480d](https://github.com/googleapis/java-storage-nio/commit/021480d13c7254bf3834327cf1251557b20dbbfb)) +* Update dependency com.google.cloud:google-cloud-storage to v2.26.0 ([#1233](https://github.com/googleapis/java-storage-nio/issues/1233)) ([7ecc6a7](https://github.com/googleapis/java-storage-nio/commit/7ecc6a721d1ee3b5ecbb62c5cdccdb6e471bba02)) + +## [0.127.0](https://github.com/googleapis/java-storage-nio/compare/v0.126.19...v0.127.0) (2023-07-26) + + +### Features + +* Store bucket name in URI authority ([#1218](https://github.com/googleapis/java-storage-nio/issues/1218)) ([#1219](https://github.com/googleapis/java-storage-nio/issues/1219)) ([99179e8](https://github.com/googleapis/java-storage-nio/commit/99179e822421e5280040f7baa95ada91b52c9f04)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.13.1 ([#1223](https://github.com/googleapis/java-storage-nio/issues/1223)) ([7f5921d](https://github.com/googleapis/java-storage-nio/commit/7f5921d185a2bc5876be8511d80716c982de9c16)) +* Update dependency com.google.cloud:google-cloud-storage to v2.25.0 ([#1215](https://github.com/googleapis/java-storage-nio/issues/1215)) ([1fe262d](https://github.com/googleapis/java-storage-nio/commit/1fe262d683a6a6976bbb3ab34f5fcafb1d158a7e)) + +## [0.126.19](https://github.com/googleapis/java-storage-nio/compare/v0.126.18...v0.126.19) (2023-07-10) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.13.0 ([#1213](https://github.com/googleapis/java-storage-nio/issues/1213)) ([dff430b](https://github.com/googleapis/java-storage-nio/commit/dff430bc27da1f0ba890c28806762269a5fd6eee)) +* Update dependency com.google.cloud:google-cloud-storage to v2.22.6 ([#1208](https://github.com/googleapis/java-storage-nio/issues/1208)) ([002c776](https://github.com/googleapis/java-storage-nio/commit/002c776fb04ab284c1495a2e99f68c28792f2e7a)) + +## [0.126.18](https://github.com/googleapis/java-storage-nio/compare/v0.126.17...v0.126.18) (2023-06-22) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230617-2.0.0 ([#1202](https://github.com/googleapis/java-storage-nio/issues/1202)) ([9f7b1bb](https://github.com/googleapis/java-storage-nio/commit/9f7b1bba58f63cdfc42f42b1f219f65dbcc8d310)) +* Update dependency com.google.cloud:google-cloud-storage to v2.22.5 ([#1205](https://github.com/googleapis/java-storage-nio/issues/1205)) ([4127953](https://github.com/googleapis/java-storage-nio/commit/412795338eda33cccddd71620cd94894e6fd0b5a)) + +## [0.126.17](https://github.com/googleapis/java-storage-nio/compare/v0.126.16...v0.126.17) (2023-06-08) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.11.0 ([#1191](https://github.com/googleapis/java-storage-nio/issues/1191)) ([db3cf28](https://github.com/googleapis/java-storage-nio/commit/db3cf2898027be6169fb109b4d64866f24d74fb4)) +* Update dependency com.google.cloud:google-cloud-storage to v2.22.4 ([#1193](https://github.com/googleapis/java-storage-nio/issues/1193)) ([6018385](https://github.com/googleapis/java-storage-nio/commit/6018385114a0e190026a19826a8ff85d994c40c3)) + +## [0.126.16](https://github.com/googleapis/java-storage-nio/compare/v0.126.15...v0.126.16) (2023-05-30) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.10.1 ([#1181](https://github.com/googleapis/java-storage-nio/issues/1181)) ([8a441b9](https://github.com/googleapis/java-storage-nio/commit/8a441b9728e9d1e85e544e9a048415a8c72cab29)) +* Update dependency com.google.cloud:google-cloud-storage to v2.22.3 ([#1183](https://github.com/googleapis/java-storage-nio/issues/1183)) ([454f4dc](https://github.com/googleapis/java-storage-nio/commit/454f4dc3bb1821dc26119cc39ae90c9dd1500f07)) + +## [0.126.15](https://github.com/googleapis/java-storage-nio/compare/v0.126.14...v0.126.15) (2023-05-10) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.9.0 ([#1171](https://github.com/googleapis/java-storage-nio/issues/1171)) ([8eff3ba](https://github.com/googleapis/java-storage-nio/commit/8eff3ba700dcfb0f775faaf40cf415cbac51ed56)) +* Update dependency com.google.cloud:google-cloud-storage to v2.22.2 ([#1174](https://github.com/googleapis/java-storage-nio/issues/1174)) ([7e4a794](https://github.com/googleapis/java-storage-nio/commit/7e4a794a5dff00f7d4edd5f72f848ef719c37c9c)) + +## [0.126.14](https://github.com/googleapis/java-storage-nio/compare/v0.126.13...v0.126.14) (2023-04-26) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.8.0 ([#1164](https://github.com/googleapis/java-storage-nio/issues/1164)) ([66370be](https://github.com/googleapis/java-storage-nio/commit/66370bea5caf6067f12a78b34dffe6c3926ff788)) +* Update dependency com.google.cloud:google-cloud-storage to v2.22.1 ([#1166](https://github.com/googleapis/java-storage-nio/issues/1166)) ([adde354](https://github.com/googleapis/java-storage-nio/commit/adde3540871437a32a0d5cde4041b4e5ac4f2717)) + +## [0.126.13](https://github.com/googleapis/java-storage-nio/compare/v0.126.12...v0.126.13) (2023-04-13) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.22.0 ([#1158](https://github.com/googleapis/java-storage-nio/issues/1158)) ([2b9d909](https://github.com/googleapis/java-storage-nio/commit/2b9d9091b781bf62d498193966665c88a660c07a)) + +## [0.126.12](https://github.com/googleapis/java-storage-nio/compare/v0.126.11...v0.126.12) (2023-04-12) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.7.0 ([#1154](https://github.com/googleapis/java-storage-nio/issues/1154)) ([1e1590d](https://github.com/googleapis/java-storage-nio/commit/1e1590de0dee2d2d2dc995978460d1dedec30799)) + +## [0.126.11](https://github.com/googleapis/java-storage-nio/compare/v0.126.10...v0.126.11) (2023-04-04) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.6.0 ([#1147](https://github.com/googleapis/java-storage-nio/issues/1147)) ([a650f7b](https://github.com/googleapis/java-storage-nio/commit/a650f7be397440a107e672fc94c87cfdbc626b33)) +* Update dependency com.google.cloud:google-cloud-storage to v2.21.0 ([#1149](https://github.com/googleapis/java-storage-nio/issues/1149)) ([61f1c2c](https://github.com/googleapis/java-storage-nio/commit/61f1c2ceb224f207a114f75700ee8c268d806c4a)) + +## [0.126.10](https://github.com/googleapis/java-storage-nio/compare/v0.126.9...v0.126.10) (2023-03-23) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.5.0 ([#1140](https://github.com/googleapis/java-storage-nio/issues/1140)) ([c6db426](https://github.com/googleapis/java-storage-nio/commit/c6db4262f3d45dc39f39ccf2d33c4e63f0bd5477)) +* Update dependency com.google.cloud:google-cloud-storage to v2.20.2 ([#1142](https://github.com/googleapis/java-storage-nio/issues/1142)) ([c56b30c](https://github.com/googleapis/java-storage-nio/commit/c56b30cc83bf54dc6dd4eed7837c0ca500fde16a)) + +## [0.126.9](https://github.com/googleapis/java-storage-nio/compare/v0.126.8...v0.126.9) (2023-03-16) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20230301-2.0.0 ([#1136](https://github.com/googleapis/java-storage-nio/issues/1136)) ([7defd38](https://github.com/googleapis/java-storage-nio/commit/7defd387cf453f8dcdc8e0cbb037bfaa07658442)) + +## [0.126.8](https://github.com/googleapis/java-storage-nio/compare/v0.126.7...v0.126.8) (2023-03-07) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.20.0 ([#1130](https://github.com/googleapis/java-storage-nio/issues/1130)) ([56b056f](https://github.com/googleapis/java-storage-nio/commit/56b056f28325919d2bd2cfe9f0b5f0ab943ad5f3)) +* Update dependency com.google.cloud:google-cloud-storage to v2.20.1 ([#1132](https://github.com/googleapis/java-storage-nio/issues/1132)) ([7fa62ae](https://github.com/googleapis/java-storage-nio/commit/7fa62ae19721118d5b76a1c0724b5646b6c8f510)) + +## [0.126.7](https://github.com/googleapis/java-storage-nio/compare/v0.126.6...v0.126.7) (2023-03-06) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.4.0 ([#1126](https://github.com/googleapis/java-storage-nio/issues/1126)) ([ace0b31](https://github.com/googleapis/java-storage-nio/commit/ace0b31ed68fd8e7e55ebbca3f3fc0e70aaccb3d)) + +## [0.126.6](https://github.com/googleapis/java-storage-nio/compare/v0.126.5...v0.126.6) (2023-02-21) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.19.0 ([#1118](https://github.com/googleapis/java-storage-nio/issues/1118)) ([ce48ec5](https://github.com/googleapis/java-storage-nio/commit/ce48ec510471507467fbabab2ee3eab5962f2570)) + +## [0.126.5](https://github.com/googleapis/java-storage-nio/compare/v0.126.4...v0.126.5) (2023-02-21) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.3.0 ([#1114](https://github.com/googleapis/java-storage-nio/issues/1114)) ([83d3586](https://github.com/googleapis/java-storage-nio/commit/83d3586c503b3c54e341d1cde79ae356aab48fcf)) + +## [0.126.4](https://github.com/googleapis/java-storage-nio/compare/v0.126.3...v0.126.4) (2023-02-07) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.2.0 ([#1104](https://github.com/googleapis/java-storage-nio/issues/1104)) ([068e0cb](https://github.com/googleapis/java-storage-nio/commit/068e0cbf6fb32de40d4a1baa04c88697584431c4)) +* Update dependency com.google.cloud:google-cloud-storage to v2.18.0 ([#1106](https://github.com/googleapis/java-storage-nio/issues/1106)) ([92e13eb](https://github.com/googleapis/java-storage-nio/commit/92e13ebb6e21f1810bf081cf5c3cecd9c0148aa8)) + +## [0.126.3](https://github.com/googleapis/java-storage-nio/compare/v0.126.2...v0.126.3) (2023-01-23) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.17.2 ([#1098](https://github.com/googleapis/java-storage-nio/issues/1098)) ([43acd0d](https://github.com/googleapis/java-storage-nio/commit/43acd0d6feb453b0eb81cd631a2957cfa12563a2)) + +## [0.126.2](https://github.com/googleapis/java-storage-nio/compare/v0.126.1...v0.126.2) (2023-01-20) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.1.2 ([#1094](https://github.com/googleapis/java-storage-nio/issues/1094)) ([dc43479](https://github.com/googleapis/java-storage-nio/commit/dc43479a4396b8d2dfa59888279f6f0d0f581bdf)) + +## [0.126.1](https://github.com/googleapis/java-storage-nio/compare/v0.126.0...v0.126.1) (2023-01-13) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.1.1 ([#1082](https://github.com/googleapis/java-storage-nio/issues/1082)) ([e42f097](https://github.com/googleapis/java-storage-nio/commit/e42f097ae9f95e2eda794dc298dcd6c33abc40d8)) +* Update dependency com.google.cloud:google-cloud-storage to v2.17.1 ([#1086](https://github.com/googleapis/java-storage-nio/issues/1086)) ([28b525d](https://github.com/googleapis/java-storage-nio/commit/28b525d9b58ff468f56133f49d83bfb8de2651b1)) + +## [0.126.0](https://github.com/googleapis/java-storage-nio/compare/v0.125.0...v0.126.0) (2022-12-09) + + +### Features + +* FileSystemProvider::readAttributes faked posix support ([#1067](https://github.com/googleapis/java-storage-nio/issues/1067)) ([b813ccc](https://github.com/googleapis/java-storage-nio/commit/b813ccc87ecff40f56fccf2ce981f16422c190f1)) + +## [0.125.0](https://github.com/googleapis/java-storage-nio/compare/v0.124.21...v0.125.0) (2022-12-07) + + +### Features + +* FileSystemProvider::readAttributes for basic and gcs views ([#1066](https://github.com/googleapis/java-storage-nio/issues/1066)) ([2f13792](https://github.com/googleapis/java-storage-nio/commit/2f13792040302899b6d7db00aa9c673dd470d460)) + + +### Bug Fixes + +* FileSystemProvider::checkAccess fails on '/' with StorageException ([d287cf5](https://github.com/googleapis/java-storage-nio/commit/d287cf5a9292221db251c987ff3a3ec736a815ac)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.1.0 ([#1070](https://github.com/googleapis/java-storage-nio/issues/1070)) ([a6e55e7](https://github.com/googleapis/java-storage-nio/commit/a6e55e77d06eba990b81400e4973a9a4098175cb)) +* Update dependency com.google.cloud:google-cloud-storage to v2.16.0 ([#1071](https://github.com/googleapis/java-storage-nio/issues/1071)) ([882e06b](https://github.com/googleapis/java-storage-nio/commit/882e06b13630259bd5254e8b4f10b6ab9dddbf84)) + +## [0.124.21](https://github.com/googleapis/java-storage-nio/compare/v0.124.20...v0.124.21) (2022-11-18) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.15.1 ([#1057](https://github.com/googleapis/java-storage-nio/issues/1057)) ([88b1aac](https://github.com/googleapis/java-storage-nio/commit/88b1aace911f8b2187b5cba246ea13a143bad7ad)) + +## [0.124.20](https://github.com/googleapis/java-storage-nio/compare/v0.124.19...v0.124.20) (2022-11-09) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.0.6 ([#1049](https://github.com/googleapis/java-storage-nio/issues/1049)) ([44270ca](https://github.com/googleapis/java-storage-nio/commit/44270cada9438ac105134713b81fb0005649f0cd)) +* Update dependency com.google.cloud:google-cloud-storage to v2.15.0 ([#1050](https://github.com/googleapis/java-storage-nio/issues/1050)) ([23a7cfa](https://github.com/googleapis/java-storage-nio/commit/23a7cfa4308be2f20355f7aaa605725d13c8eb44)) + +## [0.124.19](https://github.com/googleapis/java-storage-nio/compare/v0.124.18...v0.124.19) (2022-10-26) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.14.0 ([#1039](https://github.com/googleapis/java-storage-nio/issues/1039)) ([26ef9be](https://github.com/googleapis/java-storage-nio/commit/26ef9beee22a380a810ab6a6b9f2e3e67b03fd40)) + +## [0.124.18](https://github.com/googleapis/java-storage-nio/compare/v0.124.17...v0.124.18) (2022-10-21) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.0.5 ([#1031](https://github.com/googleapis/java-storage-nio/issues/1031)) ([00e8b8d](https://github.com/googleapis/java-storage-nio/commit/00e8b8d5fa19d858fc888233295804653d90bd4e)) +* Update dependency com.google.cloud:google-cloud-storage to v2.13.1 ([#1032](https://github.com/googleapis/java-storage-nio/issues/1032)) ([91cda8d](https://github.com/googleapis/java-storage-nio/commit/91cda8d672ee374cee9bcd09be704f5bcd0354d1)) + +## [0.124.17](https://github.com/googleapis/java-storage-nio/compare/v0.124.16...v0.124.17) (2022-10-07) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-storage to v2.13.0 ([#1024](https://github.com/googleapis/java-storage-nio/issues/1024)) ([98aba70](https://github.com/googleapis/java-storage-nio/commit/98aba7052ce506400c326867bf1c4e4e01e92790)) + +## [0.124.16](https://github.com/googleapis/java-storage-nio/compare/v0.124.15...v0.124.16) (2022-10-03) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.0.4 ([#1020](https://github.com/googleapis/java-storage-nio/issues/1020)) ([fb8115e](https://github.com/googleapis/java-storage-nio/commit/fb8115e6583c437fdb26e59c75258c4325f63598)) + +## [0.124.15](https://github.com/googleapis/java-storage-nio/compare/v0.124.14...v0.124.15) (2022-09-15) + + +### Documentation + +* Update documentation to be clearer on pseudo-directory pitfalls ([#980](https://github.com/googleapis/java-storage-nio/issues/980)) ([5f6ac74](https://github.com/googleapis/java-storage-nio/commit/5f6ac74cc50c819b5871a1a1d0b86e865801caec)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.0.2 ([#987](https://github.com/googleapis/java-storage-nio/issues/987)) ([d7061e0](https://github.com/googleapis/java-storage-nio/commit/d7061e03a7cf02138dc7ddfc13d3da6d77e301fd)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.0.3 ([#990](https://github.com/googleapis/java-storage-nio/issues/990)) ([44bb128](https://github.com/googleapis/java-storage-nio/commit/44bb12894f8459ad34c9ed314e4c75b5277f3496)) +* Update dependency com.google.cloud:google-cloud-storage to v2.12.0 ([#991](https://github.com/googleapis/java-storage-nio/issues/991)) ([b5b565f](https://github.com/googleapis/java-storage-nio/commit/b5b565f284ce2fcde66ee1d0e90d984b6aa6e703)) + +## [0.124.14](https://github.com/googleapis/java-storage-nio/compare/v0.124.13...v0.124.14) (2022-08-08) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.11.3 ([#977](https://github.com/googleapis/java-storage-nio/issues/977)) ([f044437](https://github.com/googleapis/java-storage-nio/commit/f044437af7e1ce07183ceb18b90a50e2c13c8498)) + +## [0.124.13](https://github.com/googleapis/java-storage-nio/compare/v0.124.12...v0.124.13) (2022-08-05) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.11.2 ([#973](https://github.com/googleapis/java-storage-nio/issues/973)) ([ab3cd7f](https://github.com/googleapis/java-storage-nio/commit/ab3cd7f6756899072cc6f54df98cb25d12e9835c)) + +## [0.124.12](https://github.com/googleapis/java-storage-nio/compare/v0.124.11...v0.124.12) (2022-08-04) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.11.1 ([#969](https://github.com/googleapis/java-storage-nio/issues/969)) ([abe838f](https://github.com/googleapis/java-storage-nio/commit/abe838f8c334c07bc22a12665c14b490cbde7a8b)) + +## [0.124.11](https://github.com/googleapis/java-storage-nio/compare/v0.124.10...v0.124.11) (2022-08-03) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v3 ([#963](https://github.com/googleapis/java-storage-nio/issues/963)) ([0794f54](https://github.com/googleapis/java-storage-nio/commit/0794f54070c4303298b4879f346eacbb7ba12826)) + +## [0.124.10](https://github.com/googleapis/java-storage-nio/compare/v0.124.9...v0.124.10) (2022-07-21) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.10.0 ([#955](https://github.com/googleapis/java-storage-nio/issues/955)) ([967a6da](https://github.com/googleapis/java-storage-nio/commit/967a6da8296096db0b1061556e8e19265ad21ec3)) + +## [0.124.9](https://github.com/googleapis/java-storage-nio/compare/v0.124.8...v0.124.9) (2022-07-13) + + +### Bug Fixes + +* enable longpaths support for windows test ([#1485](https://github.com/googleapis/java-storage-nio/issues/1485)) ([#951](https://github.com/googleapis/java-storage-nio/issues/951)) ([aa3f827](https://github.com/googleapis/java-storage-nio/commit/aa3f827744f97d03de46ea18a122ac10fbb2d900)) + +## [0.124.8](https://github.com/googleapis/java-storage-nio/compare/v0.124.7...v0.124.8) (2022-07-11) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20220705-1.32.1 ([#946](https://github.com/googleapis/java-storage-nio/issues/946)) ([513c21e](https://github.com/googleapis/java-storage-nio/commit/513c21e7f8337da26c7dfd8a47d059af872f6aeb)) +* update dependency com.google.cloud:google-cloud-storage to v2.9.3 ([#945](https://github.com/googleapis/java-storage-nio/issues/945)) ([39e7451](https://github.com/googleapis/java-storage-nio/commit/39e7451712aa2e754cb28c47dcc77de81273ad54)) + +## [0.124.7](https://github.com/googleapis/java-storage-nio/compare/v0.124.6...v0.124.7) (2022-06-29) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.9.0 ([#939](https://github.com/googleapis/java-storage-nio/issues/939)) ([ae5d294](https://github.com/googleapis/java-storage-nio/commit/ae5d294253a1e6aa43c0fbd6ea765acd24d43645)) + +## [0.124.6](https://github.com/googleapis/java-storage-nio/compare/v0.124.5...v0.124.6) (2022-06-23) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.13.0 ([#933](https://github.com/googleapis/java-storage-nio/issues/933)) ([d3f032c](https://github.com/googleapis/java-storage-nio/commit/d3f032cdab59ee1a23987975d3b79025ea5c4f41)) + +## [0.124.5](https://github.com/googleapis/java-storage-nio/compare/v0.124.4...v0.124.5) (2022-06-13) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20220608-1.32.1 ([#928](https://github.com/googleapis/java-storage-nio/issues/928)) ([a5cc78e](https://github.com/googleapis/java-storage-nio/commit/a5cc78e8e33204477baa9facfd59a89b9edf8ed7)) +* update dependency com.google.cloud:google-cloud-storage to v2.8.1 ([#930](https://github.com/googleapis/java-storage-nio/issues/930)) ([458b29f](https://github.com/googleapis/java-storage-nio/commit/458b29f31ee911a7131ac7c5eac84eb9e5cb0014)) + +## [0.124.4](https://github.com/googleapis/java-storage-nio/compare/v0.124.3...v0.124.4) (2022-06-09) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.8.0 ([#923](https://github.com/googleapis/java-storage-nio/issues/923)) ([f3ae73e](https://github.com/googleapis/java-storage-nio/commit/f3ae73edc6474493ff36ccbcfa0744aa8f8a9a26)) + +## [0.124.3](https://github.com/googleapis/java-storage-nio/compare/v0.124.2...v0.124.3) (2022-06-08) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20220604-1.32.1 ([#919](https://github.com/googleapis/java-storage-nio/issues/919)) ([262784d](https://github.com/googleapis/java-storage-nio/commit/262784df98b011a9712d46c12630a55bb66fdc59)) + +### [0.124.2](https://github.com/googleapis/java-storage-nio/compare/v0.124.1...v0.124.2) (2022-05-27) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.7.2 ([#912](https://github.com/googleapis/java-storage-nio/issues/912)) ([768e8c8](https://github.com/googleapis/java-storage-nio/commit/768e8c8e31fbd994c9349a031c13feb7d95da200)) + +### [0.124.1](https://github.com/googleapis/java-storage-nio/compare/v0.124.0...v0.124.1) (2022-05-25) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.7.1 ([#906](https://github.com/googleapis/java-storage-nio/issues/906)) ([f38d419](https://github.com/googleapis/java-storage-nio/commit/f38d4198488a65b3cc8aecca148c02b5aafc62c8)) + +## [0.124.0](https://github.com/googleapis/java-storage-nio/compare/v0.123.28...v0.124.0) (2022-05-24) + + +### Features + +* add build scripts for native image testing in Java 17 ([#1440](https://github.com/googleapis/java-storage-nio/issues/1440)) ([#902](https://github.com/googleapis/java-storage-nio/issues/902)) ([a21ed71](https://github.com/googleapis/java-storage-nio/commit/a21ed71257853b2a1c7549c36fd0dda8ef806b82)) + + +### Bug Fixes + +* **build:** revert maven-shade-plugin 3.3.0 ([533c6dd](https://github.com/googleapis/java-storage-nio/commit/533c6dd510327bff310fbd6ddb2ceb72b2afcbce)) +* check generation for nullability before incrementing it ([#888](https://github.com/googleapis/java-storage-nio/issues/888)) ([cadf081](https://github.com/googleapis/java-storage-nio/commit/cadf081c4ba35b33307c38cddd08959b142edf23)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20220509-1.32.1 ([#899](https://github.com/googleapis/java-storage-nio/issues/899)) ([52d2969](https://github.com/googleapis/java-storage-nio/commit/52d2969e1089072a86ac4f458c93932adfbf89d9)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.11.0 ([#901](https://github.com/googleapis/java-storage-nio/issues/901)) ([847afcb](https://github.com/googleapis/java-storage-nio/commit/847afcbe6c0e6d214677f2d408e429aaf330bf6d)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.12.0 ([#903](https://github.com/googleapis/java-storage-nio/issues/903)) ([d11b6c5](https://github.com/googleapis/java-storage-nio/commit/d11b6c5dc8b500d705c230ac712571a44bbf2df7)) +* update dependency com.google.cloud:google-cloud-storage to v2.7.0 ([#904](https://github.com/googleapis/java-storage-nio/issues/904)) ([6ef8605](https://github.com/googleapis/java-storage-nio/commit/6ef8605590f4ab835e3adfce2e602bcf2fe4376c)) + +### [0.123.28](https://github.com/googleapis/java-storage-nio/compare/v0.123.27...v0.123.28) (2022-04-18) + + +### Bug Fixes + +* **native:** initialize classes at build-time to fix Native Image compilation ([#874](https://github.com/googleapis/java-storage-nio/issues/874)) ([df905cb](https://github.com/googleapis/java-storage-nio/commit/df905cbb615f2a6f57d45397423556357b23fa38)) +* **test:** replace HashMap with ConcurrentHashMap to avoid ConcurrentModificatio… ([#883](https://github.com/googleapis/java-storage-nio/issues/883)) ([d2fe2a0](https://github.com/googleapis/java-storage-nio/commit/d2fe2a09de6881ce6ce949c8e99bec7825c5c833)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20220401-1.32.1 ([#881](https://github.com/googleapis/java-storage-nio/issues/881)) ([a598199](https://github.com/googleapis/java-storage-nio/commit/a598199247e5d29712aeba0b709ce9f37e641154)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.10.0 ([#884](https://github.com/googleapis/java-storage-nio/issues/884)) ([c08f115](https://github.com/googleapis/java-storage-nio/commit/c08f11569e4337127960b69a315f255ef08a035a)) +* update dependency com.google.cloud:google-cloud-storage to v2.6.0 ([#878](https://github.com/googleapis/java-storage-nio/issues/878)) ([0bd1985](https://github.com/googleapis/java-storage-nio/commit/0bd19858569d4181be49e60525d9ad5736b9e1ab)) +* update dependency com.google.cloud:google-cloud-storage to v2.6.1 ([#885](https://github.com/googleapis/java-storage-nio/issues/885)) ([cf24525](https://github.com/googleapis/java-storage-nio/commit/cf24525d52ea45da50e5152041d25d6ad6246cfc)) + +### [0.123.27](https://github.com/googleapis/java-storage-nio/compare/v0.123.26...v0.123.27) (2022-03-29) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.9.0 ([#873](https://github.com/googleapis/java-storage-nio/issues/873)) ([485cf03](https://github.com/googleapis/java-storage-nio/commit/485cf032c981a1a11792710107bca9814eafefc5)) +* update dependency com.google.cloud:google-cloud-storage to v2.5.1 ([#872](https://github.com/googleapis/java-storage-nio/issues/872)) ([5a6a664](https://github.com/googleapis/java-storage-nio/commit/5a6a664c1c818e5f9232bc00b3f068dfcbb608b1)) + +### [0.123.26](https://github.com/googleapis/java-storage-nio/compare/v0.123.25...v0.123.26) (2022-03-28) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.5.0 ([#868](https://github.com/googleapis/java-storage-nio/issues/868)) ([9be5945](https://github.com/googleapis/java-storage-nio/commit/9be594506f95d911a54ab56eca068f8f0b49798b)) + +### [0.123.25](https://github.com/googleapis/java-storage-nio/compare/v0.123.24...v0.123.25) (2022-03-17) + + +### Bug Fixes + +* prevent crash when checking if a missing file exists [#856](https://github.com/googleapis/java-storage-nio/issues/856) ([#858](https://github.com/googleapis/java-storage-nio/issues/858)) ([d6b7b5e](https://github.com/googleapis/java-storage-nio/commit/d6b7b5e5e7ca243583e8852edfcf83d57021c9e6)) +* Prevent NPE when bucket doesn't exist [#857](https://github.com/googleapis/java-storage-nio/issues/857) ([#860](https://github.com/googleapis/java-storage-nio/issues/860)) ([69cab9e](https://github.com/googleapis/java-storage-nio/commit/69cab9ef10072bbf504f5afe0817937bb38aef11)) + +### [0.123.24](https://github.com/googleapis/java-storage-nio/compare/v0.123.23...v0.123.24) (2022-03-09) + + +### Bug Fixes + +* prevent NPE when checking requester pays status ([#850](https://github.com/googleapis/java-storage-nio/issues/850)) ([ce50209](https://github.com/googleapis/java-storage-nio/commit/ce502094f0d983e136a60569424992acbd126041)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.4.5 ([#852](https://github.com/googleapis/java-storage-nio/issues/852)) ([0163189](https://github.com/googleapis/java-storage-nio/commit/01631891d1f6477d9021b57d097d0d6fbcccab1e)) + +### [0.123.23](https://github.com/googleapis/java-storage-nio/compare/v0.123.22...v0.123.23) (2022-03-04) + + +### Bug Fixes + +* Requester pays to check reason and fallback to error message validation ([#841](https://github.com/googleapis/java-storage-nio/issues/841)) ([9f30db3](https://github.com/googleapis/java-storage-nio/commit/9f30db3ccca16121fc5260334d603f8d272af2d9)) + +### [0.123.22](https://github.com/googleapis/java-storage-nio/compare/v0.123.21...v0.123.22) (2022-03-02) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.8.0 ([#835](https://github.com/googleapis/java-storage-nio/issues/835)) ([55a7cd7](https://github.com/googleapis/java-storage-nio/commit/55a7cd745d76a5d6781cf5a1f0432447cf8eb94b)) + +### [0.123.21](https://github.com/googleapis/java-storage-nio/compare/v0.123.20...v0.123.21) (2022-03-01) + + +### Bug Fixes + +* change CloudStorageFileSystemProvider to throw a FileAlreadyExistsException if copy receives a 412 ([#815](https://github.com/googleapis/java-storage-nio/issues/815)) ([33889c3](https://github.com/googleapis/java-storage-nio/commit/33889c352d1a0fcebe4613615907f01fbca04186)) + + +### Dependencies + +* update actions/github-script action to v6 ([#818](https://github.com/googleapis/java-storage-nio/issues/818)) ([59c9baf](https://github.com/googleapis/java-storage-nio/commit/59c9baf28ceda12d0926124d137f792e940c153e)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20220210-1.32.1 ([#823](https://github.com/googleapis/java-storage-nio/issues/823)) ([6c94667](https://github.com/googleapis/java-storage-nio/commit/6c946674931d6f5e88bba7cc93bfb1849480ce0d)) +* update dependency com.google.cloud:google-cloud-storage to v2.4.1 ([#817](https://github.com/googleapis/java-storage-nio/issues/817)) ([2ebe674](https://github.com/googleapis/java-storage-nio/commit/2ebe67495be0b2e71eed4a5c0d6aed5954c32593)) +* update dependency com.google.cloud:google-cloud-storage to v2.4.2 ([#820](https://github.com/googleapis/java-storage-nio/issues/820)) ([397ee84](https://github.com/googleapis/java-storage-nio/commit/397ee84b7530607665c725eb8392d12429f6b6a7)) +* update dependency com.google.cloud:google-cloud-storage to v2.4.4 ([#830](https://github.com/googleapis/java-storage-nio/issues/830)) ([2b3f88f](https://github.com/googleapis/java-storage-nio/commit/2b3f88f967f7fcf0a1475dae5dfc74f3e3f0cb6c)) +* update dependency com.google.guava:guava to v31.1-android ([#829](https://github.com/googleapis/java-storage-nio/issues/829)) ([70c1983](https://github.com/googleapis/java-storage-nio/commit/70c198347c8709d1302915bddda49d2402566ea7)) + +### [0.123.20](https://github.com/googleapis/java-storage-nio/compare/v0.123.19...v0.123.20) (2022-02-08) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.4.0 ([#806](https://github.com/googleapis/java-storage-nio/issues/806)) ([4849120](https://github.com/googleapis/java-storage-nio/commit/484912059a54926cba4cfc5149d70fd8f2bc5ba8)) + +### [0.123.19](https://github.com/googleapis/java-storage-nio/compare/v0.123.18...v0.123.19) (2022-02-03) + + +### Dependencies + +* **java:** update actions/github-script action to v5 ([#1339](https://github.com/googleapis/java-storage-nio/issues/1339)) ([#800](https://github.com/googleapis/java-storage-nio/issues/800)) ([4c82c37](https://github.com/googleapis/java-storage-nio/commit/4c82c37860af4e0ae43e31446e35b6b3bae7ebbb)) +* update actions/github-script action to v5 ([#799](https://github.com/googleapis/java-storage-nio/issues/799)) ([40febb2](https://github.com/googleapis/java-storage-nio/commit/40febb21b20c414c7e9443e95245e469cdafef76)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.7.0 ([#802](https://github.com/googleapis/java-storage-nio/issues/802)) ([2beefb6](https://github.com/googleapis/java-storage-nio/commit/2beefb699619f72f2a3d50cd0d63140d15d08a0b)) +* update dependency com.google.cloud:google-cloud-storage to v2.2.3 ([#786](https://github.com/googleapis/java-storage-nio/issues/786)) ([b82657c](https://github.com/googleapis/java-storage-nio/commit/b82657ca045dee5e3a73f89a8f087cd957fb370f)) +* update dependency com.google.cloud:google-cloud-storage to v2.3.0 ([#796](https://github.com/googleapis/java-storage-nio/issues/796)) ([e822be5](https://github.com/googleapis/java-storage-nio/commit/e822be5c411420c79ead2276904126b8e3ad022f)) + +### [0.123.18](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.17...v0.123.18) (2022-01-07) + + +### Bug Fixes + +* add nio entry to user-agent ([#774](https://www.github.com/googleapis/java-storage-nio/issues/774)) ([2d30f78](https://www.github.com/googleapis/java-storage-nio/commit/2d30f789e475afe8d2b0cd70bba05f3cbf105caf)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.6.0 ([#783](https://www.github.com/googleapis/java-storage-nio/issues/783)) ([55f2e8d](https://www.github.com/googleapis/java-storage-nio/commit/55f2e8d0c49d506261e3839003b4dd652e6301fc)) + +### [0.123.17](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.16...v0.123.17) (2021-12-06) + + +### Bug Fixes + +* **java:** java 17 dependency arguments ([#1266](https://www.github.com/googleapis/java-storage-nio/issues/1266)) ([#750](https://www.github.com/googleapis/java-storage-nio/issues/750)) ([3f866da](https://www.github.com/googleapis/java-storage-nio/commit/3f866da0aa3b381477c883b4f11239640c577d88)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20211018-1.32.1 ([#749](https://www.github.com/googleapis/java-storage-nio/issues/749)) ([7e99920](https://www.github.com/googleapis/java-storage-nio/commit/7e99920cfeef90ad5af5503a63338c48a57cb674)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20211201-1.32.1 ([#768](https://www.github.com/googleapis/java-storage-nio/issues/768)) ([650d9b5](https://www.github.com/googleapis/java-storage-nio/commit/650d9b5d76869f31d0456f9f3a9c01c9e9ae6ef9)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.5.0 ([#759](https://www.github.com/googleapis/java-storage-nio/issues/759)) ([7cb3292](https://www.github.com/googleapis/java-storage-nio/commit/7cb32926da5f910e3636d12a1ad412b4e3531b05)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.5.1 ([#767](https://www.github.com/googleapis/java-storage-nio/issues/767)) ([7eb3dfa](https://www.github.com/googleapis/java-storage-nio/commit/7eb3dfab9644f48f0063fb1be4f9805062b2eae8)) +* update dependency com.google.cloud:google-cloud-storage to v2.2.0 ([#753](https://www.github.com/googleapis/java-storage-nio/issues/753)) ([fcbc986](https://www.github.com/googleapis/java-storage-nio/commit/fcbc986c020ab48412218ab3825c2423a90d70d4)) +* update dependency com.google.cloud:google-cloud-storage to v2.2.1 ([#760](https://www.github.com/googleapis/java-storage-nio/issues/760)) ([33fffbf](https://www.github.com/googleapis/java-storage-nio/commit/33fffbf3d79335b36e78af95e0f26cb9ec5f5f4e)) +* update dependency com.google.cloud:google-cloud-storage to v2.2.2 ([#769](https://www.github.com/googleapis/java-storage-nio/issues/769)) ([4f48a96](https://www.github.com/googleapis/java-storage-nio/commit/4f48a96feb32a43c590a286d8c622b2ba845b8f2)) + +### [0.123.16](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.15...v0.123.16) (2021-10-21) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.4.0 ([#744](https://www.github.com/googleapis/java-storage-nio/issues/744)) ([5a3ac26](https://www.github.com/googleapis/java-storage-nio/commit/5a3ac269778ecfd237714d42803f27957b02c96d)) +* update dependency com.google.cloud:google-cloud-storage to v2.1.9 ([#743](https://www.github.com/googleapis/java-storage-nio/issues/743)) ([261d18b](https://www.github.com/googleapis/java-storage-nio/commit/261d18b885a03bb748f06b19eb5b19a4dbbf0d55)) + +### [0.123.15](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.14...v0.123.15) (2021-10-05) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.1.7 ([#735](https://www.github.com/googleapis/java-storage-nio/issues/735)) ([536e238](https://www.github.com/googleapis/java-storage-nio/commit/536e238267d0435138bb48390e97886e293d5d7f)) + +### [0.123.14](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.13...v0.123.14) (2021-10-01) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20210918-1.32.1 ([#721](https://www.github.com/googleapis/java-storage-nio/issues/721)) ([d0f8fc0](https://www.github.com/googleapis/java-storage-nio/commit/d0f8fc01ece5020306b46af8e85c94f8762f148a)) +* update dependency com.google.cloud:google-cloud-storage to v2.1.6 ([#722](https://www.github.com/googleapis/java-storage-nio/issues/722)) ([a51c47f](https://www.github.com/googleapis/java-storage-nio/commit/a51c47f6f25e80538d7f14883e9c9676c261441b)) +* update dependency com.google.guava:guava to v31 ([#720](https://www.github.com/googleapis/java-storage-nio/issues/720)) ([062edfc](https://www.github.com/googleapis/java-storage-nio/commit/062edfc6c9cb5105623f7582378a003f12f4e746)) + +### [0.123.13](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.12...v0.123.13) (2021-09-27) + + +### Bug Fixes + +* make CloudStorageFileSystem#forBucket thread safe ([#719](https://www.github.com/googleapis/java-storage-nio/issues/719)) ([ac8bfee](https://www.github.com/googleapis/java-storage-nio/commit/ac8bfeee367269a06d67c7a81adc770fb5bd83e4)), closes [#691](https://www.github.com/googleapis/java-storage-nio/issues/691) [#698](https://www.github.com/googleapis/java-storage-nio/issues/698) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20210914-1.32.1 ([#711](https://www.github.com/googleapis/java-storage-nio/issues/711)) ([1b6e324](https://www.github.com/googleapis/java-storage-nio/commit/1b6e3241c8950ba9255074efa3356b81818f514f)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.3.0 ([#715](https://www.github.com/googleapis/java-storage-nio/issues/715)) ([2f42aa8](https://www.github.com/googleapis/java-storage-nio/commit/2f42aa8cf29af5d593285ba0c69902771b60c393)) +* update dependency com.google.cloud:google-cloud-storage to v2.1.4 ([#713](https://www.github.com/googleapis/java-storage-nio/issues/713)) ([7ae21d0](https://www.github.com/googleapis/java-storage-nio/commit/7ae21d0032c5c1b65f07c45b4c9d631dd19bedf8)) +* update dependency com.google.cloud:google-cloud-storage to v2.1.5 ([#716](https://www.github.com/googleapis/java-storage-nio/issues/716)) ([f3f2037](https://www.github.com/googleapis/java-storage-nio/commit/f3f2037af5b889d0e70afff730848cbd073e21e4)) + +### [0.123.12](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.11...v0.123.12) (2021-09-15) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.1.3 ([#706](https://www.github.com/googleapis/java-storage-nio/issues/706)) ([8589fc4](https://www.github.com/googleapis/java-storage-nio/commit/8589fc4187e97491badffc408b3142524bb4c7b3)) + +### [0.123.11](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.10...v0.123.11) (2021-09-14) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-core to v2.1.3 ([#699](https://www.github.com/googleapis/java-storage-nio/issues/699)) ([4982236](https://www.github.com/googleapis/java-storage-nio/commit/49822364396ec062a64060d127a8c4748ed15ba4)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.2.1 ([#700](https://www.github.com/googleapis/java-storage-nio/issues/700)) ([42e354b](https://www.github.com/googleapis/java-storage-nio/commit/42e354b712d999dbe9cce4de45d117d5d1899600)) +* update dependency com.google.cloud:google-cloud-storage to v2.1.2 ([#703](https://www.github.com/googleapis/java-storage-nio/issues/703)) ([1d07d07](https://www.github.com/googleapis/java-storage-nio/commit/1d07d073504d469777795b6922214233acecf31f)) + +### [0.123.10](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.9...v0.123.10) (2021-09-03) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.1.1 ([#682](https://www.github.com/googleapis/java-storage-nio/issues/682)) ([2ba3c5d](https://www.github.com/googleapis/java-storage-nio/commit/2ba3c5df2c697060fab61a1f8db66b9a95f5e8fe)) + +### [0.123.9](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.8...v0.123.9) (2021-09-03) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-core to v2.1.2 ([#662](https://www.github.com/googleapis/java-storage-nio/issues/662)) ([79548fe](https://www.github.com/googleapis/java-storage-nio/commit/79548fef0e7cce2d3233e0d49a81e8a881afc100)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.2.0 ([#674](https://www.github.com/googleapis/java-storage-nio/issues/674)) ([40f2c32](https://www.github.com/googleapis/java-storage-nio/commit/40f2c3298472546973f118520699fc41a0a90df2)) + +### [0.123.8](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.7...v0.123.8) (2021-08-25) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.1.0 ([#663](https://www.github.com/googleapis/java-storage-nio/issues/663)) ([0d06199](https://www.github.com/googleapis/java-storage-nio/commit/0d061999400d043b90d61f2cc4c6aeb697dc7efb)) +* update dependency com.google.cloud:google-cloud-storage to v2.1.0 ([#666](https://www.github.com/googleapis/java-storage-nio/issues/666)) ([6a3d1e7](https://www.github.com/googleapis/java-storage-nio/commit/6a3d1e7e0eeb6594dbb3d447214f7a12c5b3f2b1)) + +### [0.123.7](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.6...v0.123.7) (2021-08-19) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v2.0.2 ([#657](https://www.github.com/googleapis/java-storage-nio/issues/657)) ([5a42693](https://www.github.com/googleapis/java-storage-nio/commit/5a42693dc9603d8e1f3d2123dfd80ac6e3b1f724)) + +### [0.123.6](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.5...v0.123.6) (2021-08-12) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-core to v2.0.5 ([#643](https://www.github.com/googleapis/java-storage-nio/issues/643)) ([bea87ba](https://www.github.com/googleapis/java-storage-nio/commit/bea87ba9099b1f995c8cc4dc907a32e3ad8b843c)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.0.1 ([#644](https://www.github.com/googleapis/java-storage-nio/issues/644)) ([0185cae](https://www.github.com/googleapis/java-storage-nio/commit/0185cae96f25bd80f4f1b97c1e81573c68af3ee0)) +* update dependency com.google.cloud:google-cloud-storage to v2.0.1 ([#647](https://www.github.com/googleapis/java-storage-nio/issues/647)) ([d9c5aa9](https://www.github.com/googleapis/java-storage-nio/commit/d9c5aa92637f7140dfe5f9cf49bc7a5ede921b8d)) + +### [0.123.5](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.4...v0.123.5) (2021-08-11) + + +### Dependencies + +* update google-cloud-* libraries to v2 ([#639](https://www.github.com/googleapis/java-storage-nio/issues/639)) ([178011c](https://www.github.com/googleapis/java-storage-nio/commit/178011cc52a3e0876d02df3d2db1c8d6e8eee893)) + +### [0.123.4](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.3...v0.123.4) (2021-07-27) + + +### Bug Fixes + +* Make the StorageOption returned by LocalStorageHelper.getOptions() serializable ([#606](https://www.github.com/googleapis/java-storage-nio/issues/606)) ([12e872b](https://www.github.com/googleapis/java-storage-nio/commit/12e872b00cfefc0949e09918bcdbcd5517c750d9)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20210127-1.32.1 ([#608](https://www.github.com/googleapis/java-storage-nio/issues/608)) ([efae4cd](https://www.github.com/googleapis/java-storage-nio/commit/efae4cd411259c9e8a5c0cd465475c4994448659)) +* update dependency com.google.cloud:google-cloud-storage to v1.118.0 ([#615](https://www.github.com/googleapis/java-storage-nio/issues/615)) ([797b10c](https://www.github.com/googleapis/java-storage-nio/commit/797b10cd7aee58bae8a50e45a43975e3040e6025)) + +### [0.123.3](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.2...v0.123.3) (2021-06-30) + + +### Bug Fixes + +* Add `shopt -s nullglob` to dependencies script ([#596](https://www.github.com/googleapis/java-storage-nio/issues/596)) ([3017a27](https://www.github.com/googleapis/java-storage-nio/commit/3017a27618cb068d54c64af1abedcb817405d30d)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20210127-1.31.5 ([#594](https://www.github.com/googleapis/java-storage-nio/issues/594)) ([8d9ac59](https://www.github.com/googleapis/java-storage-nio/commit/8d9ac598196df954337b98f340ec1ae2afe782d2)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v1.4.0 ([#601](https://www.github.com/googleapis/java-storage-nio/issues/601)) ([ab08d72](https://www.github.com/googleapis/java-storage-nio/commit/ab08d72a4492f6c23886f26b6bfc5d5d58d78a18)) +* update dependency com.google.cloud:google-cloud-storage to v1.116.0 ([#590](https://www.github.com/googleapis/java-storage-nio/issues/590)) ([c2ed328](https://www.github.com/googleapis/java-storage-nio/commit/c2ed328893e49379666f7016a6944da2455f69d7)) +* update dependency com.google.cloud:google-cloud-storage to v1.117.1 ([#602](https://www.github.com/googleapis/java-storage-nio/issues/602)) ([6717f0d](https://www.github.com/googleapis/java-storage-nio/commit/6717f0d073a3467cdf644d26d03f47d4295603e2)) + +### [0.123.2](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.1...v0.123.2) (2021-06-15) + + +### Bug Fixes + +* Update dependencies.sh to not break on mac ([#588](https://www.github.com/googleapis/java-storage-nio/issues/588)) ([d13d01f](https://www.github.com/googleapis/java-storage-nio/commit/d13d01f20f32fb29366ab8d0b56d77c7e073c546)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v1.3.0 ([#583](https://www.github.com/googleapis/java-storage-nio/issues/583)) ([cad70f7](https://www.github.com/googleapis/java-storage-nio/commit/cad70f735fbd97efba6639fa637630625c85ca26)) + +### [0.123.1](https://www.github.com/googleapis/java-storage-nio/compare/v0.123.0...v0.123.1) (2021-06-02) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v1.115.0 ([#576](https://www.github.com/googleapis/java-storage-nio/issues/576)) ([eae323d](https://www.github.com/googleapis/java-storage-nio/commit/eae323d3373fe21deaf3b636834511defc6bb348)) + +## [0.123.0](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.14...v0.123.0) (2021-05-31) + + +### Features + +* add `gcf-owl-bot[bot]` to `ignoreAuthors` ([#566](https://www.github.com/googleapis/java-storage-nio/issues/566)) ([12db992](https://www.github.com/googleapis/java-storage-nio/commit/12db9929092d36f1fa175b23cb83452477d8fe74)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v1.2.0 ([#565](https://www.github.com/googleapis/java-storage-nio/issues/565)) ([25f79b4](https://www.github.com/googleapis/java-storage-nio/commit/25f79b4a9bdcb8614bd0145c43675d838c65f96f)) + +### [0.122.14](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.13...v0.122.14) (2021-05-12) + + +### Bug Fixes + +* **test:** update NIOTest to ensure LocalStorageHelper is used ([#552](https://www.github.com/googleapis/java-storage-nio/issues/552)) ([e0cd38d](https://www.github.com/googleapis/java-storage-nio/commit/e0cd38d93d5ca28207540dbdf70151010ed35c47)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v1.1.0 ([#550](https://www.github.com/googleapis/java-storage-nio/issues/550)) ([bf60a21](https://www.github.com/googleapis/java-storage-nio/commit/bf60a21f3b7f242961eb5d6ed215e49becf3ba9a)) + +### [0.122.13](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.12...v0.122.13) (2021-04-26) + + +### Bug Fixes + +* release scripts from issuing overlapping phases ([#528](https://www.github.com/googleapis/java-storage-nio/issues/528)) ([e037375](https://www.github.com/googleapis/java-storage-nio/commit/e0373758a003f57b26e2bb6cf0c0cd69549078ba)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-core to v1.94.7 ([#486](https://www.github.com/googleapis/java-storage-nio/issues/486)) ([42b281d](https://www.github.com/googleapis/java-storage-nio/commit/42b281daf441d9a93cea6b9c29ed27e0e6155908)) +* update dependency com.google.cloud:google-cloud-core to v1.94.8 ([#535](https://www.github.com/googleapis/java-storage-nio/issues/535)) ([cae4e43](https://www.github.com/googleapis/java-storage-nio/commit/cae4e43ec8bb2379fbf266fc1d95885fb6357064)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.21.1 ([#532](https://www.github.com/googleapis/java-storage-nio/issues/532)) ([5250ad7](https://www.github.com/googleapis/java-storage-nio/commit/5250ad79c2b5b8a10ec5e991e9905bcfee2d9be8)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v1 ([#536](https://www.github.com/googleapis/java-storage-nio/issues/536)) ([f70c1f6](https://www.github.com/googleapis/java-storage-nio/commit/f70c1f6904b51ac1f3e17e2937ca1107b56827ca)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.15 ([#525](https://www.github.com/googleapis/java-storage-nio/issues/525)) ([c3d8486](https://www.github.com/googleapis/java-storage-nio/commit/c3d84864b525eb7f79bef0067df48d6bc77df2d3)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.16 ([#537](https://www.github.com/googleapis/java-storage-nio/issues/537)) ([1107a79](https://www.github.com/googleapis/java-storage-nio/commit/1107a7994967ccda5d588afdf4b6f31a4b8e3a97)) + +### [0.122.12](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.11...v0.122.12) (2021-04-12) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.21.0 ([#516](https://www.github.com/googleapis/java-storage-nio/issues/516)) ([d6c79c0](https://www.github.com/googleapis/java-storage-nio/commit/d6c79c020112b7321cc9781522bddf98d131f990)) + +### [0.122.11](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.10...v0.122.11) (2021-03-16) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.20.1 ([#492](https://www.github.com/googleapis/java-storage-nio/issues/492)) ([684ae0a](https://www.github.com/googleapis/java-storage-nio/commit/684ae0ab43d8bed5dff07472866769b07ecd0dd5)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.12 ([#472](https://www.github.com/googleapis/java-storage-nio/issues/472)) ([34819d9](https://www.github.com/googleapis/java-storage-nio/commit/34819d986dfebe953eec603718ab339018d69d12)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.13 ([#491](https://www.github.com/googleapis/java-storage-nio/issues/491)) ([14bacab](https://www.github.com/googleapis/java-storage-nio/commit/14bacaba6336835b1686d91fb98c394785d7821d)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.14 ([#495](https://www.github.com/googleapis/java-storage-nio/issues/495)) ([5eb9102](https://www.github.com/googleapis/java-storage-nio/commit/5eb9102c61fade3412c0398f18b57feb37fa0695)) + +### [0.122.10](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.9...v0.122.10) (2021-02-25) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.20.0 ([#466](https://www.github.com/googleapis/java-storage-nio/issues/466)) ([0e6a967](https://www.github.com/googleapis/java-storage-nio/commit/0e6a967879b830487d58045311ea50d54f939148)) + +### [0.122.9](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.8...v0.122.9) (2021-02-24) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v1.113.11 ([#454](https://www.github.com/googleapis/java-storage-nio/issues/454)) ([5287939](https://www.github.com/googleapis/java-storage-nio/commit/5287939496da59719e3c1f9281c4cc2967b40559)) + +### [0.122.8](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.7...v0.122.8) (2021-02-23) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-core to v1.94.1 ([#405](https://www.github.com/googleapis/java-storage-nio/issues/405)) ([21622b1](https://www.github.com/googleapis/java-storage-nio/commit/21622b1718aebfba85c6c50d06f885dcda42b593)) + +### [0.122.7](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.6...v0.122.7) (2021-02-19) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.19.0 ([#445](https://www.github.com/googleapis/java-storage-nio/issues/445)) ([114e292](https://www.github.com/googleapis/java-storage-nio/commit/114e29250af32aadd9770f725acc7ad5ac72745f)) + +### [0.122.6](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.5...v0.122.6) (2021-02-18) + + +### Bug Fixes + +* cleanup use of non-preferred terms ([#411](https://www.github.com/googleapis/java-storage-nio/issues/411)) ([3a3d465](https://www.github.com/googleapis/java-storage-nio/commit/3a3d46599b463801714cc415c0e49e112f8d8838)) +* move autovalue to annotation processor path ([#179](https://www.github.com/googleapis/java-storage-nio/issues/179)) ([a5023f1](https://www.github.com/googleapis/java-storage-nio/commit/a5023f1e44448cdb42a7cddde24baf6a8e18f110)) +* Set storage update time in FakeStorageRpc ([#174](https://www.github.com/googleapis/java-storage-nio/issues/174)) ([1328de4](https://www.github.com/googleapis/java-storage-nio/commit/1328de4adf15450f055cae0506ffc97a97309b33)), closes [#173](https://www.github.com/googleapis/java-storage-nio/issues/173) +* use projectId from CloudStorageConfig ([#429](https://www.github.com/googleapis/java-storage-nio/issues/429)) ([b6ec240](https://www.github.com/googleapis/java-storage-nio/commit/b6ec240876c66262be3ea99782f8abaec4372c96)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20210127-1.31.0 ([#428](https://www.github.com/googleapis/java-storage-nio/issues/428)) ([7de6c68](https://www.github.com/googleapis/java-storage-nio/commit/7de6c68e368455832bfb8404a8ef917b2db6d9e6)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.18.0 ([#355](https://www.github.com/googleapis/java-storage-nio/issues/355)) ([1c9e80f](https://www.github.com/googleapis/java-storage-nio/commit/1c9e80f7d9d0a3f747e1eac9900f3a40084f752a)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.10 ([#439](https://www.github.com/googleapis/java-storage-nio/issues/439)) ([b6c7718](https://www.github.com/googleapis/java-storage-nio/commit/b6c7718dc0bd7d697eca526ca275b72e602f7dfe)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.9 ([#356](https://www.github.com/googleapis/java-storage-nio/issues/356)) ([6cdc367](https://www.github.com/googleapis/java-storage-nio/commit/6cdc367e842d04dcb02ad52e9ae92a715748cd44)) + +### [0.122.5](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.4...v0.122.5) (2021-01-12) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-storage to v1.113.8 ([#314](https://www.github.com/googleapis/java-storage-nio/issues/314)) ([577eaed](https://www.github.com/googleapis/java-storage-nio/commit/577eaed2478d749515e89f4961a42004c5654a07)) +* update dependency com.google.guava:guava to v30.1-android ([#319](https://www.github.com/googleapis/java-storage-nio/issues/319)) ([8814bc3](https://www.github.com/googleapis/java-storage-nio/commit/8814bc31dedd85f5d3c50ce1891055bb334bab0b)) + +### [0.122.4](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.3...v0.122.4) (2020-12-16) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20201112-1.31.0 ([#312](https://www.github.com/googleapis/java-storage-nio/issues/312)) ([6f2e7e6](https://www.github.com/googleapis/java-storage-nio/commit/6f2e7e6e5d791a8164e9b5fcb7b1541230830c43)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.16.1 ([#320](https://www.github.com/googleapis/java-storage-nio/issues/320)) ([c56e6c0](https://www.github.com/googleapis/java-storage-nio/commit/c56e6c08d2d65123e0272a85fc36842a2e8986c4)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.17.0 ([#321](https://www.github.com/googleapis/java-storage-nio/issues/321)) ([c26c2a6](https://www.github.com/googleapis/java-storage-nio/commit/c26c2a6cb9c10e88454e008cba865f7eea5fae19)) + +### [0.122.3](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.2...v0.122.3) (2020-12-03) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.16.0 ([#305](https://www.github.com/googleapis/java-storage-nio/issues/305)) ([79245ae](https://www.github.com/googleapis/java-storage-nio/commit/79245ae5ded370d310f31939aa36572563437bb3)) + +### [0.122.2](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.1...v0.122.2) (2020-11-23) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20201112-1.30.10 ([#290](https://www.github.com/googleapis/java-storage-nio/issues/290)) ([56ec0eb](https://www.github.com/googleapis/java-storage-nio/commit/56ec0eb1b39665a6a93c48b62282a1e3c7d5c5f1)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.4 ([#286](https://www.github.com/googleapis/java-storage-nio/issues/286)) ([8a02f78](https://www.github.com/googleapis/java-storage-nio/commit/8a02f7880e849bbc6cb857b5610a8a88c770998c)) + +### [0.122.1](https://www.github.com/googleapis/java-storage-nio/compare/v0.122.0...v0.122.1) (2020-11-13) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.15.0 ([#287](https://www.github.com/googleapis/java-storage-nio/issues/287)) ([24938be](https://www.github.com/googleapis/java-storage-nio/commit/24938be0e6765ad391bdd9664dec7b33b384be06)) + +## [0.122.0](https://www.github.com/googleapis/java-storage-nio/compare/v0.121.2...v0.122.0) (2020-11-06) + + +### Features + +* update cloudstorageconfiguration to improve copy accross cross-bucket performance ([#168](https://www.github.com/googleapis/java-storage-nio/issues/168)) ([db74524](https://www.github.com/googleapis/java-storage-nio/commit/db74524d68487df71c80b122d8c0ff384dc9ace3)) +* **deps:** adopt flatten plugin and google-cloud-shared-dependencies ([#156](https://www.github.com/googleapis/java-storage-nio/issues/156)) ([510f6a5](https://www.github.com/googleapis/java-storage-nio/commit/510f6a5efc4a13b010020a8d67ec1511bbd46564)) + + +### Bug Fixes + +* FakeStorageRpc#list to filtering the files by bucket and prefix ([#208](https://www.github.com/googleapis/java-storage-nio/issues/208)) ([21f606e](https://www.github.com/googleapis/java-storage-nio/commit/21f606eca67b1c8a471ee28f7e2dd3851a0c493e)) +* implement writeWithResponse in FakeStorageRpc ([#187](https://www.github.com/googleapis/java-storage-nio/issues/187)) ([10ddfae](https://www.github.com/googleapis/java-storage-nio/commit/10ddfae3923f1d5e64b151293db238f1af433d03)) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20200611-1.30.9 ([#166](https://www.github.com/googleapis/java-storage-nio/issues/166)) ([3cab5f2](https://www.github.com/googleapis/java-storage-nio/commit/3cab5f25e081c0198d06903a42c7de3dbd34bbad)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20200727-1.30.10 ([#171](https://www.github.com/googleapis/java-storage-nio/issues/171)) ([62998f0](https://www.github.com/googleapis/java-storage-nio/commit/62998f0adfb22b194e2dd633532e13bf27a79479)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20200814-1.30.10 ([#206](https://www.github.com/googleapis/java-storage-nio/issues/206)) ([0d7cd44](https://www.github.com/googleapis/java-storage-nio/commit/0d7cd44e73b0b9456a8ba96cc6e6d0e60a0b7d4e)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20200927-1.30.10 ([#233](https://www.github.com/googleapis/java-storage-nio/issues/233)) ([91b7918](https://www.github.com/googleapis/java-storage-nio/commit/91b7918539186763ecaa377cfb6f3908105df279)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.10.0 ([#226](https://www.github.com/googleapis/java-storage-nio/issues/226)) ([83acbd7](https://www.github.com/googleapis/java-storage-nio/commit/83acbd766b06d66fc9a41385fdd49aa57d9a0291)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.10.1 ([#239](https://www.github.com/googleapis/java-storage-nio/issues/239)) ([bd1928d](https://www.github.com/googleapis/java-storage-nio/commit/bd1928da234dae6fc2fb1ad6658d89a9ebc7af82)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.10.2 ([#243](https://www.github.com/googleapis/java-storage-nio/issues/243)) ([4ca9869](https://www.github.com/googleapis/java-storage-nio/commit/4ca9869d99fc3d2e578fb86fbd9053f855f1e51c)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.12.0 ([#246](https://www.github.com/googleapis/java-storage-nio/issues/246)) ([3e6171b](https://www.github.com/googleapis/java-storage-nio/commit/3e6171b2d4c4c7832b62288faf30f767c5278762)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.12.1 ([#255](https://www.github.com/googleapis/java-storage-nio/issues/255)) ([73d7dd2](https://www.github.com/googleapis/java-storage-nio/commit/73d7dd2ec6ca371480955c844ffb5e6aed4f608f)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.13.0 ([#260](https://www.github.com/googleapis/java-storage-nio/issues/260)) ([54895cf](https://www.github.com/googleapis/java-storage-nio/commit/54895cf441f1e1723cf941aa1f6cab249acae6b8)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.14.1 ([#271](https://www.github.com/googleapis/java-storage-nio/issues/271)) ([058d7c9](https://www.github.com/googleapis/java-storage-nio/commit/058d7c9c1471846f3d897cac31cb5f6d42ce5a7d)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.2 ([#167](https://www.github.com/googleapis/java-storage-nio/issues/167)) ([3b14bbc](https://www.github.com/googleapis/java-storage-nio/commit/3b14bbc05d5658c57d4f272fdfdf152d7d9ee18d)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.3 ([#169](https://www.github.com/googleapis/java-storage-nio/issues/169)) ([4e7bac1](https://www.github.com/googleapis/java-storage-nio/commit/4e7bac10c2466d1407e8f1a6de838261e6c1b097)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.4 ([#189](https://www.github.com/googleapis/java-storage-nio/issues/189)) ([af492b8](https://www.github.com/googleapis/java-storage-nio/commit/af492b8068101727793e64e89da2778205fa08b6)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.6 ([#191](https://www.github.com/googleapis/java-storage-nio/issues/191)) ([9bbc1fc](https://www.github.com/googleapis/java-storage-nio/commit/9bbc1fc755271f2ac6b798d50f6c0ba8e022c589)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.9.0 ([#202](https://www.github.com/googleapis/java-storage-nio/issues/202)) ([cf312a3](https://www.github.com/googleapis/java-storage-nio/commit/cf312a3fe46ec446d1b5ee418849b3ab9abee87e)) +* update dependency com.google.cloud:google-cloud-storage to v1.110.0 ([#154](https://www.github.com/googleapis/java-storage-nio/issues/154)) ([fd6de38](https://www.github.com/googleapis/java-storage-nio/commit/fd6de38ee63376afff03c7e794dce1f4524a4375)) +* update dependency com.google.cloud:google-cloud-storage to v1.112.0 ([#199](https://www.github.com/googleapis/java-storage-nio/issues/199)) ([8a38817](https://www.github.com/googleapis/java-storage-nio/commit/8a3881732551bcef1d76f76026a29071fc50dd43)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.0 ([#205](https://www.github.com/googleapis/java-storage-nio/issues/205)) ([068002e](https://www.github.com/googleapis/java-storage-nio/commit/068002e1145418ecb848ed066346d1fc6ec14807)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.1 ([#212](https://www.github.com/googleapis/java-storage-nio/issues/212)) ([acc1fe8](https://www.github.com/googleapis/java-storage-nio/commit/acc1fe8f23c5926000c7a869ca9a71372850c46a)) +* update dependency com.google.cloud:google-cloud-storage to v1.113.2 ([#266](https://www.github.com/googleapis/java-storage-nio/issues/266)) ([9d2c11d](https://www.github.com/googleapis/java-storage-nio/commit/9d2c11d009394f3f3ffd1b115b97b350aa64d1b7)) +* update dependency com.google.guava:guava to v30 ([#263](https://www.github.com/googleapis/java-storage-nio/issues/263)) ([4e81dab](https://www.github.com/googleapis/java-storage-nio/commit/4e81daba421e0e42695203c6fe74670b32a3f576)) +* update dependency org.mockito:mockito-core to v3.4.4 ([#170](https://www.github.com/googleapis/java-storage-nio/issues/170)) ([3e06bd5](https://www.github.com/googleapis/java-storage-nio/commit/3e06bd5b07065acb1b41a03ebbb862487c046eeb)) +* update dependency org.mockito:mockito-core to v3.5.10 ([#203](https://www.github.com/googleapis/java-storage-nio/issues/203)) ([33fdc31](https://www.github.com/googleapis/java-storage-nio/commit/33fdc315a5b4bccdebb13262cbc5868011abe444)) +* update dependency org.mockito:mockito-core to v3.5.11 ([#214](https://www.github.com/googleapis/java-storage-nio/issues/214)) ([575c308](https://www.github.com/googleapis/java-storage-nio/commit/575c30882ab84e1b2b1ce34d5176b4f8688a8136)) +* update dependency org.mockito:mockito-core to v3.5.7 ([#194](https://www.github.com/googleapis/java-storage-nio/issues/194)) ([8cc7616](https://www.github.com/googleapis/java-storage-nio/commit/8cc76166baf3f5f49223bb8eafc8704679c4d427)) + +### [0.121.2](https://www.github.com/googleapis/java-storage-nio/compare/v0.120.1...v0.121.2) (2020-06-18) + + +### Bug Fixes + +* update FakeStorageRpc to extend StorageRpcTestBase [#135](https://www.github.com/googleapis/java-storage-nio/issues/135) ([7614295](https://www.github.com/googleapis/java-storage-nio/commit/761429571eea15b11b2d44b4f5a2c65b4f649484)) + +### [0.121.1](https://www.github.com/googleapis/java-storage-nio/compare/v0.120.0...v0.121.1) (2020-06-16) + + +### Dependencies + +* update dependency com.google.apis:google-api-services-storage to v1-rev20200410-1.30.9 ([#97](https://www.github.com/googleapis/java-storage-nio/issues/97)) ([b86aed8](https://www.github.com/googleapis/java-storage-nio/commit/b86aed82b7959f2866d3430a4ab79f5983ea6e6f)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20200430-1.30.9 ([#110](https://www.github.com/googleapis/java-storage-nio/issues/110)) ([9350ed4](https://www.github.com/googleapis/java-storage-nio/commit/9350ed430c656fc3622025299008de934c80b99d)) +* update dependency com.google.auto.value:auto-value to v1.7.1 ([#98](https://www.github.com/googleapis/java-storage-nio/issues/98)) ([3f07925](https://www.github.com/googleapis/java-storage-nio/commit/3f07925cb97b578774d721640588a83d3426b58d)) +* update dependency com.google.auto.value:auto-value to v1.7.2 ([#104](https://www.github.com/googleapis/java-storage-nio/issues/104)) ([ff05184](https://www.github.com/googleapis/java-storage-nio/commit/ff05184b1c151ebea2dd45a818cdf5d65f437adc)) +* update dependency com.google.auto.value:auto-value to v1.7.3 ([#131](https://www.github.com/googleapis/java-storage-nio/issues/131)) ([a225b87](https://www.github.com/googleapis/java-storage-nio/commit/a225b878e793e111390ce9e403b2a0018cff7c4c)) +* update dependency com.google.auto.value:auto-value-annotations to v1.7.1 ([#99](https://www.github.com/googleapis/java-storage-nio/issues/99)) ([6aef5d6](https://www.github.com/googleapis/java-storage-nio/commit/6aef5d60f1d30e861b5ca6899f4942c3831c1bf8)) +* update dependency com.google.auto.value:auto-value-annotations to v1.7.2 ([#105](https://www.github.com/googleapis/java-storage-nio/issues/105)) ([c0f3611](https://www.github.com/googleapis/java-storage-nio/commit/c0f36119bafbef4536f2ec66a8c417fe3eae624f)) +* update dependency com.google.auto.value:auto-value-annotations to v1.7.3 ([#132](https://www.github.com/googleapis/java-storage-nio/issues/132)) ([3a46dd5](https://www.github.com/googleapis/java-storage-nio/commit/3a46dd59a3777aea490e6100a20425183d982cb3)) +* update dependency com.google.cloud:google-cloud-core-bom to v1.93.5 ([#115](https://www.github.com/googleapis/java-storage-nio/issues/115)) ([7088962](https://www.github.com/googleapis/java-storage-nio/commit/70889626de81793e19c85b8c886dbea3456449d0)) +* update dependency com.google.cloud:google-cloud-storage to v1.108.0 ([#100](https://www.github.com/googleapis/java-storage-nio/issues/100)) ([6a9a281](https://www.github.com/googleapis/java-storage-nio/commit/6a9a28105561d4408110d54bb92d4e1099ed7544)) +* update dependency com.google.cloud:google-cloud-storage to v1.109.0 ([#133](https://www.github.com/googleapis/java-storage-nio/issues/133)) ([cb23faf](https://www.github.com/googleapis/java-storage-nio/commit/cb23faf0132db61bfdc8d0c6409dc9fa98ae2d11)) +* update dependency com.google.http-client:google-http-client-bom to v1.35.0 ([#93](https://www.github.com/googleapis/java-storage-nio/issues/93)) ([0683dd2](https://www.github.com/googleapis/java-storage-nio/commit/0683dd21731b86d22a9d9f7201dad9deb1878d8a)) +* update dependency com.google.protobuf:protobuf-bom to v3.12.0 ([#111](https://www.github.com/googleapis/java-storage-nio/issues/111)) ([4e592fb](https://www.github.com/googleapis/java-storage-nio/commit/4e592fb506a9b85c8d38c662605d4f97ad60b730)) +* update dependency com.google.protobuf:protobuf-bom to v3.12.1 ([#114](https://www.github.com/googleapis/java-storage-nio/issues/114)) ([1b9b5e5](https://www.github.com/googleapis/java-storage-nio/commit/1b9b5e5a780f44abcba2b7eb25a2a46f5bf8c1ba)) +* update dependency com.google.protobuf:protobuf-bom to v3.12.2 ([#116](https://www.github.com/googleapis/java-storage-nio/issues/116)) ([c297693](https://www.github.com/googleapis/java-storage-nio/commit/c297693dee0747d259ff1bcbc40d05218a782709)) + + +### Documentation + +* update CONTRIBUTING.md to include code formatting ([#534](https://www.github.com/googleapis/java-storage-nio/issues/534)) ([#103](https://www.github.com/googleapis/java-storage-nio/issues/103)) ([c329a58](https://www.github.com/googleapis/java-storage-nio/commit/c329a58086501f77d7c7e0db44eb06cb68eb933b)) + +## [0.121.0](https://www.github.com/googleapis/java-storage-nio/compare/0.120.0-alpha...v0.121.0) (2020-04-24) + + +### Features + +* make repo releasable, add parent pom ([#6](https://www.github.com/googleapis/java-storage-nio/issues/6)) ([6bca496](https://www.github.com/googleapis/java-storage-nio/commit/6bca49650fa5bac4682836149c48db95908909a5)) + + +### Bug Fixes + +* point to correct api documentation ([#68](https://www.github.com/googleapis/java-storage-nio/issues/68)) ([43675b6](https://www.github.com/googleapis/java-storage-nio/commit/43675b6416d9bec224704d92b8b9abf1fc9db10b)) + + +### Dependencies + +* update core dependencies ([#27](https://www.github.com/googleapis/java-storage-nio/issues/27)) ([b59ae15](https://www.github.com/googleapis/java-storage-nio/commit/b59ae1587bc08714c549b4d22dc078e16ef48d98)) +* update core dependencies to v29 ([#78](https://www.github.com/googleapis/java-storage-nio/issues/78)) ([63d9a56](https://www.github.com/googleapis/java-storage-nio/commit/63d9a56dc0a4e7c7e1eb9e83a6fda6d715d82edb)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20191127-1.30.8 ([#33](https://www.github.com/googleapis/java-storage-nio/issues/33)) ([bd4d1b2](https://www.github.com/googleapis/java-storage-nio/commit/bd4d1b2c6dfdf2497f68fb328778e751c3a0813a)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20191127-1.30.9 ([#48](https://www.github.com/googleapis/java-storage-nio/issues/48)) ([6a63920](https://www.github.com/googleapis/java-storage-nio/commit/6a63920d440229b0657f5464b122c3eb8bb6d882)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20200226-1.30.9 ([#53](https://www.github.com/googleapis/java-storage-nio/issues/53)) ([5319c7e](https://www.github.com/googleapis/java-storage-nio/commit/5319c7e63de772983884d9d2e275102aab30055c)) +* update dependency com.google.apis:google-api-services-storage to v1-rev20200326-1.30.9 ([#77](https://www.github.com/googleapis/java-storage-nio/issues/77)) ([315cc86](https://www.github.com/googleapis/java-storage-nio/commit/315cc8660bf670098c498397f6c7f6b7b9d4376c)) +* update dependency com.google.cloud:google-cloud-core-bom to v1.93.3 ([#43](https://www.github.com/googleapis/java-storage-nio/issues/43)) ([58a7c03](https://www.github.com/googleapis/java-storage-nio/commit/58a7c038fa3bd2b72486aa40019153004b7e7958)) +* update dependency com.google.cloud:google-cloud-storage to v1.104.0 ([#26](https://www.github.com/googleapis/java-storage-nio/issues/26)) ([020d7cf](https://www.github.com/googleapis/java-storage-nio/commit/020d7cf350d05dfd30be14ad7aab8ee051f4797f)) +* update dependency com.google.cloud:google-cloud-storage to v1.105.2 ([#44](https://www.github.com/googleapis/java-storage-nio/issues/44)) ([1bfa769](https://www.github.com/googleapis/java-storage-nio/commit/1bfa7697d39f7856defe997da1016986632be4cd)) +* update dependency com.google.cloud:google-cloud-storage to v1.106.0 ([#56](https://www.github.com/googleapis/java-storage-nio/issues/56)) ([57fff87](https://www.github.com/googleapis/java-storage-nio/commit/57fff872878942210db056974132c4123ae08e0b)) +* update dependency com.google.cloud.samples:shared-configuration to v1.0.13 ([#63](https://www.github.com/googleapis/java-storage-nio/issues/63)) ([ecac9a9](https://www.github.com/googleapis/java-storage-nio/commit/ecac9a9a839cfe9649ab53b5eb675c16ddeeea6a)) +* update dependency com.google.guava:guava to v23 ([#52](https://www.github.com/googleapis/java-storage-nio/issues/52)) ([ef0baaa](https://www.github.com/googleapis/java-storage-nio/commit/ef0baaa6805b0fa57854af8ae903262c55ee7d5d)) +* update dependency com.google.http-client:google-http-client-bom to v1.34.2 ([#25](https://www.github.com/googleapis/java-storage-nio/issues/25)) ([c1c3269](https://www.github.com/googleapis/java-storage-nio/commit/c1c326989940f660df0095b6a0efe3253c2da1b0)) +* update dependency com.google.protobuf:protobuf-bom to v3.11.4 ([#30](https://www.github.com/googleapis/java-storage-nio/issues/30)) ([ddd0555](https://www.github.com/googleapis/java-storage-nio/commit/ddd05553cb5af81b0a56277076c81a2fa4ad1abd)) diff --git a/java-storage-nio/EnableAutoValue.txt b/java-storage-nio/EnableAutoValue.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java-storage-nio/README.md b/java-storage-nio/README.md new file mode 100644 index 000000000000..8d5ecdc153a2 --- /dev/null +++ b/java-storage-nio/README.md @@ -0,0 +1,291 @@ +# Google NIO Filesystem Provider for Google Cloud Storage Client for Java + +Java idiomatic client for [NIO Filesystem Provider for Google Cloud Storage][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] + +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. + + +## Quickstart + +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file + +```xml + + + + com.google.cloud + libraries-bom + 22.0.0 + pom + import + + + + + + + com.google.cloud + google-cloud-nio + + + +``` + +If you are using Maven without BOM, add this to your dependencies: + + +```xml + + com.google.cloud + google-cloud-nio + 0.123.10 + + +``` + +If you are using Gradle 5.x or later, add this to your dependencies + +```Groovy +implementation platform('com.google.cloud:libraries-bom:23.0.0') + +implementation 'com.google.cloud:google-cloud-nio' +``` +If you are using Gradle without BOM, add this to your dependencies + +```Groovy +implementation 'com.google.cloud:google-cloud-nio:0.123.10' +``` + +If you are using SBT, add this to your dependencies + +```Scala +libraryDependencies += "com.google.cloud" % "google-cloud-nio" % "0.123.10" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired NIO Filesystem Provider for Google Cloud Storage APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the NIO Filesystem Provider for Google Cloud Storage API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the NIO Filesystem Provider for Google Cloud Storage [API enabled][enable-api]. + +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud SDK][cloud-sdk] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `google-cloud-nio` library. See the [Quickstart](#quickstart) section +to add `google-cloud-nio` as a dependency in your code. + +## About NIO Filesystem Provider for Google Cloud Storage + + +[NIO Filesystem Provider for Google Cloud Storage][product-docs] provides a Google Cloud Storage extension for Java's NIO Filesystem. + +See the [NIO Filesystem Provider for Google Cloud Storage client library docs][javadocs] to learn how to +use this NIO Filesystem Provider for Google Cloud Storage Client Library. + + +#### About Google Cloud Storage + +[Google Cloud Storage](https://cloud.google.com/storage/) is a durable and highly available +object storage service. Google Cloud Storage is almost infinitely scalable +and guarantees consistency: when a write succeeds, the latest copy of the +object will be returned to any GET, globally. + +See the [Google Cloud Storage docs](https://cloud.google.com/storage/docs/signup?hl=en) for more details +on how to activate Cloud Storage for your project. + +#### About Java NIO Providers + +Java NIO Providers is an extension mechanism that is part of Java and allows +third parties to extend Java's [normal File API](https://docs.oracle.com/javase/7/docs/api/java/nio/file/Files.html) to support +additional filesystems. + +#### Accessing files + +The simplest way to get started is with `Paths` and `Files`: + +```java +Path path = Paths.get(URI.create("gs://bucket/lolcat.csv")); +List lines = Files.readAllLines(path, StandardCharsets.UTF_8); +``` + +If you know the paths will point to Google Cloud Storage, you can also use the +direct formulation: + +```java +try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + Path path = fs.getPath("lolcat.csv"); + List lines = Files.readAllLines(path, StandardCharsets.UTF_8); +} +``` + +Once you have a `Path` you can use it as you would a normal file. For example +you can use `InputStream` and `OutputStream` for streaming: + +```java +try (InputStream input = Files.openInputStream(path)) { + // ... +} +``` + +You can also set various attributes using CloudStorageOptions static helpers: + +```java +Files.write(csvPath, csvLines, StandardCharsets.UTF_8, + withMimeType(MediaType.CSV_UTF8), + withoutCaching()); +``` + +#### Limitations + +This library is usable, but not yet complete. The following features are not +yet implemented: + * Resuming upload or download + * Generations + * File attributes + * (more - list is not exhaustive) + +Some features are not on the roadmap: this library would be a poor choice to +mirror a local filesystem onto the cloud because Google Cloud Storage has a +different set of features from your local disk. This library, by design, +does not mask those differences. Rather, it aims to expose the common +subset via a familiar interface. + +**NOTE:** Cloud Storage uses a flat namespace and therefore doesn't support real +directories. So this library supports what's known as "pseudo-directories". Any +path that includes a trailing slash, will be considered a directory. It will +always be assumed to exist, without performing any I/O. Paths without the trailing +slash will result in an I/O operation to check a file is present in that "directory". +This allows you to do path manipulation in the same manner as you would with the normal UNIX file +system implementation. Using this feature with buckets or "directory" paths that do not exist +is not recommended, as at the time I/O is performed the failure may not be handled gracefully. +You can disable this feature with `CloudStorageConfiguration.usePseudoDirectories()`. + +#### Complete source code + +There are examples in [google-cloud-nio-examples](google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/) +for your perusal. + + + + + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +## Supported Java Versions + +Java 7 or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + + +This library follows [Semantic Versioning](http://semver.org/). + + +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. + + +## Contributing + + +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. + + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: https://cloud.google.com/storage/docs +[javadocs]: https://googleapis.dev/java/google-cloud-nio/latest +[stability-image]: https://img.shields.io/badge/stability-beta-yellow +[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-nio.svg +[maven-version-link]: https://search.maven.org/search?q=g:com.google.cloud%20AND%20a:google-cloud-nio&core=gav +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-sdk]: https://cloud.google.com/sdk/ +[troubleshooting]: https://github.com/googleapis/google-cloud-common/blob/main/troubleshooting/readme.md#troubleshooting +[contributing]: https://github.com/googleapis/java-storage-nio/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/googleapis/java-storage-nio/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/googleapis/java-storage-nio/blob/main/LICENSE + +[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid=storage.googleapis.com +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/java-storage-nio/google-cloud-nio-bom/pom.xml b/java-storage-nio/google-cloud-nio-bom/pom.xml new file mode 100644 index 000000000000..01700564f3a1 --- /dev/null +++ b/java-storage-nio/google-cloud-nio-bom/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + com.google.cloud + google-cloud-nio-bom + 0.128.14 + pom + + com.google.cloud + google-cloud-pom-parent + 1.83.0-SNAPSHOT + ../../google-cloud-pom-parent/pom.xml + + + Google Cloud NIO BOM + https://github.com/googleapis/google-cloud-java + + BOM for Google Cloud NIO + + + + Google LLC + + + + + suztomo + Tomo Suzuki + suztomo@google.com + Google LLC + + Developer + + + + + + scm:git:https://github.com/googleapis/google-cloud-java.git + scm:git:git@github.com:googleapis/google-cloud-java.git + https://github.com/googleapis/google-cloud-java + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + com.google.cloud + google-cloud-nio + 0.128.14 + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + + org.apache.maven.plugins + maven-site-plugin + + + false + + + + + diff --git a/java-storage-nio/google-cloud-nio-examples/pom.xml b/java-storage-nio/google-cloud-nio-examples/pom.xml new file mode 100644 index 000000000000..a79a1301b591 --- /dev/null +++ b/java-storage-nio/google-cloud-nio-examples/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + google-cloud-nio-examples + 0.128.14 + jar + Google Cloud NIO Examples + https://github.com/googleapis/google-cloud-java + + Examples for google-cloud-nio (Java idiomatic client for Google Cloud + Storage). + + + com.google.cloud + google-cloud-nio-parent + 0.128.14 + + + + com.google.guava + guava + + + + google-cloud-nio-examples + + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + org.codehaus.mojo + clirr-maven-plugin + + true + + + + org.codehaus.mojo + flatten-maven-plugin + + + + diff --git a/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java b/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java new file mode 100644 index 000000000000..75ac7f1cf704 --- /dev/null +++ b/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java @@ -0,0 +1,117 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.examples.nio; + +import com.google.common.base.Stopwatch; +import com.google.common.io.BaseEncoding; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.util.concurrent.TimeUnit; + +/** + * CountBytes will read through the whole file given as input. + * + *

This example shows how to read a file size using NIO. File.size returns the size of the file + * as saved in Storage metadata. This class also shows how to read all of the file's contents using + * NIO, computes a MD5 hash, and reports how long it took. + * + *

See the + * README for compilation instructions. Run this code with + * + *

{@code target/appassembler/bin/CountBytes }
+ */ +public class CountBytes { + + /** See the class documentation. */ + public static void main(String[] args) throws IOException { + if (args.length == 0 || args[0].equals("--help")) { + help(); + return; + } + for (String a : args) { + countFile(a); + } + } + + /** + * Print the length of the indicated file. + * + *

This uses the normal Java NIO Api, so it can take advantage of any installed NIO Filesystem + * provider without any extra effort. + */ + private static void countFile(String fname) { + // large buffers pay off + final int bufSize = 50 * 1024 * 1024; + try { + Path path = Paths.get(new URI(fname)); + long size = Files.size(path); + System.out.println(fname + ": " + size + " bytes."); + ByteBuffer buf = ByteBuffer.allocate(bufSize); + System.out.println("Reading the whole file..."); + Stopwatch sw = Stopwatch.createStarted(); + try (SeekableByteChannel chan = Files.newByteChannel(path)) { + long total = 0; + int readCalls = 0; + MessageDigest md = MessageDigest.getInstance("MD5"); + while (chan.read(buf) > 0) { + readCalls++; + md.update(buf.array(), 0, buf.position()); + total += buf.position(); + buf.flip(); + } + readCalls++; // We must count the last call + long elapsed = sw.elapsed(TimeUnit.SECONDS); + System.out.println( + "Read all " + + total + + " bytes in " + + elapsed + + "s. " + + "(" + + readCalls + + " calls to chan.read)"); + String hex = String.valueOf(BaseEncoding.base16().encode(md.digest())); + System.out.println("The MD5 is: 0x" + hex); + if (total != size) { + System.out.println( + "Wait, this doesn't match! We saw " + + total + + " bytes, " + + "yet the file size is listed at " + + size + + " bytes."); + } + } + } catch (Exception ex) { + System.out.println(fname + ": " + ex.toString()); + } + } + + private static void help() { + String[] help = {"The argument is a ", "and we show the length of that file."}; + for (String s : help) { + System.out.println(s); + } + } +} diff --git a/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/ParallelCountBytes.java b/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/ParallelCountBytes.java new file mode 100644 index 000000000000..820f922f7147 --- /dev/null +++ b/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/ParallelCountBytes.java @@ -0,0 +1,170 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.examples.nio; + +import com.google.common.base.Stopwatch; +import com.google.common.io.BaseEncoding; +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * ParallelCountBytes will read through the whole file given as input. + * + *

This example shows how to go through all the contents of a file, in order, using multithreaded + * NIO reads. It prints a MD5 hash and reports how long it took. + * + *

See the + * README for compilation instructions. Run this code with + * + *

{@code target/appassembler/bin/ParallelCountBytes }
+ */ +public class ParallelCountBytes { + + /** + * WorkUnit holds a buffer and the instructions for what to put in it. + * + *

Use it like this: + * + *

    + *
  1. call() + *
  2. the data is now in buf, you can access it directly + *
  3. if need more, call resetForIndex(...) and go back to the top. + *
  4. else, call close() + *
+ */ + private static class WorkUnit implements Callable, Closeable { + public final ByteBuffer buf; + final SeekableByteChannel chan; + final int blockSize; + int blockIndex; + + public WorkUnit(SeekableByteChannel chan, int blockSize, int blockIndex) { + this.chan = chan; + this.buf = ByteBuffer.allocate(blockSize); + this.blockSize = blockSize; + this.blockIndex = blockIndex; + } + + @Override + public WorkUnit call() throws IOException { + long pos = ((long) blockSize) * blockIndex; + if (pos > chan.size()) { + return this; + } + chan.position(pos); + // read until buffer is full, or EOF + while (chan.read(buf) > 0) {} + ; + return this; + } + + public WorkUnit resetForIndex(int blockIndex) { + this.blockIndex = blockIndex; + buf.flip(); + return this; + } + + public void close() throws IOException { + chan.close(); + } + } + + /** See the class documentation. */ + public static void main(String[] args) throws Exception { + if (args.length == 0 || args[0].equals("--help")) { + help(); + return; + } + for (String a : args) { + countFile(a); + } + } + + /** + * Print the length and MD5 of the indicated file. + * + *

This uses the normal Java NIO Api, so it can take advantage of any installed NIO Filesystem + * provider without any extra effort. + */ + private static void countFile(String fname) throws Exception { + // large buffers pay off + final int bufSize = 50 * 1024 * 1024; + Queue> work = new ArrayDeque<>(); + Path path = Paths.get(new URI(fname)); + long size = Files.size(path); + System.out.println(fname + ": " + size + " bytes."); + int nThreads = (int) Math.ceil(size / (double) bufSize); + if (nThreads > 4) nThreads = 4; + System.out.println("Reading the whole file using " + nThreads + " threads..."); + Stopwatch sw = Stopwatch.createStarted(); + long total = 0; + MessageDigest md = MessageDigest.getInstance("MD5"); + + ExecutorService exec = Executors.newFixedThreadPool(nThreads); + int blockIndex; + for (blockIndex = 0; blockIndex < nThreads; blockIndex++) { + work.add(exec.submit(new WorkUnit(Files.newByteChannel(path), bufSize, blockIndex))); + } + while (!work.isEmpty()) { + WorkUnit full = work.remove().get(); + md.update(full.buf.array(), 0, full.buf.position()); + total += full.buf.position(); + if (full.buf.hasRemaining()) { + full.close(); + } else { + work.add(exec.submit(full.resetForIndex(blockIndex++))); + } + } + exec.shutdown(); + + long elapsed = sw.elapsed(TimeUnit.SECONDS); + System.out.println("Read all " + total + " bytes in " + elapsed + "s. "); + String hex = String.valueOf(BaseEncoding.base16().encode(md.digest())); + System.out.println("The MD5 is: 0x" + hex); + if (total != size) { + System.out.println( + "Wait, this doesn't match! We saw " + + total + + " bytes, " + + "yet the file size is listed at " + + size + + " bytes."); + } + } + + private static void help() { + String[] help = {"The argument is a ", "and we show the length of that file."}; + for (String s : help) { + System.out.println(s); + } + } +} diff --git a/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/Stat.java b/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/Stat.java new file mode 100644 index 000000000000..e68b0ea1d877 --- /dev/null +++ b/java-storage-nio/google-cloud-nio-examples/src/main/java/com/google/cloud/examples/nio/Stat.java @@ -0,0 +1,124 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.examples.nio; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.spi.FileSystemProvider; + +/** + * Stat is a super-simple program that just displays the size of the file passed as argument. + * + *

It's meant to be used to test Google Cloud's integration with Java NIO. + * + *

You can either use the '--check' argument to see whether Google Cloud Storage is enabled, or + * you can directly pass in a Google Cloud Storage file name to use. In that case you have to be + * logged in (using e.g. the gcloud auth command). + * + *

See the + * README for compilation instructions. Run this code with + * + *

{@code target/appassembler/bin/Stat --help | --check | --list | }
+ * + *

In short, this version (in google-cloud-examples) is in a package that lists google-cloud-nio + * as a dependency, so it will work directly without having to do any special work. + */ +public class Stat { + + /** See the class documentation. */ + public static void main(String[] args) throws IOException { + if (args.length == 0 || args[0].equals("--help")) { + help(); + return; + } + if (args[0].equals("--list")) { + listFilesystems(); + return; + } + if (args[0].equals("--check")) { + checkGcs(); + return; + } + for (String a : args) { + statFile(a); + } + } + + /** + * Print the length of the indicated file. + * + *

This uses the normal Java NIO Api, so it can take advantage of any installed NIO Filesystem + * provider without any extra effort. + */ + private static void statFile(String fname) { + try { + Path path = Paths.get(new URI(fname)); + long size = Files.size(path); + System.out.println(fname + ": " + size + " bytes."); + } catch (Exception ex) { + System.out.println(fname + ": " + ex.toString()); + } + } + + private static void help() { + String[] help = { + "The arguments can be one of:", + " * ", + " to display the length of that file.", + "", + " * --list", + " to list the filesystem providers.", + "", + " * --check", + " to double-check the Google Cloud Storage provider is installed.", + "", + "The purpose of this tool is to demonstrate that the Google Cloud NIO filesystem provider", + "can add Google Cloud Storage support to programs not explicitly designed for it.", + "", + "This tool normally knows nothing of Google Cloud Storage. If you pass it --check", + "or a Google Cloud Storage file name (e.g. gs://mybucket/myfile), it will show an error.", + "However, by just adding the google-cloud-nio jar as a dependency and recompiling, this", + "tool is made aware of gs:// paths and can access files on the cloud.", + "", + "The Google Cloud NIO filesystem provider can similarly enable existing Java 7 programs", + "to read and write cloud files, even if they have no special built-in cloud support." + }; + for (String s : help) { + System.out.println(s); + } + } + + private static void listFilesystems() { + System.out.println("Installed filesystem providers:"); + for (FileSystemProvider p : FileSystemProvider.installedProviders()) { + System.out.println(" " + p.getScheme()); + } + } + + private static void checkGcs() { + FileSystem fs = FileSystems.getFileSystem(URI.create("gs://domain-registry-alpha")); + System.out.println("Success! We can instantiate a gs:// filesystem."); + System.out.println("isOpen: " + fs.isOpen()); + System.out.println("isReadOnly: " + fs.isReadOnly()); + } +} diff --git a/java-storage-nio/google-cloud-nio-retrofit/README.md b/java-storage-nio/google-cloud-nio-retrofit/README.md new file mode 100644 index 000000000000..319fad186fd0 --- /dev/null +++ b/java-storage-nio/google-cloud-nio-retrofit/README.md @@ -0,0 +1,53 @@ +Example of adding the Google Cloud Storage NIO Provider to a legacy jar +======================================================================= + +This project shows how to add (retrofit) Google Cloud Storage capabilities to a +jar file for a Java 7 application that uses Java NIO without the need to +recompile. + +Note that whenever possible, you instead want to recompile the app and use the normal +dependency mechanism to add a dependency to google-cloud-nio. You can see examples of +this in the +[google-cloud-examples](https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-examples) +project, +[under nio](https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-examples/src/main/java/com/google/cloud/examples/nio). + +To run this example: + +1. Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled. + +2. Log in using gcloud SDK (`gcloud auth login` in command line) + +3. Compile the JAR with: + ``` + mvn package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true + ``` + +4. Run the sample with: + +[//]: # ({x-version-update-start:google-cloud-nio:current}) + ``` + java -cp google-cloud-nio/target/google-cloud-nio-0.120.1-alpha-SNAPSHOT-shaded.jar:google-cloud-nio-retrofit/target/google-cloud-nio-retrofit-0.120.1-alpha-SNAPSHOT.jar com.google.cloud.nio.retrofit.ListFilesystems + ``` + + Notice that it lists Google Cloud Storage ("gs"), which it wouldn't if you ran it without the NIO jar: + ``` + java -cp google-cloud-nio-retrofit/target/google-cloud-nio-retrofit-0.120.1-alpha-SNAPSHOT.jar com.google.cloud.nio.retrofit.ListFilesystems + ``` +[//]: # ({x-version-update-end}) + +The sample doesn't have anything about Google Cloud Storage in it. It gets that ability from the NIO +jar that we're adding to the classpath. We use the NIO "shaded" jar for this purpose as it +also includes the dependencies for google-cloud-nio. +The underlying mechanism is Java's standard +[ServiceLoader](https://docs.oracle.com/javase/7/docs/api/java/util/ServiceLoader.html) +facility, the +[standard way](http://docs.oracle.com/javase/7/docs/technotes/guides/io/fsp/filesystemprovider.html) +to plug in NIO providers like this one. + +If you have access to a project's source code you can also simply add google-cloud-nio as +a dependency and let Maven pull in the required dependencies (this is what the NIO unit tests do). +This approach is preferable as the shaded jar approach may waste memory on multiple copies of +dependencies. + +[developers-console]:https://console.developers.google.com/ diff --git a/java-storage-nio/google-cloud-nio-retrofit/pom.xml b/java-storage-nio/google-cloud-nio-retrofit/pom.xml new file mode 100644 index 000000000000..057db7e3ef61 --- /dev/null +++ b/java-storage-nio/google-cloud-nio-retrofit/pom.xml @@ -0,0 +1,35 @@ + + + 4.0.0 + google-cloud-nio-retrofit + 0.128.14 + jar + Google Cloud NIO Retrofit Example + https://github.com/googleapis/google-cloud-java + + Demonstrates how to use the google-cloud-nio jar to add Google Cloud Storage functionality to legacy code. + + + com.google.cloud + google-cloud-nio-parent + 0.128.14 + + + google-cloud-nio-retrofit + + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + org.codehaus.mojo + flatten-maven-plugin + + + + \ No newline at end of file diff --git a/java-storage-nio/google-cloud-nio-retrofit/src/main/java/com/google/cloud/nio/retrofit/ListFilesystems.java b/java-storage-nio/google-cloud-nio-retrofit/src/main/java/com/google/cloud/nio/retrofit/ListFilesystems.java new file mode 100644 index 000000000000..831350bf8dad --- /dev/null +++ b/java-storage-nio/google-cloud-nio-retrofit/src/main/java/com/google/cloud/nio/retrofit/ListFilesystems.java @@ -0,0 +1,44 @@ +/* + * Copyright 2015 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.nio.retrofit; + +import java.io.IOException; +import java.nio.file.spi.FileSystemProvider; + +/** + * ListFilesystems is a simple program that lists the available NIO filesystems. + * + *

Note that the code here doesn't do anything special to link in the google-cloud-nio code. It + * doesn't use any of its methods. + * + *

The README explains how, by just adding the google-cloud-nio JAR in the classpath, this + * program will magically gain the ability to read files on Google Cloud Storage. + */ +public class ListFilesystems { + + /** See the class documentation. */ + public static void main(String[] args) throws IOException { + listFilesystems(); + } + + private static void listFilesystems() { + System.out.println("Installed filesystem providers:"); + for (FileSystemProvider p : FileSystemProvider.installedProviders()) { + System.out.println(" " + p.getScheme()); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/EnableAutoValue.txt b/java-storage-nio/google-cloud-nio/EnableAutoValue.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java-storage-nio/google-cloud-nio/pom.xml b/java-storage-nio/google-cloud-nio/pom.xml new file mode 100644 index 000000000000..305f416c4b8f --- /dev/null +++ b/java-storage-nio/google-cloud-nio/pom.xml @@ -0,0 +1,201 @@ + + + 4.0.0 + google-cloud-nio + 0.128.14 + jar + Google Cloud NIO + https://github.com/googleapis/google-cloud-java + + FileSystemProvider for Java NIO to access Google Cloud Storage transparently. + + + com.google.cloud + google-cloud-nio-parent + 0.128.14 + + + google-cloud-nio + + + + com.google.cloud + google-cloud-storage + + + com.google.apis + google-api-services-storage + + + com.google.guava + guava + + + com.google.api + gax + 2.76.1-SNAPSHOT + + + com.google.cloud + google-cloud-core + 2.66.1-SNAPSHOT + + + com.google.http-client + google-http-client + + + com.google.http-client + google-http-client-gson + + + com.google.auto.value + auto-value-annotations + + + javax.inject + javax.inject + 1 + + + com.google.code.findbugs + jsr305 + + + com.google.api + gax + testlib + test + 2.76.1-SNAPSHOT + + + + junit + junit + test + + + com.google.guava + guava-testlib + test + + + com.google.truth + truth + test + + + com.google.auth + google-auth-library-credentials + test + + + + org.mockito + mockito-core + 4.11.0 + test + + + com.google.cloud + google-cloud-core + test + tests + 2.66.1-SNAPSHOT + + + + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + + org.apache.maven.plugins + maven-shade-plugin + + 3.2.4 + + + org.ow2.asm + asm + 9.9.1 + + + org.ow2.asm + asm-commons + 9.9.1 + + + + true + + + com + shaded.cloud_nio.com + + com.google.** + com.fasterxml.** + + + com.google.cloud.storage.** + com.google.auto.** + + + + org + shaded.cloud_nio.org + + org.apache.** + org.threeten.** + org.codehaus.** + + + + io + shaded.cloud_nio.io + + io.opencensus.** + io.grpc.** + + + + okio + shaded.cloud_nio.okio + + + google + shaded.cloud_nio.google + + + + + + + + + package + + shade + + + + + + maven-dependency-plugin + + + + com.google.api:gax:jar + javax.annotation:javax.annotation-api + + + + + + diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java new file mode 100644 index 000000000000..26b35fcc8963 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java @@ -0,0 +1,316 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import java.io.EOFException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.util.Map; +import javax.annotation.Nullable; +import javax.net.ssl.SSLException; + +/** Configuration for {@link CloudStorageFileSystem} instances. */ +@AutoValue +public abstract class CloudStorageConfiguration { + + public static final CloudStorageConfiguration DEFAULT = builder().build(); + + // Users can change this: then this affects every filesystem object created + // later, including via SPI. This is meant to be done only once, at the beginning + // of some main program, in order to force all libraries to use some settings we like. + // Libraries should never call this. It'll cause surprise to the writers of the main + // program and they'll be unhappy. Instead, create your own filesystem object with the + // right configuration and pass it along. + private static CloudStorageConfiguration userSpecifiedDefault = CloudStorageConfiguration.DEFAULT; + + // Don't call this one, call the one in CloudStorageFileSystemProvider. + static void setUserSpecifiedDefault(@Nullable CloudStorageConfiguration config) { + if (null == config) { + userSpecifiedDefault = CloudStorageConfiguration.DEFAULT; + } else { + userSpecifiedDefault = config; + } + } + + static CloudStorageConfiguration getUserSpecifiedDefault() { + return userSpecifiedDefault; + } + + /** Returns path of current working directory. This defaults to the root directory. */ + public abstract String workingDirectory(); + + /** + * Returns {@code true} if we shouldn't throw an exception when encountering object names + * containing superfluous slashes, e.g. {@code a//b}. + */ + public abstract boolean permitEmptyPathComponents(); + + /** + * Returns {@code true} if '/' prefix on absolute object names should be removed before I/O. + * + *

If you disable this feature, please take into consideration that all paths created from a + * URI will have the leading slash. + */ + public abstract boolean stripPrefixSlash(); + + /** + * Returns {@code true} if directories and paths with a trailing slash should be treated as fake + * directories. + * + *

With this feature, if file "foo/bar.txt" exists then both "foo" and "foo/" will be treated + * as if they were existing directories. On path construction no I/O will be performed, bucket and + * "directory" will treated as if they exist. + */ + public abstract boolean usePseudoDirectories(); + + /** Returns block size (in bytes) used when talking to the Google Cloud Storage HTTP server. */ + public abstract int blockSize(); + + /** + * Returns the number of times we try re-opening a channel if it's closed unexpectedly while + * reading. + */ + public abstract int maxChannelReopens(); + + /** + * Returns the project to be billed when accessing buckets. Leave empty for normal semantics, set + * to bill that project (project you own) for all accesses. This is required for accessing + * requester-pays buckets. This value cannot be null. + */ + public abstract @Nullable String userProject(); + + /** + * Returns whether userProject will be cleared for non-requester-pays buckets. That is, if false + * (the default value), setting userProject causes that project to be billed regardless of whether + * the bucket is requester-pays or not. If true, setting userProject will only cause that project + * to be billed when the project is requester-pays. + * + *

Setting this will cause the bucket to be accessed when the CloudStorageFileSystem object is + * created. + */ + public abstract boolean useUserProjectOnlyForRequesterPaysBuckets(); + + /** + * Returns the set of HTTP error codes that will be retried, in addition to the normally retryable + * ones. + */ + public abstract ImmutableList retryableHttpCodes(); + + /** + * Returns the set of exceptions for which we'll try a channel reopen if maxChannelReopens is + * positive. + */ + public abstract ImmutableList> reopenableExceptions(); + + /** + * Creates a new builder, initialized with the following settings: + * + *

    + *
  • Performing I/O on paths with extra slashes, e.g. {@code a//b} will throw an error. + *
  • The prefix slash on absolute paths will be removed when converting to an object name. + *
  • Pseudo-directories are enabled, so any path with a trailing slash is a fake directory. + *
  • Channel re-opens are disabled. + *
+ */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for {@link CloudStorageConfiguration}. */ + public static final class Builder { + + private String workingDirectory = UnixPath.ROOT; + private boolean permitEmptyPathComponents; + private boolean stripPrefixSlash = true; + private boolean usePseudoDirectories = true; + private int blockSize = CloudStorageFileSystem.BLOCK_SIZE_DEFAULT; + private int maxChannelReopens = 0; + private @Nullable String userProject = null; + // Think of this as "clear userProject if not RequesterPays" + private boolean useUserProjectOnlyForRequesterPaysBuckets = false; + private ImmutableList retryableHttpCodes = ImmutableList.of(500, 502, 503); + private ImmutableList> reopenableExceptions = + ImmutableList.>of( + SSLException.class, + EOFException.class, + SocketException.class, + SocketTimeoutException.class); + + /** + * Changes current working directory for new filesystem. This defaults to the root directory. + * The working directory cannot be changed once it's been set. You'll need to create another + * {@link CloudStorageFileSystem} object. + * + * @throws IllegalArgumentException if {@code path} is not absolute. + */ + public Builder workingDirectory(String path) { + checkNotNull(path); + checkArgument(UnixPath.getPath(false, path).isAbsolute(), "not absolute: %s", path); + workingDirectory = path; + return this; + } + + /** + * Configures whether or not we should throw an exception when encountering object names + * containing superfluous slashes, e.g. {@code a//b}. + */ + public Builder permitEmptyPathComponents(boolean value) { + permitEmptyPathComponents = value; + return this; + } + + /** + * Configures if the '/' prefix on absolute object names should be removed before I/O. + * + *

If you disable this feature, please take into consideration that all paths created from a + * URI will have the leading slash. + */ + public Builder stripPrefixSlash(boolean value) { + stripPrefixSlash = value; + return this; + } + + /** Configures if paths with a trailing slash should be treated as fake directories. */ + public Builder usePseudoDirectories(boolean value) { + usePseudoDirectories = value; + return this; + } + + /** + * Sets the block size in bytes that should be used for each HTTP request to the API. + * + *

The default is {@value CloudStorageFileSystem#BLOCK_SIZE_DEFAULT}. + */ + public Builder blockSize(int value) { + blockSize = value; + return this; + } + + public Builder maxChannelReopens(int value) { + maxChannelReopens = value; + return this; + } + + public Builder userProject(String value) { + userProject = value; + return this; + } + + public Builder autoDetectRequesterPays(boolean value) { + useUserProjectOnlyForRequesterPaysBuckets = value; + return this; + } + + public Builder retryableHttpCodes(ImmutableList value) { + retryableHttpCodes = value; + return this; + } + + public Builder reopenableExceptions(ImmutableList> values) { + reopenableExceptions = values; + return this; + } + + /** Creates new instance without destroying builder. */ + public CloudStorageConfiguration build() { + return new AutoValue_CloudStorageConfiguration( + workingDirectory, + permitEmptyPathComponents, + stripPrefixSlash, + usePseudoDirectories, + blockSize, + maxChannelReopens, + userProject, + useUserProjectOnlyForRequesterPaysBuckets, + retryableHttpCodes, + reopenableExceptions); + } + + Builder(CloudStorageConfiguration toModify) { + workingDirectory = toModify.workingDirectory(); + permitEmptyPathComponents = toModify.permitEmptyPathComponents(); + stripPrefixSlash = toModify.stripPrefixSlash(); + usePseudoDirectories = toModify.usePseudoDirectories(); + blockSize = toModify.blockSize(); + maxChannelReopens = toModify.maxChannelReopens(); + userProject = toModify.userProject(); + useUserProjectOnlyForRequesterPaysBuckets = + toModify.useUserProjectOnlyForRequesterPaysBuckets(); + retryableHttpCodes = toModify.retryableHttpCodes(); + reopenableExceptions = toModify.reopenableExceptions(); + } + + Builder() {} + } + + static CloudStorageConfiguration fromMap(Map env) { + return fromMap(builder(), env); + } + + static CloudStorageConfiguration fromMap( + CloudStorageConfiguration defaultValues, Map env) { + return fromMap(new Builder(defaultValues), env); + } + + private static CloudStorageConfiguration fromMap(Builder builder, Map env) { + for (Map.Entry entry : env.entrySet()) { + switch (entry.getKey()) { + case "workingDirectory": + builder.workingDirectory((String) entry.getValue()); + break; + case "permitEmptyPathComponents": + builder.permitEmptyPathComponents((Boolean) entry.getValue()); + break; + case "stripPrefixSlash": + builder.stripPrefixSlash((Boolean) entry.getValue()); + break; + case "usePseudoDirectories": + builder.usePseudoDirectories((Boolean) entry.getValue()); + break; + case "blockSize": + builder.blockSize((Integer) entry.getValue()); + break; + case "maxChannelReopens": + builder.maxChannelReopens((Integer) entry.getValue()); + break; + case "userProject": + builder.userProject((String) entry.getValue()); + break; + case "useUserProjectOnlyForRequesterPaysBuckets": + builder.autoDetectRequesterPays((Boolean) entry.getValue()); + break; + case "retryableHttpCodes": + builder.retryableHttpCodes((ImmutableList) entry.getValue()); + break; + case "reopenableExceptions": + builder.reopenableExceptions( + (ImmutableList>) entry.getValue()); + break; + default: + throw new IllegalArgumentException(entry.getKey()); + } + } + return builder.build(); + } + + CloudStorageConfiguration() {} +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java new file mode 100644 index 000000000000..0624a94abc8d --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java @@ -0,0 +1,85 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.common.base.MoreObjects; +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.Objects; +import javax.annotation.concurrent.Immutable; + +/** Metadata view for a Google Cloud Storage object. */ +@Immutable +public final class CloudStorageFileAttributeView implements BasicFileAttributeView { + + private final Storage storage; + private final CloudStoragePath path; + + CloudStorageFileAttributeView(Storage storage, CloudStoragePath path) { + this.storage = checkNotNull(storage); + this.path = checkNotNull(path); + } + + /** Returns {@value CloudStorageFileSystem#GCS_VIEW}. */ + @Override + public String name() { + return CloudStorageFileSystem.GCS_VIEW; + } + + @Override + public CloudStorageFileAttributes readAttributes() throws IOException { + if (path.seemsLikeADirectory() && path.getFileSystem().config().usePseudoDirectories()) { + return new CloudStoragePseudoDirectoryAttributes(path); + } + BlobInfo blobInfo = storage.get(path.getBlobId()); + if (blobInfo == null) { + throw new NoSuchFileException(path.toUri().toString()); + } + + return new CloudStorageObjectAttributes(blobInfo); + } + + /** This feature is not supported, since Cloud Storage objects are immutable. */ + @Override + public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) { + throw new CloudStorageObjectImmutableException(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageFileAttributeView + && Objects.equals(storage, ((CloudStorageFileAttributeView) other).storage) + && Objects.equals(path, ((CloudStorageFileAttributeView) other).path); + } + + @Override + public int hashCode() { + return Objects.hash(storage, path); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("storage", storage).add("path", path).toString(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java new file mode 100644 index 000000000000..17f93cc6a0ef --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java @@ -0,0 +1,76 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.storage.Acl; +import com.google.common.base.Optional; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; +import java.util.Map; + +/** Interface for attributes on a Cloud Storage file or pseudo-directory. */ +public interface CloudStorageFileAttributes extends BasicFileAttributes { + + /** + * Returns HTTP etag hash of object contents. + * + * @see "https://developers.google.com/storage/docs/hashes-etags" + */ + Optional etag(); + + /** + * Returns mime type (e.g. text/plain), if set. + * + * @see "http://en.wikipedia.org/wiki/Internet_media_type#List_of_common_media_types" + */ + Optional mimeType(); + + /** + * Returns access control list. + * + * @see "https://developers.google.com/storage/docs/reference-headers#acl" + */ + Optional> acl(); + + /** + * Returns {@code Cache-Control} HTTP header value, if set. + * + * @see "https://developers.google.com/storage/docs/reference-headers#cachecontrol" + */ + Optional cacheControl(); + + /** + * Returns {@code Content-Encoding} HTTP header value, if set. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentencoding" + */ + Optional contentEncoding(); + + /** + * Returns {@code Content-Disposition} HTTP header value, if set. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + Optional contentDisposition(); + + /** + * Returns user-specified metadata. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + Map userMetadata(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java new file mode 100644 index 000000000000..7b930d4a722e --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java @@ -0,0 +1,376 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.requireNonNull; + +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.base.Strings; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.UncheckedExecutionException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.WatchService; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.HashMap; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import javax.annotation.CheckReturnValue; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Google Cloud Storage {@link FileSystem} implementation. + * + * @see Concepts + * and Terminology + * @see Bucket and Object Naming + * Guidelines + */ +@ThreadSafe +public final class CloudStorageFileSystem extends FileSystem { + + public static final String URI_SCHEME = "gs"; + public static final String GCS_VIEW = "gcs"; + public static final String BASIC_VIEW = "basic"; + public static final String POSIX_VIEW = "posix"; + public static final int BLOCK_SIZE_DEFAULT = 2 * 1024 * 1024; + public static final FileTime FILE_TIME_UNKNOWN = FileTime.fromMillis(0); + public static final Set SUPPORTED_VIEWS = + ImmutableSet.of(BASIC_VIEW, GCS_VIEW, POSIX_VIEW); + private final CloudStorageFileSystemProvider provider; + private final String bucket; + private final CloudStorageConfiguration config; + private static final LoadingCache + PROVIDER_CACHE_BY_CONFIG = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + @Override + public CloudStorageFileSystemProvider load(ProviderCacheKey key) { + CloudStorageConfiguration config = key.cloudStorageConfiguration; + StorageOptions storageOptions = key.storageOptions; + String userProject = config.userProject(); + return (storageOptions == null) + ? new CloudStorageFileSystemProvider(userProject) + : new CloudStorageFileSystemProvider(userProject, storageOptions); + } + }); + + // Don't call this one, call the one in CloudStorageFileSystemProvider. + static void setDefaultCloudStorageConfiguration(CloudStorageConfiguration config) { + CloudStorageConfiguration.setUserSpecifiedDefault(config); + } + + static CloudStorageConfiguration getDefaultCloudStorageConfiguration() { + return CloudStorageConfiguration.getUserSpecifiedDefault(); + } + + /** + * Lists the project's buckets. Pass "null" to use the default project. + * + *

Example of listing buckets, specifying the page size and a name prefix. + * + *

{@code
+   * String prefix = "bucket_";
+   * Page buckets = CloudStorageFileSystem.listBuckets("my-project", BucketListOption.prefix(prefix));
+   * Iterator bucketIterator = buckets.iterateAll();
+   * while (bucketIterator.hasNext()) {
+   *   Bucket bucket = bucketIterator.next();
+   *   // do something with the bucket
+   * }
+   * }
+ * + * @throws StorageException upon failure + */ + public static Page listBuckets( + @Nullable String project, Storage.BucketListOption... options) { + CloudStorageFileSystemProvider provider = + new CloudStorageFileSystemProvider( + null, StorageOptions.newBuilder().setProjectId(project).build()); + return provider.listBuckets(options); + } + + /** + * Returns Google Cloud Storage {@link FileSystem} object for {@code bucket}. + * + *

NOTE: You may prefer to use Java's standard API instead: + * + *

{@code
+   * FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"));
+   * }
+ * + *

However some systems and build environments might be flaky when it comes to Java SPI. This + * is because services are generally runtime dependencies and depend on a META-INF file being + * present in your jar (generated by Google Auto at compile-time). In such cases, this method + * provides a simpler alternative. + * + * @see #forBucket(String, CloudStorageConfiguration) + * @see java.nio.file.FileSystems#getFileSystem(java.net.URI) + */ + @CheckReturnValue + public static CloudStorageFileSystem forBucket(String bucket) { + return forBucket(bucket, CloudStorageConfiguration.getUserSpecifiedDefault()); + } + + /** + * Creates new file system instance for {@code bucket}, with customizable settings. + * + * @see #forBucket(String) + */ + @CheckReturnValue + public static CloudStorageFileSystem forBucket(String bucket, CloudStorageConfiguration config) { + return forBucket(bucket, config, null); + } + + /** + * Returns Google Cloud Storage {@link FileSystem} object for {@code bucket}. + * + *

Google Cloud Storage file system objects are basically free. You can create as many as you + * want, even if you have multiple instances for the same bucket. There's no actual system + * resources associated with this object. Therefore calling {@link #close()} on the returned value + * is optional. + * + *

Note: It is also possible to instantiate this class via Java's {@code + * FileSystems.getFileSystem(URI.create("gs://bucket"))}. We discourage you from using that if + * possible, for the reasons documented in {@link + * CloudStorageFileSystemProvider#newFileSystem(URI, java.util.Map)} + * + * @see java.nio.file.FileSystems#getFileSystem(URI) + */ + @CheckReturnValue + public static CloudStorageFileSystem forBucket( + String bucket, CloudStorageConfiguration config, @Nullable StorageOptions storageOptions) { + checkArgument( + !bucket.startsWith(URI_SCHEME + ":"), "Bucket name must not have schema: %s", bucket); + checkNotNull(config); + CloudStorageFileSystemProvider result; + ProviderCacheKey providerCacheKey = new ProviderCacheKey(config, storageOptions); + try { + result = PROVIDER_CACHE_BY_CONFIG.get(providerCacheKey); + } catch (ExecutionException | UncheckedExecutionException e) { + throw new IllegalStateException( + "Unable to resolve CloudStorageFileSystemProvider for the provided configuration", e); + } + return new CloudStorageFileSystem(result, bucket, config); + } + + CloudStorageFileSystem( + CloudStorageFileSystemProvider provider, String bucket, CloudStorageConfiguration config) { + checkArgument(!bucket.isEmpty(), "bucket"); + this.bucket = bucket; + if (config.useUserProjectOnlyForRequesterPaysBuckets()) { + if (Strings.isNullOrEmpty(config.userProject())) { + throw new IllegalArgumentException( + "If useUserProjectOnlyForRequesterPaysBuckets is set, then userProject must be set too."); + } + // detect whether we want to pay for these accesses or not. + if (!provider.requesterPays(bucket)) { + // update config (just to ease debugging, we're not actually using config.userProject later. + HashMap disableUserProject = new HashMap<>(); + disableUserProject.put("userProject", ""); + config = CloudStorageConfiguration.fromMap(config, disableUserProject); + // update the provider (this is the most important bit) + provider = provider.withNoUserProject(); + } + } + this.provider = provider; + this.config = config; + } + + @Override + public CloudStorageFileSystemProvider provider() { + return provider; + } + + /** Returns Cloud Storage bucket name being served by this file system. */ + public String bucket() { + return bucket; + } + + /** Returns configuration object for this file system instance. */ + public CloudStorageConfiguration config() { + return config; + } + + /** Converts Cloud Storage object name to a {@link Path} object. */ + @Override + public CloudStoragePath getPath(String first, String... more) { + checkArgument( + !first.startsWith(URI_SCHEME + ":"), + "Google Cloud Storage FileSystem.getPath() must not have schema and bucket name: %s", + first); + return CloudStoragePath.getPath(this, first, more); + } + + /** + * Does nothing currently. This method might be updated in the future to close all channels + * associated with this file system object. However it's unlikely that even then, calling this + * method will become mandatory. + */ + @Override + public void close() throws IOException { + // TODO(#809): Synchronously close all channels associated with this FileSystem instance. + } + + /** Returns {@code true}, even if you previously called the {@link #close()} method. */ + @Override + public boolean isOpen() { + return true; + } + + /** Returns {@code false}. */ + @Override + public boolean isReadOnly() { + return false; + } + + /** Returns {@value UnixPath#SEPARATOR}. */ + @Override + public String getSeparator() { + return Character.toString(UnixPath.SEPARATOR); + } + + @Override + public Iterable getRootDirectories() { + return ImmutableSet.of(CloudStoragePath.getPath(this, UnixPath.ROOT)); + } + + /** + * Returns nothing because Google Cloud Storage doesn't have disk partitions of limited size, or + * anything similar. + */ + @Override + public Iterable getFileStores() { + return ImmutableSet.of(); + } + + @Override + public Set supportedFileAttributeViews() { + return SUPPORTED_VIEWS; + } + + @Override + public PathMatcher getPathMatcher(String syntaxAndPattern) { + return FileSystems.getDefault().getPathMatcher(syntaxAndPattern); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public WatchService newWatchService() throws IOException { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageFileSystem + && Objects.equals(config, ((CloudStorageFileSystem) other).config) + && Objects.equals(bucket, ((CloudStorageFileSystem) other).bucket); + } + + @Override + public int hashCode() { + return Objects.hash(bucket); + } + + @Override + public String toString() { + try { + // Store GCS bucket name in the URI authority, see + // https://github.com/googleapis/java-storage-nio/issues/1218 + return new URI(URI_SCHEME, bucket, null, null, null).toString(); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + } + + /** + * In order to cache a {@link CloudStorageFileSystemProvider} we track the config used to + * instantiate that provider. This class creates an immutable key encapsulating the config to + * allow reliable resolution from the cache. + */ + private static final class ProviderCacheKey { + private final CloudStorageConfiguration cloudStorageConfiguration; + @Nullable private final StorageOptions storageOptions; + + public ProviderCacheKey( + CloudStorageConfiguration cloudStorageConfiguration, + @Nullable StorageOptions storageOptions) { + this.cloudStorageConfiguration = + requireNonNull(cloudStorageConfiguration, "cloudStorageConfiguration must be non null"); + this.storageOptions = storageOptions; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ProviderCacheKey)) { + return false; + } + ProviderCacheKey that = (ProviderCacheKey) o; + return cloudStorageConfiguration.equals(that.cloudStorageConfiguration) + && Objects.equals(storageOptions, that.storageOptions); + } + + @Override + public int hashCode() { + return Objects.hash(cloudStorageConfiguration, storageOptions); + } + + @Override + public String toString() { + return "ConfigTuple{" + + "cloudStorageConfiguration=" + + cloudStorageConfiguration + + ", storageOptions=" + + storageOptions + + '}'; + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java new file mode 100644 index 000000000000..b702464c8b6b --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -0,0 +1,1311 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE; +import static java.nio.file.attribute.PosixFilePermission.GROUP_READ; +import static java.nio.file.attribute.PosixFilePermission.GROUP_WRITE; +import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE; +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; + +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.CopyWriter; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobGetOption; +import com.google.cloud.storage.Storage.BlobSourceOption; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.collect.AbstractIterator; +import com.google.common.net.UrlEscapers; +import com.google.common.primitives.Ints; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessMode; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.DirectoryStream; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.UserPrincipal; +import java.nio.file.spi.FileSystemProvider; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; +import javax.inject.Singleton; + +/** + * Google Cloud Storage {@link FileSystemProvider} implementation. + * + *

Note: This class should never be used directly. This class is instantiated by the + * service loader and called through a standardized API, e.g. {@link java.nio.file.Files}. However + * the javadocs in this class serve as useful documentation for the behavior of the Google Cloud + * Storage NIO library. + */ +@Singleton +@ThreadSafe +public final class CloudStorageFileSystemProvider extends FileSystemProvider { + + private Storage storage; + // if null, use StorageOptionsUtil.getDefaultInstance() + private final @Nullable StorageOptions storageOptions; + // if non-null, we pay via this project. + private final @Nullable String userProject; + + // used only when we create a new instance of CloudStorageFileSystemProvider. + private static StorageOptions futureStorageOptions = null; + + private static class LazyPathIterator extends AbstractIterator { + private final Iterator blobIterator; + private final Filter filter; + private final CloudStorageFileSystem fileSystem; + private final String prefix; + // whether to make the paths absolute before returning them. + private final boolean absolutePaths; + + LazyPathIterator( + CloudStorageFileSystem fileSystem, + String prefix, + Iterator blobIterator, + Filter filter, + boolean absolutePaths) { + this.prefix = prefix; + this.blobIterator = blobIterator; + this.filter = filter; + this.fileSystem = fileSystem; + this.absolutePaths = absolutePaths; + } + + @Override + protected Path computeNext() { + while (blobIterator.hasNext()) { + Path path = fileSystem.getPath(blobIterator.next().getName()); + try { + if (path.toString().equals(prefix)) { + // do not return ourselves, because that confuses recursive descents. + continue; + } + if (filter.accept(path)) { + if (absolutePaths) { + return path.toAbsolutePath(); + } + return path; + } + } catch (IOException ex) { + throw new DirectoryIteratorException(ex); + } + } + return endOfData(); + } + } + + /** + * Sets options that are only used by the constructor. + * + *

Instead of calling this, when possible use CloudStorageFileSystem.forBucket and pass + * StorageOptions as an argument. + */ + @VisibleForTesting + public static void setStorageOptions(@Nullable StorageOptions newStorageOptions) { + futureStorageOptions = newStorageOptions; + } + + /** + * Changes the default configuration for every filesystem object created from here on, including + * via SPI. If null then future filesystem objects will have the factory default configuration. + * + *

If options are specified later then they override the defaults. Methods that take a whole + * CloudStorageConfiguration (eg. CloudStorageFileSystem.forBucket) will completely override the + * defaults. Methods that take individual options (eg. + * CloudStorageFileSystemProvier.newFileSystem) will override only these options; the rest will be + * taken from the defaults specified here. + * + *

This is meant to be done only once, at the beginning of some main program, in order to force + * all libraries to use some settings we like. + * + *

Libraries should never call this. If you're a library then, instead, create your own + * filesystem object with the right configuration and pass it along. + * + * @param newDefault new default CloudStorageConfiguration + */ + public static void setDefaultCloudStorageConfiguration( + @Nullable CloudStorageConfiguration newDefault) { + CloudStorageConfiguration.setUserSpecifiedDefault(newDefault); + } + + /** + * Default constructor which should only be called by Java SPI. + * + * @see java.nio.file.FileSystems#getFileSystem(URI) + * @see CloudStorageFileSystem#forBucket(String) + */ + public CloudStorageFileSystemProvider() { + this(CloudStorageConfiguration.getUserSpecifiedDefault().userProject(), futureStorageOptions); + } + + /** + * Internal constructor to use the user-provided default config, and a given userProject setting. + */ + CloudStorageFileSystemProvider(@Nullable String userProject) { + this(userProject, futureStorageOptions); + } + + /** + * Internal constructor, fully configurable. Note that null options means to use the system + * defaults (NOT the user-provided ones). + */ + CloudStorageFileSystemProvider( + @Nullable String userProject, @Nullable StorageOptions gcsStorageOptions) { + this.storageOptions = + gcsStorageOptions != null + ? StorageOptionsUtil.mergeOptionsWithUserAgent(gcsStorageOptions) + : null; + this.userProject = userProject; + } + + // Initialize this.storage, once. This may throw an exception if default authentication + // credentials are not available (hence not doing it in the ctor). + private void initStorage() { + if (this.storage == null) { + doInitStorage(); + } + } + + @Override + public String getScheme() { + return CloudStorageFileSystem.URI_SCHEME; + } + + /** Returns Cloud Storage file system, provided a URI with no path, e.g. {@code gs://bucket}. */ + @Override + public CloudStorageFileSystem getFileSystem(URI uri) { + initStorage(); + return newFileSystem(uri, Collections.emptyMap()); + } + + /** + * Returns Cloud Storage file system, provided a URI, e.g. {@code gs://bucket}. The URI can + * include a path component (that will be ignored). + * + * @param uri bucket and current working directory, e.g. {@code gs://bucket} + * @param env map of configuration options, whose keys correspond to the method names of {@link + * CloudStorageConfiguration.Builder}. However you are not allowed to set the working + * directory, as that should be provided in the {@code uri} + * @throws IllegalArgumentException if {@code uri} specifies a port, user, query, or fragment, or + * if scheme is not {@value CloudStorageFileSystem#URI_SCHEME} + */ + @Override + public CloudStorageFileSystem newFileSystem(URI uri, Map env) { + checkArgument( + uri.getScheme().equalsIgnoreCase(CloudStorageFileSystem.URI_SCHEME), + "Cloud Storage URIs must have '%s' scheme: %s", + CloudStorageFileSystem.URI_SCHEME, + uri); + // Bucket names may not be compatible with getHost(), see + // https://github.com/googleapis/java-storage-nio/issues/1218 + // However, there may be existing code expecting the exception message to refer to the bucket + // name as the "host". + checkArgument( + !isNullOrEmpty(uri.getAuthority()), + "%s:// URIs must have a host: %s", + CloudStorageFileSystem.URI_SCHEME, + uri); + checkArgument( + uri.getPort() == -1 + && isNullOrEmpty(uri.getQuery()) + && isNullOrEmpty(uri.getFragment()) + && isNullOrEmpty(uri.getUserInfo()), + "GCS FileSystem URIs mustn't have: port, userinfo, query, or fragment: %s", + uri); + CloudStorageUtil.checkBucket(uri.getAuthority()); + initStorage(); + return new CloudStorageFileSystem( + this, + uri.getAuthority(), + CloudStorageConfiguration.fromMap( + CloudStorageFileSystem.getDefaultCloudStorageConfiguration(), env)); + } + + @Override + public CloudStoragePath getPath(URI uri) { + initStorage(); + return CloudStoragePath.getPath( + getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath()); + } + + /** Convenience method: replaces spaces with "%20", builds a URI, and calls getPath(uri). */ + public CloudStoragePath getPath(String uriInStringForm) { + String escaped = UrlEscapers.urlFragmentEscaper().escape(uriInStringForm); + return getPath(URI.create(escaped)); + } + + /** + * Open a file for reading or writing. To read receiver-pays buckets, specify the + * BlobSourceOption.userProject option. + * + * @param path: the path to the file to open or create + * @param options: options specifying how the file is opened, e.g. StandardOpenOption.WRITE or + * BlobSourceOption.userProject + * @param attrs: (not supported, values will be ignored) + * @return + * @throws IOException + */ + @Override + public SeekableByteChannel newByteChannel( + Path path, Set options, FileAttribute... attrs) throws IOException { + checkNotNull(path); + initStorage(); + CloudStorageUtil.checkNotNullArray(attrs); + if (options.contains(StandardOpenOption.WRITE)) { + // TODO: Make our OpenOptions implement FileAttribute. Also remove buffer option. + return newWriteChannel(path, options); + } else { + return newReadChannel(path, options); + } + } + + /** + * Open a file for reading OR writing. The {@link FileChannel} that is returned will only allow + * reads or writes depending on the {@link OpenOption}s that are specified. If any of the + * following have been specified, the {@link FileChannel} will be write-only: {@link + * StandardOpenOption#CREATE} + * + *

    + *
  • {@link StandardOpenOption#CREATE} + *
  • {@link StandardOpenOption#CREATE_NEW} + *
  • {@link StandardOpenOption#WRITE} + *
  • {@link StandardOpenOption#TRUNCATE_EXISTING} + *
+ * + * In all other cases the {@link FileChannel} will be read-only. + * + * @param path The path to the file to open or create + * @param options The options specifying how the file should be opened, and whether the {@link + * FileChannel} should be read-only or write-only. + * @param attrs (not supported, the values will be ignored) + * @throws IOException + */ + @Override + public FileChannel newFileChannel( + Path path, Set options, FileAttribute... attrs) throws IOException { + checkNotNull(path); + initStorage(); + CloudStorageUtil.checkNotNullArray(attrs); + if (options.contains(StandardOpenOption.CREATE_NEW)) { + Files.createFile(path, attrs); + } else if (options.contains(StandardOpenOption.CREATE) && !Files.exists(path)) { + Files.createFile(path, attrs); + } + if (options.contains(StandardOpenOption.WRITE) + || options.contains(StandardOpenOption.CREATE) + || options.contains(StandardOpenOption.CREATE_NEW) + || options.contains(StandardOpenOption.TRUNCATE_EXISTING)) { + return new CloudStorageWriteFileChannel(newWriteChannel(path, options)); + } else { + return new CloudStorageReadFileChannel(newReadChannel(path, options)); + } + } + + private SeekableByteChannel newReadChannel(Path path, Set options) + throws IOException { + initStorage(); + int maxChannelReopens = CloudStorageUtil.getMaxChannelReopensFromPath(path); + List blobSourceOptions = new ArrayList<>(); + for (OpenOption option : options) { + if (option instanceof StandardOpenOption) { + switch ((StandardOpenOption) option) { + case READ: + // Default behavior. + break; + case SPARSE: + case TRUNCATE_EXISTING: + // Ignored by specification. + break; + case WRITE: + throw new IllegalArgumentException("READ+WRITE not supported yet"); + case APPEND: + case CREATE: + case CREATE_NEW: + case DELETE_ON_CLOSE: + case DSYNC: + case SYNC: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else if (option instanceof OptionMaxChannelReopens) { + maxChannelReopens = ((OptionMaxChannelReopens) option).maxChannelReopens(); + } else if (option instanceof BlobSourceOption) { + blobSourceOptions.add((BlobSourceOption) option); + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + // passing false since we just want to check if it ends with / + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories(null)) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + return CloudStorageReadChannel.create( + storage, + cloudPath.getBlobId(), + 0, + maxChannelReopens, + cloudPath.getFileSystem().config(), + userProject, + blobSourceOptions.toArray(new BlobSourceOption[blobSourceOptions.size()])); + } + + private SeekableByteChannel newWriteChannel(Path path, Set options) + throws IOException { + initStorage(); + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + boolean allowSlash = options.contains(OptionAllowTrailingSlash.getInstance()); + if (!allowSlash && cloudPath.seemsLikeADirectoryAndUsePseudoDirectories(null)) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + BlobId file = cloudPath.getBlobId(); + BlobInfo.Builder infoBuilder = BlobInfo.newBuilder(file); + List writeOptions = new ArrayList<>(); + List acls = new ArrayList<>(); + + HashMap metas = new HashMap<>(); + for (OpenOption option : options) { + if (option instanceof OptionMimeType) { + infoBuilder.setContentType(((OptionMimeType) option).mimeType()); + } else if (option instanceof OptionCacheControl) { + infoBuilder.setCacheControl(((OptionCacheControl) option).cacheControl()); + } else if (option instanceof OptionContentDisposition) { + infoBuilder.setContentDisposition(((OptionContentDisposition) option).contentDisposition()); + } else if (option instanceof OptionContentEncoding) { + infoBuilder.setContentEncoding(((OptionContentEncoding) option).contentEncoding()); + } else if (option instanceof OptionUserMetadata) { + OptionUserMetadata opMeta = (OptionUserMetadata) option; + metas.put(opMeta.key(), opMeta.value()); + } else if (option instanceof OptionAcl) { + acls.add(((OptionAcl) option).acl()); + } else if (option instanceof OptionBlockSize) { + // TODO: figure out how to plumb in block size. + } else if (option instanceof StandardOpenOption) { + switch ((StandardOpenOption) option) { + case CREATE: + case TRUNCATE_EXISTING: + case WRITE: + // Default behavior. + break; + case SPARSE: + // Ignored by specification. + break; + case CREATE_NEW: + writeOptions.add(Storage.BlobWriteOption.doesNotExist()); + break; + case READ: + throw new IllegalArgumentException("READ+WRITE not supported yet"); + case APPEND: + case DELETE_ON_CLOSE: + case DSYNC: + case SYNC: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else if (option instanceof CloudStorageOption) { + // XXX: We need to interpret these later + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + if (!isNullOrEmpty(userProject)) { + writeOptions.add(Storage.BlobWriteOption.userProject(userProject)); + } + + if (!metas.isEmpty()) { + infoBuilder.setMetadata(metas); + } + if (!acls.isEmpty()) { + infoBuilder.setAcl(acls); + } + + try { + return new CloudStorageWriteChannel( + storage.writer( + infoBuilder.build(), + writeOptions.toArray(new Storage.BlobWriteOption[writeOptions.size()]))); + } catch (StorageException oops) { + throw asIoException(oops, false); + } + } + + @Override + public InputStream newInputStream(Path path, OpenOption... options) throws IOException { + initStorage(); + InputStream result = super.newInputStream(path, options); + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + int blockSize = cloudPath.getFileSystem().config().blockSize(); + for (OpenOption option : options) { + if (option instanceof OptionBlockSize) { + blockSize = ((OptionBlockSize) option).size(); + } + } + return new BufferedInputStream(result, blockSize); + } + + @Override + public boolean deleteIfExists(Path path) throws IOException { + initStorage(); + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories(storage)) { + // if the "folder" is empty then we're fine, otherwise complain + // that we cannot act on folders. + try (DirectoryStream paths = Files.newDirectoryStream(path)) { + if (!paths.iterator().hasNext()) { + // "folder" isn't actually there in the first place, so: success! + // (we must return true so delete doesn't freak out) + return true; + } + } + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + + BlobId idWithGeneration = cloudPath.getBlobId(); + if (idWithGeneration.getGeneration() == null) { + Storage.BlobGetOption[] options = new BlobGetOption[0]; + if (!isNullOrEmpty(userProject)) { + options = new BlobGetOption[] {Storage.BlobGetOption.userProject(userProject)}; + } + Blob blob = storage.get(idWithGeneration, options); + if (blob == null) { + // not found + return false; + } + idWithGeneration = blob.getBlobId(); + } + + final CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(cloudPath.getFileSystem().config()); + // Loop will terminate via an exception if all retries are exhausted + while (true) { + try { + if (isNullOrEmpty(userProject)) { + return storage.delete(idWithGeneration, Storage.BlobSourceOption.generationMatch()); + } else { + return storage.delete( + idWithGeneration, + Storage.BlobSourceOption.generationMatch(), + Storage.BlobSourceOption.userProject(userProject)); + } + } catch (StorageException exs) { + // Will rethrow a StorageException if all retries/reopens are exhausted + retryHandler.handleStorageException(exs); + // we're being aggressive by retrying even on scenarios where we'd normally reopen. + } + } + } + + @Override + public void delete(Path path) throws IOException { + initStorage(); + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (!deleteIfExists(cloudPath)) { + throw new NoSuchFileException(cloudPath.toString()); + } + } + + @Override + public void move(Path source, Path target, CopyOption... options) throws IOException { + initStorage(); + + boolean replaceExisting = false; + boolean atomicMove = false; + boolean hasCloudStorageOptions = false; + for (CopyOption option : options) { + if (option instanceof StandardCopyOption) { + switch ((StandardCopyOption) option) { + case COPY_ATTRIBUTES: + // The Objects: move API copies attributes by default. + break; + case REPLACE_EXISTING: + replaceExisting = true; + break; + case ATOMIC_MOVE: + atomicMove = true; + break; + default: + throw new UnsupportedOperationException(option.toString()); + } + } + hasCloudStorageOptions = option instanceof CloudStorageOption; + } + + CloudStoragePath fromPath = CloudStorageUtil.checkPath(source); + if (fromPath.seemsLikeADirectoryAndUsePseudoDirectories(null)) { + throw new CloudStoragePseudoDirectoryException(fromPath); + } + CloudStoragePath toPath = CloudStorageUtil.checkPath(target); + if (toPath.seemsLikeADirectoryAndUsePseudoDirectories(null)) { + throw new CloudStoragePseudoDirectoryException(toPath); + } + if (fromPath.seemsLikeADirectory() && toPath.seemsLikeADirectory()) { + if (fromPath.getFileSystem().config().usePseudoDirectories() + && toPath.getFileSystem().config().usePseudoDirectories()) { + // NOOP: This would normally create an empty directory. + return; + } else { + checkArgument( + !fromPath.getFileSystem().config().usePseudoDirectories() + && !toPath.getFileSystem().config().usePseudoDirectories(), + "File systems associated with paths don't agree on pseudo-directories."); + } + } + boolean crossBucketMove = !fromPath.bucket().equals(toPath.bucket()); + if (atomicMove) { + if (hasCloudStorageOptions) { + throw new AtomicMoveNotSupportedException( + source.toString(), + target.toString(), + "Cloud Storage does not support atomic move operations with CloudStorageOptions."); + } + if (crossBucketMove) { + throw new AtomicMoveNotSupportedException( + source.toString(), + target.toString(), + "Cloud Storage does not support atomic move operations between buckets."); + } + } else if (hasCloudStorageOptions || crossBucketMove) { + // Fall back to copy and delete if atomic move is not possible. + copy(source, target, options); + delete(source); + return; + } + + Storage.MoveBlobRequest.Builder builder = + Storage.MoveBlobRequest.newBuilder() + .setSource(fromPath.getBlobId()) + .setTarget(toPath.getBlobId()); + if (!replaceExisting) { + builder.setTargetOptions(BlobTargetOption.doesNotExist()); + } + Storage.MoveBlobRequest request = builder.build(); + CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(fromPath.getFileSystem().config()); + while (true) { + try { + storage.moveBlob(request); + break; + } catch (StorageException e) { + try { + retryHandler.handleStorageException(e); + } catch (StorageException retriesExhaustedException) { + throw asIoException(retriesExhaustedException, true); + } + } + } + } + + @Override + public void copy(Path source, Path target, CopyOption... options) throws IOException { + initStorage(); + boolean wantCopyAttributes = false; + boolean wantReplaceExisting = false; + // true if the option was set manually (so we shouldn't copy the parent's) + boolean overrideContentType = false; + boolean overrideCacheControl = false; + boolean overrideContentEncoding = false; + boolean overrideContentDisposition = false; + + CloudStoragePath toPath = CloudStorageUtil.checkPath(target); + BlobInfo.Builder tgtInfoBuilder = BlobInfo.newBuilder(toPath.getBlobId()).setContentType(""); + + int blockSize = -1; + for (CopyOption option : options) { + if (option instanceof StandardCopyOption) { + switch ((StandardCopyOption) option) { + case COPY_ATTRIBUTES: + wantCopyAttributes = true; + break; + case REPLACE_EXISTING: + wantReplaceExisting = true; + break; + case ATOMIC_MOVE: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else if (option instanceof CloudStorageOption) { + if (option instanceof OptionBlockSize) { + blockSize = ((OptionBlockSize) option).size(); + } else if (option instanceof OptionMimeType) { + tgtInfoBuilder.setContentType(((OptionMimeType) option).mimeType()); + overrideContentType = true; + } else if (option instanceof OptionCacheControl) { + tgtInfoBuilder.setCacheControl(((OptionCacheControl) option).cacheControl()); + overrideCacheControl = true; + } else if (option instanceof OptionContentEncoding) { + tgtInfoBuilder.setContentEncoding(((OptionContentEncoding) option).contentEncoding()); + overrideContentEncoding = true; + } else if (option instanceof OptionContentDisposition) { + tgtInfoBuilder.setContentDisposition( + ((OptionContentDisposition) option).contentDisposition()); + overrideContentDisposition = true; + } else { + throw new UnsupportedOperationException(option.toString()); + } + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + + CloudStoragePath fromPath = CloudStorageUtil.checkPath(source); + + blockSize = + blockSize != -1 + ? blockSize + : Ints.max( + fromPath.getFileSystem().config().blockSize(), + toPath.getFileSystem().config().blockSize()); + // TODO: actually use blockSize + + if (fromPath.seemsLikeADirectory() && toPath.seemsLikeADirectory()) { + if (fromPath.getFileSystem().config().usePseudoDirectories() + && toPath.getFileSystem().config().usePseudoDirectories()) { + // NOOP: This would normally create an empty directory. + return; + } else { + checkArgument( + !fromPath.getFileSystem().config().usePseudoDirectories() + && !toPath.getFileSystem().config().usePseudoDirectories(), + "File systems associated with paths don't agree on pseudo-directories."); + } + } + // We refuse to use paths that end in '/'. If the user puts a folder name + // but without the '/', they'll get whatever error GCS will return normally + // (if any). + if (fromPath.seemsLikeADirectoryAndUsePseudoDirectories(null)) { + throw new CloudStoragePseudoDirectoryException(fromPath); + } + if (toPath.seemsLikeADirectoryAndUsePseudoDirectories(null)) { + throw new CloudStoragePseudoDirectoryException(toPath); + } + + final CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(fromPath.getFileSystem().config()); + // Loop will terminate via an exception if all retries are exhausted + while (true) { + try { + if (wantCopyAttributes) { + BlobInfo blobInfo; + if (isNullOrEmpty(userProject)) { + blobInfo = storage.get(fromPath.getBlobId()); + } else { + blobInfo = storage.get(fromPath.getBlobId(), BlobGetOption.userProject(userProject)); + } + if (null == blobInfo) { + throw new NoSuchFileException(fromPath.toString()); + } + if (!overrideCacheControl) { + tgtInfoBuilder.setCacheControl(blobInfo.getCacheControl()); + } + if (!overrideContentType) { + tgtInfoBuilder.setContentType(blobInfo.getContentType()); + } + if (!overrideContentEncoding) { + tgtInfoBuilder.setContentEncoding(blobInfo.getContentEncoding()); + } + if (!overrideContentDisposition) { + tgtInfoBuilder.setContentDisposition(blobInfo.getContentDisposition()); + } + tgtInfoBuilder.setAcl(blobInfo.getAcl()); + tgtInfoBuilder.setMetadata(blobInfo.getMetadata()); + } + + BlobInfo tgtInfo = tgtInfoBuilder.build(); + Storage.CopyRequest.Builder copyReqBuilder = + Storage.CopyRequest.newBuilder().setSource(fromPath.getBlobId()); + if (wantReplaceExisting) { + copyReqBuilder = copyReqBuilder.setTarget(tgtInfo); + } else { + copyReqBuilder = + copyReqBuilder.setTarget(tgtInfo, Storage.BlobTargetOption.doesNotExist()); + } + if (!isNullOrEmpty(fromPath.getFileSystem().config().userProject())) { + copyReqBuilder = + copyReqBuilder.setSourceOptions( + BlobSourceOption.userProject(fromPath.getFileSystem().config().userProject())); + } else if (!isNullOrEmpty(toPath.getFileSystem().config().userProject())) { + copyReqBuilder = + copyReqBuilder.setSourceOptions( + BlobSourceOption.userProject(toPath.getFileSystem().config().userProject())); + } + CopyWriter copyWriter = storage.copy(copyReqBuilder.build()); + copyWriter.getResult(); + break; + } catch (StorageException oops) { + try { + // Will rethrow a StorageException if all retries/reopens are exhausted + retryHandler.handleStorageException(oops); + // we're being aggressive by retrying even on scenarios where we'd normally reopen. + } catch (StorageException retriesExhaustedException) { + throw asIoException(retriesExhaustedException, true); + } + } + } + } + + @Override + public boolean isSameFile(Path path, Path path2) { + return CloudStorageUtil.checkPath(path).equals(CloudStorageUtil.checkPath(path2)); + } + + /** Always returns {@code false}, because Google Cloud Storage doesn't support hidden files. */ + @Override + public boolean isHidden(Path path) { + CloudStorageUtil.checkPath(path); + return false; + } + + @Override + public void checkAccess(Path path, AccessMode... modes) throws IOException { + initStorage(); + for (AccessMode mode : modes) { + switch (mode) { + case READ: + case WRITE: + break; + case EXECUTE: + default: + throw new UnsupportedOperationException(mode.toString()); + } + } + + final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + final CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(cloudPath.getFileSystem().config()); + // Loop will terminate via an exception if all retries are exhausted + while (true) { + try { + // Edge case is the root directory which triggers the storage.get to throw a + // StorageException. + if (cloudPath.normalize().equals(cloudPath.getRoot())) { + return; + } + boolean nullId; + if (isNullOrEmpty(userProject)) { + nullId = + storage.get(cloudPath.getBlobId(), Storage.BlobGetOption.fields(Storage.BlobField.ID)) + == null; + } else { + nullId = + storage.get( + cloudPath.getBlobId(), + Storage.BlobGetOption.fields(Storage.BlobField.ID), + Storage.BlobGetOption.userProject(userProject)) + == null; + } + if (nullId) { + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories(storage)) { + // there is no such file, but we're not signalling error because the + // path is a pseudo-directory. + return; + } + throw new NoSuchFileException(path.toString()); + } + break; + } catch (StorageException exs) { + // Will rethrow a StorageException if all retries/reopens are exhausted + retryHandler.handleStorageException(exs); + // we're being aggressive by retrying even on scenarios where we'd normally reopen. + } catch (IllegalArgumentException exs) { + if (CloudStorageUtil.checkPath(path).seemsLikeADirectoryAndUsePseudoDirectories(storage)) { + // there is no such file, but we're not signalling error because the + // path is a pseudo-directory. + return; + } + // Other cause for IAE, forward the exception. + throw exs; + } + } + } + + @Override + public A readAttributes( + Path path, Class type, LinkOption... options) throws IOException { + checkNotNull(type); + CloudStorageUtil.checkNotNullArray(options); + if (type != CloudStorageFileAttributes.class && type != BasicFileAttributes.class) { + throw new UnsupportedOperationException(type.getSimpleName()); + } + initStorage(); + + final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + final CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(cloudPath.getFileSystem().config()); + // Loop will terminate via an exception if all retries are exhausted + while (true) { + try { + BlobInfo blobInfo = null; + try { + BlobId blobId = cloudPath.getBlobId(); + // Null or empty name won't give us a file, so skip. But perhaps it's a pseudodir. + if (!isNullOrEmpty(blobId.getName())) { + if (isNullOrEmpty(userProject)) { + blobInfo = storage.get(blobId); + } else { + blobInfo = storage.get(blobId, BlobGetOption.userProject(userProject)); + } + } + } catch (IllegalArgumentException ex) { + // the path may be invalid but look like a folder. In that case, return a folder. + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories(storage)) { + @SuppressWarnings("unchecked") + A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath); + return result; + } + // No? Propagate. + throw ex; + } + // null size indicate a file that we haven't closed yet, so GCS treats it as not there yet. + if (null == blobInfo || blobInfo.getSize() == null) { + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories(storage)) { + @SuppressWarnings("unchecked") + A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath); + return result; + } + throw new NoSuchFileException( + "gs://" + cloudPath.getBlobId().getBucket() + "/" + cloudPath.getBlobId().getName()); + } + CloudStorageObjectAttributes ret; + ret = new CloudStorageObjectAttributes(blobInfo); + /* + There exists a file with this name, yes. But should we pretend it's a directory? + The web UI will allow the user to "create directories" by creating files + whose name ends in slash (and these files aren't always zero-size). + If we're set to use pseudo directories and the file name looks like a path, + then say it's a directory. We pass null to avoid trying to actually list files; + if the path doesn't end in "/" we'll truthfully say it's a file. Yes it may also be + a directory but we don't want to do a prefix search every time the user stats a file. + */ + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories(null)) { + @SuppressWarnings("unchecked") + A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath); + return result; + } + @SuppressWarnings("unchecked") + A result = (A) ret; + return result; + } catch (StorageException exs) { + // Will rethrow a StorageException if all retries/reopens are exhausted + retryHandler.handleStorageException(exs); + // we're being aggressive by retrying even on scenarios where we'd normally reopen. + } + } + } + + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) + throws IOException { + // TODO(#811): Java 7 NIO defines at least eleven string attributes we'd want to support + // (eg. BasicFileAttributeView and PosixFileAttributeView), so rather than a partial + // implementation we rely on the other overload for now. + + // Partial implementation for a few commonly used ones: basic, gcs, posix + String[] split = attributes.split(":", 2); + if (split.length != 2) { + throw new UnsupportedOperationException(); + } + String view = split[0]; + List attributeNames = Arrays.asList(split[1].split(",")); + boolean allAttributes = attributeNames.size() == 1 && attributeNames.get(0).equals("*"); + + BasicFileAttributes fileAttributes; + + Map results = new TreeMap<>(); + switch (view) { + case "gcs": + fileAttributes = readAttributes(path, CloudStorageFileAttributes.class, options); + break; + case "posix": + // There is no real support for posix. + // Some systems expect Posix support for everything so these attributes are faked. + case "basic": + fileAttributes = readAttributes(path, BasicFileAttributes.class, options); + break; + default: + throw new UnsupportedOperationException(); + } + + if (fileAttributes == null) { + throw new UnsupportedOperationException(); + } + + // BasicFileAttributes + if (allAttributes || attributeNames.contains("lastModifiedTime")) { + results.put("lastModifiedTime", fileAttributes.lastModifiedTime()); + } + if (allAttributes || attributeNames.contains("lastAccessTime")) { + results.put("lastAccessTime", fileAttributes.lastAccessTime()); + } + if (allAttributes || attributeNames.contains("creationTime")) { + results.put("creationTime", fileAttributes.creationTime()); + } + if (allAttributes || attributeNames.contains("isRegularFile")) { + results.put("isRegularFile", fileAttributes.isRegularFile()); + } + if (allAttributes || attributeNames.contains("isDirectory")) { + results.put("isDirectory", fileAttributes.isDirectory()); + } + if (allAttributes || attributeNames.contains("isSymbolicLink")) { + results.put("isSymbolicLink", fileAttributes.isSymbolicLink()); + } + if (allAttributes || attributeNames.contains("isOther")) { + results.put("isOther", fileAttributes.isOther()); + } + if (allAttributes || attributeNames.contains("size")) { + results.put("size", fileAttributes.size()); + } + + // There is no real support for posix. + // Some systems fail if there is no posix support at all. + // To let these systems use this FileSystem these attributes are faked. + if (view.equals("posix")) { + if (allAttributes || attributeNames.contains("owner")) { + results.put( + "owner", + new UserPrincipal() { + @Override + public String getName() { + return "fakeowner"; + } + + @Override + public String toString() { + return "fakeowner"; + } + }); + } + if (allAttributes || attributeNames.contains("group")) { + results.put( + "group", + new GroupPrincipal() { + @Override + public String getName() { + return "fakegroup"; + } + + @Override + public String toString() { + return "fakegroup"; + } + }); + } + if (allAttributes || attributeNames.contains("permissions")) { + if (fileAttributes.isRegularFile()) { + results.put("permissions", EnumSet.of(OWNER_READ, OWNER_WRITE, GROUP_READ, GROUP_WRITE)); + } else { + // Directories, Symlinks and Other: + results.put( + "permissions", + EnumSet.of( + OWNER_READ, OWNER_WRITE, OWNER_EXECUTE, GROUP_READ, GROUP_WRITE, GROUP_EXECUTE)); + } + } + } + + // CloudStorageFileAttributes + if (fileAttributes instanceof CloudStorageFileAttributes) { + CloudStorageFileAttributes cloudStorageFileAttributes = + (CloudStorageFileAttributes) fileAttributes; + if (allAttributes || attributeNames.contains("etag")) { + results.put("etag", cloudStorageFileAttributes.etag()); + } + if (allAttributes || attributeNames.contains("mimeType")) { + results.put("mimeType", cloudStorageFileAttributes.mimeType()); + } + if (allAttributes || attributeNames.contains("acl")) { + results.put("acl", cloudStorageFileAttributes.acl()); + } + if (allAttributes || attributeNames.contains("cacheControl")) { + results.put("cacheControl", cloudStorageFileAttributes.cacheControl()); + } + if (allAttributes || attributeNames.contains("contentEncoding")) { + results.put("contentEncoding", cloudStorageFileAttributes.contentEncoding()); + } + if (allAttributes || attributeNames.contains("contentDisposition")) { + results.put("contentDisposition", cloudStorageFileAttributes.contentDisposition()); + } + if (allAttributes || attributeNames.contains("userMetadata")) { + results.put("userMetadata", cloudStorageFileAttributes.userMetadata()); + } + } + + return results; + } + + @Override + public V getFileAttributeView( + Path path, Class type, LinkOption... options) { + checkNotNull(path); + checkNotNull(type); + CloudStorageUtil.checkNotNullArray(options); + if (type != CloudStorageFileAttributeView.class && type != BasicFileAttributeView.class) { + // the javadocs for getFileAttributeView specify the following for @return + // a file attribute view of the specified type, or null if the attribute view type is not + // available + // Similar type of issue from the JDK itself https://bugs.openjdk.org/browse/JDK-8273935 + return null; + } + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + @SuppressWarnings("unchecked") + V result = (V) new CloudStorageFileAttributeView(storage, cloudPath); + return result; + } + + /** Does nothing since Google Cloud Storage uses fake directories. */ + @Override + public void createDirectory(Path dir, FileAttribute... attrs) { + CloudStorageUtil.checkPath(dir); + CloudStorageUtil.checkNotNullArray(attrs); + } + + @Override + public DirectoryStream newDirectoryStream( + final Path dir, final Filter filter) { + final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir); + checkNotNull(filter); + initStorage(); + + final CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(cloudPath.getFileSystem().config()); + // Loop will terminate via an exception if all retries are exhausted + while (true) { + try { + String prePrefix = cloudPath.normalize().toRealPath().toString(); + // we can recognize paths without the final "/" as folders, + // but storage.list doesn't do the right thing with those, we need to append a "/". + if (!prePrefix.isEmpty() && !prePrefix.endsWith("/")) { + prePrefix += "/"; + } + final String prefix = prePrefix; + Page dirList; + if (isNullOrEmpty(userProject)) { + dirList = + storage.list( + cloudPath.bucket(), + Storage.BlobListOption.prefix(prefix), + Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.fields()); + } else { + dirList = + storage.list( + cloudPath.bucket(), + Storage.BlobListOption.prefix(prefix), + Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.fields(), + Storage.BlobListOption.userProject(userProject)); + } + final Iterator blobIterator = dirList.iterateAll().iterator(); + return new DirectoryStream() { + @Override + public Iterator iterator() { + return new LazyPathIterator( + cloudPath.getFileSystem(), prefix, blobIterator, filter, dir.isAbsolute()); + } + + @Override + public void close() throws IOException { + // Does nothing since there's nothing to close. Commenting this method to quiet codacy. + } + }; + } catch (StorageException exs) { + // Will rethrow a StorageException if all retries/reopens are exhausted + retryHandler.handleStorageException(exs); + // we're being aggressive by retrying even on scenarios where we'd normally reopen. + } + } + } + + /** Throws {@link UnsupportedOperationException} because Cloud Storage objects are immutable. */ + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) { + throw new CloudStorageObjectImmutableException(); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public FileStore getFileStore(Path path) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageFileSystemProvider + && Objects.equals(storage, ((CloudStorageFileSystemProvider) other).storage); + } + + @Override + public int hashCode() { + return Objects.hash(storage); + } + + @Override + public String toString() { + initStorage(); + return MoreObjects.toStringHelper(this).add("storage", storage).toString(); + } + + /** + * @param bucketName the name of the bucket to check + * @return whether requester pays is enabled for that bucket + */ + public boolean requesterPays(String bucketName) { + initStorage(); + try { + final Bucket bucket = storage.get(bucketName); + // If the bucket doesn't exist it can't be requester pays. + if (bucket == null) { + return false; + } + // instead of true/false, this method returns true/null + Boolean isRP = bucket.requesterPays(); + return isRP != null && isRP.booleanValue(); + } catch (StorageException ex) { + if ("userProjectMissing".equals(ex.getReason())) { + return true; + // fallback to checking the error code and error message. + } else if (ex.getCode() == 400 + && ex.getMessage() != null + && ex.getMessage().contains("requester pays")) { + return true; + } + throw ex; + } + } + + /** + * Returns a NEW CloudStorageFileSystemProvider identical to this one, but with userProject + * removed. + * + *

Perhaps you want to call this is you realize you'll be working on a bucket that is not + * requester-pays. + */ + public CloudStorageFileSystemProvider withNoUserProject() { + return new CloudStorageFileSystemProvider("", this.storageOptions); + } + + /** Returns the project that is assigned to this provider. */ + public String getProject() { + initStorage(); + return storage.getOptions().getProjectId(); + } + + /** + * Lists the project's buckets. But use the one in CloudStorageFileSystem. + * + *

Example of listing buckets, specifying the page size and a name prefix. + * + *

{@code
+   * String prefix = "bucket_";
+   * Page buckets = provider.listBuckets(BucketListOption.prefix(prefix));
+   * Iterator bucketIterator = buckets.iterateAll();
+   * while (bucketIterator.hasNext()) {
+   *   Bucket bucket = bucketIterator.next();
+   *   // do something with the bucket
+   * }
+   * }
+ * + * @throws StorageException upon failure + */ + Page listBuckets(Storage.BucketListOption... options) { + initStorage(); + return storage.list(options); + } + + private IOException asIoException(StorageException oops, boolean operationWasCopy) { + // RPC API can only throw StorageException, but CloudStorageFileSystemProvider + // can only throw IOException. Square peg, round hole. + // TODO(#810): Research if other codes should be translated similarly. + if (oops.getCode() == 404) { + return new NoSuchFileException(oops.getReason()); + } else if (operationWasCopy && oops.getCode() == 412) { + return new FileAlreadyExistsException( + String.format( + "Copy failed for reason '%s'. This most likely means the destination already exists" + + " and java.nio.file.StandardCopyOption.REPLACE_EXISTING was not specified.", + oops.getReason())); + } + + Throwable cause = oops.getCause(); + try { + if (cause instanceof FileAlreadyExistsException) { + throw new FileAlreadyExistsException(((FileAlreadyExistsException) cause).getReason()); + } + // fallback + if (cause != null && cause instanceof IOException) { + return (IOException) cause; + } + } catch (IOException okEx) { + return okEx; + } + return new IOException(oops.getMessage(), oops); + } + + @VisibleForTesting + void doInitStorage() { + this.storage = + storageOptions != null + ? storageOptions.getService() + : StorageOptionsUtil.getDefaultInstance().getService(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java new file mode 100644 index 000000000000..6f24fff06a99 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java @@ -0,0 +1,146 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.BlobInfo; +import com.google.common.base.MoreObjects; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; +import java.nio.file.attribute.FileTime; +import java.util.List; +import java.util.Objects; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.Immutable; + +/** Metadata for a Google Cloud Storage file. */ +@Immutable +final class CloudStorageObjectAttributes implements CloudStorageFileAttributes { + + @Nonnull private final BlobInfo info; + + CloudStorageObjectAttributes(BlobInfo info) { + this.info = checkNotNull(info); + } + + @Override + public long size() { + return info.getSize(); + } + + @Override + public FileTime creationTime() { + if (info.getUpdateTime() == null) { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + return FileTime.fromMillis(info.getUpdateTime()); + } + + @Override + public FileTime lastModifiedTime() { + return creationTime(); + } + + @Override + public Optional etag() { + return Optional.fromNullable(info.getEtag()); + } + + @Override + public Optional mimeType() { + return Optional.fromNullable(info.getContentType()); + } + + @Override + public Optional> acl() { + return Optional.fromNullable(info.getAcl()); + } + + @Override + public Optional cacheControl() { + return Optional.fromNullable(info.getCacheControl()); + } + + @Override + public Optional contentEncoding() { + return Optional.fromNullable(info.getContentEncoding()); + } + + @Override + public Optional contentDisposition() { + return Optional.fromNullable(info.getContentDisposition()); + } + + @Override + public ImmutableMap userMetadata() { + if (null == info.getMetadata()) { + return ImmutableMap.of(); + } + return ImmutableMap.copyOf(info.getMetadata()); + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isRegularFile() { + return true; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public FileTime lastAccessTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public Object fileKey() { + return info.getBlobId().getBucket() + + info.getBlobId().getName() + + info.getBlobId().getGeneration(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageObjectAttributes + && Objects.equals(info, ((CloudStorageObjectAttributes) other).info); + } + + @Override + public int hashCode() { + return info.hashCode(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("info", info).toString(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectImmutableException.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectImmutableException.java new file mode 100644 index 000000000000..fe15298fff82 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectImmutableException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +/** Exception reminding user that Cloud Storage objects can't be mutated. */ +public final class CloudStorageObjectImmutableException extends UnsupportedOperationException { + + CloudStorageObjectImmutableException() { + super("Cloud Storage objects are immutable."); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOption.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOption.java new file mode 100644 index 000000000000..98342eba49c8 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOption.java @@ -0,0 +1,35 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import java.nio.file.CopyOption; +import java.nio.file.OpenOption; + +/** Main interface for file operation option classes related to Google Cloud Storage. */ +public interface CloudStorageOption { + + /** Interface for Google Cloud Storage options that can be specified when opening files. */ + interface Open extends CloudStorageOption, OpenOption {} + + /** Interface for Google Cloud Storage options that can be specified when copying files. */ + interface Copy extends CloudStorageOption, CopyOption {} + + /** + * Interface for Google Cloud Storage options that can be specified when opening or copying files. + */ + interface OpenCopy extends Open, Copy {} +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOptions.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOptions.java new file mode 100644 index 000000000000..2d0548a112e2 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOptions.java @@ -0,0 +1,107 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.storage.Acl; + +/** Helper class for specifying options when opening and copying Cloud Storage files. */ +public final class CloudStorageOptions { + + /** Sets the mime type header on an object, e.g. {@code "text/plain"}. */ + public static CloudStorageOption.OpenCopy withMimeType(String mimeType) { + return OptionMimeType.create(mimeType); + } + + /** Disables caching on an object. Same as: {@code withCacheControl("no-cache")}. */ + public static CloudStorageOption.OpenCopy withoutCaching() { + return withCacheControl("no-cache"); + } + + /** + * Sets the {@code Cache-Control} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#cachecontrol" + */ + public static CloudStorageOption.OpenCopy withCacheControl(String cacheControl) { + return OptionCacheControl.create(cacheControl); + } + + /** + * Sets the {@code Content-Disposition} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + public static CloudStorageOption.OpenCopy withContentDisposition(String contentDisposition) { + return OptionContentDisposition.create(contentDisposition); + } + + /** + * Sets the {@code Content-Encoding} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentencoding" + */ + public static CloudStorageOption.OpenCopy withContentEncoding(String contentEncoding) { + return OptionContentEncoding.create(contentEncoding); + } + + /** + * Sets the ACL value on a Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#acl" + */ + public static CloudStorageOption.OpenCopy withAcl(Acl acl) { + return OptionAcl.create(acl); + } + + /** + * Sets an unmodifiable piece of user metadata on a Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#xgoogmeta" + */ + public static CloudStorageOption.OpenCopy withUserMetadata(String key, String value) { + return OptionUserMetadata.create(key, value); + } + + /** + * Sets the block size (in bytes) when talking to the Google Cloud Storage server. + * + *

The default is {@value CloudStorageFileSystem#BLOCK_SIZE_DEFAULT}. + */ + public static CloudStorageOption.OpenCopy withBlockSize(int size) { + return OptionBlockSize.create(size); + } + + /** + * Sets the max number of times that the channel can be reopened if reading fails because the + * channel unexpectedly closes. + * + *

The default is 0. + */ + public static CloudStorageOption.OpenCopy withChannelReopen(int count) { + return OptionMaxChannelReopens.create(count); + } + + /** + * Allows one to use trailing slashes in file names. You really shouldn't (this is here for tests + * only). + */ + static CloudStorageOption.Open allowTrailingSlash() { + return OptionAllowTrailingSlash.getInstance(); + } + + private CloudStorageOptions() {} +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java new file mode 100644 index 000000000000..1c49f8e1c1e0 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java @@ -0,0 +1,423 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.Storage; +import com.google.common.base.Strings; +import com.google.common.collect.UnmodifiableIterator; +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.WatchEvent.Kind; +import java.nio.file.WatchEvent.Modifier; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.Collections; +import java.util.Iterator; +import java.util.Objects; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * A Google Cloud Storage specific implementation of the {@code java.nio.file.Path} interface. An + * instance of this class locates an object or a "pseudo-directory" in GCS. This implementation + * allows one to use Java's standard file system API to deal with remote objects as if they are + * local files. + * + *

Example of using {@code java.nio.file.Files} to read all lines from a remote object: + * + *

{@code
+ * Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ * List lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+ * }
+ */ +@Immutable +public final class CloudStoragePath implements Path { + + private static final Pattern EXTRA_SLASHES_OR_DOT_DIRS_PATTERN = + Pattern.compile("^\\.\\.?/|//|/\\.\\.?/|/\\.\\.?$"); + + private final CloudStorageFileSystem fileSystem; + private final UnixPath path; + + private CloudStoragePath(CloudStorageFileSystem fileSystem, UnixPath path) { + this.fileSystem = fileSystem; + this.path = path; + } + + static CloudStoragePath getPath(CloudStorageFileSystem fileSystem, String path, String... more) { + return new CloudStoragePath( + fileSystem, UnixPath.getPath(fileSystem.config().permitEmptyPathComponents(), path, more)); + } + + /** Returns the Cloud Storage bucket name being served by this file system. */ + public String bucket() { + return fileSystem.bucket(); + } + + /** Returns path converted to a {@link BlobId} so I/O can be performed. */ + BlobId getBlobId() { + checkArgument(!path.toString().isEmpty(), "Object names cannot be empty."); + return BlobId.of(bucket(), toRealPath().path.toString()); + } + + boolean seemsLikeADirectory() { + return path.seemsLikeADirectory(); + } + + // True if this path may be a directory (and pseudo-directories are enabled) + // Checks: + // 1) does the path end in / ? + // 2) (optional, if storage is set) is there a file whose name starts with path+/ ? + boolean seemsLikeADirectoryAndUsePseudoDirectories(Storage storage) { + if (!fileSystem.config().usePseudoDirectories()) { + return false; + } + if (path.seemsLikeADirectory()) { + return true; + } + // fancy case: the file name doesn't end in slash, but we've been asked to have pseudo dirs. + // Let's see if there are any files with this prefix. + if (storage == null) { + // we are in a context where we don't want to access the storage, so we conservatively + // say this isn't a directory. + return false; + } + // Using the provided path + "/" as a prefix, can we find one file? If so, the path + // is a directory. + String prefix = path.removeBeginningSeparator().toString(); + if (!prefix.endsWith("/")) { + prefix += "/"; + } + String userProject = fileSystem.config().userProject(); + Page list = null; + if (!Strings.isNullOrEmpty(userProject)) { + list = + storage.list( + this.bucket(), + Storage.BlobListOption.prefix(prefix), + // we only look at the first result, so no need for a bigger page. + Storage.BlobListOption.pageSize(1), + Storage.BlobListOption.userProject(userProject)); + } else { + list = + storage.list( + this.bucket(), + Storage.BlobListOption.prefix(prefix), + // we only look at the first result, so no need for a bigger page. + Storage.BlobListOption.pageSize(1)); + } + for (Blob b : list.getValues()) { + // if this blob starts with our prefix and then a slash, then prefix is indeed a folder! + if (b.getBlobId() == null) { + continue; + } + String name = b.getBlobId().getName(); + if (name == null) { + continue; + } + if (("/" + name).startsWith(this.path.toAbsolutePath() + "/")) { + return true; + } + } + // no match, so it's not a directory + return false; + } + + @Override + public CloudStorageFileSystem getFileSystem() { + return fileSystem; + } + + @Nullable + @Override + public CloudStoragePath getRoot() { + return newPath(path.getRoot()); + } + + @Override + public boolean isAbsolute() { + return path.isAbsolute(); + } + + /** + * Changes relative path to be absolute, using {@link CloudStorageConfiguration#workingDirectory() + * workingDirectory} as current dir. + */ + @Override + public CloudStoragePath toAbsolutePath() { + return newPath(path.toAbsolutePath(getWorkingDirectory())); + } + + /** + * Returns this path rewritten to the Cloud Storage object name that'd be used to perform i/o. + * + *

This method makes path {@link #toAbsolutePath() absolute} and removes the prefix slash from + * the absolute path when {@link CloudStorageConfiguration#stripPrefixSlash() stripPrefixSlash} is + * {@code true}. + * + * @throws IllegalArgumentException if path contains extra slashes or dot-dirs when {@link + * CloudStorageConfiguration#permitEmptyPathComponents() permitEmptyPathComponents} is {@code + * false}, or if the resulting path is empty. + */ + @Override + public CloudStoragePath toRealPath(LinkOption... options) { + CloudStorageUtil.checkNotNullArray(options); + return newPath(toRealPathInternal(true)); + } + + private UnixPath toRealPathInternal(boolean errorCheck) { + UnixPath objectName = path.toAbsolutePath(getWorkingDirectory()); + if (errorCheck && !fileSystem.config().permitEmptyPathComponents()) { + checkArgument( + !EXTRA_SLASHES_OR_DOT_DIRS_PATTERN.matcher(objectName).find(), + "I/O not allowed on dot-dirs or extra slashes when !permitEmptyPathComponents: %s", + objectName); + } + if (fileSystem.config().stripPrefixSlash()) { + objectName = objectName.removeBeginningSeparator(); + } + return objectName; + } + + /** + * Returns path without extra slashes or {@code .} and {@code ..} and preserves trailing slash. + */ + @Override + public CloudStoragePath normalize() { + return newPath(path.normalize()); + } + + @Override + public CloudStoragePath resolve(Path object) { + return newPath(path.resolve(CloudStorageUtil.checkPath(object).path)); + } + + @Override + public CloudStoragePath resolve(String other) { + return newPath(path.resolve(getUnixPath(other))); + } + + @Override + public CloudStoragePath resolveSibling(Path other) { + return newPath(path.resolveSibling(CloudStorageUtil.checkPath(other).path)); + } + + @Override + public CloudStoragePath resolveSibling(String other) { + return newPath(path.resolveSibling(getUnixPath(other))); + } + + @Override + public CloudStoragePath relativize(Path object) { + return newPath(path.relativize(CloudStorageUtil.checkPath(object).path)); + } + + @Nullable + @Override + public CloudStoragePath getParent() { + return newPath(path.getParent()); + } + + @Nullable + @Override + public CloudStoragePath getFileName() { + return newPath(path.getFileName()); + } + + @Override + public CloudStoragePath subpath(int beginIndex, int endIndex) { + return newPath(path.subpath(beginIndex, endIndex)); + } + + @Override + public int getNameCount() { + return path.getNameCount(); + } + + @Override + public CloudStoragePath getName(int index) { + return newPath(path.getName(index)); + } + + @Override + public boolean startsWith(Path other) { + if (!(checkNotNull(other) instanceof CloudStoragePath)) { + return false; + } + CloudStoragePath that = (CloudStoragePath) other; + if (!bucket().equals(that.bucket())) { + return false; + } + return path.startsWith(that.path); + } + + @Override + public boolean startsWith(String other) { + return path.startsWith(getUnixPath(other)); + } + + @Override + public boolean endsWith(Path other) { + if (!(checkNotNull(other) instanceof CloudStoragePath)) { + return false; + } + CloudStoragePath that = (CloudStoragePath) other; + if (!bucket().equals(that.bucket())) { + return false; + } + return path.endsWith(that.path); + } + + @Override + public boolean endsWith(String other) { + return path.endsWith(getUnixPath(other)); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public WatchKey register(WatchService watcher, Kind[] events, Modifier... modifiers) { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public WatchKey register(WatchService watcher, Kind... events) { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link UnsupportedOperationException} because Google Cloud Storage files are not backed + * by the local file system. + */ + @Override + public File toFile() { + throw new UnsupportedOperationException("GCS objects aren't available locally"); + } + + @Override + public Iterator iterator() { + if (path.isEmpty()) { + return Collections.singleton(this).iterator(); + } else if (path.isRoot()) { + return Collections.emptyIterator(); + } else { + return new PathIterator(); + } + } + + @Override + public int compareTo(Path other) { + // Documented to throw CCE if other is associated with a different FileSystemProvider. + CloudStoragePath that = (CloudStoragePath) other; + int res = bucket().compareTo(that.bucket()); + if (res != 0) { + return res; + } + return toRealPathInternal(false).compareTo(that.toRealPathInternal(false)); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStoragePath + && Objects.equals(bucket(), ((CloudStoragePath) other).bucket()) + && Objects.equals( + toRealPathInternal(false), ((CloudStoragePath) other).toRealPathInternal(false)); + } + + @Override + public int hashCode() { + return Objects.hash(bucket(), toRealPathInternal(false)); + } + + @Override + public String toString() { + return path.toString(); + } + + @Override + public URI toUri() { + try { + // First try storing GCS bucket name in the hostname for compatibility with earlier behavior. + return new URI( + CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null); + } catch (URISyntaxException e) { + try { + // Store GCS bucket name in the URI authority, see + // https://github.com/googleapis/java-storage-nio/issues/1218 + return new URI( + CloudStorageFileSystem.URI_SCHEME, + bucket(), + path.toAbsolutePath().toString(), + null, + null); + } catch (URISyntaxException unused) { + throw new AssertionError(e); + } + } + } + + @Nullable + private CloudStoragePath newPath(@Nullable UnixPath newPath) { + if (newPath == path) { // Nonuse of equals is intentional. + return this; + } else if (newPath != null) { + return new CloudStoragePath(fileSystem, newPath); + } else { + return null; + } + } + + private UnixPath getUnixPath(String newPath) { + return UnixPath.getPath(fileSystem.config().permitEmptyPathComponents(), newPath); + } + + private UnixPath getWorkingDirectory() { + return getUnixPath(fileSystem.config().workingDirectory()); + } + + /** Transform iterator providing a slight performance boost over {@code FluentIterable}. */ + private final class PathIterator extends UnmodifiableIterator { + private final Iterator delegate = path.split(); + + @Override + public Path next() { + return newPath(getUnixPath(delegate.next())); + } + + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java new file mode 100644 index 000000000000..c55f4d13ea7f --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java @@ -0,0 +1,113 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.storage.Acl; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; +import java.nio.file.attribute.FileTime; +import java.util.List; + +/** Metadata for a Cloud Storage pseudo-directory. */ +final class CloudStoragePseudoDirectoryAttributes implements CloudStorageFileAttributes { + + private final String id; + + CloudStoragePseudoDirectoryAttributes(CloudStoragePath path) { + this.id = path.toUri().toString(); + } + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isRegularFile() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public Object fileKey() { + return id; + } + + @Override + public long size() { + return 1; // Allow I/O to happen before we fail. + } + + @Override + public FileTime lastModifiedTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public FileTime creationTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public FileTime lastAccessTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public Optional etag() { + return Optional.absent(); + } + + @Override + public Optional mimeType() { + return Optional.absent(); + } + + @Override + public Optional> acl() { + return Optional.absent(); + } + + @Override + public Optional cacheControl() { + return Optional.absent(); + } + + @Override + public Optional contentEncoding() { + return Optional.absent(); + } + + @Override + public Optional contentDisposition() { + return Optional.absent(); + } + + @Override + public ImmutableMap userMetadata() { + return ImmutableMap.of(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java new file mode 100644 index 000000000000..38dc88c8d08c --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java @@ -0,0 +1,27 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import java.nio.file.InvalidPathException; + +/** Exception thrown when erroneously trying to operate on a path with a trailing slash. */ +public final class CloudStoragePseudoDirectoryException extends InvalidPathException { + + CloudStoragePseudoDirectoryException(CloudStoragePath path) { + super(path.toString(), "Can't perform I/O on pseudo-directories (trailing slash)"); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java new file mode 100644 index 000000000000..5e62b7ca79b7 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java @@ -0,0 +1,278 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.cloud.ReadChannel; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobSourceOption; +import com.google.cloud.storage.StorageException; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.NoSuchFileException; +import java.util.List; +import javax.annotation.CheckReturnValue; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Cloud Storage read channel. + * + * @see CloudStorageWriteChannel + */ +@ThreadSafe +final class CloudStorageReadChannel implements SeekableByteChannel { + + private final Storage gcsStorage; + private final BlobId file; + // max # of times we may reopen the file + @VisibleForTesting final int maxChannelReopens; + // max # of times we may retry a GCS operation + final int maxRetries; + // open options, we keep them around for reopens. + final BlobSourceOption[] blobSourceOptions; + final CloudStorageConfiguration config; + private ReadChannel channel; + private long position; + private long size; + // generation at time of first open, to make sure reopens don't give us a different version of the + // file. + // It can be null if not implemented, in which case we don't check. + private Long generation; + + /** + * @param maxChannelReopens max number of times to try re-opening the channel if it closes on us + * unexpectedly. + * @param config configuration for what to retry on. + * @param blobSourceOptions BlobSourceOption.userProject if you want to pay the charges (required + * for requester-pays buckets). Note: Buckets that have Requester Pays disabled still accept + * requests that include a billing project, and charges are applied to the billing project + * supplied in the request. Consider any billing implications prior to including a billing + * project in all of your requests. Source: + * https://cloud.google.com/storage/docs/requester-pays + * @param userProject: the project you want billed (set this for requester-pays buckets). Leave + * empty otherwise. + */ + @CheckReturnValue + @SuppressWarnings("resource") + static CloudStorageReadChannel create( + Storage gcsStorage, + BlobId file, + long position, + int maxChannelReopens, + final CloudStorageConfiguration config, + @Nullable String userProject, + BlobSourceOption... blobSourceOptions) + throws IOException { + return new CloudStorageReadChannel( + gcsStorage, file, position, maxChannelReopens, config, userProject, blobSourceOptions); + } + + private CloudStorageReadChannel( + Storage gcsStorage, + BlobId file, + long position, + int maxChannelReopens, + final CloudStorageConfiguration config, + @Nullable String userProject, + BlobSourceOption... blobSourceOptions) + throws IOException { + this.gcsStorage = gcsStorage; + this.file = file; + this.position = position; + this.maxChannelReopens = maxChannelReopens; + this.maxRetries = Math.max(3, maxChannelReopens); + this.config = config; + // get the generation, enshrine that in our options + fetchSize(gcsStorage, userProject, file); + List options = Lists.newArrayList(blobSourceOptions); + if (null != generation) { + options.add(Storage.BlobSourceOption.generationMatch(generation)); + } + if (!Strings.isNullOrEmpty(userProject)) { + options.add(BlobSourceOption.userProject(userProject)); + } + this.blobSourceOptions = options.toArray(new BlobSourceOption[options.size()]); + + // innerOpen checks that it sees the same generation as fetchSize did, + // which ensure the file hasn't changed. + innerOpen(); + } + + private void innerOpen() throws IOException { + this.channel = gcsStorage.reader(file, blobSourceOptions); + if (position > 0) { + channel.seek(position); + } + } + + @Override + public boolean isOpen() { + synchronized (this) { + return channel.isOpen(); + } + } + + @Override + public void close() throws IOException { + synchronized (this) { + channel.close(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + synchronized (this) { + checkOpen(); + int amt; + final CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(maxRetries, maxChannelReopens, config); + dst.mark(); + while (true) { + try { + dst.reset(); + amt = channel.read(dst); + break; + } catch (StorageException exs) { + // Will rethrow a StorageException if all retries/reopens are exhausted + handleStorageException(exs, retryHandler); + } + } + if (amt > 0) { + position += amt; + // This can only happen if the file changed under us and we didn't notice. + if (position > size) { + size = position; + } + } + return amt; + } + } + + @Override + public long size() throws IOException { + synchronized (this) { + checkOpen(); + return size; + } + } + + @Override + public long position() throws IOException { + synchronized (this) { + checkOpen(); + return position; + } + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + checkArgument(newPosition >= 0); + synchronized (this) { + checkOpen(); + if (newPosition == position) { + return this; + } + channel.seek(newPosition); + position = newPosition; + return this; + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new NonWritableChannelException(); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + throw new NonWritableChannelException(); + } + + private void checkOpen() throws ClosedChannelException { + if (!channel.isOpen()) { + throw new ClosedChannelException(); + } + } + + private long fetchSize(Storage gcsStorage, @Nullable String userProject, BlobId file) + throws IOException { + final CloudStorageRetryHandler retryHandler = + new CloudStorageRetryHandler(maxRetries, maxChannelReopens, config); + + while (true) { + try { + BlobInfo blobInfo; + if (Strings.isNullOrEmpty(userProject)) { + blobInfo = + gcsStorage.get( + file, + Storage.BlobGetOption.fields( + Storage.BlobField.GENERATION, Storage.BlobField.SIZE)); + } else { + blobInfo = + gcsStorage.get( + file, + Storage.BlobGetOption.fields( + Storage.BlobField.GENERATION, Storage.BlobField.SIZE), + Storage.BlobGetOption.userProject(userProject)); + } + if (blobInfo == null) { + throw new NoSuchFileException( + String.format("gs://%s/%s", file.getBucket(), file.getName())); + } + this.generation = blobInfo.getGeneration(); + this.size = blobInfo.getSize(); + return this.size; + } catch (StorageException exs) { + // Will rethrow a StorageException if all retries/reopens are exhausted + retryHandler.handleStorageException(exs); + // there's nothing to reopen yet, but retry even for a reopenable error. + } + } + } + + /** + * Handles a StorageException by reopening the channel or sleeping for a retry attempt if retry + * count is not exhausted. Throws a StorageException if all reopens/retries are exhausted, or if + * the StorageException is not reopenable/retryable. + * + * @param exs StorageException thrown by a GCS operation + * @param retryHandler Keeps track of reopens/retries performed so far on this operation + * @throws StorageException if all reopens/retries are exhausted, or if exs is not + * reopenable/retryable + * @throws IOException if a reopen operation fails + */ + private void handleStorageException( + final StorageException exs, final CloudStorageRetryHandler retryHandler) throws IOException { + boolean shouldReopen = retryHandler.handleStorageException(exs); + if (shouldReopen) { + // these errors aren't marked as retryable since the channel is closed; + // but here at this higher level we can retry them. + innerOpen(); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadFileChannel.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadFileChannel.java new file mode 100644 index 000000000000..db89365b939d --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadFileChannel.java @@ -0,0 +1,154 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage.contrib.nio; + +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.channels.WritableByteChannel; + +class CloudStorageReadFileChannel extends FileChannel { + private static final String READ_ONLY = "This FileChannel is read-only"; + private final SeekableByteChannel readChannel; + + CloudStorageReadFileChannel(SeekableByteChannel readChannel) { + Preconditions.checkNotNull(readChannel); + this.readChannel = readChannel; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + return readChannel.read(dst); + } + + @Override + public synchronized long read(ByteBuffer[] dsts, int offset, int length) throws IOException { + long res = 0L; + for (int i = offset; i < offset + length; i++) { + res += readChannel.read(dsts[i]); + } + return res; + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new UnsupportedOperationException(READ_ONLY); + } + + @Override + public long write(ByteBuffer[] srcs, int offset, int length) throws IOException { + throw new UnsupportedOperationException(READ_ONLY); + } + + @Override + public long position() throws IOException { + return readChannel.position(); + } + + @Override + public FileChannel position(long newPosition) throws IOException { + readChannel.position(newPosition); + return this; + } + + @Override + public long size() throws IOException { + return readChannel.size(); + } + + @Override + public FileChannel truncate(long size) throws IOException { + throw new UnsupportedOperationException(READ_ONLY); + } + + @Override + public void force(boolean metaData) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public synchronized long transferTo( + long transferFromPosition, long count, WritableByteChannel target) throws IOException { + long res = 0L; + long originalPosition = position(); + try { + position(transferFromPosition); + int blockSize = (int) Math.min(count, 0xfffffL); + int bytesRead = 0; + ByteBuffer buffer = ByteBuffer.allocate(blockSize); + while (res < count && bytesRead >= 0) { + buffer.position(0); + bytesRead = read(buffer); + if (bytesRead > 0) { + buffer.position(0); + buffer.limit(bytesRead); + target.write(buffer); + res += bytesRead; + } + } + return res; + } finally { + position(originalPosition); + } + } + + @Override + public long transferFrom(ReadableByteChannel src, long position, long count) throws IOException { + throw new UnsupportedOperationException(READ_ONLY); + } + + @Override + public synchronized int read(ByteBuffer dst, long readFromPosition) throws IOException { + long originalPosition = position(); + try { + position(readFromPosition); + int res = readChannel.read(dst); + return res; + } finally { + position(originalPosition); + } + } + + @Override + public int write(ByteBuffer src, long position) throws IOException { + throw new UnsupportedOperationException(READ_ONLY); + } + + @Override + public MappedByteBuffer map(MapMode mode, long position, long size) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FileLock lock(long position, long size, boolean shared) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FileLock tryLock(long position, long size, boolean shared) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + protected void implCloseChannel() throws IOException { + readChannel.close(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java new file mode 100644 index 000000000000..ef912a3e0d08 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java @@ -0,0 +1,224 @@ +/* + * Copyright 2017 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.storage.StorageException; +import com.google.common.annotations.VisibleForTesting; + +/** + * Simple counter class to keep track of retry and reopen attempts when StorageExceptions are + * encountered. Handles sleeping between retry/reopen attempts, as well as throwing an exception + * when all retries/reopens are exhausted. + */ +public class CloudStorageRetryHandler { + private int retries; + private int reopens; + private long totalWaitTime; // in milliseconds + private final int maxRetries; + private final int maxReopens; + private final CloudStorageConfiguration config; + + /** + * Create a CloudStorageRetryHandler with the maximum retries and reopens set to the same value. + * + * @param maxRetriesAndReopens value for both maxRetries and maxReopens + * @deprecated use CloudStorageRetryHandler(CloudStorageConfiguration) instead. + */ + @java.lang.Deprecated + public CloudStorageRetryHandler(final int maxRetriesAndReopens) { + this.maxRetries = maxRetriesAndReopens; + this.maxReopens = maxRetriesAndReopens; + // we're just using the retry parameters from the config, so it's OK to have a default. + this.config = CloudStorageConfiguration.DEFAULT; + } + + /** + * Create a CloudStorageRetryHandler with the maximum retries and reopens set to different values. + * + * @param maxRetries maximum number of retries + * @param maxReopens maximum number of reopens + * @deprecated use CloudStorageRetryHandler(CloudStorageConfiguration) instead. + */ + @java.lang.Deprecated + public CloudStorageRetryHandler(final int maxRetries, final int maxReopens) { + this.maxRetries = maxRetries; + this.maxReopens = maxReopens; + // we're just using the retry parameters from the config, so it's OK to have a default. + this.config = CloudStorageConfiguration.DEFAULT; + } + + /** + * Create a CloudStorageRetryHandler with the maximum retries and reopens set to the same value. + * + * @param config - configuration for reopens and retryable codes. + */ + public CloudStorageRetryHandler(final CloudStorageConfiguration config) { + this.maxRetries = config.maxChannelReopens(); + this.maxReopens = config.maxChannelReopens(); + this.config = config; + } + + /** + * Create a CloudStorageRetryHandler with the maximum retries and reopens set to different values. + * + * @param maxRetries maximum number of retries + * @param maxReopens maximum number of reopens (overrides what's in the config) + * @param config http codes we'll retry on, and exceptions we'll reopen on. + */ + public CloudStorageRetryHandler( + final int maxRetries, final int maxReopens, final CloudStorageConfiguration config) { + this.maxRetries = maxRetries; + this.maxReopens = maxReopens; + this.config = config; + } + + /** + * @return number of retries we've performed + */ + public int retries() { + return retries; + } + + /** + * @return number of reopens we've performed + */ + public int reopens() { + return reopens; + } + + /** + * Checks whether we should retry, reopen, or give up. + * + *

In the latter case it throws an exception (this includes the scenario where we exhausted the + * retry count). + * + *

Otherwise, it sleeps for a bit and returns whether we should reopen. The sleep time is + * dependent on the retry number. + * + * @param exs caught StorageException + * @return True if you need to reopen and then try again. False if you can just try again. + * @throws StorageException if the exception is not retryable, or if you ran out of retries. + */ + public boolean handleStorageException(final StorageException exs) throws StorageException { + // None of the retryable exceptions are reopenable, so it's OK to write the code this way. + if (isRetryable(exs)) { + handleRetryForStorageException(exs); + return false; + } + if (isReopenable(exs)) { + handleReopenForStorageException(exs); + return true; + } + throw exs; + } + + /** + * Records a retry attempt for the given StorageException, sleeping for an amount of time + * dependent on the attempt number. Throws a StorageException if we've exhausted all retries. + * + * @param exs The StorageException error that prompted this retry attempt. + */ + private void handleRetryForStorageException(final StorageException exs) throws StorageException { + retries++; + if (retries > maxRetries) { + throw new StorageException( + exs.getCode(), + "All " + + maxRetries + + " retries failed. Waited a total of " + + totalWaitTime + + " ms between attempts", + exs); + } + sleepForAttempt(retries); + } + + /** + * Records a reopen attempt for the given StorageException, sleeping for an amount of time + * dependent on the attempt number. Throws a StorageException if we've exhausted all reopens. + * + * @param exs The StorageException error that prompted this reopen attempt. + */ + private void handleReopenForStorageException(final StorageException exs) throws StorageException { + reopens++; + if (reopens > maxReopens) { + throw new StorageException( + exs.getCode(), + "All " + + maxReopens + + " reopens failed. Waited a total of " + + totalWaitTime + + " ms between attempts", + exs); + } + sleepForAttempt(reopens); + } + + void sleepForAttempt(int attempt) { + // exponential backoff, but let's bound it around 2min. + // aggressive backoff because we're dealing with unusual cases. + long delay = 1000L * (1L << Math.min(attempt, 7)); + try { + Thread.sleep(delay); + totalWaitTime += delay; + } catch (InterruptedException iex) { + // reset interrupt flag + Thread.currentThread().interrupt(); + } + } + + /** + * @param exs StorageException to test + * @return true if exs is a retryable error, otherwise false + */ + @VisibleForTesting + boolean isRetryable(final StorageException exs) { + if (exs.isRetryable()) { + return true; + } + for (int code : config.retryableHttpCodes()) { + if (exs.getCode() == code) { + return true; + } + } + return false; + } + + /** + * @param exs StorageException to test + * @return true if exs is an error that can be resolved via a channel reopen, otherwise false + */ + @VisibleForTesting + boolean isReopenable(final StorageException exs) { + Throwable throwable = exs; + // ensures finite iteration + int maxDepth = 20; + while (throwable != null && maxDepth-- > 0) { + for (Class reopenableException : config.reopenableExceptions()) { + if (reopenableException.isInstance(throwable)) { + return true; + } + } + if (throwable.getMessage() != null + && throwable.getMessage().contains("Connection closed prematurely")) { + return true; + } + throwable = throwable.getCause(); + } + return false; + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageUtil.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageUtil.java new file mode 100644 index 000000000000..3802fef0ca8e --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageUtil.java @@ -0,0 +1,89 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.util.regex.Pattern; + +final class CloudStorageUtil { + + private static final Pattern BUCKET_PATTERN = Pattern.compile("[a-z0-9][-._a-z0-9]+[a-z0-9]"); + + static void checkBucket(String bucket) { + // TODO: The true check is actually more complicated. Consider implementing it. + checkArgument( + BUCKET_PATTERN.matcher(bucket).matches(), + "Invalid bucket name: '" + + bucket + + "'. " + + "Google Cloud Storage bucket names must contain only lowercase letters, numbers, " + + "dashes (-), underscores (_), and dots (.). Bucket names must start and end with a " + + "number or a letter. See the following page for more details: " + + "https://developers.google.com/storage/docs/bucketnaming"); + } + + static CloudStoragePath checkPath(Path path) { + if (!(checkNotNull(path) instanceof CloudStoragePath)) { + throw new ProviderMismatchException( + String.format( + "Not a Cloud Storage path: %s (%s)", path, path.getClass().getSimpleName())); + } + return (CloudStoragePath) path; + } + + static URI stripPathFromUri(URI uri) { + try { + return new URI( + uri.getScheme(), + uri.getUserInfo(), + uri.getHost(), + uri.getPort(), + null, + uri.getQuery(), + uri.getFragment()); + } catch (URISyntaxException e) { + try { + // Store GCS bucket name in the URI authority, see + // https://github.com/googleapis/java-storage-nio/issues/1218 + return new URI( + uri.getScheme(), uri.getAuthority(), null, uri.getQuery(), uri.getFragment()); + } catch (URISyntaxException unused) { + throw new IllegalArgumentException(e.getMessage()); + } + } + } + + /** Makes {@code NullPointerTester} happy. */ + @SafeVarargs + static void checkNotNullArray(T... values) { + for (T value : values) { + checkNotNull(value); + } + } + + static int getMaxChannelReopensFromPath(Path path) { + return ((CloudStorageFileSystem) path.getFileSystem()).config().maxChannelReopens(); + } + + private CloudStorageUtil() {} +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannel.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannel.java new file mode 100644 index 000000000000..2994b831c8a7 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannel.java @@ -0,0 +1,110 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.WriteChannel; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonReadableChannelException; +import java.nio.channels.SeekableByteChannel; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Cloud Storage write channel. + * + *

This class does not support seeking, reading, or append. + * + * @see CloudStorageReadChannel + */ +@ThreadSafe +final class CloudStorageWriteChannel implements SeekableByteChannel { + + private final WriteChannel channel; + private long position; + private long size; + + CloudStorageWriteChannel(WriteChannel channel) { + this.channel = channel; + } + + @Override + public boolean isOpen() { + synchronized (this) { + return channel.isOpen(); + } + } + + @Override + public void close() throws IOException { + synchronized (this) { + channel.close(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + throw new NonReadableChannelException(); + } + + @Override + public int write(ByteBuffer src) throws IOException { + synchronized (this) { + checkOpen(); + int amt = channel.write(src); + if (amt > 0) { + position += amt; + size += amt; + } + return amt; + } + } + + @Override + public long position() throws IOException { + synchronized (this) { + checkOpen(); + return position; + } + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long size() throws IOException { + synchronized (this) { + checkOpen(); + return size; + } + } + + @Override + public SeekableByteChannel truncate(long newSize) throws IOException { + // TODO: Emulate this functionality by closing and rewriting old file up to newSize. + // Or maybe just swap out GcsStorage for the API client. + throw new UnsupportedOperationException(); + } + + private void checkOpen() throws ClosedChannelException { + if (!channel.isOpen()) { + throw new ClosedChannelException(); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteFileChannel.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteFileChannel.java new file mode 100644 index 000000000000..3e930f7cd2a8 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteFileChannel.java @@ -0,0 +1,180 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage.contrib.nio; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.channels.WritableByteChannel; + +class CloudStorageWriteFileChannel extends FileChannel { + private static final String WRITE_ONLY = "This FileChannel is write-only"; + private SeekableByteChannel writeChannel; + private boolean valid = true; + + CloudStorageWriteFileChannel(SeekableByteChannel writeChannel) { + this.writeChannel = writeChannel; + } + + private void checkValid() throws IOException { + if (!valid) { + // These methods are only supported to be called once, because the underlying channel does not + // support changing the position. + throw new IOException( + "This FileChannel is no longer valid. " + + "A Cloud Storage FileChannel is invalidated after calling one of " + + "the methods FileChannel#write(ByteBuffer, long) or " + + "FileChannel#transferFrom(ReadableByteChannel, long, long)"); + } + if (!writeChannel.isOpen()) { + throw new IOException("This FileChannel is closed"); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + throw new UnsupportedOperationException(WRITE_ONLY); + } + + @Override + public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { + throw new UnsupportedOperationException(WRITE_ONLY); + } + + @Override + public synchronized int write(ByteBuffer src) throws IOException { + checkValid(); + return writeChannel.write(src); + } + + @Override + public synchronized long write(ByteBuffer[] srcs, int offset, int length) throws IOException { + checkValid(); + long res = 0L; + for (int i = offset; i < offset + length; i++) { + res += writeChannel.write(srcs[i]); + } + return res; + } + + @Override + public synchronized long position() throws IOException { + checkValid(); + return writeChannel.position(); + } + + @Override + public synchronized FileChannel position(long newPosition) throws IOException { + if (newPosition != position()) { + writeChannel.position(newPosition); + } + return this; + } + + @Override + public synchronized long size() throws IOException { + checkValid(); + return writeChannel.size(); + } + + @Override + public synchronized FileChannel truncate(long size) throws IOException { + checkValid(); + writeChannel.truncate(size); + return this; + } + + @Override + public void force(boolean metaData) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long transferTo(long position, long count, WritableByteChannel target) throws IOException { + throw new UnsupportedOperationException(WRITE_ONLY); + } + + @Override + public synchronized long transferFrom(ReadableByteChannel src, long position, long count) + throws IOException { + if (position != position()) { + throw new UnsupportedOperationException( + "This FileChannel only supports transferFrom at the current position"); + } + int blockSize = (int) Math.min(count, 0xfffffL); + long res = 0L; + int bytesRead = 0; + ByteBuffer buffer = ByteBuffer.allocate(blockSize); + while (res < count && bytesRead >= 0) { + buffer.position(0); + bytesRead = src.read(buffer); + if (bytesRead > 0) { + buffer.position(0); + buffer.limit(bytesRead); + write(buffer); + res += bytesRead; + } + } + // The channel is no longer valid as the position has been updated, and there is no way of + // resetting it, but this way we at least support the write-at-position and transferFrom + // methods being called once. + this.valid = false; + return res; + } + + @Override + public int read(ByteBuffer dst, long position) throws IOException { + throw new UnsupportedOperationException(WRITE_ONLY); + } + + @Override + public synchronized int write(ByteBuffer src, long position) throws IOException { + if (position != position()) { + throw new UnsupportedOperationException( + "This FileChannel only supports write at the current position"); + } + int res = writeChannel.write(src); + // The channel is no longer valid as the position has been updated, and there is no way of + // resetting it, but this way we at least support the write-at-position and transferFrom + // methods being called once. + this.valid = false; + return res; + } + + @Override + public MappedByteBuffer map(MapMode mode, long position, long size) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FileLock lock(long position, long size, boolean shared) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FileLock tryLock(long position, long size, boolean shared) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + protected void implCloseChannel() throws IOException { + writeChannel.close(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAcl.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAcl.java new file mode 100644 index 000000000000..ebbe1583be01 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAcl.java @@ -0,0 +1,30 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; +import com.google.cloud.storage.Acl; + +@AutoValue +abstract class OptionAcl implements CloudStorageOption.OpenCopy { + + static OptionAcl create(Acl acl) { + return new AutoValue_OptionAcl(acl); + } + + abstract Acl acl(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAllowTrailingSlash.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAllowTrailingSlash.java new file mode 100644 index 000000000000..ccb7039dcfea --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAllowTrailingSlash.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +class OptionAllowTrailingSlash implements CloudStorageOption.Open { + + static OptionAllowTrailingSlash instance; + + private OptionAllowTrailingSlash() {} + ; + + public static synchronized OptionAllowTrailingSlash getInstance() { + if (null == instance) { + instance = new OptionAllowTrailingSlash(); + } + return instance; + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionBlockSize.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionBlockSize.java new file mode 100644 index 000000000000..c2796fd241f3 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionBlockSize.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionBlockSize implements CloudStorageOption.OpenCopy { + + static OptionBlockSize create(int size) { + return new AutoValue_OptionBlockSize(size); + } + + abstract int size(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionCacheControl.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionCacheControl.java new file mode 100644 index 000000000000..d0654f90b8c5 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionCacheControl.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionCacheControl implements CloudStorageOption.OpenCopy { + + static OptionCacheControl create(String cacheControl) { + return new AutoValue_OptionCacheControl(cacheControl); + } + + abstract String cacheControl(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentDisposition.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentDisposition.java new file mode 100644 index 000000000000..a6e09c3a7dbe --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentDisposition.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionContentDisposition implements CloudStorageOption.OpenCopy { + + static OptionContentDisposition create(String contentDisposition) { + return new AutoValue_OptionContentDisposition(contentDisposition); + } + + abstract String contentDisposition(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentEncoding.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentEncoding.java new file mode 100644 index 000000000000..9dc83aaee7a9 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentEncoding.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionContentEncoding implements CloudStorageOption.OpenCopy { + + static OptionContentEncoding create(String contentEncoding) { + return new AutoValue_OptionContentEncoding(contentEncoding); + } + + abstract String contentEncoding(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMaxChannelReopens.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMaxChannelReopens.java new file mode 100644 index 000000000000..7fedd0cfc3be --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMaxChannelReopens.java @@ -0,0 +1,30 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionMaxChannelReopens implements CloudStorageOption.OpenCopy { + + /** Re-open the channel if it's closed unexpectedly while we're reading it. */ + static OptionMaxChannelReopens create(int retryCount) { + return new AutoValue_OptionMaxChannelReopens(retryCount); + } + + abstract int maxChannelReopens(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMimeType.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMimeType.java new file mode 100644 index 000000000000..40b59684c388 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMimeType.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionMimeType implements CloudStorageOption.OpenCopy { + + static OptionMimeType create(String mimeType) { + return new AutoValue_OptionMimeType(mimeType); + } + + abstract String mimeType(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionUserMetadata.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionUserMetadata.java new file mode 100644 index 000000000000..7cf31ffe2366 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionUserMetadata.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionUserMetadata implements CloudStorageOption.OpenCopy { + + static OptionUserMetadata create(String key, String value) { + return new AutoValue_OptionUserMetadata(key, value); + } + + abstract String key(); + + abstract String value(); +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/SeekableByteChannelPrefetcher.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/SeekableByteChannelPrefetcher.java new file mode 100644 index 000000000000..41632f8c8f9d --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/SeekableByteChannelPrefetcher.java @@ -0,0 +1,566 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.UnknownFormatConversionException; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +/** + * SeekableByteChannelPrefetcher wraps an existing SeekableByteChannel to add prefetching. The + * prefetching is done on a different thread, so you can write simple code that repeatedly calls + * read() to get data, processes it, and then calls read again -- and yet this simple code overlaps + * computation and communication for you. (Of course this is only worthwhile if the underlying + * SeekableByteChannel doesn't already implement prefetching). + */ +public final class SeekableByteChannelPrefetcher implements SeekableByteChannel { + + // Only one thread at a time should use chan. + // To ensure this is the case, only the prefetching thread uses it. + private final SeekableByteChannel chan; + private final int bufSize; + private final ExecutorService exec; + private final long size; + private final List full = new ArrayList<>(); + private WorkUnit fetching; + // total number of buffers + private static final int BUF_COUNT = 2; + // where we pretend to be, wrt returning bytes from read() + private long position; + private boolean open; + private Stopwatch betweenCallsToRead = Stopwatch.createUnstarted(); + private static int prefetcherCount; + + // statistics, for profiling. + // time spent blocking the user because we're waiting on the network + private long msWaitingForData; + // time spent blocking the user because we're copying bytes + private long msCopyingData; + // total number of bytes returned by read (if the user asks for the same bytes multiple times, + // they count) + private long bytesReturned; + // total number of bytes read over the network (whether returned to the user or not) + private long bytesRead; + // time spend in between calls to Read, ie. presumably while the user is processing the data we + // returned. + private long msBetweenCallsToRead; + // number of times we had the user's data already ready, didn't have to grab it from the net. + private long nbHit; + // number of times we had already started to prefetch the user's data (but it hadn't arrived yet). + private long nbNearHit; + // number of times we don't have what the user's asking for, we have to wait for a prefetch to + // finish, + // and the prefetch didn't return what the user wanted (either they are going backward, or jumping + // forward) + private long nbMiss; + // number of times the user asks for data with a lower index than what we already have + // (so they're not following the expected pattern of increasing indexes) + private long nbGoingBack; + // number of times the user asks for data past the end of the file + private long nbReadsPastEnd; + // timing statistics have an overhead, so only turn them on when debugging performance + // issues. + private static final boolean trackTime = false; + + public static class Statistics { + // statistics, for profiling. + // time spent blocking the user because we're waiting on the network + public final long msWaitingForData; + // time spent blocking the user because we're copying bytes + public final long msCopyingData; + // total number of bytes returned by read (if the user asks for the same bytes multiple times, + // they count) + public final long bytesReturned; + // total number of bytes read over the network (whether returned to the user or not) + public final long bytesRead; + // time spend in between calls to Read, ie. presumably while the user is processing the data we + // returned. + public final long msBetweenCallsToRead; + // number of times we had the user's data already ready, didn't have to grab it from the net. + public final long nbHit; + // number of times we had already started to prefetch the user's data (but it hadn't arrived + // yet). + public final long nbNearHit; + // number of times we don't have what the user's asking for, we have to wait for a prefetch to + // finish, + // and the prefetch didn't return what the user wanted (either they are going backward, or + // jumping forward) + public final long nbMiss; + // number of times the user asks for data with a lower index than what we already have + // (so they're not following the expected pattern of increasing indexes) + public final long nbGoingBack; + // number of times the user asks for data past the end of the file + public final long nbReadsPastEnd; + + private Statistics( + long msWaitingForData, + long msCopyingData, + long bytesReturned, + long bytesRead, + long msBetweenCallsToRead, + long nbHit, + long nbNearHit, + long nbMiss, + long nbGoingBack, + long nbReadsPastEnd) { + this.msWaitingForData = msWaitingForData; + this.msCopyingData = msCopyingData; + this.bytesReturned = bytesReturned; + this.bytesRead = bytesRead; + this.msBetweenCallsToRead = msBetweenCallsToRead; + this.nbHit = nbHit; + this.nbNearHit = nbNearHit; + this.nbMiss = nbMiss; + this.nbGoingBack = nbGoingBack; + this.nbReadsPastEnd = nbReadsPastEnd; + } + + public String toString() { + try { + double returnedPct = (bytesRead > 0 ? 100.0 * bytesReturned / bytesRead : 100.0); + return String.format( + "Bytes read: %12d\n returned: %12d ( %3.2f %% )", + bytesRead, bytesReturned, returnedPct) + + String.format("\nReads past the end: %3d", nbReadsPastEnd) + + String.format("\nReads forcing re-fetching of an earlier block: %3d", nbGoingBack) + // A near-hit is when we're already fetching the data the user is asking for, + // but we're not done loading it in. + + String.format( + "\nCache\n hits: %12d\n near-hits: %12d\n misses: %12d", + nbHit, nbNearHit, nbMiss); + } catch (UnknownFormatConversionException x) { + // let's not crash the whole program, instead just return no info + return "(error while formatting statistics)"; + } + } + } + + /** + * Wraps the provided SeekableByteChannel within a SeekableByteChannelPrefetcher, using the + * provided buffer size + * + * @param bufferSizeMB buffer size in MB + * @param channel channel to wrap in the prefetcher + * @return wrapped channel + */ + public static SeekableByteChannel addPrefetcher(int bufferSizeMB, SeekableByteChannel channel) + throws IOException { + return new SeekableByteChannelPrefetcher(channel, bufferSizeMB * 1024 * 1024); + } + + /** + * WorkUnit holds a buffer and the instructions for what to put in it. + * + *

Use it like this: + * + *

    + *
  1. call() + *
  2. the data is now in buf, you can access it directly + *
  3. if need more, call resetForIndex(...) and go back to the top. + *
  4. else, call close() + *
+ */ + private static class WorkUnit implements Callable, Closeable { + public final ByteBuffer buf; + public long blockIndex; + private final SeekableByteChannel chan; + private final int blockSize; + private Future futureBuf; + + public WorkUnit(SeekableByteChannel chan, int blockSize, long blockIndex) { + this.chan = chan; + this.buf = ByteBuffer.allocate(blockSize); + this.futureBuf = null; + this.blockSize = blockSize; + this.blockIndex = blockIndex; + } + + @Override + public ByteBuffer call() throws IOException { + long pos = ((long) blockSize) * blockIndex; + if (pos > chan.size()) { + return null; + } + if (pos < 0) { + // This should never happen, if the code's correct. + throw new IllegalArgumentException( + "blockIndex " + + blockIndex + + " has position " + + pos + + ": negative position is not valid."); + } + chan.position(pos); + // read until buffer is full, or EOF + while (chan.read(buf) >= 0 && buf.hasRemaining()) {} + return buf; + } + + public ByteBuffer getBuf() throws ExecutionException, InterruptedException { + return futureBuf.get(); + } + + public WorkUnit resetForIndex(long blockIndex) { + this.blockIndex = blockIndex; + buf.clear(); + futureBuf = null; + return this; + } + + @Override + public void close() throws IOException { + chan.close(); + } + } + + /** + * Wraps the provided SeekableByteChannel within a SeekableByteChannelPrefetcher, using the + * provided buffer size. + * + * @param bufSize buffer size in bytes + * @param chan channel to wrap in the prefetcher + */ + private SeekableByteChannelPrefetcher(SeekableByteChannel chan, int bufSize) throws IOException { + Preconditions.checkArgument( + !(chan instanceof SeekableByteChannelPrefetcher), + "Cannot wrap a prefetcher with a prefetcher."); + + if (!chan.isOpen()) { + throw new IllegalArgumentException("channel must be open"); + } + this.chan = chan; + if (bufSize <= 0) { + throw new IllegalArgumentException("bufSize must be positive"); + } + this.size = chan.size(); + if (bufSize > this.size) { + this.bufSize = (int) this.size; + } else { + this.bufSize = bufSize; + } + this.open = true; + int prefetcherIndex = prefetcherCount++; + // Make sure the prefetching thread's name indicate what it is and + // which prefetcher it belongs to (for debugging purposes only, naturally). + String nameFormat = "nio-prefetcher-" + prefetcherIndex + "-thread-%d"; + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat(nameFormat).setDaemon(true).build(); + // Single thread to ensure no concurrent access to chan. + exec = Executors.newFixedThreadPool(1, threadFactory); + } + + public Statistics getStatistics() { + return new Statistics( + msWaitingForData, + msCopyingData, + bytesReturned, + bytesRead, + msBetweenCallsToRead, + nbHit, + nbNearHit, + nbMiss, + nbGoingBack, + nbReadsPastEnd); + } + + // if we don't already have that block and the fetching thread is idle, + // make sure it now goes looking for that block index. + private void ensureFetching(long blockIndex) { + if (fetching != null) { + if (fetching.futureBuf.isDone()) { + full.add(fetching); + fetching = null; + } else { + return; + } + } + for (WorkUnit w : full) { + if (w.blockIndex == blockIndex) { + return; + } + } + if (full.size() < BUF_COUNT) { + fetching = new WorkUnit(chan, bufSize, blockIndex); + } else { + // reuse the oldest full buffer + fetching = full.remove(0); + fetching.resetForIndex(blockIndex); + } + bytesRead += bufSize; + fetching.futureBuf = exec.submit(fetching); + } + + // Return a buffer at this position, blocking if necessary. + // Start a background read of the buffer after this one (if there isn't one already). + public ByteBuffer fetch(long position) throws InterruptedException, ExecutionException { + long blockIndex = position / bufSize; + boolean goingBack = false; + for (WorkUnit w : full) { + if (w.blockIndex == blockIndex) { + ensureFetching(blockIndex + 1); + nbHit++; + return w.buf; + } else if (w.blockIndex > blockIndex) { + goingBack = true; + } + } + if (goingBack) { + // user is asking for a block with a lower index than we've already fetched - + // in other words they are not following the expected pattern of increasing indexes. + nbGoingBack++; + } + if (null == fetching) { + ensureFetching(blockIndex); + } + WorkUnit candidate = fetching; + // block until we have the buffer + ByteBuffer buf = candidate.getBuf(); + full.add(candidate); + fetching = null; + if (candidate.blockIndex == blockIndex) { + // this is who we were waiting for + nbNearHit++; + ensureFetching(blockIndex + 1); + return buf; + } else { + // wrong block. Let's fetch the right one now. + nbMiss++; + ensureFetching(blockIndex); + candidate = fetching; + if (candidate != null) { + buf = candidate.getBuf(); + full.add(candidate); + } + fetching = null; + ensureFetching(blockIndex + 1); + return buf; + } + } + + /** + * Reads a sequence of bytes from this channel into the given buffer. + * + *

Bytes are read starting at this channel's current position, and then the position is updated + * with the number of bytes actually read. Otherwise this method behaves exactly as specified in + * the {@link java.nio.channels.ReadableByteChannel} interface. + * + * @param dst buffer to write into + */ + @Override + public synchronized int read(ByteBuffer dst) throws IOException { + if (!open) { + throw new ClosedChannelException(); + } + try { + if (trackTime) { + msBetweenCallsToRead += betweenCallsToRead.elapsed(TimeUnit.MILLISECONDS); + } + ByteBuffer src; + try { + Stopwatch waitingForData; + if (trackTime) { + waitingForData = Stopwatch.createStarted(); + } + src = fetch(position); + if (trackTime) { + msWaitingForData += waitingForData.elapsed(TimeUnit.MILLISECONDS); + } + } catch (InterruptedException e) { + // Restore interrupted status + Thread.currentThread().interrupt(); + return 0; + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + if (null == src) { + // the caller is asking for a block past EOF + nbReadsPastEnd++; + return -1; // EOF + } + Stopwatch copyingData; + if (trackTime) { + copyingData = Stopwatch.createStarted(); + } + // src.position is how far we've written into the array + long blockIndex = position / bufSize; + int offset = (int) (position - (blockIndex * bufSize)); + // src |==============---------------------| + // :<---src.pos-->------src.limit----->: + // |---:--position-> + // :<--offset--> + // ^ blockIndex*bufSize + int availableToCopy = src.position() - offset; + if (availableToCopy < 0) { + // the caller is asking to read past the end of the file + nbReadsPastEnd++; + return -1; // EOF + } + int bytesToCopy = dst.remaining(); + byte[] array = src.array(); + if (availableToCopy < bytesToCopy) { + bytesToCopy = availableToCopy; + } + dst.put(array, offset, bytesToCopy); + position += bytesToCopy; + if (trackTime) { + msCopyingData += copyingData.elapsed(TimeUnit.MILLISECONDS); + } + bytesReturned += bytesToCopy; + if (availableToCopy == 0) { + // EOF + return -1; + } + return bytesToCopy; + } finally { + if (trackTime) { + betweenCallsToRead.reset(); + betweenCallsToRead.start(); + } + } + } + + /** Writing isn't supported. */ + @Override + public int write(ByteBuffer src) throws IOException { + throw new NonWritableChannelException(); + } + + /** + * Returns this channel's position. + * + * @return This channel's position, a non-negative integer counting the number of bytes from the + * beginning of the entity to the current position + * @throws ClosedChannelException If this channel is closed + * @throws IOException If some other I/O error occurs + */ + @Override + public long position() throws IOException { + if (!open) throw new ClosedChannelException(); + return position; + } + + /** + * Sets this channel's position. + * + *

+ * + *

Setting the position to a value that is greater than the current size is legal but does not + * change the size of the entity. A later attempt to read bytes at such a position will + * immediately return an end-of-file indication. A later attempt to write bytes at such a position + * will cause the entity to grow to accommodate the new bytes; the values of any bytes between the + * previous end-of-file and the newly-written bytes are unspecified. + * + *

Setting the channel's position is not recommended when connected to an entity, typically a + * file, that is opened with the {@link java.nio.file.StandardOpenOption#APPEND APPEND} option. + * When opened for append, the position is first advanced to the end before writing. + * + * @param newPosition The new position, a non-negative integer counting the number of bytes from + * the beginning of the entity + * @return This channel + * @throws ClosedChannelException If this channel is closed + * @throws IllegalArgumentException If the new position is negative + * @throws IOException If some other I/O error occurs + */ + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + if (!open) throw new ClosedChannelException(); + position = newPosition; + return this; + } + + /** + * Returns the current size of entity to which this channel is connected. + * + * @return The current size, measured in bytes + * @throws ClosedChannelException If this channel is closed + * @throws IOException If some other I/O error occurs + */ + @Override + public long size() throws IOException { + if (!open) throw new ClosedChannelException(); + return size; + } + + /** Not supported. */ + @Override + public SeekableByteChannel truncate(long size) throws IOException { + throw new NonWritableChannelException(); + } + + /** + * Tells whether or not this channel is open. + * + * @return {@code true} if, and only if, this channel is open + */ + @Override + public boolean isOpen() { + return open; + } + + /** + * Closes this channel. + * + *

+ * + *

After a channel is closed, any further attempt to invoke I/O operations upon it will cause a + * {@link ClosedChannelException} to be thrown. + * + *

+ * + *

If this channel is already closed then invoking this method has no effect. + * + *

+ * + *

This method may be invoked at any time. If some other thread has already invoked it, + * however, then another invocation will block until the first invocation is complete, after which + * it will return without effect. + * + * @throws IOException If an I/O error occurs + */ + @Override + public void close() throws IOException { + if (open) { + // stop accepting work, interrupt worker thread. + exec.shutdownNow(); + try { + // give worker thread a bit of time to process the interruption. + exec.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // Restore interrupted status + Thread.currentThread().interrupt(); + } + chan.close(); + open = false; + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/StorageOptionsUtil.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/StorageOptionsUtil.java new file mode 100644 index 000000000000..27c28bf33be0 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/StorageOptionsUtil.java @@ -0,0 +1,119 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.api.gax.rpc.HeaderProvider; +import com.google.cloud.storage.StorageOptions; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +final class StorageOptionsUtil { + static final String USER_AGENT_ENTRY_NAME = "gcloud-java-nio"; + static final String USER_AGENT_ENTRY_VERSION = getVersion(); + private static final String USER_AGENT_ENTRY = + String.format("%s/%s", USER_AGENT_ENTRY_NAME, USER_AGENT_ENTRY_VERSION); + private static final FixedHeaderProvider DEFAULT_HEADER_PROVIDER = + FixedHeaderProvider.create("user-agent", USER_AGENT_ENTRY); + + private static final StorageOptions DEFAULT_STORAGE_OPTIONS_INSTANCE = + StorageOptions.newBuilder().setHeaderProvider(DEFAULT_HEADER_PROVIDER).build(); + private static final FixedHeaderProvider EMTPY_HEADER_PROVIDER = + FixedHeaderProvider.create(Collections.emptyMap()); + + private StorageOptionsUtil() {} + + static StorageOptions getDefaultInstance() { + return DEFAULT_STORAGE_OPTIONS_INSTANCE; + } + + static StorageOptions mergeOptionsWithUserAgent(StorageOptions providedStorageOptions) { + if (providedStorageOptions == DEFAULT_STORAGE_OPTIONS_INSTANCE) { + return providedStorageOptions; + } + + String userAgent = providedStorageOptions.getUserAgent(); + if (userAgent == null) { + return nullSafeSet(providedStorageOptions, DEFAULT_HEADER_PROVIDER); + } else { + if (!userAgent.contains(USER_AGENT_ENTRY_NAME)) { + HeaderProvider providedHeaderProvider = getHeaderProvider(providedStorageOptions); + Map newHeaders = new HashMap<>(providedHeaderProvider.getHeaders()); + newHeaders.put("user-agent", String.format("%s %s", userAgent, USER_AGENT_ENTRY)); + FixedHeaderProvider headerProvider = + FixedHeaderProvider.create(ImmutableMap.copyOf(newHeaders)); + return nullSafeSet(providedStorageOptions, headerProvider); + } else { + return providedStorageOptions; + } + } + } + + /** + * Due to some complex interactions between init and mocking, it's possible that the builder + * instance returned from {@link StorageOptions#toBuilder()} can be null. This utility method will + * attempt to create the builder and set the new header provider. If however the builder instance + * is null, the orignal options will be returned without setting the header provider. + * + *

Since this method is only every called by us trying to add our user-agent entry to the + * headers this makes our attempt effectively a no-op, which is much better than failing customer + * code. + */ + private static StorageOptions nullSafeSet( + StorageOptions storageOptions, HeaderProvider headerProvider) { + StorageOptions.Builder builder = storageOptions.toBuilder(); + if (builder == null) { + return storageOptions; + } else { + return builder.setHeaderProvider(headerProvider).build(); + } + } + + /** Resolve the version of google-cloud-nio for inclusion in request meta-data */ + private static String getVersion() { + // attempt to read the library's version from a properties file generated during the build + // this value should be read and cached for later use + String version = ""; + try (InputStream inputStream = + CloudStorageFileSystemProvider.class.getResourceAsStream( + "/META-INF/maven/com.google.cloud/google-cloud-nio/pom.properties")) { + if (inputStream != null) { + final Properties properties = new Properties(); + properties.load(inputStream); + version = properties.getProperty("version"); + } + } catch (IOException e) { + // ignore + } + return version; + } + + /** + * {@link com.google.cloud.ServiceOptions} does not specify a getter for the headerProvider, so + * instead merge with an empty provider. + */ + @VisibleForTesting + static HeaderProvider getHeaderProvider(StorageOptions options) { + return options.getMergedHeaderProvider(EMTPY_HEADER_PROVIDER); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/UnixPath.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/UnixPath.java new file mode 100644 index 000000000000..35ec2890ce86 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/UnixPath.java @@ -0,0 +1,509 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import com.google.common.collect.PeekingIterator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Unix file system path. + * + *

This class is helpful for writing {@link java.nio.file.Path Path} implementations. + * + *

This implementation behaves almost identically to {@code sun.nio.fs.UnixPath}. The only + * difference is that some methods (like {@link #relativize(UnixPath)} go to greater lengths to + * preserve trailing backslashes, in order to ensure the path will continue to be recognized as a + * directory. + * + *

Note: This code might not play nice with Supplementary + * Characters as Surrogates. + */ +@Immutable +final class UnixPath implements CharSequence { + + public static final char DOT = '.'; + public static final char SEPARATOR = '/'; + public static final String ROOT = "" + SEPARATOR; + public static final String CURRENT_DIR = "" + DOT; + public static final String PARENT_DIR = "" + DOT + DOT; + public static final UnixPath EMPTY_PATH = new UnixPath(false, ""); + public static final UnixPath ROOT_PATH = new UnixPath(false, ROOT); + + private static final Splitter SPLITTER = Splitter.on(SEPARATOR).omitEmptyStrings(); + private static final Splitter SPLITTER_PERMIT_EMPTY_COMPONENTS = Splitter.on(SEPARATOR); + private static final Joiner JOINER = Joiner.on(SEPARATOR); + private static final Ordering> ORDERING = Ordering.natural().lexicographical(); + + private final String path; + private List lazyStringParts; + private final boolean permitEmptyComponents; + + private UnixPath(boolean permitEmptyComponents, String path) { + this.path = checkNotNull(path); + this.permitEmptyComponents = permitEmptyComponents; + } + + /** Returns new path of {@code first}. */ + public static UnixPath getPath(boolean permitEmptyComponents, String path) { + if (path.isEmpty()) { + return EMPTY_PATH; + } else if (isRootInternal(path)) { + return ROOT_PATH; + } else { + return new UnixPath(permitEmptyComponents, path); + } + } + + /** + * Returns new path of {@code first} with {@code more} components resolved against it. + * + * @see #resolve(UnixPath) + * @see java.nio.file.FileSystem#getPath(String, String...) + */ + public static UnixPath getPath(boolean permitEmptyComponents, String first, String... more) { + if (more.length == 0) { + return getPath(permitEmptyComponents, first); + } + StringBuilder builder = new StringBuilder(first); + for (int i = 0; i < more.length; i++) { + String part = more[i]; + if (part.isEmpty()) { + continue; + } else if (isAbsoluteInternal(part)) { + if (i == more.length - 1) { + return new UnixPath(permitEmptyComponents, part); + } else { + builder.replace(0, builder.length(), part); + } + } else if (hasTrailingSeparatorInternal(builder)) { + builder.append(part); + } else { + builder.append(SEPARATOR); + builder.append(part); + } + } + return new UnixPath(permitEmptyComponents, builder.toString()); + } + + /** Returns {@code true} consists only of {@code separator}. */ + public boolean isRoot() { + return isRootInternal(path); + } + + private static boolean isRootInternal(String path) { + return path.length() == 1 && path.charAt(0) == SEPARATOR; + } + + /** Returns {@code true} if path starts with {@code separator}. */ + public boolean isAbsolute() { + return isAbsoluteInternal(path); + } + + private static boolean isAbsoluteInternal(String path) { + return !path.isEmpty() && path.charAt(0) == SEPARATOR; + } + + /** Returns {@code true} if path ends with {@code separator}. */ + public boolean hasTrailingSeparator() { + return hasTrailingSeparatorInternal(path); + } + + private static boolean hasTrailingSeparatorInternal(CharSequence path) { + return path.length() != 0 && path.charAt(path.length() - 1) == SEPARATOR; + } + + /** Returns {@code true} if path ends with a trailing slash, or would after normalization. */ + public boolean seemsLikeADirectory() { + int length = path.length(); + return path.isEmpty() + || path.charAt(length - 1) == SEPARATOR + || path.endsWith(".") && (length == 1 || path.charAt(length - 2) == SEPARATOR) + || path.endsWith("..") && (length == 2 || path.charAt(length - 3) == SEPARATOR); + } + + /** + * Returns last component in {@code path}. + * + * @see java.nio.file.Path#getFileName() + */ + @Nullable + public UnixPath getFileName() { + if (path.isEmpty()) { + return EMPTY_PATH; + } else if (isRoot()) { + return null; + } else { + List parts = getParts(); + String last = parts.get(parts.size() - 1); + return parts.size() == 1 && path.equals(last) + ? this + : new UnixPath(permitEmptyComponents, last); + } + } + + /** + * Returns parent directory (including trailing separator) or {@code null} if no parent remains. + * + * @see java.nio.file.Path#getParent() + */ + @Nullable + public UnixPath getParent() { + if (path.isEmpty() || isRoot()) { + return null; + } + int index = + hasTrailingSeparator() + ? path.lastIndexOf(SEPARATOR, path.length() - 2) + : path.lastIndexOf(SEPARATOR); + if (index == -1) { + return isAbsolute() ? ROOT_PATH : null; + } else { + return new UnixPath(permitEmptyComponents, path.substring(0, index + 1)); + } + } + + /** + * Returns root component if an absolute path, otherwise {@code null}. + * + * @see java.nio.file.Path#getRoot() + */ + @Nullable + public UnixPath getRoot() { + return isAbsolute() ? ROOT_PATH : null; + } + + /** + * Returns specified range of sub-components in path joined together. + * + * @see java.nio.file.Path#subpath(int, int) + */ + public UnixPath subpath(int beginIndex, int endIndex) { + if (path.isEmpty() && beginIndex == 0 && endIndex == 1) { + return this; + } + checkArgument(beginIndex >= 0 && endIndex > beginIndex); + List subList; + try { + subList = getParts().subList(beginIndex, endIndex); + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(); + } + return new UnixPath(permitEmptyComponents, JOINER.join(subList)); + } + + /** + * Returns number of components in {@code path}. + * + * @see java.nio.file.Path#getNameCount() + */ + public int getNameCount() { + if (path.isEmpty()) { + return 1; + } else if (isRoot()) { + return 0; + } else { + return getParts().size(); + } + } + + /** + * Returns component in {@code path} at {@code index}. + * + * @see java.nio.file.Path#getName(int) + */ + public UnixPath getName(int index) { + if (path.isEmpty()) { + return this; + } + try { + return new UnixPath(permitEmptyComponents, getParts().get(index)); + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(); + } + } + + /** + * Returns path without extra separators or {@code .} and {@code ..}, preserving trailing slash. + * + * @see java.nio.file.Path#normalize() + */ + public UnixPath normalize() { + List parts = new ArrayList<>(); + boolean mutated = false; + int resultLength = 0; + int mark = 0; + int index; + do { + index = path.indexOf(SEPARATOR, mark); + String part = path.substring(mark, index == -1 ? path.length() : index + 1); + switch (part) { + case CURRENT_DIR: + case CURRENT_DIR + SEPARATOR: + mutated = true; + break; + case PARENT_DIR: + case PARENT_DIR + SEPARATOR: + mutated = true; + if (!parts.isEmpty()) { + resultLength -= parts.remove(parts.size() - 1).length(); + } + break; + default: + if (index != mark || index == 0) { + parts.add(part); + resultLength = part.length(); + } else { + mutated = true; + } + } + mark = index + 1; + } while (index != -1); + if (!mutated) { + return this; + } + StringBuilder result = new StringBuilder(resultLength); + for (String part : parts) { + result.append(part); + } + return new UnixPath(permitEmptyComponents, result.toString()); + } + + /** + * Returns {@code other} appended to {@code path}. + * + * @see java.nio.file.Path#resolve(java.nio.file.Path) + */ + public UnixPath resolve(UnixPath other) { + if (other.path.isEmpty()) { + return this; + } else if (other.isAbsolute()) { + return other; + } else if (hasTrailingSeparator()) { + return new UnixPath(permitEmptyComponents, path + other.path); + } else { + return new UnixPath(permitEmptyComponents, path + SEPARATOR + other.path); + } + } + + /** + * Returns {@code other} resolved against parent of {@code path}. + * + * @see java.nio.file.Path#resolveSibling(java.nio.file.Path) + */ + public UnixPath resolveSibling(UnixPath other) { + checkNotNull(other); + UnixPath parent = getParent(); + return parent == null ? other : parent.resolve(other); + } + + /** + * Returns {@code other} made relative to {@code path}. + * + * @see java.nio.file.Path#relativize(java.nio.file.Path) + */ + public UnixPath relativize(UnixPath other) { + checkArgument(isAbsolute() == other.isAbsolute(), "'other' is different type of Path"); + if (path.isEmpty()) { + return other; + } + PeekingIterator left = Iterators.peekingIterator(split()); + PeekingIterator right = Iterators.peekingIterator(other.split()); + while (left.hasNext() && right.hasNext()) { + if (!left.peek().equals(right.peek())) { + break; + } + left.next(); + right.next(); + } + StringBuilder result = new StringBuilder(path.length() + other.path.length()); + while (left.hasNext()) { + result.append(PARENT_DIR); + result.append(SEPARATOR); + left.next(); + } + while (right.hasNext()) { + result.append(right.next()); + result.append(SEPARATOR); + } + if (result.length() > 0 && !other.hasTrailingSeparator()) { + result.deleteCharAt(result.length() - 1); + } + return new UnixPath(permitEmptyComponents, result.toString()); + } + + /** + * Returns {@code true} if {@code path} starts with {@code other}. + * + * @see java.nio.file.Path#startsWith(java.nio.file.Path) + */ + public boolean startsWith(UnixPath other) { + UnixPath me = removeTrailingSeparator(); + other = other.removeTrailingSeparator(); + if (other.path.length() > me.path.length()) { + return false; + } else if (me.isAbsolute() != other.isAbsolute()) { + return false; + } else if (!me.path.isEmpty() && other.path.isEmpty()) { + return false; + } + return startsWith(split(), other.split()); + } + + private static boolean startsWith(Iterator lefts, Iterator rights) { + while (rights.hasNext()) { + if (!lefts.hasNext() || !rights.next().equals(lefts.next())) { + return false; + } + } + return true; + } + + /** + * Returns {@code true} if {@code path} ends with {@code other}. + * + * @see java.nio.file.Path#endsWith(java.nio.file.Path) + */ + public boolean endsWith(UnixPath other) { + UnixPath me = removeTrailingSeparator(); + other = other.removeTrailingSeparator(); + if (other.path.length() > me.path.length()) { + return false; + } else if (!me.path.isEmpty() && other.path.isEmpty()) { + return false; + } else if (other.isAbsolute()) { + return me.isAbsolute() && me.path.equals(other.path); + } + return startsWith(me.splitReverse(), other.splitReverse()); + } + + /** + * Compares two paths lexicographically for ordering. + * + * @see java.nio.file.Path#compareTo(java.nio.file.Path) + */ + public int compareTo(UnixPath other) { + return ORDERING.compare(getParts(), other.getParts()); + } + + /** Converts relative path to an absolute path. */ + public UnixPath toAbsolutePath(UnixPath currentWorkingDirectory) { + checkArgument(currentWorkingDirectory.isAbsolute()); + return isAbsolute() ? this : currentWorkingDirectory.resolve(this); + } + + /** Returns {@code toAbsolutePath(ROOT_PATH)}. */ + public UnixPath toAbsolutePath() { + return toAbsolutePath(ROOT_PATH); + } + + /** Removes beginning separator from path, if an absolute path. */ + public UnixPath removeBeginningSeparator() { + return isAbsolute() ? new UnixPath(permitEmptyComponents, path.substring(1)) : this; + } + + /** Adds trailing separator to path, if it isn't present. */ + public UnixPath addTrailingSeparator() { + return hasTrailingSeparator() ? this : new UnixPath(permitEmptyComponents, path + SEPARATOR); + } + + /** Removes trailing separator from path, unless it's root. */ + public UnixPath removeTrailingSeparator() { + if (!isRoot() && hasTrailingSeparator()) { + return new UnixPath(permitEmptyComponents, path.substring(0, path.length() - 1)); + } else { + return this; + } + } + + /** Splits path into components, excluding separators and empty strings. */ + public Iterator split() { + return getParts().iterator(); + } + + /** Splits path into components in reverse, excluding separators and empty strings. */ + public Iterator splitReverse() { + return Lists.reverse(getParts()).iterator(); + } + + @Override + public boolean equals(Object other) { + return this == other || other instanceof UnixPath && path.equals(((UnixPath) other).path); + } + + @Override + public int hashCode() { + return path.hashCode(); + } + + /** Returns path as a string. */ + @Override + public String toString() { + return path; + } + + @Override + public int length() { + return path.length(); + } + + @Override + public char charAt(int index) { + return path.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return path.subSequence(start, end); + } + + /** Returns {@code true} if this path is an empty string. */ + public boolean isEmpty() { + return path.isEmpty(); + } + + /** Returns list of path components, excluding slashes. */ + private List getParts() { + List result = lazyStringParts; + return result != null + ? result + : (lazyStringParts = + path.isEmpty() || isRoot() ? Collections.emptyList() : createParts()); + } + + private List createParts() { + if (permitEmptyComponents) { + return SPLITTER_PERMIT_EMPTY_COMPONENTS.splitToList( + path.charAt(0) == SEPARATOR ? path.substring(1) : path); + } else { + return SPLITTER.splitToList(path); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/package-info.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/package-info.java new file mode 100644 index 000000000000..0c7f8f65ce08 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/package-info.java @@ -0,0 +1,103 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Java 7 nio FileSystem client library for Google Cloud Storage. + * + *

This client library allows you to easily interact with Google Cloud Storage, using Java's + * standard file system API, introduced in Java 7. + * + *

How It Works

+ * + * The simplest way to get started is with {@code Paths} and {@code Files}: + * + *
{@code
+ * Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ * List lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+ * }
+ * + *

For the complete source code see + * ReadAllLines.java. + * + *

If you want to configure the bucket per-environment, it might make more sense to use the + * {@code FileSystem} API: + * + *

{@code
+ * FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"));
+ * byte[] data = "hello world".getBytes(StandardCharsets.UTF_8);
+ * Path path = fs.getPath("/object");
+ * Files.write(path, data);
+ * List lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+ * }
+ * + *

For the complete source code see + * GetFileSystem.java. + * + *

You can also use {@code InputStream} and {@code OutputStream} for streaming: + * + *

+ *   Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ *   try (InputStream input = Files.newInputStream(path)) {
+ *     // use input stream
+ *   }
+ * 
+ * + *

For the complete source code see + * CreateInputStream.java. + * + *

You can set various attributes using {@link + * com.google.cloud.storage.contrib.nio.CloudStorageOptions CloudStorageOptions} static helpers: + * + *

+ *   Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ *   Files.write(path, csvLines, StandardCharsets.UTF_8,
+ *       withMimeType("text/csv; charset=UTF-8"),
+ *       withoutCaching());
+ * 
+ * + *

For the complete source code see + * WriteFileWithAttributes.java. + * + *

NOTE: Cloud Storage uses a flat namespace and therefore doesn't support real + * directories. So this library supports what's known as "pseudo-directories". Any path that + * includes a trailing slash, will be considered a directory. It will always be assumed to exist, + * without performing any I/O. This allows you to do path manipulation in the same manner as you + * would with the normal UNIX file system implementation. You can disable this feature with {@link + * com.google.cloud.storage.contrib.nio.CloudStorageConfiguration#usePseudoDirectories()}. + * + *

Non-SPI Interface

+ * + *

If you don't want to rely on Java SPI, which requires a META-INF file in your jar generated by + * Google Auto, you can instantiate this file system directly as follows: + * + *

+ *   CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("bucket");
+ *   byte[] data = "hello world".getBytes(StandardCharsets.UTF_8);
+ *   Path path = fs.getPath("/object");
+ *   Files.write(path, data);
+ *   data = Files.readAllBytes(path);
+ * 
+ * + *

For the complete source code see + * CreateCloudStorageFileSystem.java. + */ +@javax.annotation.ParametersAreNonnullByDefault +package com.google.cloud.storage.contrib.nio; diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java new file mode 100644 index 000000000000..08f485385fb8 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java @@ -0,0 +1,765 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio.testing; + +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.client.util.DateTime; +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.ServiceAccount; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.Tuple; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.testing.StorageRpcTestBase; +import com.google.common.base.Preconditions; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.net.URLDecoder; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * A bare-bones in-memory implementation of StorageRpc, meant for testing. + * + *

This class is not thread-safe. It's also (currently) limited in the following ways: + * + *

    + *
  • Supported + *
      + *
    • object create + *
    • object get + *
    • object delete + *
    • list the contents of a bucket + *
    + *
  • Unsupported + *
      + *
    • bucket create + *
    • bucket get + *
    • bucket delete + *
    • list all buckets + *
    • generations + *
    • file attributes + *
    • patch + *
    • continueRewrite + *
    • createBatch + *
    • checksums, etags + *
    • IAM operations + *
    • BucketLock operations + *
    • HMAC key operations + *
    + *
+ */ +@NotThreadSafe +class FakeStorageRpc extends StorageRpcTestBase { + + private static final SimpleDateFormat RFC_3339_FORMATTER = + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); + + private static final int OK = 200; + private static final int PARTIAL_CONTENT = 206; + private static final int NOT_FOUND = 404; + private static final byte[] EMPTY_BYTES = new byte[0]; + + // fullname -> metadata + Map metadata = new ConcurrentHashMap<>(); + // fullname -> contents + Map contents = new ConcurrentHashMap<>(); + // fullname -> future contents that will be visible on close. + Map futureContents = new ConcurrentHashMap<>(); + + private final boolean throwIfOption; + + /** + * @param throwIfOption if true, we throw when given any option + */ + public FakeStorageRpc(boolean throwIfOption) { + this.throwIfOption = throwIfOption; + } + + // remove all files + void reset() { + metadata = new ConcurrentHashMap<>(); + contents = new ConcurrentHashMap<>(); + } + + @Override + public StorageObject create(StorageObject object, InputStream content, Map options) + throws StorageException { + potentiallyThrow(options); + String key = fullname(object); + object.setUpdated(now()); + metadata.put(key, object); + try { + contents.put(key, com.google.common.io.ByteStreams.toByteArray(content)); + } catch (IOException e) { + throw new StorageException(e); + } + // TODO: crc, etc + return object; + } + + @Override + public Tuple> list(String bucket, Map options) + throws StorageException { + String delimiter = null; + String preprefix = ""; + String pageToken = null; + long maxResults = Long.MAX_VALUE; + for (Map.Entry e : options.entrySet()) { + switch (e.getKey()) { + case PAGE_TOKEN: + pageToken = (String) e.getValue(); + break; + case PREFIX: + preprefix = (String) e.getValue(); + if (preprefix.startsWith("/")) { + preprefix = preprefix.substring(1); + } + break; + case DELIMITER: + delimiter = (String) e.getValue(); + break; + case FIELDS: + // ignore and return all the fields + break; + case MAX_RESULTS: + maxResults = (Long) e.getValue(); + break; + case USER_PROJECT: + // prevent unsupported operation + break; + default: + throw new UnsupportedOperationException("Unknown option: " + e.getKey()); + } + } + final String prefix = preprefix; + + List values = new ArrayList<>(); + Map folders = new ConcurrentHashMap<>(); + for (StorageObject so : metadata.values()) { + if (!so.getBucket().equals(bucket) || !so.getName().startsWith(prefix)) { + continue; + } + if (processedAsFolder(so, delimiter, prefix, folders)) { + continue; + } + so.setSize(size(so)); + values.add(so); + } + values.addAll(folders.values()); + + // truncate to max allowed length + if (values.size() > maxResults) { + List newValues = new ArrayList<>(); + for (int i = 0; i < maxResults; i++) { + newValues.add(values.get(i)); + } + values = newValues; + } + + // null cursor to indicate there is no more data (empty string would cause us to be called + // again). + // The type cast seems to be necessary to help Java's typesystem remember that collections are + // iterable. + return Tuple.of(pageToken, (Iterable) values); + } + + /** Returns the requested bucket or {@code null} if not found. */ + @Override + public Bucket get(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + /** Returns the requested storage object or {@code null} if not found. */ + @Override + public StorageObject get(StorageObject object, Map options) throws StorageException { + // we allow the "ID" option because we need to, but then we give a whole answer anyways + // because the caller won't mind the extra fields. + if (throwIfOption + && !options.isEmpty() + && options.size() > 1 + && options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) { + throw new UnsupportedOperationException(); + } + + String key = fullname(object); + if (metadata.containsKey(key)) { + StorageObject ret = metadata.get(key); + ret.setSize(size(ret)); + ret.setId(key); + + return ret; + } + return null; + } + + @Override + public Bucket patch(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public StorageObject patch(StorageObject storageObject, Map options) + throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public boolean delete(Bucket bucket, Map options) throws StorageException { + return false; + } + + @Override + public boolean delete(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + contents.remove(key); + return null != metadata.remove(key); + } + + @Override + public StorageObject compose( + Iterable sources, StorageObject target, Map targetOptions) + throws StorageException { + return null; + } + + @Override + public byte[] load(StorageObject storageObject, Map options) throws StorageException { + String key = fullname(storageObject); + if (!contents.containsKey(key)) { + throw new StorageException(NOT_FOUND, "File not found: " + key); + } + return contents.get(key); + } + + @Override + public Tuple read( + StorageObject from, Map options, long zposition, int zbytes) + throws StorageException { + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option op : options.keySet()) { + if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) { + generationMatch = (Long) options.get(op); + } else { + throw new UnsupportedOperationException("Unknown option: " + op); + } + } + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(NOT_FOUND, "File not found: " + key); + } + checkGeneration(key, generationMatch); + long position = zposition; + int bytes = zbytes; + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + if ((int) position + bytes > full.length) { + bytes = full.length - (int) position; + } + if (bytes <= 0) { + // special case: you're trying to read past the end + return Tuple.of("etag-goes-here", new byte[0]); + } + byte[] ret = new byte[bytes]; + System.arraycopy(full, (int) position, ret, 0, bytes); + return Tuple.of("etag-goes-here", ret); + } + + @Override + public long read( + StorageObject from, Map options, long position, OutputStream outputStream) { + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option op : options.keySet()) { + if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) { + generationMatch = (Long) options.get(op); + } else { + throw new UnsupportedOperationException("Unknown option: " + op); + } + } + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(NOT_FOUND, "File not found: " + key); + } + checkGeneration(key, generationMatch); + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + int bytes = (int) (full.length - position); + if (bytes <= 0) { + // special case: you're trying to read past the end + return 0; + } + try { + outputStream.write(full, (int) position, bytes); + } catch (IOException e) { + throw new StorageException(500, "Failed to write to file", e); + } + return bytes; + } + + @Override + public String open(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option option : options.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH) { + generationMatch = (Long) options.get(option); + } + } + checkGeneration(key, generationMatch); + metadata.put(key, object); + futureContents.put(key, new byte[0]); + + return key; + } + + @Override + public String open(String signedURL) { + return null; + } + + @Override + public void write( + String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, int length, boolean last) + throws StorageException { + writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last); + } + + @Override + public StorageObject writeWithResponse( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last) { + // this may have a lot more allocations than ideal, but it'll work. + byte[] bytes; + if (futureContents.containsKey(uploadId)) { + bytes = futureContents.get(uploadId); + if (bytes.length < length + destOffset) { + byte[] newBytes = new byte[(int) (length + destOffset)]; + System.arraycopy(bytes, 0, newBytes, (int) 0, bytes.length); + bytes = newBytes; + } + } else { + bytes = new byte[(int) (length + destOffset)]; + } + System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length); + // we want to mimic the GCS behavior that file contents are only visible on close. + StorageObject storageObject = null; + if (last) { + contents.put(uploadId, bytes); + futureContents.remove(uploadId); + if (metadata.containsKey(uploadId)) { + storageObject = metadata.get(uploadId); + storageObject.setUpdated(now()); + Long generation = storageObject.getGeneration(); + if (null == generation) { + generation = Long.valueOf(0); + } + storageObject.setGeneration(++generation); + metadata.put(uploadId, storageObject); + } + } else { + futureContents.put(uploadId, bytes); + } + return storageObject; + } + + @Override + public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException { + String sourceKey = fullname(rewriteRequest.source); + + // a little hackish, just good enough for the tests to work. + if (!contents.containsKey(sourceKey)) { + throw new StorageException(NOT_FOUND, "File not found: " + sourceKey); + } + + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option option : rewriteRequest.targetOptions.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH) { + generationMatch = (Long) rewriteRequest.targetOptions.get(option); + } + } + + String destKey = fullname(rewriteRequest.target); + + // if this is a new file, set generation to 1, else increment the existing generation + long generation = 1; + if (metadata.containsKey(destKey)) { + Long storedGeneration = metadata.get(destKey).getGeneration(); + if (null != storedGeneration) { + generation = storedGeneration + 1; + } + } + + checkGeneration(destKey, generationMatch); + + byte[] data = contents.get(sourceKey); + + rewriteRequest.target.setGeneration(generation); + rewriteRequest.target.setSize(BigInteger.valueOf(data.length)); + rewriteRequest.target.setUpdated(metadata.get(sourceKey).getUpdated()); + + metadata.put(destKey, rewriteRequest.target); + + contents.put(destKey, Arrays.copyOf(data, data.length)); + return new RewriteResponse( + rewriteRequest, + rewriteRequest.target, + data.length, + true, + "rewriteToken goes here", + data.length); + } + + @Override + public StorageObject moveObject( + String bucket, + String sourceObject, + String destinationObject, + Map sourceOptions, + Map targetOptions) + throws StorageException { + // This logic doesn't exactly match the semantics of the Objects: move API. But it should be + // close enough for the test. + String sourceKey = fullname(bucket, sourceObject); + if (!contents.containsKey(sourceKey)) { + throw new StorageException(NOT_FOUND, "File not found: " + sourceKey); + } + String destKey = fullname(bucket, destinationObject); + StorageObject sourceMetadata = metadata.get(sourceKey); + DateTime currentTime = now(); + sourceMetadata.setGeneration(sourceMetadata.getGeneration() + 1); + sourceMetadata.setTimeCreated(currentTime); + sourceMetadata.setUpdated(currentTime); + byte[] sourceData = contents.get(sourceKey); + metadata.put(destKey, sourceMetadata); + contents.put(destKey, Arrays.copyOf(sourceData, sourceData.length)); + metadata.remove(sourceKey); + contents.remove(sourceKey); + return sourceMetadata; + } + + private static DateTime now() { + return DateTime.parseRfc3339(RFC_3339_FORMATTER.format(new Date())); + } + + private String fullname(StorageObject so) { + return fullname(so.getBucket(), so.getName()); + } + + private BigInteger size(StorageObject so) { + String key = fullname(so); + + if (contents.containsKey(key)) { + return BigInteger.valueOf(contents.get(key).length); + } + + return null; + } + + private void potentiallyThrow(Map options) throws UnsupportedOperationException { + if (throwIfOption && !options.isEmpty()) { + throw new UnsupportedOperationException(); + } + } + + /** + * Throw if we're asking for generation 0 and the file exists, or if the requested generation + * number doesn't match what is asked. + * + * @param key + * @param generationMatch + */ + private void checkGeneration(String key, Long generationMatch) { + if (null == generationMatch) { + return; + } + if (generationMatch == 0 && metadata.containsKey(key)) { + throw new StorageException(new FileAlreadyExistsException(key)); + } + if (generationMatch != 0) { + Long generation = metadata.get(key).getGeneration(); + if (!generationMatch.equals(generation)) { + throw new StorageException( + NOT_FOUND, + "Generation mismatch. Requested " + generationMatch + " but got " + generation); + } + } + } + + // Returns true if this is a folder. Adds it to folders if it isn't already there. + private static boolean processedAsFolder( + StorageObject so, + String delimiter, + String prefix, /* inout */ + Map folders) { + if (delimiter == null) { + return false; + } + int nextSlash = so.getName().indexOf(delimiter, prefix.length()); + if (nextSlash < 0) { + return false; + } + String folderName = so.getName().substring(0, nextSlash + 1); + if (folders.containsKey(folderName)) { + return true; + } + StorageObject fakeFolder = new StorageObject(); + fakeFolder.setName(folderName); + fakeFolder.setBucket(so.getBucket()); + fakeFolder.setGeneration(so.getGeneration()); + fakeFolder.set("isDirectory", true); + fakeFolder.setSize(BigInteger.ZERO); + folders.put(folderName, fakeFolder); + return true; + } + + @Override + public ServiceAccount getServiceAccount(String projectId) { + return null; + } + + @Override + public com.google.api.services.storage.Storage getStorage() { + HttpTransport transport = new FakeStorageRpcHttpTransport(); + HttpRequestInitializer httpRequestInitializer = request -> {}; + return new com.google.api.services.storage.Storage( + transport, new GsonFactory(), httpRequestInitializer); + } + + private static String fullname(String bucket, String object) { + return String.format("http://localhost:65555/b/%s/o/%s", bucket, object); + } + + private static final String KEY_PATTERN_DEFINITION = "^.*?/b/(.*?)/o/(.*?)(?:[?].*|$)"; + private static final Pattern KEY_PATTERN = Pattern.compile(KEY_PATTERN_DEFINITION); + + MyMockLowLevelHttpRequest create(String method, String url) { + Matcher m = KEY_PATTERN.matcher(url); + Preconditions.checkArgument( + m.matches(), + "Provided url '%s' does not match expected pattern '%s'", + url, + KEY_PATTERN_DEFINITION); + + String bucket = m.group(1); + String object = m.group(2); + + String decode = urlDecode(object); + String key = fullname(bucket, decode); + return new MyMockLowLevelHttpRequest(method, url, key); + } + + private static String urlDecode(String object) { + try { + return URLDecoder.decode(object, StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + private class MyMockLowLevelHttpRequest extends MockLowLevelHttpRequest { + + private final String method; + private final String key; + + private MyMockLowLevelHttpRequest(String method, String url, String key) { + super(url); + this.method = method; + this.key = key; + } + + /** + * {@link MockLowLevelHttpRequest#execute} tries to return a value, but we need to compute based + * upon possible request mutations. So override it to call {@link #getResponse()}. + */ + @Override + public LowLevelHttpResponse execute() throws IOException { + return getResponse(); + } + + @Override + public MockLowLevelHttpResponse getResponse() { + + MockLowLevelHttpResponse resp = new MockLowLevelHttpResponse(); + byte[] bytes = contents.get(key); + StorageObject storageObject = metadata.get(key); + if (bytes == null && storageObject == null) { + resp.setStatusCode(NOT_FOUND); + } else if (storageObject != null && method.equals("PUT")) { + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + this.getStreamingContent().writeTo(baos); + bytes = futureContents.get(key); + byte[] byteArray = baos.toByteArray(); + if (bytes != null) { + futureContents.put(key, concat(bytes, byteArray)); + } else { + futureContents.put(key, byteArray); + } + + List contentRanges = getHeaders().get("content-range"); + if (contentRanges != null && !contentRanges.isEmpty()) { + String contentRange = contentRanges.get(0); + if ("bytes */*".equals(contentRange) || contentRange.endsWith("/*")) { + // query or incremental put + resp.addHeader( + "Range", String.format("bytes=0-%d", futureContents.get(key).length - 1)); + resp.setStatusCode(308); + } else if (contentRange.startsWith("bytes */") + || contentRange.matches("bytes \\d+-\\d+/\\d+$")) { + // finalize + byte[] finalBytes = futureContents.get(key); + BigInteger size = BigInteger.valueOf(finalBytes.length); + storageObject.setGeneration(System.currentTimeMillis()); + DateTime now = now(); + storageObject.setTimeCreated(now); + storageObject.setUpdated(now); + storageObject.setSize(size); + String string = GsonFactory.getDefaultInstance().toPrettyString(storageObject); + resp.addHeader("Content-Type", "application/json;charset=utf-8"); + resp.addHeader("Content-Length", String.valueOf(string.length())); + resp.setContent(string); + resp.setStatusCode(200); + contents.put(key, finalBytes); + futureContents.remove(key); + } else { + resp.setStatusCode(NOT_FOUND); + } + } else { + resp.setStatusCode(NOT_FOUND); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + int length = bytes.length; + Map> headers = getHeaders(); + List range = headers.get("range"); + int begin = 0; + int endInclusive = length - 1; + if (range != null && !range.isEmpty()) { + String rangeString = range.get(0).substring("range=".length()); + if ("0-".equals(rangeString)) { + resp.setStatusCode(OK); + } else if (rangeString.startsWith("-")) { + // we don't support negative offsets yet + resp.setStatusCode(400); + return resp; + } else if (rangeString.endsWith("-")) { + // only lower bounded + String beginS = rangeString.substring(0, rangeString.length() - 1); + begin = Integer.parseInt(beginS); + resp.setStatusCode(PARTIAL_CONTENT); + } else { + // otherwise a lower and upper bound + int i = rangeString.indexOf('-'); + if (i == -1) { + resp.setStatusCode(400); + return resp; + } else { + String beginS = rangeString.substring(0, i); + String endInclusiveS = rangeString.substring(i + 1); + begin = Integer.parseInt(beginS); + endInclusive = Integer.parseInt(endInclusiveS); + resp.setStatusCode(PARTIAL_CONTENT); + } + } + + if (begin > length) { + resp.addHeader("Content-Range", String.format("bytes */%d", length)); + resp.setContent(EMPTY_BYTES); + resp.setContent(new ByteArrayInputStream(EMPTY_BYTES)); + } else { + int newLength = endInclusive - begin + 1; + resp.addHeader("Content-Length", String.valueOf(newLength)); + // Content-Range: bytes 4-9/512 + resp.addHeader( + "Content-Range", String.format("bytes %d-%d/%d", begin, endInclusive, length)); + byte[] content = Arrays.copyOfRange(bytes, begin, endInclusive + 1); + resp.setContent(content); + resp.setContent(new ByteArrayInputStream(content)); + } + } else { + resp.addHeader("Content-Length", String.valueOf(length)); + resp.setContent(bytes); + resp.setContent(new ByteArrayInputStream(bytes)); + resp.setStatusCode(OK); + } + } + return resp; + } + } + + private static byte[] concat(byte[]... arrays) { + int total = Arrays.stream(arrays).filter(Objects::nonNull).mapToInt(a -> a.length).sum(); + byte[] retVal = new byte[total]; + + ByteBuffer wrap = ByteBuffer.wrap(retVal); + Arrays.stream(arrays).filter(Objects::nonNull).forEach(wrap::put); + + return retVal; + } + + private class FakeStorageRpcHttpTransport extends MockHttpTransport { + + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return create(method, url); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/LocalStorageHelper.java b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/LocalStorageHelper.java new file mode 100644 index 000000000000..436b9cf9dd10 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/LocalStorageHelper.java @@ -0,0 +1,106 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio.testing; + +import com.google.cloud.ServiceRpc; +import com.google.cloud.spi.ServiceRpcFactory; +import com.google.cloud.storage.StorageOptions; + +/** + * Utility to create an in-memory storage configuration for testing. Storage options can be obtained + * via the {@link #getOptions()} method. Returned options will point to FakeStorageRpc. + * + *

Note, the created in-memory storage configuration supports limited set of operations and is + * not thread-safe: + * + *

    + *
  • Supported operations + *
      + *
    • object create + *
    • object get + *
    • object delete + *
    • list the contents of a bucket + *
    + *
  • Unsupported operations + *
      + *
    • bucket create + *
    • bucket get + *
    • bucket delete + *
    • list all buckets + *
    • generations + *
    • file attributes + *
    • patch + *
    • continueRewrite + *
    • createBatch + *
    • checksums, etags + *
    • IAM operations + *
    + *
+ * + * {@link FakeStorageRpc#list(String, java.util.Map)} lists all the objects that have been created + * rather than the objects in the provided bucket. Since this class does not support creating, + * listing and deleting buckets, the parameter bucket here is not actually used and on serves as a + * placeholder. + */ +public final class LocalStorageHelper { + + // used for testing. Will throw if you pass it an option. + private static final FakeStorageRpc instance = new FakeStorageRpc(true); + + private LocalStorageHelper() {} + + /** + * Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, and resets it + * first so you start from a clean slate. That instance will throw if you pass it any option. + */ + public static StorageOptions getOptions() { + instance.reset(); + return StorageOptions.newBuilder() + .setProjectId("fake-project-for-testing") + .setServiceRpcFactory(new FakeStorageRpcFactory()) + .build(); + } + + /** + * Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance with the given + * option. + */ + public static StorageOptions customOptions(final boolean throwIfOptions) { + return StorageOptions.newBuilder() + .setProjectId("fake-project-for-testing") + .setServiceRpcFactory(new FakeStorageRpcFactory(new FakeStorageRpc(throwIfOptions))) + .build(); + } + + public static class FakeStorageRpcFactory implements ServiceRpcFactory { + + private final FakeStorageRpc instance; + + public FakeStorageRpcFactory() { + this(LocalStorageHelper.instance); + } + + public FakeStorageRpcFactory(FakeStorageRpc instance) { + this.instance = instance; + } + + @Override + public ServiceRpc create(StorageOptions storageOptions) { + return instance; + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/main/resources/META-INF/native-image/com/google/cloud/google-cloud-nio/native-image.properties b/java-storage-nio/google-cloud-nio/src/main/resources/META-INF/native-image/com/google/cloud/google-cloud-nio/native-image.properties new file mode 100644 index 000000000000..be1730e46a74 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/resources/META-INF/native-image/com/google/cloud/google-cloud-nio/native-image.properties @@ -0,0 +1,9 @@ +# Using META-INF/services with Native Image compilation results in +# FileSystemProvider being initialized at build time. This results +# CloudStorageFileSystemProvider and some classes referenced by +# this class (for example, CloudStorageConfiguration) +# being unexpectedly and recursively initialized at +# build time. +Args = --initialize-at-build-time=com.google.cloud.storage.contrib.nio.CloudStorageFileSystemProvider,\ + com.google.cloud.storage.contrib.nio.CloudStorageConfiguration,\ + com.google.cloud.storage.contrib.nio.AutoValue_CloudStorageConfiguration,\ diff --git a/java-storage-nio/google-cloud-nio/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/java-storage-nio/google-cloud-nio/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider new file mode 100644 index 000000000000..cdacea417537 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -0,0 +1,14 @@ +# Copyright 2021 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +com.google.cloud.storage.contrib.nio.CloudStorageFileSystemProvider diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageConfigurationTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageConfigurationTest.java new file mode 100644 index 000000000000..d7715623325f --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageConfigurationTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.net.SocketTimeoutException; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageConfiguration}. */ +@RunWith(JUnit4.class) +public class CloudStorageConfigurationTest { + + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + @Test + public void testBuilder() { + CloudStorageConfiguration config = + CloudStorageConfiguration.builder() + .workingDirectory("/omg") + .permitEmptyPathComponents(true) + .stripPrefixSlash(false) + .usePseudoDirectories(false) + .blockSize(666) + .retryableHttpCodes(ImmutableList.of(1, 2, 3)) + .reopenableExceptions( + ImmutableList.>of(SocketTimeoutException.class)) + .build(); + assertThat(config.workingDirectory()).isEqualTo("/omg"); + assertThat(config.permitEmptyPathComponents()).isTrue(); + assertThat(config.stripPrefixSlash()).isFalse(); + assertThat(config.usePseudoDirectories()).isFalse(); + assertThat(config.blockSize()).isEqualTo(666); + assertThat(config.retryableHttpCodes()).isEqualTo(ImmutableList.of(1, 2, 3)); + assertThat(config.reopenableExceptions()) + .isEqualTo(ImmutableList.>of(SocketTimeoutException.class)); + } + + @Test + public void testFromMap() { + CloudStorageConfiguration config = + CloudStorageConfiguration.fromMap( + new ImmutableMap.Builder() + .put("workingDirectory", "/omg") + .put("permitEmptyPathComponents", true) + .put("stripPrefixSlash", false) + .put("usePseudoDirectories", false) + .put("blockSize", 666) + .put("retryableHttpCodes", ImmutableList.of(1, 2, 3)) + .put( + "reopenableExceptions", + ImmutableList.>of(SocketTimeoutException.class)) + .build()); + assertThat(config.workingDirectory()).isEqualTo("/omg"); + assertThat(config.permitEmptyPathComponents()).isTrue(); + assertThat(config.stripPrefixSlash()).isFalse(); + assertThat(config.usePseudoDirectories()).isFalse(); + assertThat(config.blockSize()).isEqualTo(666); + assertThat(config.retryableHttpCodes()).isEqualTo(ImmutableList.of(1, 2, 3)); + assertThat(config.reopenableExceptions()) + .isEqualTo(ImmutableList.>of(SocketTimeoutException.class)); + } + + @Test + public void testFromMap_badKey_throwsIae() { + try { + CloudStorageConfiguration.fromMap(ImmutableMap.of("lol", "/omg")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + /** Spot check that our defaults are applied. */ + public void testSomeDefaults() { + for (CloudStorageConfiguration config : + ImmutableList.of( + CloudStorageConfiguration.DEFAULT, CloudStorageConfiguration.builder().build())) { + assertThat(config.retryableHttpCodes()).contains(503); + assertThat(config.reopenableExceptions()).contains(SocketTimeoutException.class); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java new file mode 100644 index 000000000000..fc673c0ae26e --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java @@ -0,0 +1,120 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageFileAttributeView}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileAttributeViewTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private static final byte[] HAPPY = "(✿◕ ‿◕ )ノ".getBytes(UTF_8); + + private Path path; + + @Before + public void before() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + path = Paths.get(URI.create("gs://red/water")); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void testReadAttributes() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withCacheControl("potato")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.readAttributes().cacheControl().get()).isEqualTo("potato"); + } + + @Test + public void testReadAttributes_notFound_throwsNoSuchFileException() throws IOException { + try { + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + lazyAttributes.readAttributes(); + Assert.fail(); + } catch (NoSuchFileException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testReadAttributes_pseudoDirectory() throws IOException { + Path dir = Paths.get(URI.create("gs://red/rum/")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(dir, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.readAttributes()) + .isInstanceOf(CloudStoragePseudoDirectoryAttributes.class); + } + + @Test + public void testName() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withCacheControl("potato")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.name()).isEqualTo("gcs"); + } + + @Test + public void testEquals_equalsTester() { + new EqualsTester() + .addEqualityGroup( + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/rum")), CloudStorageFileAttributeView.class), + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/rum")), CloudStorageFileAttributeView.class)) + .addEqualityGroup( + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/lol/dog")), CloudStorageFileAttributeView.class)) + .testEquals(); + } + + @Test + public void testNullness() throws NoSuchMethodException, SecurityException { + new NullPointerTester() + .ignore(CloudStorageFileAttributeView.class.getMethod("equals", Object.class)) + .setDefault(FileTime.class, FileTime.fromMillis(0)) + .testAllPublicInstanceMethods( + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class)); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributesTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributesTest.java new file mode 100644 index 000000000000..92efba8e2c24 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributesTest.java @@ -0,0 +1,193 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageFileAttributes}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileAttributesTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private static final byte[] HAPPY = "(✿◕ ‿◕ )ノ".getBytes(UTF_8); + private static final byte[] EMPTY = "".getBytes(UTF_8); + + private Path path; + private Path dir; + + @Before + public void before() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + path = Paths.get(URI.create("gs://bucket/randompath")); + dir = Paths.get(URI.create("gs://bucket/randompath/")); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void testCacheControl() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withCacheControl("potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("potato"); + } + + @Test + public void testMimeType() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).mimeType().get()) + .isEqualTo("text/potato"); + } + + @Test + public void testAcl() throws IOException { + Acl acl = Acl.of(new Acl.User("serf@example.com"), Acl.Role.READER); + Files.write(path, HAPPY, CloudStorageOptions.withAcl(acl)); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).acl().get()) + .contains(acl); + } + + @Test + public void testContentDisposition() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withContentDisposition("crash call")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).contentDisposition().get()) + .isEqualTo("crash call"); + } + + @Test + public void testContentEncoding() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withContentEncoding("my content encoding")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).contentEncoding().get()) + .isEqualTo("my content encoding"); + } + + @Test + public void testUserMetadata() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withUserMetadata("green", "bean")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class) + .userMetadata() + .get("green")) + .isEqualTo("bean"); + } + + @Test + public void testIsDirectory() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isDirectory()) + .isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isDirectory()).isTrue(); + } + + @Test + public void testIsPseudoDirectory() throws IOException { + Files.write(path, EMPTY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isDirectory()) + .isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isDirectory()).isTrue(); + } + + @Test + public void testIsRegularFile() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isRegularFile()) + .isTrue(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isRegularFile()) + .isFalse(); + } + + @Test + public void testIsOther() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isOther()).isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isOther()).isFalse(); + } + + @Test + public void testIsSymbolicLink() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isSymbolicLink()) + .isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isSymbolicLink()) + .isFalse(); + } + + @Test + public void testEquals_equalsTester() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/plain")); + CloudStorageFileAttributes a1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes a2 = Files.readAttributes(path, CloudStorageFileAttributes.class); + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/potato")); + CloudStorageFileAttributes b1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes b2 = Files.readAttributes(path, CloudStorageFileAttributes.class); + new EqualsTester().addEqualityGroup(a1, a2).addEqualityGroup(b1, b2).testEquals(); + } + + @Test + public void testFilekey() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/plain")); + Path path2 = Paths.get(URI.create("gs://bucket/anotherrandompath")); + Files.write(path2, HAPPY, CloudStorageOptions.withMimeType("text/plain")); + + // diff files cannot have same filekey + CloudStorageFileAttributes a1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes a2 = Files.readAttributes(path2, CloudStorageFileAttributes.class); + assertThat(a1.fileKey()).isNotEqualTo(a2.fileKey()); + + // same for directories + CloudStorageFileAttributes b1 = Files.readAttributes(dir, CloudStorageFileAttributes.class); + CloudStorageFileAttributes b2 = + Files.readAttributes( + Paths.get(URI.create("gs://bucket/jacket/")), CloudStorageFileAttributes.class); + assertThat(a1.fileKey()).isNotEqualTo(b1.fileKey()); + assertThat(b1.fileKey()).isNotEqualTo(b2.fileKey()); + } + + @Test + public void testNullness() throws IOException, NoSuchMethodException, SecurityException { + Files.write(path, HAPPY); + CloudStorageFileAttributes pathAttributes = + Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes dirAttributes = + Files.readAttributes(dir, CloudStorageFileAttributes.class); + NullPointerTester tester = new NullPointerTester(); + tester.ignore(CloudStorageObjectAttributes.class.getMethod("equals", Object.class)); + tester.testAllPublicInstanceMethods(pathAttributes); + tester.testAllPublicInstanceMethods(dirAttributes); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java new file mode 100644 index 000000000000..36a9c2c4ffb3 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java @@ -0,0 +1,1044 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.cloud.storage.Acl.Role.OWNER; +import static com.google.cloud.storage.contrib.nio.CloudStorageFileSystem.forBucket; +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.Files.readAllBytes; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.Acl.User; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.testing.NullPointerTester; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.UserPrincipal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageFileSystemProvider}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileSystemProviderTest { + // @Rule(order = 1) public final MultipleAttemptsRule multipleAttemptsRule = new + // MultipleAttemptsRule(3); + @Rule(order = 1) + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static final List FILE_CONTENTS = + ImmutableList.of( + "Lorem ipsum dolor sit amet, consectetur ", + "adipisicing elit. Ab, corporis deleniti ducimus ", + "ea esse est fuga illum inventore itaque maiores ", + "mollitia necessitatibus nesciunt nisi nobis non, ", + "nulla officia omnis placeat quibusdam unde? Alias ", + "delectus dignissimos, ducimus enim et expedita ", + "iste molestiae mollitia porro sunt! Debitis ", + "doloribus earum modi nam neque nulla optio ", + "quisquam reprehenderit. Autem consequatur ", + "delectus vitae. Aut consectetur cum eaque facere ", + "illum in molestiae nam, nulla obcaecati officia ", + "optio perspiciatis, quisquam reiciendis sequi ", + "tempora, velit veritatis vitae? Alias ", + "consequuntur dolor doloremque eius et fugiat ", + "fugit harum illo incidunt ipsa maxime molestiae ", + "nostrum officia pariatur, quam quidem similique ", + "velit veniam voluptatem voluptatibus. Ab at ", + "commodi ea expedita optio. Ab cumque eos et, ", + "libero non quam quia recusandae tempora vitae! ", + "Debitis libero quidem reprehenderit voluptas. ", + "Architecto consectetur cum dignissimos, dolorem ", + "eos, eum explicabo fugiat magnam maiores modi ", + "numquam odio pariatur provident quae quasi quos ", + "ratione recusandae repellendus similique ullam ", + "velit!"); + + private static final String SINGULARITY = "A string"; + + @Before + public void before() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void testSize() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + assertThat(Files.size(path)).isEqualTo(SINGULARITY.getBytes(UTF_8).length); + } + + @Test + public void testSize_trailingSlash_returnsFakePseudoDirectorySize() throws Exception { + assertThat(Files.size(Paths.get(URI.create("gs://bucket/wat/")))).isEqualTo(1); + } + + @Test + public void test_trailingSlash_isFolder() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + Files.write(path, SINGULARITY.getBytes(UTF_8), CloudStorageOptions.allowTrailingSlash()); + assertThat(Files.isDirectory(path)).isTrue(); + } + + @Test + public void testSize_trailingSlash_disablePseudoDirectories() throws Exception { + try (CloudStorageFileSystem fs = forBucket("doodle", usePseudoDirectories(false))) { + Path path = fs.getPath("wat/"); + byte[] rapture = SINGULARITY.getBytes(UTF_8); + Files.write(path, rapture); + assertThat(Files.size(path)).isEqualTo(rapture.length); + Files.delete(path); + } + } + + @Test + public void testReadAllBytes() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + assertThat(new String(readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + Files.delete(path); + } + + @Test + public void testReadAllBytes_trailingSlash() throws Exception { + try { + readAllBytes(Paths.get(URI.create("gs://bucket/wat/"))); + Assert.fail(); + } catch (CloudStoragePseudoDirectoryException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testNewByteChannelRead() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + byte[] data = SINGULARITY.getBytes(UTF_8); + Files.write(path, data); + try (ReadableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(data.length); + assertThat(input.read(buffer)).isEqualTo(data.length); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo(SINGULARITY); + buffer.rewind(); + assertThat(input.read(buffer)).isEqualTo(-1); + } + Files.delete(path); + } + + @Test + public void testNewByteChannelRead_seeking() throws Exception { + Path path = Paths.get(URI.create("gs://lol/cat")); + Files.write(path, "helloworld".getBytes(UTF_8)); + try (SeekableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(5); + input.position(5); + assertThat(input.position()).isEqualTo(5); + assertThat(input.read(buffer)).isEqualTo(5); + assertThat(input.position()).isEqualTo(10); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo("world"); + buffer.rewind(); + assertThat(input.read(buffer)).isEqualTo(-1); + input.position(0); + assertThat(input.position()).isEqualTo(0); + assertThat(input.read(buffer)).isEqualTo(5); + assertThat(input.position()).isEqualTo(5); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo("hello"); + } + Files.delete(path); + } + + @Test + public void testNewByteChannelRead_seekBeyondSize_reportsEofOnNextRead() throws Exception { + Path path = Paths.get(URI.create("gs://lol/cat")); + Files.write(path, "hellocat".getBytes(UTF_8)); + try (SeekableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(5); + input.position(10); + assertThat(input.read(buffer)).isEqualTo(-1); + input.position(11); + assertThat(input.read(buffer)).isEqualTo(-1); + assertThat(input.size()).isEqualTo(8); + } + } + + @Test + public void testNewByteChannelRead_trailingSlash() throws Exception { + try { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + Files.newByteChannel(path); + Assert.fail(); + } catch (CloudStoragePseudoDirectoryException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testNewByteChannelRead_notFound() throws Exception { + try { + Path path = Paths.get(URI.create("gs://bucket/wednesday")); + Files.newByteChannel(path); + Assert.fail(); + } catch (NoSuchFileException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testNewByteChannelWrite() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/tests")); + try (SeekableByteChannel output = Files.newByteChannel(path, WRITE)) { + assertThat(output.position()).isEqualTo(0); + assertThat(output.size()).isEqualTo(0); + ByteBuffer buffer = ByteBuffer.wrap("filec".getBytes(UTF_8)); + assertThat(output.write(buffer)).isEqualTo(5); + assertThat(output.position()).isEqualTo(5); + assertThat(output.size()).isEqualTo(5); + buffer = ByteBuffer.wrap("onten".getBytes(UTF_8)); + assertThat(output.write(buffer)).isEqualTo(5); + assertThat(output.position()).isEqualTo(10); + assertThat(output.size()).isEqualTo(10); + } + assertThat(new String(readAllBytes(path), UTF_8)).isEqualTo("fileconten"); + } + + @Test + public void testNewInputStream() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + try (InputStream input = Files.newInputStream(path)) { + byte[] data = new byte[SINGULARITY.getBytes(UTF_8).length]; + input.read(data); + assertThat(new String(data, UTF_8)).isEqualTo(SINGULARITY); + } + } + + @Test + public void testNewInputStream_trailingSlash() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + try (InputStream input = Files.newInputStream(path)) { + input.read(); + Assert.fail(); + } catch (CloudStoragePseudoDirectoryException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testNewInputStream_notFound() throws Exception { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + try (InputStream input = Files.newInputStream(path)) { + input.read(); + Assert.fail(); + } catch (NoSuchFileException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testNewOutputStream() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_truncateByDefault() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + Files.write(path, "hello".getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_truncateExplicitly() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + Files.write(path, "hello".getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path, TRUNCATE_EXISTING)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_trailingSlash() throws Exception { + try { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + Files.newOutputStream(path); + Assert.fail(); + } catch (CloudStoragePseudoDirectoryException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testNewOutputStream_createNew() throws Exception { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + Files.newOutputStream(path, CREATE_NEW); + } + + @Test + public void testNewOutputStream_createNew_alreadyExists() throws Exception { + try { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + Files.newOutputStream(path, CREATE_NEW); + Assert.fail(); + } catch (FileAlreadyExistsException expected) { + } + } + + @Test + public void testWrite_objectNameWithExtraSlashes_throwsIae() throws Exception { + try { + Path path = Paths.get(URI.create("gs://double/slash//yep")); + Files.write(path, FILE_CONTENTS, UTF_8); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testWrite_objectNameWithExtraSlashes_canBeNormalized() throws Exception { + try (CloudStorageFileSystem fs = forBucket("greenbean", permitEmptyPathComponents(false))) { + Path path = fs.getPath("adipose//yep").normalize(); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(fs.getPath("adipose", "yep"))).isTrue(); + } + } + + @Test + public void testWrite_objectNameWithExtraSlashes_permitEmptyPathComponents() throws Exception { + try (CloudStorageFileSystem fs = forBucket("greenbean", permitEmptyPathComponents(true))) { + Path path = fs.getPath("adipose//yep"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(path)).isTrue(); + } + } + + @Test + public void testWrite_absoluteObjectName_prefixSlashGetsRemoved() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/adipose/yep")); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testWrite_absoluteObjectName_disableStrip_slashGetsPreserved() throws Exception { + try (CloudStorageFileSystem fs = + forBucket( + "greenbean", CloudStorageConfiguration.builder().stripPrefixSlash(false).build())) { + Path path = fs.getPath("/adipose/yep"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(path)).isTrue(); + } + } + + @Test + public void testWrite() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + } + + @Test + public void testWriteOnClose() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/adipose")); + try (SeekableByteChannel chan = Files.newByteChannel(path, WRITE)) { + // writing lots of contents to defeat channel-internal buffering. + for (int i = 0; i < 9999; i++) { + for (String s : FILE_CONTENTS) { + chan.write(ByteBuffer.wrap(s.getBytes(UTF_8))); + } + } + try { + Files.size(path); + // we shouldn't make it to this line. Not using thrown.expect because + // I still want to run a few lines after the exception. + assertThat(false).isTrue(); + } catch (NoSuchFileException nsf) { + // that's what we wanted, we're good. + } + } + // channel now closed, the file should be there and with the new contents. + assertThat(Files.exists(path)).isTrue(); + assertThat(Files.size(path)).isGreaterThan(100L); + } + + @Test + public void testWrite_trailingSlash() throws Exception { + try { + Files.write(Paths.get(URI.create("gs://greenbean/adipose/")), FILE_CONTENTS, UTF_8); + Assert.fail(); + } catch (CloudStoragePseudoDirectoryException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testExists() throws Exception { + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion")))).isFalse(); + Files.write(Paths.get(URI.create("gs://military/fashion")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion")))).isTrue(); + } + + @Test + public void testExists_trailingSlash() { + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/")))).isTrue(); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/.")))).isTrue(); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/..")))).isTrue(); + } + + @Test + public void testExists_trailingSlash_disablePseudoDirectories() throws Exception { + try (CloudStorageFileSystem fs = forBucket("military", usePseudoDirectories(false))) { + assertThat(Files.exists(fs.getPath("fashion/"))).isFalse(); + } + } + + @Test + public void testFakeDirectories() throws IOException { + try (FileSystem fs = forBucket("military")) { + List paths = new ArrayList<>(); + paths.add(fs.getPath("dir/angel")); + paths.add(fs.getPath("dir/deepera")); + paths.add(fs.getPath("dir/deeperb")); + paths.add(fs.getPath("dir/deeper_")); + paths.add(fs.getPath("dir/deeper.sea/hasfish")); + paths.add(fs.getPath("dir/deeper/fish")); + for (Path path : paths) { + Files.createFile(path); + } + + // ends with slash, must be a directory + assertThat(Files.isDirectory(fs.getPath("dir/"))).isTrue(); + // files are not directories + assertThat(Files.exists(fs.getPath("dir/angel"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/angel"))).isFalse(); + // directories are recognized even without the trailing "/" + assertThat(Files.isDirectory(fs.getPath("dir"))).isTrue(); + // also works for absolute paths + assertThat(Files.isDirectory(fs.getPath("/dir"))).isTrue(); + // non-existent files are not directories (but they don't make us crash) + assertThat(Files.isDirectory(fs.getPath("di"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("dirs"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("dir/deep"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper/fi"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("/dir/deeper/fi"))).isFalse(); + // also works for subdirectories + assertThat(Files.isDirectory(fs.getPath("dir/deeper/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("/dir/deeper/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("/dir/deeper"))).isTrue(); + // dot and .. folders are directories + assertThat(Files.isDirectory(fs.getPath("dir/deeper/."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper/.."))).isTrue(); + // dots in the name are fine + assertThat(Files.isDirectory(fs.getPath("dir/deeper.sea/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper.sea"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper.seax"))).isFalse(); + // the root folder is a directory + assertThat(Files.isDirectory(fs.getPath("/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath(""))).isTrue(); + } + } + + @Test + public void testDelete() throws Exception { + Files.write(Paths.get(URI.create("gs://love/fashion")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.exists(Paths.get(URI.create("gs://love/fashion")))).isTrue(); + Files.delete(Paths.get(URI.create("gs://love/fashion"))); + assertThat(Files.exists(Paths.get(URI.create("gs://love/fashion")))).isFalse(); + } + + @Test + public void testDelete_dotDirNotNormalized_throwsIae() throws Exception { + try { + Files.delete(Paths.get(URI.create("gs://love/fly/../passion"))); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testDelete_trailingSlash() throws Exception { + Files.delete(Paths.get(URI.create("gs://love/passion/"))); + } + + @Test + public void testDelete_trailingSlash_disablePseudoDirectories() throws Exception { + try (CloudStorageFileSystem fs = forBucket("pumpkin", usePseudoDirectories(false))) { + Path path = fs.getPath("wat/"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.exists(path)); + Files.delete(path); + assertThat(!Files.exists(path)); + } + } + + @Test + public void testDelete_notFound() throws Exception { + try { + Files.delete(Paths.get(URI.create("gs://loveh/passionehu"))); + Assert.fail(); + } catch (NoSuchFileException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testDeleteIfExists() throws Exception { + Files.write(Paths.get(URI.create("gs://love/passionz")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.deleteIfExists(Paths.get(URI.create("gs://love/passionz")))).isTrue(); + // call does not fail, the folder just doesn't exist + Files.deleteIfExists(Paths.get(URI.create("gs://love/passion/"))); + } + + @Test + public void testDeleteIfExists_trailingSlash_disablePseudoDirectories() throws Exception { + try (CloudStorageFileSystem fs = forBucket("doodle", usePseudoDirectories(false))) { + // Doesn't exist, no error + Files.deleteIfExists(Paths.get(URI.create("gs://love/passion/"))); + } + } + + @Test + public void testCopy() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target); + assertThat(new String(readAllBytes(target), UTF_8)).isEqualTo("(✿◕ ‿◕ )ノ"); + assertThat(Files.exists(source)).isTrue(); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + public void testCopy_sourceMissing_throwsNoSuchFileException() throws Exception { + try { + Files.copy( + Paths.get(URI.create("gs://military/fashion.show")), + Paths.get(URI.create("gs://greenbean/adipose"))); + Assert.fail(); + } catch (NoSuchFileException expected) { + } + } + + @Test + public void testCopy_targetExists_throwsFileAlreadyExistsException() throws Exception { + try { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.write(target, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target); + Assert.fail(); + } catch (FileAlreadyExistsException expected) { + } + } + + @Test + public void testCopyReplace_targetExists_works() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.write(target, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target, REPLACE_EXISTING); + } + + @Test + public void testCopy_directory_doesNothing() throws Exception { + Path source = Paths.get(URI.create("gs://military/fundir/")); + Path target = Paths.get(URI.create("gs://greenbean/loldir/")); + Files.copy(source, target); + } + + @Test + public void testCopy_atomic_throwsUnsupported() throws Exception { + try { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target, ATOMIC_MOVE); + Assert.fail(); + } catch (UnsupportedOperationException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testMove_atomic() throws Exception { + Path source = Paths.get(URI.create("gs://greenbean/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.move(source, target, ATOMIC_MOVE); + assertThat(Files.exists(source)).isFalse(); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + public void testMove_crossBucket() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.move(source, target); + assertThat(Files.exists(source)).isFalse(); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + public void testMove_atomicCrossBucket_throwsUnsupported() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + try { + Files.move(source, target, ATOMIC_MOVE); + Assert.fail(); + } catch (AtomicMoveNotSupportedException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testCreateDirectory() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/dir/")); + Files.createDirectory(path); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testIsDirectory() throws Exception { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://doodle"))) { + assertThat(Files.isDirectory(fs.getPath(""))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("./"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("cat/.."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("hello/cat/.."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("cat/../"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("hello/cat/../"))).isTrue(); + } + } + + @Test + public void testIsDirectory_trailingSlash_alwaysTrue() { + assertThat(Files.isDirectory(Paths.get(URI.create("gs://military/fundir/")))).isTrue(); + } + + @Test + public void testIsDirectory_trailingSlash_pseudoDirectoriesDisabled_false() throws Exception { + try (CloudStorageFileSystem fs = forBucket("doodle", usePseudoDirectories(false))) { + assertThat(Files.isDirectory(fs.getPath("fundir/"))).isFalse(); + } + } + + @Test + public void testCopy_withCopyAttributes_preservesAttributes() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write( + source, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withMimeType("text/lolcat"), + CloudStorageOptions.withCacheControl("public; max-age=666"), + CloudStorageOptions.withContentEncoding("foobar"), + CloudStorageOptions.withContentDisposition("my-content-disposition"), + CloudStorageOptions.withUserMetadata("answer", "42")); + Files.copy(source, target, COPY_ATTRIBUTES); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/lolcat"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + assertThat(attributes.contentEncoding()).hasValue("foobar"); + assertThat(attributes.contentDisposition()).hasValue("my-content-disposition"); + assertThat(attributes.userMetadata().containsKey("answer")).isTrue(); + assertThat(attributes.userMetadata().get("answer")).isEqualTo("42"); + } + + @Test + public void testCopy_withoutOptions_doesntPreservesAttributes() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write( + source, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withMimeType("text/lolcat"), + CloudStorageOptions.withCacheControl("public; max-age=666"), + CloudStorageOptions.withUserMetadata("answer", "42")); + Files.copy(source, target); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target, CloudStorageFileAttributes.class); + String mimeType = attributes.mimeType().orNull(); + String cacheControl = attributes.cacheControl().orNull(); + assertThat(mimeType).isNotEqualTo("text/lolcat"); + assertThat(cacheControl).isNull(); + assertThat(attributes.userMetadata().containsKey("answer")).isFalse(); + } + + @Test + public void testCopy_overwriteAttributes() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target1 = Paths.get(URI.create("gs://greenbean/adipose")); + Path target2 = Paths.get(URI.create("gs://greenbean/round")); + Files.write( + source, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withMimeType("text/lolcat"), + CloudStorageOptions.withCacheControl("public; max-age=666")); + Files.copy(source, target1, COPY_ATTRIBUTES); + Files.copy(source, target2, COPY_ATTRIBUTES, CloudStorageOptions.withMimeType("text/palfun")); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target1, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/lolcat"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + + attributes = Files.readAttributes(target2, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/palfun"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + Files.delete(source); + Files.delete(target1); + Files.delete(target2); + } + + @Test + public void testCopy_path_toLocalFileSystem() throws IOException { + Path source = Paths.get(URI.create("gs://mybucket/myobject")); + byte[] helloWorldBytes = "Hello, World!".getBytes(UTF_8); + Files.write(source, helloWorldBytes); + + Path path = temporaryFolder.newFile().toPath(); + // The new file created by temporaryFolder is an empty file on disk, specify REPLACE_EXISTING + // so we can overwrite its contents. + Files.copy(source, path, REPLACE_EXISTING); + assertThat(Files.readAllBytes(path)).isEqualTo(helloWorldBytes); + } + + @Test + public void testNullness() throws Exception { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://blood"))) { + NullPointerTester tester = new NullPointerTester(); + tester.ignore(CloudStorageFileSystemProvider.class.getMethod("equals", Object.class)); + tester.setDefault(URI.class, URI.create("gs://blood")); + tester.setDefault(Path.class, fs.getPath("and/one")); + tester.setDefault(OpenOption.class, CREATE); + tester.setDefault(CopyOption.class, COPY_ATTRIBUTES); + tester.testAllPublicStaticMethods(CloudStorageFileSystemProvider.class); + tester.testAllPublicInstanceMethods(new CloudStorageFileSystemProvider()); + } + } + + @Test + public void testProviderEquals() { + Path path1 = Paths.get(URI.create("gs://bucket/tuesday")); + Path path2 = Paths.get(URI.create("gs://blood/wednesday")); + Path path3 = Paths.get("tmp"); + assertThat(path1.getFileSystem().provider()).isEqualTo(path2.getFileSystem().provider()); + assertThat(path1.getFileSystem().provider()).isNotEqualTo(path3.getFileSystem().provider()); + } + + @Test + public void testNewFileSystem() throws Exception { + Map env = new HashMap<>(); + FileSystems.newFileSystem(URI.create("gs://bucket/path/to/file"), env); + } + + @Test + public void testFromSpace() throws Exception { + // User should be able to create paths to files whose name contains a space. + // Traditional way 1: manually escape the spaces + Path path1 = Paths.get(URI.create("gs://bucket/with/a%20space")); + CloudStorageFileSystemProvider provider = + (CloudStorageFileSystemProvider) path1.getFileSystem().provider(); + // Traditional way 2: use UrlEscapers.urlFragmentEscaper().escape + // to escape the string for you. + // (Not tested because UrlEscapers isn't the unit under test). + + // Non-traditional way: use our convenience method to work around URIs not being allowed to + // contain spaces. + Path path3 = provider.getPath("gs://bucket/with/a space"); + // Both approaches should be equivalent + assertThat(path1.getFileSystem().provider()).isEqualTo(path3.getFileSystem().provider()); + assertThat(path1.toUri()).isEqualTo(path3.toUri()); + + // getPath does not interpret the string at all. + Path path4 = provider.getPath("gs://bucket/with/a%20percent"); + assertThat(path4.toString()).isEqualTo("/with/a%20percent"); + } + + @Test + public void testBucketWithHost() { + // User should be able to create buckets whose name contains a host name. + Path path1 = Paths.get(URI.create("gs://bucket-with-host/path")); + CloudStorageFileSystemProvider provider = + (CloudStorageFileSystemProvider) path1.getFileSystem().provider(); + + Path path2 = provider.getPath("gs://bucket-with-host/path"); + // Both approaches should be equivalent + assertThat(path1.getFileSystem().provider()).isEqualTo(path2.getFileSystem().provider()); + assertThat(path1.toUri()).isEqualTo(path2.toUri()); + assertThat(path1.toUri().getHost()).isEqualTo("bucket-with-host"); + assertThat(path1.toUri().getAuthority()).isEqualTo("bucket-with-host"); + } + + @Test + public void testBucketWithAuthority() { + // User should be able to create buckets whose name contains an authority that is not a host. + Path path1 = Paths.get(URI.create("gs://bucket_with_authority/path")); + CloudStorageFileSystemProvider provider = + (CloudStorageFileSystemProvider) path1.getFileSystem().provider(); + + Path path2 = provider.getPath("gs://bucket_with_authority/path"); + // Both approaches should be equivalent + assertThat(path1.getFileSystem().provider()).isEqualTo(path2.getFileSystem().provider()); + assertThat(path1.toUri()).isEqualTo(path2.toUri()); + assertThat(path1.toUri().getHost()).isNull(); + assertThat(path1.toUri().getAuthority()).isEqualTo("bucket_with_authority"); + } + + @Test + public void testBucketWithoutAuthority() { + Path path1 = Paths.get(URI.create("gs://bucket_with_authority/path")); + CloudStorageFileSystemProvider provider = + (CloudStorageFileSystemProvider) path1.getFileSystem().provider(); + + try { + provider.getFileSystem(URI.create("gs:///path")); + Assert.fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage()).isEqualTo("gs:// URIs must have a host: gs:///path"); + } + } + + @Test + public void testVersion_matchesAcceptablePatterns() { + String acceptableVersionPattern = "|(?:\\d+\\.\\d+\\.\\d+(?:-.*?)?(?:-SNAPSHOT)?)"; + String version = StorageOptionsUtil.USER_AGENT_ENTRY_VERSION; + assertTrue( + String.format("the loaded version '%s' did not match the acceptable pattern", version), + version.matches(acceptableVersionPattern)); + } + + @Test + public void getUserAgentStartsWithCorrectToken() { + assertThat(String.format("gcloud-java-nio/%s", StorageOptionsUtil.USER_AGENT_ENTRY_VERSION)) + .startsWith("gcloud-java-nio/"); + } + + @Test + public void testReadAttributes() throws IOException { + CloudStorageFileSystem fileSystem = forBucket("dummy"); + CloudStorageFileSystemProvider fileSystemProvider = spy(fileSystem.provider()); + + BasicFileAttributes attributesBasic = mock(BasicFileAttributes.class); + // BasicFileAttributes + when(attributesBasic.creationTime()).thenReturn(FileTime.fromMillis(1L)); + when(attributesBasic.lastModifiedTime()).thenReturn(FileTime.fromMillis(2L)); + when(attributesBasic.lastAccessTime()).thenReturn(FileTime.fromMillis(3L)); + when(attributesBasic.isRegularFile()).thenReturn(true); + when(attributesBasic.isDirectory()).thenReturn(true); + when(attributesBasic.isSymbolicLink()).thenReturn(true); + when(attributesBasic.isOther()).thenReturn(true); + when(attributesBasic.size()).thenReturn(42L); + + CloudStorageFileAttributes attributesGcs = mock(CloudStorageFileAttributes.class); + // BasicFileAttributes + when(attributesGcs.creationTime()).thenReturn(FileTime.fromMillis(1L)); + when(attributesGcs.lastModifiedTime()).thenReturn(FileTime.fromMillis(2L)); + when(attributesGcs.lastAccessTime()).thenReturn(FileTime.fromMillis(3L)); + when(attributesGcs.isRegularFile()).thenReturn(true); + when(attributesGcs.isDirectory()).thenReturn(true); + when(attributesGcs.isSymbolicLink()).thenReturn(true); + when(attributesGcs.isOther()).thenReturn(true); + when(attributesGcs.size()).thenReturn(42L); + + List acls = ImmutableList.of(Acl.newBuilder(new User("Foo"), OWNER).build()); + + // CloudStorageFileAttributes + when(attributesGcs.etag()).thenReturn(Optional.of("TheEtag")); + when(attributesGcs.mimeType()).thenReturn(Optional.of("TheMimeType")); + when(attributesGcs.acl()).thenReturn(Optional.of(acls)); + when(attributesGcs.cacheControl()).thenReturn(Optional.of("TheCacheControl")); + when(attributesGcs.contentEncoding()).thenReturn(Optional.of("TheContentEncoding")); + when(attributesGcs.contentDisposition()).thenReturn(Optional.of("TheContentDisposition")); + when(attributesGcs.userMetadata()).thenReturn(new TreeMap<>()); + + CloudStoragePath path1 = CloudStoragePath.getPath(fileSystem, "/"); + when(fileSystemProvider.readAttributes(path1, BasicFileAttributes.class)) + .thenReturn(attributesBasic); + when(fileSystemProvider.readAttributes(path1, CloudStorageFileAttributes.class)) + .thenReturn(attributesGcs); + + Map expectedBasic = new TreeMap<>(); + // BasicFileAttributes + expectedBasic.put("creationTime", FileTime.fromMillis(1L)); + expectedBasic.put("lastModifiedTime", FileTime.fromMillis(2L)); + expectedBasic.put("lastAccessTime", FileTime.fromMillis(3L)); + expectedBasic.put("isRegularFile", true); + expectedBasic.put("isDirectory", true); + expectedBasic.put("isSymbolicLink", true); + expectedBasic.put("isOther", true); + expectedBasic.put("size", 42L); + + assertEquals(expectedBasic, fileSystemProvider.readAttributes(path1, "basic:*")); + + Map expectedGcs = new TreeMap<>(expectedBasic); + // CloudStorageFileAttributes + expectedGcs.put("etag", Optional.of("TheEtag")); + expectedGcs.put("mimeType", Optional.of("TheMimeType")); + expectedGcs.put("acl", Optional.of(acls)); + expectedGcs.put("cacheControl", Optional.of("TheCacheControl")); + expectedGcs.put("contentEncoding", Optional.of("TheContentEncoding")); + expectedGcs.put("contentDisposition", Optional.of("TheContentDisposition")); + expectedGcs.put("userMetadata", new TreeMap<>()); + + assertEquals(expectedGcs, fileSystemProvider.readAttributes(path1, "gcs:*")); + + Map expectedSpecific = new TreeMap<>(); + expectedSpecific.put("lastModifiedTime", FileTime.fromMillis(2L)); + expectedSpecific.put("isSymbolicLink", true); + expectedSpecific.put("isOther", true); + + // Asking for attributes that should NOT be known because we ask for basic view ! + assertEquals( + expectedSpecific, + fileSystemProvider.readAttributes( + path1, "basic:lastModifiedTime,isSymbolicLink,isOther,etag,cacheControl,owner,group")); + + // Add the attributes that are only known in posix view + // These are all fake values + expectedSpecific.put( + "owner", + new UserPrincipal() { + @Override + public String getName() { + return "fakeowner"; + } + + @Override + public String toString() { + return "fakeowner"; + } + }); + + expectedSpecific.put( + "group", + new GroupPrincipal() { + @Override + public String getName() { + return "fakegroup"; + } + + @Override + public String toString() { + return "fakegroup"; + } + }); + + // The equals between two anonymous classes (the UserPrincipal and GroupPrincipal) is always + // false + // so we compare the toString() instead. + assertEquals( + expectedSpecific.toString(), + fileSystemProvider + .readAttributes( + path1, + "posix:lastModifiedTime,isSymbolicLink,isOther,etag,cacheControl,owner,group") + .toString()); + + expectedSpecific.remove("owner"); + expectedSpecific.remove("group"); + + // Add the attributes that are only known in gcs view + expectedSpecific.put("etag", Optional.of("TheEtag")); + expectedSpecific.put("cacheControl", Optional.of("TheCacheControl")); + + assertEquals( + expectedSpecific, + fileSystemProvider.readAttributes( + path1, "gcs:lastModifiedTime,isSymbolicLink,isOther,etag,cacheControl,owner,group")); + } + + private static CloudStorageConfiguration permitEmptyPathComponents(boolean value) { + return CloudStorageConfiguration.builder().permitEmptyPathComponents(value).build(); + } + + private static CloudStorageConfiguration usePseudoDirectories(boolean value) { + return CloudStorageConfiguration.builder().usePseudoDirectories(value).build(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java new file mode 100644 index 000000000000..6c63a7e65eb2 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java @@ -0,0 +1,512 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import com.google.api.gax.rpc.internal.QuotaProjectIdHidingCredentials; +import com.google.auth.Credentials; +import com.google.cloud.NoCredentials; +import com.google.cloud.ServiceOptions; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URI; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageFileSystem}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileSystemTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private static final String ALONE = + "To be, or not to be, that is the question—\n" + + "Whether 'tis Nobler in the mind to suffer\n" + + "The Slings and Arrows of outrageous Fortune,\n" + + "Or to take Arms against a Sea of troubles,\n" + + "And by opposing, end them? To die, to sleep—\n" + + "No more; and by a sleep, to say we end\n" + + "The Heart-ache, and the thousand Natural shocks\n" + + "That Flesh is heir to? 'Tis a consummation\n"; + + @Before + public void before() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void checkDefaultOptions() throws IOException { + // 1. We get the normal default if we don't do anything special. + Path path = Paths.get(URI.create("gs://bucket/file")); + CloudStorageFileSystem gcs = (CloudStorageFileSystem) path.getFileSystem(); + assertThat(gcs.config().maxChannelReopens()).isEqualTo(0); + + // 2(a). Override the default, and see it reflected. + CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration( + CloudStorageConfiguration.builder().maxChannelReopens(123).build()); + Path path2 = Paths.get(URI.create("gs://newbucket/file")); + CloudStorageFileSystem gcs2 = (CloudStorageFileSystem) path2.getFileSystem(); + assertThat(gcs2.config().maxChannelReopens()).isEqualTo(123); + + // 2(b) ...even reflected if we try to open a file. + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + CloudStorageFileSystem csfs = (CloudStorageFileSystem) fs; + assertThat(csfs.config().maxChannelReopens()).isEqualTo(123); + Files.write(fs.getPath("/angel"), ALONE.getBytes(UTF_8)); + path2 = Paths.get(URI.create("gs://bucket/angel")); + try (SeekableByteChannel seekableByteChannel = Files.newByteChannel(path2)) { + CloudStorageReadChannel cloudChannel = (CloudStorageReadChannel) seekableByteChannel; + assertThat(cloudChannel.maxChannelReopens).isEqualTo(123); + } + } + + // 4. Clean up. + CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(null); + Path path3 = Paths.get(URI.create("gs://newbucket/file")); + CloudStorageFileSystem gcs3 = (CloudStorageFileSystem) path3.getFileSystem(); + assertThat(gcs3.config().maxChannelReopens()).isEqualTo(0); + } + + @Test + public void canOverrideDefaultOptions() throws IOException { + // Set a new default. + CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration( + CloudStorageConfiguration.builder().maxChannelReopens(123).build()); + + // This code wants its own value. + try (FileSystem fs = + CloudStorageFileSystem.forBucket( + "bucket", CloudStorageConfiguration.builder().maxChannelReopens(7).build())) { + CloudStorageFileSystem csfs = (CloudStorageFileSystem) fs; + assertThat(csfs.config().maxChannelReopens()).isEqualTo(7); + } + + // Clean up. + CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(null); + } + + @Test + public void testGetPath() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(fs.getPath("/angel").toString()).isEqualTo("/angel"); + assertThat(fs.getPath("/angel").toUri().toString()).isEqualTo("gs://bucket/angel"); + } + } + + @Test + public void testGetHost_valid_dns() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket-with-host")) { + assertThat(fs.getPath("/angel").toUri().getHost()).isEqualTo("bucket-with-host"); + } + } + + @Test + public void testGetHost_invalid_dns() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket_with_authority")) { + assertThat(fs.getPath("/angel").toUri().getHost()).isNull(); + } + } + + @Test + public void testGetAuthority_valid_dns() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket-with-host")) { + assertThat(fs.getPath("/angel").toUri().getAuthority()).isEqualTo("bucket-with-host"); + } + } + + @Test + public void testGetAuthority_invalid_dns() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket_with_authority")) { + assertThat(fs.getPath("/angel").toUri().getAuthority()).isEqualTo("bucket_with_authority"); + } + } + + @Test + public void testToString_valid_dns() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket-with-host")) { + assertThat(fs.toString()).isEqualTo("gs://bucket-with-host"); + } + } + + @Test + public void testToString_invalid_dns() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket_with_authority")) { + assertThat(fs.toString()).isEqualTo("gs://bucket_with_authority"); + } + } + + @Test + public void testWrite() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + Files.write(fs.getPath("/angel"), ALONE.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(Paths.get(URI.create("gs://bucket/angel"))), UTF_8)) + .isEqualTo(ALONE); + } + + @Test + public void testRead() throws IOException { + Files.write(Paths.get(URI.create("gs://bucket/angel")), ALONE.getBytes(UTF_8)); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(new String(Files.readAllBytes(fs.getPath("/angel")), UTF_8)).isEqualTo(ALONE); + } + } + + @Test + public void testExists_false() throws IOException { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"))) { + assertThat(Files.exists(fs.getPath("/angel"))).isFalse(); + } + } + + @Test + public void testExists_true() throws IOException { + Files.write(Paths.get(URI.create("gs://bucket/angel")), ALONE.getBytes(UTF_8)); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(Files.exists(fs.getPath("/angel"))).isTrue(); + } + } + + @Test + public void testGetters() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(fs.isOpen()).isTrue(); + assertThat(fs.isReadOnly()).isFalse(); + assertThat(fs.getRootDirectories()).containsExactly(fs.getPath("/")); + assertThat(fs.getFileStores()).isEmpty(); + assertThat(fs.getSeparator()).isEqualTo("/"); + assertThat(fs.supportedFileAttributeViews()).containsExactly("basic", "gcs", "posix"); + } + } + + @Test + public void testEquals() throws IOException { + try (FileSystem bucket1 = CloudStorageFileSystem.forBucket("bucket"); + FileSystem bucket2 = FileSystems.getFileSystem(URI.create("gs://bucket")); + FileSystem doge1 = CloudStorageFileSystem.forBucket("doge"); + FileSystem doge2 = FileSystems.getFileSystem(URI.create("gs://doge"))) { + new EqualsTester() + .addEqualityGroup(bucket1, bucket2) + .addEqualityGroup(doge1, doge2) + .testEquals(); + } + } + + @Test + public void testNullness() throws IOException, NoSuchMethodException, SecurityException { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"))) { + NullPointerTester tester = + new NullPointerTester() + .ignore(CloudStorageFileSystem.class.getMethod("equals", Object.class)) + .setDefault(CloudStorageConfiguration.class, CloudStorageConfiguration.DEFAULT) + .setDefault(StorageOptions.class, LocalStorageHelper.getOptions()); + tester.testAllPublicStaticMethods(CloudStorageFileSystem.class); + tester.testAllPublicInstanceMethods(fs); + } + } + + @Test + public void testListFiles() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + goodPaths.add(fs.getPath("dir/dir2/")); + for (Path path : paths) { + Files.write(path, ALONE.getBytes(UTF_8)); + } + + List got = new ArrayList<>(); + for (Path path : Files.newDirectoryStream(fs.getPath("/dir/"))) { + got.add(path); + } + assertThat(got).containsExactlyElementsIn(goodPaths); + + // Must also work with relative path + got.clear(); + for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) { + got.add(path); + } + assertThat(got).containsExactlyElementsIn(goodPaths); + } + } + + @Test + public void testMatcher() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + String pattern1 = "glob:*.java"; + PathMatcher javaFileMatcher = fs.getPathMatcher(pattern1); + assertMatches(fs, javaFileMatcher, "a.java", true); + assertMatches(fs, javaFileMatcher, "a.text", false); + assertMatches(fs, javaFileMatcher, "folder/c.java", true); + assertMatches(fs, javaFileMatcher, "d", false); + + String pattern2 = "glob:*.{java,text}"; + PathMatcher javaAndTextFileMatcher = fs.getPathMatcher(pattern2); + assertMatches(fs, javaAndTextFileMatcher, "a.java", true); + assertMatches(fs, javaAndTextFileMatcher, "a.text", true); + assertMatches(fs, javaAndTextFileMatcher, "folder/c.java", true); + assertMatches(fs, javaAndTextFileMatcher, "d", false); + } + } + + @Test + public void testDeleteEmptyFolder() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + List paths = new ArrayList<>(); + paths.add(fs.getPath("dir/angel")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + for (Path path : paths) { + Files.write(path, ALONE.getBytes(UTF_8)); + } + // we can delete non-existent folders, because they are not represented on disk anyways. + Files.delete(fs.getPath("ghost/")); + Files.delete(fs.getPath("dir/ghost/")); + Files.delete(fs.getPath("dir/dir2/ghost/")); + // likewise, deleteIfExists works. + Files.deleteIfExists(fs.getPath("ghost/")); + Files.deleteIfExists(fs.getPath("dir/ghost/")); + Files.deleteIfExists(fs.getPath("dir/dir2/ghost/")); + } + } + + @Test + public void testDeleteFullFolder() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + Files.write(fs.getPath("dir/angel"), ALONE.getBytes(UTF_8)); + // we cannot delete existing folders if they contain something + Files.delete(fs.getPath("dir/")); + Assert.fail(); + } catch (CloudStoragePseudoDirectoryException ex) { + assertThat(ex.getMessage()) + .isEqualTo("Can't perform I/O on pseudo-directories (trailing slash): dir/"); + } + } + + @Test + public void testDelete() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + List paths = new ArrayList<>(); + paths.add(fs.getPath("dir/angel")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + for (Path path : paths) { + Files.write(path, ALONE.getBytes(UTF_8)); + } + Files.delete(fs.getPath("atroot")); + Files.delete(fs.getPath("dir/angel")); + Files.deleteIfExists(fs.getPath("dir/dir2/another_angel")); + + for (Path path : paths) { + assertThat(Files.exists(path)).isFalse(); + } + } + } + + @Test + public void testDeleteEmptiedFolder() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + List paths = new ArrayList<>(); + paths.add(fs.getPath("dir/angel")); + paths.add(fs.getPath("dir/dir2/another_angel")); + for (Path path : paths) { + Files.write(path, ALONE.getBytes(UTF_8)); + } + Files.delete(fs.getPath("dir/angel")); + Files.deleteIfExists(fs.getPath("dir/dir2/another_angel")); + // delete folder (trailing slash is required) + Path dir2 = fs.getPath("dir/dir2/"); + Files.deleteIfExists(dir2); + Path dir = fs.getPath("dir/"); + Files.deleteIfExists(dir); + // We can't check Files.exists on a folder (since GCS fakes folders) + } + } + + @Test + public void testDeleteRecursive() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + List paths = new ArrayList<>(); + paths.add(fs.getPath("atroot")); + paths.add(fs.getPath("dir/angel")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("dir/dir2/angel3")); + paths.add(fs.getPath("dir/dir3/cloud")); + for (Path path : paths) { + Files.write(path, ALONE.getBytes(UTF_8)); + } + + deleteRecursive(fs.getPath("dir/")); + assertThat(Files.exists(fs.getPath("dir/angel"))).isFalse(); + assertThat(Files.exists(fs.getPath("dir/dir3/cloud"))).isFalse(); + assertThat(Files.exists(fs.getPath("atroot"))).isTrue(); + } + } + + @Test + public void testSameProvider() throws IOException { + try (CloudStorageFileSystem sourceFileSystem = + CloudStorageFileSystem.forBucket( + "bucket", + CloudStorageConfiguration.builder().permitEmptyPathComponents(true).build())) { + CloudStorageFileSystem destFileSystem = + CloudStorageFileSystem.forBucket( + "new-bucket", + CloudStorageConfiguration.builder().permitEmptyPathComponents(true).build()); + assertSame(sourceFileSystem.provider(), destFileSystem.provider()); + assertEquals(sourceFileSystem.config(), destFileSystem.config()); + assertEquals("bucket", sourceFileSystem.bucket()); + assertEquals("new-bucket", destFileSystem.bucket()); + } + } + + @Test + public void testDifferentProvider() throws IOException { + try (CloudStorageFileSystem sourceFileSystem = + CloudStorageFileSystem.forBucket( + "bucket", + CloudStorageConfiguration.builder().permitEmptyPathComponents(true).build())) { + CloudStorageFileSystem destFileSystem = + CloudStorageFileSystem.forBucket( + "new-bucket", + CloudStorageConfiguration.builder().permitEmptyPathComponents(false).build()); + assertNotSame(sourceFileSystem.provider(), destFileSystem.provider()); + assertNotEquals(sourceFileSystem.config(), destFileSystem.config()); + assertEquals("bucket", sourceFileSystem.bucket()); + assertEquals("new-bucket", destFileSystem.bucket()); + } + } + + // port of test from + // https://github.com/broadinstitute/cromwell/pull/6491/files#diff-758dbbe823e71cc26fee7bc89cd5c434dfb76e604d51005b8327db59aab96068R300-R336 + @Test + public void ensureMultipleInstancesDoNotCorruptCredentials() throws Exception { + + CloudStorageConfiguration config = + CloudStorageConfiguration.builder() + .permitEmptyPathComponents(true) + .stripPrefixSlash(true) + .usePseudoDirectories(true) + .build(); + + Credentials noCredentials = NoCredentials.getInstance(); + Credentials saCredentials = new QuotaProjectIdHidingCredentials(noCredentials); + + StorageOptions noOptions = + StorageOptions.newBuilder() + .setProjectId("public-project") + .setCredentials(noCredentials) + .build(); + + StorageOptions saOptions = + StorageOptions.newBuilder() + .setProjectId("private-project") + .setCredentials(saCredentials) + .build(); + + CloudStorageFileSystem noFs = + CloudStorageFileSystem.forBucket("public-bucket", config, noOptions); + CloudStorageFileSystem saFs = + CloudStorageFileSystem.forBucket("private-bucket", config, saOptions); + + CloudStoragePath noPath = noFs.getPath("public-file"); + CloudStoragePath saPath = saFs.getPath("private-file"); + + assertThat(credentialsForPath(noPath)).isEqualTo(noCredentials); + assertThat(credentialsForPath(saPath)).isEqualTo(saCredentials); + } + + /** + * Delete the given directory and all of its contents if non-empty. + * + * @param directory the directory to delete + * @throws IOException + */ + private static void deleteRecursive(Path directory) throws IOException { + Files.walkFileTree( + directory, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + + private void assertMatches(FileSystem fs, PathMatcher matcher, String toMatch, boolean expected) { + assertThat(matcher.matches(fs.getPath(toMatch).getFileName())).isEqualTo(expected); + } + + private static Credentials credentialsForPath(Path p) + throws NoSuchFieldException, IllegalAccessException { + CloudStorageFileSystemProvider cloudFilesystemProvider = + (CloudStorageFileSystemProvider) p.getFileSystem().provider(); + Field storageOptionsField = + cloudFilesystemProvider.getClass().getDeclaredField("storageOptions"); + storageOptionsField.setAccessible(true); + StorageOptions storageOptions = + (StorageOptions) storageOptionsField.get(cloudFilesystemProvider); + Field credentialsField = ServiceOptions.class.getDeclaredField("credentials"); + credentialsField.setAccessible(true); + return (Credentials) credentialsField.get(storageOptions); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageIsDirectoryTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageIsDirectoryTest.java new file mode 100644 index 000000000000..e78d6859de6f --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageIsDirectoryTest.java @@ -0,0 +1,112 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.collect.Lists; +import java.nio.file.Files; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@code Files.isDirectory()}. */ +@RunWith(JUnit4.class) +public class CloudStorageIsDirectoryTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + @Rule public final TestName testName = new TestName(); + + private StorageOptions mockOptions; + private Storage mockStorage; + + @Before + public void before() { + mockOptions = + mock( + StorageOptions.class, + String.format("storage-options-mock_%s", testName.getMethodName())); + mockStorage = mock(Storage.class); + when(mockOptions.getService()).thenReturn(mockStorage); + CloudStorageFileSystemProvider.setStorageOptions(mockOptions); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void testIsDirectoryNoUserProject() { + CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("bucket", CloudStorageConfiguration.DEFAULT, mockOptions); + when(mockStorage.get(BlobId.of("bucket", "test", null))) + .thenThrow(new IllegalArgumentException()); + Page pages = mock(Page.class); + Blob blob = mock(Blob.class); + when(blob.getBlobId()).thenReturn(BlobId.of("bucket", "test/hello.txt")); + when(pages.getValues()).thenReturn(Lists.newArrayList(blob)); + when(mockStorage.list( + "bucket", Storage.BlobListOption.prefix("test/"), Storage.BlobListOption.pageSize(1))) + .thenReturn(pages); + + Files.isDirectory(fs.getPath("test")); + verify(mockStorage, times(1)) + .list("bucket", Storage.BlobListOption.prefix("test/"), Storage.BlobListOption.pageSize(1)); + } + + @Test + public void testIsDirectoryWithUserProject() { + CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket( + "bucket", + CloudStorageConfiguration.builder().userProject("project-id").build(), + mockOptions); + when(mockStorage.get(BlobId.of("bucket", "test", null))) + .thenThrow(new IllegalArgumentException()); + Page pages = mock(Page.class); + Blob blob = mock(Blob.class); + when(blob.getBlobId()).thenReturn(BlobId.of("bucket", "test/hello.txt")); + when(pages.getValues()).thenReturn(Lists.newArrayList(blob)); + when(mockStorage.list( + "bucket", + Storage.BlobListOption.prefix("test/"), + Storage.BlobListOption.pageSize(1), + Storage.BlobListOption.userProject("project-id"))) + .thenReturn(pages); + Files.isDirectory(fs.getPath("test")); + verify(mockStorage, times(1)) + .list( + "bucket", + Storage.BlobListOption.prefix("test/"), + Storage.BlobListOption.pageSize(1), + Storage.BlobListOption.userProject("project-id")); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageLateInitializationTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageLateInitializationTest.java new file mode 100644 index 000000000000..f0cecea01556 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageLateInitializationTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.net.URI; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +/** Unit tests for {@link CloudStorageFileSystemProvider} late initialization. */ +@RunWith(MockitoJUnitRunner.class) +public class CloudStorageLateInitializationTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + @Spy private final CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + + @Test + public void ctorDoesNotCreateStorage() { + verify(provider, never()).doInitStorage(); + } + + @Test + public void getPathCreatesStorageOnce() { + provider.getPath(URI.create("gs://bucket1/wat")); + provider.getPath(URI.create("gs://bucket2/wat")); + verify(provider, times(1)).doInitStorage(); + } + + @Test + public void getFileSystemCreatesStorageOnce() { + provider.getFileSystem(URI.create("gs://bucket1")); + provider.getFileSystem(URI.create("gs://bucket2")); + verify(provider, times(1)).doInitStorage(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageOptionsTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageOptionsTest.java new file mode 100644 index 000000000000..f541e9031c7a --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageOptionsTest.java @@ -0,0 +1,129 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.testing.NullPointerTester; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageOptions}. */ +@RunWith(JUnit4.class) +public class CloudStorageOptionsTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + @Before + public void before() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void testWithoutCaching() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withoutCaching()); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("no-cache"); + } + + @Test + public void testCacheControl() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withCacheControl("potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("potato"); + } + + @Test + public void testWithAcl() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Acl acl = Acl.of(new Acl.User("king@example.com"), Acl.Role.OWNER); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withAcl(acl)); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).acl().get()) + .contains(acl); + } + + @Test + public void testWithContentDisposition() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write( + path, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withContentDisposition("bubbly fun")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).contentDisposition().get()) + .isEqualTo("bubbly fun"); + } + + @Test + public void testWithContentEncoding() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withContentEncoding("gzip")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).contentEncoding().get()) + .isEqualTo("gzip"); + } + + @Test + public void testWithUserMetadata() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write( + path, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withUserMetadata("nolo", "contendere"), + CloudStorageOptions.withUserMetadata("eternal", "sadness")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).userMetadata().get("nolo")) + .isEqualTo("contendere"); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class) + .userMetadata() + .get("eternal")) + .isEqualTo("sadness"); + } + + @Test + public void testWithMimeType_string() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withMimeType("text/plain")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).mimeType().get()) + .isEqualTo("text/plain"); + } + + @Test + public void testNullness() { + NullPointerTester tester = new NullPointerTester(); + tester.testAllPublicStaticMethods(CloudStorageOptions.class); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStoragePathTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStoragePathTest.java new file mode 100644 index 000000000000..2a2d47bb7a9b --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStoragePathTest.java @@ -0,0 +1,547 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.collect.Iterables; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; +import java.io.IOException; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStoragePath}. */ +@RunWith(JUnit4.class) +public class CloudStoragePathTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + @Before + public void before() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void testCreate_neverRemoveExtraSlashes() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("lol//cat").toString()).isEqualTo("lol//cat"); + assertThat((Object) fs.getPath("lol//cat")).isEqualTo(fs.getPath("lol//cat")); + } + } + + @Test + public void testCreate_preservesTrailingSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("lol/cat/").toString()).isEqualTo("lol/cat/"); + assertThat((Object) fs.getPath("lol/cat/")).isEqualTo(fs.getPath("lol/cat/")); + } + } + + @Test + public void testGetGcsFilename_empty_notAllowed() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("").getBlobId(); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isEqualTo("Object names cannot be empty."); + } + } + + @Test + public void testGetGcsFilename_stripsPrefixSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi").getBlobId().getName()).isEqualTo("hi"); + } + } + + @Test + public void testGetGcsFilename_overrideStripPrefixSlash_doesntStripPrefixSlash() + throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", stripPrefixSlash(false))) { + assertThat(fs.getPath("/hi").getBlobId().getName()).isEqualTo("/hi"); + } + } + + @Test + public void testGetGcsFilename_extraSlashes_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("a//b").getBlobId().getName(); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testGetGcsFilename_overridepermitEmptyPathComponents() throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("a//b").getBlobId().getName()).isEqualTo("a//b"); + } + } + + @Test + public void testGetGcsFilename_freaksOutOnExtraSlashesAndDotDirs() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("a//b/..").getBlobId().getName(); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testNameCount() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").getNameCount()).isEqualTo(1); + assertThat(fs.getPath("/").getNameCount()).isEqualTo(0); + assertThat(fs.getPath("/hi/").getNameCount()).isEqualTo(1); + assertThat(fs.getPath("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(fs.getPath("hi/yo").getNameCount()).isEqualTo(2); + } + } + + @Test + public void testGetName() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").getName(0).toString()).isEqualTo(""); + assertThat(fs.getPath("/hi").getName(0).toString()).isEqualTo("hi"); + assertThat(fs.getPath("hi/there").getName(1).toString()).isEqualTo("there"); + } + } + + @Test + public void testGetName_negative_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("angel").getName(-1); + Assert.fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testGetName_overflow_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("angel").getName(1); + Assert.fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testIterator() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(Iterables.get(fs.getPath("/dog/mog"), 0).toString()).isEqualTo("dog"); + assertThat(Iterables.get(fs.getPath("/dog/mog"), 1).toString()).isEqualTo("mog"); + assertThat(Iterables.size(fs.getPath("/"))).isEqualTo(0); + assertThat(Iterables.size(fs.getPath(""))).isEqualTo(1); + assertThat(Iterables.get(fs.getPath(""), 0).toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/").normalize().toString()).isEqualTo("/"); + assertThat(fs.getPath("a/x/../b/x/..").normalize().toString()).isEqualTo("a/b/"); + assertThat(fs.getPath("/x/x/../../♡").normalize().toString()).isEqualTo("/♡"); + assertThat(fs.getPath("/x/x/./.././.././♡").normalize().toString()).isEqualTo("/♡"); + } + } + + @Test + public void testNormalize_dot_becomesBlank() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").normalize().toString()).isEqualTo(""); + assertThat(fs.getPath(".").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_trailingSlash_isPreserved() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("o/").normalize().toString()).isEqualTo("o/"); + } + } + + @Test + public void testNormalize_doubleDot_becomesBlank() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("..").normalize().toString()).isEqualTo(""); + assertThat(fs.getPath("../..").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_extraSlashes_getRemoved() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("//life///b/good//").normalize().toString()).isEqualTo("/life/b/good/"); + } + } + + @Test + public void testToRealPath_hasDotDir_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("a/hi./b").toRealPath(); + fs.getPath("a/.hi/b").toRealPath(); + fs.getPath("a/./b").toRealPath(); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testToRealPath_hasDotDotDir_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("a/hi../b").toRealPath(); + fs.getPath("a/..hi/b").toRealPath(); + fs.getPath("a/../b").toRealPath(); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()) + .contains("I/O not allowed on dot-dirs or extra slashes when !permitEmptyPathComponents"); + } + } + + @Test + public void testToRealPath_extraSlashes_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("a//b").toRealPath(); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()) + .contains("I/O not allowed on dot-dirs or extra slashes when !permitEmptyPathComponents"); + } + } + + @Test + public void testToRealPath_overridePermitEmptyPathComponents_extraSlashes_slashesRemain() + throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("/life///b/./good/").toRealPath().toString()) + .isEqualTo("life///b/./good/"); + } + } + + @Test + public void testToRealPath_permitEmptyPathComponents_doesNotNormalize() throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("a"); + assertThat(fs.getPath("a//b").toRealPath().toString()).isEqualTo("a//b"); + assertThat(fs.getPath("a//./b//..").toRealPath().toString()).isEqualTo("a//./b//.."); + } + } + + @Test + public void testToRealPath_withWorkingDirectory_makesAbsolute() throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", workingDirectory("/lol"))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("lol/a"); + } + } + + @Test + public void testToRealPath_disableStripPrefixSlash_makesPathAbsolute() throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", stripPrefixSlash(false))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("/a"); + assertThat(fs.getPath("/a").toRealPath().toString()).isEqualTo("/a"); + } + } + + @Test + public void testToRealPath_trailingSlash_getsPreserved() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("a/b/").toRealPath().toString()).isEqualTo("a/b/"); + } + } + + @Test + public void testNormalize_empty_returnsEmpty() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_preserveTrailingSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("a/b/../c/").normalize().toString()).isEqualTo("a/c/"); + assertThat(fs.getPath("a/b/./c/").normalize().toString()).isEqualTo("a/b/c/"); + } + } + + @Test + public void testGetParent_preserveTrailingSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("a/b/c").getParent().toString()).isEqualTo("a/b/"); + assertThat(fs.getPath("a/b/c/").getParent().toString()).isEqualTo("a/b/"); + assertThat((Object) fs.getPath("").getParent()).isNull(); + assertThat((Object) fs.getPath("/").getParent()).isNull(); + assertThat((Object) fs.getPath("aaa").getParent()).isNull(); + assertThat((Object) (fs.getPath("aaa/").getParent())).isNull(); + } + } + + @Test + public void testGetRoot() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hello").getRoot().toString()).isEqualTo("/"); + assertThat((Object) fs.getPath("hello").getRoot()).isNull(); + } + } + + @Test + public void testRelativize() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat( + fs.getPath("/foo/bar/lol/cat").relativize(fs.getPath("/foo/a/b/../../c")).toString()) + .isEqualTo("../../../a/b/../../c"); + } + } + + @Test + public void testRelativize_providerMismatch() throws IOException { + try (CloudStorageFileSystem gcs = CloudStorageFileSystem.forBucket("doodle")) { + gcs.getPath("/etc").relativize(FileSystems.getDefault().getPath("/dog")); + Assert.fail(); + } catch (ProviderMismatchException ex) { + assertThat(ex.getMessage()).contains("Not a Cloud Storage path"); + } + } + + @Test + @SuppressWarnings("ReturnValueIgnored") // testing that an Exception is thrown + public void testRelativize_providerMismatch2() throws IOException { + try (CloudStorageFileSystem gcs = CloudStorageFileSystem.forBucket("doodle")) { + gcs.getPath("/dog").relativize(FileSystems.getDefault().getPath("/etc")); + Assert.fail(); + } catch (ProviderMismatchException ex) { + assertThat(ex.getMessage()).contains("Not a Cloud Storage path"); + } + } + + @Test + public void testResolve() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi").resolve("there").toString()).isEqualTo("/hi/there"); + assertThat(fs.getPath("hi").resolve("there").toString()).isEqualTo("hi/there"); + } + } + + @Test + public void testResolve_providerMismatch() throws IOException { + try (CloudStorageFileSystem gcs = CloudStorageFileSystem.forBucket("doodle")) { + gcs.getPath("etc").resolve(FileSystems.getDefault().getPath("/dog")); + Assert.fail(); + } catch (ProviderMismatchException ex) { + assertThat(ex.getMessage()).contains("Not a Cloud Storage path"); + } + } + + @Test + public void testIsAbsolute() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi").isAbsolute()).isTrue(); + assertThat(fs.getPath("hi").isAbsolute()).isFalse(); + } + } + + @Test + public void testToAbsolutePath() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat((Object) fs.getPath("/hi").toAbsolutePath()).isEqualTo(fs.getPath("/hi")); + assertThat((Object) fs.getPath("hi").toAbsolutePath()).isEqualTo(fs.getPath("/hi")); + } + } + + @Test + public void testToAbsolutePath_withWorkingDirectory() throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", workingDirectory("/lol"))) { + assertThat(fs.getPath("a").toAbsolutePath().toString()).isEqualTo("/lol/a"); + } + } + + @Test + public void testGetFileName() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").getFileName().toString()).isEqualTo("there"); + assertThat(fs.getPath("military/fashion/show").getFileName().toString()).isEqualTo("show"); + } + } + + @Test + public void testCompareTo() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/there"))).isEqualTo(0); + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/therf"))).isEqualTo(-1); + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/therd"))).isEqualTo(1); + } + } + + @Test + public void testStartsWith() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/there"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("hi"))).isFalse(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath(""))).isFalse(); + } + } + + @Test + public void testEndsWith() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("there"))).isTrue(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/blag/therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/hi/there"))).isTrue(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/there"))).isFalse(); + assertThat(fs.getPath("/human/that/you/cry").endsWith(fs.getPath("that/you/cry"))).isTrue(); + assertThat(fs.getPath("/human/that/you/cry").endsWith(fs.getPath("that/you/cry/"))).isTrue(); + assertThat(fs.getPath("/hi/there/").endsWith(fs.getPath("/"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath(""))).isFalse(); + assertThat(fs.getPath("").endsWith(fs.getPath(""))).isTrue(); + } + } + + @Test + public void testResolve_willWorkWithRecursiveCopy() throws IOException { + // See: http://stackoverflow.com/a/10068306 + try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); + FileSystem fsTarget = FileSystems.getFileSystem(URI.create("gs://cat"))) { + Path targetPath = fsTarget.getPath("/some/folder/"); + Path relSrcPath = fsSource.getPath("file.txt"); + assertThat((Object) targetPath.resolve(relSrcPath)) + .isEqualTo(fsTarget.getPath("/some/folder/file.txt")); + } + } + + @Test + public void testRelativize_willWorkWithRecursiveCopy() throws IOException { + // See: http://stackoverflow.com/a/10068306 + try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); + FileSystem fsTarget = FileSystems.getFileSystem(URI.create("gs://cat"))) { + Path targetPath = fsTarget.getPath("/some/folder/"); + Path sourcePath = fsSource.getPath("/sloth/"); + Path file = fsSource.getPath("/sloth/file.txt"); + assertThat((Object) targetPath.resolve(sourcePath.relativize(file))) + .isEqualTo(fsTarget.getPath("/some/folder/file.txt")); + } + } + + @Test + public void testToFile_unsupported() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + Path path = fs.getPath("/lol"); + path.toFile(); + Assert.fail(); + } catch (UnsupportedOperationException ex) { + assertThat(ex.getMessage()).isEqualTo("GCS objects aren't available locally"); + } + } + + @Test + public void testEquals() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + new EqualsTester() + // These are obviously equal. + .addEqualityGroup(fs.getPath("/hello/cat"), fs.getPath("/hello/cat")) + // These are equal because equals() runs things through toRealPath() + .addEqualityGroup(fs.getPath("great/commandment"), fs.getPath("/great/commandment")) + .addEqualityGroup(fs.getPath("great/commandment/"), fs.getPath("/great/commandment/")) + // Equals shouldn't do error checking or normalization. + .addEqualityGroup(fs.getPath("foo/../bar"), fs.getPath("foo/../bar")) + .addEqualityGroup(fs.getPath("bar")) + .testEquals(); + } + } + + @Test + public void testEquals_currentDirectoryIsTakenIntoConsideration() throws IOException { + try (CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket("doodle", workingDirectory("/hello"))) { + new EqualsTester() + .addEqualityGroup(fs.getPath("cat"), fs.getPath("/hello/cat")) + .addEqualityGroup(fs.getPath(""), fs.getPath("/hello")) + .testEquals(); + } + } + + @Test + public void testNullness() throws IOException, NoSuchMethodException, SecurityException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + NullPointerTester tester = new NullPointerTester(); + tester.ignore(CloudStoragePath.class.getMethod("equals", Object.class)); + tester.setDefault(Path.class, fs.getPath("sup")); + tester.testAllPublicStaticMethods(CloudStoragePath.class); + tester.testAllPublicInstanceMethods(fs.getPath("sup")); + } + } + + @Test + public void testSpaces() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + Path path = fs.getPath("/with/a space"); + // we can also go via a URI. Decoding should give us the space back. + String toUri = URLDecoder.decode(path.toUri().toString(), "UTF-8"); + assertThat(toUri).isEqualTo("gs://doodle/with/a space"); + + Path path2 = fs.getPath("/with/a%20percent"); + String toUri2 = URLDecoder.decode(path2.toUri().toString(), "UTF-8"); + assertThat(toUri2).isEqualTo("gs://doodle/with/a%20percent"); + } + } + + private static CloudStorageConfiguration stripPrefixSlash(boolean value) { + return CloudStorageConfiguration.builder().stripPrefixSlash(value).build(); + } + + private static CloudStorageConfiguration permitEmptyPathComponents(boolean value) { + return CloudStorageConfiguration.builder().permitEmptyPathComponents(value).build(); + } + + private static CloudStorageConfiguration workingDirectory(String value) { + return CloudStorageConfiguration.builder().workingDirectory(value).build(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java new file mode 100644 index 000000000000..7b827d2710c0 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java @@ -0,0 +1,267 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.cloud.ReadChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import javax.net.ssl.SSLHandshakeException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; + +/** Unit tests for {@link CloudStorageReadChannel}. */ +@RunWith(JUnit4.class) +public class CloudStorageReadChannelTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private CloudStorageReadChannel chan; + + private final Storage gcsStorage = mock(Storage.class); + private final BlobId file = BlobId.of("blob", "attack"); + private final Blob metadata = mock(Blob.class); + private final ReadChannel gcsChannel = mock(ReadChannel.class); + + @Before + public void before() throws IOException { + when(metadata.getSize()).thenReturn(42L); + when(metadata.getGeneration()).thenReturn(2L); + when(gcsStorage.get( + file, + Storage.BlobGetOption.fields(Storage.BlobField.GENERATION, Storage.BlobField.SIZE))) + .thenReturn(metadata); + when(gcsStorage.reader(file, Storage.BlobSourceOption.generationMatch(2L))) + .thenReturn(gcsChannel); + when(gcsChannel.isOpen()).thenReturn(true); + chan = + CloudStorageReadChannel.create( + gcsStorage, file, 0, 1, CloudStorageConfiguration.DEFAULT, ""); + verify(gcsStorage) + .get( + eq(file), + eq(Storage.BlobGetOption.fields(Storage.BlobField.GENERATION, Storage.BlobField.SIZE))); + verify(gcsStorage).reader(eq(file), eq(Storage.BlobSourceOption.generationMatch(2L))); + } + + @Test + public void testRead() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + when(gcsChannel.read(eq(buffer))).thenReturn(1); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.read(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + verify(gcsChannel).read(any(ByteBuffer.class)); + verify(gcsChannel, times(3)).isOpen(); + } + + @Test + public void testReadRetry() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + when(gcsChannel.read(eq(buffer))) + .thenThrow( + new StorageException( + new IOException( + "outer", + new IOException( + "Connection closed prematurely: bytesRead = 33554432, Content-Length = 41943040")))) + .thenReturn(1); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.read(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + verify(gcsChannel, times(2)).read(any(ByteBuffer.class)); + } + + @Test + public void testReadRetrySSLHandshake() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + when(gcsChannel.read(eq(buffer))) + .thenThrow( + new StorageException( + new IOException( + "something", + new IOException( + "thing", + new SSLHandshakeException("connection closed due to throttling"))))) + .thenReturn(1); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.read(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + verify(gcsChannel, times(2)).read(any(ByteBuffer.class)); + } + + @Test + public void testReadRetryEventuallyGivesUp() throws IOException { + try { + ByteBuffer buffer = ByteBuffer.allocate(1); + when(gcsChannel.read(eq(buffer))) + .thenThrow( + new StorageException( + new IOException( + "Connection closed prematurely: bytesRead = 33554432, Content-Length = 41943040"))) + .thenThrow( + new StorageException( + new IOException( + "Connection closed prematurely: bytesRead = 33554432, Content-Length = 41943040"))) + .thenReturn(1); + assertThat(chan.position()).isEqualTo(0L); + chan.read(buffer); + Assert.fail(); + } catch (StorageException ex) { + assertThat(ex.getMessage()).isNotNull(); + } + } + + @Test + public void testRead_whenClosed_throwsCce() throws IOException { + try { + when(gcsChannel.isOpen()).thenReturn(false); + chan.read(ByteBuffer.allocate(1)); + Assert.fail(); + } catch (ClosedChannelException expected) { + } + } + + @Test + public void testWrite_throwsNonWritableChannelException() throws IOException { + try { + chan.write(ByteBuffer.allocate(1)); + Assert.fail(); + } catch (NonWritableChannelException expected) { + } + } + + @Test + public void testTruncate_throwsNonWritableChannelException() throws IOException { + try { + chan.truncate(0); + Assert.fail(); + } catch (NonWritableChannelException ex) { + assertThat(ex.getClass()).isEqualTo(NonWritableChannelException.class); + } + } + + @Test + public void testIsOpen() throws IOException { + when(gcsChannel.isOpen()).thenReturn(true).thenReturn(false); + assertThat(chan.isOpen()).isTrue(); + chan.close(); + assertThat(chan.isOpen()).isFalse(); + verify(gcsChannel, times(2)).isOpen(); + verify(gcsChannel).close(); + } + + @Test + public void testSize() throws IOException { + assertThat(chan.size()).isEqualTo(42L); + verify(gcsChannel).isOpen(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testSize_whenClosed_throwsCce() throws IOException { + try { + when(gcsChannel.isOpen()).thenReturn(false); + chan.size(); + Assert.fail(); + } catch (ClosedChannelException ex) { + assertThat(ex.getClass()).isEqualTo(ClosedChannelException.class); + } + } + + @Test + public void testPosition_whenClosed_throwsCce() throws IOException { + try { + when(gcsChannel.isOpen()).thenReturn(false); + chan.position(); + Assert.fail(); + } catch (ClosedChannelException ex) { + assertThat(ex.getClass()).isEqualTo(ClosedChannelException.class); + } + } + + @Test + public void testSetPosition_whenClosed_throwsCce() throws IOException { + try { + when(gcsChannel.isOpen()).thenReturn(false); + chan.position(0); + Assert.fail(); + } catch (ClosedChannelException ex) { + assertThat(ex.getClass()).isEqualTo(ClosedChannelException.class); + } + } + + @Test + public void testClose_calledMultipleTimes_doesntThrowAnError() throws IOException { + chan.close(); + chan.close(); + chan.close(); + } + + @Test + public void testSetPosition() throws IOException { + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.size()).isEqualTo(42L); + chan.position(1L); + assertThat(chan.position()).isEqualTo(1L); + assertThat(chan.size()).isEqualTo(42L); + verify(gcsChannel).seek(1); + verify(gcsChannel, times(5)).isOpen(); + } + + /* + * This test case was crafted in response to a bug in CloudStorageReadChannel in which the + * channel position (a long) was getting truncated to an int when seeking on the encapsulated + * ReadChannel in innerOpen(). This test case fails when the bad long -> int cast is present, + * and passes when it's removed. + */ + @Test + public void testChannelPositionDoesNotGetTruncatedToInt() throws IOException { + // This position value will overflow to a negative value if a long -> int cast is attempted + long startPosition = 11918483280L; + ArgumentCaptor captor = ArgumentCaptor.forClass(Long.class); + + // Invoke CloudStorageReadChannel.create() to trigger a call to the private + // CloudStorageReadChannel.innerOpen() method, which does a seek on our gcsChannel. + CloudStorageReadChannel.create( + gcsStorage, file, startPosition, 1, CloudStorageConfiguration.DEFAULT, ""); + + // Confirm that our position did not overflow during the seek in + // CloudStorageReadChannel.innerOpen() + verify(gcsChannel).seek(captor.capture()); + assertThat(captor.getValue()).isEqualTo(startPosition); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadFileChannelTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadFileChannelTest.java new file mode 100644 index 000000000000..6bcfaa6859df --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadFileChannelTest.java @@ -0,0 +1,185 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CloudStorageReadFileChannelTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private static final class SeekableByteChannelImpl implements SeekableByteChannel { + private boolean open = true; + private ByteBuffer data; + + private SeekableByteChannelImpl(ByteBuffer data) { + this.data = data; + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() throws IOException { + open = false; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + byte[] tmp = new byte[Math.min(dst.remaining(), data.remaining())]; + if (tmp.length == 0) { + return -1; + } else { + data.get(tmp); + dst.put(tmp); + return tmp.length; + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + int res = src.remaining(); + if (data.position() + res > data.limit()) { + data.limit(data.limit() + res); + } + data.put(src); + return res; + } + + @Override + public long position() throws IOException { + return data.position(); + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + if (newPosition >= data.limit()) { + data.limit((int) newPosition); + } + data.position((int) newPosition); + return this; + } + + @Override + public long size() throws IOException { + return data.limit(); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + if (size < data.limit()) { + if (data.position() >= size) { + data.position((int) size - 1); + } + data.limit((int) size); + } + return this; + } + } + + private CloudStorageReadFileChannel fileChannel; + private SeekableByteChannel readChannel; + private ByteBuffer data; + + @Before + public void before() throws IOException { + data = ByteBuffer.allocate(5000); + data.limit(3); + data.put(new byte[] {1, 2, 3}); + data.position(0); + readChannel = new SeekableByteChannelImpl(data); + fileChannel = new CloudStorageReadFileChannel(readChannel); + } + + @Test + public void testRead() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.read(buffer)).isEqualTo(1); + assertThat(fileChannel.position()).isEqualTo(1L); + assertThat(buffer.get(0)).isEqualTo((byte) 1); + } + + @Test + public void testReadArray() throws IOException { + ByteBuffer[] buffers = + new ByteBuffer[] {ByteBuffer.allocate(1), ByteBuffer.allocate(1), ByteBuffer.allocate(1)}; + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.read(buffers)).isEqualTo(3L); + assertThat(fileChannel.position()).isEqualTo(3L); + assertThat(buffers[0].get(0)).isEqualTo((byte) 1); + assertThat(buffers[1].get(0)).isEqualTo((byte) 2); + assertThat(buffers[2].get(0)).isEqualTo((byte) 3); + } + + @Test + public void testPosition() throws IOException { + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.position(1L)).isEqualTo(fileChannel); + assertThat(fileChannel.position()).isEqualTo(1L); + assertThat(fileChannel.position(0L)).isEqualTo(fileChannel); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.position(100L)).isEqualTo(fileChannel); + assertThat(fileChannel.position()).isEqualTo(100L); + } + + @Test + public void testSize() throws IOException { + assertThat(fileChannel.size()).isEqualTo(3L); + } + + @Test + public void testTransferTo() throws IOException { + SeekableByteChannelImpl target = new SeekableByteChannelImpl(ByteBuffer.allocate(100)); + assertThat(fileChannel.transferTo(0L, 3L, target)).isEqualTo(3L); + assertThat(target.position()).isEqualTo(3L); + ByteBuffer dst = ByteBuffer.allocate(3); + target.position(0L); + target.read(dst); + assertThat(dst.get(0)).isEqualTo((byte) 1); + assertThat(dst.get(1)).isEqualTo((byte) 2); + assertThat(dst.get(2)).isEqualTo((byte) 3); + } + + @Test + public void testReadOnPosition() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.read(buffer, 1L)).isEqualTo(1); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(buffer.get(0)).isEqualTo((byte) 2); + } + + @Test + public void testReadBeyondEnd() throws IOException { + fileChannel.position(3L); + assertThat(fileChannel.read(ByteBuffer.allocate(1))).isEqualTo(-1); + fileChannel.position(2L); + assertThat(fileChannel.read(ByteBuffer.allocate(2))).isEqualTo(1); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadTest.java new file mode 100644 index 000000000000..a202cc3fb408 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2017 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageFileSystem}. */ +@RunWith(JUnit4.class) +public class CloudStorageReadTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private static final String ALONE = + "To be, or not to be, that is the question—\n" + + "Whether 'tis Nobler in the mind to suffer\n" + + "The Slings and Arrows of outrageous Fortune,\n" + + "Or to take Arms against a Sea of troubles,\n" + + "And by opposing, end them? To die, to sleep—\n" + + "No more; and by a sleep, to say we end\n" + + "The Heart-ache, and the thousand Natural shocks\n" + + "That Flesh is heir to? 'Tis a consummation\n"; + + // Large enough value that we write more than one "chunk", for interesting behavior. + private static final int repeat = 10000; + + @Before + public void before() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + @Test + public void testInputStreamReads() throws IOException, InterruptedException { + // fill in the file + byte[] bytes = ALONE.getBytes(UTF_8); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + Path p = fillFile(fs, bytes, repeat); + + try (InputStream is = Files.newInputStream(p)) { + byte[] buf = new byte[bytes.length]; + for (int i = 0; i < repeat; i++) { + Arrays.fill(buf, (byte) 0); + for (int off = 0; off < bytes.length; ) { + int delta = is.read(buf, off, bytes.length - off); + if (delta < 0) { + // EOF + break; + } + off += delta; + } + assertWithMessage("Wrong bytes from input stream at repeat " + i) + .that(new String(buf, UTF_8)) + .isEqualTo(ALONE); + } + // reading past the end + int eof = is.read(buf, 0, 1); + assertWithMessage("EOF should return -1").that(eof).isEqualTo(-1); + } finally { + // clean up + Files.delete(p); + } + } + } + + @Test + public void testChannelReads() throws IOException, InterruptedException { + // fill in the file + byte[] bytes = ALONE.getBytes(UTF_8); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + Path p = fillFile(fs, bytes, repeat); + + try (SeekableByteChannel chan = Files.newByteChannel(p, StandardOpenOption.READ)) { + ByteBuffer buf = ByteBuffer.allocate(bytes.length); + for (int i = 0; i < repeat; i++) { + buf.clear(); + for (int off = 0; off < bytes.length; ) { + int read = chan.read(buf); + if (read < 0) { + // EOF + break; + } + off += read; + } + assertWithMessage("Wrong bytes from channel at repeat " + i) + .that(new String(buf.array(), UTF_8)) + .isEqualTo(ALONE); + } + // reading past the end + buf.clear(); + int eof = chan.read(buf); + assertWithMessage("EOF should return -1").that(eof).isEqualTo(-1); + } finally { + // clean up + Files.delete(p); + } + } + } + + private Path fillFile(FileSystem fs, byte[] bytes, int repeat) throws IOException { + Path p = fs.getPath("/alone"); + try (OutputStream os = Files.newOutputStream(p)) { + for (int i = 0; i < repeat; i++) { + os.write(bytes); + } + } + assertThat(Files.size(p) == repeat * bytes.length); + return p; + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandlerTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandlerTest.java new file mode 100644 index 000000000000..a8c957d83b25 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandlerTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.StorageException; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.collect.ImmutableList; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CloudStorageRetryHandlerTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + @Test + public void testIsRetryable() throws Exception { + CloudStorageConfiguration config = + CloudStorageConfiguration.builder().retryableHttpCodes(ImmutableList.of(1, 2, 3)).build(); + CloudStorageRetryHandler retrier = new CloudStorageRetryHandler(config); + assertThat(retrier.isRetryable(new StorageException(1, "pretend error code 1"))).isTrue(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannelTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannelTest.java new file mode 100644 index 000000000000..a098e10f96e2 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannelTest.java @@ -0,0 +1,132 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.cloud.WriteChannel; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonReadableChannelException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageWriteChannel}. */ +@RunWith(JUnit4.class) +public class CloudStorageWriteChannelTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private final WriteChannel gcsChannel = mock(WriteChannel.class); + private final CloudStorageWriteChannel chan = new CloudStorageWriteChannel(gcsChannel); + + @Before + public void before() { + when(gcsChannel.isOpen()).thenReturn(true); + } + + @Test + public void testRead_throwsNonReadableChannelException() throws IOException { + try { + chan.read(ByteBuffer.allocate(1)); + Assert.fail(); + } catch (NonReadableChannelException expected) { + } + } + + @Test + public void testWrite() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 'B'); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.size()).isEqualTo(0L); + when(gcsChannel.write(eq(buffer))).thenReturn(1); + assertThat(chan.write(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + assertThat(chan.size()).isEqualTo(1L); + verify(gcsChannel).write(any(ByteBuffer.class)); + verify(gcsChannel, times(5)).isOpen(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testWrite_whenClosed_throwsCce() throws IOException { + try { + when(gcsChannel.isOpen()).thenReturn(false); + chan.write(ByteBuffer.allocate(1)); + Assert.fail(); + } catch (ClosedChannelException expected) { + } + } + + @Test + public void testIsOpen() throws IOException { + when(gcsChannel.isOpen()).thenReturn(true).thenReturn(false); + assertThat(chan.isOpen()).isTrue(); + chan.close(); + assertThat(chan.isOpen()).isFalse(); + verify(gcsChannel, times(2)).isOpen(); + verify(gcsChannel).close(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testSize() throws IOException { + assertThat(chan.size()).isEqualTo(0L); + verify(gcsChannel).isOpen(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testSize_whenClosed_throwsCce() throws IOException { + try { + when(gcsChannel.isOpen()).thenReturn(false); + chan.size(); + Assert.fail(); + } catch (ClosedChannelException expected) { + } + } + + @Test + public void testPosition_whenClosed_throwsCce() throws IOException { + try { + when(gcsChannel.isOpen()).thenReturn(false); + chan.position(); + Assert.fail(); + } catch (ClosedChannelException expected) { + } + } + + @Test + public void testClose_calledMultipleTimes_doesntThrowAnError() throws IOException { + chan.close(); + chan.close(); + chan.close(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteFileChannelTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteFileChannelTest.java new file mode 100644 index 000000000000..f0e3a4daaca1 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteFileChannelTest.java @@ -0,0 +1,200 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CloudStorageWriteFileChannelTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private static final class SeekableByteChannelImpl implements SeekableByteChannel { + private boolean open = true; + private ByteBuffer data; + + private SeekableByteChannelImpl(ByteBuffer data) { + this.data = data; + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() throws IOException { + open = false; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + byte[] tmp = new byte[Math.min(dst.remaining(), data.remaining())]; + if (tmp.length == 0) { + return -1; + } else { + data.get(tmp); + dst.put(tmp); + return tmp.length; + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + int res = src.remaining(); + if (data.position() + res > data.limit()) { + data.limit(data.limit() + res); + } + data.put(src); + return res; + } + + @Override + public long position() throws IOException { + return data.position(); + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + if (newPosition >= data.limit()) { + data.limit((int) newPosition); + } + data.position((int) newPosition); + return this; + } + + @Override + public long size() throws IOException { + return data.limit(); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + if (size < data.limit()) { + if (data.position() >= size) { + data.position((int) size - 1); + } + data.limit((int) size); + } + return this; + } + } + + private CloudStorageWriteFileChannel fileChannel; + private SeekableByteChannel writeChannel; + private ByteBuffer data; + + @Before + public void before() throws IOException { + data = ByteBuffer.allocate(5000); + data.limit(3); + data.put(new byte[] {1, 2, 3}); + data.position(0); + writeChannel = new SeekableByteChannelImpl(data); + fileChannel = new CloudStorageWriteFileChannel(writeChannel); + } + + @Test + public void testWrite() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 100).position(0); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.write(buffer)).isEqualTo(1); + assertThat(fileChannel.position()).isEqualTo(1L); + assertThat(data.get(0)).isEqualTo((byte) 100); + } + + @Test + public void testWriteArray() throws IOException { + ByteBuffer[] buffers = + new ByteBuffer[] {ByteBuffer.allocate(1), ByteBuffer.allocate(1), ByteBuffer.allocate(1)}; + buffers[0].put((byte) 10).position(0); + buffers[1].put((byte) 20).position(0); + buffers[2].put((byte) 30).position(0); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.write(buffers)).isEqualTo(3L); + assertThat(fileChannel.position()).isEqualTo(3L); + + assertThat(data.get(0)).isEqualTo((byte) 10); + assertThat(data.get(1)).isEqualTo((byte) 20); + assertThat(data.get(2)).isEqualTo((byte) 30); + } + + @Test + public void testPosition() throws IOException { + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.position(1L)).isEqualTo(fileChannel); + assertThat(fileChannel.position()).isEqualTo(1L); + assertThat(fileChannel.position(0L)).isEqualTo(fileChannel); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.position(100L)).isEqualTo(fileChannel); + assertThat(fileChannel.position()).isEqualTo(100L); + } + + @Test + public void testSizeAndTruncate() throws IOException { + assertThat(fileChannel.size()).isEqualTo(3L); + fileChannel.truncate(1L); + assertThat(fileChannel.size()).isEqualTo(1L); + fileChannel.truncate(10L); + assertThat(fileChannel.size()).isEqualTo(1L); + assertThat(fileChannel.position()).isEqualTo(0L); + } + + @Test + public void testTransferFrom() throws IOException { + SeekableByteChannelImpl src = new SeekableByteChannelImpl(ByteBuffer.allocate(100)); + src.write(ByteBuffer.wrap(new byte[] {10, 20, 30})); + src.position(0L); + fileChannel.position(0L); + assertThat(fileChannel.transferFrom(src, 0L, 3L)).isEqualTo(3L); + assertThat(src.position()).isEqualTo(3L); + + assertThat(data.get(0)).isEqualTo((byte) 10); + assertThat(data.get(1)).isEqualTo((byte) 20); + assertThat(data.get(2)).isEqualTo((byte) 30); + } + + @Test + public void testWriteOnPosition() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 100).position(0); + assertThat(fileChannel.position()).isEqualTo(0L); + assertThat(fileChannel.write(buffer, 0)).isEqualTo(1); + assertThat(data.get(0)).isEqualTo((byte) 100); + } + + @Test + public void testWriteBeyondEnd() throws IOException { + fileChannel.position(3L); + ByteBuffer src = ByteBuffer.wrap(new byte[] {10, 20, 30}); + assertThat(fileChannel.write(src)).isEqualTo(3); + assertThat(fileChannel.position()).isEqualTo(6L); + fileChannel.position(3L); + assertThat(data.get(3)).isEqualTo((byte) 10); + assertThat(data.get(4)).isEqualTo((byte) 20); + assertThat(data.get(5)).isEqualTo((byte) 30); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/NIOTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/NIOTest.java new file mode 100644 index 000000000000..2c34bc02d643 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/NIOTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link CloudStorageFileSystemProvider}. Makes sure the NIO system picks it up + * properly. + */ +@RunWith(JUnit4.class) +public class NIOTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private URI uriToCloudStorage; + + @Before + public void setUp() { + CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions()); + uriToCloudStorage = URI.create("gs://bucket/file.txt"); + } + + @After + public void after() { + CloudStorageFileSystemProvider.setStorageOptions(StorageOptionsUtil.getDefaultInstance()); + } + + /** We can create a Path object from a gs:// URI. * */ + @Test + public void testCreatePath() { + // Return value ignored on purpose, we just want to check + // no exception is thrown. + Path path = Paths.get(uriToCloudStorage); + // Truth bug workaround, see https://github.com/google/truth/issues/285 + assertThat((Object) path).isNotNull(); + } + + /** The created Path object has the correct scheme. * */ + @Test + public void testCreatedPathIsGS() { + Path path = Paths.get(uriToCloudStorage); + assertThat(path.getFileSystem().provider().getScheme()).isEqualTo("gs"); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/SeekableByteChannelPrefetcherTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/SeekableByteChannelPrefetcherTest.java new file mode 100644 index 000000000000..ed4f130eee43 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/SeekableByteChannelPrefetcherTest.java @@ -0,0 +1,191 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SeekableByteChannelPrefetcherTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + // A file big enough to try seeks on. + private static Path input; + + /** Sets up an input file to play with. */ + @BeforeClass + public static void setUp() throws IOException { + // create input, fill with data + input = Files.createTempFile("tmp_big_file", ".tmp"); + try (BufferedOutputStream writer = new BufferedOutputStream(Files.newOutputStream(input))) { + byte[] buffer = new byte[1024]; + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) i; + } + for (int i = 0; i < 1024; i++) { + writer.write(buffer); + } + } + } + + /** Deletes the test file. */ + @AfterClass + public static void tearDown() throws IOException { + Files.delete(input); + } + + @Test + public void testRead() throws Exception { + SeekableByteChannel chan1 = Files.newByteChannel(input); + SeekableByteChannel chan2 = + SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); + + testReading(chan1, chan2, 0); + testReading(chan1, chan2, 128); + testReading(chan1, chan2, 1024); + testReading(chan1, chan2, 1500); + testReading(chan1, chan2, 2048); + testReading(chan1, chan2, 3000); + testReading(chan1, chan2, 6000); + } + + @Test + public void testSeek() throws Exception { + SeekableByteChannel chan1 = Files.newByteChannel(input); + SeekableByteChannel chan2 = + SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); + + testSeeking(chan1, chan2, 1024); + testSeeking(chan1, chan2, 1500); + testSeeking(chan1, chan2, 128); + testSeeking(chan1, chan2, 256); + testSeeking(chan1, chan2, 128); + // yes, testReading - let's make sure that reading more than one block still works + // even after a seek. + testReading(chan1, chan2, 1500); + testSeeking(chan1, chan2, 2048); + testSeeking(chan1, chan2, 0); + testSeeking(chan1, chan2, 3000); + testSeeking(chan1, chan2, 6000); + testSeeking(chan1, chan2, (int) chan1.size() - 127); + testSeeking(chan1, chan2, (int) chan1.size() - 128); + testSeeking(chan1, chan2, (int) chan1.size() - 129); + } + + @Test + public void testPartialBuffers() throws Exception { + SeekableByteChannel chan1 = Files.newByteChannel(input); + SeekableByteChannel chan2 = + SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); + // get a partial buffer + testSeeking(chan1, chan2, (int) chan1.size() - 127); + // make sure normal reads can use the full buffer + for (int i = 0; i < 2; i++) { + testSeeking(chan1, chan2, i * 1024); + } + // get a partial buffer, replacing one of the full ones + testSeeking(chan1, chan2, (int) chan1.size() - 127); + // make sure the buffers are still OK + for (int i = 0; i < 2; i++) { + testSeeking(chan1, chan2, i * 1024); + } + } + + @Test + public void testEOF() throws Exception { + SeekableByteChannel chan1 = Files.newByteChannel(input); + SeekableByteChannel chan2 = + SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); + // read the final 128 bytes, exactly. + testSeeking(chan1, chan2, (int) chan1.size() - 128); + // read truncated because we're asking for beyond EOF + testSeeking(chan1, chan2, (int) chan1.size() - 64); + // read starting past EOF + testSeeking(chan1, chan2, (int) chan1.size() + 128); + // read more than a whole block past EOF + testSeeking(chan1, chan2, (int) chan1.size() + 1024 * 2); + } + + @Test(expected = IllegalArgumentException.class) + public void testDoubleWrapping() throws IOException { + SeekableByteChannel chan1 = + SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); + SeekableByteChannelPrefetcher.addPrefetcher(1, chan1); + } + + @Test + public void testCloseWhilePrefetching() throws Exception { + SeekableByteChannel chan = + SeekableByteChannelPrefetcher.addPrefetcher(10, Files.newByteChannel(input)); + // read just 1 byte, get the prefetching going + ByteBuffer one = ByteBuffer.allocate(1); + readFully(chan, one); + // closing must not throw an exception, even if the prefetching + // thread is active. + chan.close(); + } + + private void testReading(SeekableByteChannel chan1, SeekableByteChannel chan2, int howMuch) + throws IOException { + ByteBuffer one = ByteBuffer.allocate(howMuch); + ByteBuffer two = ByteBuffer.allocate(howMuch); + + readFully(chan1, one); + readFully(chan2, two); + + assertThat(one.position()).isEqualTo(two.position()); + assertThat(one.array()).isEqualTo(two.array()); + } + + private void testSeeking(SeekableByteChannel chan1, SeekableByteChannel chan2, int position) + throws IOException { + ByteBuffer one = ByteBuffer.allocate(128); + ByteBuffer two = ByteBuffer.allocate(128); + + chan1.position(position); + chan2.position(position); + + readFully(chan1, one); + readFully(chan2, two); + + assertThat(one.position()).isEqualTo(two.position()); + assertThat(one.array()).isEqualTo(two.array()); + } + + private void readFully(ReadableByteChannel chan, ByteBuffer buf) throws IOException { + // the countdown isn't strictly necessary but it protects us against infinite loops + // for some potential bugs in the channel implementation. + int countdown = buf.capacity(); + while (chan.read(buf) > 0 && countdown > 0) { + countdown--; + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/StorageOptionsUtilTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/StorageOptionsUtilTest.java new file mode 100644 index 000000000000..9a1f79738d21 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/StorageOptionsUtilTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.api.gax.rpc.HeaderProvider; +import com.google.cloud.NoCredentials; +import com.google.cloud.storage.StorageOptions; +import java.util.Map; +import org.junit.Test; + +public final class StorageOptionsUtilTest { + + @Test + public void ensureDefaultInstanceCarriedThrough() { + StorageOptions defaultInstance = StorageOptionsUtil.getDefaultInstance(); + StorageOptions merged = StorageOptionsUtil.mergeOptionsWithUserAgent(defaultInstance); + + assertSame(defaultInstance, merged); + } + + @Test + public void ensureUserAgentEntryAddedIfUserAgentSpecified() { + StorageOptions defaultInstance = + defaults().toBuilder() + .setHeaderProvider(FixedHeaderProvider.create("user-agent", "asdf/ gcloud-java-nio/")) + .build(); + StorageOptions merged = StorageOptionsUtil.mergeOptionsWithUserAgent(defaultInstance); + + String mergedUserAgent = merged.getUserAgent(); + assertNotNull(mergedUserAgent); + assertTrue(mergedUserAgent.contains("gcloud-java-nio/")); + } + + @Test + public void ensureUserAgentEntryLeftInTactIfAlreadySpecifiesOurEntry() { + StorageOptions defaultInstance = defaults(); + StorageOptions merged = StorageOptionsUtil.mergeOptionsWithUserAgent(defaultInstance); + + String mergedUserAgent = merged.getUserAgent(); + assertNotNull(mergedUserAgent); + assertTrue(mergedUserAgent.contains("gcloud-java-nio/")); + } + + @Test + public void ensureAddingUserAgentEntryDoesNotClobberAnyProvidedHeaders() { + StorageOptions base = + defaults().toBuilder() + .setHeaderProvider( + FixedHeaderProvider.create( + "user-agent", "custom/", + "x-custom-header", "value")) + .build(); + StorageOptions merged = StorageOptionsUtil.mergeOptionsWithUserAgent(base); + + String mergedUserAgent = merged.getUserAgent(); + + HeaderProvider mergedHeaderProvider = StorageOptionsUtil.getHeaderProvider(merged); + Map headers = mergedHeaderProvider.getHeaders(); + String customHeader = headers.get("x-custom-header"); + + assertNotNull(mergedUserAgent); + assertNotNull(customHeader); + assertEquals("value", customHeader); + assertTrue(mergedUserAgent.contains("custom/ gcloud-java-nio/")); + } + + private StorageOptions defaults() { + return StorageOptions.newBuilder() + .setCredentials(NoCredentials.getInstance()) + .setProjectId("proj") + .build(); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/UnixPathTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/UnixPathTest.java new file mode 100644 index 000000000000..8dd762a8875e --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/UnixPathTest.java @@ -0,0 +1,418 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link UnixPath}. */ +@RunWith(JUnit4.class) +public class UnixPathTest { + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + @Test + public void testNormalize() { + assertThat(p(".").normalize()).isEqualTo(p("")); + assertThat(p("/").normalize()).isEqualTo(p("/")); + assertThat(p("/.").normalize()).isEqualTo(p("/")); + assertThat(p("/a/b/../c").normalize()).isEqualTo(p("/a/c")); + assertThat(p("/a/b/./c").normalize()).isEqualTo(p("/a/b/c")); + assertThat(p("a/b/../c").normalize()).isEqualTo(p("a/c")); + assertThat(p("a/b/./c").normalize()).isEqualTo(p("a/b/c")); + assertThat(p("/a/b/../../c").normalize()).isEqualTo(p("/c")); + assertThat(p("/a/b/./.././.././c").normalize()).isEqualTo(p("/c")); + } + + @Test + public void testNormalize_empty_returnsEmpty() { + assertThat(p("").normalize()).isEqualTo(p("")); + } + + @Test + public void testNormalize_underflow_isAllowed() { + assertThat(p("../").normalize()).isEqualTo(p("")); + } + + @Test + public void testNormalize_extraSlashes_getRemoved() { + assertThat(p("///").normalize()).isEqualTo(p("/")); + assertThat(p("/hi//there").normalize()).isEqualTo(p("/hi/there")); + assertThat(p("/hi////.///there").normalize()).isEqualTo(p("/hi/there")); + } + + @Test + public void testNormalize_trailingSlash() { + assertThat(p("/hi/there/").normalize()).isEqualTo(p("/hi/there/")); + assertThat(p("/hi/there/../").normalize()).isEqualTo(p("/hi/")); + assertThat(p("/hi/there/..").normalize()).isEqualTo(p("/hi/")); + assertThat(p("hi/../").normalize()).isEqualTo(p("")); + assertThat(p("/hi/../").normalize()).isEqualTo(p("/")); + assertThat(p("hi/..").normalize()).isEqualTo(p("")); + assertThat(p("/hi/..").normalize()).isEqualTo(p("/")); + } + + @Test + public void testNormalize_sameObjectOptimization() { + UnixPath path = p("/hi/there"); + assertThat(path.normalize()).isSameInstanceAs(path); + path = p("/hi/there/"); + assertThat(path.normalize()).isSameInstanceAs(path); + } + + @Test + public void testResolve() { + assertThat(p("/hello").resolve(p("cat"))).isEqualTo(p("/hello/cat")); + assertThat(p("/hello/").resolve(p("cat"))).isEqualTo(p("/hello/cat")); + assertThat(p("hello/").resolve(p("cat"))).isEqualTo(p("hello/cat")); + assertThat(p("hello/").resolve(p("cat/"))).isEqualTo(p("hello/cat/")); + assertThat(p("hello/").resolve(p(""))).isEqualTo(p("hello/")); + assertThat(p("hello/").resolve(p("/hi/there"))).isEqualTo(p("/hi/there")); + } + + @Test + public void testResolve_sameObjectOptimization() { + UnixPath path = p("/hi/there"); + assertThat(path.resolve(p(""))).isSameInstanceAs(path); + assertThat(p("hello").resolve(path)).isSameInstanceAs(path); + } + + @Test + public void testGetPath() { + assertThat(UnixPath.getPath(false, "hello")).isEqualTo(p("hello")); + assertThat(UnixPath.getPath(false, "hello", "cat")).isEqualTo(p("hello/cat")); + assertThat(UnixPath.getPath(false, "/hello", "cat")).isEqualTo(p("/hello/cat")); + assertThat(UnixPath.getPath(false, "/hello", "cat", "inc.")).isEqualTo(p("/hello/cat/inc.")); + assertThat(UnixPath.getPath(false, "hello/", "/hi/there")).isEqualTo(p("/hi/there")); + } + + @Test + public void testResolveSibling() { + assertThat(p("/hello/cat").resolveSibling(p("dog"))).isEqualTo(p("/hello/dog")); + assertThat(p("/").resolveSibling(p("dog"))).isEqualTo(p("dog")); + } + + @Test + public void testResolveSibling_preservesTrailingSlash() { + assertThat(p("/hello/cat").resolveSibling(p("dog/"))).isEqualTo(p("/hello/dog/")); + assertThat(p("/").resolveSibling(p("dog/"))).isEqualTo(p("dog/")); + } + + @Test + public void testRelativize() { + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/mop/top"))) + .isEqualTo(p("../../../mop/top")); + assertThat(p("/foo/bar/dog").relativize(p("/foo/mop/top"))).isEqualTo(p("../../mop/top")); + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/mop/top/../../mog"))) + .isEqualTo(p("../../../mop/top/../../mog")); + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/../mog"))).isEqualTo(p("../../../../mog")); + assertThat(p("").relativize(p("foo/mop/top/"))).isEqualTo(p("foo/mop/top/")); + } + + @Test + public void testRelativize_absoluteMismatch_notAllowed() { + try { + p("/a/b/").relativize(p("")); + Assert.fail(); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isEqualTo("'other' is different type of Path"); + } + } + + @Test + public void testRelativize_preservesTrailingSlash() { + // This behavior actually diverges from sun.nio.fs.UnixPath: + // bsh % print(Paths.get("/a/b/").relativize(Paths.get("/etc/"))); + // ../../etc + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/../mog/"))) + .isEqualTo(p("../../../../mog/")); + assertThat(p("/a/b/").relativize(p("/etc/"))).isEqualTo(p("../../etc/")); + } + + @Test + public void testStartsWith() { + assertThat(p("/hi/there").startsWith(p("/hi/there"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("/hi/therf"))).isFalse(); + assertThat(p("/hi/there").startsWith(p("/hi"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("/hi/"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("hi"))).isFalse(); + assertThat(p("/hi/there").startsWith(p("/"))).isTrue(); + assertThat(p("/hi/there").startsWith(p(""))).isFalse(); + assertThat(p("/a/b").startsWith(p("a/b/"))).isFalse(); + assertThat(p("/a/b/").startsWith(p("a/b/"))).isFalse(); + assertThat(p("/hi/there").startsWith(p(""))).isFalse(); + assertThat(p("").startsWith(p(""))).isTrue(); + } + + @Test + public void testStartsWith_comparesComponentsIndividually() { + assertThat(p("/hello").startsWith(p("/hell"))).isFalse(); + assertThat(p("/hello").startsWith(p("/hello"))).isTrue(); + } + + @Test + public void testEndsWith() { + assertThat(p("/hi/there").endsWith(p("there"))).isTrue(); + assertThat(p("/hi/there").endsWith(p("therf"))).isFalse(); + assertThat(p("/hi/there").endsWith(p("/blag/therf"))).isFalse(); + assertThat(p("/hi/there").endsWith(p("/hi/there"))).isTrue(); + assertThat(p("/hi/there").endsWith(p("/there"))).isFalse(); + assertThat(p("/human/that/you/cry").endsWith(p("that/you/cry"))).isTrue(); + assertThat(p("/human/that/you/cry").endsWith(p("that/you/cry/"))).isTrue(); + assertThat(p("/hi/there/").endsWith(p("/"))).isFalse(); + assertThat(p("/hi/there").endsWith(p(""))).isFalse(); + assertThat(p("").endsWith(p(""))).isTrue(); + } + + @Test + public void testEndsWith_comparesComponentsIndividually() { + assertThat(p("/hello").endsWith(p("lo"))).isFalse(); + assertThat(p("/hello").endsWith(p("hello"))).isTrue(); + } + + @Test + public void testGetParent() { + assertThat(p("").getParent()).isNull(); + assertThat(p("/").getParent()).isNull(); + assertThat(p("aaa/").getParent()).isNull(); + assertThat(p("aaa").getParent()).isNull(); + assertThat(p("/aaa/").getParent()).isEqualTo(p("/")); + assertThat(p("a/b/c").getParent()).isEqualTo(p("a/b/")); + assertThat(p("a/b/c/").getParent()).isEqualTo(p("a/b/")); + assertThat(p("a/b/").getParent()).isEqualTo(p("a/")); + } + + @Test + public void testGetRoot() { + assertThat(p("/hello").getRoot()).isEqualTo(p("/")); + assertThat(p("hello").getRoot()).isNull(); + } + + @Test + public void testGetFileName() { + assertThat(p("").getFileName()).isEqualTo(p("")); + assertThat(p("/").getFileName()).isNull(); + assertThat(p("/dark").getFileName()).isEqualTo(p("dark")); + assertThat(p("/angels/").getFileName()).isEqualTo(p("angels")); + } + + @Test + public void testEquals() { + assertThat(p("/a/").equals(p("/a/"))).isTrue(); + assertThat(p("/a/").equals(p("/b/"))).isFalse(); + assertThat(p("/b/").equals(p("/b"))).isFalse(); + assertThat(p("/b").equals(p("/b/"))).isFalse(); + assertThat(p("b").equals(p("/b"))).isFalse(); + assertThat(p("b").equals(p("b"))).isTrue(); + } + + @Test + public void testSplit() { + assertThat(p("").split().hasNext()).isFalse(); + assertThat(p("hi/there").split().hasNext()).isTrue(); + assertThat(p(p("hi/there").split().next())).isEqualTo(p("hi")); + } + + @Test + public void testToAbsolute() { + assertThat(p("lol").toAbsolutePath(UnixPath.ROOT_PATH)).isEqualTo(p("/lol")); + assertThat(p("lol/cat").toAbsolutePath(UnixPath.ROOT_PATH)).isEqualTo(p("/lol/cat")); + } + + @Test + public void testToAbsolute_withCurrentDirectory() { + assertThat(p("cat").toAbsolutePath(p("/lol"))).isEqualTo(p("/lol/cat")); + assertThat(p("cat").toAbsolutePath(p("/lol/"))).isEqualTo(p("/lol/cat")); + assertThat(p("/hi/there").toAbsolutePath(p("/lol"))).isEqualTo(p("/hi/there")); + } + + @Test + public void testToAbsolute_preservesTrailingSlash() { + assertThat(p("cat/").toAbsolutePath(p("/lol"))).isEqualTo(p("/lol/cat/")); + } + + @Test + public void testSubpath() { + assertThat(p("/eins/zwei/drei/vier").subpath(0, 1)).isEqualTo(p("eins")); + assertThat(p("/eins/zwei/drei/vier").subpath(0, 2)).isEqualTo(p("eins/zwei")); + assertThat(p("eins/zwei/drei/vier/").subpath(1, 4)).isEqualTo(p("zwei/drei/vier")); + assertThat(p("eins/zwei/drei/vier/").subpath(2, 4)).isEqualTo(p("drei/vier")); + } + + @Test + public void testSubpath_empty_returnsEmpty() { + assertThat(p("").subpath(0, 1)).isEqualTo(p("")); + } + + @Test + public void testSubpath_root_throwsIae() { + try { + p("/").subpath(0, 1); + Assert.fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testSubpath_negativeIndex_throwsIae() { + try { + p("/eins/zwei/drei/vier").subpath(-1, 1); + Assert.fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testSubpath_notEnoughElements_throwsIae() { + try { + p("/eins/zwei/drei/vier").subpath(0, 5); + Assert.fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testSubpath_beginAboveEnd_throwsIae() { + try { + p("/eins/zwei/drei/vier").subpath(1, 0); + Assert.fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testSubpath_beginAndEndEqual_throwsIae() { + try { + p("/eins/zwei/drei/vier").subpath(0, 0); + Assert.fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testNameCount() { + assertThat(p("").getNameCount()).isEqualTo(1); + assertThat(p("/").getNameCount()).isEqualTo(0); + assertThat(p("/hi/").getNameCount()).isEqualTo(1); + assertThat(p("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(p("hi/yo").getNameCount()).isEqualTo(2); + } + + @Test + public void testNameCount_dontPermitEmptyComponents_emptiesGetIgnored() { + assertThat(p("hi//yo").getNameCount()).isEqualTo(2); + assertThat(p("//hi//yo//").getNameCount()).isEqualTo(2); + } + + @Test + public void testNameCount_permitEmptyComponents_emptiesGetCounted() { + assertThat(pp("hi//yo").getNameCount()).isEqualTo(3); + assertThat(pp("hi//yo/").getNameCount()).isEqualTo(4); + assertThat(pp("hi//yo//").getNameCount()).isEqualTo(5); + } + + @Test + public void testNameCount_permitEmptyComponents_rootComponentDoesntCount() { + assertThat(pp("hi/yo").getNameCount()).isEqualTo(2); + assertThat(pp("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(pp("//hi/yo").getNameCount()).isEqualTo(3); + } + + @Test + public void testGetName() { + assertThat(p("").getName(0)).isEqualTo(p("")); + assertThat(p("/hi").getName(0)).isEqualTo(p("hi")); + assertThat(p("hi/there").getName(1)).isEqualTo(p("there")); + } + + @Test + public void testCompareTo() { + assertThat(p("/hi/there").compareTo(p("/hi/there"))).isEqualTo(0); + assertThat(p("/hi/there").compareTo(p("/hi/therf"))).isEqualTo(-1); + assertThat(p("/hi/there").compareTo(p("/hi/therd"))).isEqualTo(1); + } + + @Test + public void testCompareTo_dontPermitEmptyComponents_emptiesGetIgnored() { + assertThat(p("a/b").compareTo(p("a//b"))).isEqualTo(0); + } + + @Test + public void testCompareTo_permitEmptyComponents_behaviorChanges() { + assertThat(p("a/b").compareTo(pp("a//b"))).isEqualTo(1); + assertThat(pp("a/b").compareTo(pp("a//b"))).isEqualTo(1); + } + + @Test + public void testCompareTo_comparesComponentsIndividually() { + assumeTrue('.' < '/'); + assertThat("hi./there".compareTo("hi/there")).isEqualTo(-1); + assertThat("hi.".compareTo("hi")).isEqualTo(1); + assertThat(p("hi./there").compareTo(p("hi/there"))).isEqualTo(1); + assertThat(p("hi./there").compareTo(p("hi/there"))).isEqualTo(1); + assumeTrue('0' > '/'); + assertThat("hi0/there".compareTo("hi/there")).isEqualTo(1); + assertThat("hi0".compareTo("hi")).isEqualTo(1); + assertThat(p("hi0/there").compareTo(p("hi/there"))).isEqualTo(1); + } + + @Test + public void testSeemsLikeADirectory() { + assertThat(p("a").seemsLikeADirectory()).isFalse(); + assertThat(p("a.").seemsLikeADirectory()).isFalse(); + assertThat(p("a..").seemsLikeADirectory()).isFalse(); + assertThat(p("").seemsLikeADirectory()).isTrue(); + assertThat(p("/").seemsLikeADirectory()).isTrue(); + assertThat(p(".").seemsLikeADirectory()).isTrue(); + assertThat(p("/.").seemsLikeADirectory()).isTrue(); + assertThat(p("..").seemsLikeADirectory()).isTrue(); + assertThat(p("/..").seemsLikeADirectory()).isTrue(); + } + + @Test + public void testEquals_equalsTester() { + new EqualsTester() + .addEqualityGroup(p("/lol"), p("/lol")) + .addEqualityGroup(p("/lol//"), p("/lol//")) + .addEqualityGroup(p("dust")) + .testEquals(); + } + + @Test + public void testNullness() throws Exception { + NullPointerTester tester = new NullPointerTester(); + tester.ignore(UnixPath.class.getMethod("equals", Object.class)); + tester.testAllPublicStaticMethods(UnixPath.class); + tester.testAllPublicInstanceMethods(p("solo")); + } + + private static UnixPath p(String path) { + return UnixPath.getPath(false, path); + } + + private static UnixPath pp(String path) { + return UnixPath.getPath(true, path); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java new file mode 100644 index 000000000000..40d60611b05b --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java @@ -0,0 +1,1259 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio.it; + +import static com.google.common.collect.ImmutableList.copyOf; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import com.google.api.client.http.HttpResponseException; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration; +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem; +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystemProvider; +import com.google.cloud.storage.contrib.nio.CloudStoragePath; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.cloud.storage.testing.RemoteStorageHelper; +import com.google.cloud.testing.junit4.MultipleAttemptsRule; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessMode; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.spi.FileSystemProvider; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Integration test for google-cloud-nio. + * + *

This test actually talks to Google Cloud Storage (you need an account) and tests both reading + * and writing. You *must* set the {@code GOOGLE_APPLICATION_CREDENTIALS} environment variable for + * this test to work. It must contain the name of a local file that contains your Service Account + * JSON Key. We use the project in those credentials. + * + *

See + * Service Accounts for instructions on how to get the Service Account JSON Key. + * + *

The short version is this: go to cloud.google.com/console, select your project, search for + * "API manager", click "Credentials", click "create credentials/service account key", new service + * account, JSON. The contents of the file that's sent to your browsers is your "Service Account + * JSON Key". + */ +@RunWith(JUnit4.class) +public class ITGcsNio { + + private static final List FILE_CONTENTS = + ImmutableList.of( + "Tous les êtres humains naissent libres et égaux en dignité et en droits.", + "Ils sont doués de raison et de conscience et doivent agir ", + "les uns envers les autres dans un esprit de fraternité."); + + private static final Logger log = Logger.getLogger(ITGcsNio.class.getName()); + private static final String BUCKET = RemoteStorageHelper.generateBucketName(); + private static final String TARGET_BUCKET = RemoteStorageHelper.generateBucketName(); + private static final String REQUESTER_PAYS_BUCKET = + RemoteStorageHelper.generateBucketName() + "_rp"; + private static final String SML_FILE = "tmp-test-small-file.txt"; + private static final String TMP_FILE = "tmp/tmp-test-rnd-file.txt"; + private static final int SML_SIZE = 100; + private static final String BIG_FILE = "tmp-test-big-file.txt"; // it's big, relatively speaking. + private static final int BIG_SIZE = 2 * 1024 * 1024 - 50; // arbitrary size that's not too round. + private static final String PREFIX = "tmp-test-file"; + private static String project; + private static Storage storage; + private static StorageOptions storageOptions; + + @Rule public final MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3); + + private final Random rnd = new Random(); + + @BeforeClass + public static void beforeClass() throws IOException { + // loads the credentials from local disk as par README + RemoteStorageHelper gcsHelper = RemoteStorageHelper.create(); + storageOptions = gcsHelper.getOptions(); + project = storageOptions.getProjectId(); + storage = storageOptions.getService(); + // create and populate test bucket + storage.create(BucketInfo.of(BUCKET)); + storage.create(BucketInfo.of(TARGET_BUCKET)); + fillFile(storage, BUCKET, SML_FILE, SML_SIZE); + fillFile(storage, BUCKET, BIG_FILE, BIG_SIZE); + BucketInfo requesterPaysBucket = + BucketInfo.newBuilder(REQUESTER_PAYS_BUCKET).setRequesterPays(true).build(); + storage.create(requesterPaysBucket); + fillRequesterPaysFile(storage, SML_FILE, SML_SIZE); + } + + @AfterClass + public static void afterClass() throws ExecutionException, InterruptedException { + if (storage != null) { + for (String bucket : new String[] {BUCKET, REQUESTER_PAYS_BUCKET}) { + if (!RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS, project) + && log.isLoggable(Level.WARNING)) { + log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", bucket); + } + } + } + } + + private static byte[] randomContents(int size) { + byte[] bytes = new byte[size]; + new Random(size).nextBytes(bytes); + return bytes; + } + + private static void fillFile(Storage storage, String bucket, String fname, int size) + throws IOException { + storage.create(BlobInfo.newBuilder(bucket, fname).build(), randomContents(size)); + } + + private static void fillRequesterPaysFile(Storage storage, String fname, int size) + throws IOException { + storage.create( + BlobInfo.newBuilder(REQUESTER_PAYS_BUCKET, fname).build(), + randomContents(size), + BlobTargetOption.userProject(project)); + } + + // Start of tests related to the "requester pays" feature + @Test + public void testFileExistsRequesterPaysNoUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, ""); + Path path = testBucket.getPath(SML_FILE); + try { + // fails because we must pay for every access, including metadata. + Files.exists(path); + Assert.fail("It should have thrown an exception."); + } catch (StorageException ex) { + assertIsRequesterPaysException("testFileExistsRequesterPaysNoUserProject", ex); + } + } + + @Test + public void testFileExistsRequesterPays() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, project); + Path path = testBucket.getPath(SML_FILE); + // should succeed because we specified a project + Files.exists(path); + } + + @Test + public void testFileExistsRequesterPaysWithAutodetect() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(true, project); + Path path = testBucket.getPath(SML_FILE); + // should succeed because we specified a project + Files.exists(path); + } + + @Test + public void testCantCreateWithoutUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, ""); + Path path = testBucket.getPath(TMP_FILE); + try { + // fails + Files.write(path, "I would like to write".getBytes()); + Assert.fail("It should have thrown an exception."); + } catch (IOException ex) { + assertIsRequesterPaysException("testCantCreateWithoutUserProject", ex); + } + } + + @Test + public void testCanCreateWithUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, project); + Path path = testBucket.getPath(TMP_FILE); + // should succeed because we specified a project + Files.write(path, "I would like to write, please?".getBytes()); + } + + @Test + public void testCantReadWithoutUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, ""); + Path path = testBucket.getPath(SML_FILE); + try { + // fails + Files.readAllBytes(path); + Assert.fail("It should have thrown an exception."); + } catch (StorageException ex) { + assertIsRequesterPaysException("testCantReadWithoutUserProject", ex); + } + } + + @Test + public void testCanReadWithUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, project); + Path path = testBucket.getPath(SML_FILE); + // should succeed because we specified a project + Files.readAllBytes(path); + } + + @Test + public void testCantCopyWithoutUserProject() throws IOException { + CloudStorageFileSystem testRPBucket = getRequesterPaysBucket(false, ""); + CloudStorageFileSystem testBucket = getTestBucket(); + CloudStoragePath[] sources = + new CloudStoragePath[] {testBucket.getPath(SML_FILE), testRPBucket.getPath(SML_FILE)}; + CloudStoragePath[] dests = + new CloudStoragePath[] {testBucket.getPath(TMP_FILE), testRPBucket.getPath(TMP_FILE)}; + for (int s = 0; s < 2; s++) { + for (int d = 0; d < 2; d++) { + // normal to normal is out of scope of RP testing. + if (s == 0 && d == 0) { + continue; + } + innerTestCantCopyWithoutUserProject(s == 0, d == 0, sources[s], dests[d]); + } + } + } + + // Try to copy the file, make sure that we were prevented. + private void innerTestCantCopyWithoutUserProject( + boolean sourceNormal, boolean destNormal, Path source, Path dest) throws IOException { + String sdesc = (sourceNormal ? "normal bucket" : "requester-pays bucket"); + String ddesc = (destNormal ? "normal bucket" : "requester-pays bucket"); + String description = "Copying from " + sdesc + " to " + ddesc; + try { + Files.copy(source, dest); + Assert.fail("Shouldn't have been able to copy from " + sdesc + " to " + ddesc); + // for some reason this throws "GoogleJsonResponseException" instead of "StorageException" + // when going from requester pays bucket to requester pays bucket, but otherwise we get a + // normal StorageException. + } catch (HttpResponseException hex) { + Assert.assertEquals(description, hex.getStatusCode(), 400); + Assert.assertTrue(description, hex.getMessage().contains("requester pays")); + } catch (StorageException ex) { + assertIsRequesterPaysException(description, ex); + } + } + + @Test + public void testCanCopyWithUserProject() throws IOException { + CloudStorageFileSystem testRPBucket = getRequesterPaysBucket(false, project); + CloudStorageFileSystem testBucket = getTestBucket(); + CloudStoragePath[] sources = + new CloudStoragePath[] {testBucket.getPath(SML_FILE), testRPBucket.getPath(SML_FILE)}; + CloudStoragePath[] dests = + new CloudStoragePath[] {testBucket.getPath(TMP_FILE), testRPBucket.getPath(TMP_FILE)}; + for (int s = 0; s < 2; s++) { + for (int d = 0; d < 2; d++) { + // normal to normal is out of scope of RP testing. + if (s == 0 && d == 0) { + continue; + } + Files.copy(sources[s], dests[d], StandardCopyOption.REPLACE_EXISTING); + } + } + } + + @Test + public void testAutodetectWhenRequesterPays() throws IOException { + CloudStorageFileSystem testRPBucket = getRequesterPaysBucket(true, project); + Assert.assertEquals( + "Autodetect should have detected the RP bucket", + testRPBucket.config().userProject(), + project); + } + + @Test + public void testAutodetectWhenNotRequesterPays() throws IOException { + CloudStorageConfiguration config = + CloudStorageConfiguration.builder() + .autoDetectRequesterPays(true) + .userProject(project) + .build(); + CloudStorageFileSystem testBucket = + CloudStorageFileSystem.forBucket(BUCKET, config, storageOptions); + Assert.assertEquals( + "Autodetect should have detected the bucket is not RP", + testBucket.config().userProject(), + ""); + } + + @Test + public void testRequesterPaysOnNonexistentBucket() { + CloudStorageConfiguration config = + CloudStorageConfiguration.builder() + .autoDetectRequesterPays(true) + .userProject(project) + .usePseudoDirectories(true) + .build(); + + final String bucketThatDoesntExist = "abuckethatdoesntexist"; + final String subPath = "hello"; + CloudStorageFileSystem testBucket = + CloudStorageFileSystem.forBucket(bucketThatDoesntExist, config, storageOptions); + final CloudStoragePath aPathThatDoesntExist = testBucket.getPath(subPath); + Assert.assertEquals( + aPathThatDoesntExist.toUri().toString(), "gs://" + bucketThatDoesntExist + "/" + subPath); + Assert.assertFalse(testBucket.provider().requesterPays(bucketThatDoesntExist)); + } + + @Test + public void testFilesExistBehaviorRequesterPays() { + CloudStorageConfiguration config = + CloudStorageConfiguration.builder() + .autoDetectRequesterPays(true) + .userProject(project) + .build(); + CloudStorageFileSystem testBucket = + CloudStorageFileSystem.forBucket(BUCKET, config, storageOptions); + Assert.assertFalse(Files.exists(testBucket.getPath("path"))); + } + + @Test + public void testAutoDetectNoUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, ""); + Assert.assertTrue(testBucket.provider().requesterPays(testBucket.bucket())); + } + + private void assertIsRequesterPaysException(String message, StorageException ex) { + Assert.assertEquals(message, ex.getCode(), 400); + Assert.assertTrue(message, ex.getMessage().contains("requester pays")); + } + + private void assertIsRequesterPaysException(String message, IOException ioex) { + Assert.assertTrue(message, ioex.getMessage().startsWith("400")); + Assert.assertTrue(message, ioex.getMessage().contains("requester pays")); + } + + // End of tests related to the "requester pays" feature + + @Test + public void testListBuckets() throws IOException { + boolean bucketFound = false; + boolean rpBucketFound = false; + for (Bucket b : CloudStorageFileSystem.listBuckets(project).iterateAll()) { + bucketFound |= BUCKET.equals(b.getName()); + rpBucketFound |= REQUESTER_PAYS_BUCKET.equals(b.getName()); + } + assertWithMessage("listBucket should have found the test bucket").that(bucketFound).isTrue(); + assertWithMessage("listBucket should have found the test requester-pays bucket") + .that(rpBucketFound) + .isTrue(); + } + + @Test + public void testFileExists() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testFileSize() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + assertThat(Files.size(path)).isEqualTo(SML_SIZE); + } + + @Test(timeout = 60_000) + public void testReadByteChannel() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + long size = Files.size(path); + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + assertThat(chan.size()).isEqualTo(size); + ByteBuffer buf = ByteBuffer.allocate(SML_SIZE); + int read = 0; + while (chan.isOpen()) { + buf.clear(); + int rc = chan.read(buf); + assertThat(chan.size()).isEqualTo(size); + if (rc < 0) { + // EOF + break; + } + assertThat(rc).isGreaterThan(0); + read += rc; + assertThat(chan.position()).isEqualTo(read); + } + assertThat(read).isEqualTo(size); + byte[] expected = new byte[SML_SIZE]; + new Random(SML_SIZE).nextBytes(expected); + assertThat(Arrays.equals(buf.array(), expected)).isTrue(); + } + + @Test + public void testSeek() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(BIG_FILE); + int size = BIG_SIZE; + byte[] contents = randomContents(size); + byte[] sample = new byte[100]; + byte[] wanted; + byte[] wanted2; + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + assertThat(chan.size()).isEqualTo(size); + + // check seek + int dest = size / 2; + chan.position(dest); + readFully(chan, sample); + wanted = Arrays.copyOfRange(contents, dest, dest + 100); + assertThat(wanted).isEqualTo(sample); + // now go back and check the beginning + // (we do 2 locations because 0 is sometimes a special case). + chan.position(0); + readFully(chan, sample); + wanted2 = Arrays.copyOf(contents, 100); + assertThat(wanted2).isEqualTo(sample); + // if the two spots in the file have the same contents, then this isn't a good file for this + // test. + assertThat(wanted).isNotEqualTo(wanted2); + } + + @Test + public void testCreate() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially. If it does it's either because it's a leftover + // from a previous run (so we should delete the file) + // or because we're misconfigured and pointing to an actually important file + // (so we should absolutely not delete it). + // So if the file's here, don't try to fix it automatically, let the user deal with it. + assertThat(Files.exists(path)).isFalse(); + try { + Files.createFile(path); + // now it does, and it has size 0. + assertThat(Files.exists(path)).isTrue(); + long size = Files.size(path); + assertThat(size).isEqualTo(0); + } finally { + // let's not leave files around + Files.deleteIfExists(path); + } + } + + @Test + public void testWrite() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially. If it does it's either because it's a leftover + // from a previous run (so we should delete the file) + // or because we're misconfigured and pointing to an actually important file + // (so we should absolutely not delete it). + // So if the file's here, don't try to fix it automatically, let the user deal with it. + assertThat(Files.exists(path)).isFalse(); + try { + Files.write(path, FILE_CONTENTS, UTF_8); + // now it does. + assertThat(Files.exists(path)).isTrue(); + + // let's check that the contents is OK. + ByteArrayOutputStream wantBytes = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(wantBytes, UTF_8)); + for (String content : FILE_CONTENTS) { + writer.println(content); + } + writer.close(); + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + byte[] gotBytes = new byte[(int) chan.size()]; + readFully(chan, gotBytes); + assertThat(gotBytes).isEqualTo(wantBytes.toByteArray()); + } finally { + // let's not leave files around + Files.deleteIfExists(path); + } + } + + @Test + public void testCreateAndWrite() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially (see above). + assertThat(Files.exists(path)).isFalse(); + try { + Files.createFile(path); + Files.write(path, FILE_CONTENTS, UTF_8); + // now it does. + assertThat(Files.exists(path)).isTrue(); + + // let's check that the contents is OK. + ByteArrayOutputStream wantBytes = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(wantBytes, UTF_8)); + for (String content : FILE_CONTENTS) { + writer.println(content); + } + writer.close(); + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + byte[] gotBytes = new byte[(int) chan.size()]; + readFully(chan, gotBytes); + assertThat(gotBytes).isEqualTo(wantBytes.toByteArray()); + } finally { + // let's not leave files around + Files.deleteIfExists(path); + } + } + + @Test + public void testWriteOnClose() throws Exception { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially (see above) + assertThat(Files.exists(path)).isFalse(); + try { + long expectedSize = 0; + try (SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.WRITE)) { + // writing lots of contents to defeat channel-internal buffering. + for (String s : FILE_CONTENTS) { + byte[] sBytes = s.getBytes(UTF_8); + expectedSize += sBytes.length * 9999; + for (int i = 0; i < 9999; i++) { + chan.write(ByteBuffer.wrap(sBytes)); + } + } + try { + Files.size(path); + // we shouldn't make it to this line. Not using thrown.expect because + // I still want to run a few lines after the exception. + Assert.fail("Files.size should have thrown an exception"); + } catch (NoSuchFileException nsf) { + // that's what we wanted, we're good. + } + } + // channel now closed, the file should be there and with the new contents. + assertThat(Files.exists(path)).isTrue(); + assertThat(Files.size(path)).isEqualTo(expectedSize); + } finally { + Files.deleteIfExists(path); + } + } + + @Test + public void testCopy() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path src = testBucket.getPath(SML_FILE); + Path dst = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially (see above). + assertThat(Files.exists(dst)).isFalse(); + try { + Files.copy(src, dst); + + assertThat(Files.exists(dst)).isTrue(); + assertThat(Files.size(dst)).isEqualTo(SML_SIZE); + byte[] got = new byte[SML_SIZE]; + readFully(Files.newByteChannel(dst), got); + assertThat(got).isEqualTo(randomContents(SML_SIZE)); + } finally { + // let's not leave files around + Files.deleteIfExists(dst); + } + } + + @Test + public void testListFiles() throws IOException { + try (FileSystem fs = getTestBucket()) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + goodPaths.add(fs.getPath("dir/dir2/")); + for (Path path : paths) { + fillFile(storage, BUCKET, path.toString(), SML_SIZE); + } + + List got = new ArrayList<>(); + for (String folder : new String[] {"/dir/", "/dir", "dir/", "dir"}) { + got.clear(); + for (Path path : Files.newDirectoryStream(fs.getPath(folder))) { + got.add(path); + } + assertWithMessage("Listing " + folder + ": ") + .that(got) + .containsExactlyElementsIn(goodPaths); + } + + // clean up + for (Path path : paths) { + Files.delete(path); + } + } + } + + @Test + public void testRelativityOfResolve() throws IOException { + try (FileSystem fs = getTestBucket()) { + Path abs1 = fs.getPath("/dir"); + Path abs2 = abs1.resolve("subdir/"); + Path rel1 = fs.getPath("dir"); + Path rel2 = rel1.resolve("subdir/"); + // children of absolute paths are absolute, + // children of relative paths are relative. + assertThat(abs1.isAbsolute()).isTrue(); + assertThat(abs2.isAbsolute()).isTrue(); + assertThat(rel1.isAbsolute()).isFalse(); + assertThat(rel2.isAbsolute()).isFalse(); + } + } + + @Test + public void testWalkFiles() throws IOException { + try (FileSystem fs = getTestBucket()) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + for (Path path : paths) { + fillFile(storage, BUCKET, path.toString(), SML_SIZE); + } + // Given a relative path as starting point, walkFileTree must return only relative paths. + List relativePaths = PostTraversalWalker.walkFileTree(fs.getPath("dir/")); + for (Path p : relativePaths) { + assertWithMessage("Should have been relative: " + p.toString()) + .that(p.isAbsolute()) + .isFalse(); + } + // The 5 paths are: + // dir/, dir/angel, dir/alone, dir/dir2/, dir/dir2/another_angel. + assertThat(relativePaths.size()).isEqualTo(5); + + // Given an absolute path as starting point, walkFileTree must return only relative paths. + List absolutePaths = PostTraversalWalker.walkFileTree(fs.getPath("/dir/")); + for (Path p : absolutePaths) { + assertWithMessage("Should have been absolute: " + p.toString()) + .that(p.isAbsolute()) + .isTrue(); + } + assertThat(absolutePaths.size()).isEqualTo(5); + } + } + + @Test + public void testDeleteRecursive() throws IOException { + try (FileSystem fs = getTestBucket()) { + List paths = new ArrayList<>(); + paths.add(fs.getPath("Racine")); + paths.add(fs.getPath("playwrights/Moliere")); + paths.add(fs.getPath("playwrights/French/Corneille")); + for (Path path : paths) { + Files.write(path, FILE_CONTENTS, UTF_8); + } + deleteRecursive(fs.getPath("playwrights/")); + assertThat(Files.exists(fs.getPath("playwrights/Moliere"))).isFalse(); + assertThat(Files.exists(fs.getPath("playwrights/French/Corneille"))).isFalse(); + assertThat(Files.exists(fs.getPath("Racine"))).isTrue(); + Files.deleteIfExists(fs.getPath("Racine")); + assertThat(Files.exists(fs.getPath("Racine"))).isFalse(); + } + } + + @Test + public void testListFilesInRootDirectory() throws IOException { + // We must explicitly set the storageOptions, because the unit tests + // set the fake storage as default but we want to access the real storage. + CloudStorageFileSystem fs = + CloudStorageFileSystem.forBucket( + BUCKET, + CloudStorageConfiguration.builder().permitEmptyPathComponents(true).build(), + storageOptions); + + // test absolute path, relative path. + for (String check : new String[] {".", "/", ""}) { + Path rootPath = fs.getPath(check); + List pathsFound = new ArrayList<>(); + for (Path path : Files.newDirectoryStream(rootPath)) { + // The returned paths will match the absolute-ness of the root path + // (this matches the behavior of the built-in UNIX file system). + assertWithMessage("Absolute/relative for " + check + ": ") + .that(path.isAbsolute()) + .isEqualTo(rootPath.isAbsolute()); + // To simplify the check that we found our files, we normalize here. + pathsFound.add(path.toAbsolutePath()); + } + assertWithMessage("Listing " + check + ": ") + .that(pathsFound) + .containsExactly( + fs.getPath(BIG_FILE).toAbsolutePath(), fs.getPath(SML_FILE).toAbsolutePath()); + } + } + + @Test + public void testFakeDirectories() throws IOException { + try (FileSystem fs = getTestBucket()) { + List paths = new ArrayList<>(); + paths.add(fs.getPath("dir/angel")); + paths.add(fs.getPath("dir/deepera")); + paths.add(fs.getPath("dir/deeperb")); + paths.add(fs.getPath("dir/deeper_")); + paths.add(fs.getPath("dir/deeper.sea/hasfish")); + paths.add(fs.getPath("dir/deeper/fish")); + for (Path path : paths) { + Files.createFile(path); + } + + // ends with slash, must be a directory + assertThat(Files.isDirectory(fs.getPath("dir/"))).isTrue(); + // files are not directories + assertThat(Files.exists(fs.getPath("dir/angel"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/angel"))).isFalse(); + // directories are recognized even without the trailing "/" + assertThat(Files.isDirectory(fs.getPath("dir"))).isTrue(); + // also works for absolute paths + assertThat(Files.isDirectory(fs.getPath("/dir"))).isTrue(); + // non-existent files are not directories (but they don't make us crash) + assertThat(Files.isDirectory(fs.getPath("di"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("dirs"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("dir/deep"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper/fi"))).isFalse(); + assertThat(Files.isDirectory(fs.getPath("/dir/deeper/fi"))).isFalse(); + // also works for subdirectories + assertThat(Files.isDirectory(fs.getPath("dir/deeper/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("/dir/deeper/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("/dir/deeper"))).isTrue(); + // dot and .. folders are directories + assertThat(Files.isDirectory(fs.getPath("dir/deeper/."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper/.."))).isTrue(); + // dots in the name are fine + assertThat(Files.isDirectory(fs.getPath("dir/deeper.sea/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper.sea"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("dir/deeper.seax"))).isFalse(); + // the root folder is a directory + assertThat(Files.isDirectory(fs.getPath("/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath(""))).isTrue(); + + // clean up + for (Path path : paths) { + Files.delete(path); + } + } + } + + @Test + public void testFileChannelRead() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + FileChannel chan = provider.newFileChannel(path, Sets.newHashSet(StandardOpenOption.READ)); + long size = Files.size(path); + assertThat(chan.size()).isEqualTo(size); + ByteBuffer buf = ByteBuffer.allocate(SML_SIZE); + int read = 0; + while (chan.isOpen()) { + buf.clear(); + int rc = chan.read(buf); + assertThat(chan.size()).isEqualTo(size); + if (rc < 0) { + // EOF + break; + } + assertThat(rc).isGreaterThan(0); + read += rc; + assertThat(chan.position()).isEqualTo(read); + } + chan.close(); + assertThat(read).isEqualTo(size); + byte[] expected = new byte[SML_SIZE]; + new Random(SML_SIZE).nextBytes(expected); + assertThat(Arrays.equals(buf.array(), expected)).isTrue(); + } + + @Test + public void testFileChannelCreate() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + assertThat(Files.exists(path)).isFalse(); + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + try { + FileChannel channel = + provider.newFileChannel(path, Sets.newHashSet(StandardOpenOption.CREATE)); + channel.close(); + assertThat(Files.exists(path)).isTrue(); + long size = Files.size(path); + assertThat(size).isEqualTo(0); + } finally { + Files.deleteIfExists(path); + } + } + + @Test + public void testFileChannelWrite() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + assertThat(Files.exists(path)).isFalse(); + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + try { + FileChannel channel = + provider.newFileChannel( + path, Sets.newHashSet(StandardOpenOption.CREATE, StandardOpenOption.WRITE)); + ByteBuffer src = ByteBuffer.allocate(5000); + int written = 0; + for (String s : FILE_CONTENTS) { + byte[] bytes = (s + "\n").getBytes(UTF_8); + written += bytes.length; + src.put(bytes); + } + src.limit(written); + src.position(0); + channel.write(src); + channel.close(); + assertThat(Files.exists(path)).isTrue(); + + ByteArrayOutputStream wantBytes = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(wantBytes, UTF_8)); + for (String content : FILE_CONTENTS) { + writer.println(content); + } + writer.close(); + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + byte[] gotBytes = new byte[(int) chan.size()]; + readFully(chan, gotBytes); + assertThat(gotBytes).isEqualTo(wantBytes.toByteArray()); + } finally { + Files.deleteIfExists(path); + } + } + + @Test + public void testFileChannelWriteOnClose() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + assertThat(Files.exists(path)).isFalse(); + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + try { + long expectedSize = 0; + try (FileChannel chan = + provider.newFileChannel(path, Sets.newHashSet(StandardOpenOption.WRITE))) { + for (String s : FILE_CONTENTS) { + byte[] sBytes = s.getBytes(UTF_8); + expectedSize += sBytes.length * 9999; + for (int i = 0; i < 9999; i++) { + chan.write(ByteBuffer.wrap(sBytes)); + } + } + try { + Files.size(path); + // we shouldn't make it to this line. Not using thrown.expect because + // I still want to run a few lines after the exception. + Assert.fail("Files.size should have thrown an exception"); + } catch (NoSuchFileException nsf) { + // that's what we wanted, we're good. + } + } + // channel now closed, the file should be there and with the new contents. + assertThat(Files.exists(path)).isTrue(); + assertThat(Files.size(path)).isEqualTo(expectedSize); + } finally { + Files.deleteIfExists(path); + } + } + + @Test + public void testFileChannelSeek() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(BIG_FILE); + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + int size = BIG_SIZE; + byte[] contents = randomContents(size); + byte[] sample = new byte[100]; + byte[] wanted; + byte[] wanted2; + FileChannel chan = provider.newFileChannel(path, Sets.newHashSet(StandardOpenOption.READ)); + assertThat(chan.size()).isEqualTo(size); + + // check seek + int dest = size / 2; + chan.position(dest); + readFully(chan, sample); + wanted = Arrays.copyOfRange(contents, dest, dest + 100); + assertThat(wanted).isEqualTo(sample); + // now go back and check the beginning + // (we do 2 locations because 0 is sometimes a special case). + chan.position(0); + readFully(chan, sample); + wanted2 = Arrays.copyOf(contents, 100); + assertThat(wanted2).isEqualTo(sample); + // if the two spots in the file have the same contents, then this isn't a good file for this + // test. + assertThat(wanted).isNotEqualTo(wanted2); + } + + @Test + public void testFileChannelTransferFrom() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + Path destPath = testBucket.getPath(PREFIX + randomSuffix()); + assertThat(Files.exists(destPath)).isFalse(); + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + try { + try (FileChannel source = + provider.newFileChannel(path, Sets.newHashSet(StandardOpenOption.READ)); + FileChannel dest = + provider.newFileChannel( + destPath, Sets.newHashSet(StandardOpenOption.CREATE, StandardOpenOption.WRITE))) { + dest.transferFrom(source, 0L, SML_SIZE); + } + + FileChannel source = + provider.newFileChannel(destPath, Sets.newHashSet(StandardOpenOption.READ)); + long size = Files.size(destPath); + assertThat(source.size()).isEqualTo(size); + ByteBuffer buf = ByteBuffer.allocate(SML_SIZE); + int read = 0; + while (source.isOpen()) { + buf.clear(); + int rc = source.read(buf); + assertThat(source.size()).isEqualTo(size); + if (rc < 0) { + // EOF + break; + } + assertThat(rc).isGreaterThan(0); + read += rc; + assertThat(source.position()).isEqualTo(read); + } + source.close(); + assertThat(read).isEqualTo(size); + byte[] expected = new byte[SML_SIZE]; + new Random(SML_SIZE).nextBytes(expected); + assertThat(Arrays.equals(buf.array(), expected)).isTrue(); + } finally { + Files.deleteIfExists(destPath); + } + } + + @Test + public void testFileChannelTransferTo() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + Path destPath = testBucket.getPath(PREFIX + randomSuffix()); + assertThat(Files.exists(destPath)).isFalse(); + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(); + try { + try (FileChannel source = + provider.newFileChannel(path, Sets.newHashSet(StandardOpenOption.READ)); + FileChannel dest = + provider.newFileChannel( + destPath, Sets.newHashSet(StandardOpenOption.CREATE, StandardOpenOption.WRITE))) { + source.transferTo(0L, SML_SIZE, dest); + } + + FileChannel source = + provider.newFileChannel(destPath, Sets.newHashSet(StandardOpenOption.READ)); + long size = Files.size(destPath); + assertThat(source.size()).isEqualTo(size); + ByteBuffer buf = ByteBuffer.allocate(SML_SIZE); + int read = 0; + while (source.isOpen()) { + buf.clear(); + int rc = source.read(buf); + assertThat(source.size()).isEqualTo(size); + if (rc < 0) { + // EOF + break; + } + assertThat(rc).isGreaterThan(0); + read += rc; + assertThat(source.position()).isEqualTo(read); + } + source.close(); + assertThat(read).isEqualTo(size); + byte[] expected = new byte[SML_SIZE]; + new Random(SML_SIZE).nextBytes(expected); + assertThat(Arrays.equals(buf.array(), expected)).isTrue(); + } finally { + Files.deleteIfExists(destPath); + } + } + + /** + * Delete the given directory and all of its contents if non-empty. + * + * @param directory the directory to delete + * @throws IOException + */ + private static void deleteRecursive(Path directory) throws IOException { + Files.walkFileTree( + directory, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + + private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException { + ByteBuffer buf = ByteBuffer.wrap(outputBuf); + int sofar = 0; + int bytes = buf.remaining(); + while (sofar < bytes) { + int read = chan.read(buf); + if (read < 0) { + throw new EOFException("channel EOF"); + } + sofar += read; + } + return sofar; + } + + private String randomSuffix() { + return "-" + rnd.nextInt(99999); + } + + private static class PostTraversalWalker extends SimpleFileVisitor { + private final List paths = new ArrayList<>(); + + // Traverse the tree, return the list of files and folders. + public static ImmutableList walkFileTree(Path start) throws IOException { + PostTraversalWalker walker = new PostTraversalWalker(); + Files.walkFileTree(start, walker); + return walker.getPaths(); + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + paths.add(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + paths.add(dir); + return FileVisitResult.CONTINUE; + } + + public ImmutableList getPaths() { + return copyOf(paths); + } + } + + @Test + public void testCopyWithSameProvider() throws IOException { + CloudStorageFileSystem sourceFileSystem = getTestBucket(); + CloudStorageFileSystem targetFileSystem = + CloudStorageFileSystem.forBucket( + TARGET_BUCKET, CloudStorageConfiguration.DEFAULT, storageOptions); + Path sourceFileSystemPath = sourceFileSystem.getPath(SML_FILE); + Path targetFileSystemPath = targetFileSystem.getPath(PREFIX + randomSuffix()); + Files.copy(sourceFileSystemPath, targetFileSystemPath); + assertSame(sourceFileSystem.provider(), targetFileSystem.provider()); + assertEquals(sourceFileSystem.config(), targetFileSystem.config()); + } + + @Test + public void testCopyWithDifferentProvider() throws IOException { + CloudStorageFileSystem sourceFileSystem = getTestBucket(); + CloudStorageFileSystem targetFileSystem = + CloudStorageFileSystem.forBucket( + TARGET_BUCKET, + CloudStorageConfiguration.builder().permitEmptyPathComponents(true).build(), + storageOptions); + Path sourceFileSystemPath = sourceFileSystem.getPath(SML_FILE); + Path targetFileSystemPath = targetFileSystem.getPath(PREFIX + randomSuffix()); + Files.copy(sourceFileSystemPath, targetFileSystemPath); + assertNotSame(sourceFileSystem.provider(), targetFileSystem.provider()); + assertNotEquals(sourceFileSystem.config(), targetFileSystem.config()); + } + + @Test + public void testMove() throws Exception { + ImmutableMap metadata = ImmutableMap.of("k", "v"); + BlobInfo info1 = BlobInfo.newBuilder(BUCKET, "testMove-0001").setMetadata(metadata).build(); + BlobInfo info2 = BlobInfo.newBuilder(BUCKET, "testMove-0002").build(); + storage.create(info1, "Hello".getBytes(UTF_8), BlobTargetOption.doesNotExist()); + + CloudStorageFileSystem fs = getTestBucket(); + CloudStoragePath src = fs.getPath(info1.getName()); + CloudStoragePath dst = fs.getPath(info2.getName()); + + Path moved = Files.move(src, dst); + assertThat(moved).isNotNull(); + + BlobInfo movedInfo = storage.get(info2.getBlobId()); + assertThat(movedInfo).isNotNull(); + assertThat(movedInfo.getMetadata()).isEqualTo(metadata); + } + + @Test + public void testListObject() throws IOException { + String firstBucket = "first-bucket-" + UUID.randomUUID().toString(); + String secondBucket = "second-bucket" + UUID.randomUUID().toString(); + Storage localStorageService = LocalStorageHelper.customOptions(true).getService(); + fillFile(localStorageService, firstBucket, "object", SML_SIZE); + fillFile(localStorageService, firstBucket, "test-object", SML_SIZE); + fillFile(localStorageService, secondBucket, "test-object", SML_SIZE); + + // Listing objects from first bucket without prefix. + List objects = Lists.newArrayList(localStorageService.list(firstBucket).getValues()); + assertThat(objects.size()).isEqualTo(2); + + // Listing objects from first bucket with prefix. + objects = + Lists.newArrayList( + localStorageService + .list(firstBucket, Storage.BlobListOption.prefix("test-")) + .getValues()); + assertThat(objects.size()).isEqualTo(1); + + // Listing objects from second bucket. + objects = Lists.newArrayList(localStorageService.list(secondBucket).getValues()); + assertThat(objects.size()).isEqualTo(1); + } + + @Test(expected = FileAlreadyExistsException.class) + public void testCopy_replaceFile_withoutOption() throws IOException { + CloudStorageFileSystem fs = getTestBucket(); + String uuid = UUID.randomUUID().toString(); + + CloudStoragePath foo = fs.getPath(uuid, "foo.txt"); + CloudStoragePath bar = fs.getPath(uuid, "bar.txt"); + + try { + Files.createFile(foo); + Files.createFile(bar); + + Files.copy(foo, bar); + } finally { + Files.deleteIfExists(foo); + Files.deleteIfExists(bar); + } + } + + @Test + public void testCopy_replaceFile_withOption() throws IOException { + CloudStorageFileSystem fs = getTestBucket(); + String uuid = UUID.randomUUID().toString(); + + CloudStoragePath foo = fs.getPath(uuid, "foo.txt"); + CloudStoragePath bar = fs.getPath(uuid, "bar.txt"); + + try { + Files.createFile(foo); + Files.createFile(bar); + + Files.copy(foo, bar, StandardCopyOption.REPLACE_EXISTING); + } finally { + Files.deleteIfExists(foo); + Files.deleteIfExists(bar); + } + } + + @Test(expected = NoSuchFileException.class) + public void testCopy_replaceFile_withOption_srcDoesNotExist() throws IOException { + CloudStorageFileSystem fs = getTestBucket(); + String uuid = UUID.randomUUID().toString(); + + CloudStoragePath foo = fs.getPath(uuid, "foo.txt"); + CloudStoragePath bar = fs.getPath(uuid, "bar.txt"); + + try { + // explicitly do not create foo + Files.createFile(bar); + + Files.copy(foo, bar); + } finally { + Files.deleteIfExists(foo); + Files.deleteIfExists(bar); + } + } + + @Test + public void testCheckAccessRoot() throws Exception { + FileSystem fileSystem = getTestBucket(); + Path path = fileSystem.getPath("/"); + FileSystemProvider provider = fileSystem.provider(); + + // Against the real cloud storage this used to throw a StorageException. + provider.checkAccess(path, AccessMode.READ, AccessMode.WRITE); + } + + private CloudStorageFileSystem getTestBucket() throws IOException { + // in typical usage we use the single-argument version of forBucket + // and rely on the user being logged into their project with the + // gcloud tool, and then everything authenticates automagically + // (or we just use paths that start with "gs://" and rely on NIO's magic). + // + // However for the tests we want to be able to run in automated environments + // where we can set environment variables but not necessarily install gcloud + // or run it. That's why we're setting the credentials programmatically. + return CloudStorageFileSystem.forBucket( + BUCKET, CloudStorageConfiguration.DEFAULT, storageOptions); + } + + // same as getTestBucket, but for the requester-pays bucket. + private CloudStorageFileSystem getRequesterPaysBucket(boolean autodetect, String userProject) + throws IOException { + CloudStorageConfiguration config = + CloudStorageConfiguration.builder() + .autoDetectRequesterPays(autodetect) + .userProject(userProject) + .build(); + return CloudStorageFileSystem.forBucket(REQUESTER_PAYS_BUCKET, config, storageOptions); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpcTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpcTest.java new file mode 100644 index 000000000000..54a2985734c1 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpcTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage.contrib.nio.testing; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.WriteChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.HttpStorageOptions; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobWriteOption; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper.FakeStorageRpcFactory; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import org.junit.Test; + +public final class FakeStorageRpcTest { + + @Test + public void overwritingAnObjectOverwritesItsContent() throws IOException { + Storage storage = + StorageOptions.http() + .setServiceRpcFactory(new FakeStorageRpcFactory()) + .build() + .getService(); + + try (WriteChannel writer = + storage.writer( + BlobInfo.newBuilder(BlobId.of("bucket", "obj", 0L)).build(), + BlobWriteOption.generationMatch())) { + writer.write(ByteBuffer.wrap("abc".getBytes(StandardCharsets.UTF_8))); + } + Blob gen1 = storage.get(BlobId.of("bucket", "obj")); + // get existing generation + String gen1read1 = new String(gen1.getContent(), StandardCharsets.UTF_8); + assertThat(gen1read1).isEqualTo("abc"); + System.out.println("gen1read1 = " + gen1read1); + + // start an upload that will overwrite the existing generation + WriteChannel writer = storage.writer(gen1, BlobWriteOption.generationMatch()); + writer.write(ByteBuffer.wrap("def".getBytes(StandardCharsets.UTF_8))); + // make sure we can still read the existing generations value after starting but before closing + String gen1read2 = new String(gen1.getContent(), StandardCharsets.UTF_8); + assertThat(gen1read2).isEqualTo("abc"); + writer.close(); + + Blob gen2 = storage.get(BlobId.of("bucket", "obj")); + String gen2read1 = new String(gen2.getContent(), StandardCharsets.UTF_8); + System.out.println("gen2read1 = " + gen2read1); + assertThat(gen2read1).isEqualTo("def"); + } + + @Test + public void multiChunkUploadWorks() throws Exception { + FakeStorageRpcFactory serviceRpcFactory = new FakeStorageRpcFactory(); + HttpStorageOptions options = + StorageOptions.http().setServiceRpcFactory(serviceRpcFactory).build(); + try (Storage storage = options.getService()) { + + byte[] bytes = new byte[256 * 1024 + 37]; + Arrays.fill(bytes, (byte) 'A'); + + BlobId id = BlobId.of("bucket", "object"); + BlobInfo info = BlobInfo.newBuilder(id).build(); + try (WriteChannel writeChannel = storage.writer(info)) { + writeChannel.setChunkSize(256 * 1024); + writeChannel.write(ByteBuffer.wrap(bytes)); + } + byte[] actual = storage.readAllBytes(id); + assertThat(actual).isEqualTo(bytes); + } + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/testing/LocalStorageHelperTest.java b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/testing/LocalStorageHelperTest.java new file mode 100644 index 000000000000..6cac05f8d202 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/testing/LocalStorageHelperTest.java @@ -0,0 +1,183 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio.testing; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.WriteChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link LocalStorageHelper}. */ +@RunWith(JUnit4.class) +public class LocalStorageHelperTest { + + Storage localStorageService = null; + private final String testBucket = "bucket"; + private final String sourceFile = "testSource"; + private final String destinationFile = "testDestination"; + private final String payload = "copy me"; + + @Before + public void before() throws IOException { + localStorageService = LocalStorageHelper.getOptions().getService(); + + byte[] payloadBytes = payload.getBytes(); + BlobId id = BlobId.of(testBucket, sourceFile); + BlobInfo info = BlobInfo.newBuilder(id).build(); + + try (WriteChannel writer = localStorageService.writer(info)) { + writer.write(ByteBuffer.wrap(payloadBytes)); + } + } + + @After + public void after() { + localStorageService.delete(testBucket, sourceFile); + localStorageService.delete(testBucket, destinationFile); + } + + private Storage.CopyRequest copyRequest() { + Storage.CopyRequest request = + Storage.CopyRequest.newBuilder() + .setSource(BlobId.of(testBucket, sourceFile)) + .setTarget(BlobId.of(testBucket, destinationFile)) + .build(); + + return request; + } + + @Test + public void testCopyCanBeRead() throws Exception { + Storage.CopyRequest request = copyRequest(); + localStorageService.copy(request).getResult(); + Blob obj = localStorageService.get(BlobId.of(testBucket, destinationFile)); + String copiedContents = new String(obj.getContent(Blob.BlobSourceOption.generationMatch())); + File file = File.createTempFile("file", ".txt"); + file.deleteOnExit(); + obj.downloadTo(file.toPath()); + Assert.assertArrayEquals(payload.getBytes(), Files.readAllBytes(file.toPath())); + + assertThat(copiedContents).isEqualTo(payload); + assertThat(obj.getGeneration()).isEqualTo(1); + assertThat(obj.getSize()).isEqualTo(7); + } + + @Test + public void testCopyIncrementsGenerations() { + Storage.CopyRequest request = copyRequest(); + + localStorageService.copy(request).getResult(); + localStorageService.copy(request).getResult(); + Blob obj = localStorageService.get(BlobId.of(testBucket, destinationFile)); + String copiedContents = new String(obj.getContent(Blob.BlobSourceOption.generationMatch())); + + assertThat(copiedContents).isEqualTo(payload); + assertThat(obj.getGeneration()).isEqualTo(2); + assertThat(obj.getSize()).isEqualTo(7); + } + + @Test + public void testWriteNewFileSetsUpdateTime() { + Blob obj = localStorageService.get(BlobId.of(testBucket, sourceFile)); + + assertThat(obj.getUpdateTime()).isNotNull(); + } + + @Test + public void testCreateNewFileSetsUpdateTime() { + BlobInfo info = BlobInfo.newBuilder(BlobId.of(testBucket, "newFile")).build(); + Blob obj = localStorageService.create(info); + + assertThat(obj.getUpdateTime()).isNotNull(); + } + + @Test + public void testStorageOptionIsSerializable() throws Exception { + StorageOptions storageOptions = LocalStorageHelper.getOptions(); + byte[] bytes; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(storageOptions); + oos.flush(); + oos.close(); + bytes = baos.toByteArray(); + } + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais)) { + assertThat(ois.readObject()).isEqualTo(storageOptions); + } + } + + @Test + public void testStorageOptionIsSerializable_customOptions() throws Exception { + StorageOptions storageOptions = LocalStorageHelper.customOptions(false); + byte[] bytes; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(storageOptions); + oos.flush(); + oos.close(); + bytes = baos.toByteArray(); + } + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais)) { + assertThat(ois.readObject()).isEqualTo(storageOptions); + } + } + + @Test + public void testCopyOperationOverwritesExistingFile() { + String bucket = "bucket"; + String original = "original"; + String replacement = "replacement"; + byte[] originalContent = "original content".getBytes(); + byte[] replacementContent = "replacement content".getBytes(); + + localStorageService.create(BlobInfo.newBuilder(bucket, original).build(), originalContent); + localStorageService.create( + BlobInfo.newBuilder(bucket, replacement).build(), replacementContent); + + final Storage.CopyRequest request = + Storage.CopyRequest.newBuilder() + .setSource(BlobId.of(bucket, replacement)) + .setTarget(BlobId.of(bucket, original)) + .build(); + + localStorageService.copy(request).getResult(); + + assertThat(localStorageService.readAllBytes(BlobId.of(bucket, original))) + .isEqualTo(replacementContent); + } +} diff --git a/java-storage-nio/google-cloud-nio/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/java-storage-nio/google-cloud-nio/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 000000000000..1f0955d450f0 --- /dev/null +++ b/java-storage-nio/google-cloud-nio/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/java-storage-nio/pom.xml b/java-storage-nio/pom.xml new file mode 100644 index 000000000000..4c672c503e41 --- /dev/null +++ b/java-storage-nio/pom.xml @@ -0,0 +1,114 @@ + + + 4.0.0 + com.google.cloud + google-cloud-nio-parent + pom + 0.128.14 + Storage Parent + https://github.com/googleapis/google-cloud-java + + FileSystemProvider for Java NIO to access Google Cloud Storage transparently. + + + + com.google.cloud + google-cloud-jar-parent + 1.83.0-SNAPSHOT + ../google-cloud-jar-parent/pom.xml + + + + + jart + Justine Tunney + jart@google.com + Google + + Developer + + + + jean-philippe-martin + Jean-Philippe Martin + jpmartin@verily.com + Verily + + Developer + + + + + Google LLC + + + scm:git:git@github.com:googleapis/google-cloud-java.git + scm:git:git@github.com:googleapis/google-cloud-java.git + https://github.com/googleapis/google-cloud-java + HEAD + + + https://github.com/googleapis/google-cloud-java/issues + GitHub Issues + + + + + Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + UTF-8 + UTF-8 + github + google-cloud-storage-nio-parent + + + + + + + com.google.cloud + google-cloud-storage + 2.64.1 + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + + com.google.truth + truth + 1.4.5 + test + + + org.checkerframework + checker-qual + + + + + + + + google-cloud-nio + google-cloud-nio-retrofit + google-cloud-nio-examples + google-cloud-nio-bom + + + + + include-samples + + google-cloud-nio-retrofit + google-cloud-nio-examples + + + + diff --git a/java-storage-nio/samples/install-without-bom/pom.xml b/java-storage-nio/samples/install-without-bom/pom.xml new file mode 100644 index 000000000000..965a2c9f5693 --- /dev/null +++ b/java-storage-nio/samples/install-without-bom/pom.xml @@ -0,0 +1,84 @@ + + + 4.0.0 + com.google.cloud + google-cloud-nio-install-without-bom + jar + Google NIO Filesystem Provider for Google Cloud Storage Install Without Bom + https://github.com/googleapis/java-storage-nio + + + + com.google.cloud.samples + shared-configuration + 1.2.2 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + com.google.cloud + google-cloud-nio + 0.128.8 + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.4.5 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.1 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + diff --git a/java-storage-nio/samples/pom.xml b/java-storage-nio/samples/pom.xml new file mode 100644 index 000000000000..df0c94a1ad1e --- /dev/null +++ b/java-storage-nio/samples/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + com.google.cloud + google-cloud-google-cloud-nio-samples + 0.0.1-SNAPSHOT + pom + Google NIO Filesystem Provider for Google Cloud Storage Samples Parent + https://github.com/googleapis/java-storage-nio + + Java idiomatic client for Google Cloud Platform services. + + + + + com.google.cloud.samples + shared-configuration + 1.2.2 + + + + 1.8 + 1.8 + UTF-8 + + + + install-without-bom + snapshot + snippets + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.4 + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.7.0 + + true + + + + + diff --git a/java-storage-nio/samples/snapshot/pom.xml b/java-storage-nio/samples/snapshot/pom.xml new file mode 100644 index 000000000000..ca9191241640 --- /dev/null +++ b/java-storage-nio/samples/snapshot/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + com.google.cloud + google-cloud-nio-snapshot + jar + Google NIO Filesystem Provider for Google Cloud Storage Snapshot Samples + https://github.com/googleapis/java-storage-nio + + + + com.google.cloud.samples + shared-configuration + 1.2.2 + + + + 1.8 + 1.8 + UTF-8 + + + + + + com.google.cloud + google-cloud-nio + 0.128.14 + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.4.5 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.1 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + \ No newline at end of file diff --git a/java-storage-nio/samples/snippets/pom.xml b/java-storage-nio/samples/snippets/pom.xml new file mode 100644 index 000000000000..e2f8edd4dd52 --- /dev/null +++ b/java-storage-nio/samples/snippets/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + com.google.cloud + google-cloud-nio-snippets + jar + Google NIO Filesystem Provider for Google Cloud Storage Snippets + https://github.com/googleapis/java-storage-nio + + + + com.google.cloud.samples + shared-configuration + 1.2.2 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + + com.google.cloud + libraries-bom + 26.78.0 + pom + import + + + + + + + com.google.cloud + google-cloud-nio + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.4.5 + test + + + + + diff --git a/pom.xml b/pom.xml index ba802530f02d..e9892122714e 100644 --- a/pom.xml +++ b/pom.xml @@ -226,6 +226,7 @@ java-spanner-jdbc java-spanneradapter java-speech + java-storage-nio java-storage-transfer java-storagebatchoperations java-storageinsights diff --git a/versions.txt b/versions.txt index 342eebe90cde..25a8149d09d5 100644 --- a/versions.txt +++ b/versions.txt @@ -1021,3 +1021,4 @@ google-auth-library-parent:1.43.0:1.43.1-SNAPSHOT google-auth-library-appengine:1.43.0:1.43.1-SNAPSHOT google-auth-library-credentials:1.43.0:1.43.1-SNAPSHOT google-auth-library-oauth2-http:1.43.0:1.43.1-SNAPSHOT +google-cloud-nio:0.128.14:0.128.14